From eb8ad64a24606c9ebe9a3e49712bed496784fd62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20St=C3=BCrmer?= Date: Wed, 23 Dec 2020 13:12:49 +0100 Subject: [PATCH 01/39] Move log entry types out of http directory --- .../results/log_entry_category_examples.ts | 2 +- .../common/http_api/log_entries/entries.ts | 51 +-------- .../common/http_api/log_entries/highlights.ts | 3 +- .../infra/common/log_entry/log_entry.ts | 105 ++++++++++++------ .../log_entries/log_entry.ts | 12 +- .../log_entry_fields_table.tsx | 8 +- .../infra/public/utils/log_entry/log_entry.ts | 8 +- .../log_entries/log_entry_search_strategy.ts | 2 +- 8 files changed, 82 insertions(+), 109 deletions(-) diff --git a/x-pack/plugins/infra/common/http_api/log_analysis/results/log_entry_category_examples.ts b/x-pack/plugins/infra/common/http_api/log_analysis/results/log_entry_category_examples.ts index e9e3c6e0ca3f98..3166d40d703921 100644 --- a/x-pack/plugins/infra/common/http_api/log_analysis/results/log_entry_category_examples.ts +++ b/x-pack/plugins/infra/common/http_api/log_analysis/results/log_entry_category_examples.ts @@ -12,7 +12,7 @@ import { timeRangeRT, routeTimingMetadataRT, } from '../../shared'; -import { logEntryContextRT } from '../../log_entries'; +import { logEntryContextRT } from '../../../log_entry'; export const LOG_ANALYSIS_GET_LOG_ENTRY_CATEGORY_EXAMPLES_PATH = '/api/infra/log_analysis/results/log_entry_category_examples'; diff --git a/x-pack/plugins/infra/common/http_api/log_entries/entries.ts b/x-pack/plugins/infra/common/http_api/log_entries/entries.ts index 31bc62f48791a5..b4d9a5744d5ac0 100644 --- a/x-pack/plugins/infra/common/http_api/log_entries/entries.ts +++ b/x-pack/plugins/infra/common/http_api/log_entries/entries.ts @@ -5,8 +5,7 @@ */ import * as rt from 'io-ts'; -import { logEntryCursorRT } from '../../log_entry'; -import { jsonArrayRT } from '../../typed_json'; +import { logEntryCursorRT, logEntryRT } from '../../log_entry'; import { logSourceColumnConfigurationRT } from '../log_sources'; export const LOG_ENTRIES_PATH = '/api/log_entries/entries'; @@ -52,54 +51,6 @@ export type LogEntriesAfterRequest = rt.TypeOf; export type LogEntriesCenteredRequest = rt.TypeOf; export type LogEntriesRequest = rt.TypeOf; -export const logMessageConstantPartRT = rt.type({ - constant: rt.string, -}); -export const logMessageFieldPartRT = rt.type({ - field: rt.string, - value: jsonArrayRT, - highlights: rt.array(rt.string), -}); - -export const logMessagePartRT = rt.union([logMessageConstantPartRT, logMessageFieldPartRT]); - -export const logTimestampColumnRT = rt.type({ columnId: rt.string, timestamp: rt.number }); -export const logFieldColumnRT = rt.type({ - columnId: rt.string, - field: rt.string, - value: jsonArrayRT, - highlights: rt.array(rt.string), -}); -export const logMessageColumnRT = rt.type({ - columnId: rt.string, - message: rt.array(logMessagePartRT), -}); - -export const logColumnRT = rt.union([logTimestampColumnRT, logFieldColumnRT, logMessageColumnRT]); - -export const logEntryContextRT = rt.union([ - rt.type({}), - rt.type({ 'container.id': rt.string }), - rt.type({ 'host.name': rt.string, 'log.file.path': rt.string }), -]); - -export const logEntryRT = rt.type({ - id: rt.string, - cursor: logEntryCursorRT, - columns: rt.array(logColumnRT), - context: logEntryContextRT, -}); - -export type LogMessageConstantPart = rt.TypeOf; -export type LogMessageFieldPart = rt.TypeOf; -export type LogMessagePart = rt.TypeOf; -export type LogTimestampColumn = rt.TypeOf; -export type LogFieldColumn = rt.TypeOf; -export type LogMessageColumn = rt.TypeOf; -export type LogColumn = rt.TypeOf; -export type LogEntryContext = rt.TypeOf; -export type LogEntry = rt.TypeOf; - export const logEntriesResponseRT = rt.type({ data: rt.intersection([ rt.type({ diff --git a/x-pack/plugins/infra/common/http_api/log_entries/highlights.ts b/x-pack/plugins/infra/common/http_api/log_entries/highlights.ts index 648da43134a276..96bf8beb29021f 100644 --- a/x-pack/plugins/infra/common/http_api/log_entries/highlights.ts +++ b/x-pack/plugins/infra/common/http_api/log_entries/highlights.ts @@ -5,13 +5,12 @@ */ import * as rt from 'io-ts'; -import { logEntryCursorRT } from '../../log_entry'; +import { logEntryCursorRT, logEntryRT } from '../../log_entry'; import { logEntriesBaseRequestRT, logEntriesBeforeRequestRT, logEntriesAfterRequestRT, logEntriesCenteredRequestRT, - logEntryRT, } from './entries'; export const LOG_ENTRIES_HIGHLIGHTS_PATH = '/api/log_entries/highlights'; diff --git a/x-pack/plugins/infra/common/log_entry/log_entry.ts b/x-pack/plugins/infra/common/log_entry/log_entry.ts index e02acebe277113..2795a18cf7a242 100644 --- a/x-pack/plugins/infra/common/log_entry/log_entry.ts +++ b/x-pack/plugins/infra/common/log_entry/log_entry.ts @@ -4,41 +4,80 @@ * you may not use this file except in compliance with the Elastic License. */ +import * as rt from 'io-ts'; import { TimeKey } from '../time'; -import { InfraLogEntry } from '../graphql/types'; +import { jsonArrayRT } from '../typed_json'; +import { logEntryCursorRT } from './log_entry_cursor'; -export type LogEntry = InfraLogEntry; +export type LogEntryTime = TimeKey; + +/** + * message parts + */ -export interface LogEntryOrigin { - id: string; - index: string; - type: string; -} +export const logMessageConstantPartRT = rt.type({ + constant: rt.string, +}); +export type LogMessageConstantPart = rt.TypeOf; -export type LogEntryTime = TimeKey; +export const logMessageFieldPartRT = rt.type({ + field: rt.string, + value: jsonArrayRT, + highlights: rt.array(rt.string), +}); +export type LogMessageFieldPart = rt.TypeOf; + +export const logMessagePartRT = rt.union([logMessageConstantPartRT, logMessageFieldPartRT]); +export type LogMessagePart = rt.TypeOf; + +/** + * columns + */ + +export const logTimestampColumnRT = rt.type({ columnId: rt.string, timestamp: rt.number }); +export type LogTimestampColumn = rt.TypeOf; + +export const logFieldColumnRT = rt.type({ + columnId: rt.string, + field: rt.string, + value: jsonArrayRT, + highlights: rt.array(rt.string), +}); +export type LogFieldColumn = rt.TypeOf; + +export const logMessageColumnRT = rt.type({ + columnId: rt.string, + message: rt.array(logMessagePartRT), +}); +export type LogMessageColumn = rt.TypeOf; + +export const logColumnRT = rt.union([logTimestampColumnRT, logFieldColumnRT, logMessageColumnRT]); +export type LogColumn = rt.TypeOf; + +/** + * fields + */ +export const logEntryContextRT = rt.union([ + rt.type({}), + rt.type({ 'container.id': rt.string }), + rt.type({ 'host.name': rt.string, 'log.file.path': rt.string }), +]); +export type LogEntryContext = rt.TypeOf; + +export const logEntryFieldRT = rt.type({ + field: rt.string, + value: jsonArrayRT, +}); +export type LogEntryField = rt.TypeOf; + +/** + * entry + */ -export interface LogEntryFieldsMapping { - message: string; - tiebreaker: string; - time: string; -} - -export function isEqual(time1: LogEntryTime, time2: LogEntryTime) { - return time1.time === time2.time && time1.tiebreaker === time2.tiebreaker; -} - -export function isLess(time1: LogEntryTime, time2: LogEntryTime) { - return ( - time1.time < time2.time || (time1.time === time2.time && time1.tiebreaker < time2.tiebreaker) - ); -} - -export function isLessOrEqual(time1: LogEntryTime, time2: LogEntryTime) { - return ( - time1.time < time2.time || (time1.time === time2.time && time1.tiebreaker <= time2.tiebreaker) - ); -} - -export function isBetween(min: LogEntryTime, max: LogEntryTime, operand: LogEntryTime) { - return isLessOrEqual(min, operand) && isLessOrEqual(operand, max); -} +export const logEntryRT = rt.type({ + id: rt.string, + cursor: logEntryCursorRT, + columns: rt.array(logColumnRT), + context: logEntryContextRT, +}); +export type LogEntry = rt.TypeOf; diff --git a/x-pack/plugins/infra/common/search_strategies/log_entries/log_entry.ts b/x-pack/plugins/infra/common/search_strategies/log_entries/log_entry.ts index af6bd203f980ea..986f6baf044881 100644 --- a/x-pack/plugins/infra/common/search_strategies/log_entries/log_entry.ts +++ b/x-pack/plugins/infra/common/search_strategies/log_entries/log_entry.ts @@ -5,8 +5,7 @@ */ import * as rt from 'io-ts'; -import { logEntryCursorRT } from '../../log_entry'; -import { jsonArrayRT } from '../../typed_json'; +import { logEntryCursorRT, logEntryFieldRT } from '../../log_entry'; import { searchStrategyErrorRT } from '../common/errors'; export const LOG_ENTRY_SEARCH_STRATEGY = 'infra-log-entry'; @@ -18,18 +17,11 @@ export const logEntrySearchRequestParamsRT = rt.type({ export type LogEntrySearchRequestParams = rt.TypeOf; -const logEntryFieldRT = rt.type({ - field: rt.string, - value: jsonArrayRT, -}); - -export type LogEntryField = rt.TypeOf; - export const logEntryRT = rt.type({ id: rt.string, index: rt.string, fields: rt.array(logEntryFieldRT), - key: logEntryCursorRT, + cursor: logEntryCursorRT, }); export type LogEntry = rt.TypeOf; diff --git a/x-pack/plugins/infra/public/components/logging/log_entry_flyout/log_entry_fields_table.tsx b/x-pack/plugins/infra/public/components/logging/log_entry_flyout/log_entry_fields_table.tsx index 44e9902e0413f1..b3c80a3a4924a2 100644 --- a/x-pack/plugins/infra/public/components/logging/log_entry_flyout/log_entry_fields_table.tsx +++ b/x-pack/plugins/infra/public/components/logging/log_entry_flyout/log_entry_fields_table.tsx @@ -7,10 +7,8 @@ import { EuiBasicTableColumn, EuiInMemoryTable } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; import React, { useMemo } from 'react'; -import { - LogEntry, - LogEntryField, -} from '../../../../common/search_strategies/log_entries/log_entry'; +import { LogEntryField } from '../../../../common/log_entry'; +import { LogEntry } from '../../../../common/search_strategies/log_entries/log_entry'; import { TimeKey } from '../../../../common/time'; import { FieldValue } from '../log_text_stream/field_value'; @@ -22,7 +20,7 @@ export const LogEntryFieldsTable: React.FC<{ () => onSetFieldFilter ? (field: LogEntryField) => () => { - onSetFieldFilter?.(`${field.field}:"${field.value}"`, logEntry.id, logEntry.key); + onSetFieldFilter?.(`${field.field}:"${field.value}"`, logEntry.id, logEntry.cursor); } : undefined, [logEntry, onSetFieldFilter] diff --git a/x-pack/plugins/infra/public/utils/log_entry/log_entry.ts b/x-pack/plugins/infra/public/utils/log_entry/log_entry.ts index bb528ee5b18c5c..60034aea6be639 100644 --- a/x-pack/plugins/infra/public/utils/log_entry/log_entry.ts +++ b/x-pack/plugins/infra/public/utils/log_entry/log_entry.ts @@ -5,9 +5,7 @@ */ import { bisector } from 'd3-array'; - import { compareToTimeKey, getIndexAtTimeKey, TimeKey, UniqueTimeKey } from '../../../common/time'; -import { InfraLogEntryFields } from '../../graphql/types'; import { LogEntry, LogColumn, @@ -17,11 +15,7 @@ import { LogMessagePart, LogMessageFieldPart, LogMessageConstantPart, -} from '../../../common/http_api'; - -export type LogEntryMessageSegment = InfraLogEntryFields.Message; -export type LogEntryConstantMessageSegment = InfraLogEntryFields.InfraLogMessageConstantSegmentInlineFragment; -export type LogEntryFieldMessageSegment = InfraLogEntryFields.InfraLogMessageFieldSegmentInlineFragment; +} from '../../../common/log_entry'; export const getLogEntryKey = (entry: { cursor: TimeKey }) => entry.cursor; diff --git a/x-pack/plugins/infra/server/services/log_entries/log_entry_search_strategy.ts b/x-pack/plugins/infra/server/services/log_entries/log_entry_search_strategy.ts index a0dfe3d7176fd4..ab2b72055e4a41 100644 --- a/x-pack/plugins/infra/server/services/log_entries/log_entry_search_strategy.ts +++ b/x-pack/plugins/infra/server/services/log_entries/log_entry_search_strategy.ts @@ -119,6 +119,6 @@ const { asyncInitialRequestRT, asyncRecoveredRequestRT, asyncRequestRT } = creat const createLogEntryFromHit = (hit: LogEntryHit) => ({ id: hit._id, index: hit._index, - key: getLogEntryCursorFromHit(hit), + cursor: getLogEntryCursorFromHit(hit), fields: Object.entries(hit.fields).map(([field, value]) => ({ field, value })), }); From d38126cc9cf33da591add61297cdf3452ba5296a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20St=C3=BCrmer?= Date: Wed, 23 Dec 2020 17:03:10 +0100 Subject: [PATCH 02/39] Create basic (dysfunctional) log entries search strategy --- .../log_entries/log_entries.ts | 70 ++++++++++ .../log_entries_search_strategy.ts | 124 ++++++++++++++++++ .../log_entries/log_entries_service.ts | 6 + .../log_entries/queries/log_entries.ts | 50 +++++++ 4 files changed, 250 insertions(+) create mode 100644 x-pack/plugins/infra/common/search_strategies/log_entries/log_entries.ts create mode 100644 x-pack/plugins/infra/server/services/log_entries/log_entries_search_strategy.ts create mode 100644 x-pack/plugins/infra/server/services/log_entries/queries/log_entries.ts diff --git a/x-pack/plugins/infra/common/search_strategies/log_entries/log_entries.ts b/x-pack/plugins/infra/common/search_strategies/log_entries/log_entries.ts new file mode 100644 index 00000000000000..1b28f43d4df231 --- /dev/null +++ b/x-pack/plugins/infra/common/search_strategies/log_entries/log_entries.ts @@ -0,0 +1,70 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import * as rt from 'io-ts'; +import { logSourceColumnConfigurationRT } from '../../http_api/log_sources'; +import { logEntryCursorRT, logEntryRT } from '../../log_entry'; +import { searchStrategyErrorRT } from '../common/errors'; + +export const LOG_ENTRIES_SEARCH_STRATEGY = 'infra-log-entries'; + +export const logEntriesBaseSearchRequestParamsRT = rt.intersection([ + rt.type({ + sourceId: rt.string, + startTimestamp: rt.number, + endTimestamp: rt.number, + }), + rt.partial({ + query: rt.union([rt.string, rt.null]), + size: rt.number, + columns: rt.array(logSourceColumnConfigurationRT), + }), +]); + +export const logEntriesBeforeSearchRequestParamsRT = rt.intersection([ + logEntriesBaseSearchRequestParamsRT, + rt.type({ before: rt.union([logEntryCursorRT, rt.literal('last')]) }), +]); + +export const logEntriesAfterSearchRequestParamsRT = rt.intersection([ + logEntriesBaseSearchRequestParamsRT, + rt.type({ after: rt.union([logEntryCursorRT, rt.literal('first')]) }), +]); + +export const logEntriesCenteredSearchRequestParamsRT = rt.intersection([ + logEntriesBaseSearchRequestParamsRT, + rt.type({ center: logEntryCursorRT }), +]); + +export const logEntriesSearchRequestParamsRT = rt.union([ + logEntriesBaseSearchRequestParamsRT, + logEntriesBeforeSearchRequestParamsRT, + logEntriesAfterSearchRequestParamsRT, + logEntriesCenteredSearchRequestParamsRT, +]); + +export type LogEntriesSearchRequestParams = rt.TypeOf; + +export const logEntriesSearchResponsePayloadRT = rt.intersection([ + rt.type({ + data: rt.intersection([ + rt.type({ + entries: rt.array(logEntryRT), + topCursor: rt.union([logEntryCursorRT, rt.null]), + bottomCursor: rt.union([logEntryCursorRT, rt.null]), + }), + rt.partial({ + hasMoreBefore: rt.boolean, + hasMoreAfter: rt.boolean, + }), + ]), + }), + rt.partial({ + errors: rt.array(searchStrategyErrorRT), + }), +]); + +export type LogEntriesSearchResponsePayload = rt.TypeOf; diff --git a/x-pack/plugins/infra/server/services/log_entries/log_entries_search_strategy.ts b/x-pack/plugins/infra/server/services/log_entries/log_entries_search_strategy.ts new file mode 100644 index 00000000000000..c8aaef535dde25 --- /dev/null +++ b/x-pack/plugins/infra/server/services/log_entries/log_entries_search_strategy.ts @@ -0,0 +1,124 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import * as rt from 'io-ts'; +import { concat, defer, of } from 'rxjs'; +import { concatMap, filter, map, shareReplay, take } from 'rxjs/operators'; +import type { + IEsSearchRequest, + IKibanaSearchRequest, + IKibanaSearchResponse, +} from '../../../../../../src/plugins/data/common'; +import type { + ISearchStrategy, + PluginStart as DataPluginStart, +} from '../../../../../../src/plugins/data/server'; +import { getLogEntryCursorFromHit } from '../../../common/log_entry'; +import { decodeOrThrow } from '../../../common/runtime_types'; +import { + LogEntriesSearchRequestParams, + logEntriesSearchRequestParamsRT, + LogEntriesSearchResponsePayload, + logEntriesSearchResponsePayloadRT, +} from '../../../common/search_strategies/log_entries/log_entries'; +import type { IInfraSources } from '../../lib/sources'; +import { + createAsyncRequestRTs, + createErrorFromShardFailure, + jsonFromBase64StringRT, +} from '../../utils/typed_search_strategy'; +import { + createGetLogEntriesQuery, + getLogEntriesResponseRT, + LogEntryHit, +} from './queries/log_entries'; + +type LogEntriesSearchRequest = IKibanaSearchRequest; +type LogEntriesSearchResponse = IKibanaSearchResponse; + +export const logEntriesSearchStrategyProvider = ({ + data, + sources, +}: { + data: DataPluginStart; + sources: IInfraSources; +}): ISearchStrategy => { + const esSearchStrategy = data.search.getSearchStrategy('ese'); + + return { + search: (rawRequest, options, dependencies) => + defer(() => { + const request = decodeOrThrow(asyncRequestRT)(rawRequest); + + const sourceConfiguration$ = defer(() => + sources.getSourceConfiguration(dependencies.savedObjectsClient, request.params.sourceId) + ).pipe(shareReplay(1)); + + const recoveredRequest$ = of(request).pipe( + filter(asyncRecoveredRequestRT.is), + map(({ id: { esRequestId } }) => ({ id: esRequestId })) + ); + + const initialRequest$ = of(request).pipe( + filter(asyncInitialRequestRT.is), + concatMap(({ params }) => + sourceConfiguration$.pipe( + map( + ({ configuration }): IEsSearchRequest => ({ + params: createGetLogEntriesQuery(configuration.logAlias), + }) + ) + ) + ) + ); + + return concat(recoveredRequest$, initialRequest$).pipe( + take(1), + concatMap((esRequest) => esSearchStrategy.search(esRequest, options, dependencies)), + map((esResponse) => ({ + ...esResponse, + rawResponse: decodeOrThrow(getLogEntriesResponseRT)(esResponse.rawResponse), + })), + map((esResponse) => ({ + ...esResponse, + ...(esResponse.id + ? { id: logEntriesSearchRequestStateRT.encode({ esRequestId: esResponse.id }) } + : {}), + rawResponse: logEntriesSearchResponsePayloadRT.encode({ + data: { entries: [], topCursor: null, bottomCursor: null }, + // data: esResponse.rawResponse.hits.hits.map(createLogEntryFromHit)[0] ?? null, + errors: (esResponse.rawResponse._shards.failures ?? []).map( + createErrorFromShardFailure + ), + }), + })) + ); + }), + cancel: async (id, options, dependencies) => { + const { esRequestId } = decodeOrThrow(logEntriesSearchRequestStateRT)(id); + return await esSearchStrategy.cancel?.(esRequestId, options, dependencies); + }, + }; +}; + +// exported for tests +export const logEntriesSearchRequestStateRT = rt.string.pipe(jsonFromBase64StringRT).pipe( + rt.type({ + esRequestId: rt.string, + }) +); + +const { asyncInitialRequestRT, asyncRecoveredRequestRT, asyncRequestRT } = createAsyncRequestRTs( + logEntriesSearchRequestStateRT, + logEntriesSearchRequestParamsRT +); + +const createLogEntryFromHit = (hit: LogEntryHit) => ({ + id: hit._id, + index: hit._index, + cursor: getLogEntryCursorFromHit(hit), + fields: Object.entries(hit.fields).map(([field, value]) => ({ field, value })), +}); diff --git a/x-pack/plugins/infra/server/services/log_entries/log_entries_service.ts b/x-pack/plugins/infra/server/services/log_entries/log_entries_service.ts index edd53be9db8411..9aba69428f2575 100644 --- a/x-pack/plugins/infra/server/services/log_entries/log_entries_service.ts +++ b/x-pack/plugins/infra/server/services/log_entries/log_entries_service.ts @@ -6,12 +6,18 @@ import { CoreSetup } from 'src/core/server'; import { LOG_ENTRY_SEARCH_STRATEGY } from '../../../common/search_strategies/log_entries/log_entry'; +import { LOG_ENTRIES_SEARCH_STRATEGY } from '../../../common/search_strategies/log_entries/log_entries'; +import { logEntriesSearchStrategyProvider } from './log_entries_search_strategy'; import { logEntrySearchStrategyProvider } from './log_entry_search_strategy'; import { LogEntriesServiceSetupDeps, LogEntriesServiceStartDeps } from './types'; export class LogEntriesService { public setup(core: CoreSetup, setupDeps: LogEntriesServiceSetupDeps) { core.getStartServices().then(([, startDeps]) => { + setupDeps.data.search.registerSearchStrategy( + LOG_ENTRIES_SEARCH_STRATEGY, + logEntriesSearchStrategyProvider({ ...setupDeps, ...startDeps }) + ); setupDeps.data.search.registerSearchStrategy( LOG_ENTRY_SEARCH_STRATEGY, logEntrySearchStrategyProvider({ ...setupDeps, ...startDeps }) diff --git a/x-pack/plugins/infra/server/services/log_entries/queries/log_entries.ts b/x-pack/plugins/infra/server/services/log_entries/queries/log_entries.ts new file mode 100644 index 00000000000000..7aaf2a295d1fd7 --- /dev/null +++ b/x-pack/plugins/infra/server/services/log_entries/queries/log_entries.ts @@ -0,0 +1,50 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import type { RequestParams } from '@elastic/elasticsearch'; +import * as rt from 'io-ts'; +import { jsonArrayRT } from '../../../../common/typed_json'; +import { + commonHitFieldsRT, + commonSearchSuccessResponseFieldsRT, +} from '../../../utils/elasticsearch_runtime_types'; + +export const createGetLogEntriesQuery = ( + logEntryIndex: string +): RequestParams.AsyncSearchSubmit> => ({ + index: logEntryIndex, + terminate_after: 1, + track_scores: false, + track_total_hits: false, + body: { + size: 1, + query: {}, + fields: ['*'], + // sort: [{ [timestampField]: 'desc' }, { [tiebreakerField]: 'desc' }], + _source: false, + }, +}); + +export const logEntryHitRT = rt.intersection([ + commonHitFieldsRT, + rt.type({ + fields: rt.record(rt.string, jsonArrayRT), + sort: rt.tuple([rt.number, rt.number]), + }), +]); + +export type LogEntryHit = rt.TypeOf; + +export const getLogEntriesResponseRT = rt.intersection([ + commonSearchSuccessResponseFieldsRT, + rt.type({ + hits: rt.type({ + hits: rt.array(logEntryHitRT), + }), + }), +]); + +export type GetLogEntriesResponse = rt.TypeOf; From 6f0df9629ace38936e74dd63352ac848d9142c96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20St=C3=BCrmer?= Date: Wed, 23 Dec 2020 17:25:59 +0100 Subject: [PATCH 03/39] Add log entries query params --- .../common/log_entry/log_entry_cursor.ts | 16 ++++++- .../log_entries/log_entries.ts | 16 +++++-- .../log_entries/queries/log_entries.ts | 46 +++++++++++++++++-- 3 files changed, 69 insertions(+), 9 deletions(-) diff --git a/x-pack/plugins/infra/common/log_entry/log_entry_cursor.ts b/x-pack/plugins/infra/common/log_entry/log_entry_cursor.ts index 280403dd5438d4..b11a48822e758f 100644 --- a/x-pack/plugins/infra/common/log_entry/log_entry_cursor.ts +++ b/x-pack/plugins/infra/common/log_entry/log_entry_cursor.ts @@ -11,9 +11,23 @@ export const logEntryCursorRT = rt.type({ time: rt.number, tiebreaker: rt.number, }); - export type LogEntryCursor = rt.TypeOf; +export const logEntryBeforeCursorRT = rt.type({ + before: rt.union([logEntryCursorRT, rt.literal('last')]), +}); +export type LogEntryBeforeCursor = rt.TypeOf; + +export const logEntryAfterCursorRT = rt.type({ + after: rt.union([logEntryCursorRT, rt.literal('first')]), +}); +export type LogEntryAfterCursor = rt.TypeOf; + +export const logEntryAroundCursorRT = rt.type({ + center: logEntryCursorRT, +}); +export type LogEntryAroundCursor = rt.TypeOf; + export const getLogEntryCursorFromHit = (hit: { sort: [number, number] }) => decodeOrThrow(logEntryCursorRT)({ time: hit.sort[0], diff --git a/x-pack/plugins/infra/common/search_strategies/log_entries/log_entries.ts b/x-pack/plugins/infra/common/search_strategies/log_entries/log_entries.ts index 1b28f43d4df231..af268600902562 100644 --- a/x-pack/plugins/infra/common/search_strategies/log_entries/log_entries.ts +++ b/x-pack/plugins/infra/common/search_strategies/log_entries/log_entries.ts @@ -6,12 +6,18 @@ import * as rt from 'io-ts'; import { logSourceColumnConfigurationRT } from '../../http_api/log_sources'; -import { logEntryCursorRT, logEntryRT } from '../../log_entry'; +import { + logEntryAfterCursorRT, + logEntryAroundCursorRT, + logEntryBeforeCursorRT, + logEntryCursorRT, + logEntryRT, +} from '../../log_entry'; import { searchStrategyErrorRT } from '../common/errors'; export const LOG_ENTRIES_SEARCH_STRATEGY = 'infra-log-entries'; -export const logEntriesBaseSearchRequestParamsRT = rt.intersection([ +const logEntriesBaseSearchRequestParamsRT = rt.intersection([ rt.type({ sourceId: rt.string, startTimestamp: rt.number, @@ -26,17 +32,17 @@ export const logEntriesBaseSearchRequestParamsRT = rt.intersection([ export const logEntriesBeforeSearchRequestParamsRT = rt.intersection([ logEntriesBaseSearchRequestParamsRT, - rt.type({ before: rt.union([logEntryCursorRT, rt.literal('last')]) }), + logEntryBeforeCursorRT, ]); export const logEntriesAfterSearchRequestParamsRT = rt.intersection([ logEntriesBaseSearchRequestParamsRT, - rt.type({ after: rt.union([logEntryCursorRT, rt.literal('first')]) }), + logEntryAfterCursorRT, ]); export const logEntriesCenteredSearchRequestParamsRT = rt.intersection([ logEntriesBaseSearchRequestParamsRT, - rt.type({ center: logEntryCursorRT }), + logEntryAroundCursorRT, ]); export const logEntriesSearchRequestParamsRT = rt.union([ diff --git a/x-pack/plugins/infra/server/services/log_entries/queries/log_entries.ts b/x-pack/plugins/infra/server/services/log_entries/queries/log_entries.ts index 7aaf2a295d1fd7..c2b9d2bc57e437 100644 --- a/x-pack/plugins/infra/server/services/log_entries/queries/log_entries.ts +++ b/x-pack/plugins/infra/server/services/log_entries/queries/log_entries.ts @@ -6,16 +6,30 @@ import type { RequestParams } from '@elastic/elasticsearch'; import * as rt from 'io-ts'; -import { jsonArrayRT } from '../../../../common/typed_json'; +import { + LogEntryAfterCursor, + LogEntryBeforeCursor, + LogEntryCursor, +} from '../../../../common/log_entry'; +import { jsonArrayRT, JsonObject } from '../../../../common/typed_json'; import { commonHitFieldsRT, commonSearchSuccessResponseFieldsRT, } from '../../../utils/elasticsearch_runtime_types'; export const createGetLogEntriesQuery = ( - logEntryIndex: string + logEntriesIndex: string, + startTimestamp: number, + endTimestamp: number, + cursor: LogEntryBeforeCursor | LogEntryAfterCursor | null | undefined, + size: number, + timestampField: string, + tiebreakerField: string, + fields: string[], + query?: JsonObject, + highlightTerm?: string ): RequestParams.AsyncSearchSubmit> => ({ - index: logEntryIndex, + index: logEntriesIndex, terminate_after: 1, track_scores: false, track_total_hits: false, @@ -28,6 +42,32 @@ export const createGetLogEntriesQuery = ( }, }); +function createSortAndSearchAfterClause( + cursor: LogEntryBeforeCursor | LogEntryAfterCursor | null | undefined +): { + sortDirection: 'asc' | 'desc'; + searchAfterClause: { search_after?: readonly [number, number] }; +} { + if (cursor) { + if ('before' in cursor) { + return { + sortDirection: 'desc', + searchAfterClause: + cursor.before !== 'last' + ? { search_after: [cursor.before.time, cursor.before.tiebreaker] as const } + : {}, + }; + } else if (cursor.after !== 'first') { + return { + sortDirection: 'asc', + searchAfterClause: { search_after: [cursor.after.time, cursor.after.tiebreaker] as const }, + }; + } + } + + return { sortDirection: 'asc', searchAfterClause: {} }; +} + export const logEntryHitRT = rt.intersection([ commonHitFieldsRT, rt.type({ From 61cd20b87e792a42cae91ab95fec40bf57af5856 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20St=C3=BCrmer?= Date: Thu, 7 Jan 2021 17:07:30 +0100 Subject: [PATCH 04/39] Expand the log entries query --- .../log_entries/log_entries.ts | 6 +- .../log_entries_search_strategy.ts | 14 +- .../services/log_entries/queries/common.ts | 32 +++++ .../log_entries/queries/log_entries.ts | 122 ++++++++++++------ 4 files changed, 135 insertions(+), 39 deletions(-) create mode 100644 x-pack/plugins/infra/server/services/log_entries/queries/common.ts diff --git a/x-pack/plugins/infra/common/search_strategies/log_entries/log_entries.ts b/x-pack/plugins/infra/common/search_strategies/log_entries/log_entries.ts index af268600902562..1325fb651ec507 100644 --- a/x-pack/plugins/infra/common/search_strategies/log_entries/log_entries.ts +++ b/x-pack/plugins/infra/common/search_strategies/log_entries/log_entries.ts @@ -13,6 +13,7 @@ import { logEntryCursorRT, logEntryRT, } from '../../log_entry'; +import { jsonObjectRT } from '../../typed_json'; import { searchStrategyErrorRT } from '../common/errors'; export const LOG_ENTRIES_SEARCH_STRATEGY = 'infra-log-entries'; @@ -22,11 +23,12 @@ const logEntriesBaseSearchRequestParamsRT = rt.intersection([ sourceId: rt.string, startTimestamp: rt.number, endTimestamp: rt.number, + size: rt.number, }), rt.partial({ - query: rt.union([rt.string, rt.null]), - size: rt.number, + query: jsonObjectRT, columns: rt.array(logSourceColumnConfigurationRT), + highlightTerms: rt.array(rt.string), }), ]); diff --git a/x-pack/plugins/infra/server/services/log_entries/log_entries_search_strategy.ts b/x-pack/plugins/infra/server/services/log_entries/log_entries_search_strategy.ts index c8aaef535dde25..0e293d430f37cc 100644 --- a/x-pack/plugins/infra/server/services/log_entries/log_entries_search_strategy.ts +++ b/x-pack/plugins/infra/server/services/log_entries/log_entries_search_strategy.ts @@ -4,6 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ +import { pick } from '@kbn/std'; import * as rt from 'io-ts'; import { concat, defer, of } from 'rxjs'; import { concatMap, filter, map, shareReplay, take } from 'rxjs/operators'; @@ -68,7 +69,18 @@ export const logEntriesSearchStrategyProvider = ({ sourceConfiguration$.pipe( map( ({ configuration }): IEsSearchRequest => ({ - params: createGetLogEntriesQuery(configuration.logAlias), + params: createGetLogEntriesQuery( + configuration.logAlias, + params.startTimestamp, + params.endTimestamp, + undefined, // TODO: determine cursor + params.size, + configuration.fields.timestamp, + configuration.fields.tiebreaker, + [], // TODO: determine fields list + params.query, + undefined // TODO: map over highlight terms OR reduce to one term in request + ), }) ) ) diff --git a/x-pack/plugins/infra/server/services/log_entries/queries/common.ts b/x-pack/plugins/infra/server/services/log_entries/queries/common.ts new file mode 100644 index 00000000000000..f170fa337a8b9e --- /dev/null +++ b/x-pack/plugins/infra/server/services/log_entries/queries/common.ts @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export const createSortClause = ( + sortDirection: 'asc' | 'desc', + timestampField: string, + tiebreakerField: string +) => ({ + sort: { + [timestampField]: sortDirection, + [tiebreakerField]: sortDirection, + }, +}); + +export const createTimeRangeFilterClauses = ( + startTimestamp: number, + endTimestamp: number, + timestampField: string +) => [ + { + range: { + [timestampField]: { + gte: startTimestamp, + lte: endTimestamp, + format: 'epoch_millis', + }, + }, + }, +]; diff --git a/x-pack/plugins/infra/server/services/log_entries/queries/log_entries.ts b/x-pack/plugins/infra/server/services/log_entries/queries/log_entries.ts index c2b9d2bc57e437..c727bb7d808601 100644 --- a/x-pack/plugins/infra/server/services/log_entries/queries/log_entries.ts +++ b/x-pack/plugins/infra/server/services/log_entries/queries/log_entries.ts @@ -8,14 +8,16 @@ import type { RequestParams } from '@elastic/elasticsearch'; import * as rt from 'io-ts'; import { LogEntryAfterCursor, + logEntryAfterCursorRT, LogEntryBeforeCursor, - LogEntryCursor, + logEntryBeforeCursorRT, } from '../../../../common/log_entry'; import { jsonArrayRT, JsonObject } from '../../../../common/typed_json'; import { commonHitFieldsRT, commonSearchSuccessResponseFieldsRT, } from '../../../utils/elasticsearch_runtime_types'; +import { createSortClause, createTimeRangeFilterClauses } from './common'; export const createGetLogEntriesQuery = ( logEntriesIndex: string, @@ -28,50 +30,98 @@ export const createGetLogEntriesQuery = ( fields: string[], query?: JsonObject, highlightTerm?: string -): RequestParams.AsyncSearchSubmit> => ({ - index: logEntriesIndex, - terminate_after: 1, - track_scores: false, - track_total_hits: false, - body: { - size: 1, - query: {}, - fields: ['*'], - // sort: [{ [timestampField]: 'desc' }, { [tiebreakerField]: 'desc' }], - _source: false, - }, -}); +): RequestParams.AsyncSearchSubmit> => { + const sortDirection = getSortDirection(cursor); + const highlightQuery = createHighlightQuery(highlightTerm, fields); -function createSortAndSearchAfterClause( + return { + index: logEntriesIndex, + allow_no_indices: true, + terminate_after: 1, + track_scores: false, + track_total_hits: false, + body: { + size, + query: { + bool: { + filter: [ + ...(query ? [query] : []), + ...(highlightQuery ? [highlightQuery] : []), + ...createTimeRangeFilterClauses(startTimestamp, endTimestamp, timestampField), + ], + }, + }, + fields, + _source: false, + ...createSortClause(sortDirection, timestampField, tiebreakerField), + ...createSearchAfterClause(cursor), + ...createHighlightClause(highlightQuery, fields), + }, + }; +}; + +const getSortDirection = ( + cursor: LogEntryBeforeCursor | LogEntryAfterCursor | null | undefined +): 'asc' | 'desc' => (logEntryBeforeCursorRT.is(cursor) ? 'desc' : 'asc'); + +const createSearchAfterClause = ( cursor: LogEntryBeforeCursor | LogEntryAfterCursor | null | undefined -): { - sortDirection: 'asc' | 'desc'; - searchAfterClause: { search_after?: readonly [number, number] }; -} { - if (cursor) { - if ('before' in cursor) { - return { - sortDirection: 'desc', - searchAfterClause: - cursor.before !== 'last' - ? { search_after: [cursor.before.time, cursor.before.tiebreaker] as const } - : {}, - }; - } else if (cursor.after !== 'first') { - return { - sortDirection: 'asc', - searchAfterClause: { search_after: [cursor.after.time, cursor.after.tiebreaker] as const }, - }; - } +): { search_after?: [number, number] } => { + if (logEntryBeforeCursorRT.is(cursor) && cursor.before !== 'last') { + return { + search_after: [cursor.before.time, cursor.before.tiebreaker], + }; + } else if (logEntryAfterCursorRT.is(cursor) && cursor.after !== 'first') { + return { + search_after: [cursor.after.time, cursor.after.tiebreaker], + }; } - return { sortDirection: 'asc', searchAfterClause: {} }; -} + return {}; +}; + +const createHighlightClause = (highlightQuery: JsonObject | undefined, fields: string[]) => + highlightQuery + ? { + highlight: { + boundary_scanner: 'word', + fields: fields.reduce( + (highlightFieldConfigs, fieldName) => ({ + ...highlightFieldConfigs, + [fieldName]: {}, + }), + {} + ), + fragment_size: 1, + number_of_fragments: 100, + post_tags: [''], + pre_tags: [''], + highlight_query: highlightQuery, + }, + } + : {}; + +const createHighlightQuery = ( + highlightTerm: string | undefined, + fields: string[] +): JsonObject | undefined => { + if (highlightTerm) { + return { + multi_match: { + fields, + lenient: true, + query: highlightTerm, + type: 'phrase', + }, + }; + } +}; export const logEntryHitRT = rt.intersection([ commonHitFieldsRT, rt.type({ fields: rt.record(rt.string, jsonArrayRT), + highlight: rt.record(rt.string, rt.array(rt.string)), sort: rt.tuple([rt.number, rt.number]), }), ]); From a51b6245aae7a8666b11d6fdafb696810f398ef8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20St=C3=BCrmer?= Date: Mon, 11 Jan 2021 19:26:54 +0100 Subject: [PATCH 05/39] remove the center cursor correctly extract the others --- .../log_entries/log_entries.ts | 12 ++--- .../log_entries_search_strategy.ts | 50 +++++++++++++------ 2 files changed, 41 insertions(+), 21 deletions(-) diff --git a/x-pack/plugins/infra/common/search_strategies/log_entries/log_entries.ts b/x-pack/plugins/infra/common/search_strategies/log_entries/log_entries.ts index 1325fb651ec507..6b168f91e6ff22 100644 --- a/x-pack/plugins/infra/common/search_strategies/log_entries/log_entries.ts +++ b/x-pack/plugins/infra/common/search_strategies/log_entries/log_entries.ts @@ -8,7 +8,7 @@ import * as rt from 'io-ts'; import { logSourceColumnConfigurationRT } from '../../http_api/log_sources'; import { logEntryAfterCursorRT, - logEntryAroundCursorRT, + // logEntryAroundCursorRT, logEntryBeforeCursorRT, logEntryCursorRT, logEntryRT, @@ -42,16 +42,16 @@ export const logEntriesAfterSearchRequestParamsRT = rt.intersection([ logEntryAfterCursorRT, ]); -export const logEntriesCenteredSearchRequestParamsRT = rt.intersection([ - logEntriesBaseSearchRequestParamsRT, - logEntryAroundCursorRT, -]); +// export const logEntriesCenteredSearchRequestParamsRT = rt.intersection([ +// logEntriesBaseSearchRequestParamsRT, +// logEntryAroundCursorRT, +// ]); export const logEntriesSearchRequestParamsRT = rt.union([ logEntriesBaseSearchRequestParamsRT, logEntriesBeforeSearchRequestParamsRT, logEntriesAfterSearchRequestParamsRT, - logEntriesCenteredSearchRequestParamsRT, + // logEntriesCenteredSearchRequestParamsRT, ]); export type LogEntriesSearchRequestParams = rt.TypeOf; diff --git a/x-pack/plugins/infra/server/services/log_entries/log_entries_search_strategy.ts b/x-pack/plugins/infra/server/services/log_entries/log_entries_search_strategy.ts index 0e293d430f37cc..d4e09c54683d7a 100644 --- a/x-pack/plugins/infra/server/services/log_entries/log_entries_search_strategy.ts +++ b/x-pack/plugins/infra/server/services/log_entries/log_entries_search_strategy.ts @@ -17,7 +17,13 @@ import type { ISearchStrategy, PluginStart as DataPluginStart, } from '../../../../../../src/plugins/data/server'; -import { getLogEntryCursorFromHit } from '../../../common/log_entry'; +import { + getLogEntryCursorFromHit, + LogEntryAfterCursor, + logEntryAfterCursorRT, + LogEntryBeforeCursor, + logEntryBeforeCursorRT, +} from '../../../common/log_entry'; import { decodeOrThrow } from '../../../common/runtime_types'; import { LogEntriesSearchRequestParams, @@ -68,20 +74,22 @@ export const logEntriesSearchStrategyProvider = ({ concatMap(({ params }) => sourceConfiguration$.pipe( map( - ({ configuration }): IEsSearchRequest => ({ - params: createGetLogEntriesQuery( - configuration.logAlias, - params.startTimestamp, - params.endTimestamp, - undefined, // TODO: determine cursor - params.size, - configuration.fields.timestamp, - configuration.fields.tiebreaker, - [], // TODO: determine fields list - params.query, - undefined // TODO: map over highlight terms OR reduce to one term in request - ), - }) + ({ configuration }): IEsSearchRequest => { + return { + params: createGetLogEntriesQuery( + configuration.logAlias, + params.startTimestamp, + params.endTimestamp, + pickCursor(params), + params.size, + configuration.fields.timestamp, + configuration.fields.tiebreaker, + [], // TODO: determine fields list + params.query, + undefined // TODO: map over highlight terms OR reduce to one term in request + ), + }; + } ) ) ) @@ -134,3 +142,15 @@ const createLogEntryFromHit = (hit: LogEntryHit) => ({ cursor: getLogEntryCursorFromHit(hit), fields: Object.entries(hit.fields).map(([field, value]) => ({ field, value })), }); + +const pickCursor = ( + params: LogEntriesSearchRequestParams +): LogEntryAfterCursor | LogEntryBeforeCursor | null => { + if (logEntryAfterCursorRT.is(params)) { + return pick(params, ['after']); + } else if (logEntryBeforeCursorRT.is(params)) { + return pick(params, ['before']); + } + + return null; +}; From 1ce7b3df0dd53baad00ed27650ce2842e175d9f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20St=C3=BCrmer?= Date: Tue, 12 Jan 2021 16:28:31 +0100 Subject: [PATCH 06/39] Fix cursor property on single log entry test --- .../services/log_entries/log_entry_search_strategy.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/infra/server/services/log_entries/log_entry_search_strategy.test.ts b/x-pack/plugins/infra/server/services/log_entries/log_entry_search_strategy.test.ts index 38626675f5ae7b..b3e1a31f73b7ab 100644 --- a/x-pack/plugins/infra/server/services/log_entries/log_entry_search_strategy.test.ts +++ b/x-pack/plugins/infra/server/services/log_entries/log_entry_search_strategy.test.ts @@ -121,7 +121,7 @@ describe('LogEntry search strategy', () => { expect(response.rawResponse.data).toEqual({ id: 'HIT_ID', index: 'HIT_INDEX', - key: { + cursor: { time: 1605116827143, tiebreaker: 1, }, From 6527f4cfec47eb8939ca4634de58537cba200bf2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20St=C3=BCrmer?= Date: Tue, 12 Jan 2021 19:54:38 +0100 Subject: [PATCH 07/39] Wire up field fetching and message reconstruction --- .../log_sources/log_source_configuration.ts | 1 + .../infra/common/log_entry/log_entry.ts | 1 + .../log_entries/log_entries.ts | 2 +- .../log_entries_domain/log_entries_domain.ts | 8 +- .../log_entries_search_strategy.test.ts | 313 ++++++++++++++++++ .../log_entries_search_strategy.ts | 125 +++++-- .../builtin_rules/filebeat_apache2.test.ts | 0 .../builtin_rules/filebeat_apache2.ts | 0 .../builtin_rules/filebeat_auditd.test.ts | 0 .../message}/builtin_rules/filebeat_auditd.ts | 0 .../builtin_rules/filebeat_haproxy.test.ts | 0 .../builtin_rules/filebeat_haproxy.ts | 0 .../builtin_rules/filebeat_icinga.test.ts | 0 .../message}/builtin_rules/filebeat_icinga.ts | 0 .../builtin_rules/filebeat_iis.test.ts | 0 .../message}/builtin_rules/filebeat_iis.ts | 0 .../builtin_rules/filebeat_kafka.test.ts | 0 .../builtin_rules/filebeat_logstash.test.ts | 0 .../builtin_rules/filebeat_logstash.ts | 0 .../builtin_rules/filebeat_mongodb.test.ts | 0 .../builtin_rules/filebeat_mongodb.ts | 0 .../builtin_rules/filebeat_mysql.test.ts | 0 .../message}/builtin_rules/filebeat_mysql.ts | 0 .../builtin_rules/filebeat_nginx.test.ts | 0 .../message}/builtin_rules/filebeat_nginx.ts | 0 .../builtin_rules/filebeat_osquery.test.ts | 0 .../builtin_rules/filebeat_osquery.ts | 0 .../message}/builtin_rules/filebeat_redis.ts | 0 .../message}/builtin_rules/filebeat_system.ts | 0 .../builtin_rules/filebeat_traefik.test.ts | 0 .../builtin_rules/filebeat_traefik.ts | 0 .../message}/builtin_rules/generic.test.ts | 0 .../message}/builtin_rules/generic.ts | 0 .../builtin_rules/generic_webserver.ts | 0 .../message}/builtin_rules/helpers.ts | 0 .../message}/builtin_rules/index.ts | 0 .../services/log_entries/message/index.ts | 8 + .../log_entries/message}/message.ts | 2 +- .../log_entries/message}/rule_types.ts | 0 .../log_entries/queries/log_entries.ts | 4 +- 40 files changed, 431 insertions(+), 33 deletions(-) create mode 100644 x-pack/plugins/infra/server/services/log_entries/log_entries_search_strategy.test.ts rename x-pack/plugins/infra/server/{lib/domains/log_entries_domain => services/log_entries/message}/builtin_rules/filebeat_apache2.test.ts (100%) rename x-pack/plugins/infra/server/{lib/domains/log_entries_domain => services/log_entries/message}/builtin_rules/filebeat_apache2.ts (100%) rename x-pack/plugins/infra/server/{lib/domains/log_entries_domain => services/log_entries/message}/builtin_rules/filebeat_auditd.test.ts (100%) rename x-pack/plugins/infra/server/{lib/domains/log_entries_domain => services/log_entries/message}/builtin_rules/filebeat_auditd.ts (100%) rename x-pack/plugins/infra/server/{lib/domains/log_entries_domain => services/log_entries/message}/builtin_rules/filebeat_haproxy.test.ts (100%) rename x-pack/plugins/infra/server/{lib/domains/log_entries_domain => services/log_entries/message}/builtin_rules/filebeat_haproxy.ts (100%) rename x-pack/plugins/infra/server/{lib/domains/log_entries_domain => services/log_entries/message}/builtin_rules/filebeat_icinga.test.ts (100%) rename x-pack/plugins/infra/server/{lib/domains/log_entries_domain => services/log_entries/message}/builtin_rules/filebeat_icinga.ts (100%) rename x-pack/plugins/infra/server/{lib/domains/log_entries_domain => services/log_entries/message}/builtin_rules/filebeat_iis.test.ts (100%) rename x-pack/plugins/infra/server/{lib/domains/log_entries_domain => services/log_entries/message}/builtin_rules/filebeat_iis.ts (100%) rename x-pack/plugins/infra/server/{lib/domains/log_entries_domain => services/log_entries/message}/builtin_rules/filebeat_kafka.test.ts (100%) rename x-pack/plugins/infra/server/{lib/domains/log_entries_domain => services/log_entries/message}/builtin_rules/filebeat_logstash.test.ts (100%) rename x-pack/plugins/infra/server/{lib/domains/log_entries_domain => services/log_entries/message}/builtin_rules/filebeat_logstash.ts (100%) rename x-pack/plugins/infra/server/{lib/domains/log_entries_domain => services/log_entries/message}/builtin_rules/filebeat_mongodb.test.ts (100%) rename x-pack/plugins/infra/server/{lib/domains/log_entries_domain => services/log_entries/message}/builtin_rules/filebeat_mongodb.ts (100%) rename x-pack/plugins/infra/server/{lib/domains/log_entries_domain => services/log_entries/message}/builtin_rules/filebeat_mysql.test.ts (100%) rename x-pack/plugins/infra/server/{lib/domains/log_entries_domain => services/log_entries/message}/builtin_rules/filebeat_mysql.ts (100%) rename x-pack/plugins/infra/server/{lib/domains/log_entries_domain => services/log_entries/message}/builtin_rules/filebeat_nginx.test.ts (100%) rename x-pack/plugins/infra/server/{lib/domains/log_entries_domain => services/log_entries/message}/builtin_rules/filebeat_nginx.ts (100%) rename x-pack/plugins/infra/server/{lib/domains/log_entries_domain => services/log_entries/message}/builtin_rules/filebeat_osquery.test.ts (100%) rename x-pack/plugins/infra/server/{lib/domains/log_entries_domain => services/log_entries/message}/builtin_rules/filebeat_osquery.ts (100%) rename x-pack/plugins/infra/server/{lib/domains/log_entries_domain => services/log_entries/message}/builtin_rules/filebeat_redis.ts (100%) rename x-pack/plugins/infra/server/{lib/domains/log_entries_domain => services/log_entries/message}/builtin_rules/filebeat_system.ts (100%) rename x-pack/plugins/infra/server/{lib/domains/log_entries_domain => services/log_entries/message}/builtin_rules/filebeat_traefik.test.ts (100%) rename x-pack/plugins/infra/server/{lib/domains/log_entries_domain => services/log_entries/message}/builtin_rules/filebeat_traefik.ts (100%) rename x-pack/plugins/infra/server/{lib/domains/log_entries_domain => services/log_entries/message}/builtin_rules/generic.test.ts (100%) rename x-pack/plugins/infra/server/{lib/domains/log_entries_domain => services/log_entries/message}/builtin_rules/generic.ts (100%) rename x-pack/plugins/infra/server/{lib/domains/log_entries_domain => services/log_entries/message}/builtin_rules/generic_webserver.ts (100%) rename x-pack/plugins/infra/server/{lib/domains/log_entries_domain => services/log_entries/message}/builtin_rules/helpers.ts (100%) rename x-pack/plugins/infra/server/{lib/domains/log_entries_domain => services/log_entries/message}/builtin_rules/index.ts (100%) create mode 100644 x-pack/plugins/infra/server/services/log_entries/message/index.ts rename x-pack/plugins/infra/server/{lib/domains/log_entries_domain => services/log_entries/message}/message.ts (99%) rename x-pack/plugins/infra/server/{lib/domains/log_entries_domain => services/log_entries/message}/rule_types.ts (100%) diff --git a/x-pack/plugins/infra/common/http_api/log_sources/log_source_configuration.ts b/x-pack/plugins/infra/common/http_api/log_sources/log_source_configuration.ts index 7581e29692356f..df7d80d33f1e60 100644 --- a/x-pack/plugins/infra/common/http_api/log_sources/log_source_configuration.ts +++ b/x-pack/plugins/infra/common/http_api/log_sources/log_source_configuration.ts @@ -53,6 +53,7 @@ export const logSourceColumnConfigurationRT = rt.union([ logSourceMessageColumnConfigurationRT, logSourceFieldColumnConfigurationRT, ]); +export type LogSourceColumnConfiguration = rt.TypeOf; export const logSourceConfigurationPropertiesRT = rt.strict({ name: rt.string, diff --git a/x-pack/plugins/infra/common/log_entry/log_entry.ts b/x-pack/plugins/infra/common/log_entry/log_entry.ts index 2795a18cf7a242..bf3f9ceb0b0887 100644 --- a/x-pack/plugins/infra/common/log_entry/log_entry.ts +++ b/x-pack/plugins/infra/common/log_entry/log_entry.ts @@ -76,6 +76,7 @@ export type LogEntryField = rt.TypeOf; export const logEntryRT = rt.type({ id: rt.string, + index: rt.string, cursor: logEntryCursorRT, columns: rt.array(logColumnRT), context: logEntryContextRT, diff --git a/x-pack/plugins/infra/common/search_strategies/log_entries/log_entries.ts b/x-pack/plugins/infra/common/search_strategies/log_entries/log_entries.ts index 6b168f91e6ff22..00ac40cf9e220a 100644 --- a/x-pack/plugins/infra/common/search_strategies/log_entries/log_entries.ts +++ b/x-pack/plugins/infra/common/search_strategies/log_entries/log_entries.ts @@ -28,7 +28,7 @@ const logEntriesBaseSearchRequestParamsRT = rt.intersection([ rt.partial({ query: jsonObjectRT, columns: rt.array(logSourceColumnConfigurationRT), - highlightTerms: rt.array(rt.string), + highlightPhrase: rt.string, }), ]); diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/log_entries_domain.ts b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/log_entries_domain.ts index 0b1df3abd465a2..0808b241cb15a4 100644 --- a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/log_entries_domain.ts +++ b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/log_entries_domain.ts @@ -10,8 +10,6 @@ import type { InfraPluginRequestHandlerContext } from '../../../types'; import { LogEntriesSummaryBucket, LogEntriesSummaryHighlightsBucket, - LogEntry, - LogColumn, LogEntriesRequest, } from '../../../../common/http_api'; import { @@ -19,13 +17,13 @@ import { InfraSources, SavedSourceConfigurationFieldColumnRuntimeType, } from '../../sources'; -import { getBuiltinRules } from './builtin_rules'; +import { getBuiltinRules } from '../../../services/log_entries/message/builtin_rules'; import { CompiledLogMessageFormattingRule, Fields, Highlights, compileFormattingRules, -} from './message'; +} from '../../../services/log_entries/message/message'; import { KibanaFramework } from '../../adapters/framework/kibana_framework_adapter'; import { decodeOrThrow } from '../../../../common/runtime_types'; import { @@ -34,7 +32,7 @@ import { CompositeDatasetKey, createLogEntryDatasetsQuery, } from './queries/log_entry_datasets'; -import { LogEntryCursor } from '../../../../common/log_entry'; +import { LogEntry, LogColumn, LogEntryCursor } from '../../../../common/log_entry'; export interface LogEntriesParams { startTimestamp: number; diff --git a/x-pack/plugins/infra/server/services/log_entries/log_entries_search_strategy.test.ts b/x-pack/plugins/infra/server/services/log_entries/log_entries_search_strategy.test.ts new file mode 100644 index 00000000000000..b86b4727b33779 --- /dev/null +++ b/x-pack/plugins/infra/server/services/log_entries/log_entries_search_strategy.test.ts @@ -0,0 +1,313 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { ResponseError } from '@elastic/elasticsearch/lib/errors'; +import { of, throwError } from 'rxjs'; +import { + elasticsearchServiceMock, + savedObjectsClientMock, + uiSettingsServiceMock, +} from 'src/core/server/mocks'; +import { + IEsSearchRequest, + IEsSearchResponse, + ISearchStrategy, + SearchStrategyDependencies, +} from 'src/plugins/data/server'; +import { InfraSource } from '../../lib/sources'; +import { createInfraSourcesMock } from '../../lib/sources/mocks'; +import { + logEntriesSearchRequestStateRT, + logEntriesSearchStrategyProvider, +} from './log_entries_search_strategy'; + +describe('LogEntries search strategy', () => { + it('handles initial search requests', async () => { + const esSearchStrategyMock = createEsSearchStrategyMock({ + id: 'ASYNC_REQUEST_ID', + isRunning: true, + rawResponse: { + took: 0, + _shards: { total: 1, failed: 0, skipped: 0, successful: 0 }, + timed_out: false, + hits: { total: 0, max_score: 0, hits: [] }, + }, + }); + + const dataMock = createDataPluginMock(esSearchStrategyMock); + const sourcesMock = createInfraSourcesMock(); + sourcesMock.getSourceConfiguration.mockResolvedValue(createSourceConfigurationMock()); + const mockDependencies = createSearchStrategyDependenciesMock(); + + const logEntriesSearchStrategy = logEntriesSearchStrategyProvider({ + data: dataMock, + sources: sourcesMock, + }); + + const response = await logEntriesSearchStrategy + .search( + { + params: { + sourceId: 'SOURCE_ID', + startTimestamp: 100, + endTimestamp: 200, + size: 3, + }, + }, + {}, + mockDependencies + ) + .toPromise(); + + expect(sourcesMock.getSourceConfiguration).toHaveBeenCalled(); + expect(esSearchStrategyMock.search).toHaveBeenCalledWith( + expect.objectContaining({ + params: expect.objectContaining({ + index: 'log-indices-*', + body: expect.objectContaining({ + fields: expect.arrayContaining(['event.dataset', 'message']), + }), + }), + }), + expect.anything(), + expect.anything() + ); + expect(response.id).toEqual(expect.any(String)); + expect(response.isRunning).toBe(true); + }); + + it('handles subsequent polling requests', async () => { + const esSearchStrategyMock = createEsSearchStrategyMock({ + id: 'ASYNC_REQUEST_ID', + isRunning: false, + rawResponse: { + took: 1, + _shards: { total: 1, failed: 0, skipped: 0, successful: 1 }, + timed_out: false, + hits: { + total: 0, + max_score: 0, + hits: [ + { + _id: 'HIT_ID', + _index: 'HIT_INDEX', + _type: '_doc', + _score: 0, + _source: null, + fields: { + '@timestamp': [1605116827143], + 'event.dataset': ['HIT_DATASET'], + MESSAGE_FIELD: ['HIT_MESSAGE'], + 'container.id': ['HIT_CONTAINER_ID'], + }, + sort: [1605116827143 as any, 1 as any], // incorrectly typed as string upstream + }, + ], + }, + }, + }); + const dataMock = createDataPluginMock(esSearchStrategyMock); + const sourcesMock = createInfraSourcesMock(); + sourcesMock.getSourceConfiguration.mockResolvedValue(createSourceConfigurationMock()); + const mockDependencies = createSearchStrategyDependenciesMock(); + + const logEntriesSearchStrategy = logEntriesSearchStrategyProvider({ + data: dataMock, + sources: sourcesMock, + }); + const requestId = logEntriesSearchRequestStateRT.encode({ + esRequestId: 'ASYNC_REQUEST_ID', + }); + + const response = await logEntriesSearchStrategy + .search( + { + id: requestId, + params: { + sourceId: 'SOURCE_ID', + startTimestamp: 100, + endTimestamp: 200, + size: 3, + }, + }, + {}, + mockDependencies + ) + .toPromise(); + + expect(sourcesMock.getSourceConfiguration).toHaveBeenCalled(); + expect(esSearchStrategyMock.search).toHaveBeenCalled(); + expect(response.id).toEqual(requestId); + expect(response.isRunning).toBe(false); + expect(response.rawResponse.data.entries).toEqual([ + { + id: 'HIT_ID', + index: 'HIT_INDEX', + cursor: { + time: 1605116827143, + tiebreaker: 1, + }, + columns: [ + { + columnId: 'TIMESTAMP_COLUMN_ID', + timestamp: 1605116827143, + }, + { + columnId: 'DATASET_COLUMN_ID', + field: 'event.dataset', + value: ['HIT_DATASET'], + highlights: [], + }, + { + columnId: 'MESSAGE_COLUMN_ID', + message: [ + { + field: 'MESSAGE_FIELD', + value: ['HIT_MESSAGE'], + highlights: [], + }, + ], + }, + ], + context: { + 'container.id': 'HIT_CONTAINER_ID', + }, + }, + ]); + }); + + it.skip('forwards errors from the underlying search strategy', async () => { + const esSearchStrategyMock = createEsSearchStrategyMock({ + id: 'ASYNC_REQUEST_ID', + isRunning: false, + rawResponse: { + took: 1, + _shards: { total: 1, failed: 0, skipped: 0, successful: 1 }, + timed_out: false, + hits: { total: 0, max_score: 0, hits: [] }, + }, + }); + const dataMock = createDataPluginMock(esSearchStrategyMock); + const sourcesMock = createInfraSourcesMock(); + sourcesMock.getSourceConfiguration.mockResolvedValue(createSourceConfigurationMock()); + const mockDependencies = createSearchStrategyDependenciesMock(); + + const logEntriesSearchStrategy = logEntriesSearchStrategyProvider({ + data: dataMock, + sources: sourcesMock, + }); + + const response = logEntriesSearchStrategy.search( + { + id: logEntriesSearchRequestStateRT.encode({ esRequestId: 'UNKNOWN_ID' }), + params: { sourceId: 'SOURCE_ID', logEntryId: 'LOG_ENTRY_ID' }, + }, + {}, + mockDependencies + ); + + await expect(response.toPromise()).rejects.toThrowError(ResponseError); + }); + + it.skip('forwards cancellation to the underlying search strategy', async () => { + const esSearchStrategyMock = createEsSearchStrategyMock({ + id: 'ASYNC_REQUEST_ID', + isRunning: false, + rawResponse: { + took: 1, + _shards: { total: 1, failed: 0, skipped: 0, successful: 1 }, + timed_out: false, + hits: { total: 0, max_score: 0, hits: [] }, + }, + }); + const dataMock = createDataPluginMock(esSearchStrategyMock); + const sourcesMock = createInfraSourcesMock(); + sourcesMock.getSourceConfiguration.mockResolvedValue(createSourceConfigurationMock()); + const mockDependencies = createSearchStrategyDependenciesMock(); + + const logEntriesSearchStrategy = logEntriesSearchStrategyProvider({ + data: dataMock, + sources: sourcesMock, + }); + const requestId = logEntriesSearchRequestStateRT.encode({ + esRequestId: 'ASYNC_REQUEST_ID', + }); + + await logEntriesSearchStrategy.cancel?.(requestId, {}, mockDependencies); + + expect(esSearchStrategyMock.cancel).toHaveBeenCalled(); + }); +}); + +const createSourceConfigurationMock = (): InfraSource => ({ + id: 'SOURCE_ID', + origin: 'stored' as const, + configuration: { + name: 'SOURCE_NAME', + description: 'SOURCE_DESCRIPTION', + logAlias: 'log-indices-*', + metricAlias: 'metric-indices-*', + inventoryDefaultView: 'DEFAULT_VIEW', + metricsExplorerDefaultView: 'DEFAULT_VIEW', + logColumns: [ + { timestampColumn: { id: 'TIMESTAMP_COLUMN_ID' } }, + { + fieldColumn: { + id: 'DATASET_COLUMN_ID', + field: 'event.dataset', + }, + }, + { + messageColumn: { id: 'MESSAGE_COLUMN_ID' }, + }, + ], + fields: { + pod: 'POD_FIELD', + host: 'HOST_FIELD', + container: 'CONTAINER_FIELD', + message: ['MESSAGE_FIELD'], + timestamp: 'TIMESTAMP_FIELD', + tiebreaker: 'TIEBREAKER_FIELD', + }, + }, +}); + +const createEsSearchStrategyMock = (esSearchResponse: IEsSearchResponse) => ({ + search: jest.fn((esSearchRequest: IEsSearchRequest) => { + if (typeof esSearchRequest.id === 'string') { + if (esSearchRequest.id === esSearchResponse.id) { + return of(esSearchResponse); + } else { + return throwError( + new ResponseError({ + body: {}, + headers: {}, + meta: {} as any, + statusCode: 404, + warnings: [], + }) + ); + } + } else { + return of(esSearchResponse); + } + }), + cancel: jest.fn().mockResolvedValue(undefined), +}); + +const createSearchStrategyDependenciesMock = (): SearchStrategyDependencies => ({ + uiSettingsClient: uiSettingsServiceMock.createClient(), + esClient: elasticsearchServiceMock.createScopedClusterClient(), + savedObjectsClient: savedObjectsClientMock.create(), +}); + +// using the official data mock from within x-pack doesn't type-check successfully, +// because the `licensing` plugin modifies the `RequestHandlerContext` core type. +const createDataPluginMock = (esSearchStrategyMock: ISearchStrategy): any => ({ + search: { + getSearchStrategy: jest.fn().mockReturnValue(esSearchStrategyMock), + }, +}); diff --git a/x-pack/plugins/infra/server/services/log_entries/log_entries_search_strategy.ts b/x-pack/plugins/infra/server/services/log_entries/log_entries_search_strategy.ts index d4e09c54683d7a..39bc0d68dc0aa5 100644 --- a/x-pack/plugins/infra/server/services/log_entries/log_entries_search_strategy.ts +++ b/x-pack/plugins/infra/server/services/log_entries/log_entries_search_strategy.ts @@ -6,7 +6,7 @@ import { pick } from '@kbn/std'; import * as rt from 'io-ts'; -import { concat, defer, of } from 'rxjs'; +import { concat, defer, forkJoin, of } from 'rxjs'; import { concatMap, filter, map, shareReplay, take } from 'rxjs/operators'; import type { IEsSearchRequest, @@ -17,12 +17,16 @@ import type { ISearchStrategy, PluginStart as DataPluginStart, } from '../../../../../../src/plugins/data/server'; +import { LogSourceColumnConfiguration } from '../../../common/http_api/log_sources'; import { getLogEntryCursorFromHit, + LogColumn, + LogEntry, LogEntryAfterCursor, logEntryAfterCursorRT, LogEntryBeforeCursor, logEntryBeforeCursorRT, + LogEntryContext, } from '../../../common/log_entry'; import { decodeOrThrow } from '../../../common/runtime_types'; import { @@ -37,6 +41,11 @@ import { createErrorFromShardFailure, jsonFromBase64StringRT, } from '../../utils/typed_search_strategy'; +import { + CompiledLogMessageFormattingRule, + compileFormattingRules, + getBuiltinRules, +} from './message'; import { createGetLogEntriesQuery, getLogEntriesResponseRT, @@ -62,7 +71,15 @@ export const logEntriesSearchStrategyProvider = ({ const sourceConfiguration$ = defer(() => sources.getSourceConfiguration(dependencies.savedObjectsClient, request.params.sourceId) - ).pipe(shareReplay(1)); + ).pipe(take(1), shareReplay(1)); + + const messageFormattingRules$ = defer(() => + sourceConfiguration$.pipe( + map(({ configuration }) => + compileFormattingRules(getBuiltinRules(configuration.fields.message)) + ) + ) + ).pipe(take(1), shareReplay(1)); const recoveredRequest$ = of(request).pipe( filter(asyncRecoveredRequestRT.is), @@ -72,9 +89,9 @@ export const logEntriesSearchStrategyProvider = ({ const initialRequest$ = of(request).pipe( filter(asyncInitialRequestRT.is), concatMap(({ params }) => - sourceConfiguration$.pipe( + forkJoin([sourceConfiguration$, messageFormattingRules$]).pipe( map( - ({ configuration }): IEsSearchRequest => { + ([{ configuration }, messageFormattingRules]): IEsSearchRequest => { return { params: createGetLogEntriesQuery( configuration.logAlias, @@ -84,9 +101,9 @@ export const logEntriesSearchStrategyProvider = ({ params.size, configuration.fields.timestamp, configuration.fields.tiebreaker, - [], // TODO: determine fields list + messageFormattingRules.requiredFields, params.query, - undefined // TODO: map over highlight terms OR reduce to one term in request + params.highlightPhrase ), }; } @@ -102,19 +119,31 @@ export const logEntriesSearchStrategyProvider = ({ ...esResponse, rawResponse: decodeOrThrow(getLogEntriesResponseRT)(esResponse.rawResponse), })), - map((esResponse) => ({ - ...esResponse, - ...(esResponse.id - ? { id: logEntriesSearchRequestStateRT.encode({ esRequestId: esResponse.id }) } - : {}), - rawResponse: logEntriesSearchResponsePayloadRT.encode({ - data: { entries: [], topCursor: null, bottomCursor: null }, - // data: esResponse.rawResponse.hits.hits.map(createLogEntryFromHit)[0] ?? null, - errors: (esResponse.rawResponse._shards.failures ?? []).map( - createErrorFromShardFailure - ), - }), - })) + concatMap((esResponse) => + forkJoin([sourceConfiguration$, messageFormattingRules$]).pipe( + map(([{ configuration }, messageFormattingRules]) => { + const entries = esResponse.rawResponse.hits.hits.map( + createLogEntryFromHit(configuration.logColumns, messageFormattingRules) + ); + const topCursor = null; + const bottomCursor = null; + + return { + ...esResponse, + ...(esResponse.id + ? { id: logEntriesSearchRequestStateRT.encode({ esRequestId: esResponse.id }) } + : {}), + rawResponse: logEntriesSearchResponsePayloadRT.encode({ + data: { entries, topCursor, bottomCursor }, + // data: esResponse.rawResponse.hits.hits.map(createLogEntryFromHit)[0] ?? null, + errors: (esResponse.rawResponse._shards.failures ?? []).map( + createErrorFromShardFailure + ), + }), + }; + }) + ) + ) ); }), cancel: async (id, options, dependencies) => { @@ -136,12 +165,41 @@ const { asyncInitialRequestRT, asyncRecoveredRequestRT, asyncRequestRT } = creat logEntriesSearchRequestParamsRT ); -const createLogEntryFromHit = (hit: LogEntryHit) => ({ - id: hit._id, - index: hit._index, - cursor: getLogEntryCursorFromHit(hit), - fields: Object.entries(hit.fields).map(([field, value]) => ({ field, value })), -}); +const createLogEntryFromHit = ( + columnDefinitions: LogSourceColumnConfiguration[], + messageFormattingRules: CompiledLogMessageFormattingRule +) => (hit: LogEntryHit): LogEntry => { + const cursor = getLogEntryCursorFromHit(hit); + return { + id: hit._id, + index: hit._index, + cursor, + columns: columnDefinitions.map( + (column): LogColumn => { + if ('timestampColumn' in column) { + return { + columnId: column.timestampColumn.id, + timestamp: cursor.time, + }; + } else if ('messageColumn' in column) { + return { + columnId: column.messageColumn.id, + message: messageFormattingRules.format(hit.fields, hit.highlight || {}), + }; + } else { + return { + columnId: column.fieldColumn.id, + field: column.fieldColumn.field, + value: hit.fields[column.fieldColumn.field] ?? [], + highlights: hit.highlight?.[column.fieldColumn.field] ?? [], + }; + } + } + ), + context: getContextFromHit(hit), + // fields: Object.entries(hit.fields).map(([field, value]) => ({ field, value })), + }; +}; const pickCursor = ( params: LogEntriesSearchRequestParams @@ -154,3 +212,20 @@ const pickCursor = ( return null; }; + +const getContextFromHit = (hit: LogEntryHit): LogEntryContext => { + // Get all context fields, then test for the presence and type of the ones that go together + const containerId = hit.fields['container.id']?.[0]; + const hostName = hit.fields['host.name']?.[0]; + const logFilePath = hit.fields['log.file.path']?.[0]; + + if (typeof containerId === 'string') { + return { 'container.id': containerId }; + } + + if (typeof hostName === 'string' && typeof logFilePath === 'string') { + return { 'host.name': hostName, 'log.file.path': logFilePath }; + } + + return {}; +}; diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_apache2.test.ts b/x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_apache2.test.ts similarity index 100% rename from x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_apache2.test.ts rename to x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_apache2.test.ts diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_apache2.ts b/x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_apache2.ts similarity index 100% rename from x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_apache2.ts rename to x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_apache2.ts diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_auditd.test.ts b/x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_auditd.test.ts similarity index 100% rename from x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_auditd.test.ts rename to x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_auditd.test.ts diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_auditd.ts b/x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_auditd.ts similarity index 100% rename from x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_auditd.ts rename to x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_auditd.ts diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_haproxy.test.ts b/x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_haproxy.test.ts similarity index 100% rename from x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_haproxy.test.ts rename to x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_haproxy.test.ts diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_haproxy.ts b/x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_haproxy.ts similarity index 100% rename from x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_haproxy.ts rename to x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_haproxy.ts diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_icinga.test.ts b/x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_icinga.test.ts similarity index 100% rename from x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_icinga.test.ts rename to x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_icinga.test.ts diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_icinga.ts b/x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_icinga.ts similarity index 100% rename from x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_icinga.ts rename to x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_icinga.ts diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_iis.test.ts b/x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_iis.test.ts similarity index 100% rename from x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_iis.test.ts rename to x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_iis.test.ts diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_iis.ts b/x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_iis.ts similarity index 100% rename from x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_iis.ts rename to x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_iis.ts diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_kafka.test.ts b/x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_kafka.test.ts similarity index 100% rename from x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_kafka.test.ts rename to x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_kafka.test.ts diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_logstash.test.ts b/x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_logstash.test.ts similarity index 100% rename from x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_logstash.test.ts rename to x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_logstash.test.ts diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_logstash.ts b/x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_logstash.ts similarity index 100% rename from x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_logstash.ts rename to x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_logstash.ts diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_mongodb.test.ts b/x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_mongodb.test.ts similarity index 100% rename from x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_mongodb.test.ts rename to x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_mongodb.test.ts diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_mongodb.ts b/x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_mongodb.ts similarity index 100% rename from x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_mongodb.ts rename to x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_mongodb.ts diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_mysql.test.ts b/x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_mysql.test.ts similarity index 100% rename from x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_mysql.test.ts rename to x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_mysql.test.ts diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_mysql.ts b/x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_mysql.ts similarity index 100% rename from x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_mysql.ts rename to x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_mysql.ts diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_nginx.test.ts b/x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_nginx.test.ts similarity index 100% rename from x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_nginx.test.ts rename to x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_nginx.test.ts diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_nginx.ts b/x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_nginx.ts similarity index 100% rename from x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_nginx.ts rename to x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_nginx.ts diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_osquery.test.ts b/x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_osquery.test.ts similarity index 100% rename from x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_osquery.test.ts rename to x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_osquery.test.ts diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_osquery.ts b/x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_osquery.ts similarity index 100% rename from x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_osquery.ts rename to x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_osquery.ts diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_redis.ts b/x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_redis.ts similarity index 100% rename from x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_redis.ts rename to x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_redis.ts diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_system.ts b/x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_system.ts similarity index 100% rename from x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_system.ts rename to x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_system.ts diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_traefik.test.ts b/x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_traefik.test.ts similarity index 100% rename from x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_traefik.test.ts rename to x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_traefik.test.ts diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_traefik.ts b/x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_traefik.ts similarity index 100% rename from x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_traefik.ts rename to x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_traefik.ts diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/generic.test.ts b/x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/generic.test.ts similarity index 100% rename from x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/generic.test.ts rename to x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/generic.test.ts diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/generic.ts b/x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/generic.ts similarity index 100% rename from x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/generic.ts rename to x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/generic.ts diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/generic_webserver.ts b/x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/generic_webserver.ts similarity index 100% rename from x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/generic_webserver.ts rename to x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/generic_webserver.ts diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/helpers.ts b/x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/helpers.ts similarity index 100% rename from x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/helpers.ts rename to x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/helpers.ts diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/index.ts b/x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/index.ts similarity index 100% rename from x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/index.ts rename to x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/index.ts diff --git a/x-pack/plugins/infra/server/services/log_entries/message/index.ts b/x-pack/plugins/infra/server/services/log_entries/message/index.ts new file mode 100644 index 00000000000000..05126eea075afe --- /dev/null +++ b/x-pack/plugins/infra/server/services/log_entries/message/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export * from './message'; +export { getBuiltinRules } from './builtin_rules'; diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/message.ts b/x-pack/plugins/infra/server/services/log_entries/message/message.ts similarity index 99% rename from x-pack/plugins/infra/server/lib/domains/log_entries_domain/message.ts rename to x-pack/plugins/infra/server/services/log_entries/message/message.ts index 19ab82c9c5ac11..d04e036b33b215 100644 --- a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/message.ts +++ b/x-pack/plugins/infra/server/services/log_entries/message/message.ts @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import { LogMessagePart } from '../../../../common/http_api/log_entries'; +import { LogMessagePart } from '../../../../common/log_entry'; import { JsonArray, JsonValue } from '../../../../../../../src/plugins/kibana_utils/common'; import { LogMessageFormattingCondition, diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/rule_types.ts b/x-pack/plugins/infra/server/services/log_entries/message/rule_types.ts similarity index 100% rename from x-pack/plugins/infra/server/lib/domains/log_entries_domain/rule_types.ts rename to x-pack/plugins/infra/server/services/log_entries/message/rule_types.ts diff --git a/x-pack/plugins/infra/server/services/log_entries/queries/log_entries.ts b/x-pack/plugins/infra/server/services/log_entries/queries/log_entries.ts index c727bb7d808601..9bd05caf9c191e 100644 --- a/x-pack/plugins/infra/server/services/log_entries/queries/log_entries.ts +++ b/x-pack/plugins/infra/server/services/log_entries/queries/log_entries.ts @@ -121,9 +121,11 @@ export const logEntryHitRT = rt.intersection([ commonHitFieldsRT, rt.type({ fields: rt.record(rt.string, jsonArrayRT), - highlight: rt.record(rt.string, rt.array(rt.string)), sort: rt.tuple([rt.number, rt.number]), }), + rt.partial({ + highlight: rt.record(rt.string, rt.array(rt.string)), + }), ]); export type LogEntryHit = rt.TypeOf; From d13086af70b103b5b713897ca175ca3f27fc4209 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20St=C3=BCrmer?= Date: Wed, 13 Jan 2021 12:54:03 +0100 Subject: [PATCH 08/39] Remove unused graphql types --- x-pack/plugins/infra/common/graphql/types.ts | 337 --------- .../log_entry_message_column.tsx | 2 +- .../log_entry_highlights.gql_query.ts | 42 -- .../log_highlights/log_entry_highlights.tsx | 6 +- .../infra/public/graphql/introspection.json | 660 ------------------ x-pack/plugins/infra/public/graphql/types.ts | 337 --------- .../utils/log_entry/log_entry_highlight.ts | 10 +- x-pack/plugins/infra/server/graphql/types.ts | 406 ----------- 8 files changed, 5 insertions(+), 1795 deletions(-) delete mode 100644 x-pack/plugins/infra/public/containers/logs/log_highlights/log_entry_highlights.gql_query.ts diff --git a/x-pack/plugins/infra/common/graphql/types.ts b/x-pack/plugins/infra/common/graphql/types.ts index 4a18c3d5ff3340..ee536feb1ce653 100644 --- a/x-pack/plugins/infra/common/graphql/types.ts +++ b/x-pack/plugins/infra/common/graphql/types.ts @@ -28,12 +28,6 @@ export interface InfraSource { configuration: InfraSourceConfiguration; /** The status of the source */ status: InfraSourceStatus; - /** A consecutive span of log entries surrounding a point in time */ - logEntriesAround: InfraLogEntryInterval; - /** A consecutive span of log entries within an interval */ - logEntriesBetween: InfraLogEntryInterval; - /** Sequences of log entries matching sets of highlighting queries within an interval */ - logEntryHighlights: InfraLogEntryInterval[]; /** A snapshot of nodes */ snapshot?: InfraSnapshotResponse | null; @@ -129,80 +123,6 @@ export interface InfraIndexField { /** Whether the field should be displayed based on event.module and a ECS allowed list */ displayable: boolean; } -/** A consecutive sequence of log entries */ -export interface InfraLogEntryInterval { - /** The key corresponding to the start of the interval covered by the entries */ - start?: InfraTimeKey | null; - /** The key corresponding to the end of the interval covered by the entries */ - end?: InfraTimeKey | null; - /** Whether there are more log entries available before the start */ - hasMoreBefore: boolean; - /** Whether there are more log entries available after the end */ - hasMoreAfter: boolean; - /** The query the log entries were filtered by */ - filterQuery?: string | null; - /** The query the log entries were highlighted with */ - highlightQuery?: string | null; - /** A list of the log entries */ - entries: InfraLogEntry[]; -} -/** A representation of the log entry's position in the event stream */ -export interface InfraTimeKey { - /** The timestamp of the event that the log entry corresponds to */ - time: number; - /** The tiebreaker that disambiguates events with the same timestamp */ - tiebreaker: number; -} -/** A log entry */ -export interface InfraLogEntry { - /** A unique representation of the log entry's position in the event stream */ - key: InfraTimeKey; - /** The log entry's id */ - gid: string; - /** The source id */ - source: string; - /** The columns used for rendering the log entry */ - columns: InfraLogEntryColumn[]; -} -/** A special built-in column that contains the log entry's timestamp */ -export interface InfraLogEntryTimestampColumn { - /** The id of the corresponding column configuration */ - columnId: string; - /** The timestamp */ - timestamp: number; -} -/** A special built-in column that contains the log entry's constructed message */ -export interface InfraLogEntryMessageColumn { - /** The id of the corresponding column configuration */ - columnId: string; - /** A list of the formatted log entry segments */ - message: InfraLogMessageSegment[]; -} -/** A segment of the log entry message that was derived from a field */ -export interface InfraLogMessageFieldSegment { - /** The field the segment was derived from */ - field: string; - /** The segment's message */ - value: string; - /** A list of highlighted substrings of the value */ - highlights: string[]; -} -/** A segment of the log entry message that was derived from a string literal */ -export interface InfraLogMessageConstantSegment { - /** The segment's message */ - constant: string; -} -/** A column that contains the value of a field of the log entry */ -export interface InfraLogEntryFieldColumn { - /** The id of the corresponding column configuration */ - columnId: string; - /** The field name of the column */ - field: string; - /** The value of the field in the log entry */ - value: string; - /** A list of highlighted substrings of the value */ - highlights: string[]; -} export interface InfraSnapshotResponse { /** Nodes of type host, container or pod grouped by 0, 1 or 2 terms */ @@ -276,21 +196,6 @@ export interface DeleteSourceResult { // InputTypes // ==================================================== -export interface InfraTimeKeyInput { - time: number; - - tiebreaker: number; -} -/** A highlighting definition */ -export interface InfraLogEntryHighlightInput { - /** The query to highlight by */ - query: string; - /** The number of highlighted documents to include beyond the beginning of the interval */ - countBefore: number; - /** The number of highlighted documents to include beyond the end of the interval */ - countAfter: number; -} - export interface InfraTimerangeInput { /** The interval string to use for last bucket. The format is '{value}{unit}'. For example '5m' would return the metrics for the last 5 minutes of the timespan. */ interval: string; @@ -381,34 +286,6 @@ export interface SourceQueryArgs { /** The id of the source */ id: string; } -export interface LogEntriesAroundInfraSourceArgs { - /** The sort key that corresponds to the point in time */ - key: InfraTimeKeyInput; - /** The maximum number of preceding to return */ - countBefore?: number | null; - /** The maximum number of following to return */ - countAfter?: number | null; - /** The query to filter the log entries by */ - filterQuery?: string | null; -} -export interface LogEntriesBetweenInfraSourceArgs { - /** The sort key that corresponds to the start of the interval */ - startKey: InfraTimeKeyInput; - /** The sort key that corresponds to the end of the interval */ - endKey: InfraTimeKeyInput; - /** The query to filter the log entries by */ - filterQuery?: string | null; -} -export interface LogEntryHighlightsInfraSourceArgs { - /** The sort key that corresponds to the start of the interval */ - startKey: InfraTimeKeyInput; - /** The sort key that corresponds to the end of the interval */ - endKey: InfraTimeKeyInput; - /** The query to filter the log entries by */ - filterQuery?: string | null; - /** The highlighting to apply to the log entries */ - highlights: InfraLogEntryHighlightInput[]; -} export interface SnapshotInfraSourceArgs { timerange: InfraTimerangeInput; @@ -565,15 +442,6 @@ export type InfraSourceLogColumn = | InfraSourceMessageLogColumn | InfraSourceFieldLogColumn; -/** A column of a log entry */ -export type InfraLogEntryColumn = - | InfraLogEntryTimestampColumn - | InfraLogEntryMessageColumn - | InfraLogEntryFieldColumn; - -/** A segment of the log entry message */ -export type InfraLogMessageSegment = InfraLogMessageFieldSegment | InfraLogMessageConstantSegment; - // ==================================================== // END: Typescript template // ==================================================== @@ -582,46 +450,6 @@ export type InfraLogMessageSegment = InfraLogMessageFieldSegment | InfraLogMessa // Documents // ==================================================== -export namespace LogEntryHighlightsQuery { - export type Variables = { - sourceId?: string | null; - startKey: InfraTimeKeyInput; - endKey: InfraTimeKeyInput; - filterQuery?: string | null; - highlights: InfraLogEntryHighlightInput[]; - }; - - export type Query = { - __typename?: 'Query'; - - source: Source; - }; - - export type Source = { - __typename?: 'InfraSource'; - - id: string; - - logEntryHighlights: LogEntryHighlights[]; - }; - - export type LogEntryHighlights = { - __typename?: 'InfraLogEntryInterval'; - - start?: Start | null; - - end?: End | null; - - entries: Entries[]; - }; - - export type Start = InfraTimeKeyFields.Fragment; - - export type End = InfraTimeKeyFields.Fragment; - - export type Entries = InfraLogEntryHighlightFields.Fragment; -} - export namespace MetricsQuery { export type Variables = { sourceId: string; @@ -820,50 +648,6 @@ export namespace WaffleNodesQuery { }; } -export namespace LogEntries { - export type Variables = { - sourceId?: string | null; - timeKey: InfraTimeKeyInput; - countBefore?: number | null; - countAfter?: number | null; - filterQuery?: string | null; - }; - - export type Query = { - __typename?: 'Query'; - - source: Source; - }; - - export type Source = { - __typename?: 'InfraSource'; - - id: string; - - logEntriesAround: LogEntriesAround; - }; - - export type LogEntriesAround = { - __typename?: 'InfraLogEntryInterval'; - - start?: Start | null; - - end?: End | null; - - hasMoreBefore: boolean; - - hasMoreAfter: boolean; - - entries: Entries[]; - }; - - export type Start = InfraTimeKeyFields.Fragment; - - export type End = InfraTimeKeyFields.Fragment; - - export type Entries = InfraLogEntryFields.Fragment; -} - export namespace SourceConfigurationFields { export type Fragment = { __typename?: 'InfraSourceConfiguration'; @@ -994,124 +778,3 @@ export namespace InfraSourceFields { origin: string; }; } - -export namespace InfraLogEntryFields { - export type Fragment = { - __typename?: 'InfraLogEntry'; - - gid: string; - - key: Key; - - columns: Columns[]; - }; - - export type Key = { - __typename?: 'InfraTimeKey'; - - time: number; - - tiebreaker: number; - }; - - export type Columns = - | InfraLogEntryTimestampColumnInlineFragment - | InfraLogEntryMessageColumnInlineFragment - | InfraLogEntryFieldColumnInlineFragment; - - export type InfraLogEntryTimestampColumnInlineFragment = { - __typename?: 'InfraLogEntryTimestampColumn'; - - columnId: string; - - timestamp: number; - }; - - export type InfraLogEntryMessageColumnInlineFragment = { - __typename?: 'InfraLogEntryMessageColumn'; - - columnId: string; - - message: Message[]; - }; - - export type Message = - | InfraLogMessageFieldSegmentInlineFragment - | InfraLogMessageConstantSegmentInlineFragment; - - export type InfraLogMessageFieldSegmentInlineFragment = { - __typename?: 'InfraLogMessageFieldSegment'; - - field: string; - - value: string; - }; - - export type InfraLogMessageConstantSegmentInlineFragment = { - __typename?: 'InfraLogMessageConstantSegment'; - - constant: string; - }; - - export type InfraLogEntryFieldColumnInlineFragment = { - __typename?: 'InfraLogEntryFieldColumn'; - - columnId: string; - - field: string; - - value: string; - }; -} - -export namespace InfraLogEntryHighlightFields { - export type Fragment = { - __typename?: 'InfraLogEntry'; - - gid: string; - - key: Key; - - columns: Columns[]; - }; - - export type Key = { - __typename?: 'InfraTimeKey'; - - time: number; - - tiebreaker: number; - }; - - export type Columns = - | InfraLogEntryMessageColumnInlineFragment - | InfraLogEntryFieldColumnInlineFragment; - - export type InfraLogEntryMessageColumnInlineFragment = { - __typename?: 'InfraLogEntryMessageColumn'; - - columnId: string; - - message: Message[]; - }; - - export type Message = InfraLogMessageFieldSegmentInlineFragment; - - export type InfraLogMessageFieldSegmentInlineFragment = { - __typename?: 'InfraLogMessageFieldSegment'; - - field: string; - - highlights: string[]; - }; - - export type InfraLogEntryFieldColumnInlineFragment = { - __typename?: 'InfraLogEntryFieldColumn'; - - columnId: string; - - field: string; - - highlights: string[]; - }; -} diff --git a/x-pack/plugins/infra/public/components/logging/log_text_stream/log_entry_message_column.tsx b/x-pack/plugins/infra/public/components/logging/log_text_stream/log_entry_message_column.tsx index bfc160ada2e6a3..92214dee9de220 100644 --- a/x-pack/plugins/infra/public/components/logging/log_text_stream/log_entry_message_column.tsx +++ b/x-pack/plugins/infra/public/components/logging/log_text_stream/log_entry_message_column.tsx @@ -6,7 +6,7 @@ import React, { memo, useMemo } from 'react'; import { euiStyled } from '../../../../../../../src/plugins/kibana_react/common'; -import { LogColumn, LogMessagePart } from '../../../../common/http_api'; +import { LogColumn, LogMessagePart } from '../../../../common/log_entry'; import { isConstantSegment, isFieldSegment, diff --git a/x-pack/plugins/infra/public/containers/logs/log_highlights/log_entry_highlights.gql_query.ts b/x-pack/plugins/infra/public/containers/logs/log_highlights/log_entry_highlights.gql_query.ts deleted file mode 100644 index 9d9fab58754275..00000000000000 --- a/x-pack/plugins/infra/public/containers/logs/log_highlights/log_entry_highlights.gql_query.ts +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import gql from 'graphql-tag'; - -import { sharedFragments } from '../../../../common/graphql/shared'; - -export const logEntryHighlightsQuery = gql` - query LogEntryHighlightsQuery( - $sourceId: ID = "default" - $startKey: InfraTimeKeyInput! - $endKey: InfraTimeKeyInput! - $filterQuery: String - $highlights: [InfraLogEntryHighlightInput!]! - ) { - source(id: $sourceId) { - id - logEntryHighlights( - startKey: $startKey - endKey: $endKey - filterQuery: $filterQuery - highlights: $highlights - ) { - start { - ...InfraTimeKeyFields - } - end { - ...InfraTimeKeyFields - } - entries { - ...InfraLogEntryHighlightFields - } - } - } - } - - ${sharedFragments.InfraTimeKey} - ${sharedFragments.InfraLogEntryHighlightFields} -`; diff --git a/x-pack/plugins/infra/public/containers/logs/log_highlights/log_entry_highlights.tsx b/x-pack/plugins/infra/public/containers/logs/log_highlights/log_entry_highlights.tsx index b4edebe8f8207c..caac28a0756a16 100644 --- a/x-pack/plugins/infra/public/containers/logs/log_highlights/log_entry_highlights.tsx +++ b/x-pack/plugins/infra/public/containers/logs/log_highlights/log_entry_highlights.tsx @@ -5,12 +5,12 @@ */ import { useEffect, useMemo, useState } from 'react'; - +import { LogEntriesHighlightsResponse } from '../../../../common/http_api'; +import { LogEntry } from '../../../../common/log_entry'; import { TimeKey } from '../../../../common/time'; +import { useKibanaContextForPlugin } from '../../../hooks/use_kibana'; import { useTrackedPromise } from '../../../utils/use_tracked_promise'; import { fetchLogEntriesHighlights } from './api/fetch_log_entries_highlights'; -import { LogEntry, LogEntriesHighlightsResponse } from '../../../../common/http_api'; -import { useKibanaContextForPlugin } from '../../../hooks/use_kibana'; export const useLogEntryHighlights = ( sourceId: string, diff --git a/x-pack/plugins/infra/public/graphql/introspection.json b/x-pack/plugins/infra/public/graphql/introspection.json index 5d351f3259ac5f..efdca72c1383aa 100644 --- a/x-pack/plugins/infra/public/graphql/introspection.json +++ b/x-pack/plugins/infra/public/graphql/introspection.json @@ -137,155 +137,6 @@ "isDeprecated": false, "deprecationReason": null }, - { - "name": "logEntriesAround", - "description": "A consecutive span of log entries surrounding a point in time", - "args": [ - { - "name": "key", - "description": "The sort key that corresponds to the point in time", - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { "kind": "INPUT_OBJECT", "name": "InfraTimeKeyInput", "ofType": null } - }, - "defaultValue": null - }, - { - "name": "countBefore", - "description": "The maximum number of preceding to return", - "type": { "kind": "SCALAR", "name": "Int", "ofType": null }, - "defaultValue": "0" - }, - { - "name": "countAfter", - "description": "The maximum number of following to return", - "type": { "kind": "SCALAR", "name": "Int", "ofType": null }, - "defaultValue": "0" - }, - { - "name": "filterQuery", - "description": "The query to filter the log entries by", - "type": { "kind": "SCALAR", "name": "String", "ofType": null }, - "defaultValue": null - } - ], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { "kind": "OBJECT", "name": "InfraLogEntryInterval", "ofType": null } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "logEntriesBetween", - "description": "A consecutive span of log entries within an interval", - "args": [ - { - "name": "startKey", - "description": "The sort key that corresponds to the start of the interval", - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { "kind": "INPUT_OBJECT", "name": "InfraTimeKeyInput", "ofType": null } - }, - "defaultValue": null - }, - { - "name": "endKey", - "description": "The sort key that corresponds to the end of the interval", - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { "kind": "INPUT_OBJECT", "name": "InfraTimeKeyInput", "ofType": null } - }, - "defaultValue": null - }, - { - "name": "filterQuery", - "description": "The query to filter the log entries by", - "type": { "kind": "SCALAR", "name": "String", "ofType": null }, - "defaultValue": null - } - ], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { "kind": "OBJECT", "name": "InfraLogEntryInterval", "ofType": null } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "logEntryHighlights", - "description": "Sequences of log entries matching sets of highlighting queries within an interval", - "args": [ - { - "name": "startKey", - "description": "The sort key that corresponds to the start of the interval", - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { "kind": "INPUT_OBJECT", "name": "InfraTimeKeyInput", "ofType": null } - }, - "defaultValue": null - }, - { - "name": "endKey", - "description": "The sort key that corresponds to the end of the interval", - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { "kind": "INPUT_OBJECT", "name": "InfraTimeKeyInput", "ofType": null } - }, - "defaultValue": null - }, - { - "name": "filterQuery", - "description": "The query to filter the log entries by", - "type": { "kind": "SCALAR", "name": "String", "ofType": null }, - "defaultValue": null - }, - { - "name": "highlights", - "description": "The highlighting to apply to the log entries", - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "INPUT_OBJECT", - "name": "InfraLogEntryHighlightInput", - "ofType": null - } - } - } - }, - "defaultValue": null - } - ], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { "kind": "OBJECT", "name": "InfraLogEntryInterval", "ofType": null } - } - } - }, - "isDeprecated": false, - "deprecationReason": null - }, { "name": "snapshot", "description": "A snapshot of nodes", @@ -993,37 +844,6 @@ "enumValues": null, "possibleTypes": null }, - { - "kind": "INPUT_OBJECT", - "name": "InfraTimeKeyInput", - "description": "", - "fields": null, - "inputFields": [ - { - "name": "time", - "description": "", - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { "kind": "SCALAR", "name": "Float", "ofType": null } - }, - "defaultValue": null - }, - { - "name": "tiebreaker", - "description": "", - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { "kind": "SCALAR", "name": "Float", "ofType": null } - }, - "defaultValue": null - } - ], - "interfaces": null, - "enumValues": null, - "possibleTypes": null - }, { "kind": "SCALAR", "name": "Int", @@ -1034,486 +854,6 @@ "enumValues": null, "possibleTypes": null }, - { - "kind": "OBJECT", - "name": "InfraLogEntryInterval", - "description": "A consecutive sequence of log entries", - "fields": [ - { - "name": "start", - "description": "The key corresponding to the start of the interval covered by the entries", - "args": [], - "type": { "kind": "OBJECT", "name": "InfraTimeKey", "ofType": null }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "end", - "description": "The key corresponding to the end of the interval covered by the entries", - "args": [], - "type": { "kind": "OBJECT", "name": "InfraTimeKey", "ofType": null }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "hasMoreBefore", - "description": "Whether there are more log entries available before the start", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { "kind": "SCALAR", "name": "Boolean", "ofType": null } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "hasMoreAfter", - "description": "Whether there are more log entries available after the end", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { "kind": "SCALAR", "name": "Boolean", "ofType": null } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "filterQuery", - "description": "The query the log entries were filtered by", - "args": [], - "type": { "kind": "SCALAR", "name": "String", "ofType": null }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "highlightQuery", - "description": "The query the log entries were highlighted with", - "args": [], - "type": { "kind": "SCALAR", "name": "String", "ofType": null }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "entries", - "description": "A list of the log entries", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { "kind": "OBJECT", "name": "InfraLogEntry", "ofType": null } - } - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "InfraTimeKey", - "description": "A representation of the log entry's position in the event stream", - "fields": [ - { - "name": "time", - "description": "The timestamp of the event that the log entry corresponds to", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { "kind": "SCALAR", "name": "Float", "ofType": null } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "tiebreaker", - "description": "The tiebreaker that disambiguates events with the same timestamp", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { "kind": "SCALAR", "name": "Float", "ofType": null } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "InfraLogEntry", - "description": "A log entry", - "fields": [ - { - "name": "key", - "description": "A unique representation of the log entry's position in the event stream", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { "kind": "OBJECT", "name": "InfraTimeKey", "ofType": null } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "gid", - "description": "The log entry's id", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { "kind": "SCALAR", "name": "String", "ofType": null } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "source", - "description": "The source id", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { "kind": "SCALAR", "name": "String", "ofType": null } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "columns", - "description": "The columns used for rendering the log entry", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { "kind": "UNION", "name": "InfraLogEntryColumn", "ofType": null } - } - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "UNION", - "name": "InfraLogEntryColumn", - "description": "A column of a log entry", - "fields": null, - "inputFields": null, - "interfaces": null, - "enumValues": null, - "possibleTypes": [ - { "kind": "OBJECT", "name": "InfraLogEntryTimestampColumn", "ofType": null }, - { "kind": "OBJECT", "name": "InfraLogEntryMessageColumn", "ofType": null }, - { "kind": "OBJECT", "name": "InfraLogEntryFieldColumn", "ofType": null } - ] - }, - { - "kind": "OBJECT", - "name": "InfraLogEntryTimestampColumn", - "description": "A special built-in column that contains the log entry's timestamp", - "fields": [ - { - "name": "columnId", - "description": "The id of the corresponding column configuration", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { "kind": "SCALAR", "name": "ID", "ofType": null } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "timestamp", - "description": "The timestamp", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { "kind": "SCALAR", "name": "Float", "ofType": null } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "InfraLogEntryMessageColumn", - "description": "A special built-in column that contains the log entry's constructed message", - "fields": [ - { - "name": "columnId", - "description": "The id of the corresponding column configuration", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { "kind": "SCALAR", "name": "ID", "ofType": null } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "message", - "description": "A list of the formatted log entry segments", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { "kind": "UNION", "name": "InfraLogMessageSegment", "ofType": null } - } - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "UNION", - "name": "InfraLogMessageSegment", - "description": "A segment of the log entry message", - "fields": null, - "inputFields": null, - "interfaces": null, - "enumValues": null, - "possibleTypes": [ - { "kind": "OBJECT", "name": "InfraLogMessageFieldSegment", "ofType": null }, - { "kind": "OBJECT", "name": "InfraLogMessageConstantSegment", "ofType": null } - ] - }, - { - "kind": "OBJECT", - "name": "InfraLogMessageFieldSegment", - "description": "A segment of the log entry message that was derived from a field", - "fields": [ - { - "name": "field", - "description": "The field the segment was derived from", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { "kind": "SCALAR", "name": "String", "ofType": null } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "value", - "description": "The segment's message", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { "kind": "SCALAR", "name": "String", "ofType": null } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "highlights", - "description": "A list of highlighted substrings of the value", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { "kind": "SCALAR", "name": "String", "ofType": null } - } - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "InfraLogMessageConstantSegment", - "description": "A segment of the log entry message that was derived from a string literal", - "fields": [ - { - "name": "constant", - "description": "The segment's message", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { "kind": "SCALAR", "name": "String", "ofType": null } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "InfraLogEntryFieldColumn", - "description": "A column that contains the value of a field of the log entry", - "fields": [ - { - "name": "columnId", - "description": "The id of the corresponding column configuration", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { "kind": "SCALAR", "name": "ID", "ofType": null } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "field", - "description": "The field name of the column", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { "kind": "SCALAR", "name": "String", "ofType": null } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "value", - "description": "The value of the field in the log entry", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { "kind": "SCALAR", "name": "String", "ofType": null } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "highlights", - "description": "A list of highlighted substrings of the value", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { "kind": "SCALAR", "name": "String", "ofType": null } - } - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "INPUT_OBJECT", - "name": "InfraLogEntryHighlightInput", - "description": "A highlighting definition", - "fields": null, - "inputFields": [ - { - "name": "query", - "description": "The query to highlight by", - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { "kind": "SCALAR", "name": "String", "ofType": null } - }, - "defaultValue": null - }, - { - "name": "countBefore", - "description": "The number of highlighted documents to include beyond the beginning of the interval", - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { "kind": "SCALAR", "name": "Int", "ofType": null } - }, - "defaultValue": null - }, - { - "name": "countAfter", - "description": "The number of highlighted documents to include beyond the end of the interval", - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { "kind": "SCALAR", "name": "Int", "ofType": null } - }, - "defaultValue": null - } - ], - "interfaces": null, - "enumValues": null, - "possibleTypes": null - }, { "kind": "INPUT_OBJECT", "name": "InfraTimerangeInput", diff --git a/x-pack/plugins/infra/public/graphql/types.ts b/x-pack/plugins/infra/public/graphql/types.ts index f0f74c34a19e6c..eb025ee4efd79b 100644 --- a/x-pack/plugins/infra/public/graphql/types.ts +++ b/x-pack/plugins/infra/public/graphql/types.ts @@ -30,12 +30,6 @@ export interface InfraSource { configuration: InfraSourceConfiguration; /** The status of the source */ status: InfraSourceStatus; - /** A consecutive span of log entries surrounding a point in time */ - logEntriesAround: InfraLogEntryInterval; - /** A consecutive span of log entries within an interval */ - logEntriesBetween: InfraLogEntryInterval; - /** Sequences of log entries matching sets of highlighting queries within an interval */ - logEntryHighlights: InfraLogEntryInterval[]; /** A snapshot of nodes */ snapshot?: InfraSnapshotResponse | null; @@ -135,80 +129,6 @@ export interface InfraIndexField { /** Whether the field should be displayed based on event.module and a ECS allowed list */ displayable: boolean; } -/** A consecutive sequence of log entries */ -export interface InfraLogEntryInterval { - /** The key corresponding to the start of the interval covered by the entries */ - start?: InfraTimeKey | null; - /** The key corresponding to the end of the interval covered by the entries */ - end?: InfraTimeKey | null; - /** Whether there are more log entries available before the start */ - hasMoreBefore: boolean; - /** Whether there are more log entries available after the end */ - hasMoreAfter: boolean; - /** The query the log entries were filtered by */ - filterQuery?: string | null; - /** The query the log entries were highlighted with */ - highlightQuery?: string | null; - /** A list of the log entries */ - entries: InfraLogEntry[]; -} -/** A representation of the log entry's position in the event stream */ -export interface InfraTimeKey { - /** The timestamp of the event that the log entry corresponds to */ - time: number; - /** The tiebreaker that disambiguates events with the same timestamp */ - tiebreaker: number; -} -/** A log entry */ -export interface InfraLogEntry { - /** A unique representation of the log entry's position in the event stream */ - key: InfraTimeKey; - /** The log entry's id */ - gid: string; - /** The source id */ - source: string; - /** The columns used for rendering the log entry */ - columns: InfraLogEntryColumn[]; -} -/** A special built-in column that contains the log entry's timestamp */ -export interface InfraLogEntryTimestampColumn { - /** The id of the corresponding column configuration */ - columnId: string; - /** The timestamp */ - timestamp: number; -} -/** A special built-in column that contains the log entry's constructed message */ -export interface InfraLogEntryMessageColumn { - /** The id of the corresponding column configuration */ - columnId: string; - /** A list of the formatted log entry segments */ - message: InfraLogMessageSegment[]; -} -/** A segment of the log entry message that was derived from a field */ -export interface InfraLogMessageFieldSegment { - /** The field the segment was derived from */ - field: string; - /** The segment's message */ - value: string; - /** A list of highlighted substrings of the value */ - highlights: string[]; -} -/** A segment of the log entry message that was derived from a string literal */ -export interface InfraLogMessageConstantSegment { - /** The segment's message */ - constant: string; -} -/** A column that contains the value of a field of the log entry */ -export interface InfraLogEntryFieldColumn { - /** The id of the corresponding column configuration */ - columnId: string; - /** The field name of the column */ - field: string; - /** The value of the field in the log entry */ - value: string; - /** A list of highlighted substrings of the value */ - highlights: string[]; -} export interface InfraSnapshotResponse { /** Nodes of type host, container or pod grouped by 0, 1 or 2 terms */ @@ -282,21 +202,6 @@ export interface DeleteSourceResult { // InputTypes // ==================================================== -export interface InfraTimeKeyInput { - time: number; - - tiebreaker: number; -} -/** A highlighting definition */ -export interface InfraLogEntryHighlightInput { - /** The query to highlight by */ - query: string; - /** The number of highlighted documents to include beyond the beginning of the interval */ - countBefore: number; - /** The number of highlighted documents to include beyond the end of the interval */ - countAfter: number; -} - export interface InfraTimerangeInput { /** The interval string to use for last bucket. The format is '{value}{unit}'. For example '5m' would return the metrics for the last 5 minutes of the timespan. */ interval: string; @@ -387,34 +292,6 @@ export interface SourceQueryArgs { /** The id of the source */ id: string; } -export interface LogEntriesAroundInfraSourceArgs { - /** The sort key that corresponds to the point in time */ - key: InfraTimeKeyInput; - /** The maximum number of preceding to return */ - countBefore?: number | null; - /** The maximum number of following to return */ - countAfter?: number | null; - /** The query to filter the log entries by */ - filterQuery?: string | null; -} -export interface LogEntriesBetweenInfraSourceArgs { - /** The sort key that corresponds to the start of the interval */ - startKey: InfraTimeKeyInput; - /** The sort key that corresponds to the end of the interval */ - endKey: InfraTimeKeyInput; - /** The query to filter the log entries by */ - filterQuery?: string | null; -} -export interface LogEntryHighlightsInfraSourceArgs { - /** The sort key that corresponds to the start of the interval */ - startKey: InfraTimeKeyInput; - /** The sort key that corresponds to the end of the interval */ - endKey: InfraTimeKeyInput; - /** The query to filter the log entries by */ - filterQuery?: string | null; - /** The highlighting to apply to the log entries */ - highlights: InfraLogEntryHighlightInput[]; -} export interface SnapshotInfraSourceArgs { timerange: InfraTimerangeInput; @@ -571,15 +448,6 @@ export type InfraSourceLogColumn = | InfraSourceMessageLogColumn | InfraSourceFieldLogColumn; -/** A column of a log entry */ -export type InfraLogEntryColumn = - | InfraLogEntryTimestampColumn - | InfraLogEntryMessageColumn - | InfraLogEntryFieldColumn; - -/** A segment of the log entry message */ -export type InfraLogMessageSegment = InfraLogMessageFieldSegment | InfraLogMessageConstantSegment; - // ==================================================== // END: Typescript template // ==================================================== @@ -588,46 +456,6 @@ export type InfraLogMessageSegment = InfraLogMessageFieldSegment | InfraLogMessa // Documents // ==================================================== -export namespace LogEntryHighlightsQuery { - export type Variables = { - sourceId?: string | null; - startKey: InfraTimeKeyInput; - endKey: InfraTimeKeyInput; - filterQuery?: string | null; - highlights: InfraLogEntryHighlightInput[]; - }; - - export type Query = { - __typename?: 'Query'; - - source: Source; - }; - - export type Source = { - __typename?: 'InfraSource'; - - id: string; - - logEntryHighlights: LogEntryHighlights[]; - }; - - export type LogEntryHighlights = { - __typename?: 'InfraLogEntryInterval'; - - start?: Start | null; - - end?: End | null; - - entries: Entries[]; - }; - - export type Start = InfraTimeKeyFields.Fragment; - - export type End = InfraTimeKeyFields.Fragment; - - export type Entries = InfraLogEntryHighlightFields.Fragment; -} - export namespace MetricsQuery { export type Variables = { sourceId: string; @@ -826,50 +654,6 @@ export namespace WaffleNodesQuery { }; } -export namespace LogEntries { - export type Variables = { - sourceId?: string | null; - timeKey: InfraTimeKeyInput; - countBefore?: number | null; - countAfter?: number | null; - filterQuery?: string | null; - }; - - export type Query = { - __typename?: 'Query'; - - source: Source; - }; - - export type Source = { - __typename?: 'InfraSource'; - - id: string; - - logEntriesAround: LogEntriesAround; - }; - - export type LogEntriesAround = { - __typename?: 'InfraLogEntryInterval'; - - start?: Start | null; - - end?: End | null; - - hasMoreBefore: boolean; - - hasMoreAfter: boolean; - - entries: Entries[]; - }; - - export type Start = InfraTimeKeyFields.Fragment; - - export type End = InfraTimeKeyFields.Fragment; - - export type Entries = InfraLogEntryFields.Fragment; -} - export namespace SourceConfigurationFields { export type Fragment = { __typename?: 'InfraSourceConfiguration'; @@ -1000,124 +784,3 @@ export namespace InfraSourceFields { origin: string; }; } - -export namespace InfraLogEntryFields { - export type Fragment = { - __typename?: 'InfraLogEntry'; - - gid: string; - - key: Key; - - columns: Columns[]; - }; - - export type Key = { - __typename?: 'InfraTimeKey'; - - time: number; - - tiebreaker: number; - }; - - export type Columns = - | InfraLogEntryTimestampColumnInlineFragment - | InfraLogEntryMessageColumnInlineFragment - | InfraLogEntryFieldColumnInlineFragment; - - export type InfraLogEntryTimestampColumnInlineFragment = { - __typename?: 'InfraLogEntryTimestampColumn'; - - columnId: string; - - timestamp: number; - }; - - export type InfraLogEntryMessageColumnInlineFragment = { - __typename?: 'InfraLogEntryMessageColumn'; - - columnId: string; - - message: Message[]; - }; - - export type Message = - | InfraLogMessageFieldSegmentInlineFragment - | InfraLogMessageConstantSegmentInlineFragment; - - export type InfraLogMessageFieldSegmentInlineFragment = { - __typename?: 'InfraLogMessageFieldSegment'; - - field: string; - - value: string; - }; - - export type InfraLogMessageConstantSegmentInlineFragment = { - __typename?: 'InfraLogMessageConstantSegment'; - - constant: string; - }; - - export type InfraLogEntryFieldColumnInlineFragment = { - __typename?: 'InfraLogEntryFieldColumn'; - - columnId: string; - - field: string; - - value: string; - }; -} - -export namespace InfraLogEntryHighlightFields { - export type Fragment = { - __typename?: 'InfraLogEntry'; - - gid: string; - - key: Key; - - columns: Columns[]; - }; - - export type Key = { - __typename?: 'InfraTimeKey'; - - time: number; - - tiebreaker: number; - }; - - export type Columns = - | InfraLogEntryMessageColumnInlineFragment - | InfraLogEntryFieldColumnInlineFragment; - - export type InfraLogEntryMessageColumnInlineFragment = { - __typename?: 'InfraLogEntryMessageColumn'; - - columnId: string; - - message: Message[]; - }; - - export type Message = InfraLogMessageFieldSegmentInlineFragment; - - export type InfraLogMessageFieldSegmentInlineFragment = { - __typename?: 'InfraLogMessageFieldSegment'; - - field: string; - - highlights: string[]; - }; - - export type InfraLogEntryFieldColumnInlineFragment = { - __typename?: 'InfraLogEntryFieldColumn'; - - columnId: string; - - field: string; - - highlights: string[]; - }; -} diff --git a/x-pack/plugins/infra/public/utils/log_entry/log_entry_highlight.ts b/x-pack/plugins/infra/public/utils/log_entry/log_entry_highlight.ts index abb004911214b2..e14d938c426f90 100644 --- a/x-pack/plugins/infra/public/utils/log_entry/log_entry_highlight.ts +++ b/x-pack/plugins/infra/public/utils/log_entry/log_entry_highlight.ts @@ -4,7 +4,6 @@ * you may not use this file except in compliance with the Elastic License. */ -import { InfraLogEntryHighlightFields } from '../../graphql/types'; import { LogEntry, LogColumn, @@ -12,14 +11,7 @@ import { LogFieldColumn, LogMessagePart, LogMessageFieldPart, -} from '../../../common/http_api'; - -export type LogEntryHighlightColumn = InfraLogEntryHighlightFields.Columns; -export type LogEntryHighlightMessageColumn = InfraLogEntryHighlightFields.InfraLogEntryMessageColumnInlineFragment; -export type LogEntryHighlightFieldColumn = InfraLogEntryHighlightFields.InfraLogEntryFieldColumnInlineFragment; - -export type LogEntryHighlightMessageSegment = InfraLogEntryHighlightFields.Message | {}; -export type LogEntryHighlightFieldMessageSegment = InfraLogEntryHighlightFields.InfraLogMessageFieldSegmentInlineFragment; +} from '../../../common/log_entry'; export interface LogEntryHighlightsMap { [entryId: string]: LogEntry[]; diff --git a/x-pack/plugins/infra/server/graphql/types.ts b/x-pack/plugins/infra/server/graphql/types.ts index 02dcd76e8b34c8..712438ce2bfe02 100644 --- a/x-pack/plugins/infra/server/graphql/types.ts +++ b/x-pack/plugins/infra/server/graphql/types.ts @@ -56,12 +56,6 @@ export interface InfraSource { configuration: InfraSourceConfiguration; /** The status of the source */ status: InfraSourceStatus; - /** A consecutive span of log entries surrounding a point in time */ - logEntriesAround: InfraLogEntryInterval; - /** A consecutive span of log entries within an interval */ - logEntriesBetween: InfraLogEntryInterval; - /** Sequences of log entries matching sets of highlighting queries within an interval */ - logEntryHighlights: InfraLogEntryInterval[]; /** A snapshot of nodes */ snapshot?: InfraSnapshotResponse | null; @@ -157,80 +151,6 @@ export interface InfraIndexField { /** Whether the field should be displayed based on event.module and a ECS allowed list */ displayable: boolean; } -/** A consecutive sequence of log entries */ -export interface InfraLogEntryInterval { - /** The key corresponding to the start of the interval covered by the entries */ - start?: InfraTimeKey | null; - /** The key corresponding to the end of the interval covered by the entries */ - end?: InfraTimeKey | null; - /** Whether there are more log entries available before the start */ - hasMoreBefore: boolean; - /** Whether there are more log entries available after the end */ - hasMoreAfter: boolean; - /** The query the log entries were filtered by */ - filterQuery?: string | null; - /** The query the log entries were highlighted with */ - highlightQuery?: string | null; - /** A list of the log entries */ - entries: InfraLogEntry[]; -} -/** A representation of the log entry's position in the event stream */ -export interface InfraTimeKey { - /** The timestamp of the event that the log entry corresponds to */ - time: number; - /** The tiebreaker that disambiguates events with the same timestamp */ - tiebreaker: number; -} -/** A log entry */ -export interface InfraLogEntry { - /** A unique representation of the log entry's position in the event stream */ - key: InfraTimeKey; - /** The log entry's id */ - gid: string; - /** The source id */ - source: string; - /** The columns used for rendering the log entry */ - columns: InfraLogEntryColumn[]; -} -/** A special built-in column that contains the log entry's timestamp */ -export interface InfraLogEntryTimestampColumn { - /** The id of the corresponding column configuration */ - columnId: string; - /** The timestamp */ - timestamp: number; -} -/** A special built-in column that contains the log entry's constructed message */ -export interface InfraLogEntryMessageColumn { - /** The id of the corresponding column configuration */ - columnId: string; - /** A list of the formatted log entry segments */ - message: InfraLogMessageSegment[]; -} -/** A segment of the log entry message that was derived from a field */ -export interface InfraLogMessageFieldSegment { - /** The field the segment was derived from */ - field: string; - /** The segment's message */ - value: string; - /** A list of highlighted substrings of the value */ - highlights: string[]; -} -/** A segment of the log entry message that was derived from a string literal */ -export interface InfraLogMessageConstantSegment { - /** The segment's message */ - constant: string; -} -/** A column that contains the value of a field of the log entry */ -export interface InfraLogEntryFieldColumn { - /** The id of the corresponding column configuration */ - columnId: string; - /** The field name of the column */ - field: string; - /** The value of the field in the log entry */ - value: string; - /** A list of highlighted substrings of the value */ - highlights: string[]; -} export interface InfraSnapshotResponse { /** Nodes of type host, container or pod grouped by 0, 1 or 2 terms */ @@ -304,21 +224,6 @@ export interface DeleteSourceResult { // InputTypes // ==================================================== -export interface InfraTimeKeyInput { - time: number; - - tiebreaker: number; -} -/** A highlighting definition */ -export interface InfraLogEntryHighlightInput { - /** The query to highlight by */ - query: string; - /** The number of highlighted documents to include beyond the beginning of the interval */ - countBefore: number; - /** The number of highlighted documents to include beyond the end of the interval */ - countAfter: number; -} - export interface InfraTimerangeInput { /** The interval string to use for last bucket. The format is '{value}{unit}'. For example '5m' would return the metrics for the last 5 minutes of the timespan. */ interval: string; @@ -409,34 +314,6 @@ export interface SourceQueryArgs { /** The id of the source */ id: string; } -export interface LogEntriesAroundInfraSourceArgs { - /** The sort key that corresponds to the point in time */ - key: InfraTimeKeyInput; - /** The maximum number of preceding to return */ - countBefore?: number | null; - /** The maximum number of following to return */ - countAfter?: number | null; - /** The query to filter the log entries by */ - filterQuery?: string | null; -} -export interface LogEntriesBetweenInfraSourceArgs { - /** The sort key that corresponds to the start of the interval */ - startKey: InfraTimeKeyInput; - /** The sort key that corresponds to the end of the interval */ - endKey: InfraTimeKeyInput; - /** The query to filter the log entries by */ - filterQuery?: string | null; -} -export interface LogEntryHighlightsInfraSourceArgs { - /** The sort key that corresponds to the start of the interval */ - startKey: InfraTimeKeyInput; - /** The sort key that corresponds to the end of the interval */ - endKey: InfraTimeKeyInput; - /** The query to filter the log entries by */ - filterQuery?: string | null; - /** The highlighting to apply to the log entries */ - highlights: InfraLogEntryHighlightInput[]; -} export interface SnapshotInfraSourceArgs { timerange: InfraTimerangeInput; @@ -593,15 +470,6 @@ export type InfraSourceLogColumn = | InfraSourceMessageLogColumn | InfraSourceFieldLogColumn; -/** A column of a log entry */ -export type InfraLogEntryColumn = - | InfraLogEntryTimestampColumn - | InfraLogEntryMessageColumn - | InfraLogEntryFieldColumn; - -/** A segment of the log entry message */ -export type InfraLogMessageSegment = InfraLogMessageFieldSegment | InfraLogMessageConstantSegment; - // ==================================================== // END: Typescript template // ==================================================== @@ -650,12 +518,6 @@ export namespace InfraSourceResolvers { configuration?: ConfigurationResolver; /** The status of the source */ status?: StatusResolver; - /** A consecutive span of log entries surrounding a point in time */ - logEntriesAround?: LogEntriesAroundResolver; - /** A consecutive span of log entries within an interval */ - logEntriesBetween?: LogEntriesBetweenResolver; - /** Sequences of log entries matching sets of highlighting queries within an interval */ - logEntryHighlights?: LogEntryHighlightsResolver; /** A snapshot of nodes */ snapshot?: SnapshotResolver; @@ -693,51 +555,6 @@ export namespace InfraSourceResolvers { Parent = InfraSource, Context = InfraContext > = Resolver; - export type LogEntriesAroundResolver< - R = InfraLogEntryInterval, - Parent = InfraSource, - Context = InfraContext - > = Resolver; - export interface LogEntriesAroundArgs { - /** The sort key that corresponds to the point in time */ - key: InfraTimeKeyInput; - /** The maximum number of preceding to return */ - countBefore?: number | null; - /** The maximum number of following to return */ - countAfter?: number | null; - /** The query to filter the log entries by */ - filterQuery?: string | null; - } - - export type LogEntriesBetweenResolver< - R = InfraLogEntryInterval, - Parent = InfraSource, - Context = InfraContext - > = Resolver; - export interface LogEntriesBetweenArgs { - /** The sort key that corresponds to the start of the interval */ - startKey: InfraTimeKeyInput; - /** The sort key that corresponds to the end of the interval */ - endKey: InfraTimeKeyInput; - /** The query to filter the log entries by */ - filterQuery?: string | null; - } - - export type LogEntryHighlightsResolver< - R = InfraLogEntryInterval[], - Parent = InfraSource, - Context = InfraContext - > = Resolver; - export interface LogEntryHighlightsArgs { - /** The sort key that corresponds to the start of the interval */ - startKey: InfraTimeKeyInput; - /** The sort key that corresponds to the end of the interval */ - endKey: InfraTimeKeyInput; - /** The query to filter the log entries by */ - filterQuery?: string | null; - /** The highlighting to apply to the log entries */ - highlights: InfraLogEntryHighlightInput[]; - } export type SnapshotResolver< R = InfraSnapshotResponse | null, @@ -1059,229 +876,6 @@ export namespace InfraIndexFieldResolvers { Context = InfraContext > = Resolver; } -/** A consecutive sequence of log entries */ -export namespace InfraLogEntryIntervalResolvers { - export interface Resolvers { - /** The key corresponding to the start of the interval covered by the entries */ - start?: StartResolver; - /** The key corresponding to the end of the interval covered by the entries */ - end?: EndResolver; - /** Whether there are more log entries available before the start */ - hasMoreBefore?: HasMoreBeforeResolver; - /** Whether there are more log entries available after the end */ - hasMoreAfter?: HasMoreAfterResolver; - /** The query the log entries were filtered by */ - filterQuery?: FilterQueryResolver; - /** The query the log entries were highlighted with */ - highlightQuery?: HighlightQueryResolver; - /** A list of the log entries */ - entries?: EntriesResolver; - } - - export type StartResolver< - R = InfraTimeKey | null, - Parent = InfraLogEntryInterval, - Context = InfraContext - > = Resolver; - export type EndResolver< - R = InfraTimeKey | null, - Parent = InfraLogEntryInterval, - Context = InfraContext - > = Resolver; - export type HasMoreBeforeResolver< - R = boolean, - Parent = InfraLogEntryInterval, - Context = InfraContext - > = Resolver; - export type HasMoreAfterResolver< - R = boolean, - Parent = InfraLogEntryInterval, - Context = InfraContext - > = Resolver; - export type FilterQueryResolver< - R = string | null, - Parent = InfraLogEntryInterval, - Context = InfraContext - > = Resolver; - export type HighlightQueryResolver< - R = string | null, - Parent = InfraLogEntryInterval, - Context = InfraContext - > = Resolver; - export type EntriesResolver< - R = InfraLogEntry[], - Parent = InfraLogEntryInterval, - Context = InfraContext - > = Resolver; -} -/** A representation of the log entry's position in the event stream */ -export namespace InfraTimeKeyResolvers { - export interface Resolvers { - /** The timestamp of the event that the log entry corresponds to */ - time?: TimeResolver; - /** The tiebreaker that disambiguates events with the same timestamp */ - tiebreaker?: TiebreakerResolver; - } - - export type TimeResolver = Resolver< - R, - Parent, - Context - >; - export type TiebreakerResolver< - R = number, - Parent = InfraTimeKey, - Context = InfraContext - > = Resolver; -} -/** A log entry */ -export namespace InfraLogEntryResolvers { - export interface Resolvers { - /** A unique representation of the log entry's position in the event stream */ - key?: KeyResolver; - /** The log entry's id */ - gid?: GidResolver; - /** The source id */ - source?: SourceResolver; - /** The columns used for rendering the log entry */ - columns?: ColumnsResolver; - } - - export type KeyResolver< - R = InfraTimeKey, - Parent = InfraLogEntry, - Context = InfraContext - > = Resolver; - export type GidResolver = Resolver< - R, - Parent, - Context - >; - export type SourceResolver = Resolver< - R, - Parent, - Context - >; - export type ColumnsResolver< - R = InfraLogEntryColumn[], - Parent = InfraLogEntry, - Context = InfraContext - > = Resolver; -} -/** A special built-in column that contains the log entry's timestamp */ -export namespace InfraLogEntryTimestampColumnResolvers { - export interface Resolvers { - /** The id of the corresponding column configuration */ - columnId?: ColumnIdResolver; - /** The timestamp */ - timestamp?: TimestampResolver; - } - - export type ColumnIdResolver< - R = string, - Parent = InfraLogEntryTimestampColumn, - Context = InfraContext - > = Resolver; - export type TimestampResolver< - R = number, - Parent = InfraLogEntryTimestampColumn, - Context = InfraContext - > = Resolver; -} -/** A special built-in column that contains the log entry's constructed message */ -export namespace InfraLogEntryMessageColumnResolvers { - export interface Resolvers { - /** The id of the corresponding column configuration */ - columnId?: ColumnIdResolver; - /** A list of the formatted log entry segments */ - message?: MessageResolver; - } - - export type ColumnIdResolver< - R = string, - Parent = InfraLogEntryMessageColumn, - Context = InfraContext - > = Resolver; - export type MessageResolver< - R = InfraLogMessageSegment[], - Parent = InfraLogEntryMessageColumn, - Context = InfraContext - > = Resolver; -} -/** A segment of the log entry message that was derived from a field */ -export namespace InfraLogMessageFieldSegmentResolvers { - export interface Resolvers { - /** The field the segment was derived from */ - field?: FieldResolver; - /** The segment's message */ - value?: ValueResolver; - /** A list of highlighted substrings of the value */ - highlights?: HighlightsResolver; - } - - export type FieldResolver< - R = string, - Parent = InfraLogMessageFieldSegment, - Context = InfraContext - > = Resolver; - export type ValueResolver< - R = string, - Parent = InfraLogMessageFieldSegment, - Context = InfraContext - > = Resolver; - export type HighlightsResolver< - R = string[], - Parent = InfraLogMessageFieldSegment, - Context = InfraContext - > = Resolver; -} -/** A segment of the log entry message that was derived from a string literal */ -export namespace InfraLogMessageConstantSegmentResolvers { - export interface Resolvers { - /** The segment's message */ - constant?: ConstantResolver; - } - - export type ConstantResolver< - R = string, - Parent = InfraLogMessageConstantSegment, - Context = InfraContext - > = Resolver; -} -/** A column that contains the value of a field of the log entry */ -export namespace InfraLogEntryFieldColumnResolvers { - export interface Resolvers { - /** The id of the corresponding column configuration */ - columnId?: ColumnIdResolver; - /** The field name of the column */ - field?: FieldResolver; - /** The value of the field in the log entry */ - value?: ValueResolver; - /** A list of highlighted substrings of the value */ - highlights?: HighlightsResolver; - } - - export type ColumnIdResolver< - R = string, - Parent = InfraLogEntryFieldColumn, - Context = InfraContext - > = Resolver; - export type FieldResolver< - R = string, - Parent = InfraLogEntryFieldColumn, - Context = InfraContext - > = Resolver; - export type ValueResolver< - R = string, - Parent = InfraLogEntryFieldColumn, - Context = InfraContext - > = Resolver; - export type HighlightsResolver< - R = string[], - Parent = InfraLogEntryFieldColumn, - Context = InfraContext - > = Resolver; -} export namespace InfraSnapshotResponseResolvers { export interface Resolvers { From 2802b936bab3baf107b8282e583904d796a2947c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20St=C3=BCrmer?= Date: Wed, 13 Jan 2021 13:05:28 +0100 Subject: [PATCH 09/39] Fix LogEntry imports after moving it --- .../log_entry_actions_menu.test.tsx | 16 ++++++++-------- .../components/logging/log_text_stream/item.ts | 3 +-- .../log_entry_field_column.test.tsx | 2 +- .../log_text_stream/log_entry_field_column.tsx | 2 +- .../log_entry_message_column.test.tsx | 2 +- .../logging/log_text_stream/log_entry_row.tsx | 3 +-- .../scrollable_log_text_stream_view.tsx | 2 +- .../public/containers/logs/log_entries/index.ts | 2 +- .../public/containers/logs/log_stream/index.ts | 3 +-- .../view_log_in_context/view_log_in_context.ts | 2 +- .../public/containers/logs/with_stream_items.ts | 2 +- .../top_categories/category_example_message.tsx | 2 +- .../logs/stream/page_view_log_in_context.tsx | 2 +- .../log_entry_categories_analysis.ts | 2 +- 14 files changed, 21 insertions(+), 24 deletions(-) diff --git a/x-pack/plugins/infra/public/components/logging/log_entry_flyout/log_entry_actions_menu.test.tsx b/x-pack/plugins/infra/public/components/logging/log_entry_flyout/log_entry_actions_menu.test.tsx index f578292d6d6fcb..447e6afbbf1fdb 100644 --- a/x-pack/plugins/infra/public/components/logging/log_entry_flyout/log_entry_actions_menu.test.tsx +++ b/x-pack/plugins/infra/public/components/logging/log_entry_flyout/log_entry_actions_menu.test.tsx @@ -32,7 +32,7 @@ describe('LogEntryActionsMenu component', () => { fields: [{ field: 'host.ip', value: ['HOST_IP'] }], id: 'ITEM_ID', index: 'INDEX', - key: { + cursor: { time: 0, tiebreaker: 0, }, @@ -62,7 +62,7 @@ describe('LogEntryActionsMenu component', () => { fields: [{ field: 'container.id', value: ['CONTAINER_ID'] }], id: 'ITEM_ID', index: 'INDEX', - key: { + cursor: { time: 0, tiebreaker: 0, }, @@ -92,7 +92,7 @@ describe('LogEntryActionsMenu component', () => { fields: [{ field: 'kubernetes.pod.uid', value: ['POD_UID'] }], id: 'ITEM_ID', index: 'INDEX', - key: { + cursor: { time: 0, tiebreaker: 0, }, @@ -126,7 +126,7 @@ describe('LogEntryActionsMenu component', () => { ], id: 'ITEM_ID', index: 'INDEX', - key: { + cursor: { time: 0, tiebreaker: 0, }, @@ -158,7 +158,7 @@ describe('LogEntryActionsMenu component', () => { fields: [], id: 'ITEM_ID', index: 'INDEX', - key: { + cursor: { time: 0, tiebreaker: 0, }, @@ -192,7 +192,7 @@ describe('LogEntryActionsMenu component', () => { fields: [{ field: 'trace.id', value: ['1234567'] }], id: 'ITEM_ID', index: 'INDEX', - key: { + cursor: { time: 0, tiebreaker: 0, }, @@ -226,7 +226,7 @@ describe('LogEntryActionsMenu component', () => { ], id: 'ITEM_ID', index: 'INDEX', - key: { + cursor: { time: 0, tiebreaker: 0, }, @@ -256,7 +256,7 @@ describe('LogEntryActionsMenu component', () => { fields: [], id: 'ITEM_ID', index: 'INDEX', - key: { + cursor: { time: 0, tiebreaker: 0, }, diff --git a/x-pack/plugins/infra/public/components/logging/log_text_stream/item.ts b/x-pack/plugins/infra/public/components/logging/log_text_stream/item.ts index 19e8108ee50e85..13d5b7b889465a 100644 --- a/x-pack/plugins/infra/public/components/logging/log_text_stream/item.ts +++ b/x-pack/plugins/infra/public/components/logging/log_text_stream/item.ts @@ -5,9 +5,8 @@ */ import { bisector } from 'd3-array'; - import { compareToTimeKey, TimeKey } from '../../../../common/time'; -import { LogEntry } from '../../../../common/http_api'; +import { LogEntry } from '../../../../common/log_entry'; export type StreamItem = LogEntryStreamItem; diff --git a/x-pack/plugins/infra/public/components/logging/log_text_stream/log_entry_field_column.test.tsx b/x-pack/plugins/infra/public/components/logging/log_text_stream/log_entry_field_column.test.tsx index 8de9e565b00be1..2b30d43f8c38da 100644 --- a/x-pack/plugins/infra/public/components/logging/log_text_stream/log_entry_field_column.test.tsx +++ b/x-pack/plugins/infra/public/components/logging/log_text_stream/log_entry_field_column.test.tsx @@ -7,7 +7,7 @@ import { render } from '@testing-library/react'; import React from 'react'; import { EuiThemeProvider } from '../../../../../../../src/plugins/kibana_react/common'; -import { LogFieldColumn } from '../../../../common/http_api'; +import { LogFieldColumn } from '../../../../common/log_entry'; import { LogEntryFieldColumn } from './log_entry_field_column'; describe('LogEntryFieldColumn', () => { diff --git a/x-pack/plugins/infra/public/components/logging/log_text_stream/log_entry_field_column.tsx b/x-pack/plugins/infra/public/components/logging/log_text_stream/log_entry_field_column.tsx index 4a9b0d0906a76e..0d295b4df5566b 100644 --- a/x-pack/plugins/infra/public/components/logging/log_text_stream/log_entry_field_column.tsx +++ b/x-pack/plugins/infra/public/components/logging/log_text_stream/log_entry_field_column.tsx @@ -7,7 +7,7 @@ import React from 'react'; import { JsonValue } from '../../../../../../../src/plugins/kibana_utils/common'; import { euiStyled } from '../../../../../../../src/plugins/kibana_react/common'; -import { LogColumn } from '../../../../common/http_api'; +import { LogColumn } from '../../../../common/log_entry'; import { isFieldColumn, isHighlightFieldColumn } from '../../../utils/log_entry'; import { FieldValue } from './field_value'; import { LogEntryColumnContent } from './log_entry_column'; diff --git a/x-pack/plugins/infra/public/components/logging/log_text_stream/log_entry_message_column.test.tsx b/x-pack/plugins/infra/public/components/logging/log_text_stream/log_entry_message_column.test.tsx index 5d36e5cd47c59b..00281c2df3133f 100644 --- a/x-pack/plugins/infra/public/components/logging/log_text_stream/log_entry_message_column.test.tsx +++ b/x-pack/plugins/infra/public/components/logging/log_text_stream/log_entry_message_column.test.tsx @@ -7,7 +7,7 @@ import { render } from '@testing-library/react'; import React from 'react'; import { EuiThemeProvider } from '../../../../../../../src/plugins/kibana_react/common'; -import { LogMessageColumn } from '../../../../common/http_api'; +import { LogMessageColumn } from '../../../../common/log_entry'; import { LogEntryMessageColumn } from './log_entry_message_column'; describe('LogEntryMessageColumn', () => { diff --git a/x-pack/plugins/infra/public/components/logging/log_text_stream/log_entry_row.tsx b/x-pack/plugins/infra/public/components/logging/log_text_stream/log_entry_row.tsx index 93c657fbdda971..036818317011c2 100644 --- a/x-pack/plugins/infra/public/components/logging/log_text_stream/log_entry_row.tsx +++ b/x-pack/plugins/infra/public/components/logging/log_text_stream/log_entry_row.tsx @@ -7,7 +7,6 @@ import React, { memo, useState, useCallback, useMemo } from 'react'; import { i18n } from '@kbn/i18n'; import { isEmpty } from 'lodash'; - import { euiStyled } from '../../../../../../../src/plugins/kibana_react/common'; import { useUiTracker } from '../../../../../observability/public'; import { isTimestampColumn } from '../../../utils/log_entry'; @@ -17,7 +16,7 @@ import { LogEntryFieldColumn } from './log_entry_field_column'; import { LogEntryMessageColumn } from './log_entry_message_column'; import { LogEntryTimestampColumn } from './log_entry_timestamp_column'; import { monospaceTextStyle, hoveredContentStyle, highlightedContentStyle } from './text_styles'; -import { LogEntry, LogColumn } from '../../../../common/http_api'; +import { LogEntry, LogColumn } from '../../../../common/log_entry'; import { LogEntryContextMenu } from './log_entry_context_menu'; import { LogColumnRenderConfiguration, diff --git a/x-pack/plugins/infra/public/components/logging/log_text_stream/scrollable_log_text_stream_view.tsx b/x-pack/plugins/infra/public/components/logging/log_text_stream/scrollable_log_text_stream_view.tsx index d399e47a735624..8fb63533cf61bf 100644 --- a/x-pack/plugins/infra/public/components/logging/log_text_stream/scrollable_log_text_stream_view.tsx +++ b/x-pack/plugins/infra/public/components/logging/log_text_stream/scrollable_log_text_stream_view.tsx @@ -25,7 +25,7 @@ import { MeasurableItemView } from './measurable_item_view'; import { VerticalScrollPanel } from './vertical_scroll_panel'; import { useColumnWidths, LogEntryColumnWidths } from './log_entry_column'; import { LogDateRow } from './log_date_row'; -import { LogEntry } from '../../../../common/http_api'; +import { LogEntry } from '../../../../common/log_entry'; import { LogColumnRenderConfiguration } from '../../../utils/log_column_render_configuration'; interface ScrollableLogTextStreamViewProps { diff --git a/x-pack/plugins/infra/public/containers/logs/log_entries/index.ts b/x-pack/plugins/infra/public/containers/logs/log_entries/index.ts index bf4c5fbe0b13bf..f1b820857e3407 100644 --- a/x-pack/plugins/infra/public/containers/logs/log_entries/index.ts +++ b/x-pack/plugins/infra/public/containers/logs/log_entries/index.ts @@ -10,10 +10,10 @@ import { pick, throttle } from 'lodash'; import { TimeKey, timeKeyIsBetween } from '../../../../common/time'; import { LogEntriesResponse, - LogEntry, LogEntriesRequest, LogEntriesBaseRequest, } from '../../../../common/http_api'; +import { LogEntry } from '../../../../common/log_entry'; import { fetchLogEntries } from './api/fetch_log_entries'; import { useKibanaContextForPlugin } from '../../../hooks/use_kibana'; diff --git a/x-pack/plugins/infra/public/containers/logs/log_stream/index.ts b/x-pack/plugins/infra/public/containers/logs/log_stream/index.ts index ff30e993aa3a92..f17e6c15604008 100644 --- a/x-pack/plugins/infra/public/containers/logs/log_stream/index.ts +++ b/x-pack/plugins/infra/public/containers/logs/log_stream/index.ts @@ -10,8 +10,7 @@ import usePrevious from 'react-use/lib/usePrevious'; import { esKuery } from '../../../../../../../src/plugins/data/public'; import { fetchLogEntries } from '../log_entries/api/fetch_log_entries'; import { useTrackedPromise } from '../../../utils/use_tracked_promise'; -import { LogEntry } from '../../../../common/http_api'; -import { LogEntryCursor } from '../../../../common/log_entry'; +import { LogEntry, LogEntryCursor } from '../../../../common/log_entry'; import { useKibanaContextForPlugin } from '../../../hooks/use_kibana'; import { LogSourceConfigurationProperties } from '../log_source'; diff --git a/x-pack/plugins/infra/public/containers/logs/view_log_in_context/view_log_in_context.ts b/x-pack/plugins/infra/public/containers/logs/view_log_in_context/view_log_in_context.ts index 61e1ea353880a0..2888e5a2b3ac50 100644 --- a/x-pack/plugins/infra/public/containers/logs/view_log_in_context/view_log_in_context.ts +++ b/x-pack/plugins/infra/public/containers/logs/view_log_in_context/view_log_in_context.ts @@ -5,7 +5,7 @@ */ import { useState } from 'react'; import createContainer from 'constate'; -import { LogEntry } from '../../../../common/http_api'; +import { LogEntry } from '../../../../common/log_entry'; interface ViewLogInContextProps { sourceId: string; diff --git a/x-pack/plugins/infra/public/containers/logs/with_stream_items.ts b/x-pack/plugins/infra/public/containers/logs/with_stream_items.ts index 2b8986820d5a43..89b5d993fa01e7 100644 --- a/x-pack/plugins/infra/public/containers/logs/with_stream_items.ts +++ b/x-pack/plugins/infra/public/containers/logs/with_stream_items.ts @@ -11,7 +11,7 @@ import { RendererFunction } from '../../utils/typed_react'; import { LogHighlightsState } from './log_highlights/log_highlights'; import { LogEntriesState, LogEntriesStateParams, LogEntriesCallbacks } from './log_entries'; import { UniqueTimeKey } from '../../../common/time'; -import { LogEntry } from '../../../common/http_api'; +import { LogEntry } from '../../../common/log_entry'; export const WithStreamItems: React.FunctionComponent<{ children: RendererFunction< diff --git a/x-pack/plugins/infra/public/pages/logs/log_entry_categories/sections/top_categories/category_example_message.tsx b/x-pack/plugins/infra/public/pages/logs/log_entry_categories/sections/top_categories/category_example_message.tsx index 84d7e198636e92..26f68635246ddb 100644 --- a/x-pack/plugins/infra/public/pages/logs/log_entry_categories/sections/top_categories/category_example_message.tsx +++ b/x-pack/plugins/infra/public/pages/logs/log_entry_categories/sections/top_categories/category_example_message.tsx @@ -9,7 +9,7 @@ import { i18n } from '@kbn/i18n'; import { encode } from 'rison-node'; import moment from 'moment'; -import { LogEntry, LogEntryContext } from '../../../../../../common/http_api'; +import { LogEntry, LogEntryContext } from '../../../../../../common/log_entry'; import { TimeRange } from '../../../../../../common/http_api/shared'; import { getFriendlyNameForPartitionId, diff --git a/x-pack/plugins/infra/public/pages/logs/stream/page_view_log_in_context.tsx b/x-pack/plugins/infra/public/pages/logs/stream/page_view_log_in_context.tsx index 3fa89da5b5e51a..011653fd6eb478 100644 --- a/x-pack/plugins/infra/public/pages/logs/stream/page_view_log_in_context.tsx +++ b/x-pack/plugins/infra/public/pages/logs/stream/page_view_log_in_context.tsx @@ -16,7 +16,7 @@ import { import { FormattedMessage } from '@kbn/i18n/react'; import { isEmpty } from 'lodash'; import React, { useCallback, useContext, useMemo } from 'react'; -import { LogEntry } from '../../../../common/http_api'; +import { LogEntry } from '../../../../common/log_entry'; import { ViewLogInContext } from '../../../containers/logs/view_log_in_context'; import { useViewportDimensions } from '../../../utils/use_viewport_dimensions'; import { euiStyled } from '../../../../../../../src/plugins/kibana_react/common'; diff --git a/x-pack/plugins/infra/server/lib/log_analysis/log_entry_categories_analysis.ts b/x-pack/plugins/infra/server/lib/log_analysis/log_entry_categories_analysis.ts index 7dd5aae9784f53..7d4a296bf1c139 100644 --- a/x-pack/plugins/infra/server/lib/log_analysis/log_entry_categories_analysis.ts +++ b/x-pack/plugins/infra/server/lib/log_analysis/log_entry_categories_analysis.ts @@ -5,7 +5,6 @@ */ import type { ILegacyScopedClusterClient } from 'src/core/server'; -import { LogEntryContext } from '../../../common/http_api'; import { compareDatasetsByMaximumAnomalyScore, getJobId, @@ -13,6 +12,7 @@ import { logEntryCategoriesJobTypes, } from '../../../common/log_analysis'; import { CategorySort } from '../../../common/http_api/log_analysis'; +import { LogEntryContext } from '../../../common/log_entry'; import { startTracingSpan } from '../../../common/performance_tracing'; import { decodeOrThrow } from '../../../common/runtime_types'; import type { MlAnomalyDetectors, MlSystem } from '../../types'; From 4994d05ce7899f4e78eed8c24bbee60f186ebaa0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20St=C3=BCrmer?= Date: Wed, 13 Jan 2021 13:13:04 +0100 Subject: [PATCH 10/39] Include index in old log entry API response --- .../sections/top_categories/category_example_message.tsx | 1 + .../lib/adapters/log_entries/kibana_log_entries_adapter.ts | 1 + .../server/lib/domains/log_entries_domain/log_entries_domain.ts | 2 ++ 3 files changed, 4 insertions(+) diff --git a/x-pack/plugins/infra/public/pages/logs/log_entry_categories/sections/top_categories/category_example_message.tsx b/x-pack/plugins/infra/public/pages/logs/log_entry_categories/sections/top_categories/category_example_message.tsx index 26f68635246ddb..3c7b1c362afba2 100644 --- a/x-pack/plugins/infra/public/pages/logs/log_entry_categories/sections/top_categories/category_example_message.tsx +++ b/x-pack/plugins/infra/public/pages/logs/log_entry_categories/sections/top_categories/category_example_message.tsx @@ -127,6 +127,7 @@ export const CategoryExampleMessage: React.FunctionComponent<{ onClick: () => { const logEntry: LogEntry = { id, + index: '', // TODO: use real index when loading via async search context, cursor: { time: timestamp, tiebreaker }, columns: [], diff --git a/x-pack/plugins/infra/server/lib/adapters/log_entries/kibana_log_entries_adapter.ts b/x-pack/plugins/infra/server/lib/adapters/log_entries/kibana_log_entries_adapter.ts index ffbc750af14f82..6702a43cb2316e 100644 --- a/x-pack/plugins/infra/server/lib/adapters/log_entries/kibana_log_entries_adapter.ts +++ b/x-pack/plugins/infra/server/lib/adapters/log_entries/kibana_log_entries_adapter.ts @@ -215,6 +215,7 @@ function mapHitsToLogEntryDocuments(hits: SortedSearchHit[], fields: string[]): return { id: hit._id, + index: hit._index, cursor: { time: hit.sort[0], tiebreaker: hit.sort[1] }, fields: logFields, highlights: hit.highlight || {}, diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/log_entries_domain.ts b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/log_entries_domain.ts index 0808b241cb15a4..c00e74647c3a1d 100644 --- a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/log_entries_domain.ts +++ b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/log_entries_domain.ts @@ -155,6 +155,7 @@ export class InfraLogEntriesDomain { const entries = documents.map((doc) => { return { id: doc.id, + index: doc.index, cursor: doc.cursor, columns: columnDefinitions.map( (column): LogColumn => { @@ -316,6 +317,7 @@ export type LogEntryQuery = JsonObject; export interface LogEntryDocument { id: string; + index: string; fields: Fields; highlights: Highlights; cursor: LogEntryCursor; From 4c86d39051248502287aa05d8a5a5b5d1284138c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20St=C3=BCrmer?= Date: Wed, 13 Jan 2021 17:21:46 +0100 Subject: [PATCH 11/39] Fix import in api integration test --- .../api_integration/apis/metrics_ui/log_entries.ts | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/x-pack/test/api_integration/apis/metrics_ui/log_entries.ts b/x-pack/test/api_integration/apis/metrics_ui/log_entries.ts index 2d148f4c2c0f7d..0323d3c098851a 100644 --- a/x-pack/test/api_integration/apis/metrics_ui/log_entries.ts +++ b/x-pack/test/api_integration/apis/metrics_ui/log_entries.ts @@ -6,18 +6,17 @@ import expect from '@kbn/expect'; import { v4 as uuidv4 } from 'uuid'; - -import { decodeOrThrow } from '../../../../plugins/infra/common/runtime_types'; - import { - LOG_ENTRIES_PATH, logEntriesRequestRT, logEntriesResponseRT, - LogTimestampColumn, + LOG_ENTRIES_PATH, +} from '../../../../plugins/infra/common/http_api'; +import { LogFieldColumn, LogMessageColumn, -} from '../../../../plugins/infra/common/http_api'; - + LogTimestampColumn, +} from '../../../../plugins/infra/common/log_entry'; +import { decodeOrThrow } from '../../../../plugins/infra/common/runtime_types'; import { FtrProviderContext } from '../../ftr_provider_context'; const KEY_WITHIN_DATA_RANGE = { From 981d511887b4e43d4974c3a49b395a8f459aa976 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20St=C3=BCrmer?= Date: Thu, 14 Jan 2021 11:18:29 +0100 Subject: [PATCH 12/39] Align function names --- .../services/log_entries/log_entries_search_strategy.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/infra/server/services/log_entries/log_entries_search_strategy.ts b/x-pack/plugins/infra/server/services/log_entries/log_entries_search_strategy.ts index 39bc0d68dc0aa5..55b66b275024bd 100644 --- a/x-pack/plugins/infra/server/services/log_entries/log_entries_search_strategy.ts +++ b/x-pack/plugins/infra/server/services/log_entries/log_entries_search_strategy.ts @@ -123,7 +123,7 @@ export const logEntriesSearchStrategyProvider = ({ forkJoin([sourceConfiguration$, messageFormattingRules$]).pipe( map(([{ configuration }, messageFormattingRules]) => { const entries = esResponse.rawResponse.hits.hits.map( - createLogEntryFromHit(configuration.logColumns, messageFormattingRules) + getLogEntryFromHit(configuration.logColumns, messageFormattingRules) ); const topCursor = null; const bottomCursor = null; @@ -135,7 +135,6 @@ export const logEntriesSearchStrategyProvider = ({ : {}), rawResponse: logEntriesSearchResponsePayloadRT.encode({ data: { entries, topCursor, bottomCursor }, - // data: esResponse.rawResponse.hits.hits.map(createLogEntryFromHit)[0] ?? null, errors: (esResponse.rawResponse._shards.failures ?? []).map( createErrorFromShardFailure ), @@ -165,7 +164,7 @@ const { asyncInitialRequestRT, asyncRecoveredRequestRT, asyncRequestRT } = creat logEntriesSearchRequestParamsRT ); -const createLogEntryFromHit = ( +const getLogEntryFromHit = ( columnDefinitions: LogSourceColumnConfiguration[], messageFormattingRules: CompiledLogMessageFormattingRule ) => (hit: LogEntryHit): LogEntry => { From 668f687fa457ed7588d4b8c8d03a42eb15a11567 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20St=C3=BCrmer?= Date: Thu, 14 Jan 2021 11:18:51 +0100 Subject: [PATCH 13/39] Comment out unfinished tests to avoid type errors --- .../log_entries_search_strategy.test.ts | 108 +++++++++--------- 1 file changed, 54 insertions(+), 54 deletions(-) diff --git a/x-pack/plugins/infra/server/services/log_entries/log_entries_search_strategy.test.ts b/x-pack/plugins/infra/server/services/log_entries/log_entries_search_strategy.test.ts index b86b4727b33779..2c52b0fbf4bf89 100644 --- a/x-pack/plugins/infra/server/services/log_entries/log_entries_search_strategy.test.ts +++ b/x-pack/plugins/infra/server/services/log_entries/log_entries_search_strategy.test.ts @@ -179,67 +179,67 @@ describe('LogEntries search strategy', () => { ]); }); - it.skip('forwards errors from the underlying search strategy', async () => { - const esSearchStrategyMock = createEsSearchStrategyMock({ - id: 'ASYNC_REQUEST_ID', - isRunning: false, - rawResponse: { - took: 1, - _shards: { total: 1, failed: 0, skipped: 0, successful: 1 }, - timed_out: false, - hits: { total: 0, max_score: 0, hits: [] }, - }, - }); - const dataMock = createDataPluginMock(esSearchStrategyMock); - const sourcesMock = createInfraSourcesMock(); - sourcesMock.getSourceConfiguration.mockResolvedValue(createSourceConfigurationMock()); - const mockDependencies = createSearchStrategyDependenciesMock(); + // it.skip('forwards errors from the underlying search strategy', async () => { + // const esSearchStrategyMock = createEsSearchStrategyMock({ + // id: 'ASYNC_REQUEST_ID', + // isRunning: false, + // rawResponse: { + // took: 1, + // _shards: { total: 1, failed: 0, skipped: 0, successful: 1 }, + // timed_out: false, + // hits: { total: 0, max_score: 0, hits: [] }, + // }, + // }); + // const dataMock = createDataPluginMock(esSearchStrategyMock); + // const sourcesMock = createInfraSourcesMock(); + // sourcesMock.getSourceConfiguration.mockResolvedValue(createSourceConfigurationMock()); + // const mockDependencies = createSearchStrategyDependenciesMock(); - const logEntriesSearchStrategy = logEntriesSearchStrategyProvider({ - data: dataMock, - sources: sourcesMock, - }); + // const logEntriesSearchStrategy = logEntriesSearchStrategyProvider({ + // data: dataMock, + // sources: sourcesMock, + // }); - const response = logEntriesSearchStrategy.search( - { - id: logEntriesSearchRequestStateRT.encode({ esRequestId: 'UNKNOWN_ID' }), - params: { sourceId: 'SOURCE_ID', logEntryId: 'LOG_ENTRY_ID' }, - }, - {}, - mockDependencies - ); + // const response = logEntriesSearchStrategy.search( + // { + // id: logEntriesSearchRequestStateRT.encode({ esRequestId: 'UNKNOWN_ID' }), + // params: { sourceId: 'SOURCE_ID', logEntryId: 'LOG_ENTRY_ID' }, + // }, + // {}, + // mockDependencies + // ); - await expect(response.toPromise()).rejects.toThrowError(ResponseError); - }); + // await expect(response.toPromise()).rejects.toThrowError(ResponseError); + // }); - it.skip('forwards cancellation to the underlying search strategy', async () => { - const esSearchStrategyMock = createEsSearchStrategyMock({ - id: 'ASYNC_REQUEST_ID', - isRunning: false, - rawResponse: { - took: 1, - _shards: { total: 1, failed: 0, skipped: 0, successful: 1 }, - timed_out: false, - hits: { total: 0, max_score: 0, hits: [] }, - }, - }); - const dataMock = createDataPluginMock(esSearchStrategyMock); - const sourcesMock = createInfraSourcesMock(); - sourcesMock.getSourceConfiguration.mockResolvedValue(createSourceConfigurationMock()); - const mockDependencies = createSearchStrategyDependenciesMock(); + // it.skip('forwards cancellation to the underlying search strategy', async () => { + // const esSearchStrategyMock = createEsSearchStrategyMock({ + // id: 'ASYNC_REQUEST_ID', + // isRunning: false, + // rawResponse: { + // took: 1, + // _shards: { total: 1, failed: 0, skipped: 0, successful: 1 }, + // timed_out: false, + // hits: { total: 0, max_score: 0, hits: [] }, + // }, + // }); + // const dataMock = createDataPluginMock(esSearchStrategyMock); + // const sourcesMock = createInfraSourcesMock(); + // sourcesMock.getSourceConfiguration.mockResolvedValue(createSourceConfigurationMock()); + // const mockDependencies = createSearchStrategyDependenciesMock(); - const logEntriesSearchStrategy = logEntriesSearchStrategyProvider({ - data: dataMock, - sources: sourcesMock, - }); - const requestId = logEntriesSearchRequestStateRT.encode({ - esRequestId: 'ASYNC_REQUEST_ID', - }); + // const logEntriesSearchStrategy = logEntriesSearchStrategyProvider({ + // data: dataMock, + // sources: sourcesMock, + // }); + // const requestId = logEntriesSearchRequestStateRT.encode({ + // esRequestId: 'ASYNC_REQUEST_ID', + // }); - await logEntriesSearchStrategy.cancel?.(requestId, {}, mockDependencies); + // await logEntriesSearchStrategy.cancel?.(requestId, {}, mockDependencies); - expect(esSearchStrategyMock.cancel).toHaveBeenCalled(); - }); + // expect(esSearchStrategyMock.cancel).toHaveBeenCalled(); + // }); }); const createSourceConfigurationMock = (): InfraSource => ({ From 880485e48cc9ec34b98899e8e3eff8cd5c1de534 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20St=C3=BCrmer?= Date: Thu, 14 Jan 2021 11:19:24 +0100 Subject: [PATCH 14/39] WIP --- .../containers/logs/log_entries_async.ts | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 x-pack/plugins/infra/public/containers/logs/log_entries_async.ts diff --git a/x-pack/plugins/infra/public/containers/logs/log_entries_async.ts b/x-pack/plugins/infra/public/containers/logs/log_entries_async.ts new file mode 100644 index 00000000000000..4fe993720f41f8 --- /dev/null +++ b/x-pack/plugins/infra/public/containers/logs/log_entries_async.ts @@ -0,0 +1,56 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { useCallback } from 'react'; +import { decodeOrThrow } from '../../../common/runtime_types'; +import { + logEntriesSearchRequestParamsRT, + logEntriesSearchResponsePayloadRT, + LOG_ENTRIES_SEARCH_STRATEGY, +} from '../../../common/search_strategies/log_entries/log_entries'; +import { useDataSearch, useLatestPartialDataSearchResponse } from '../../utils/data_search'; + +export const useLogEntries = ({ sourceId }: { sourceId: string | null | undefined }) => { + const { search: fetchLogEntry, requests$: logEntrySearchRequests$ } = useDataSearch({ + getRequest: useCallback(() => { + return !!logEntryId && !!sourceId + ? { + request: { + params: logEntrySearchRequestParamsRT.encode({ sourceId, logEntryId }), + }, + options: { strategy: LOG_ENTRY_SEARCH_STRATEGY }, + } + : null; + }, [sourceId, logEntryId]), + }); + + const { + cancelRequest, + isRequestRunning, + isResponsePartial, + latestResponseData, + latestResponseErrors, + loaded, + total, + } = useLatestPartialDataSearchResponse( + logEntrySearchRequests$, + null, + decodeLogEntrySearchResponse + ); + + return { + cancelRequest, + errors: latestResponseErrors, + fetchLogEntry, + isRequestRunning, + isResponsePartial, + loaded, + logEntry: latestResponseData ?? null, + total, + }; +}; + +const decodeLogEntrySearchResponse = decodeOrThrow(logEntrySearchResponsePayloadRT); From 162ad28d85eb0f9d144e59ea20d941ad76a187c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20St=C3=BCrmer?= Date: Thu, 14 Jan 2021 19:18:39 +0100 Subject: [PATCH 15/39] Factor out handling of search responses --- .../handle_data_search_response.ts | 89 +++++++++++++++++++ ...use_latest_partial_data_search_response.ts | 67 ++------------ 2 files changed, 94 insertions(+), 62 deletions(-) create mode 100644 x-pack/plugins/infra/public/utils/data_search/handle_data_search_response.ts diff --git a/x-pack/plugins/infra/public/utils/data_search/handle_data_search_response.ts b/x-pack/plugins/infra/public/utils/data_search/handle_data_search_response.ts new file mode 100644 index 00000000000000..52a38dbb628d7f --- /dev/null +++ b/x-pack/plugins/infra/public/utils/data_search/handle_data_search_response.ts @@ -0,0 +1,89 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { Observable, of } from 'rxjs'; +import { catchError, map, startWith } from 'rxjs/operators'; +import { IKibanaSearchRequest } from '../../../../../../src/plugins/data/public'; +import { AbortError } from '../../../../../../src/plugins/kibana_utils/public'; +import { SearchStrategyError } from '../../../common/search_strategies/common/errors'; +import { DataSearchRequestDescriptor, DataSearchResponseDescriptor } from './types'; + +export type ResponseProjection = ( + rawResponse: RawResponse +) => { data: Response; errors?: SearchStrategyError[] }; + +export const handleDataSearchResponse = < + Request extends IKibanaSearchRequest, + RawResponse, + Response, + InitialResponse +>( + // these are ref objects so they can be changed without having to recreate a new pipeline + initialResponseRef: { current: InitialResponse }, + projectResponseRef: { current: ResponseProjection } +) => ({ + abortController, + options, + request, + response$, +}: DataSearchRequestDescriptor): Observable< + DataSearchResponseDescriptor +> => + response$.pipe( + map((response) => { + const { data, errors = [] } = projectResponseRef.current(response.rawResponse); + return { + abortController, + options, + request, + response: { + data, + errors, + isPartial: response.isPartial ?? false, + isRunning: response.isRunning ?? false, + loaded: response.loaded, + total: response.total, + }, + }; + }), + startWith({ + abortController, + options, + request, + response: { + data: initialResponseRef.current, + errors: [], + isPartial: true, + isRunning: true, + loaded: 0, + total: undefined, + }, + }), + catchError((error) => + of({ + abortController, + options, + request, + response: { + data: initialResponseRef.current, + errors: [ + error instanceof AbortError + ? { + type: 'aborted' as const, + } + : { + type: 'generic' as const, + message: `${error.message ?? error}`, + }, + ], + isPartial: true, + isRunning: false, + loaded: 0, + total: undefined, + }, + }) + ) + ); diff --git a/x-pack/plugins/infra/public/utils/data_search/use_latest_partial_data_search_response.ts b/x-pack/plugins/infra/public/utils/data_search/use_latest_partial_data_search_response.ts index 71fd96283d0ef9..34ab220f3e2208 100644 --- a/x-pack/plugins/infra/public/utils/data_search/use_latest_partial_data_search_response.ts +++ b/x-pack/plugins/infra/public/utils/data_search/use_latest_partial_data_search_response.ts @@ -5,12 +5,11 @@ */ import { useCallback } from 'react'; -import { Observable, of } from 'rxjs'; -import { catchError, map, startWith, switchMap } from 'rxjs/operators'; +import { Observable } from 'rxjs'; +import { switchMap } from 'rxjs/operators'; import { IKibanaSearchRequest } from '../../../../../../src/plugins/data/public'; -import { AbortError } from '../../../../../../src/plugins/kibana_utils/public'; -import { SearchStrategyError } from '../../../common/search_strategies/common/errors'; import { useLatest, useObservable, useObservableState } from '../use_observable'; +import { handleDataSearchResponse, ResponseProjection } from './handle_data_search_response'; import { DataSearchRequestDescriptor, DataSearchResponseDescriptor } from './types'; export const useLatestPartialDataSearchResponse = < @@ -21,7 +20,7 @@ export const useLatestPartialDataSearchResponse = < >( requests$: Observable>, initialResponse: InitialResponse, - projectResponse: (rawResponse: RawResponse) => { data: Response; errors?: SearchStrategyError[] } + projectResponse: ResponseProjection ) => { const latestInitialResponse = useLatest(initialResponse); const latestProjectResponse = useLatest(projectResponse); @@ -33,63 +32,7 @@ export const useLatestPartialDataSearchResponse = < inputs$.pipe( switchMap(([currentRequests$]) => currentRequests$.pipe( - switchMap(({ abortController, options, request, response$ }) => - response$.pipe( - map((response) => { - const { data, errors = [] } = latestProjectResponse.current(response.rawResponse); - return { - abortController, - options, - request, - response: { - data, - errors, - isPartial: response.isPartial ?? false, - isRunning: response.isRunning ?? false, - loaded: response.loaded, - total: response.total, - }, - }; - }), - startWith({ - abortController, - options, - request, - response: { - data: latestInitialResponse.current, - errors: [], - isPartial: true, - isRunning: true, - loaded: 0, - total: undefined, - }, - }), - catchError((error) => - of({ - abortController, - options, - request, - response: { - data: latestInitialResponse.current, - errors: [ - error instanceof AbortError - ? { - type: 'aborted' as const, - } - : { - type: 'generic' as const, - message: `${error.message ?? error}`, - }, - ], - isPartial: true, - isRunning: false, - loaded: 0, - total: undefined, - }, - }) - ) - ) - ) + switchMap(handleDataSearchResponse(latestInitialResponse, latestProjectResponse)) ) ) ), From 4d53acd1b6a013db3e6f342313380f1ea93c64f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20St=C3=BCrmer?= Date: Fri, 15 Jan 2021 01:06:39 +0100 Subject: [PATCH 16/39] Document the response handling function --- .../handle_data_search_response.ts | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/infra/public/utils/data_search/handle_data_search_response.ts b/x-pack/plugins/infra/public/utils/data_search/handle_data_search_response.ts index 52a38dbb628d7f..050d4be9b877a0 100644 --- a/x-pack/plugins/infra/public/utils/data_search/handle_data_search_response.ts +++ b/x-pack/plugins/infra/public/utils/data_search/handle_data_search_response.ts @@ -15,13 +15,31 @@ export type ResponseProjection = ( rawResponse: RawResponse ) => { data: Response; errors?: SearchStrategyError[] }; +/** + * Turns the {@link DataSearchRequestDescriptor} into a {@link + * DataSearchResponseDescriptor} by decoding or validating and unrolling the + * partial and final responses emitted. + * + * Since the parameters are refs they will be used immediately for the next + * response without the need to recreate the pipeline. + * + * + * @param initialResponseRef - A ref object containing the initial value to + * emit when a new request is handled. * + * @param projectResponseRef - A ref object containing the projection function + * to apply to each response payload. It should validate that the response + * payload is of the type {@link RawResponse} and decode it to a {@link + * Response}. + * + * @return A function that decodes and validates each response payload using + * the arguments given above. + */ export const handleDataSearchResponse = < Request extends IKibanaSearchRequest, RawResponse, Response, InitialResponse >( - // these are ref objects so they can be changed without having to recreate a new pipeline initialResponseRef: { current: InitialResponse }, projectResponseRef: { current: ResponseProjection } ) => ({ From fb087493e4be8c5e28348b0743693ffdb18c127c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20St=C3=BCrmer?= Date: Fri, 15 Jan 2021 11:58:55 +0100 Subject: [PATCH 17/39] WIP --- .../containers/logs/log_entries_async.ts | 92 +++++++++---------- .../log_stream/use_fetch_log_entries_after.ts | 61 ++++++++++++ .../use_fetch_log_entries_around.ts | 48 ++++++++++ 3 files changed, 155 insertions(+), 46 deletions(-) create mode 100644 x-pack/plugins/infra/public/containers/logs/log_stream/use_fetch_log_entries_after.ts create mode 100644 x-pack/plugins/infra/public/containers/logs/log_stream/use_fetch_log_entries_around.ts diff --git a/x-pack/plugins/infra/public/containers/logs/log_entries_async.ts b/x-pack/plugins/infra/public/containers/logs/log_entries_async.ts index 4fe993720f41f8..92302b7f304f09 100644 --- a/x-pack/plugins/infra/public/containers/logs/log_entries_async.ts +++ b/x-pack/plugins/infra/public/containers/logs/log_entries_async.ts @@ -4,53 +4,53 @@ * you may not use this file except in compliance with the Elastic License. */ -import { useCallback } from 'react'; -import { decodeOrThrow } from '../../../common/runtime_types'; -import { - logEntriesSearchRequestParamsRT, - logEntriesSearchResponsePayloadRT, - LOG_ENTRIES_SEARCH_STRATEGY, -} from '../../../common/search_strategies/log_entries/log_entries'; -import { useDataSearch, useLatestPartialDataSearchResponse } from '../../utils/data_search'; +// import { useCallback } from 'react'; +// import { decodeOrThrow } from '../../../common/runtime_types'; +// import { +// logEntriesSearchRequestParamsRT, +// logEntriesSearchResponsePayloadRT, +// LOG_ENTRIES_SEARCH_STRATEGY, +// } from '../../../common/search_strategies/log_entries/log_entries'; +// import { useDataSearch, useLatestPartialDataSearchResponse } from '../../utils/data_search'; -export const useLogEntries = ({ sourceId }: { sourceId: string | null | undefined }) => { - const { search: fetchLogEntry, requests$: logEntrySearchRequests$ } = useDataSearch({ - getRequest: useCallback(() => { - return !!logEntryId && !!sourceId - ? { - request: { - params: logEntrySearchRequestParamsRT.encode({ sourceId, logEntryId }), - }, - options: { strategy: LOG_ENTRY_SEARCH_STRATEGY }, - } - : null; - }, [sourceId, logEntryId]), - }); +// export const useLogEntries = ({ sourceId }: { sourceId: string | null | undefined }) => { +// const { search: fetchLogEntry, requests$: logEntrySearchRequests$ } = useDataSearch({ +// getRequest: useCallback(() => { +// return !!logEntryId && !!sourceId +// ? { +// request: { +// params: logEntrySearchRequestParamsRT.encode({ sourceId, logEntryId }), +// }, +// options: { strategy: LOG_ENTRY_SEARCH_STRATEGY }, +// } +// : null; +// }, [sourceId, logEntryId]), +// }); - const { - cancelRequest, - isRequestRunning, - isResponsePartial, - latestResponseData, - latestResponseErrors, - loaded, - total, - } = useLatestPartialDataSearchResponse( - logEntrySearchRequests$, - null, - decodeLogEntrySearchResponse - ); +// const { +// cancelRequest, +// isRequestRunning, +// isResponsePartial, +// latestResponseData, +// latestResponseErrors, +// loaded, +// total, +// } = useLatestPartialDataSearchResponse( +// logEntrySearchRequests$, +// null, +// decodeLogEntrySearchResponse +// ); - return { - cancelRequest, - errors: latestResponseErrors, - fetchLogEntry, - isRequestRunning, - isResponsePartial, - loaded, - logEntry: latestResponseData ?? null, - total, - }; -}; +// return { +// cancelRequest, +// errors: latestResponseErrors, +// fetchLogEntry, +// isRequestRunning, +// isResponsePartial, +// loaded, +// logEntry: latestResponseData ?? null, +// total, +// }; +// }; -const decodeLogEntrySearchResponse = decodeOrThrow(logEntrySearchResponsePayloadRT); +// const decodeLogEntrySearchResponse = decodeOrThrow(logEntrySearchResponsePayloadRT); diff --git a/x-pack/plugins/infra/public/containers/logs/log_stream/use_fetch_log_entries_after.ts b/x-pack/plugins/infra/public/containers/logs/log_stream/use_fetch_log_entries_after.ts new file mode 100644 index 00000000000000..2ce2f2dc80a65b --- /dev/null +++ b/x-pack/plugins/infra/public/containers/logs/log_stream/use_fetch_log_entries_after.ts @@ -0,0 +1,61 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { useCallback } from 'react'; +import { LogSourceColumnConfiguration } from '../../../../common/http_api/log_sources'; +import { + logEntriesSearchRequestParamsRT, + LOG_ENTRIES_SEARCH_STRATEGY, +} from '../../../../common/search_strategies/log_entries/log_entries'; +import { JsonObject } from '../../../../common/typed_json'; +import { useDataSearch } from '../../../utils/data_search'; + +export const useFetchLogEntriesAfter = ({ + columnOverrides, + endTimestamp, + highlightPhrase, + query, + sourceId, + startTimestamp, +}: { + columnOverrides?: LogSourceColumnConfiguration[]; + endTimestamp: number; + highlightPhrase?: string; + query?: JsonObject; + sourceId: string; + startTimestamp: number; +}) => { + const { search: fetchLogEntriesAfter, requests$: logEntriesAfterSearchRequests$ } = useDataSearch( + { + getRequest: useCallback( + (size: number) => { + return !!sourceId + ? { + request: { + params: logEntriesSearchRequestParamsRT.encode({ + columns: columnOverrides, + endTimestamp, + highlightPhrase, + query, + size, + sourceId, + startTimestamp, + }), + }, + options: { strategy: LOG_ENTRIES_SEARCH_STRATEGY }, + } + : null; + }, + [columnOverrides, endTimestamp, highlightPhrase, query, sourceId, startTimestamp] + ), + } + ); + + return { + fetchLogEntriesAfter, + logEntriesAfterSearchRequests$, + }; +}; diff --git a/x-pack/plugins/infra/public/containers/logs/log_stream/use_fetch_log_entries_around.ts b/x-pack/plugins/infra/public/containers/logs/log_stream/use_fetch_log_entries_around.ts new file mode 100644 index 00000000000000..c8527996d2943c --- /dev/null +++ b/x-pack/plugins/infra/public/containers/logs/log_stream/use_fetch_log_entries_around.ts @@ -0,0 +1,48 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { useCallback } from 'react'; +import { + logEntriesSearchRequestParamsRT, + LOG_ENTRIES_SEARCH_STRATEGY, +} from '../../../../common/search_strategies/log_entries/log_entries'; +import { useDataSearch } from '../../../utils/data_search'; + +export const useFetchLogEntriesAround = ({ + endTimestamp, + sourceId, + startTimestamp, +}: { + endTimestamp: number; + sourceId: string; + startTimestamp: number; +}) => { + const { search: fetchLogEntry, requests$: logEntrySearchRequests$ } = useDataSearch({ + getRequest: useCallback( + (size: number) => { + return !!sourceId + ? { + request: { + params: logEntriesSearchRequestParamsRT.encode({ + endTimestamp, + size, + sourceId, + startTimestamp, + }), + }, + options: { strategy: LOG_ENTRIES_SEARCH_STRATEGY }, + } + : null; + }, + [endTimestamp, sourceId, startTimestamp] + ), + }); + + return { + fetchLogEntry, + logEntrySearchRequests$, + }; +}; From 24d4d2b4c422e243e2293f83dd8e4b1c6a294e08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20St=C3=BCrmer?= Date: Fri, 15 Jan 2021 16:16:30 +0100 Subject: [PATCH 18/39] Remove erroneous terminate_after --- .../server/services/log_entries/log_entries_search_strategy.ts | 2 +- .../infra/server/services/log_entries/queries/log_entries.ts | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/x-pack/plugins/infra/server/services/log_entries/log_entries_search_strategy.ts b/x-pack/plugins/infra/server/services/log_entries/log_entries_search_strategy.ts index 55b66b275024bd..91706a8161a51a 100644 --- a/x-pack/plugins/infra/server/services/log_entries/log_entries_search_strategy.ts +++ b/x-pack/plugins/infra/server/services/log_entries/log_entries_search_strategy.ts @@ -125,7 +125,7 @@ export const logEntriesSearchStrategyProvider = ({ const entries = esResponse.rawResponse.hits.hits.map( getLogEntryFromHit(configuration.logColumns, messageFormattingRules) ); - const topCursor = null; + const topCursor = null; // TODO: determine cursors const bottomCursor = null; return { diff --git a/x-pack/plugins/infra/server/services/log_entries/queries/log_entries.ts b/x-pack/plugins/infra/server/services/log_entries/queries/log_entries.ts index 9bd05caf9c191e..d8181e39219b27 100644 --- a/x-pack/plugins/infra/server/services/log_entries/queries/log_entries.ts +++ b/x-pack/plugins/infra/server/services/log_entries/queries/log_entries.ts @@ -37,7 +37,6 @@ export const createGetLogEntriesQuery = ( return { index: logEntriesIndex, allow_no_indices: true, - terminate_after: 1, track_scores: false, track_total_hits: false, body: { From caacaa58ad0f2665a5444ceadeca4e38f2030892 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20St=C3=BCrmer?= Date: Fri, 15 Jan 2021 19:17:23 +0100 Subject: [PATCH 19/39] Continue to make the search strategy more complete --- .../log_entries_search_strategy.ts | 85 +++++++++++-------- .../log_entries/queries/log_entries.ts | 2 +- 2 files changed, 51 insertions(+), 36 deletions(-) diff --git a/x-pack/plugins/infra/server/services/log_entries/log_entries_search_strategy.ts b/x-pack/plugins/infra/server/services/log_entries/log_entries_search_strategy.ts index 91706a8161a51a..6ce3d4410a2dd2 100644 --- a/x-pack/plugins/infra/server/services/log_entries/log_entries_search_strategy.ts +++ b/x-pack/plugins/infra/server/services/log_entries/log_entries_search_strategy.ts @@ -6,7 +6,7 @@ import { pick } from '@kbn/std'; import * as rt from 'io-ts'; -import { concat, defer, forkJoin, of } from 'rxjs'; +import { combineLatest, concat, defer, forkJoin, of } from 'rxjs'; import { concatMap, filter, map, shareReplay, take } from 'rxjs/operators'; import type { IEsSearchRequest, @@ -49,6 +49,7 @@ import { import { createGetLogEntriesQuery, getLogEntriesResponseRT, + getSortDirection, LogEntryHit, } from './queries/log_entries'; @@ -97,8 +98,8 @@ export const logEntriesSearchStrategyProvider = ({ configuration.logAlias, params.startTimestamp, params.endTimestamp, - pickCursor(params), - params.size, + pickRequestCursor(params), + params.size + 1, configuration.fields.timestamp, configuration.fields.tiebreaker, messageFormattingRules.requiredFields, @@ -112,37 +113,44 @@ export const logEntriesSearchStrategyProvider = ({ ) ); - return concat(recoveredRequest$, initialRequest$).pipe( + const searchResponse$ = concat(recoveredRequest$, initialRequest$).pipe( take(1), - concatMap((esRequest) => esSearchStrategy.search(esRequest, options, dependencies)), - map((esResponse) => ({ - ...esResponse, - rawResponse: decodeOrThrow(getLogEntriesResponseRT)(esResponse.rawResponse), - })), - concatMap((esResponse) => - forkJoin([sourceConfiguration$, messageFormattingRules$]).pipe( - map(([{ configuration }, messageFormattingRules]) => { - const entries = esResponse.rawResponse.hits.hits.map( - getLogEntryFromHit(configuration.logColumns, messageFormattingRules) - ); - const topCursor = null; // TODO: determine cursors - const bottomCursor = null; - - return { - ...esResponse, - ...(esResponse.id - ? { id: logEntriesSearchRequestStateRT.encode({ esRequestId: esResponse.id }) } - : {}), - rawResponse: logEntriesSearchResponsePayloadRT.encode({ - data: { entries, topCursor, bottomCursor }, - errors: (esResponse.rawResponse._shards.failures ?? []).map( - createErrorFromShardFailure - ), - }), - }; - }) - ) - ) + concatMap((esRequest) => esSearchStrategy.search(esRequest, options, dependencies)) + ); + + return combineLatest([searchResponse$, sourceConfiguration$, messageFormattingRules$]).pipe( + map(([esResponse, { configuration }, messageFormattingRules]) => { + const rawResponse = decodeOrThrow(getLogEntriesResponseRT)(esResponse.rawResponse); + + const entries = rawResponse.hits.hits + .slice(0, request.params.size) + .map(getLogEntryFromHit(configuration.logColumns, messageFormattingRules)); + + const sortDirection = getSortDirection(pickRequestCursor(request.params)); + + if (sortDirection === 'desc') { + entries.reverse(); + } + + const hasMore = rawResponse.hits.hits.length > entries.length; + const hasMoreBefore = sortDirection === 'desc' ? hasMore : undefined; + const hasMoreAfter = sortDirection === 'asc' ? hasMore : undefined; + + const { topCursor, bottomCursor } = getResponseCursors(entries); + + const errors = (rawResponse._shards.failures ?? []).map(createErrorFromShardFailure); + + return { + ...esResponse, + ...(esResponse.id + ? { id: logEntriesSearchRequestStateRT.encode({ esRequestId: esResponse.id }) } + : {}), + rawResponse: logEntriesSearchResponsePayloadRT.encode({ + data: { entries, topCursor, bottomCursor, hasMoreBefore, hasMoreAfter }, + errors, + }), + }; + }) ); }), cancel: async (id, options, dependencies) => { @@ -196,11 +204,10 @@ const getLogEntryFromHit = ( } ), context: getContextFromHit(hit), - // fields: Object.entries(hit.fields).map(([field, value]) => ({ field, value })), }; }; -const pickCursor = ( +const pickRequestCursor = ( params: LogEntriesSearchRequestParams ): LogEntryAfterCursor | LogEntryBeforeCursor | null => { if (logEntryAfterCursorRT.is(params)) { @@ -228,3 +235,11 @@ const getContextFromHit = (hit: LogEntryHit): LogEntryContext => { return {}; }; + +function getResponseCursors(entries: LogEntry[]) { + const hasEntries = entries.length > 0; + const topCursor = hasEntries ? entries[0].cursor : null; + const bottomCursor = hasEntries ? entries[entries.length - 1].cursor : null; + + return { topCursor, bottomCursor }; +} diff --git a/x-pack/plugins/infra/server/services/log_entries/queries/log_entries.ts b/x-pack/plugins/infra/server/services/log_entries/queries/log_entries.ts index d8181e39219b27..81476fa2b286e2 100644 --- a/x-pack/plugins/infra/server/services/log_entries/queries/log_entries.ts +++ b/x-pack/plugins/infra/server/services/log_entries/queries/log_entries.ts @@ -59,7 +59,7 @@ export const createGetLogEntriesQuery = ( }; }; -const getSortDirection = ( +export const getSortDirection = ( cursor: LogEntryBeforeCursor | LogEntryAfterCursor | null | undefined ): 'asc' | 'desc' => (logEntryBeforeCursorRT.is(cursor) ? 'desc' : 'asc'); From 98a07ac2184007f85506024f0f6e6b5d27828c3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20St=C3=BCrmer?= Date: Fri, 15 Jan 2021 19:18:31 +0100 Subject: [PATCH 20/39] Factor out data search response flattening --- .../log_stream/use_fetch_log_entries_after.ts | 26 ++++++++++++++-- .../infra/public/utils/data_search/index.ts | 2 ++ .../data_search/use_flattened_response.ts | 31 +++++++++++++++++++ ...use_latest_partial_data_search_response.ts | 16 +++------- 4 files changed, 61 insertions(+), 14 deletions(-) create mode 100644 x-pack/plugins/infra/public/utils/data_search/use_flattened_response.ts diff --git a/x-pack/plugins/infra/public/containers/logs/log_stream/use_fetch_log_entries_after.ts b/x-pack/plugins/infra/public/containers/logs/log_stream/use_fetch_log_entries_after.ts index 2ce2f2dc80a65b..5a0aea302ef636 100644 --- a/x-pack/plugins/infra/public/containers/logs/log_stream/use_fetch_log_entries_after.ts +++ b/x-pack/plugins/infra/public/containers/logs/log_stream/use_fetch_log_entries_after.ts @@ -5,13 +5,22 @@ */ import { useCallback } from 'react'; +import { exhaustMap, switchMap } from 'rxjs/operators'; import { LogSourceColumnConfiguration } from '../../../../common/http_api/log_sources'; +import { LogEntryCursor } from '../../../../common/log_entry'; +import { decodeOrThrow } from '../../../../common/runtime_types'; import { logEntriesSearchRequestParamsRT, + logEntriesSearchResponsePayloadRT, LOG_ENTRIES_SEARCH_STRATEGY, } from '../../../../common/search_strategies/log_entries/log_entries'; import { JsonObject } from '../../../../common/typed_json'; -import { useDataSearch } from '../../../utils/data_search'; +import { + handleDataSearchResponse, + useDataSearch, + useFlattenedResponse, +} from '../../../utils/data_search'; +import { useObservable } from '../../../utils/use_observable'; export const useFetchLogEntriesAfter = ({ columnOverrides, @@ -31,11 +40,12 @@ export const useFetchLogEntriesAfter = ({ const { search: fetchLogEntriesAfter, requests$: logEntriesAfterSearchRequests$ } = useDataSearch( { getRequest: useCallback( - (size: number) => { + (cursor: LogEntryCursor, size: number) => { return !!sourceId ? { request: { params: logEntriesSearchRequestParamsRT.encode({ + after: cursor, columns: columnOverrides, endTimestamp, highlightPhrase, @@ -54,8 +64,18 @@ export const useFetchLogEntriesAfter = ({ } ); + const logEntriesAfterSearchResponse$ = useFlattenedResponse( + logEntriesAfterSearchRequests$, + exhaustMap( + handleDataSearchResponse( + { current: null }, + { current: decodeOrThrow(logEntriesSearchResponsePayloadRT) } + ) + ) + ); + return { fetchLogEntriesAfter, - logEntriesAfterSearchRequests$, + logEntriesAfterSearchResponse$, }; }; diff --git a/x-pack/plugins/infra/public/utils/data_search/index.ts b/x-pack/plugins/infra/public/utils/data_search/index.ts index c08ab0727fd904..5ff1ac9fc1da5c 100644 --- a/x-pack/plugins/infra/public/utils/data_search/index.ts +++ b/x-pack/plugins/infra/public/utils/data_search/index.ts @@ -4,6 +4,8 @@ * you may not use this file except in compliance with the Elastic License. */ +export * from './handle_data_search_response'; export * from './types'; export * from './use_data_search_request'; +export * from './use_flattened_response'; export * from './use_latest_partial_data_search_response'; diff --git a/x-pack/plugins/infra/public/utils/data_search/use_flattened_response.ts b/x-pack/plugins/infra/public/utils/data_search/use_flattened_response.ts new file mode 100644 index 00000000000000..a76bf08f6192a0 --- /dev/null +++ b/x-pack/plugins/infra/public/utils/data_search/use_flattened_response.ts @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { Observable, OperatorFunction } from 'rxjs'; +import { switchMap } from 'rxjs/operators'; +import { IKibanaSearchRequest } from '../../../../../../src/plugins/data/public'; +import { useLatest, useObservable } from '../use_observable'; +import { DataSearchRequestDescriptor, DataSearchResponseDescriptor } from './types'; + +export const useFlattenedResponse = ( + requests$: Observable>, + project: OperatorFunction< + DataSearchRequestDescriptor, + DataSearchResponseDescriptor + > +) => { + const latestProject = useLatest(project); + + const latestResponse$: Observable< + DataSearchResponseDescriptor + > = useObservable( + (inputs$) => + inputs$.pipe(switchMap(([currentRequests$]) => latestProject.current(currentRequests$))), + [requests$] as const + ); + + return latestResponse$; +}; diff --git a/x-pack/plugins/infra/public/utils/data_search/use_latest_partial_data_search_response.ts b/x-pack/plugins/infra/public/utils/data_search/use_latest_partial_data_search_response.ts index 34ab220f3e2208..459510f7600525 100644 --- a/x-pack/plugins/infra/public/utils/data_search/use_latest_partial_data_search_response.ts +++ b/x-pack/plugins/infra/public/utils/data_search/use_latest_partial_data_search_response.ts @@ -8,9 +8,10 @@ import { useCallback } from 'react'; import { Observable } from 'rxjs'; import { switchMap } from 'rxjs/operators'; import { IKibanaSearchRequest } from '../../../../../../src/plugins/data/public'; -import { useLatest, useObservable, useObservableState } from '../use_observable'; +import { useLatest, useObservableState } from '../use_observable'; import { handleDataSearchResponse, ResponseProjection } from './handle_data_search_response'; import { DataSearchRequestDescriptor, DataSearchResponseDescriptor } from './types'; +import { useFlattenedResponse } from './use_flattened_response'; export const useLatestPartialDataSearchResponse = < Request extends IKibanaSearchRequest, @@ -27,16 +28,9 @@ export const useLatestPartialDataSearchResponse = < const latestResponse$: Observable< DataSearchResponseDescriptor - > = useObservable( - (inputs$) => - inputs$.pipe( - switchMap(([currentRequests$]) => - currentRequests$.pipe( - switchMap(handleDataSearchResponse(latestInitialResponse, latestProjectResponse)) - ) - ) - ), - [requests$] as const + > = useFlattenedResponse( + requests$, + switchMap(handleDataSearchResponse(latestInitialResponse, latestProjectResponse)) ); const { latestValue } = useObservableState(latestResponse$, undefined); From 7137a5309e63320557289cf81ef54d9aeb6bd86c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20St=C3=BCrmer?= Date: Fri, 15 Jan 2021 19:19:21 +0100 Subject: [PATCH 21/39] Temporarily load "log entries after" in parallel --- .../containers/logs/log_stream/index.ts | 43 ++++++++++++++----- 1 file changed, 33 insertions(+), 10 deletions(-) diff --git a/x-pack/plugins/infra/public/containers/logs/log_stream/index.ts b/x-pack/plugins/infra/public/containers/logs/log_stream/index.ts index f17e6c15604008..d82af339b7c97e 100644 --- a/x-pack/plugins/infra/public/containers/logs/log_stream/index.ts +++ b/x-pack/plugins/infra/public/containers/logs/log_stream/index.ts @@ -4,15 +4,17 @@ * you may not use this file except in compliance with the Elastic License. */ -import { useMemo, useEffect } from 'react'; -import useSetState from 'react-use/lib/useSetState'; +import { useEffect, useMemo } from 'react'; import usePrevious from 'react-use/lib/usePrevious'; +import useSetState from 'react-use/lib/useSetState'; import { esKuery } from '../../../../../../../src/plugins/data/public'; -import { fetchLogEntries } from '../log_entries/api/fetch_log_entries'; -import { useTrackedPromise } from '../../../utils/use_tracked_promise'; import { LogEntry, LogEntryCursor } from '../../../../common/log_entry'; import { useKibanaContextForPlugin } from '../../../hooks/use_kibana'; +import { useSubscription } from '../../../utils/use_observable'; +import { useTrackedPromise } from '../../../utils/use_tracked_promise'; +import { fetchLogEntries } from '../log_entries/api/fetch_log_entries'; import { LogSourceConfigurationProperties } from '../log_source'; +import { useFetchLogEntriesAfter } from './use_fetch_log_entries_after'; interface LogStreamProps { sourceId: string; @@ -56,6 +58,8 @@ const EMPTY_DATA = { bottomCursor: null, }; +const LOG_ENTRIES_CHUNK_SIZE = 200; + export function useLogStream({ sourceId, startTimestamp, @@ -84,11 +88,13 @@ export function useLogStream({ }, [prevEndTimestamp, endTimestamp, setState]); const parsedQuery = useMemo(() => { - return query - ? JSON.stringify(esKuery.toElasticsearchQuery(esKuery.fromKueryExpression(query))) - : null; + return query ? esKuery.toElasticsearchQuery(esKuery.fromKueryExpression(query)) : undefined; }, [query]); + const serializedQuery = useMemo(() => { + return parsedQuery ? JSON.stringify(parsedQuery) : undefined; + }, [parsedQuery]); + // Callbacks const [entriesPromise, fetchEntries] = useTrackedPromise( { @@ -102,7 +108,7 @@ export function useLogStream({ sourceId, startTimestamp, endTimestamp, - query: parsedQuery, + query: serializedQuery, columns, ...fetchPosition, }, @@ -139,7 +145,7 @@ export function useLogStream({ sourceId, startTimestamp, endTimestamp, - query: parsedQuery, + query: serializedQuery, before: state.topCursor, }, services.http.fetch @@ -173,12 +179,13 @@ export function useLogStream({ return Promise.resolve({ data: EMPTY_DATA }); } + fetchLogEntriesAfter(state.bottomCursor, LOG_ENTRIES_CHUNK_SIZE); return fetchLogEntries( { sourceId, startTimestamp, endTimestamp, - query: parsedQuery, + query: serializedQuery, after: state.bottomCursor, }, services.http.fetch @@ -198,6 +205,22 @@ export function useLogStream({ [sourceId, startTimestamp, endTimestamp, query, state.bottomCursor] ); + const { fetchLogEntriesAfter, logEntriesAfterSearchResponse$ } = useFetchLogEntriesAfter({ + sourceId, + startTimestamp, + endTimestamp, + query: parsedQuery, + }); + + useSubscription(logEntriesAfterSearchResponse$, { + next: ({ response }) => { + console.log('next after', response); + }, + error: (err) => { + console.error(err); + }, + }); + const loadingState = useMemo( () => convertPromiseStateToLoadingState(entriesPromise.state), [entriesPromise.state] From 070f450c83444585118b90b278e6bdfe3591a954 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20St=C3=BCrmer?= Date: Mon, 18 Jan 2021 18:15:04 +0100 Subject: [PATCH 22/39] Factor out hook to track data search response state --- .../infra/public/utils/data_search/index.ts | 1 + .../use_data_search_response_state.ts | 35 +++++++++++++++++++ ...use_latest_partial_data_search_response.ts | 30 +++++++++------- 3 files changed, 53 insertions(+), 13 deletions(-) create mode 100644 x-pack/plugins/infra/public/utils/data_search/use_data_search_response_state.ts diff --git a/x-pack/plugins/infra/public/utils/data_search/index.ts b/x-pack/plugins/infra/public/utils/data_search/index.ts index 5ff1ac9fc1da5c..437349375df5eb 100644 --- a/x-pack/plugins/infra/public/utils/data_search/index.ts +++ b/x-pack/plugins/infra/public/utils/data_search/index.ts @@ -7,5 +7,6 @@ export * from './handle_data_search_response'; export * from './types'; export * from './use_data_search_request'; +export * from './use_data_search_response_state'; export * from './use_flattened_response'; export * from './use_latest_partial_data_search_response'; diff --git a/x-pack/plugins/infra/public/utils/data_search/use_data_search_response_state.ts b/x-pack/plugins/infra/public/utils/data_search/use_data_search_response_state.ts new file mode 100644 index 00000000000000..44ece154595304 --- /dev/null +++ b/x-pack/plugins/infra/public/utils/data_search/use_data_search_response_state.ts @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { useCallback } from 'react'; +import { Observable } from 'rxjs'; +import { IKibanaSearchRequest } from '../../../../../../src/plugins/data/public'; +import { useObservableState } from '../use_observable'; +import { DataSearchResponseDescriptor } from './types'; + +export const useDataSearchResponseState = < + Request extends IKibanaSearchRequest, + Response, + InitialResponse +>( + response$: Observable> +) => { + const { latestValue } = useObservableState(response$, undefined); + + const cancelRequest = useCallback(() => { + latestValue?.abortController.abort(); + }, [latestValue]); + + return { + cancelRequest, + isRequestRunning: latestValue?.response.isRunning ?? false, + isResponsePartial: latestValue?.response.isPartial ?? false, + latestResponseData: latestValue?.response.data, + latestResponseErrors: latestValue?.response.errors, + loaded: latestValue?.response.loaded, + total: latestValue?.response.total, + }; +}; diff --git a/x-pack/plugins/infra/public/utils/data_search/use_latest_partial_data_search_response.ts b/x-pack/plugins/infra/public/utils/data_search/use_latest_partial_data_search_response.ts index 459510f7600525..ef3199d982c172 100644 --- a/x-pack/plugins/infra/public/utils/data_search/use_latest_partial_data_search_response.ts +++ b/x-pack/plugins/infra/public/utils/data_search/use_latest_partial_data_search_response.ts @@ -4,13 +4,13 @@ * you may not use this file except in compliance with the Elastic License. */ -import { useCallback } from 'react'; import { Observable } from 'rxjs'; import { switchMap } from 'rxjs/operators'; import { IKibanaSearchRequest } from '../../../../../../src/plugins/data/public'; -import { useLatest, useObservableState } from '../use_observable'; +import { useLatest } from '../use_observable'; import { handleDataSearchResponse, ResponseProjection } from './handle_data_search_response'; import { DataSearchRequestDescriptor, DataSearchResponseDescriptor } from './types'; +import { useDataSearchResponseState } from './use_data_search_response_state'; import { useFlattenedResponse } from './use_flattened_response'; export const useLatestPartialDataSearchResponse = < @@ -33,19 +33,23 @@ export const useLatestPartialDataSearchResponse = < switchMap(handleDataSearchResponse(latestInitialResponse, latestProjectResponse)) ); - const { latestValue } = useObservableState(latestResponse$, undefined); - - const cancelRequest = useCallback(() => { - latestValue?.abortController.abort(); - }, [latestValue]); + const { + cancelRequest, + isRequestRunning, + isResponsePartial, + latestResponseData, + latestResponseErrors, + loaded, + total, + } = useDataSearchResponseState(latestResponse$); return { cancelRequest, - isRequestRunning: latestValue?.response.isRunning ?? false, - isResponsePartial: latestValue?.response.isPartial ?? false, - latestResponseData: latestValue?.response.data, - latestResponseErrors: latestValue?.response.errors, - loaded: latestValue?.response.loaded, - total: latestValue?.response.total, + isRequestRunning, + isResponsePartial, + latestResponseData, + latestResponseErrors, + loaded, + total, }; }; From 0d2bc5ce93661457fe10b01eeafc2be57e2c2699 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20St=C3=BCrmer?= Date: Mon, 18 Jan 2021 18:16:20 +0100 Subject: [PATCH 23/39] Replace the tracked promise with the new async search --- .../containers/logs/log_stream/index.ts | 94 +++++++++---------- .../log_stream/use_fetch_log_entries_after.ts | 32 +++++-- 2 files changed, 68 insertions(+), 58 deletions(-) diff --git a/x-pack/plugins/infra/public/containers/logs/log_stream/index.ts b/x-pack/plugins/infra/public/containers/logs/log_stream/index.ts index d82af339b7c97e..6ccdc0c4e54b30 100644 --- a/x-pack/plugins/infra/public/containers/logs/log_stream/index.ts +++ b/x-pack/plugins/infra/public/containers/logs/log_stream/index.ts @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import { useEffect, useMemo } from 'react'; +import { useCallback, useEffect, useMemo } from 'react'; import usePrevious from 'react-use/lib/usePrevious'; import useSetState from 'react-use/lib/useSetState'; import { esKuery } from '../../../../../../../src/plugins/data/public'; @@ -165,47 +165,12 @@ export function useLogStream({ [sourceId, startTimestamp, endTimestamp, query, state.topCursor] ); - const [nextEntriesPromise, fetchNextEntries] = useTrackedPromise( - { - cancelPreviousOn: 'creation', - createPromise: () => { - if (state.bottomCursor === null) { - throw new Error( - 'useLogState: Cannot fetch next entries. No cursor is set.\nEnsure you have called `fetchEntries` at least once.' - ); - } - - if (!state.hasMoreAfter) { - return Promise.resolve({ data: EMPTY_DATA }); - } - - fetchLogEntriesAfter(state.bottomCursor, LOG_ENTRIES_CHUNK_SIZE); - return fetchLogEntries( - { - sourceId, - startTimestamp, - endTimestamp, - query: serializedQuery, - after: state.bottomCursor, - }, - services.http.fetch - ); - }, - onResolve: ({ data }) => { - if (!data.entries.length) { - return; - } - setState((prevState) => ({ - entries: [...prevState.entries, ...data.entries], - hasMoreAfter: data.hasMoreAfter ?? prevState.hasMoreAfter, - bottomCursor: data.bottomCursor ?? prevState.bottomCursor, - })); - }, - }, - [sourceId, startTimestamp, endTimestamp, query, state.bottomCursor] - ); - - const { fetchLogEntriesAfter, logEntriesAfterSearchResponse$ } = useFetchLogEntriesAfter({ + const { + fetchLogEntriesAfter, + isRequestRunning: isLogEntriesAfterRequestRunning, + isResponsePartial: isLogEntriesAfterResponsePartial, + logEntriesAfterSearchResponse$, + } = useFetchLogEntriesAfter({ sourceId, startTimestamp, endTimestamp, @@ -213,36 +178,65 @@ export function useLogStream({ }); useSubscription(logEntriesAfterSearchResponse$, { - next: ({ response }) => { - console.log('next after', response); + next: ({ response: { data, isPartial } }) => { + if (data != null && !isPartial) { + setState((prevState) => ({ + ...prevState, + entries: [...prevState.entries, ...data.entries], + hasMoreAfter: data.hasMoreAfter ?? prevState.hasMoreAfter, + bottomCursor: data.bottomCursor ?? prevState.bottomCursor, + })); + } }, error: (err) => { console.error(err); }, }); + const fetchNextEntries = useCallback(() => { + if (state.bottomCursor === null) { + throw new Error( + 'useLogState: Cannot fetch next entries. No cursor is set.\nEnsure you have called `fetchEntries` at least once.' + ); + } + + if (!state.hasMoreAfter) { + return; + } + + fetchLogEntriesAfter(state.bottomCursor, LOG_ENTRIES_CHUNK_SIZE); + }, [fetchLogEntriesAfter, state.bottomCursor, state.hasMoreAfter]); + const loadingState = useMemo( () => convertPromiseStateToLoadingState(entriesPromise.state), [entriesPromise.state] ); const pageLoadingState = useMemo(() => { - const states = [previousEntriesPromise.state, nextEntriesPromise.state]; - - if (states.includes('pending')) { + if (previousEntriesPromise.state === 'pending' || isLogEntriesAfterRequestRunning) { return 'loading'; } - if (states.includes('rejected')) { + if ( + previousEntriesPromise.state === 'rejected' || + (!isLogEntriesAfterRequestRunning && isLogEntriesAfterResponsePartial) + ) { return 'error'; } - if (states.includes('resolved')) { + if ( + previousEntriesPromise.state === 'resolved' || + (!isLogEntriesAfterRequestRunning && !isLogEntriesAfterResponsePartial) + ) { return 'success'; } return 'uninitialized'; - }, [previousEntriesPromise.state, nextEntriesPromise.state]); + }, [ + isLogEntriesAfterRequestRunning, + isLogEntriesAfterResponsePartial, + previousEntriesPromise.state, + ]); return { ...state, diff --git a/x-pack/plugins/infra/public/containers/logs/log_stream/use_fetch_log_entries_after.ts b/x-pack/plugins/infra/public/containers/logs/log_stream/use_fetch_log_entries_after.ts index 5a0aea302ef636..8133b8a68113fd 100644 --- a/x-pack/plugins/infra/public/containers/logs/log_stream/use_fetch_log_entries_after.ts +++ b/x-pack/plugins/infra/public/containers/logs/log_stream/use_fetch_log_entries_after.ts @@ -5,7 +5,7 @@ */ import { useCallback } from 'react'; -import { exhaustMap, switchMap } from 'rxjs/operators'; +import { exhaustMap } from 'rxjs/operators'; import { LogSourceColumnConfiguration } from '../../../../common/http_api/log_sources'; import { LogEntryCursor } from '../../../../common/log_entry'; import { decodeOrThrow } from '../../../../common/runtime_types'; @@ -18,9 +18,9 @@ import { JsonObject } from '../../../../common/typed_json'; import { handleDataSearchResponse, useDataSearch, + useDataSearchResponseState, useFlattenedResponse, } from '../../../utils/data_search'; -import { useObservable } from '../../../utils/use_observable'; export const useFetchLogEntriesAfter = ({ columnOverrides, @@ -66,16 +66,32 @@ export const useFetchLogEntriesAfter = ({ const logEntriesAfterSearchResponse$ = useFlattenedResponse( logEntriesAfterSearchRequests$, - exhaustMap( - handleDataSearchResponse( - { current: null }, - { current: decodeOrThrow(logEntriesSearchResponsePayloadRT) } - ) - ) + exhaustMap(handleDataSearchResponse(initialResponseRef, initialProjectResponseRef)) ); + const { + cancelRequest, + isRequestRunning, + isResponsePartial, + loaded, + total, + } = useDataSearchResponseState(logEntriesAfterSearchResponse$); + return { + cancelRequest, fetchLogEntriesAfter, + isRequestRunning, + isResponsePartial, + loaded, logEntriesAfterSearchResponse$, + total, }; }; + +const initialResponseRef = { + current: null, +}; + +const initialProjectResponseRef = { + current: decodeOrThrow(logEntriesSearchResponsePayloadRT), +}; From 0a0ed374d3777578169a997757446c56a5f5e5ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20St=C3=BCrmer?= Date: Tue, 19 Jan 2021 21:35:42 +0100 Subject: [PATCH 24/39] Simplify useDataSearchRequest --- .../data_search/use_data_search_request.ts | 88 +++++++++---------- 1 file changed, 42 insertions(+), 46 deletions(-) diff --git a/x-pack/plugins/infra/public/utils/data_search/use_data_search_request.ts b/x-pack/plugins/infra/public/utils/data_search/use_data_search_request.ts index a23f06adc0353c..48623904899c37 100644 --- a/x-pack/plugins/infra/public/utils/data_search/use_data_search_request.ts +++ b/x-pack/plugins/infra/public/utils/data_search/use_data_search_request.ts @@ -6,7 +6,7 @@ import { useCallback } from 'react'; import { Subject } from 'rxjs'; -import { map, share, switchMap, tap } from 'rxjs/operators'; +import { shareReplay, tap } from 'rxjs/operators'; import { IKibanaSearchRequest, IKibanaSearchResponse, @@ -14,6 +14,7 @@ import { } from '../../../../../../src/plugins/data/public'; import { useKibanaContextForPlugin } from '../../hooks/use_kibana'; import { tapUnsubscribe, useObservable } from '../use_observable'; +import { DataSearchRequestDescriptor } from './types'; export type DataSearchRequestFactory = ( ...args: Args @@ -35,59 +36,54 @@ export const useDataSearch = < getRequest: DataSearchRequestFactory; }) => { const { services } = useKibanaContextForPlugin(); - const request$ = useObservable( - () => new Subject<{ request: Request; options: ISearchOptions }>(), - [] - ); const requests$ = useObservable( - (inputs$) => - inputs$.pipe( - switchMap(([currentRequest$]) => currentRequest$), - map(({ request, options }) => { - const abortController = new AbortController(); - let isAbortable = true; - - return { - abortController, - request, - options, - response$: services.data.search - .search>(request, { - abortSignal: abortController.signal, - ...options, - }) - .pipe( - // avoid aborting failed or completed requests - tap({ - error: () => { - isAbortable = false; - }, - complete: () => { - isAbortable = false; - }, - }), - tapUnsubscribe(() => { - if (isAbortable) { - abortController.abort(); - } - }), - share() - ), - }; - }) - ), - [request$] + () => new Subject>(), + [] ); const search = useCallback( (...args: RequestFactoryArgs) => { - const request = getRequest(...args); + const requestArgs = getRequest(...args); - if (request) { - request$.next(request); + if (requestArgs == null) { + return; } + + const abortController = new AbortController(); + let isAbortable = true; + + const newRequestDescriptor = { + ...requestArgs, + abortController, + response$: services.data.search + .search>(requestArgs.request, { + abortSignal: abortController.signal, + ...requestArgs.options, + }) + .pipe( + // avoid aborting failed or completed requests + tap({ + error: () => { + isAbortable = false; + }, + complete: () => { + isAbortable = false; + }, + }), + tapUnsubscribe(() => { + if (isAbortable) { + abortController.abort(); + } + }), + shareReplay(1) + ), + }; + + requests$.next(newRequestDescriptor); + + return newRequestDescriptor; }, - [getRequest, request$] + [getRequest, services.data.search, requests$] ); return { From 379bb6974983cfc85e954d859e2b9598df97e14e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20St=C3=BCrmer?= Date: Wed, 20 Jan 2021 11:18:53 +0100 Subject: [PATCH 25/39] Make data search transformations more composable --- .../infra/public/containers/logs/log_entry.ts | 18 ++- .../containers/logs/log_stream/index.ts | 81 +++++++------ .../log_stream/use_fetch_log_entries_after.ts | 78 +++++++------ .../use_fetch_log_entries_around.ts | 95 +++++++++++----- .../use_fetch_log_entries_before.ts | 101 +++++++++++++++++ .../flatten_data_search_response.ts | 29 +++++ .../handle_data_search_response.ts | 107 ------------------ .../infra/public/utils/data_search/index.ts | 4 +- .../parse_data_search_responses.ts | 84 ++++++++++++++ .../infra/public/utils/data_search/types.ts | 19 +++- .../data_search/use_data_search_request.ts | 4 +- .../use_data_search_response_state.ts | 4 +- .../data_search/use_flattened_response.ts | 31 ----- ...test_partial_data_search_response.test.tsx | 77 +++++++------ ...use_latest_partial_data_search_response.ts | 30 ++--- .../infra/public/utils/use_observable.ts | 20 +++- 16 files changed, 463 insertions(+), 319 deletions(-) create mode 100644 x-pack/plugins/infra/public/containers/logs/log_stream/use_fetch_log_entries_before.ts create mode 100644 x-pack/plugins/infra/public/utils/data_search/flatten_data_search_response.ts delete mode 100644 x-pack/plugins/infra/public/utils/data_search/handle_data_search_response.ts create mode 100644 x-pack/plugins/infra/public/utils/data_search/parse_data_search_responses.ts delete mode 100644 x-pack/plugins/infra/public/utils/data_search/use_flattened_response.ts diff --git a/x-pack/plugins/infra/public/containers/logs/log_entry.ts b/x-pack/plugins/infra/public/containers/logs/log_entry.ts index af8618b8be5657..f5303606dd0382 100644 --- a/x-pack/plugins/infra/public/containers/logs/log_entry.ts +++ b/x-pack/plugins/infra/public/containers/logs/log_entry.ts @@ -5,6 +5,7 @@ */ import { useCallback } from 'react'; +import { map } from 'rxjs/operators'; import { decodeOrThrow } from '../../../common/runtime_types'; import { logEntrySearchRequestParamsRT, @@ -12,6 +13,8 @@ import { LOG_ENTRY_SEARCH_STRATEGY, } from '../../../common/search_strategies/log_entries/log_entry'; import { useDataSearch, useLatestPartialDataSearchResponse } from '../../utils/data_search'; +import { parseDataSearchResponses } from '../../utils/data_search/parse_data_search_responses'; +import { useOperator } from '../../utils/use_observable'; export const useLogEntry = ({ sourceId, @@ -33,6 +36,11 @@ export const useLogEntry = ({ }, [sourceId, logEntryId]), }); + const parsedLogEntrySearchRequests$ = useOperator( + logEntrySearchRequests$, + parseLogEntrySearchResponses + ); + const { cancelRequest, isRequestRunning, @@ -41,11 +49,7 @@ export const useLogEntry = ({ latestResponseErrors, loaded, total, - } = useLatestPartialDataSearchResponse( - logEntrySearchRequests$, - null, - decodeLogEntrySearchResponse - ); + } = useLatestPartialDataSearchResponse(parsedLogEntrySearchRequests$); return { cancelRequest, @@ -59,4 +63,6 @@ export const useLogEntry = ({ }; }; -const decodeLogEntrySearchResponse = decodeOrThrow(logEntrySearchResponsePayloadRT); +const parseLogEntrySearchResponses = map( + parseDataSearchResponses(null, decodeOrThrow(logEntrySearchResponsePayloadRT)) +); diff --git a/x-pack/plugins/infra/public/containers/logs/log_stream/index.ts b/x-pack/plugins/infra/public/containers/logs/log_stream/index.ts index 6ccdc0c4e54b30..3c321ebcb6690f 100644 --- a/x-pack/plugins/infra/public/containers/logs/log_stream/index.ts +++ b/x-pack/plugins/infra/public/containers/logs/log_stream/index.ts @@ -15,6 +15,7 @@ import { useTrackedPromise } from '../../../utils/use_tracked_promise'; import { fetchLogEntries } from '../log_entries/api/fetch_log_entries'; import { LogSourceConfigurationProperties } from '../log_source'; import { useFetchLogEntriesAfter } from './use_fetch_log_entries_after'; +import { useFetchLogEntriesBefore } from './use_fetch_log_entries_before'; interface LogStreamProps { sourceId: string; @@ -52,12 +53,6 @@ const INITIAL_STATE: LogStreamState = { hasMoreAfter: true, }; -const EMPTY_DATA = { - entries: [], - topCursor: null, - bottomCursor: null, -}; - const LOG_ENTRIES_CHUNK_SIZE = 200; export function useLogStream({ @@ -126,44 +121,47 @@ export function useLogStream({ [sourceId, startTimestamp, endTimestamp, query] ); - const [previousEntriesPromise, fetchPreviousEntries] = useTrackedPromise( - { - cancelPreviousOn: 'creation', - createPromise: () => { - if (state.topCursor === null) { - throw new Error( - 'useLogState: Cannot fetch previous entries. No cursor is set.\nEnsure you have called `fetchEntries` at least once.' - ); - } - - if (!state.hasMoreBefore) { - return Promise.resolve({ data: EMPTY_DATA }); - } + const { + fetchLogEntriesBefore, + isRequestRunning: isLogEntriesBeforeRequestRunning, + isResponsePartial: isLogEntriesBeforeResponsePartial, + logEntriesBeforeSearchResponse$, + } = useFetchLogEntriesBefore({ + sourceId, + startTimestamp, + endTimestamp, + query: parsedQuery, + }); - return fetchLogEntries( - { - sourceId, - startTimestamp, - endTimestamp, - query: serializedQuery, - before: state.topCursor, - }, - services.http.fetch - ); - }, - onResolve: ({ data }) => { - if (!data.entries.length) { - return; - } + useSubscription(logEntriesBeforeSearchResponse$, { + next: ({ response: { data, isPartial } }) => { + if (data != null && !isPartial) { setState((prevState) => ({ + ...prevState, entries: [...data.entries, ...prevState.entries], hasMoreBefore: data.hasMoreBefore ?? prevState.hasMoreBefore, topCursor: data.topCursor ?? prevState.topCursor, })); - }, + } }, - [sourceId, startTimestamp, endTimestamp, query, state.topCursor] - ); + error: (err) => { + console.error(err); + }, + }); + + const fetchPreviousEntries = useCallback(() => { + if (state.topCursor === null) { + throw new Error( + 'useLogState: Cannot fetch previous entries. No cursor is set.\nEnsure you have called `fetchEntries` at least once.' + ); + } + + if (!state.hasMoreBefore) { + return; + } + + fetchLogEntriesBefore(state.topCursor, LOG_ENTRIES_CHUNK_SIZE); + }, [fetchLogEntriesBefore, state.topCursor, state.hasMoreBefore]); const { fetchLogEntriesAfter, @@ -213,19 +211,19 @@ export function useLogStream({ ); const pageLoadingState = useMemo(() => { - if (previousEntriesPromise.state === 'pending' || isLogEntriesAfterRequestRunning) { + if (isLogEntriesBeforeRequestRunning || isLogEntriesAfterRequestRunning) { return 'loading'; } if ( - previousEntriesPromise.state === 'rejected' || + (!isLogEntriesBeforeRequestRunning && isLogEntriesBeforeResponsePartial) || (!isLogEntriesAfterRequestRunning && isLogEntriesAfterResponsePartial) ) { return 'error'; } if ( - previousEntriesPromise.state === 'resolved' || + (!isLogEntriesBeforeRequestRunning && !isLogEntriesBeforeResponsePartial) || (!isLogEntriesAfterRequestRunning && !isLogEntriesAfterResponsePartial) ) { return 'success'; @@ -235,7 +233,8 @@ export function useLogStream({ }, [ isLogEntriesAfterRequestRunning, isLogEntriesAfterResponsePartial, - previousEntriesPromise.state, + isLogEntriesBeforeRequestRunning, + isLogEntriesBeforeResponsePartial, ]); return { diff --git a/x-pack/plugins/infra/public/containers/logs/log_stream/use_fetch_log_entries_after.ts b/x-pack/plugins/infra/public/containers/logs/log_stream/use_fetch_log_entries_after.ts index 8133b8a68113fd..9d17a0db0e03e2 100644 --- a/x-pack/plugins/infra/public/containers/logs/log_stream/use_fetch_log_entries_after.ts +++ b/x-pack/plugins/infra/public/containers/logs/log_stream/use_fetch_log_entries_after.ts @@ -5,7 +5,7 @@ */ import { useCallback } from 'react'; -import { exhaustMap } from 'rxjs/operators'; +import { exhaustMap, map } from 'rxjs/operators'; import { LogSourceColumnConfiguration } from '../../../../common/http_api/log_sources'; import { LogEntryCursor } from '../../../../common/log_entry'; import { decodeOrThrow } from '../../../../common/runtime_types'; @@ -16,11 +16,12 @@ import { } from '../../../../common/search_strategies/log_entries/log_entries'; import { JsonObject } from '../../../../common/typed_json'; import { - handleDataSearchResponse, + flattenDataSearchResponseDescriptor, + parseDataSearchResponses, useDataSearch, useDataSearchResponseState, - useFlattenedResponse, } from '../../../utils/data_search'; +import { useOperator } from '../../../utils/use_observable'; export const useFetchLogEntriesAfter = ({ columnOverrides, @@ -37,36 +38,41 @@ export const useFetchLogEntriesAfter = ({ sourceId: string; startTimestamp: number; }) => { - const { search: fetchLogEntriesAfter, requests$: logEntriesAfterSearchRequests$ } = useDataSearch( - { - getRequest: useCallback( - (cursor: LogEntryCursor, size: number) => { - return !!sourceId - ? { - request: { - params: logEntriesSearchRequestParamsRT.encode({ - after: cursor, - columns: columnOverrides, - endTimestamp, - highlightPhrase, - query, - size, - sourceId, - startTimestamp, - }), - }, - options: { strategy: LOG_ENTRIES_SEARCH_STRATEGY }, - } - : null; - }, - [columnOverrides, endTimestamp, highlightPhrase, query, sourceId, startTimestamp] - ), - } - ); + const { + search: fetchLogEntriesAfter, + requests$: rawLogEntriesAfterSearchRequests$, + } = useDataSearch({ + getRequest: useCallback( + (cursor: LogEntryCursor, size: number) => { + return !!sourceId + ? { + request: { + params: logEntriesSearchRequestParamsRT.encode({ + after: cursor, + columns: columnOverrides, + endTimestamp, + highlightPhrase, + query, + size, + sourceId, + startTimestamp, + }), + }, + options: { strategy: LOG_ENTRIES_SEARCH_STRATEGY }, + } + : null; + }, + [columnOverrides, endTimestamp, highlightPhrase, query, sourceId, startTimestamp] + ), + }); - const logEntriesAfterSearchResponse$ = useFlattenedResponse( + const logEntriesAfterSearchRequests$ = useOperator( + rawLogEntriesAfterSearchRequests$, + parseLogEntriesSearchResponses + ); + const logEntriesAfterSearchResponse$ = useOperator( logEntriesAfterSearchRequests$, - exhaustMap(handleDataSearchResponse(initialResponseRef, initialProjectResponseRef)) + flattenLogEntriesSearchResponse ); const { @@ -88,10 +94,8 @@ export const useFetchLogEntriesAfter = ({ }; }; -const initialResponseRef = { - current: null, -}; +const parseLogEntriesSearchResponses = map( + parseDataSearchResponses(null, decodeOrThrow(logEntriesSearchResponsePayloadRT)) +); -const initialProjectResponseRef = { - current: decodeOrThrow(logEntriesSearchResponsePayloadRT), -}; +const flattenLogEntriesSearchResponse = exhaustMap(flattenDataSearchResponseDescriptor); diff --git a/x-pack/plugins/infra/public/containers/logs/log_stream/use_fetch_log_entries_around.ts b/x-pack/plugins/infra/public/containers/logs/log_stream/use_fetch_log_entries_around.ts index c8527996d2943c..0534f347d63113 100644 --- a/x-pack/plugins/infra/public/containers/logs/log_stream/use_fetch_log_entries_around.ts +++ b/x-pack/plugins/infra/public/containers/logs/log_stream/use_fetch_log_entries_around.ts @@ -4,45 +4,82 @@ * you may not use this file except in compliance with the Elastic License. */ -import { useCallback } from 'react'; -import { - logEntriesSearchRequestParamsRT, - LOG_ENTRIES_SEARCH_STRATEGY, -} from '../../../../common/search_strategies/log_entries/log_entries'; -import { useDataSearch } from '../../../utils/data_search'; +import { combineLatest } from 'rxjs'; +import { map, switchMap } from 'rxjs/operators'; +import { LogSourceColumnConfiguration } from '../../../../common/http_api/log_sources'; +import { JsonObject } from '../../../../common/typed_json'; +import { useObservable, useSubscription } from '../../../utils/use_observable'; +import { useFetchLogEntriesAfter } from './use_fetch_log_entries_after'; +import { useFetchLogEntriesBefore } from './use_fetch_log_entries_before'; export const useFetchLogEntriesAround = ({ + columnOverrides, endTimestamp, + highlightPhrase, + query, sourceId, startTimestamp, }: { + columnOverrides?: LogSourceColumnConfiguration[]; endTimestamp: number; + highlightPhrase?: string; + query?: JsonObject; sourceId: string; startTimestamp: number; }) => { - const { search: fetchLogEntry, requests$: logEntrySearchRequests$ } = useDataSearch({ - getRequest: useCallback( - (size: number) => { - return !!sourceId - ? { - request: { - params: logEntriesSearchRequestParamsRT.encode({ - endTimestamp, - size, - sourceId, - startTimestamp, - }), - }, - options: { strategy: LOG_ENTRIES_SEARCH_STRATEGY }, - } - : null; - }, - [endTimestamp, sourceId, startTimestamp] - ), + const { + fetchLogEntriesBefore, + isRequestRunning: isLogEntriesBeforeRequestRunning, + isResponsePartial: isLogEntriesBeforeResponsePartial, + logEntriesBeforeSearchResponse$, + } = useFetchLogEntriesBefore({ + columnOverrides, + endTimestamp, + highlightPhrase, + query, + sourceId, + startTimestamp, }); - return { - fetchLogEntry, - logEntrySearchRequests$, - }; + const { + fetchLogEntriesAfter, + isRequestRunning: isLogEntriesAfterRequestRunning, + isResponsePartial: isLogEntriesAfterResponsePartial, + logEntriesAfterSearchResponse$, + } = useFetchLogEntriesAfter({ + columnOverrides, + endTimestamp, + highlightPhrase, + query, + sourceId, + startTimestamp, + }); + + // TODO: make result flattening and transformation more composable + + const combinedLogEntriesSearchResponse$ = useObservable( + (inputs$) => + inputs$.pipe( + switchMap(([beforeResponse$, afterResponse$]) => + combineLatest([beforeResponse$, afterResponse$]) + ), + map(([a, b]) => [a, b] as const) + ), + [logEntriesBeforeSearchResponse$, logEntriesAfterSearchResponse$] as const + ); + + useSubscription(combinedLogEntriesSearchResponse$, { + next: ([a, b]) => { + console.log('combined next', a, b); + }, + }); + + // return { + // cancelRequest, + // fetchLogEntriesAround, + // isRequestRunning, + // isResponsePartial, + // loaded, + // total, + // }; }; diff --git a/x-pack/plugins/infra/public/containers/logs/log_stream/use_fetch_log_entries_before.ts b/x-pack/plugins/infra/public/containers/logs/log_stream/use_fetch_log_entries_before.ts new file mode 100644 index 00000000000000..fe5690f2809f79 --- /dev/null +++ b/x-pack/plugins/infra/public/containers/logs/log_stream/use_fetch_log_entries_before.ts @@ -0,0 +1,101 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { useCallback } from 'react'; +import { exhaustMap, map } from 'rxjs/operators'; +import { LogSourceColumnConfiguration } from '../../../../common/http_api/log_sources'; +import { LogEntryCursor } from '../../../../common/log_entry'; +import { decodeOrThrow } from '../../../../common/runtime_types'; +import { + logEntriesSearchRequestParamsRT, + logEntriesSearchResponsePayloadRT, + LOG_ENTRIES_SEARCH_STRATEGY, +} from '../../../../common/search_strategies/log_entries/log_entries'; +import { JsonObject } from '../../../../common/typed_json'; +import { + flattenDataSearchResponseDescriptor, + parseDataSearchResponses, + useDataSearch, + useDataSearchResponseState, +} from '../../../utils/data_search'; +import { useOperator } from '../../../utils/use_observable'; + +export const useFetchLogEntriesBefore = ({ + columnOverrides, + endTimestamp, + highlightPhrase, + query, + sourceId, + startTimestamp, +}: { + columnOverrides?: LogSourceColumnConfiguration[]; + endTimestamp: number; + highlightPhrase?: string; + query?: JsonObject; + sourceId: string; + startTimestamp: number; +}) => { + const { + search: fetchLogEntriesBefore, + requests$: rawLogEntriesBeforeSearchRequests$, + } = useDataSearch({ + getRequest: useCallback( + (cursor: LogEntryCursor, size: number) => { + return !!sourceId + ? { + request: { + params: logEntriesSearchRequestParamsRT.encode({ + before: cursor, + columns: columnOverrides, + endTimestamp, + highlightPhrase, + query, + size, + sourceId, + startTimestamp, + }), + }, + options: { strategy: LOG_ENTRIES_SEARCH_STRATEGY }, + } + : null; + }, + [columnOverrides, endTimestamp, highlightPhrase, query, sourceId, startTimestamp] + ), + }); + + const logEntriesBeforeSearchRequests$ = useOperator( + rawLogEntriesBeforeSearchRequests$, + parseLogEntriesSearchResponses + ); + const logEntriesBeforeSearchResponse$ = useOperator( + logEntriesBeforeSearchRequests$, + flattenLogEntriesSearchResponse + ); + + const { + cancelRequest, + isRequestRunning, + isResponsePartial, + loaded, + total, + } = useDataSearchResponseState(logEntriesBeforeSearchResponse$); + + return { + cancelRequest, + fetchLogEntriesBefore, + isRequestRunning, + isResponsePartial, + loaded, + logEntriesBeforeSearchResponse$, + total, + }; +}; + +const parseLogEntriesSearchResponses = map( + parseDataSearchResponses(null, decodeOrThrow(logEntriesSearchResponsePayloadRT)) +); + +const flattenLogEntriesSearchResponse = exhaustMap(flattenDataSearchResponseDescriptor); diff --git a/x-pack/plugins/infra/public/utils/data_search/flatten_data_search_response.ts b/x-pack/plugins/infra/public/utils/data_search/flatten_data_search_response.ts new file mode 100644 index 00000000000000..98df6d441bd806 --- /dev/null +++ b/x-pack/plugins/infra/public/utils/data_search/flatten_data_search_response.ts @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { map } from 'rxjs/operators'; +import { IKibanaSearchRequest } from '../../../../../../src/plugins/data/public'; +import { ParsedDataSearchRequestDescriptor } from './types'; + +export const flattenDataSearchResponseDescriptor = < + Request extends IKibanaSearchRequest, + Response +>({ + abortController, + options, + request, + response$, +}: ParsedDataSearchRequestDescriptor) => + response$.pipe( + map((response) => { + return { + abortController, + options, + request, + response, + }; + }) + ); diff --git a/x-pack/plugins/infra/public/utils/data_search/handle_data_search_response.ts b/x-pack/plugins/infra/public/utils/data_search/handle_data_search_response.ts deleted file mode 100644 index 050d4be9b877a0..00000000000000 --- a/x-pack/plugins/infra/public/utils/data_search/handle_data_search_response.ts +++ /dev/null @@ -1,107 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { Observable, of } from 'rxjs'; -import { catchError, map, startWith } from 'rxjs/operators'; -import { IKibanaSearchRequest } from '../../../../../../src/plugins/data/public'; -import { AbortError } from '../../../../../../src/plugins/kibana_utils/public'; -import { SearchStrategyError } from '../../../common/search_strategies/common/errors'; -import { DataSearchRequestDescriptor, DataSearchResponseDescriptor } from './types'; - -export type ResponseProjection = ( - rawResponse: RawResponse -) => { data: Response; errors?: SearchStrategyError[] }; - -/** - * Turns the {@link DataSearchRequestDescriptor} into a {@link - * DataSearchResponseDescriptor} by decoding or validating and unrolling the - * partial and final responses emitted. - * - * Since the parameters are refs they will be used immediately for the next - * response without the need to recreate the pipeline. - * - * - * @param initialResponseRef - A ref object containing the initial value to - * emit when a new request is handled. * - * @param projectResponseRef - A ref object containing the projection function - * to apply to each response payload. It should validate that the response - * payload is of the type {@link RawResponse} and decode it to a {@link - * Response}. - * - * @return A function that decodes and validates each response payload using - * the arguments given above. - */ -export const handleDataSearchResponse = < - Request extends IKibanaSearchRequest, - RawResponse, - Response, - InitialResponse ->( - initialResponseRef: { current: InitialResponse }, - projectResponseRef: { current: ResponseProjection } -) => ({ - abortController, - options, - request, - response$, -}: DataSearchRequestDescriptor): Observable< - DataSearchResponseDescriptor -> => - response$.pipe( - map((response) => { - const { data, errors = [] } = projectResponseRef.current(response.rawResponse); - return { - abortController, - options, - request, - response: { - data, - errors, - isPartial: response.isPartial ?? false, - isRunning: response.isRunning ?? false, - loaded: response.loaded, - total: response.total, - }, - }; - }), - startWith({ - abortController, - options, - request, - response: { - data: initialResponseRef.current, - errors: [], - isPartial: true, - isRunning: true, - loaded: 0, - total: undefined, - }, - }), - catchError((error) => - of({ - abortController, - options, - request, - response: { - data: initialResponseRef.current, - errors: [ - error instanceof AbortError - ? { - type: 'aborted' as const, - } - : { - type: 'generic' as const, - message: `${error.message ?? error}`, - }, - ], - isPartial: true, - isRunning: false, - loaded: 0, - total: undefined, - }, - }) - ) - ); diff --git a/x-pack/plugins/infra/public/utils/data_search/index.ts b/x-pack/plugins/infra/public/utils/data_search/index.ts index 437349375df5eb..ed42c611de4a3e 100644 --- a/x-pack/plugins/infra/public/utils/data_search/index.ts +++ b/x-pack/plugins/infra/public/utils/data_search/index.ts @@ -4,9 +4,9 @@ * you may not use this file except in compliance with the Elastic License. */ -export * from './handle_data_search_response'; +export * from './flatten_data_search_response'; +export * from './parse_data_search_responses'; export * from './types'; export * from './use_data_search_request'; export * from './use_data_search_response_state'; -export * from './use_flattened_response'; export * from './use_latest_partial_data_search_response'; diff --git a/x-pack/plugins/infra/public/utils/data_search/parse_data_search_responses.ts b/x-pack/plugins/infra/public/utils/data_search/parse_data_search_responses.ts new file mode 100644 index 00000000000000..8bbbbe7bed267c --- /dev/null +++ b/x-pack/plugins/infra/public/utils/data_search/parse_data_search_responses.ts @@ -0,0 +1,84 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { of } from 'rxjs'; +import { catchError, map, startWith } from 'rxjs/operators'; +import { IKibanaSearchRequest } from '../../../../../../src/plugins/data/public'; +import { AbortError } from '../../../../../../src/plugins/kibana_utils/public'; +import { SearchStrategyError } from '../../../common/search_strategies/common/errors'; +import { DataSearchRequestDescriptor, ParsedDataSearchRequestDescriptor } from './types'; + +export type ResponseProjection = ( + rawResponse: RawResponse +) => { data: Response; errors?: SearchStrategyError[] }; + +/** + * Turns the {@link DataSearchRequestDescriptor} into a {@link + * ParsedDataSearchRequestDescriptor} by enhancing the response observable with + * parsing and error handling transformations. + * + * @param initialResponse - The initial value to emit when a new request is + * handled. + * @param projectResponse - The projection function to apply to each response + * payload. It should validate that the response payload is of the type {@link + * RawResponse} and decode it to a {@link Response}. + * + * @return A function that adds parsing and error handling transformations to + * each response payload using the arguments given above. + */ +export const parseDataSearchResponses = < + Request extends IKibanaSearchRequest, + RawResponse, + Response, + InitialResponse +>( + initialResponse: InitialResponse, + projectResponse: ResponseProjection +) => ( + requestDescriptor: DataSearchRequestDescriptor +): ParsedDataSearchRequestDescriptor => ({ + ...requestDescriptor, + response$: requestDescriptor.response$.pipe( + map((response) => { + const { data, errors = [] } = projectResponse(response.rawResponse); + return { + data, + errors, + isPartial: response.isPartial ?? false, + isRunning: response.isRunning ?? false, + loaded: response.loaded, + total: response.total, + }; + }), + startWith({ + data: initialResponse, + errors: [], + isPartial: true, + isRunning: true, + loaded: 0, + total: undefined, + }), + catchError((error) => + of({ + data: initialResponse, + errors: [ + error instanceof AbortError + ? { + type: 'aborted' as const, + } + : { + type: 'generic' as const, + message: `${error.message ?? error}`, + }, + ], + isPartial: true, + isRunning: false, + loaded: 0, + total: undefined, + }) + ) + ), +}); diff --git a/x-pack/plugins/infra/public/utils/data_search/types.ts b/x-pack/plugins/infra/public/utils/data_search/types.ts index ba0a4c639dae4a..4fcb5898ea5bd8 100644 --- a/x-pack/plugins/infra/public/utils/data_search/types.ts +++ b/x-pack/plugins/infra/public/utils/data_search/types.ts @@ -19,7 +19,17 @@ export interface DataSearchRequestDescriptor { +export interface ParsedDataSearchRequestDescriptor< + Request extends IKibanaSearchRequest, + ResponseData +> { + request: Request; + options: ISearchOptions; + response$: Observable>; + abortController: AbortController; +} + +export interface ParsedKibanaSearchResponse { total?: number; loaded?: number; isRunning: boolean; @@ -28,9 +38,12 @@ export interface NormalizedKibanaSearchResponse { errors: SearchStrategyError[]; } -export interface DataSearchResponseDescriptor { +export interface ParsedDataSearchResponseDescriptor< + Request extends IKibanaSearchRequest, + Response +> { request: Request; options: ISearchOptions; - response: NormalizedKibanaSearchResponse; + response: ParsedKibanaSearchResponse; abortController: AbortController; } diff --git a/x-pack/plugins/infra/public/utils/data_search/use_data_search_request.ts b/x-pack/plugins/infra/public/utils/data_search/use_data_search_request.ts index 48623904899c37..8b575fe1f5ceb5 100644 --- a/x-pack/plugins/infra/public/utils/data_search/use_data_search_request.ts +++ b/x-pack/plugins/infra/public/utils/data_search/use_data_search_request.ts @@ -6,7 +6,7 @@ import { useCallback } from 'react'; import { Subject } from 'rxjs'; -import { shareReplay, tap } from 'rxjs/operators'; +import { share, tap } from 'rxjs/operators'; import { IKibanaSearchRequest, IKibanaSearchResponse, @@ -75,7 +75,7 @@ export const useDataSearch = < abortController.abort(); } }), - shareReplay(1) + share() ), }; diff --git a/x-pack/plugins/infra/public/utils/data_search/use_data_search_response_state.ts b/x-pack/plugins/infra/public/utils/data_search/use_data_search_response_state.ts index 44ece154595304..3b37b80f26cdc2 100644 --- a/x-pack/plugins/infra/public/utils/data_search/use_data_search_response_state.ts +++ b/x-pack/plugins/infra/public/utils/data_search/use_data_search_response_state.ts @@ -8,14 +8,14 @@ import { useCallback } from 'react'; import { Observable } from 'rxjs'; import { IKibanaSearchRequest } from '../../../../../../src/plugins/data/public'; import { useObservableState } from '../use_observable'; -import { DataSearchResponseDescriptor } from './types'; +import { ParsedDataSearchResponseDescriptor } from './types'; export const useDataSearchResponseState = < Request extends IKibanaSearchRequest, Response, InitialResponse >( - response$: Observable> + response$: Observable> ) => { const { latestValue } = useObservableState(response$, undefined); diff --git a/x-pack/plugins/infra/public/utils/data_search/use_flattened_response.ts b/x-pack/plugins/infra/public/utils/data_search/use_flattened_response.ts deleted file mode 100644 index a76bf08f6192a0..00000000000000 --- a/x-pack/plugins/infra/public/utils/data_search/use_flattened_response.ts +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { Observable, OperatorFunction } from 'rxjs'; -import { switchMap } from 'rxjs/operators'; -import { IKibanaSearchRequest } from '../../../../../../src/plugins/data/public'; -import { useLatest, useObservable } from '../use_observable'; -import { DataSearchRequestDescriptor, DataSearchResponseDescriptor } from './types'; - -export const useFlattenedResponse = ( - requests$: Observable>, - project: OperatorFunction< - DataSearchRequestDescriptor, - DataSearchResponseDescriptor - > -) => { - const latestProject = useLatest(project); - - const latestResponse$: Observable< - DataSearchResponseDescriptor - > = useObservable( - (inputs$) => - inputs$.pipe(switchMap(([currentRequests$]) => latestProject.current(currentRequests$))), - [requests$] as const - ); - - return latestResponse$; -}; diff --git a/x-pack/plugins/infra/public/utils/data_search/use_latest_partial_data_search_response.test.tsx b/x-pack/plugins/infra/public/utils/data_search/use_latest_partial_data_search_response.test.tsx index 4c336aa1107a22..864d92f43bc17e 100644 --- a/x-pack/plugins/infra/public/utils/data_search/use_latest_partial_data_search_response.test.tsx +++ b/x-pack/plugins/infra/public/utils/data_search/use_latest_partial_data_search_response.test.tsx @@ -5,12 +5,9 @@ */ import { act, renderHook } from '@testing-library/react-hooks'; -import { Observable, of, Subject } from 'rxjs'; -import { - IKibanaSearchRequest, - IKibanaSearchResponse, -} from '../../../../../../src/plugins/data/public'; -import { DataSearchRequestDescriptor } from './types'; +import { BehaviorSubject, Observable, of, Subject } from 'rxjs'; +import { IKibanaSearchRequest } from '../../../../../../src/plugins/data/public'; +import { ParsedDataSearchRequestDescriptor, ParsedKibanaSearchResponse } from './types'; import { useLatestPartialDataSearchResponse } from './use_latest_partial_data_search_response'; describe('useLatestPartialDataSearchResponse hook', () => { @@ -19,25 +16,31 @@ describe('useLatestPartialDataSearchResponse hook', () => { abortController: new AbortController(), options: {}, request: { params: 'firstRequestParam' }, - response$: new Subject>(), + response$: new BehaviorSubject>({ + data: 'initial', + isRunning: true, + isPartial: true, + errors: [], + }), }; const secondRequest = { abortController: new AbortController(), options: {}, request: { params: 'secondRequestParam' }, - response$: new Subject>(), + response$: new BehaviorSubject>({ + data: 'initial', + isRunning: true, + isPartial: true, + errors: [], + }), }; const requests$ = new Subject< - DataSearchRequestDescriptor, string> + ParsedDataSearchRequestDescriptor, string> >(); - const { result } = renderHook(() => - useLatestPartialDataSearchResponse(requests$, 'initial', (response) => ({ - data: `projection of ${response}`, - })) - ); + const { result } = renderHook(() => useLatestPartialDataSearchResponse(requests$)); expect(result).toHaveProperty('current.isRequestRunning', false); expect(result).toHaveProperty('current.latestResponseData', undefined); @@ -52,37 +55,43 @@ describe('useLatestPartialDataSearchResponse hook', () => { // first response of the first request arrives act(() => { - firstRequest.response$.next({ rawResponse: 'request-1-response-1', isRunning: true }); + firstRequest.response$.next({ + data: 'request-1-response-1', + isRunning: true, + isPartial: true, + errors: [], + }); }); expect(result).toHaveProperty('current.isRequestRunning', true); - expect(result).toHaveProperty( - 'current.latestResponseData', - 'projection of request-1-response-1' - ); + expect(result).toHaveProperty('current.latestResponseData', 'request-1-response-1'); // second request is started before the second response of the first request arrives act(() => { requests$.next(secondRequest); - secondRequest.response$.next({ rawResponse: 'request-2-response-1', isRunning: true }); + secondRequest.response$.next({ + data: 'request-2-response-1', + isRunning: true, + isPartial: true, + errors: [], + }); }); expect(result).toHaveProperty('current.isRequestRunning', true); - expect(result).toHaveProperty( - 'current.latestResponseData', - 'projection of request-2-response-1' - ); + expect(result).toHaveProperty('current.latestResponseData', 'request-2-response-1'); // second response of the second request arrives act(() => { - secondRequest.response$.next({ rawResponse: 'request-2-response-2', isRunning: false }); + secondRequest.response$.next({ + data: 'request-2-response-2', + isRunning: false, + isPartial: false, + errors: [], + }); }); expect(result).toHaveProperty('current.isRequestRunning', false); - expect(result).toHaveProperty( - 'current.latestResponseData', - 'projection of request-2-response-2' - ); + expect(result).toHaveProperty('current.latestResponseData', 'request-2-response-2'); }); it("unsubscribes from the latest request's response observable on unmount", () => { @@ -92,20 +101,16 @@ describe('useLatestPartialDataSearchResponse hook', () => { abortController: new AbortController(), options: {}, request: { params: 'firstRequestParam' }, - response$: new Observable>(() => { + response$: new Observable>(() => { return onUnsubscribe; }), }; - const requests$ = of, string>>( + const requests$ = of, string>>( firstRequest ); - const { unmount } = renderHook(() => - useLatestPartialDataSearchResponse(requests$, 'initial', (response) => ({ - data: `projection of ${response}`, - })) - ); + const { unmount } = renderHook(() => useLatestPartialDataSearchResponse(requests$)); expect(onUnsubscribe).not.toHaveBeenCalled(); diff --git a/x-pack/plugins/infra/public/utils/data_search/use_latest_partial_data_search_response.ts b/x-pack/plugins/infra/public/utils/data_search/use_latest_partial_data_search_response.ts index ef3199d982c172..9366df8adbaf71 100644 --- a/x-pack/plugins/infra/public/utils/data_search/use_latest_partial_data_search_response.ts +++ b/x-pack/plugins/infra/public/utils/data_search/use_latest_partial_data_search_response.ts @@ -7,31 +7,17 @@ import { Observable } from 'rxjs'; import { switchMap } from 'rxjs/operators'; import { IKibanaSearchRequest } from '../../../../../../src/plugins/data/public'; -import { useLatest } from '../use_observable'; -import { handleDataSearchResponse, ResponseProjection } from './handle_data_search_response'; -import { DataSearchRequestDescriptor, DataSearchResponseDescriptor } from './types'; +import { useOperator } from '../use_observable'; +import { flattenDataSearchResponseDescriptor } from './flatten_data_search_response'; +import { ParsedDataSearchRequestDescriptor, ParsedDataSearchResponseDescriptor } from './types'; import { useDataSearchResponseState } from './use_data_search_response_state'; -import { useFlattenedResponse } from './use_flattened_response'; -export const useLatestPartialDataSearchResponse = < - Request extends IKibanaSearchRequest, - RawResponse, - Response, - InitialResponse ->( - requests$: Observable>, - initialResponse: InitialResponse, - projectResponse: ResponseProjection +export const useLatestPartialDataSearchResponse = ( + requests$: Observable> ) => { - const latestInitialResponse = useLatest(initialResponse); - const latestProjectResponse = useLatest(projectResponse); - const latestResponse$: Observable< - DataSearchResponseDescriptor - > = useFlattenedResponse( - requests$, - switchMap(handleDataSearchResponse(latestInitialResponse, latestProjectResponse)) - ); + ParsedDataSearchResponseDescriptor + > = useOperator(requests$, flattenLatestDataSearchResponse); const { cancelRequest, @@ -53,3 +39,5 @@ export const useLatestPartialDataSearchResponse = < total, }; }; + +const flattenLatestDataSearchResponse = switchMap(flattenDataSearchResponseDescriptor); diff --git a/x-pack/plugins/infra/public/utils/use_observable.ts b/x-pack/plugins/infra/public/utils/use_observable.ts index 342aa5aa797b13..508684f8d72686 100644 --- a/x-pack/plugins/infra/public/utils/use_observable.ts +++ b/x-pack/plugins/infra/public/utils/use_observable.ts @@ -5,7 +5,8 @@ */ import { useEffect, useRef, useState } from 'react'; -import { BehaviorSubject, Observable, PartialObserver, Subscription } from 'rxjs'; +import { BehaviorSubject, Observable, OperatorFunction, PartialObserver, Subscription } from 'rxjs'; +import { switchMap } from 'rxjs/operators'; export const useLatest = (value: Value) => { const valueRef = useRef(value); @@ -62,7 +63,9 @@ export const useSubscription = ( const fixedUnsubscribe = latestUnsubscribe.current; const subscription = input$.subscribe({ - next: (value) => latestNext.current?.(value), + next: (value) => { + return latestNext.current?.(value); + }, error: (value) => latestError.current?.(value), complete: () => latestComplete.current?.(), }); @@ -78,6 +81,19 @@ export const useSubscription = ( return latestSubscription.current; }; +export const useOperator = ( + input$: Observable, + operator: OperatorFunction +) => { + const latestOperator = useLatest(operator); + + return useObservable( + (inputs$) => + inputs$.pipe(switchMap(([currentInput$]) => latestOperator.current(currentInput$))), + [input$] as const + ); +}; + export const tapUnsubscribe = (onUnsubscribe: () => void) => (source$: Observable) => { return new Observable((subscriber) => { const subscription = source$.subscribe({ From 5fce211a3023803a551f0b56ff9db1652a9ae77a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20St=C3=BCrmer?= Date: Thu, 21 Jan 2021 20:56:00 +0100 Subject: [PATCH 26/39] Remove old loading mechanism from useLogStream --- .../public/components/log_stream/index.tsx | 18 +- .../infra/public/containers/logs/log_entry.ts | 22 +-- .../containers/logs/log_stream/index.ts | 149 ++++++--------- .../log_stream/use_fetch_log_entries_after.ts | 134 ++++++++++---- .../use_fetch_log_entries_around.ts | 175 ++++++++++++++---- .../use_fetch_log_entries_before.ts | 85 +++++++-- .../infra/public/utils/data_search/index.ts | 1 + .../normalize_data_search_responses.ts | 77 ++++++++ .../infra/public/utils/data_search/types.ts | 9 + .../data_search/use_data_search_request.ts | 22 ++- 10 files changed, 481 insertions(+), 211 deletions(-) create mode 100644 x-pack/plugins/infra/public/utils/data_search/normalize_data_search_responses.ts diff --git a/x-pack/plugins/infra/public/components/log_stream/index.tsx b/x-pack/plugins/infra/public/components/log_stream/index.tsx index b485a21221af21..b981b6cce6b873 100644 --- a/x-pack/plugins/infra/public/components/log_stream/index.tsx +++ b/x-pack/plugins/infra/public/components/log_stream/index.tsx @@ -100,14 +100,14 @@ Read more at https://github.com/elastic/kibana/blob/master/src/plugins/kibana_re // Internal state const { - loadingState, - pageLoadingState, entries, - hasMoreBefore, - hasMoreAfter, fetchEntries, - fetchPreviousEntries, fetchNextEntries, + fetchPreviousEntries, + hasMoreAfter, + hasMoreBefore, + isLoadingMore, + isReloading, } = useLogStream({ sourceId, startTimestamp, @@ -117,12 +117,6 @@ Read more at https://github.com/elastic/kibana/blob/master/src/plugins/kibana_re columns: customColumns, }); - // Derived state - const isReloading = - isLoadingSourceConfiguration || loadingState === 'uninitialized' || loadingState === 'loading'; - - const isLoadingMore = pageLoadingState === 'loading'; - const columnConfigurations = useMemo(() => { return sourceConfiguration ? customColumns ?? sourceConfiguration.configuration.logColumns : []; }, [sourceConfiguration, customColumns]); @@ -176,7 +170,7 @@ Read more at https://github.com/elastic/kibana/blob/master/src/plugins/kibana_re items={streamItems} scale="medium" wrap={true} - isReloading={isReloading} + isReloading={isLoadingSourceConfiguration || isReloading} isLoadingMore={isLoadingMore} hasMoreBeforeStart={hasMoreBefore} hasMoreAfterEnd={hasMoreAfter} diff --git a/x-pack/plugins/infra/public/containers/logs/log_entry.ts b/x-pack/plugins/infra/public/containers/logs/log_entry.ts index f5303606dd0382..60000e0b8baba1 100644 --- a/x-pack/plugins/infra/public/containers/logs/log_entry.ts +++ b/x-pack/plugins/infra/public/containers/logs/log_entry.ts @@ -5,16 +5,17 @@ */ import { useCallback } from 'react'; -import { map } from 'rxjs/operators'; import { decodeOrThrow } from '../../../common/runtime_types'; import { logEntrySearchRequestParamsRT, logEntrySearchResponsePayloadRT, LOG_ENTRY_SEARCH_STRATEGY, } from '../../../common/search_strategies/log_entries/log_entry'; -import { useDataSearch, useLatestPartialDataSearchResponse } from '../../utils/data_search'; -import { parseDataSearchResponses } from '../../utils/data_search/parse_data_search_responses'; -import { useOperator } from '../../utils/use_observable'; +import { + normalizeDataSearchResponses, + useDataSearch, + useLatestPartialDataSearchResponse, +} from '../../utils/data_search'; export const useLogEntry = ({ sourceId, @@ -34,13 +35,9 @@ export const useLogEntry = ({ } : null; }, [sourceId, logEntryId]), + parseResponses: parseLogEntrySearchResponses, }); - const parsedLogEntrySearchRequests$ = useOperator( - logEntrySearchRequests$, - parseLogEntrySearchResponses - ); - const { cancelRequest, isRequestRunning, @@ -49,7 +46,7 @@ export const useLogEntry = ({ latestResponseErrors, loaded, total, - } = useLatestPartialDataSearchResponse(parsedLogEntrySearchRequests$); + } = useLatestPartialDataSearchResponse(logEntrySearchRequests$); return { cancelRequest, @@ -63,6 +60,7 @@ export const useLogEntry = ({ }; }; -const parseLogEntrySearchResponses = map( - parseDataSearchResponses(null, decodeOrThrow(logEntrySearchResponsePayloadRT)) +const parseLogEntrySearchResponses = normalizeDataSearchResponses( + null, + decodeOrThrow(logEntrySearchResponsePayloadRT) ); diff --git a/x-pack/plugins/infra/public/containers/logs/log_stream/index.ts b/x-pack/plugins/infra/public/containers/logs/log_stream/index.ts index 3c321ebcb6690f..c0c0c8accd6764 100644 --- a/x-pack/plugins/infra/public/containers/logs/log_stream/index.ts +++ b/x-pack/plugins/infra/public/containers/logs/log_stream/index.ts @@ -9,12 +9,10 @@ import usePrevious from 'react-use/lib/usePrevious'; import useSetState from 'react-use/lib/useSetState'; import { esKuery } from '../../../../../../../src/plugins/data/public'; import { LogEntry, LogEntryCursor } from '../../../../common/log_entry'; -import { useKibanaContextForPlugin } from '../../../hooks/use_kibana'; import { useSubscription } from '../../../utils/use_observable'; -import { useTrackedPromise } from '../../../utils/use_tracked_promise'; -import { fetchLogEntries } from '../log_entries/api/fetch_log_entries'; import { LogSourceConfigurationProperties } from '../log_source'; import { useFetchLogEntriesAfter } from './use_fetch_log_entries_after'; +import { useFetchLogEntriesAround } from './use_fetch_log_entries_around'; import { useFetchLogEntriesBefore } from './use_fetch_log_entries_before'; interface LogStreamProps { @@ -34,16 +32,6 @@ interface LogStreamState { hasMoreAfter: boolean; } -type LoadingState = 'uninitialized' | 'loading' | 'success' | 'error'; - -interface LogStreamReturn extends LogStreamState { - fetchEntries: () => void; - fetchPreviousEntries: () => void; - fetchNextEntries: () => void; - loadingState: LoadingState; - pageLoadingState: LoadingState; -} - const INITIAL_STATE: LogStreamState = { entries: [], topCursor: null, @@ -62,8 +50,7 @@ export function useLogStream({ query, center, columns, -}: LogStreamProps): LogStreamReturn { - const { services } = useKibanaContextForPlugin(); +}: LogStreamProps) { const [state, setState] = useSetState(INITIAL_STATE); // Ensure the pagination keeps working when the timerange gets extended @@ -86,51 +73,47 @@ export function useLogStream({ return query ? esKuery.toElasticsearchQuery(esKuery.fromKueryExpression(query)) : undefined; }, [query]); - const serializedQuery = useMemo(() => { - return parsedQuery ? JSON.stringify(parsedQuery) : undefined; - }, [parsedQuery]); - - // Callbacks - const [entriesPromise, fetchEntries] = useTrackedPromise( - { - cancelPreviousOn: 'creation', - createPromise: () => { - setState(INITIAL_STATE); - const fetchPosition = center ? { center } : { before: 'last' }; + const { + fetchLogEntriesAround, + isRequestRunning: isLogEntriesAroundRequestRunning, + logEntriesAroundSearchResponses$, + } = useFetchLogEntriesAround({ + sourceId, + startTimestamp, + endTimestamp, + query: parsedQuery, + columnOverrides: columns, + }); - return fetchLogEntries( - { - sourceId, - startTimestamp, - endTimestamp, - query: serializedQuery, - columns, - ...fetchPosition, - }, - services.http.fetch - ); - }, - onResolve: ({ data }) => { + useSubscription(logEntriesAroundSearchResponses$, { + next: ({ before, after, combined }) => { + // if (data != null && !isPartial) { + if ((before.response.data != null || after?.response.data != null) && !combined.isPartial) { setState((prevState) => ({ - ...data, - hasMoreBefore: data.hasMoreBefore ?? prevState.hasMoreBefore, - hasMoreAfter: data.hasMoreAfter ?? prevState.hasMoreAfter, + ...prevState, + entries: combined.entries, + hasMoreAfter: combined.hasMoreAfter ?? prevState.hasMoreAfter, + hasMoreBefore: combined.hasMoreAfter ?? prevState.hasMoreAfter, + bottomCursor: combined.bottomCursor, + topCursor: combined.topCursor, })); - }, + } }, - [sourceId, startTimestamp, endTimestamp, query] - ); + error: (err) => { + console.error(err); + }, + }); const { fetchLogEntriesBefore, isRequestRunning: isLogEntriesBeforeRequestRunning, - isResponsePartial: isLogEntriesBeforeResponsePartial, logEntriesBeforeSearchResponse$, } = useFetchLogEntriesBefore({ sourceId, startTimestamp, endTimestamp, query: parsedQuery, + columnOverrides: columns, }); useSubscription(logEntriesBeforeSearchResponse$, { @@ -166,13 +149,13 @@ export function useLogStream({ const { fetchLogEntriesAfter, isRequestRunning: isLogEntriesAfterRequestRunning, - isResponsePartial: isLogEntriesAfterResponsePartial, logEntriesAfterSearchResponse$, } = useFetchLogEntriesAfter({ sourceId, startTimestamp, endTimestamp, query: parsedQuery, + columnOverrides: columns, }); useSubscription(logEntriesAfterSearchResponse$, { @@ -205,59 +188,39 @@ export function useLogStream({ fetchLogEntriesAfter(state.bottomCursor, LOG_ENTRIES_CHUNK_SIZE); }, [fetchLogEntriesAfter, state.bottomCursor, state.hasMoreAfter]); - const loadingState = useMemo( - () => convertPromiseStateToLoadingState(entriesPromise.state), - [entriesPromise.state] - ); - - const pageLoadingState = useMemo(() => { - if (isLogEntriesBeforeRequestRunning || isLogEntriesAfterRequestRunning) { - return 'loading'; - } + const fetchEntries = useCallback(() => { + setState(INITIAL_STATE); - if ( - (!isLogEntriesBeforeRequestRunning && isLogEntriesBeforeResponsePartial) || - (!isLogEntriesAfterRequestRunning && isLogEntriesAfterResponsePartial) - ) { - return 'error'; - } - - if ( - (!isLogEntriesBeforeRequestRunning && !isLogEntriesBeforeResponsePartial) || - (!isLogEntriesAfterRequestRunning && !isLogEntriesAfterResponsePartial) - ) { - return 'success'; + if (center) { + fetchLogEntriesAround(center, LOG_ENTRIES_CHUNK_SIZE); + } else { + fetchLogEntriesBefore('last', LOG_ENTRIES_CHUNK_SIZE); } + }, [center, fetchLogEntriesAround, fetchLogEntriesBefore, setState]); + + const isReloading = useMemo( + () => + isLogEntriesAroundRequestRunning || + (state.bottomCursor == null && state.topCursor == null && isLogEntriesBeforeRequestRunning), + [ + isLogEntriesAroundRequestRunning, + isLogEntriesBeforeRequestRunning, + state.bottomCursor, + state.topCursor, + ] + ); - return 'uninitialized'; - }, [ - isLogEntriesAfterRequestRunning, - isLogEntriesAfterResponsePartial, - isLogEntriesBeforeRequestRunning, - isLogEntriesBeforeResponsePartial, - ]); + const isLoadingMore = useMemo( + () => isLogEntriesBeforeRequestRunning || isLogEntriesAfterRequestRunning, + [isLogEntriesAfterRequestRunning, isLogEntriesBeforeRequestRunning] + ); return { ...state, fetchEntries, - fetchPreviousEntries, fetchNextEntries, - loadingState, - pageLoadingState, + fetchPreviousEntries, + isLoadingMore, + isReloading, }; } - -function convertPromiseStateToLoadingState( - state: 'uninitialized' | 'pending' | 'resolved' | 'rejected' -): LoadingState { - switch (state) { - case 'uninitialized': - return 'uninitialized'; - case 'pending': - return 'loading'; - case 'resolved': - return 'success'; - case 'rejected': - return 'error'; - } -} diff --git a/x-pack/plugins/infra/public/containers/logs/log_stream/use_fetch_log_entries_after.ts b/x-pack/plugins/infra/public/containers/logs/log_stream/use_fetch_log_entries_after.ts index 9d17a0db0e03e2..d460f23f8a9fe5 100644 --- a/x-pack/plugins/infra/public/containers/logs/log_stream/use_fetch_log_entries_after.ts +++ b/x-pack/plugins/infra/public/containers/logs/log_stream/use_fetch_log_entries_after.ts @@ -5,25 +5,29 @@ */ import { useCallback } from 'react'; -import { exhaustMap, map } from 'rxjs/operators'; +import { Observable } from 'rxjs'; +import { exhaustMap } from 'rxjs/operators'; +import { IKibanaSearchRequest } from '../../../../../../../src/plugins/data/public'; import { LogSourceColumnConfiguration } from '../../../../common/http_api/log_sources'; -import { LogEntryCursor } from '../../../../common/log_entry'; +import { LogEntryAfterCursor } from '../../../../common/log_entry'; import { decodeOrThrow } from '../../../../common/runtime_types'; import { logEntriesSearchRequestParamsRT, + LogEntriesSearchResponsePayload, logEntriesSearchResponsePayloadRT, LOG_ENTRIES_SEARCH_STRATEGY, } from '../../../../common/search_strategies/log_entries/log_entries'; import { JsonObject } from '../../../../common/typed_json'; import { flattenDataSearchResponseDescriptor, - parseDataSearchResponses, + normalizeDataSearchResponses, + ParsedDataSearchRequestDescriptor, useDataSearch, useDataSearchResponseState, } from '../../../utils/data_search'; import { useOperator } from '../../../utils/use_observable'; -export const useFetchLogEntriesAfter = ({ +export const useLogEntriesAfterRequest = ({ columnOverrides, endTimestamp, highlightPhrase, @@ -38,41 +42,48 @@ export const useFetchLogEntriesAfter = ({ sourceId: string; startTimestamp: number; }) => { - const { - search: fetchLogEntriesAfter, - requests$: rawLogEntriesAfterSearchRequests$, - } = useDataSearch({ - getRequest: useCallback( - (cursor: LogEntryCursor, size: number) => { - return !!sourceId - ? { - request: { - params: logEntriesSearchRequestParamsRT.encode({ - after: cursor, - columns: columnOverrides, - endTimestamp, - highlightPhrase, - query, - size, - sourceId, - startTimestamp, - }), - }, - options: { strategy: LOG_ENTRIES_SEARCH_STRATEGY }, - } - : null; - }, - [columnOverrides, endTimestamp, highlightPhrase, query, sourceId, startTimestamp] - ), - }); - - const logEntriesAfterSearchRequests$ = useOperator( - rawLogEntriesAfterSearchRequests$, - parseLogEntriesSearchResponses + const { search: fetchLogEntriesAfter, requests$: logEntriesAfterSearchRequests$ } = useDataSearch( + { + getRequest: useCallback( + (cursor: LogEntryAfterCursor['after'], size: number) => { + return !!sourceId + ? { + request: { + params: logEntriesSearchRequestParamsRT.encode({ + after: cursor, + columns: columnOverrides, + endTimestamp, + highlightPhrase, + query, + size, + sourceId, + startTimestamp, + }), + }, + options: { strategy: LOG_ENTRIES_SEARCH_STRATEGY }, + } + : null; + }, + [columnOverrides, endTimestamp, highlightPhrase, query, sourceId, startTimestamp] + ), + parseResponses: parseLogEntriesAfterSearchResponses, + } ); + + return { + fetchLogEntriesAfter, + logEntriesAfterSearchRequests$, + }; +}; + +export const useLogEntriesAfterResponse = ( + logEntriesAfterSearchRequests$: Observable< + ParsedDataSearchRequestDescriptor + > +) => { const logEntriesAfterSearchResponse$ = useOperator( logEntriesAfterSearchRequests$, - flattenLogEntriesSearchResponse + flattenLogEntriesAfterSearchResponse ); const { @@ -83,6 +94,50 @@ export const useFetchLogEntriesAfter = ({ total, } = useDataSearchResponseState(logEntriesAfterSearchResponse$); + return { + cancelRequest, + isRequestRunning, + isResponsePartial, + loaded, + logEntriesAfterSearchRequests$, + logEntriesAfterSearchResponse$, + total, + }; +}; + +export const useFetchLogEntriesAfter = ({ + columnOverrides, + endTimestamp, + highlightPhrase, + query, + sourceId, + startTimestamp, +}: { + columnOverrides?: LogSourceColumnConfiguration[]; + endTimestamp: number; + highlightPhrase?: string; + query?: JsonObject; + sourceId: string; + startTimestamp: number; +}) => { + const { fetchLogEntriesAfter, logEntriesAfterSearchRequests$ } = useLogEntriesAfterRequest({ + columnOverrides, + endTimestamp, + highlightPhrase, + query, + sourceId, + startTimestamp, + }); + + const { + cancelRequest, + isRequestRunning, + isResponsePartial, + loaded, + logEntriesAfterSearchResponse$, + total, + } = useLogEntriesAfterResponse(logEntriesAfterSearchRequests$); + return { cancelRequest, fetchLogEntriesAfter, @@ -94,8 +149,9 @@ export const useFetchLogEntriesAfter = ({ }; }; -const parseLogEntriesSearchResponses = map( - parseDataSearchResponses(null, decodeOrThrow(logEntriesSearchResponsePayloadRT)) +export const parseLogEntriesAfterSearchResponses = normalizeDataSearchResponses( + null, + decodeOrThrow(logEntriesSearchResponsePayloadRT) ); -const flattenLogEntriesSearchResponse = exhaustMap(flattenDataSearchResponseDescriptor); +const flattenLogEntriesAfterSearchResponse = exhaustMap(flattenDataSearchResponseDescriptor); diff --git a/x-pack/plugins/infra/public/containers/logs/log_stream/use_fetch_log_entries_around.ts b/x-pack/plugins/infra/public/containers/logs/log_stream/use_fetch_log_entries_around.ts index 0534f347d63113..171f00ee9b2e69 100644 --- a/x-pack/plugins/infra/public/containers/logs/log_stream/use_fetch_log_entries_around.ts +++ b/x-pack/plugins/infra/public/containers/logs/log_stream/use_fetch_log_entries_around.ts @@ -4,13 +4,16 @@ * you may not use this file except in compliance with the Elastic License. */ -import { combineLatest } from 'rxjs'; -import { map, switchMap } from 'rxjs/operators'; +import { useCallback } from 'react'; +import { combineLatest, Observable, Subject } from 'rxjs'; +import { last, map, startWith, switchMap } from 'rxjs/operators'; import { LogSourceColumnConfiguration } from '../../../../common/http_api/log_sources'; +import { LogEntryCursor } from '../../../../common/log_entry'; import { JsonObject } from '../../../../common/typed_json'; -import { useObservable, useSubscription } from '../../../utils/use_observable'; -import { useFetchLogEntriesAfter } from './use_fetch_log_entries_after'; -import { useFetchLogEntriesBefore } from './use_fetch_log_entries_before'; +import { flattenDataSearchResponseDescriptor } from '../../../utils/data_search'; +import { useObservable, useObservableState, useSubscription } from '../../../utils/use_observable'; +import { useLogEntriesAfterRequest } from './use_fetch_log_entries_after'; +import { useLogEntriesBeforeRequest } from './use_fetch_log_entries_before'; export const useFetchLogEntriesAround = ({ columnOverrides, @@ -27,12 +30,7 @@ export const useFetchLogEntriesAround = ({ sourceId: string; startTimestamp: number; }) => { - const { - fetchLogEntriesBefore, - isRequestRunning: isLogEntriesBeforeRequestRunning, - isResponsePartial: isLogEntriesBeforeResponsePartial, - logEntriesBeforeSearchResponse$, - } = useFetchLogEntriesBefore({ + const { fetchLogEntriesBefore } = useLogEntriesBeforeRequest({ columnOverrides, endTimestamp, highlightPhrase, @@ -41,12 +39,7 @@ export const useFetchLogEntriesAround = ({ startTimestamp, }); - const { - fetchLogEntriesAfter, - isRequestRunning: isLogEntriesAfterRequestRunning, - isResponsePartial: isLogEntriesAfterResponsePartial, - logEntriesAfterSearchResponse$, - } = useFetchLogEntriesAfter({ + const { fetchLogEntriesAfter } = useLogEntriesAfterRequest({ columnOverrides, endTimestamp, highlightPhrase, @@ -55,31 +48,143 @@ export const useFetchLogEntriesAround = ({ startTimestamp, }); - // TODO: make result flattening and transformation more composable + type LogEntriesBeforeRequest = NonNullable>; + type LogEntriesAfterRequest = NonNullable>; + + const logEntriesAroundSearchRequests$ = useObservable( + () => new Subject<[LogEntriesBeforeRequest, Observable]>(), + [] + ); + + const fetchLogEntriesAround = useCallback( + (cursor: LogEntryCursor, size: number) => { + const logEntriesBeforeSearchRequest = fetchLogEntriesBefore(cursor, Math.floor(size / 2)); + + if (logEntriesBeforeSearchRequest == null) { + return; + } + + const logEntriesAfterSearchRequest$ = flattenDataSearchResponseDescriptor( + logEntriesBeforeSearchRequest + ).pipe( + last(), // in the future we could start earlier if we receive partial results already + map((lastBeforeSearchResponse) => { + const cursorAfter = lastBeforeSearchResponse.response.data?.bottomCursor ?? { + time: cursor.time - 1, + tiebreaker: 0, + }; + + const logEntriesAfterSearchRequest = fetchLogEntriesAfter( + cursorAfter, + Math.ceil(size / 2) + ); + + if (logEntriesAfterSearchRequest == null) { + throw new Error('Failed to create request: no request args given'); + } + + return logEntriesAfterSearchRequest; + }) + ); - const combinedLogEntriesSearchResponse$ = useObservable( + logEntriesAroundSearchRequests$.next([ + logEntriesBeforeSearchRequest, + logEntriesAfterSearchRequest$, + ]); + }, + [fetchLogEntriesAfter, fetchLogEntriesBefore, logEntriesAroundSearchRequests$] + ); + + const logEntriesAroundSearchResponses$ = useObservable( (inputs$) => inputs$.pipe( - switchMap(([beforeResponse$, afterResponse$]) => - combineLatest([beforeResponse$, afterResponse$]) - ), - map(([a, b]) => [a, b] as const) + switchMap(([currentSearchRequests$]) => + currentSearchRequests$.pipe( + switchMap(([beforeRequest, afterRequest$]) => { + const beforeResponse$ = flattenDataSearchResponseDescriptor(beforeRequest); + const afterResponse$ = afterRequest$.pipe( + switchMap(flattenDataSearchResponseDescriptor), + startWith(undefined) // emit "before" response even if "after" hasn't started yet + ); + return combineLatest([beforeResponse$, afterResponse$]); + }), + map(([beforeResponse, afterResponse]) => { + const loadedBefore = beforeResponse.response.loaded; + const loadedAfter = afterResponse?.response.loaded; + const totalBefore = beforeResponse.response.total; + const totalAfter = afterResponse?.response.total; + + return { + before: beforeResponse, + after: afterResponse, + combined: { + isRunning: + (beforeResponse.response.isRunning || afterResponse?.response.isRunning) ?? + false, + isPartial: + (beforeResponse.response.isPartial || afterResponse?.response.isPartial) ?? + false, + loaded: + loadedBefore != null || loadedAfter != null + ? (loadedBefore ?? 0) + (loadedAfter ?? 0) + : undefined, + total: + totalBefore != null || totalAfter != null + ? (totalBefore ?? 0) + (totalAfter ?? 0) + : undefined, + entries: [ + ...(beforeResponse.response.data?.entries ?? []), + ...(afterResponse?.response.data?.entries ?? []), + ], + errors: [ + ...(beforeResponse.response.errors ?? []), + ...(afterResponse?.response.errors ?? []), + ], + hasMoreBefore: beforeResponse.response.data?.hasMoreBefore, + hasMoreAfter: afterResponse?.response.data?.hasMoreAfter, + topCursor: beforeResponse.response.data?.topCursor, + bottomCursor: afterResponse?.response.data?.bottomCursor, + }, + }; + }) + ) + ) ), - [logEntriesBeforeSearchResponse$, logEntriesAfterSearchResponse$] as const + [logEntriesAroundSearchRequests$] ); - useSubscription(combinedLogEntriesSearchResponse$, { - next: ([a, b]) => { - console.log('combined next', a, b); + const { + latestValue: { + before: latestBeforeResponse, + after: latestAfterResponse, + combined: latestCombinedResponse, + }, + } = useObservableState(logEntriesAroundSearchResponses$, initialCombinedResponse); + + useSubscription(logEntriesAroundSearchResponses$, { + next: ({ before, after, combined }) => { + console.log('combined next', before, after, combined); }, }); - // return { - // cancelRequest, - // fetchLogEntriesAround, - // isRequestRunning, - // isResponsePartial, - // loaded, - // total, - // }; + const cancelRequest = useCallback(() => { + latestBeforeResponse?.abortController.abort(); + latestAfterResponse?.abortController.abort(); + }, [latestBeforeResponse, latestAfterResponse]); + + return { + cancelRequest, + fetchLogEntriesAround, + isRequestRunning: latestCombinedResponse?.isRunning ?? false, + isResponsePartial: latestCombinedResponse?.isPartial ?? false, + loaded: latestCombinedResponse?.loaded, + logEntriesAroundSearchResponses$, + total: latestCombinedResponse?.total, + }; }; + +const initialCombinedResponse = { + before: undefined, + after: undefined, + combined: undefined, +} as const; diff --git a/x-pack/plugins/infra/public/containers/logs/log_stream/use_fetch_log_entries_before.ts b/x-pack/plugins/infra/public/containers/logs/log_stream/use_fetch_log_entries_before.ts index fe5690f2809f79..10e6bdcbf188d3 100644 --- a/x-pack/plugins/infra/public/containers/logs/log_stream/use_fetch_log_entries_before.ts +++ b/x-pack/plugins/infra/public/containers/logs/log_stream/use_fetch_log_entries_before.ts @@ -5,25 +5,29 @@ */ import { useCallback } from 'react'; -import { exhaustMap, map } from 'rxjs/operators'; +import { Observable } from 'rxjs'; +import { exhaustMap } from 'rxjs/operators'; +import { IKibanaSearchRequest } from '../../../../../../../src/plugins/data/public'; import { LogSourceColumnConfiguration } from '../../../../common/http_api/log_sources'; -import { LogEntryCursor } from '../../../../common/log_entry'; +import { LogEntryBeforeCursor } from '../../../../common/log_entry'; import { decodeOrThrow } from '../../../../common/runtime_types'; import { logEntriesSearchRequestParamsRT, + LogEntriesSearchResponsePayload, logEntriesSearchResponsePayloadRT, LOG_ENTRIES_SEARCH_STRATEGY, } from '../../../../common/search_strategies/log_entries/log_entries'; import { JsonObject } from '../../../../common/typed_json'; import { flattenDataSearchResponseDescriptor, - parseDataSearchResponses, + normalizeDataSearchResponses, + ParsedDataSearchRequestDescriptor, useDataSearch, useDataSearchResponseState, } from '../../../utils/data_search'; import { useOperator } from '../../../utils/use_observable'; -export const useFetchLogEntriesBefore = ({ +export const useLogEntriesBeforeRequest = ({ columnOverrides, endTimestamp, highlightPhrase, @@ -40,10 +44,10 @@ export const useFetchLogEntriesBefore = ({ }) => { const { search: fetchLogEntriesBefore, - requests$: rawLogEntriesBeforeSearchRequests$, + requests$: logEntriesBeforeSearchRequests$, } = useDataSearch({ getRequest: useCallback( - (cursor: LogEntryCursor, size: number) => { + (cursor: LogEntryBeforeCursor['before'], size: number) => { return !!sourceId ? { request: { @@ -64,15 +68,23 @@ export const useFetchLogEntriesBefore = ({ }, [columnOverrides, endTimestamp, highlightPhrase, query, sourceId, startTimestamp] ), + parseResponses: parseLogEntriesBeforeSearchResponses, }); - const logEntriesBeforeSearchRequests$ = useOperator( - rawLogEntriesBeforeSearchRequests$, - parseLogEntriesSearchResponses - ); + return { + fetchLogEntriesBefore, + logEntriesBeforeSearchRequests$, + }; +}; + +export const useLogEntriesBeforeResponse = ( + logEntriesBeforeSearchRequests$: Observable< + ParsedDataSearchRequestDescriptor + > +) => { const logEntriesBeforeSearchResponse$ = useOperator( logEntriesBeforeSearchRequests$, - flattenLogEntriesSearchResponse + flattenLogEntriesBeforeSearchResponse ); const { @@ -83,6 +95,50 @@ export const useFetchLogEntriesBefore = ({ total, } = useDataSearchResponseState(logEntriesBeforeSearchResponse$); + return { + cancelRequest, + isRequestRunning, + isResponsePartial, + loaded, + logEntriesBeforeSearchRequests$, + logEntriesBeforeSearchResponse$, + total, + }; +}; + +export const useFetchLogEntriesBefore = ({ + columnOverrides, + endTimestamp, + highlightPhrase, + query, + sourceId, + startTimestamp, +}: { + columnOverrides?: LogSourceColumnConfiguration[]; + endTimestamp: number; + highlightPhrase?: string; + query?: JsonObject; + sourceId: string; + startTimestamp: number; +}) => { + const { fetchLogEntriesBefore, logEntriesBeforeSearchRequests$ } = useLogEntriesBeforeRequest({ + columnOverrides, + endTimestamp, + highlightPhrase, + query, + sourceId, + startTimestamp, + }); + + const { + cancelRequest, + isRequestRunning, + isResponsePartial, + loaded, + logEntriesBeforeSearchResponse$, + total, + } = useLogEntriesBeforeResponse(logEntriesBeforeSearchRequests$); + return { cancelRequest, fetchLogEntriesBefore, @@ -94,8 +150,9 @@ export const useFetchLogEntriesBefore = ({ }; }; -const parseLogEntriesSearchResponses = map( - parseDataSearchResponses(null, decodeOrThrow(logEntriesSearchResponsePayloadRT)) +export const parseLogEntriesBeforeSearchResponses = normalizeDataSearchResponses( + null, + decodeOrThrow(logEntriesSearchResponsePayloadRT) ); -const flattenLogEntriesSearchResponse = exhaustMap(flattenDataSearchResponseDescriptor); +const flattenLogEntriesBeforeSearchResponse = exhaustMap(flattenDataSearchResponseDescriptor); diff --git a/x-pack/plugins/infra/public/utils/data_search/index.ts b/x-pack/plugins/infra/public/utils/data_search/index.ts index ed42c611de4a3e..baf12907b7a158 100644 --- a/x-pack/plugins/infra/public/utils/data_search/index.ts +++ b/x-pack/plugins/infra/public/utils/data_search/index.ts @@ -5,6 +5,7 @@ */ export * from './flatten_data_search_response'; +export * from './normalize_data_search_responses'; export * from './parse_data_search_responses'; export * from './types'; export * from './use_data_search_request'; diff --git a/x-pack/plugins/infra/public/utils/data_search/normalize_data_search_responses.ts b/x-pack/plugins/infra/public/utils/data_search/normalize_data_search_responses.ts new file mode 100644 index 00000000000000..5046cc128a835a --- /dev/null +++ b/x-pack/plugins/infra/public/utils/data_search/normalize_data_search_responses.ts @@ -0,0 +1,77 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { Observable, of } from 'rxjs'; +import { catchError, map, startWith } from 'rxjs/operators'; +import { IKibanaSearchResponse } from '../../../../../../src/plugins/data/public'; +import { AbortError } from '../../../../../../src/plugins/kibana_utils/public'; +import { SearchStrategyError } from '../../../common/search_strategies/common/errors'; +import { ParsedKibanaSearchResponse } from './types'; + +export type RawResponseParser = ( + rawResponse: RawResponse +) => { data: Response; errors?: SearchStrategyError[] }; + +/** + * An operator factory that normalizes each {@link IKibanaSearchResponse} by + * parsing it into a {@link ParsedKibanaSearchResponse} and adding initial + * responses and error handling. + * + * @param initialResponse - The initial value to emit when a new request is + * handled. + * @param projectResponse - The projection function to apply to each response + * payload. It should validate that the response payload is of the type {@link + * RawResponse} and decode it to a {@link Response}. + * + * @return An operator that adds parsing and error handling transformations to + * each response payload using the arguments given above. + */ +export const normalizeDataSearchResponses = ( + initialResponse: InitialResponse, + parseRawResponse: RawResponseParser +) => ( + response$: Observable> +): Observable> => + response$.pipe( + map((response) => { + const { data, errors = [] } = parseRawResponse(response.rawResponse); + return { + data, + errors, + isPartial: response.isPartial ?? false, + isRunning: response.isRunning ?? false, + loaded: response.loaded, + total: response.total, + }; + }), + startWith({ + data: initialResponse, + errors: [], + isPartial: true, + isRunning: true, + loaded: 0, + total: undefined, + }), + catchError((error) => + of({ + data: initialResponse, + errors: [ + error instanceof AbortError + ? { + type: 'aborted' as const, + } + : { + type: 'generic' as const, + message: `${error.message ?? error}`, + }, + ], + isPartial: true, + isRunning: false, + loaded: 0, + total: undefined, + }) + ) + ); diff --git a/x-pack/plugins/infra/public/utils/data_search/types.ts b/x-pack/plugins/infra/public/utils/data_search/types.ts index 4fcb5898ea5bd8..72172c2a036bb9 100644 --- a/x-pack/plugins/infra/public/utils/data_search/types.ts +++ b/x-pack/plugins/infra/public/utils/data_search/types.ts @@ -47,3 +47,12 @@ export interface ParsedDataSearchResponseDescriptor< response: ParsedKibanaSearchResponse; abortController: AbortController; } + +export type ParsedDataSearchResponseDescriptorOfRequest< + ParsedRequestDescriptor extends ParsedDataSearchRequestDescriptor +> = ParsedRequestDescriptor extends ParsedDataSearchRequestDescriptor< + infer Request, + infer ResponseData +> + ? ParsedDataSearchResponseDescriptor + : never; diff --git a/x-pack/plugins/infra/public/utils/data_search/use_data_search_request.ts b/x-pack/plugins/infra/public/utils/data_search/use_data_search_request.ts index 8b575fe1f5ceb5..0f1686a93be829 100644 --- a/x-pack/plugins/infra/public/utils/data_search/use_data_search_request.ts +++ b/x-pack/plugins/infra/public/utils/data_search/use_data_search_request.ts @@ -5,7 +5,7 @@ */ import { useCallback } from 'react'; -import { Subject } from 'rxjs'; +import { OperatorFunction, Subject } from 'rxjs'; import { share, tap } from 'rxjs/operators'; import { IKibanaSearchRequest, @@ -14,7 +14,7 @@ import { } from '../../../../../../src/plugins/data/public'; import { useKibanaContextForPlugin } from '../../hooks/use_kibana'; import { tapUnsubscribe, useObservable } from '../use_observable'; -import { DataSearchRequestDescriptor } from './types'; +import { ParsedDataSearchRequestDescriptor, ParsedKibanaSearchResponse } from './types'; export type DataSearchRequestFactory = ( ...args: Args @@ -26,18 +26,27 @@ export type DataSearchRequestFactory = OperatorFunction< + IKibanaSearchResponse, + ParsedKibanaSearchResponse +>; + export const useDataSearch = < RequestFactoryArgs extends any[], - Request extends IKibanaSearchRequest, - RawResponse + RequestParams, + Request extends IKibanaSearchRequest, + RawResponse, + Response >({ getRequest, + parseResponses, }: { getRequest: DataSearchRequestFactory; + parseResponses: ParseResponsesOperator; }) => { const { services } = useKibanaContextForPlugin(); const requests$ = useObservable( - () => new Subject>(), + () => new Subject>(), [] ); @@ -75,6 +84,7 @@ export const useDataSearch = < abortController.abort(); } }), + parseResponses, share() ), }; @@ -83,7 +93,7 @@ export const useDataSearch = < return newRequestDescriptor; }, - [getRequest, services.data.search, requests$] + [getRequest, services.data.search, parseResponses, requests$] ); return { From 25252d5100543cbdb7f54c81f9423a5f1c9a6f0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20St=C3=BCrmer?= Date: Tue, 26 Jan 2021 16:36:13 +0100 Subject: [PATCH 27/39] Remove unneeded code --- .../infra/public/utils/data_search/index.ts | 1 - .../parse_data_search_responses.ts | 84 ------------------- .../infra/public/utils/data_search/types.ts | 9 -- 3 files changed, 94 deletions(-) delete mode 100644 x-pack/plugins/infra/public/utils/data_search/parse_data_search_responses.ts diff --git a/x-pack/plugins/infra/public/utils/data_search/index.ts b/x-pack/plugins/infra/public/utils/data_search/index.ts index baf12907b7a158..10beba4aa4fdcd 100644 --- a/x-pack/plugins/infra/public/utils/data_search/index.ts +++ b/x-pack/plugins/infra/public/utils/data_search/index.ts @@ -6,7 +6,6 @@ export * from './flatten_data_search_response'; export * from './normalize_data_search_responses'; -export * from './parse_data_search_responses'; export * from './types'; export * from './use_data_search_request'; export * from './use_data_search_response_state'; diff --git a/x-pack/plugins/infra/public/utils/data_search/parse_data_search_responses.ts b/x-pack/plugins/infra/public/utils/data_search/parse_data_search_responses.ts deleted file mode 100644 index 8bbbbe7bed267c..00000000000000 --- a/x-pack/plugins/infra/public/utils/data_search/parse_data_search_responses.ts +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { of } from 'rxjs'; -import { catchError, map, startWith } from 'rxjs/operators'; -import { IKibanaSearchRequest } from '../../../../../../src/plugins/data/public'; -import { AbortError } from '../../../../../../src/plugins/kibana_utils/public'; -import { SearchStrategyError } from '../../../common/search_strategies/common/errors'; -import { DataSearchRequestDescriptor, ParsedDataSearchRequestDescriptor } from './types'; - -export type ResponseProjection = ( - rawResponse: RawResponse -) => { data: Response; errors?: SearchStrategyError[] }; - -/** - * Turns the {@link DataSearchRequestDescriptor} into a {@link - * ParsedDataSearchRequestDescriptor} by enhancing the response observable with - * parsing and error handling transformations. - * - * @param initialResponse - The initial value to emit when a new request is - * handled. - * @param projectResponse - The projection function to apply to each response - * payload. It should validate that the response payload is of the type {@link - * RawResponse} and decode it to a {@link Response}. - * - * @return A function that adds parsing and error handling transformations to - * each response payload using the arguments given above. - */ -export const parseDataSearchResponses = < - Request extends IKibanaSearchRequest, - RawResponse, - Response, - InitialResponse ->( - initialResponse: InitialResponse, - projectResponse: ResponseProjection -) => ( - requestDescriptor: DataSearchRequestDescriptor -): ParsedDataSearchRequestDescriptor => ({ - ...requestDescriptor, - response$: requestDescriptor.response$.pipe( - map((response) => { - const { data, errors = [] } = projectResponse(response.rawResponse); - return { - data, - errors, - isPartial: response.isPartial ?? false, - isRunning: response.isRunning ?? false, - loaded: response.loaded, - total: response.total, - }; - }), - startWith({ - data: initialResponse, - errors: [], - isPartial: true, - isRunning: true, - loaded: 0, - total: undefined, - }), - catchError((error) => - of({ - data: initialResponse, - errors: [ - error instanceof AbortError - ? { - type: 'aborted' as const, - } - : { - type: 'generic' as const, - message: `${error.message ?? error}`, - }, - ], - isPartial: true, - isRunning: false, - loaded: 0, - total: undefined, - }) - ) - ), -}); diff --git a/x-pack/plugins/infra/public/utils/data_search/types.ts b/x-pack/plugins/infra/public/utils/data_search/types.ts index 72172c2a036bb9..4fcb5898ea5bd8 100644 --- a/x-pack/plugins/infra/public/utils/data_search/types.ts +++ b/x-pack/plugins/infra/public/utils/data_search/types.ts @@ -47,12 +47,3 @@ export interface ParsedDataSearchResponseDescriptor< response: ParsedKibanaSearchResponse; abortController: AbortController; } - -export type ParsedDataSearchResponseDescriptorOfRequest< - ParsedRequestDescriptor extends ParsedDataSearchRequestDescriptor -> = ParsedRequestDescriptor extends ParsedDataSearchRequestDescriptor< - infer Request, - infer ResponseData -> - ? ParsedDataSearchResponseDescriptor - : never; From 3929ebac1bcca59b3e4d48e7734df7aed8686cc9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20St=C3=BCrmer?= Date: Tue, 26 Jan 2021 18:35:04 +0100 Subject: [PATCH 28/39] Fix the useDataSearch unit test --- .../use_data_search_request.test.tsx | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/infra/public/utils/data_search/use_data_search_request.test.tsx b/x-pack/plugins/infra/public/utils/data_search/use_data_search_request.test.tsx index 87c091f12ad90a..780476abb7b1bf 100644 --- a/x-pack/plugins/infra/public/utils/data_search/use_data_search_request.test.tsx +++ b/x-pack/plugins/infra/public/utils/data_search/use_data_search_request.test.tsx @@ -17,6 +17,7 @@ import { import { dataPluginMock } from '../../../../../../src/plugins/data/public/mocks'; import { createKibanaReactContext } from '../../../../../../src/plugins/kibana_react/public'; import { PluginKibanaContextValue } from '../../hooks/use_kibana'; +import { normalizeDataSearchResponses } from './normalize_data_search_responses'; import { useDataSearch } from './use_data_search_request'; describe('useDataSearch hook', () => { @@ -34,6 +35,7 @@ describe('useDataSearch hook', () => { () => useDataSearch({ getRequest, + parseResponses: noopParseResponse, }), { wrapper: ({ children }) => {children}, @@ -48,7 +50,7 @@ describe('useDataSearch hook', () => { expect(dataMock.search.search).not.toHaveBeenCalled(); }); - it('creates search requests with the given params and options', async () => { + it('creates search requests with the given params and options and parses the responses', async () => { const dataMock = createDataPluginMock(); const searchResponseMock$ = of({ rawResponse: { @@ -78,6 +80,7 @@ describe('useDataSearch hook', () => { () => useDataSearch({ getRequest, + parseResponses: noopParseResponse, }), { wrapper: ({ children }) => {children}, @@ -112,10 +115,11 @@ describe('useDataSearch hook', () => { }); expect(firstRequest).toHaveProperty('options.strategy', 'test-search-strategy'); expect(firstRequest).toHaveProperty('response$', expect.any(Observable)); - await expect(firstRequest.response$.toPromise()).resolves.toEqual({ - rawResponse: { - firstKey: 'firstValue', + await expect(firstRequest.response$.toPromise()).resolves.toMatchObject({ + data: { + firstKey: 'firstValue', // because this specific response parser just copies the raw response }, + errors: [], }); }); @@ -145,6 +149,7 @@ describe('useDataSearch hook', () => { () => useDataSearch({ getRequest, + parseResponses: noopParseResponse, }), { wrapper: ({ children }) => {children}, @@ -186,3 +191,8 @@ const createDataPluginMock = () => { }; return dataMock; }; + +const noopParseResponse = normalizeDataSearchResponses( + null, + (response: Response) => ({ data: response }) +); From 631cc4ab88d02e62ee17f33397281da1ea17db2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20St=C3=BCrmer?= Date: Tue, 26 Jan 2021 18:47:08 +0100 Subject: [PATCH 29/39] Update the documenting storybook --- .../utils/data_search/data_search.stories.mdx | 36 ++++++++++++------- 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/x-pack/plugins/infra/public/utils/data_search/data_search.stories.mdx b/x-pack/plugins/infra/public/utils/data_search/data_search.stories.mdx index a698b806b4cd70..a8854692caa36c 100644 --- a/x-pack/plugins/infra/public/utils/data_search/data_search.stories.mdx +++ b/x-pack/plugins/infra/public/utils/data_search/data_search.stories.mdx @@ -43,15 +43,35 @@ be issued by calling the returned `search()` function. For each new request the hook emits an object describing the request and its state in the `requests$` `Observable`. +Since the specific response shape depends on the data strategy used, the hook +takes a projection function, that is responsible for decoding the response in +an appropriate way. Because most response projections follow a similar pattern +there's a helper `normalizeDataSearchResponses(initialResponse, +parseRawResponse)`, which generates an RxJS operator, that... + +- emits an initial response containing the given `initialResponse` value +- applies `parseRawResponse` to the `rawResponse` property of each emitted response +- transforms transport layer errors as well as parsing errors into + `SearchStrategyError`s + ```typescript +const parseMyCustomSearchResponse = normalizeDataSearchResponses( + 'initial value', + decodeOrThrow(myCustomSearchResponsePayloadRT) +); + const { search, requests$ } = useDataSearch({ getRequest: useCallback((searchTerm: string) => ({ request: { params: { searchTerm - } - } - }), []); + }, + options: { + strategy: 'my-custom-search-strategy', + }, + }, + }), []), + parseResponses: parseMyCustomSearchResponse, }); ``` @@ -68,10 +88,6 @@ observables are unsubscribed from for proper cancellation if a new request has been created. This uses RxJS's `switchMap()` operator under the hood. The hook also makes sure that all observables are unsubscribed from on unmount. -Since the specific response shape depends on the data strategy used, the hook -takes a projection function, that is responsible for decoding the response in -an appropriate way. - A request can fail due to various reasons that include servers-side errors, Elasticsearch shard failures and network failures. The intention is to map all of them to a common `SearchStrategyError` interface. While the @@ -94,11 +110,7 @@ const { latestResponseErrors, loaded, total, -} = useLatestPartialDataSearchResponse( - requests$, - 'initialValue', - useMemo(() => decodeOrThrow(mySearchStrategyResponsePayloadRT), []), -); +} = useLatestPartialDataSearchResponse(requests$); ``` ## Representing the request state to the user From 532daffa50c558aace5db266d0bc3bb3fb0bb244 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20St=C3=BCrmer?= Date: Tue, 26 Jan 2021 20:51:27 +0100 Subject: [PATCH 30/39] Fix JSON types import --- x-pack/plugins/infra/common/typed_json.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/x-pack/plugins/infra/common/typed_json.ts b/x-pack/plugins/infra/common/typed_json.ts index f3e7608910e095..5aec8d3eaf2ccf 100644 --- a/x-pack/plugins/infra/common/typed_json.ts +++ b/x-pack/plugins/infra/common/typed_json.ts @@ -7,6 +7,8 @@ import * as rt from 'io-ts'; import { JsonArray, JsonObject, JsonValue } from '../../../../src/plugins/kibana_utils/common'; +export { JsonArray, JsonObject, JsonValue }; + export const jsonScalarRT = rt.union([rt.null, rt.boolean, rt.number, rt.string]); export const jsonValueRT: rt.Type = rt.recursion('JsonValue', () => From d5f34d99a92d3b2bb73f444ab757e62fc0bd1a29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20St=C3=BCrmer?= Date: Tue, 26 Jan 2021 20:52:24 +0100 Subject: [PATCH 31/39] Remove debug output --- .../logs/log_stream/use_fetch_log_entries_around.ts | 6 ------ 1 file changed, 6 deletions(-) diff --git a/x-pack/plugins/infra/public/containers/logs/log_stream/use_fetch_log_entries_around.ts b/x-pack/plugins/infra/public/containers/logs/log_stream/use_fetch_log_entries_around.ts index 171f00ee9b2e69..a3e0a7b6560966 100644 --- a/x-pack/plugins/infra/public/containers/logs/log_stream/use_fetch_log_entries_around.ts +++ b/x-pack/plugins/infra/public/containers/logs/log_stream/use_fetch_log_entries_around.ts @@ -161,12 +161,6 @@ export const useFetchLogEntriesAround = ({ }, } = useObservableState(logEntriesAroundSearchResponses$, initialCombinedResponse); - useSubscription(logEntriesAroundSearchResponses$, { - next: ({ before, after, combined }) => { - console.log('combined next', before, after, combined); - }, - }); - const cancelRequest = useCallback(() => { latestBeforeResponse?.abortController.abort(); latestAfterResponse?.abortController.abort(); From 46f30b9c7ea3a2ee9471d61db7f64b9348782671 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20St=C3=BCrmer?= Date: Tue, 26 Jan 2021 20:55:41 +0100 Subject: [PATCH 32/39] Re-enable logEntriesSearchStrategy unit tests --- .../log_entries_search_strategy.test.ts | 113 +++++++++--------- 1 file changed, 59 insertions(+), 54 deletions(-) diff --git a/x-pack/plugins/infra/server/services/log_entries/log_entries_search_strategy.test.ts b/x-pack/plugins/infra/server/services/log_entries/log_entries_search_strategy.test.ts index 2c52b0fbf4bf89..f07ee0508fa6c4 100644 --- a/x-pack/plugins/infra/server/services/log_entries/log_entries_search_strategy.test.ts +++ b/x-pack/plugins/infra/server/services/log_entries/log_entries_search_strategy.test.ts @@ -179,67 +179,72 @@ describe('LogEntries search strategy', () => { ]); }); - // it.skip('forwards errors from the underlying search strategy', async () => { - // const esSearchStrategyMock = createEsSearchStrategyMock({ - // id: 'ASYNC_REQUEST_ID', - // isRunning: false, - // rawResponse: { - // took: 1, - // _shards: { total: 1, failed: 0, skipped: 0, successful: 1 }, - // timed_out: false, - // hits: { total: 0, max_score: 0, hits: [] }, - // }, - // }); - // const dataMock = createDataPluginMock(esSearchStrategyMock); - // const sourcesMock = createInfraSourcesMock(); - // sourcesMock.getSourceConfiguration.mockResolvedValue(createSourceConfigurationMock()); - // const mockDependencies = createSearchStrategyDependenciesMock(); + it('forwards errors from the underlying search strategy', async () => { + const esSearchStrategyMock = createEsSearchStrategyMock({ + id: 'ASYNC_REQUEST_ID', + isRunning: false, + rawResponse: { + took: 1, + _shards: { total: 1, failed: 0, skipped: 0, successful: 1 }, + timed_out: false, + hits: { total: 0, max_score: 0, hits: [] }, + }, + }); + const dataMock = createDataPluginMock(esSearchStrategyMock); + const sourcesMock = createInfraSourcesMock(); + sourcesMock.getSourceConfiguration.mockResolvedValue(createSourceConfigurationMock()); + const mockDependencies = createSearchStrategyDependenciesMock(); - // const logEntriesSearchStrategy = logEntriesSearchStrategyProvider({ - // data: dataMock, - // sources: sourcesMock, - // }); + const logEntriesSearchStrategy = logEntriesSearchStrategyProvider({ + data: dataMock, + sources: sourcesMock, + }); - // const response = logEntriesSearchStrategy.search( - // { - // id: logEntriesSearchRequestStateRT.encode({ esRequestId: 'UNKNOWN_ID' }), - // params: { sourceId: 'SOURCE_ID', logEntryId: 'LOG_ENTRY_ID' }, - // }, - // {}, - // mockDependencies - // ); + const response = logEntriesSearchStrategy.search( + { + id: logEntriesSearchRequestStateRT.encode({ esRequestId: 'UNKNOWN_ID' }), + params: { + sourceId: 'SOURCE_ID', + startTimestamp: 100, + endTimestamp: 200, + size: 3, + }, + }, + {}, + mockDependencies + ); - // await expect(response.toPromise()).rejects.toThrowError(ResponseError); - // }); + await expect(response.toPromise()).rejects.toThrowError(ResponseError); + }); - // it.skip('forwards cancellation to the underlying search strategy', async () => { - // const esSearchStrategyMock = createEsSearchStrategyMock({ - // id: 'ASYNC_REQUEST_ID', - // isRunning: false, - // rawResponse: { - // took: 1, - // _shards: { total: 1, failed: 0, skipped: 0, successful: 1 }, - // timed_out: false, - // hits: { total: 0, max_score: 0, hits: [] }, - // }, - // }); - // const dataMock = createDataPluginMock(esSearchStrategyMock); - // const sourcesMock = createInfraSourcesMock(); - // sourcesMock.getSourceConfiguration.mockResolvedValue(createSourceConfigurationMock()); - // const mockDependencies = createSearchStrategyDependenciesMock(); + it('forwards cancellation to the underlying search strategy', async () => { + const esSearchStrategyMock = createEsSearchStrategyMock({ + id: 'ASYNC_REQUEST_ID', + isRunning: false, + rawResponse: { + took: 1, + _shards: { total: 1, failed: 0, skipped: 0, successful: 1 }, + timed_out: false, + hits: { total: 0, max_score: 0, hits: [] }, + }, + }); + const dataMock = createDataPluginMock(esSearchStrategyMock); + const sourcesMock = createInfraSourcesMock(); + sourcesMock.getSourceConfiguration.mockResolvedValue(createSourceConfigurationMock()); + const mockDependencies = createSearchStrategyDependenciesMock(); - // const logEntriesSearchStrategy = logEntriesSearchStrategyProvider({ - // data: dataMock, - // sources: sourcesMock, - // }); - // const requestId = logEntriesSearchRequestStateRT.encode({ - // esRequestId: 'ASYNC_REQUEST_ID', - // }); + const logEntriesSearchStrategy = logEntriesSearchStrategyProvider({ + data: dataMock, + sources: sourcesMock, + }); + const requestId = logEntriesSearchRequestStateRT.encode({ + esRequestId: 'ASYNC_REQUEST_ID', + }); - // await logEntriesSearchStrategy.cancel?.(requestId, {}, mockDependencies); + await logEntriesSearchStrategy.cancel?.(requestId, {}, mockDependencies); - // expect(esSearchStrategyMock.cancel).toHaveBeenCalled(); - // }); + expect(esSearchStrategyMock.cancel).toHaveBeenCalled(); + }); }); const createSourceConfigurationMock = (): InfraSource => ({ From 2342f1fca571fa59bcd4411a16c46344f2de9444 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20St=C3=BCrmer?= Date: Tue, 26 Jan 2021 20:58:44 +0100 Subject: [PATCH 33/39] Remove unused import --- .../containers/logs/log_stream/use_fetch_log_entries_around.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/infra/public/containers/logs/log_stream/use_fetch_log_entries_around.ts b/x-pack/plugins/infra/public/containers/logs/log_stream/use_fetch_log_entries_around.ts index a3e0a7b6560966..97c2f3a694a280 100644 --- a/x-pack/plugins/infra/public/containers/logs/log_stream/use_fetch_log_entries_around.ts +++ b/x-pack/plugins/infra/public/containers/logs/log_stream/use_fetch_log_entries_around.ts @@ -11,7 +11,7 @@ import { LogSourceColumnConfiguration } from '../../../../common/http_api/log_so import { LogEntryCursor } from '../../../../common/log_entry'; import { JsonObject } from '../../../../common/typed_json'; import { flattenDataSearchResponseDescriptor } from '../../../utils/data_search'; -import { useObservable, useObservableState, useSubscription } from '../../../utils/use_observable'; +import { useObservable, useObservableState } from '../../../utils/use_observable'; import { useLogEntriesAfterRequest } from './use_fetch_log_entries_after'; import { useLogEntriesBeforeRequest } from './use_fetch_log_entries_before'; From 73d1c204e761efee8726ba4cf9be60a3f0d3b416 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20St=C3=BCrmer?= Date: Wed, 27 Jan 2021 12:40:58 +0100 Subject: [PATCH 34/39] Try to always derive a top and bottom cursor --- .../infra/public/containers/logs/log_stream/index.ts | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/x-pack/plugins/infra/public/containers/logs/log_stream/index.ts b/x-pack/plugins/infra/public/containers/logs/log_stream/index.ts index c0c0c8accd6764..f97e034f48465f 100644 --- a/x-pack/plugins/infra/public/containers/logs/log_stream/index.ts +++ b/x-pack/plugins/infra/public/containers/logs/log_stream/index.ts @@ -87,7 +87,6 @@ export function useLogStream({ useSubscription(logEntriesAroundSearchResponses$, { next: ({ before, after, combined }) => { - // if (data != null && !isPartial) { if ((before.response.data != null || after?.response.data != null) && !combined.isPartial) { setState((prevState) => ({ ...prevState, @@ -99,9 +98,6 @@ export function useLogStream({ })); } }, - error: (err) => { - console.error(err); - }, }); const { @@ -124,12 +120,10 @@ export function useLogStream({ entries: [...data.entries, ...prevState.entries], hasMoreBefore: data.hasMoreBefore ?? prevState.hasMoreBefore, topCursor: data.topCursor ?? prevState.topCursor, + bottomCursor: prevState.bottomCursor ?? data.bottomCursor, })); } }, - error: (err) => { - console.error(err); - }, }); const fetchPreviousEntries = useCallback(() => { @@ -165,13 +159,11 @@ export function useLogStream({ ...prevState, entries: [...prevState.entries, ...data.entries], hasMoreAfter: data.hasMoreAfter ?? prevState.hasMoreAfter, + topCursor: prevState.topCursor ?? data.topCursor, bottomCursor: data.bottomCursor ?? prevState.bottomCursor, })); } }, - error: (err) => { - console.error(err); - }, }); const fetchNextEntries = useCallback(() => { From 812c6cbc5e70883b73bcfa4f6a533cdbb6a4a79e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20St=C3=BCrmer?= Date: Wed, 27 Jan 2021 12:42:54 +0100 Subject: [PATCH 35/39] Update the `` storybook --- .../log_stream/log_stream.stories.mdx | 77 +++++++++++++------ .../infra/public/test_utils/entries.ts | 3 +- 2 files changed, 57 insertions(+), 23 deletions(-) diff --git a/x-pack/plugins/infra/public/components/log_stream/log_stream.stories.mdx b/x-pack/plugins/infra/public/components/log_stream/log_stream.stories.mdx index bda52d9323eb60..901a4b6a8383e3 100644 --- a/x-pack/plugins/infra/public/components/log_stream/log_stream.stories.mdx +++ b/x-pack/plugins/infra/public/components/log_stream/log_stream.stories.mdx @@ -1,10 +1,12 @@ import { Meta, Story, Canvas, ArgsTable } from '@storybook/addon-docs/blocks'; -import { Subject } from 'rxjs'; +import { defer, of, Subject } from 'rxjs'; +import { delay } from 'rxjs/operators'; import { I18nProvider } from '@kbn/i18n/react'; import { EuiThemeProvider } from '../../../../../../src/plugins/kibana_react/common'; import { KibanaContextProvider } from '../../../../../../src/plugins/kibana_react/public'; +import { LOG_ENTRIES_SEARCH_STRATEGY } from '../../../common/search_strategies/log_entries/log_entries'; import { DEFAULT_SOURCE_CONFIGURATION } from '../../test_utils/source_configuration'; import { generateFakeEntries, ENTRIES_EMPTY } from '../../test_utils/entries'; @@ -15,30 +17,61 @@ import { LogStream } from './'; export const startTimestamp = 1595145600000; export const endTimestamp = startTimestamp + 15 * 60 * 1000; +export const dataMock = { + search: { + search: ({ params }, options) => { + return defer(() => { + switch (options.strategy) { + case LOG_ENTRIES_SEARCH_STRATEGY: + if (params.after?.time === params.endTimestamp || params.before?.time === params.startTimestamp) { + return of({ + id: 'EMPTY_FAKE_RESPONSE', + total: 1, + loaded: 1, + isRunning: false, + isPartial: false, + rawResponse: ENTRIES_EMPTY, + }); + } else { + const entries = generateFakeEntries( + 200, + params.startTimestamp, + params.endTimestamp, + params.columns || DEFAULT_SOURCE_CONFIGURATION.data.configuration.logColumns + ); + return of({ + id: 'FAKE_RESPONSE', + total: 1, + loaded: 1, + isRunning: false, + isPartial: false, + rawResponse: { + data: { + entries, + topCursor: entries[0].cursor, + bottomCursor: entries[entries.length - 1].cursor, + hasMoreBefore: false, + }, + errors: [], + } + }); + } + default: + return of({ + id: 'FAKE_RESPONSE', + rawResponse: {}, + }); + } + }).pipe(delay(2000)); + }, + }, +}; + + export const fetch = function (url, params) { switch (url) { case '/api/infra/log_source_configurations/default': return DEFAULT_SOURCE_CONFIGURATION; - case '/api/log_entries/entries': - const body = JSON.parse(params.body); - if (body.after?.time === body.endTimestamp || body.before?.time === body.startTimestamp) { - return ENTRIES_EMPTY; - } else { - const entries = generateFakeEntries( - 200, - body.startTimestamp, - body.endTimestamp, - body.columns || DEFAULT_SOURCE_CONFIGURATION.data.configuration.logColumns - ); - return { - data: { - entries, - topCursor: entries[0].cursor, - bottomCursor: entries[entries.length - 1].cursor, - hasMoreBefore: false, - }, - }; - } default: return {}; } @@ -67,7 +100,7 @@ export const Template = (args) => ; (story) => ( - + {story()} diff --git a/x-pack/plugins/infra/public/test_utils/entries.ts b/x-pack/plugins/infra/public/test_utils/entries.ts index 04c87d5f73902b..1633b9d8dc0760 100644 --- a/x-pack/plugins/infra/public/test_utils/entries.ts +++ b/x-pack/plugins/infra/public/test_utils/entries.ts @@ -5,7 +5,7 @@ */ import faker from 'faker'; -import { LogEntry } from '../../common/http_api'; +import { LogEntry } from '../../common/log_entry'; import { LogSourceConfiguration } from '../containers/logs/log_source'; export const ENTRIES_EMPTY = { @@ -28,6 +28,7 @@ export function generateFakeEntries( const timestamp = i === count - 1 ? endTimestamp : startTimestamp + timestampStep * i; entries.push({ id: `entry-${i}`, + index: 'logs-fake', context: {}, cursor: { time: timestamp, tiebreaker: i }, columns: columns.map((column) => { From acc0840fa54ef45f59e84d44b46e115b90abd585 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20St=C3=BCrmer?= Date: Wed, 27 Jan 2021 13:30:19 +0100 Subject: [PATCH 36/39] Remove more unused code --- .../log_entries/log_entries.ts | 7 --- .../containers/logs/log_entries_async.ts | 56 ------------------- 2 files changed, 63 deletions(-) delete mode 100644 x-pack/plugins/infra/public/containers/logs/log_entries_async.ts diff --git a/x-pack/plugins/infra/common/search_strategies/log_entries/log_entries.ts b/x-pack/plugins/infra/common/search_strategies/log_entries/log_entries.ts index 00ac40cf9e220a..64b59926405bde 100644 --- a/x-pack/plugins/infra/common/search_strategies/log_entries/log_entries.ts +++ b/x-pack/plugins/infra/common/search_strategies/log_entries/log_entries.ts @@ -8,7 +8,6 @@ import * as rt from 'io-ts'; import { logSourceColumnConfigurationRT } from '../../http_api/log_sources'; import { logEntryAfterCursorRT, - // logEntryAroundCursorRT, logEntryBeforeCursorRT, logEntryCursorRT, logEntryRT, @@ -42,16 +41,10 @@ export const logEntriesAfterSearchRequestParamsRT = rt.intersection([ logEntryAfterCursorRT, ]); -// export const logEntriesCenteredSearchRequestParamsRT = rt.intersection([ -// logEntriesBaseSearchRequestParamsRT, -// logEntryAroundCursorRT, -// ]); - export const logEntriesSearchRequestParamsRT = rt.union([ logEntriesBaseSearchRequestParamsRT, logEntriesBeforeSearchRequestParamsRT, logEntriesAfterSearchRequestParamsRT, - // logEntriesCenteredSearchRequestParamsRT, ]); export type LogEntriesSearchRequestParams = rt.TypeOf; diff --git a/x-pack/plugins/infra/public/containers/logs/log_entries_async.ts b/x-pack/plugins/infra/public/containers/logs/log_entries_async.ts deleted file mode 100644 index 92302b7f304f09..00000000000000 --- a/x-pack/plugins/infra/public/containers/logs/log_entries_async.ts +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -// import { useCallback } from 'react'; -// import { decodeOrThrow } from '../../../common/runtime_types'; -// import { -// logEntriesSearchRequestParamsRT, -// logEntriesSearchResponsePayloadRT, -// LOG_ENTRIES_SEARCH_STRATEGY, -// } from '../../../common/search_strategies/log_entries/log_entries'; -// import { useDataSearch, useLatestPartialDataSearchResponse } from '../../utils/data_search'; - -// export const useLogEntries = ({ sourceId }: { sourceId: string | null | undefined }) => { -// const { search: fetchLogEntry, requests$: logEntrySearchRequests$ } = useDataSearch({ -// getRequest: useCallback(() => { -// return !!logEntryId && !!sourceId -// ? { -// request: { -// params: logEntrySearchRequestParamsRT.encode({ sourceId, logEntryId }), -// }, -// options: { strategy: LOG_ENTRY_SEARCH_STRATEGY }, -// } -// : null; -// }, [sourceId, logEntryId]), -// }); - -// const { -// cancelRequest, -// isRequestRunning, -// isResponsePartial, -// latestResponseData, -// latestResponseErrors, -// loaded, -// total, -// } = useLatestPartialDataSearchResponse( -// logEntrySearchRequests$, -// null, -// decodeLogEntrySearchResponse -// ); - -// return { -// cancelRequest, -// errors: latestResponseErrors, -// fetchLogEntry, -// isRequestRunning, -// isResponsePartial, -// loaded, -// logEntry: latestResponseData ?? null, -// total, -// }; -// }; - -// const decodeLogEntrySearchResponse = decodeOrThrow(logEntrySearchResponsePayloadRT); From 42c8b5e5e821c7704e06fb06bb149faea811c437 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20St=C3=BCrmer?= Date: Thu, 28 Jan 2021 12:03:49 +0100 Subject: [PATCH 37/39] Fix incorrect imports after merge --- .../lib/domains/log_entries_domain/log_entries_domain.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/log_entries_domain.ts b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/log_entries_domain.ts index 8e0d9de0f02897..e318075045522f 100644 --- a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/log_entries_domain.ts +++ b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/log_entries_domain.ts @@ -12,7 +12,7 @@ import { LogEntriesSummaryHighlightsBucket, LogEntriesRequest, } from '../../../../common/http_api'; -import { LogEntry, LogColumn } from '../../../../common/log_entry'; +import { LogColumn, LogEntryCursor, LogEntry } from '../../../../common/log_entry'; import { InfraSourceConfiguration, InfraSources, @@ -33,7 +33,6 @@ import { CompositeDatasetKey, createLogEntryDatasetsQuery, } from './queries/log_entry_datasets'; -import { LogEntry, LogColumn, LogEntryCursor } from '../../../../common/log_entry'; export interface LogEntriesParams { startTimestamp: number; From 05669db9eb3f6f53d845af63c8f4dc31a754cf16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20St=C3=BCrmer?= Date: Mon, 1 Feb 2021 19:07:38 +0100 Subject: [PATCH 38/39] Fix types after merge --- .../common/search_strategies/log_entries/log_entries.ts | 5 ++++- .../logs/log_stream/use_fetch_log_entries_after.ts | 6 +++--- .../logs/log_stream/use_fetch_log_entries_around.ts | 4 ++-- .../logs/log_stream/use_fetch_log_entries_before.ts | 6 +++--- 4 files changed, 12 insertions(+), 9 deletions(-) diff --git a/x-pack/plugins/infra/common/search_strategies/log_entries/log_entries.ts b/x-pack/plugins/infra/common/search_strategies/log_entries/log_entries.ts index 64b59926405bde..b2a879c3b72fd3 100644 --- a/x-pack/plugins/infra/common/search_strategies/log_entries/log_entries.ts +++ b/x-pack/plugins/infra/common/search_strategies/log_entries/log_entries.ts @@ -5,6 +5,7 @@ */ import * as rt from 'io-ts'; +import { DslQuery } from '../../../../../../src/plugins/data/common'; import { logSourceColumnConfigurationRT } from '../../http_api/log_sources'; import { logEntryAfterCursorRT, @@ -12,7 +13,7 @@ import { logEntryCursorRT, logEntryRT, } from '../../log_entry'; -import { jsonObjectRT } from '../../typed_json'; +import { JsonObject, jsonObjectRT } from '../../typed_json'; import { searchStrategyErrorRT } from '../common/errors'; export const LOG_ENTRIES_SEARCH_STRATEGY = 'infra-log-entries'; @@ -49,6 +50,8 @@ export const logEntriesSearchRequestParamsRT = rt.union([ export type LogEntriesSearchRequestParams = rt.TypeOf; +export type LogEntriesSearchRequestQuery = JsonObject | DslQuery; + export const logEntriesSearchResponsePayloadRT = rt.intersection([ rt.type({ data: rt.intersection([ diff --git a/x-pack/plugins/infra/public/containers/logs/log_stream/use_fetch_log_entries_after.ts b/x-pack/plugins/infra/public/containers/logs/log_stream/use_fetch_log_entries_after.ts index d460f23f8a9fe5..c7076ec51db6a3 100644 --- a/x-pack/plugins/infra/public/containers/logs/log_stream/use_fetch_log_entries_after.ts +++ b/x-pack/plugins/infra/public/containers/logs/log_stream/use_fetch_log_entries_after.ts @@ -13,11 +13,11 @@ import { LogEntryAfterCursor } from '../../../../common/log_entry'; import { decodeOrThrow } from '../../../../common/runtime_types'; import { logEntriesSearchRequestParamsRT, + LogEntriesSearchRequestQuery, LogEntriesSearchResponsePayload, logEntriesSearchResponsePayloadRT, LOG_ENTRIES_SEARCH_STRATEGY, } from '../../../../common/search_strategies/log_entries/log_entries'; -import { JsonObject } from '../../../../common/typed_json'; import { flattenDataSearchResponseDescriptor, normalizeDataSearchResponses, @@ -38,7 +38,7 @@ export const useLogEntriesAfterRequest = ({ columnOverrides?: LogSourceColumnConfiguration[]; endTimestamp: number; highlightPhrase?: string; - query?: JsonObject; + query?: LogEntriesSearchRequestQuery; sourceId: string; startTimestamp: number; }) => { @@ -116,7 +116,7 @@ export const useFetchLogEntriesAfter = ({ columnOverrides?: LogSourceColumnConfiguration[]; endTimestamp: number; highlightPhrase?: string; - query?: JsonObject; + query?: LogEntriesSearchRequestQuery; sourceId: string; startTimestamp: number; }) => { diff --git a/x-pack/plugins/infra/public/containers/logs/log_stream/use_fetch_log_entries_around.ts b/x-pack/plugins/infra/public/containers/logs/log_stream/use_fetch_log_entries_around.ts index 97c2f3a694a280..01f6336e0d5c8b 100644 --- a/x-pack/plugins/infra/public/containers/logs/log_stream/use_fetch_log_entries_around.ts +++ b/x-pack/plugins/infra/public/containers/logs/log_stream/use_fetch_log_entries_around.ts @@ -9,7 +9,7 @@ import { combineLatest, Observable, Subject } from 'rxjs'; import { last, map, startWith, switchMap } from 'rxjs/operators'; import { LogSourceColumnConfiguration } from '../../../../common/http_api/log_sources'; import { LogEntryCursor } from '../../../../common/log_entry'; -import { JsonObject } from '../../../../common/typed_json'; +import { LogEntriesSearchRequestQuery } from '../../../../common/search_strategies/log_entries/log_entries'; import { flattenDataSearchResponseDescriptor } from '../../../utils/data_search'; import { useObservable, useObservableState } from '../../../utils/use_observable'; import { useLogEntriesAfterRequest } from './use_fetch_log_entries_after'; @@ -26,7 +26,7 @@ export const useFetchLogEntriesAround = ({ columnOverrides?: LogSourceColumnConfiguration[]; endTimestamp: number; highlightPhrase?: string; - query?: JsonObject; + query?: LogEntriesSearchRequestQuery; sourceId: string; startTimestamp: number; }) => { diff --git a/x-pack/plugins/infra/public/containers/logs/log_stream/use_fetch_log_entries_before.ts b/x-pack/plugins/infra/public/containers/logs/log_stream/use_fetch_log_entries_before.ts index 10e6bdcbf188d3..5553be11b9fef3 100644 --- a/x-pack/plugins/infra/public/containers/logs/log_stream/use_fetch_log_entries_before.ts +++ b/x-pack/plugins/infra/public/containers/logs/log_stream/use_fetch_log_entries_before.ts @@ -13,11 +13,11 @@ import { LogEntryBeforeCursor } from '../../../../common/log_entry'; import { decodeOrThrow } from '../../../../common/runtime_types'; import { logEntriesSearchRequestParamsRT, + LogEntriesSearchRequestQuery, LogEntriesSearchResponsePayload, logEntriesSearchResponsePayloadRT, LOG_ENTRIES_SEARCH_STRATEGY, } from '../../../../common/search_strategies/log_entries/log_entries'; -import { JsonObject } from '../../../../common/typed_json'; import { flattenDataSearchResponseDescriptor, normalizeDataSearchResponses, @@ -38,7 +38,7 @@ export const useLogEntriesBeforeRequest = ({ columnOverrides?: LogSourceColumnConfiguration[]; endTimestamp: number; highlightPhrase?: string; - query?: JsonObject; + query?: LogEntriesSearchRequestQuery; sourceId: string; startTimestamp: number; }) => { @@ -117,7 +117,7 @@ export const useFetchLogEntriesBefore = ({ columnOverrides?: LogSourceColumnConfiguration[]; endTimestamp: number; highlightPhrase?: string; - query?: JsonObject; + query?: LogEntriesSearchRequestQuery; sourceId: string; startTimestamp: number; }) => { From 6f2144fedd6109b3520350cfb527b4e49a2a9ba2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20St=C3=BCrmer?= Date: Tue, 2 Feb 2021 12:50:19 +0100 Subject: [PATCH 39/39] Deduplicate common arguments --- .../containers/logs/log_stream/index.ts | 35 ++++++++----------- 1 file changed, 14 insertions(+), 21 deletions(-) diff --git a/x-pack/plugins/infra/public/containers/logs/log_stream/index.ts b/x-pack/plugins/infra/public/containers/logs/log_stream/index.ts index 9c5727e6681594..8343525c828628 100644 --- a/x-pack/plugins/infra/public/containers/logs/log_stream/index.ts +++ b/x-pack/plugins/infra/public/containers/logs/log_stream/index.ts @@ -83,17 +83,22 @@ export function useLogStream({ } }, [query]); + const commonFetchArguments = useMemo( + () => ({ + sourceId, + startTimestamp, + endTimestamp, + query: parsedQuery, + columnOverrides: columns, + }), + [columns, endTimestamp, parsedQuery, sourceId, startTimestamp] + ); + const { fetchLogEntriesAround, isRequestRunning: isLogEntriesAroundRequestRunning, logEntriesAroundSearchResponses$, - } = useFetchLogEntriesAround({ - sourceId, - startTimestamp, - endTimestamp, - query: parsedQuery, - columnOverrides: columns, - }); + } = useFetchLogEntriesAround(commonFetchArguments); useSubscription(logEntriesAroundSearchResponses$, { next: ({ before, after, combined }) => { @@ -114,13 +119,7 @@ export function useLogStream({ fetchLogEntriesBefore, isRequestRunning: isLogEntriesBeforeRequestRunning, logEntriesBeforeSearchResponse$, - } = useFetchLogEntriesBefore({ - sourceId, - startTimestamp, - endTimestamp, - query: parsedQuery, - columnOverrides: columns, - }); + } = useFetchLogEntriesBefore(commonFetchArguments); useSubscription(logEntriesBeforeSearchResponse$, { next: ({ response: { data, isPartial } }) => { @@ -154,13 +153,7 @@ export function useLogStream({ fetchLogEntriesAfter, isRequestRunning: isLogEntriesAfterRequestRunning, logEntriesAfterSearchResponse$, - } = useFetchLogEntriesAfter({ - sourceId, - startTimestamp, - endTimestamp, - query: parsedQuery, - columnOverrides: columns, - }); + } = useFetchLogEntriesAfter(commonFetchArguments); useSubscription(logEntriesAfterSearchResponse$, { next: ({ response: { data, isPartial } }) => {