Skip to content

Commit

Permalink
[Inventory] Inventory plugin (elastic#191798)
Browse files Browse the repository at this point in the history
## Description

This PR adds an inventory plugin, which renders an inventory UI.
Currently only data streams are rendered. This is part of the LogsAI
initiative - basically we need a UI for tasks like structuring data,
extracting entities, listing the results etc. This is mostly POC-level
stuff. Eventually some of this code might be handed over to ECO but
let's cross that bridge when we get to it.

## Notes for reviewers:

@elastic/appex-ai-infra @elastic/security-generative-ai: added a
`truncateList` utility function that takes the first n elements of an
array and appends a `{l-n} more` string value if there are more values
than n. Really simple but I expect will also be very often used because
we cannot send a huge amount of items to the LLM.

@elastic/kibana-core @elastic/kibana-operations: just boiler plate stuff
for adding a new plugin (and thank you for enabling us to run
`quick_checks` locally!

@elastic/obs-knowledge-team: added support for streaming using an
Observable.

@elastic/obs-ux-management-team: added links to the Inventory UI in the
Observability plugin

@elastic/obs-entities: I've added an entity manager client to be able to
fetch entity definitions on the server. Maybe there's a better way? LMK.

@elastic/obs-ux-logs-team: added a deeplink to the Inventory UI. I've
also moved CODEOWNERS for this package to
@elastic/obs-ux-management-team as they own the Observability plugin
where this is mostly used.

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
  • Loading branch information
dgieselaar and kibanamachine authored Sep 12, 2024
1 parent 4d5c253 commit 98aa1ab
Show file tree
Hide file tree
Showing 110 changed files with 3,070 additions and 877 deletions.
6 changes: 5 additions & 1 deletion .github/CODEOWNERS
Validating CODEOWNERS rules …
Original file line number Diff line number Diff line change
Expand Up @@ -362,7 +362,7 @@ packages/deeplinks/devtools @elastic/kibana-management
packages/deeplinks/fleet @elastic/fleet
packages/deeplinks/management @elastic/kibana-management
packages/deeplinks/ml @elastic/ml-ui
packages/deeplinks/observability @elastic/obs-ux-logs-team
packages/deeplinks/observability @elastic/obs-ux-management-team
packages/deeplinks/search @elastic/search-kibana
packages/deeplinks/security @elastic/security-solution
packages/deeplinks/shared @elastic/appex-sharedux
Expand Down Expand Up @@ -522,6 +522,7 @@ x-pack/plugins/integration_assistant @elastic/security-scalability
src/plugins/interactive_setup @elastic/kibana-security
test/interactive_setup_api_integration/plugins/test_endpoints @elastic/kibana-security
packages/kbn-interpreter @elastic/kibana-visualizations
x-pack/plugins/observability_solution/inventory @elastic/obs-ux-infra_services-team
x-pack/plugins/observability_solution/investigate_app @elastic/obs-ux-management-team
x-pack/plugins/observability_solution/investigate @elastic/obs-ux-management-team
packages/kbn-investigation-shared @elastic/obs-ux-management-team
Expand Down Expand Up @@ -876,6 +877,9 @@ packages/kbn-sort-predicates @elastic/kibana-visualizations
x-pack/plugins/spaces @elastic/kibana-security
x-pack/test/spaces_api_integration/common/plugins/spaces_test_plugin @elastic/kibana-security
packages/kbn-spec-to-console @elastic/kibana-management
packages/kbn-sse-utils @elastic/obs-knowledge-team
packages/kbn-sse-utils-client @elastic/obs-knowledge-team
packages/kbn-sse-utils-server @elastic/obs-knowledge-team
x-pack/plugins/stack_alerts @elastic/response-ops
x-pack/plugins/stack_connectors @elastic/response-ops
x-pack/test/usage_collection/plugins/stack_management_usage_test @elastic/kibana-management
Expand Down
1 change: 1 addition & 0 deletions .i18nrc.json
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,7 @@
"searchTypes": "packages/kbn-search-types",
"securitySolutionPackages": "x-pack/packages/security-solution",
"serverlessPackages": "packages/serverless",
"sse": [ "packages/kbn-sse-utils" ],
"coloring": "packages/kbn-coloring/src",
"languageDocumentationPopover": "packages/kbn-language-documentation-popover/src",
"esql": "src/plugins/esql",
Expand Down
4 changes: 4 additions & 0 deletions docs/developer/plugin-list.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -648,6 +648,10 @@ the infrastructure monitoring use-case within Kibana.
|Team owner: Security Integrations Scalability
|{kib-repo}blob/{branch}/x-pack/plugins/observability_solution/inventory/README.md[inventory]
|Home of the Inventory plugin, which renders the... inventory.
|{kib-repo}blob/{branch}/x-pack/plugins/observability_solution/investigate/README.md[investigate]
|undefined
Expand Down
4 changes: 4 additions & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -566,6 +566,7 @@
"@kbn/interactive-setup-plugin": "link:src/plugins/interactive_setup",
"@kbn/interactive-setup-test-endpoints-plugin": "link:test/interactive_setup_api_integration/plugins/test_endpoints",
"@kbn/interpreter": "link:packages/kbn-interpreter",
"@kbn/inventory-plugin": "link:x-pack/plugins/observability_solution/inventory",
"@kbn/investigate-app-plugin": "link:x-pack/plugins/observability_solution/investigate_app",
"@kbn/investigate-plugin": "link:x-pack/plugins/observability_solution/investigate",
"@kbn/investigation-shared": "link:packages/kbn-investigation-shared",
Expand Down Expand Up @@ -888,6 +889,9 @@
"@kbn/sort-predicates": "link:packages/kbn-sort-predicates",
"@kbn/spaces-plugin": "link:x-pack/plugins/spaces",
"@kbn/spaces-test-plugin": "link:x-pack/test/spaces_api_integration/common/plugins/spaces_test_plugin",
"@kbn/sse-utils": "link:packages/kbn-sse-utils",
"@kbn/sse-utils-client": "link:packages/kbn-sse-utils-client",
"@kbn/sse-utils-server": "link:packages/kbn-sse-utils-server",
"@kbn/stack-alerts-plugin": "link:x-pack/plugins/stack_alerts",
"@kbn/stack-connectors-plugin": "link:x-pack/plugins/stack_connectors",
"@kbn/stack-management-usage-test-plugin": "link:x-pack/test/usage_collection/plugins/stack_management_usage_test",
Expand Down
2 changes: 2 additions & 0 deletions packages/deeplinks/observability/constants.ts
Original file line number Diff line number Diff line change
Expand Up @@ -30,3 +30,5 @@ export const INVESTIGATE_APP_ID = 'investigate';
export const OBLT_UX_APP_ID = 'ux';

export const OBLT_PROFILING_APP_ID = 'profiling';

export const INVENTORY_APP_ID = 'inventory';
10 changes: 8 additions & 2 deletions packages/deeplinks/observability/deep_links.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import {
AI_ASSISTANT_APP_ID,
OBLT_UX_APP_ID,
OBLT_PROFILING_APP_ID,
INVENTORY_APP_ID,
} from './constants';

type LogsApp = typeof LOGS_APP_ID;
Expand All @@ -32,6 +33,7 @@ type SloApp = typeof SLO_APP_ID;
type AiAssistantApp = typeof AI_ASSISTANT_APP_ID;
type ObltUxApp = typeof OBLT_UX_APP_ID;
type ObltProfilingApp = typeof OBLT_PROFILING_APP_ID;
type InventoryApp = typeof INVENTORY_APP_ID;

export type AppId =
| LogsApp
Expand All @@ -44,10 +46,13 @@ export type AppId =
| SloApp
| AiAssistantApp
| ObltUxApp
| ObltProfilingApp;
| ObltProfilingApp
| InventoryApp;

export type LogsLinkId = 'log-categories' | 'settings' | 'anomalies' | 'stream';

export type InventoryLinkId = 'datastreams';

export type ObservabilityOverviewLinkId =
| 'alerts'
| 'cases'
Expand Down Expand Up @@ -90,4 +95,5 @@ export type DeepLinkId =
| `${MetricsApp}:${MetricsLinkId}`
| `${ApmApp}:${ApmLinkId}`
| `${SyntheticsApp}:${SyntheticsLinkId}`
| `${ObltProfilingApp}:${ProfilingLinkId}`;
| `${ObltProfilingApp}:${ProfilingLinkId}`
| `${InventoryApp}:${InventoryLinkId}`;
2 changes: 1 addition & 1 deletion packages/deeplinks/observability/kibana.jsonc
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
{
"type": "shared-common",
"id": "@kbn/deeplinks-observability",
"owner": "@elastic/obs-ux-logs-team"
"owner": "@elastic/obs-ux-management-team"
}
2 changes: 1 addition & 1 deletion packages/kbn-es-types/src/search.ts
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ export type SearchHit<
? {
fields: Partial<Record<ValueTypeOfField<TFields>, unknown[]>>;
}
: {}) &
: { fields?: Record<string, unknown[]> }) &
(TDocValueFields extends DocValueFields
? {
fields: Partial<Record<ValueTypeOfField<TDocValueFields>, unknown[]>>;
Expand Down
1 change: 1 addition & 0 deletions packages/kbn-optimizer/limits.yml
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,7 @@ pageLoadAssetSize:
inspector: 148711
integrationAssistant: 19524
interactiveSetup: 80000
inventory: 27430
investigate: 17970
investigateApp: 91898
kibanaOverview: 56279
Expand Down
1 change: 1 addition & 0 deletions packages/kbn-server-route-repository-client/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@

export { createRepositoryClient } from './src/create_repository_client';
export { isHttpFetchError } from './src/is_http_fetch_error';
export { isRequestAbortedError } from './src/is_request_aborted_error';

export type {
DefaultClientOptions,
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/

import { createParser } from 'eventsource-parser';
import { Observable, throwError } from 'rxjs';

export interface StreamedHttpResponse {
response?: { body: ReadableStream<Uint8Array> | null | undefined };
}

class NoReadableStreamError extends Error {
constructor() {
super(`No readable stream found in response`);
}
}

export function isNoReadableStreamError(error: any): error is NoReadableStreamError {
return error instanceof NoReadableStreamError;
}

export function createObservableFromHttpResponse(
response: StreamedHttpResponse
): Observable<string> {
const rawResponse = response.response;

const body = rawResponse?.body;
if (!body) {
return throwError(() => {
throw new NoReadableStreamError();
});
}

return new Observable<string>((subscriber) => {
const parser = createParser((event) => {
if (event.type === 'event') {
subscriber.next(event.data);
}
});

const readStream = async () => {
const reader = body.getReader();
const decoder = new TextDecoder();

// Function to process each chunk
const processChunk = ({
done,
value,
}: ReadableStreamReadResult<Uint8Array>): Promise<void> => {
if (done) {
return Promise.resolve();
}

parser.feed(decoder.decode(value, { stream: true }));

return reader.read().then(processChunk);
};

// Start reading the stream
return reader.read().then(processChunk);
};

readStream()
.then(() => {
subscriber.complete();
})
.catch((error) => {
subscriber.error(error);
});
});
}
Original file line number Diff line number Diff line change
Expand Up @@ -11,28 +11,52 @@ import type { CoreSetup, CoreStart } from '@kbn/core-lifecycle-browser';
import {
RouteRepositoryClient,
ServerRouteRepository,
DefaultClientOptions,
formatRequest,
} from '@kbn/server-route-repository-utils';
import { httpResponseIntoObservable } from '@kbn/sse-utils-client';
import { from } from 'rxjs';
import { HttpFetchOptions, HttpFetchQuery, HttpResponse } from '@kbn/core-http-browser';
import { omit } from 'lodash';

export function createRepositoryClient<
TRepository extends ServerRouteRepository,
TClientOptions extends Record<string, any> = DefaultClientOptions
>(core: CoreStart | CoreSetup) {
TClientOptions extends HttpFetchOptions = {}
>(core: CoreStart | CoreSetup): RouteRepositoryClient<TRepository, TClientOptions> {
const fetch = (
endpoint: string,
params: { path?: Record<string, string>; body?: unknown; query?: HttpFetchQuery } | undefined,
options: TClientOptions
) => {
const { method, pathname, version } = formatRequest(endpoint, params?.path);

return core.http[method](pathname, {
...options,
body: params && params.body ? JSON.stringify(params.body) : undefined,
query: params?.query,
version,
});
};

return {
fetch: (endpoint, optionsWithParams) => {
const { params, ...options } = (optionsWithParams ?? { params: {} }) as unknown as {
params?: Partial<Record<string, any>>;
};
fetch: (endpoint, ...args) => {
const allOptions = args[0] ?? {};
const params = 'params' in allOptions ? (allOptions.params as Record<string, any>) : {};
const otherOptions = omit(allOptions, 'params') as TClientOptions;

const { method, pathname, version } = formatRequest(endpoint, params?.path);
return fetch(endpoint, params, otherOptions) as any;
},
stream: (endpoint, ...args) => {
const allOptions = args[0] ?? {};
const params = 'params' in allOptions ? (allOptions.params as Record<string, any>) : {};
const otherOptions = omit(allOptions, 'params') as TClientOptions;

return core.http[method](pathname, {
...options,
body: params && params.body ? JSON.stringify(params.body) : undefined,
query: params?.query,
version,
});
return from(
fetch(endpoint, params, {
...otherOptions,
asResponse: true,
rawResponse: true,
}) as Promise<HttpResponse>
).pipe(httpResponseIntoObservable()) as any;
},
} as { fetch: RouteRepositoryClient<TRepository, TClientOptions> };
};
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/

import { get } from 'lodash';

export function isRequestAbortedError(error: unknown): error is Error {
return get(error, 'name') === 'AbortError';
}
1 change: 1 addition & 0 deletions packages/kbn-server-route-repository-client/tsconfig.json
Original file line number Diff line number Diff line change
Expand Up @@ -17,5 +17,6 @@
"@kbn/server-route-repository-utils",
"@kbn/core-lifecycle-browser",
"@kbn/core-http-browser",
"@kbn/sse-utils-client",
]
}
Loading

0 comments on commit 98aa1ab

Please sign in to comment.