EP Meta Telemetry Perf (#104396)

* Add comments for other developers.

* Move OS infomation into meta key.

* Refmt endpoint metrics.

* Add helper funcs to batch sending.

* Add test to ensure opt in status.

* Add helpers test.

* Finish reshaping the document based on feedback.

* Add better type safety. Add policy package version to output.

* Fix sender implementation for aggregating EP datastreams.

* Fix type issues.

* Fix cadence inference + miss default agent id.

* Dynamically control search ranges for metrics + policy responses.

* Set back to 24h.

* Add comment for ignoring the default policy id.

* explicitly type the sub agg search query.

* Improve type safety.

* Add additional type safety + try/catch the last block.

* Remove unneeded optional chaining.

* Destructure host metrics.

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

x-pack/plugins/security_solution/server/lib/telemetry/diagnostic_task.ts

@@ -12,7 +12,7 @@ import {
   TaskManagerSetupContract,
   TaskManagerStartContract,
 } from '../../../../task_manager/server';
-import { getLastTaskExecutionTimestamp } from './helpers';
+import { getPreviousDiagTaskTimestamp } from './helpers';
 import { TelemetryEventsSender, TelemetryEvent } from './sender';
 
 export const TelemetryDiagTaskConstants = {
@@ -44,7 +44,7 @@ export class TelemetryDiagTask {
           return {
             run: async () => {
               const executeTo = moment().utc().toISOString();
-              const executeFrom = getLastTaskExecutionTimestamp(
+              const executeFrom = getPreviousDiagTaskTimestamp(
                 executeTo,
                 taskInstance.state?.lastExecutionTimestamp
               );

x-pack/plugins/security_solution/server/lib/telemetry/endpoint_task.test.ts

@@ -6,9 +6,11 @@
  */
 
 import { loggingSystemMock } from 'src/core/server/mocks';
+import { TaskStatus } from '../../../../task_manager/server';
 import { taskManagerMock } from '../../../../task_manager/server/mocks';
-import { TelemetryEndpointTask } from './endpoint_task';
-import { createMockTelemetryEventsSender } from './mocks';
+
+import { TelemetryEndpointTask, TelemetryEndpointTaskConstants } from './endpoint_task';
+import { createMockTelemetryEventsSender, MockTelemetryEndpointTask } from './mocks';
 
 describe('test', () => {
   let logger: ReturnType<typeof loggingSystemMock.createLogger>;
@@ -18,7 +20,7 @@ describe('test', () => {
   });
 
   describe('endpoint alert telemetry checks', () => {
-    test('the task can register', () => {
+    test('the endpoint task can register', () => {
       const telemetryEndpointTask = new TelemetryEndpointTask(
         logger,
         taskManagerMock.createSetup(),
@@ -48,4 +50,30 @@ describe('test', () => {
     await telemetryEndpointTask.start(mockTaskManagerStart);
     expect(mockTaskManagerStart.ensureScheduled).toHaveBeenCalled();
   });
+
+  test('endpoint task should not query elastic if telemetry is not opted in', async () => {
+    const mockSender = createMockTelemetryEventsSender(false);
+    const mockTaskManager = taskManagerMock.createSetup();
+    new MockTelemetryEndpointTask(logger, mockTaskManager, mockSender);
+
+    const mockTaskInstance = {
+      id: TelemetryEndpointTaskConstants.TYPE,
+      runAt: new Date(),
+      attempts: 0,
+      ownerId: '',
+      status: TaskStatus.Running,
+      startedAt: new Date(),
+      scheduledAt: new Date(),
+      retryAt: new Date(),
+      params: {},
+      state: {},
+      taskType: TelemetryEndpointTaskConstants.TYPE,
+    };
+    const createTaskRunner =
+      mockTaskManager.registerTaskDefinitions.mock.calls[0][0][TelemetryEndpointTaskConstants.TYPE]
+        .createTaskRunner;
+    const taskRunner = createTaskRunner({ taskInstance: mockTaskInstance });
+    await taskRunner.run();
+    expect(mockSender.fetchDiagnosticAlerts).not.toHaveBeenCalled();
+  });
 });