Add common log values to the query

services/api/src/agent/prompts.ts

@@ -150,6 +150,32 @@ export const prefix = {
 
   Begin!
   
+  Log records:
+  {logRecords}
+  `,
+  filterHighCardinalityFields: `
+  Given some log records and their fields, return the fields that don't have a high cardinality.
+  Meaning, exclude fields which seem to have a lot of different values. This usually means fields
+  such as user ids, session ids, messages, log body, timestamps, etc.
+
+  Your output should be in JSON format which contains just one key called "fields" and its value should be the array of fields that don't have a high cardinality.
+
+  For example, given the following log records:
+  \`\`\`json
+  [
+    {{"userId": 123, "timestamp": "2021-01-01T00:00:00Z", "message": "User logged in", "service": "demo-app"}},
+    {{"userId": 456, "timestamp": "2021-01-01T00:00:00Z", "message": "User logged in", "service": "demo-app"}},
+    {{"userId": 789, "timestamp": "2021-01-01T00:00:00Z", "message": "User logged in", "service": "demo-app"}}
+  ]
+  \`\`\`
+
+  The output should be:
+  \`\`\`json
+  {{"fields": ["service"]}}
+  \`\`\`
+
+  That's it! Begin!
+
   Log records:
   {logRecords}
   `,
@@ -195,3 +221,7 @@ export const investigationLeanTemplate = PromptTemplate.fromTemplate(
 export const extractLogStructureKeysPrompt = PromptTemplate.fromTemplate(
   prefix.extractLogStructureKeys,
 );
+
+export const filterHighCardinalityFieldsPrompt = PromptTemplate.fromTemplate(
+  prefix.filterHighCardinalityFields,
+);

services/api/src/agent/tools/coralogix/constants.ts

@@ -106,6 +106,10 @@ Whitespace between operators is ignored, supporting multiline queries.
   \`\`\`shell
   source logs
   \`\`\`
+- **Fetch all logs based on subsystem name**
+  \`\`\`shell
+  source logs | filter $l.subsystemname == "dlp"
+  \`\`\`
 - **Fetch all logs that contain 500 in their msg **
   \`\`\`shell
   source logs | filter msg.contains('500')

services/api/src/agent/tools/coralogix/logs_expert.ts

@@ -4,7 +4,9 @@ import { PromptTemplate } from "langchain/prompts";
 import { CoralogixIntegration } from "@merlinn/db";
 import { DATAPRIME_CHEATSHEET } from "./constants";
 import {
+  filterHighCardinalityFields,
   getCommonLogFields,
+  getCommonLogValues,
   getPrettyLogAnalysis,
   getPrettyLogSample,
   limitLogs,
@@ -19,6 +21,16 @@ import {
 } from "../../../utils/dates";
 import { RunContext } from "../../../agent/types";
 
+// const TEMP_COMMONG_TAGS = `
+// Here are the common fields that you can use in your query (they were taken from the environment itself):\n
+
+// resource.attributes.cloud_account_id, resource.attributes.cloud_availability_zone, resource.attributes.cloud_platform, resource.attributes.cloud_provider, resource.attributes.cloud_region, resource.attributes.cx_otel_integration_name, resource.attributes.host_id, resource.attributes.host_image_id, resource.attributes.host_name, resource.attributes.host_type, resource.attributes.k8s_cluster_name, resource.attributes.k8s_container_name, resource.attributes.k8s_container_restart_count, resource.attributes.k8s_deployment_name, resource.attributes.k8s_namespace_name, resource.attributes.k8s_node_name, resource.attributes.k8s_pod_name, resource.attributes.k8s_pod_uid, resource.attributes.os_type, resourceSchemaUrl, attributes.cluster_name, attributes.log_file_path, attributes.log_iostream, attributes.logtag, attributes.time, observedTimeUnixNano, timeUnixNano, date, level, service, controller, threadId, caller, message, status, time, url, method, requestAborted, tenantId, userId, correlationId, clientIp, platform, body, PRIORITY, SYSLOG_FACILITY, _UID, _GID, _CAP_EFFECTIVE, _BOOT_ID, _MACHINE_ID, _HOSTNAME, _RUNTIME_SCOPE, _SYSTEMD_SLICE, _TRANSPORT, _STREAM_ID, SYSLOG_IDENTIFIER, _PID, _COMM, _EXE, _CMDLINE, _SELINUX_CONTEXT, _SYSTEMD_CGROUP, _SYSTEMD_UNIT, _SYSTEMD_INVOCATION_ID, MESSAGE, exception, errorType, DeviceId, PatchProperties, FieldKey, AttemptedDefaultValue, FieldType, Query, connectorId, connectorGroupNfId, connectorsCount, zitiConnectorsCount, @timestamp, msg, run_time, source, name, id, AuthenticationScheme, EventId, EventName, DbContextType, fileName, queryId, connectionOpenedTime, queryExecutionTime, queryReadResultsTime, timezoneSetTime, records, ApplicationGroupId, RequestSignature, RequestUrl, startTime, categoryName, level.level, level.levelStr, level.colour, context.TENANT, context.DOCID, context.USERID, pid, query, endTime, limit, parameters, FailureMessage, applicationId, connectionType, duration, httpVersion, isConnectivityCheck, requestedHost, routeId, routeProvider, statusCode, uri, userAgent\n\n
+// `;
+
+// const TEMP_LOG_SAMPLE = `
+// resource.attributes.cloud_account_id: 636375568718\nresource.attributes.cloud_availability_zone: us-east-1c\nresource.attributes.cloud_platform: aws_ec2\nresource.attributes.cloud_provider: aws\nresource.attributes.cloud_region: us-east-1\nresource.attributes.cx_otel_integration_name: coralogix-integration-helm\nresource.attributes.host_id: i-0c74fd375858e256b\nresource.attributes.host_image_id: ami-065265c6ca4531afa\nresource.attributes.host_name: ip-10-21-70-83.ec2.internal\nresource.attributes.host_type: m6g.xlarge\nresource.attributes.k8s_cluster_name: staging-us-east-1\nresource.attributes.k8s_container_name: identity\nresource.attributes.k8s_container_restart_count: 0\nresource.attributes.k8s_deployment_name: identity\nresource.attributes.k8s_namespace_name: dev-0\nresource.attributes.k8s_node_name: ip-10-21-70-83.ec2.internal\nresource.attributes.k8s_pod_name: identity-d4f8c7575-9msqp\nresource.attributes.k8s_pod_uid: 695421c8-e8ca-473e-ad7f-bb0cb0942a46\nresource.attributes.os_type: linux\nresourceSchemaUrl: https://opentelemetry.io/schemas/1.6.1\nattributes.cluster_name: staging-us-east-1\nattributes.log_file_path: /var/log/pods/dev-0_identity-d4f8c7575-9msqp_695421c8-e8ca-473e-ad7f-bb0cb0942a46/identity/0.log\nattributes.log_iostream: stdout\nattributes.logtag: F\nattributes.time: 2024-08-28T12:59:58.578342483Z\nobservedTimeUnixNano: 1724849998635745800\ntimeUnixNano: 1724849998578342400\ndate: 2024-08-28 12:59:58.5742\nlevel: INFO\nservice: Identity\ncontroller: InternalUsersData\nthreadId: 11\ncaller: ExecutionContext.RunInternal\nmessage: Request => Status: 200, Time: 4ms, Url: "http://identity/internal/api/v1/InternalUsersData?userIds=auth0%7C66cea1b652764c475e7d1d1c&userType=1", Method: "GET", tenantId: "automation-118f4063-bc6b-4e4f-92a6-b18e8adf78c4", userId: "auth0|66cea1b5b822319b3984c3ce", requestAborted: false\nstatus: 200\ntime: 4\nurl: http://identity/internal/api/v1/InternalUsersData?userIds=auth0%7C66cea1b652764c475e7d1d1c&userType=1\nmethod: GET\nrequestAborted: false\ntenantId: automation-118f4063-bc6b-4e4f-92a6-b18e8adf78c4\nuserId: auth0|66cea1b5b822319b3984c3ce\ncorrelationId: e0ab7c2a-e5d3-4f51-9c6b-1fc14d8f5028\nclientIp: 10.21.52.41\nplatform: Browser\n\n--------------------------------------\n\nresource.attributes.cloud_account_id: 636375568718\nresource.attributes.cloud_availability_zone: us-east-1a\nresource.attributes.cloud_platform: aws_ec2\nresource.attributes.cloud_provider: aws\nresource.attributes.cloud_region: us-east-1\nresource.attributes.cx_otel_integration_name: coralogix-integration-helm\nresource.attributes.host_id: i-0d1f249ffc6cb2286\nresource.attributes.host_image_id: ami-065265c6ca4531afa\nresource.attributes.host_name: ip-10-21-18-97.ec2.internal\nresource.attributes.host_type: m6g.xlarge\nresource.attributes.k8s_cluster_name: staging-us-east-1\nresource.attributes.k8s_container_name: extensions\nresource.attributes.k8s_container_restart_count: 0\nresource.attributes.k8s_deployment_name: extensions\nresource.attributes.k8s_namespace_name: default\nresource.attributes.k8s_node_name: ip-10-21-18-97.ec2.internal\nresource.attributes.k8s_pod_name: extensions-b8cbd7c88-gm8fg\nresource.attributes.k8s_pod_uid: c0d415e8-ce20-445b-bebc-a4e812fa185b\nresource.attributes.os_type: linux\nresourceSchemaUrl: https://opentelemetry.io/schemas/1.6.1\nattributes.cluster_name: staging-us-east-1\nattributes.log_file_path: /var/log/pods/default_extensions-b8cbd7c88-gm8fg_c0d415e8-ce20-445b-bebc-a4e812fa185b/extensions/0.log\nattributes.log_iostream: stdout\nattributes.logtag: F\nattributes.time: 2024-08-28T12:59:58.895803912Z\nobservedTimeUnixNano: 1724849999077313500\ntimeUnixNano: 1724849998895804000\ndate: 2024-08-28 12:59:58.8879\nlevel: DEBUG\nservice: Extensions\nthreadId: 40\ncaller: ExecutionContext.RunInternal\nmessage: Metadata or package entity not found, skipping risk calculation for now
+// `;
+
 const PROMPT_TEMPLATE = `
 You are a Coralogix logs expert. Given a request in natural language, you should generate {nQueries} queries in a DataPrime syntax.
 DataPrime is Coralogix's proprietary query language that allows you to query logs in a more structured way.
@@ -29,6 +41,9 @@ Here is a cheatsheet of DataPrime query language:
 Here are the common fields that you can use in your query (they were taken from the environment itself):
 {commonFields}
 
+Here are the common values that you can use in your query (they were taken from the environment itself):
+{commonValues}
+
 Here is a sample of logs so you'd know how they look like:
 {logSample}
 
@@ -85,16 +100,30 @@ export default async function (
   return new DynamicStructuredTool({
     name: "logs_expert_tool",
     description: TOOL_DESCRIPTION,
-    func: async ({ request }) => {
+    func: async ({ request, timeframe }) => {
       try {
-        const commonFields = await getCommonLogFields(logsKey, region);
         const logSample = await getPrettyLogSample(logsKey, region, 2);
+        const commonFields = await getCommonLogFields(logsKey, region);
+        const filteredFields = await filterHighCardinalityFields(logSample);
+        const commonValues = await getCommonLogValues(
+          filteredFields,
+          logsKey,
+          region,
+          20,
+        );
+
+        const prettyCommonFields = commonFields.join("\n");
+        const prettyCommonValues = Object.entries(commonValues)
+          .map(([key, value]) => `${key}: ${value.join(", ")}`)
+          .join("\n");
+
         const prompt = await PromptTemplate.fromTemplate(
           PROMPT_TEMPLATE,
         ).format({
           nQueries: 3,
           cheatsheet: DATAPRIME_CHEATSHEET,
-          commonFields: commonFields.join(", "),
+          commonFields: prettyCommonFields,
+          commonValues: prettyCommonValues,
           logSample: logSample,
           request,
         });
@@ -106,7 +135,7 @@ export default async function (
           throw new Error("No queries generated");
         }
 
-        const [amount, scale] = timeframe2values["Last 2 days"];
+        const [amount, scale] = timeframe2values[timeframe];
         const startDate = getTimestamp({ amount, scale });
         const endDate = new Date().toISOString();
         const results = (
@@ -116,7 +145,7 @@ export default async function (
                 const { analysis, parsedLogs } = await getPrettyLogAnalysis({
                   query,
                   integration,
-                  timeframe: Timeframe.Last2Days,
+                  timeframe,
                 });
 
                 if (
@@ -178,6 +207,27 @@ export default async function (
     },
     schema: z.object({
       request: z.string().describe("The request to be used with Coralogix."),
+      timeframe: z
+        .enum([
+          Timeframe.Last1Minute,
+          Timeframe.Last2Minutes,
+          Timeframe.Last5Minutes,
+          Timeframe.Last15Minutes,
+          Timeframe.Last30Minutes,
+          Timeframe.Last1Hour,
+          Timeframe.Last2Hours,
+          Timeframe.Last6Hours,
+          Timeframe.Last12Hours,
+          Timeframe.Last24Hours,
+          Timeframe.Last2Days,
+          Timeframe.Last3Days,
+          Timeframe.Last5Days,
+          Timeframe.Last7Days,
+        ])
+        .describe(
+          "The period for which you wish to search the logs. Default is last 24 hours.",
+        )
+        .default(Timeframe.Last24Hours),
     }),
   });
 }

services/api/src/agent/tools/coralogix/utils.ts

@@ -7,7 +7,10 @@ import {
   timeframe2values,
 } from "../../../utils/dates";
 import { CoralogixClient } from "../../../clients/coralogix";
-import { extractLogStructureKeysPrompt } from "../../../agent/prompts";
+import {
+  extractLogStructureKeysPrompt,
+  filterHighCardinalityFieldsPrompt,
+} from "../../../agent/prompts";
 import { JsonOutputParser } from "langchain/schema/output_parser";
 import { chatModel } from "../../../agent/model";
 
@@ -54,25 +57,45 @@ export const getCommonLogFields = async (
 };
 
 export const getCommonLogValues = async (
-  field: string,
+  fields: string[],
   apiKey: string,
   region: CoralogixRegionKey,
+  limit: number = 20,
 ) => {
   const startDate = String(getTimestamp({ amount: 7, scale: "days" }));
   const endDate = String(getTimestamp({}));
 
   const client = new CoralogixClient({ logsKey: apiKey }, region);
-  const query = `source logs | distinct ${field} | limit 100`;
+  const query = `source logs | distinct ${fields.join(", ")} | limit 1000`;
   const { result } = await client.getLogs({
     query,
     startDate,
     endDate,
   });
 
-  const values = result.results.map(
-    (obj) => Object.values(JSON.parse(obj.userData))[0],
+  const values = result.results.reduce(
+    (total, obj) => {
+      const data = JSON.parse(obj.userData);
+      const keys = Object.keys(data);
+      keys.forEach((key) => {
+        if (total[key]) {
+          if (total[key].size < limit) {
+            total[key].add(data[key]);
+          }
+        } else {
+          total[key] = new Set([data[key]]);
+        }
+      });
+      return total;
+    },
+    {} as Record<string, Set<string>>,
   );
-  return values;
+  // Transform sets to lists
+  const transformedValues: Record<string, string[]> = {};
+  for (const [key, set] of Object.entries(values)) {
+    transformedValues[key] = Array.from(set);
+  }
+  return transformedValues;
 };
 
 export const getLogSample = async (
@@ -124,6 +147,25 @@ interface LogCluster extends Record<string, unknown> {
   Percentage: number;
 }
 
+export async function filterHighCardinalityFields(logSample: string) {
+  const queriesPrompt = await filterHighCardinalityFieldsPrompt.format({
+    logRecords: logSample,
+  });
+  const parser = new JsonOutputParser();
+  try {
+    const { content } = await chatModel.invoke(queriesPrompt);
+    const { fields } = await parser.parse(content as string);
+    if (!fields) {
+      throw new Error("Failed to extract log structure keys");
+    }
+    return fields;
+  } catch (error) {
+    console.error("Error generating queries", error);
+    console.error("invalid logRecords", logSample);
+    throw error;
+  }
+}
+
 interface ParseLogsResponse {
   clusters: LogCluster[];
 }
@@ -282,6 +324,11 @@ export async function getPrettyLogAnalysis({
   return { analysis, parsedLogs };
 }
 
+// export async function getQueriesHistory() {
+//   const limit = 30;
+//   const query = `source logs | filter $l.subsystemname == 'dataprime-api' && action_details.operation.operation_payload.tracingMetadata.queryText != null && action_details.operation.operation_payload.tracingMetadata.queryText != '' && !action_details.operation.operation_payload.tracingMetadata.queryText.contains('action_details.operation.operation_payload.tracingMetadata.queryText') | choose action_details.operation.operation_payload.tracingMetadata.queryText | limit ${limit}`;
+// }
+
 export function limitLogs(logsStr: string, limit = 10000) {
   return logsStr.slice(0, limit);
 }

services/api/src/clients/coralogix/constants.ts

@@ -79,4 +79,42 @@ export const domains: Record<string, CoralogixDomain> = {
       apiURL: "https://api.cx498.coralogix.com/api/v1/external/alerts",
     },
   },
+  opensearch: {
+    EU1: {
+      domain: "coralogix.com",
+      region: "eu-west-1",
+      description: "[EU1 – Ireland]",
+      apiURL: "https://api.coralogix.com/data/os-api",
+    },
+    AP1: {
+      domain: "coralogix.in",
+      region: "ap-south1",
+      description: "[AP1 – India]",
+      apiURL: "https://api.app.coralogix.in/data/os-api",
+    },
+    US1: {
+      domain: "coralogix.us",
+      region: "us-east2",
+      description: "[US1 – Ohio]",
+      apiURL: "https://api.coralogix.us/data/os-api",
+    },
+    EU2: {
+      domain: "eu2.coralogix.com",
+      region: "eu-north-1",
+      description: "[EU2 – Stockholm]",
+      apiURL: "https://api.eu2.coralogix.com/data/os-api",
+    },
+    AP2: {
+      domain: "coralogixsg.com",
+      region: "ap-southeast-1",
+      description: "[AP2 – Singapore]",
+      apiURL: "https://api.coralogixsg.com/data/os-api",
+    },
+    US2: {
+      domain: "cx498.coralogix.com",
+      region: "us-west-2",
+      description: "[US2 – Oregon]",
+      apiURL: "https://api.cx498.coralogix.com/data/os-api",
+    },
+  },
 };