L-839, L-840 Move data serializion logic to core (#98)

logtail · Nov 16, 2023 · 4327340 · 4327340
1 parent b433699
commit 4327340
Show file tree

Hide file tree

Showing 5 changed files with 105 additions and 208 deletions.
diff --git a/package.json b/package.json
@@ -6,6 +6,7 @@
   "private": true,
   "scripts": {
     "bootstrap": "lerna bootstrap",
+    "bootstrap-example": "rm -rf example-project/node_modules/@logtail && ln -s ../../packages example-project/node_modules/@logtail",
     "build": "lerna run build",
     "lint": "prettier -c \"packages/**\" \"example-project/**\"",
     "lint:save": "prettier --write \"packages/**\" \"example-project/**\"",

diff --git a/packages/core/src/base.ts b/packages/core/src/base.ts
@@ -196,6 +196,16 @@ class Logtail {
     level: ILogLevel = LogLevel.Info,
     context: TContext = {} as TContext,
   ): Promise<ILogtailLog & TContext> {
+    // Wrap context in an object, if it's not already
+    if (typeof context !== "object") {
+      const wrappedContext: unknown = { extra: context };
+      context = wrappedContext as TContext;
+    }
+    if (context instanceof Error) {
+      const wrappedContext: unknown = { error: context };
+      context = wrappedContext as TContext;
+    }
+
     if (this._options.sendLogsToConsoleOutput) {
       switch (level) {
         case "debug":
@@ -234,28 +244,10 @@ class Logtail {
 
       // Add initial context
       ...context,
-    };
-
-    // Determine the type of message...
-
-    // Is this an error?
-    if (message instanceof Error) {
-      log = {
-        // Add stub
-        ...log,
 
-        // Serialize the error and add to log
-        ...serializeError(message),
-      };
-    } else {
-      log = {
-        // Add stub
-        ...log,
-
-        // Add string message
-        message,
-      };
-    }
+      // Add string message or serialized error
+      ...(message instanceof Error ? serializeError(message) : { message }),
+    };
 
     let transformedLog = log as ILogtailLog | null;
     for (const middleware of this._middleware) {
@@ -267,6 +259,12 @@ class Logtail {
       transformedLog = newTransformedLog;
     }
 
+    // Manually serialize the log data
+    transformedLog = this.serialize(
+      transformedLog,
+      this._options.contextObjectMaxDepth,
+    );
+
     if (!this._options.sendLogsToBetterStack) {
       // Return the resulting log before sending it
       return transformedLog as ILogtailLog & TContext;
@@ -297,6 +295,82 @@ class Logtail {
     return transformedLog as ILogtailLog & TContext;
   }
 
+  private serialize(
+    value: any,
+    maxDepth: number,
+    visitedObjects: WeakSet<any> = new WeakSet(),
+  ): any {
+    if (
+      value === null ||
+      typeof value === "boolean" ||
+      typeof value === "number" ||
+      typeof value === "string"
+    ) {
+      return value;
+    } else if (value instanceof Date) {
+      // Date instances can be invalid & toISOString() will fail
+      if (isNaN(value.getTime())) {
+        return value.toString();
+      }
+
+      return value.toISOString();
+    } else if (value instanceof Error) {
+      return serializeError(value);
+    } else if (
+      (typeof value === "object" || Array.isArray(value)) &&
+      (maxDepth < 1 || visitedObjects.has(value))
+    ) {
+      if (visitedObjects.has(value)) {
+        if (this._options.contextObjectCircularRefWarn) {
+          console.warn(
+            `[Logtail] Found a circular reference when serializing logs. Please do not use circular references in your logs.`,
+          );
+        }
+        return "<omitted circular reference>";
+      }
+      if (this._options.contextObjectMaxDepthWarn) {
+        console.warn(
+          `[Logtail] Max depth of ${this._options.contextObjectMaxDepth} reached when serializing logs. Please do not use excessive object depth in your logs.`,
+        );
+      }
+      return `<omitted context beyond configured max depth: ${this._options.contextObjectMaxDepth}>`;
+    } else if (Array.isArray(value)) {
+      visitedObjects.add(value);
+      const serializedArray = value.map(item =>
+        this.serialize(item, maxDepth - 1, visitedObjects),
+      );
+      visitedObjects.delete(value);
+
+      return serializedArray;
+    } else if (typeof value === "object") {
+      const serializedObject: { [key: string]: any } = {};
+
+      visitedObjects.add(value);
+
+      Object.entries(value).forEach(item => {
+        const key = item[0];
+        const value = item[1];
+
+        const serializedValue = this.serialize(
+          value,
+          maxDepth - 1,
+          visitedObjects,
+        );
+        if (serializedValue !== undefined) {
+          serializedObject[key] = serializedValue;
+        }
+      });
+
+      visitedObjects.delete(value);
+
+      return serializedObject;
+    } else if (typeof value === "undefined") {
+      return undefined;
+    } else {
+      return `<omitted unserializable ${typeof value}>`;
+    }
+  }
+
   /**
    *
    * Debug level log, to be synced with Better Stack

diff --git a/packages/edge/src/edge.test.ts b/packages/edge/src/edge.test.ts
@@ -61,13 +61,19 @@ describe("edge tests", () => {
 
     const message: string = String(Math.random());
     const expectedLog = getRandomLog(message);
-    const edge = new Edge("valid source token", { throwExceptions: true });
+    const edge = new Edge("valid source token", {
+      throwExceptions: true,
+      warnAboutMissingExecutionContext: false,
+    });
 
     edge.setSync(async logs => logs);
 
     const echoedLog = await edge.log(message, LogLevel.Info, circularContext);
     expect(echoedLog.message).toEqual(expectedLog.message);
     expect((console.warn as Mock).mock.calls).toHaveLength(1);
+    expect((console.warn as Mock).mock.calls[0][0]).toBe(
+      "[Logtail] Found a circular reference when serializing logs. Please do not use circular references in your logs.",
+    );
   });
 
   it("should contain context info", async () => {

diff --git a/packages/edge/src/edge.ts b/packages/edge/src/edge.ts
@@ -1,5 +1,4 @@
 import { encode } from "@msgpack/msgpack";
-import { serializeError } from "serialize-error";
 
 import {
   Context,
@@ -72,16 +71,6 @@ export class Edge extends Base {
     context: any = {} as TContext,
     ctx?: ExecutionContext,
   ): Promise<ILogtailLog & TContext> {
-    // Wrap context in an object, if it's not already
-    if (typeof context !== "object") {
-      const wrappedContext: unknown = { extra: context };
-      context = wrappedContext as TContext;
-    }
-    if (context instanceof Error) {
-      const wrappedContext: unknown = { error: context };
-      context = wrappedContext as TContext;
-    }
-
     // Process/sync the log, per `Base` logic
     const stackContext = getStackContext(this);
     context = { ...stackContext, ...context };
@@ -143,93 +132,12 @@ export class Edge extends Base {
   }
 
   private encodeAsMsgpack(logs: ILogtailLog[]): Uint8Array {
-    const maxDepth = this._options.contextObjectMaxDepth;
-    const logsWithISODateFormat = logs.map(log => ({
-      ...this.sanitizeForEncoding(log, maxDepth),
-      dt: log.dt.toISOString(),
-    }));
-    const encoded = encode(logsWithISODateFormat);
+    const encoded = encode(logs);
 
     return new Uint8Array(
       encoded.buffer,
       encoded.byteOffset,
       encoded.byteLength,
     );
   }
-
-  private sanitizeForEncoding(
-    value: any,
-    maxDepth: number,
-    visitedObjects: WeakSet<any> = new WeakSet(),
-  ): any {
-    if (
-      value === null ||
-      typeof value === "boolean" ||
-      typeof value === "number" ||
-      typeof value === "string"
-    ) {
-      return value;
-    } else if (value instanceof Date) {
-      // Date instances can be invalid & toISOString() will fail
-      if (isNaN(value.getTime())) {
-        return value.toString();
-      }
-
-      return value.toISOString();
-    } else if (value instanceof Error) {
-      return serializeError(value);
-    } else if (
-      (typeof value === "object" || Array.isArray(value)) &&
-      (maxDepth < 1 || visitedObjects.has(value))
-    ) {
-      if (visitedObjects.has(value)) {
-        if (this._options.contextObjectCircularRefWarn) {
-          console.warn(
-            `[Logtail] Found a circular reference when serializing logs. Please do not use circular references in your logs.`,
-          );
-        }
-        return "<omitted circular reference>";
-      }
-      if (this._options.contextObjectMaxDepthWarn) {
-        console.warn(
-          `[Logtail] Max depth of ${this._options.contextObjectMaxDepth} reached when serializing logs. Please do not use excessive object depth in your logs.`,
-        );
-      }
-      return `<omitted context beyond configured max depth: ${this._options.contextObjectMaxDepth}>`;
-    } else if (Array.isArray(value)) {
-      visitedObjects.add(value);
-      const sanitizedArray = value.map(item =>
-        this.sanitizeForEncoding(item, maxDepth - 1, visitedObjects),
-      );
-      visitedObjects.delete(value);
-
-      return sanitizedArray;
-    } else if (typeof value === "object") {
-      const logClone: { [key: string]: any } = {};
-
-      visitedObjects.add(value);
-
-      Object.entries(value).forEach(item => {
-        const key = item[0];
-        const value = item[1];
-
-        const result = this.sanitizeForEncoding(
-          value,
-          maxDepth - 1,
-          visitedObjects,
-        );
-        if (result !== undefined) {
-          logClone[key] = result;
-        }
-      });
-
-      visitedObjects.delete(value);
-
-      return logClone;
-    } else if (typeof value === "undefined") {
-      return undefined;
-    } else {
-      return `<omitted unserializable ${typeof value}>`;
-    }
-  }
 }