From e92154b891ef6362cec511e1371f8d9ff3007e89 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Bene=C5=A1?= Date: Fri, 6 Oct 2023 15:08:35 +0200 Subject: [PATCH] chore: `foundation/src/serialization` tech debt (#2722) Fixes #2593 I marked the `deserializeArrayFromVector` as deprecated because we only use it in `SiblingPath` and everywhere else we use `BufferReader`. --- .../foundation/src/serialize/deserializer.ts | 180 ------------------ .../foundation/src/serialize/free_funcs.ts | 26 +-- .../foundation/src/serialize/index.ts | 2 - .../src/serialize/serialize.test.ts | 89 --------- .../foundation/src/serialize/serializer.ts | 128 ------------- .../types/src/logs/function_l2_logs.ts | 6 +- .../types/src/logs/l2_block_l2_logs.ts | 4 +- yarn-project/types/src/logs/tx_l2_logs.ts | 4 +- .../types/src/logs/unencrypted_l2_log.ts | 4 +- 9 files changed, 14 insertions(+), 429 deletions(-) delete mode 100644 yarn-project/foundation/src/serialize/deserializer.ts delete mode 100644 yarn-project/foundation/src/serialize/serialize.test.ts delete mode 100644 yarn-project/foundation/src/serialize/serializer.ts diff --git a/yarn-project/foundation/src/serialize/deserializer.ts b/yarn-project/foundation/src/serialize/deserializer.ts deleted file mode 100644 index 0f62d0b10b0..00000000000 --- a/yarn-project/foundation/src/serialize/deserializer.ts +++ /dev/null @@ -1,180 +0,0 @@ -import { - deserializeArrayFromVector, - deserializeBigInt, - deserializeBool, - deserializeBufferFromVector, - deserializeInt32, - deserializeUInt32, -} from './free_funcs.js'; - -/** - * DeserializeFn is a type representing a deserialization function for a specific data type. The function takes - * a buffer and an offset as input, and returns an object containing the deserialized element of the data type and - * the number of bytes advanced in the buffer. This type is used to provide custom deserialization logic for arrays, - * objects or custom data types while working with the Deserializer class. - */ -export type DeserializeFn = ( - buf: Buffer, - offset: number, -) => { - /** - * The deserialized element of the specified data type. - */ - elem: T; - /** - * The number of bytes advanced in the buffer during deserialization. - */ - adv: number; -}; - -/** - * Deserializer class provides a set of methods to deserialize different data types from a buffer. - * It maintains an internal buffer and offset, updating the offset as it deserializes each data type. - * The class supports deserialization of various data types including boolean, integers, big integers, - * buffers, strings, dates, and arrays with custom deserialization functions. - * - * @example - * const deserializer = new Deserializer(buffer); - * const boolValue = deserializer.bool(); - * const intValue = deserializer.int32(); - * const bigIntValue = deserializer.bigInt(); - * const stringValue = deserializer.string(); - * const dateValue = deserializer.date(); - * const arrayValue = deserializer.deserializeArray(customDeserializeFn); - */ -export class Deserializer { - constructor(private buf: Buffer, private offset = 0) {} - - /** - * Deserialize a boolean value from the buffer at the current offset. - * Advances the internal offset by one byte after deserialization. - * Returns 'true' if the deserialized value is non-zero, otherwise returns 'false'. - * - * @returns The deserialized boolean value. - */ - public bool() { - return this.exec(deserializeBool) ? true : false; - } - - /** - * Deserialize a 32-bit unsigned integer from the buffer at the current offset. - * Advances the internal buffer offset by 4 after successful deserialization. - * The result is returned as a JavaScript number. - * - * @returns A 32-bit unsigned integer value. - */ - public uInt32() { - return this.exec(deserializeUInt32); - } - - /** - * Deserialize a 32-bit signed integer from the internal buffer. - * Reads 4 bytes from the current offset in the buffer and interprets them as a little-endian int32 value. - * Advances the internal offset by 4 bytes after successful deserialization. - * - * @returns The deserialized 32-bit signed integer value. - */ - public int32() { - return this.exec(deserializeInt32); - } - - /** - * Deserialize a BigInt from the buffer, taking into account the specified width. - * The method reads 'width' bytes from the buffer starting at the current offset and converts it to a BigInt. - * The offset is advanced by 'width' bytes after successful deserialization. - * - * @param width - The number of bytes to read from the buffer to construct the BigInt (default is 32). - * @returns The deserialized BigInt value. - */ - public bigInt(width = 32) { - return this.exec((buf: Buffer, offset: number) => deserializeBigInt(buf, offset, width)); - } - - /** - * Deserialize a variable-length byte array from the internal buffer. - * This method reads the length of the array and then extracts the corresponding bytes. - * It advances the internal offset by the number of bytes read, including the length prefix. - * - * @returns A Buffer instance containing the deserialized byte array. - */ - public vector() { - return this.exec(deserializeBufferFromVector); - } - - /** - * Extract a sub-buffer with the specified width, advancing the internal offset. - * The function slices the buffer from the current offset to the offset plus the provided width, - * and advances the internal offset by the width. This can be useful for working with fixed-width - * structures within the original buffer. - * - * @param width - The number of bytes to include in the extracted sub-buffer. - * @returns A sub-buffer containing the specified number of bytes from the original buffer. - */ - public buffer(width: number) { - const buf = this.buf.slice(this.offset, this.offset + width); - this.offset += width; - return buf; - } - - /** - * Deserialize a string from the internal buffer. - * It first deserializes a vector representing the UTF-8 encoded string from the buffer, - * and then converts it to a string. - * - * @returns The deserialized string. - */ - public string() { - return this.vector().toString(); - } - - /** - * Deserialize a Date object from the internal buffer. - * The date value is expected to be stored as a 64-bit BigInt representing the number of milliseconds since the Unix epoch. - * Advances the internal offset by 8 bytes after deserialization. - * - * @returns A Date instance representing the deserialized date value. - */ - public date() { - return new Date(Number(this.bigInt(8))); - } - - /** - * Deserialize an array of elements using the provided deserialization function. - * This method reads the serialized data from the buffer and deserializes each element in the array - * using the given 'fn' deserialization function. The returned array contains the deserialized elements - * in their original order. - * - * @param fn - The deserialization function to be applied on each element in the array. - * @returns An array containing the deserialized elements. - */ - public deserializeArray(fn: DeserializeFn) { - return this.exec((buf: Buffer, offset: number) => deserializeArrayFromVector(fn, buf, offset)); - } - - /** - * Executes the given deserialization function on this Deserializer's buffer and updates the internal offset. - * The DeserializeFn should take a Buffer and an offset as input, and return an object containing the deserialized - * element and the number of bytes advanced in the buffer. This method is useful for custom deserialization logic - * or implementing new deserialization functions. - * - * @typeparam T - The type of the deserialized element. - * @param fn - The deserialization function to execute. - * @returns The deserialized element of type T. - */ - public exec(fn: DeserializeFn): T { - const { elem, adv } = fn(this.buf, this.offset); - this.offset += adv; - return elem; - } - - /** - * Returns the current offset value in the Deserializer instance. - * The offset is updated as elements are deserialized from the buffer. - * It can be useful for tracking the position in the buffer during complex deserialization processes. - * - * @returns The current offset value as a number. - */ - public getOffset() { - return this.offset; - } -} diff --git a/yarn-project/foundation/src/serialize/free_funcs.ts b/yarn-project/foundation/src/serialize/free_funcs.ts index e85d5ac58b3..6f4242582c6 100644 --- a/yarn-project/foundation/src/serialize/free_funcs.ts +++ b/yarn-project/foundation/src/serialize/free_funcs.ts @@ -79,14 +79,11 @@ export function numToUInt8(n: number) { } /** - * Serialize a Buffer into a vector format by encoding the length of the buffer and concatenating it with the original buffer. - * The resulting vector consists of a 4-byte header containing the big-endian representation of the original buffer's length, followed by the original buffer. - * This function is useful when storing buffers as data structures with dynamic lengths and later deserializing them using 'deserializeBufferFromVector'. - * - * @param buf - The input Buffer to be serialized into a vector format. - * @returns A Buffer containing the serialized vector with the encoded length header. + * Adds a 4-byte byte-length prefix to a buffer. + * @param buf - The input Buffer to be prefixed + * @returns A Buffer with 4-byte byte-length prefix. */ -export function serializeBufferToVector(buf: Buffer) { +export function prefixBufferWithLength(buf: Buffer) { const lengthBuf = Buffer.alloc(4); lengthBuf.writeUInt32BE(buf.length, 0); return Buffer.concat([lengthBuf, buf]); @@ -131,20 +128,6 @@ export function serializeDate(date: Date) { return serializeBigInt(BigInt(date.getTime()), 8); } -/** - * Deserialize a buffer from a vector by reading the length from its first 4 bytes, and then extracting the contents of the buffer. - * The function returns an object containing the deserialized buffer as 'elem' and the number of bytes advanced ('adv') after deserialization. - * - * @param vector - The input buffer containing the serialized vector. - * @param offset - The starting position from where the deserialization should begin (default is 0). - * @returns An object with the deserialized buffer as 'elem' and the number of bytes advanced ('adv') after deserialization. - */ -export function deserializeBufferFromVector(vector: Buffer, offset = 0) { - const length = vector.readUInt32BE(offset); - const adv = 4 + length; - return { elem: vector.subarray(offset + 4, offset + adv), adv }; -} - /** * Deserialize a boolean value from a given buffer at the specified offset. * Reads a single byte at the provided offset in the buffer and returns @@ -225,6 +208,7 @@ export function serializeBufferArrayToVector(arr: Buffer[]) { * @param vector - The input buffer containing the serialized array. * @param offset - An optional starting position in the buffer for deserializing the array. * @returns An object containing the deserialized array and the total number of bytes used during deserialization (adv). + * @deprecated User BufferReader instead. */ export function deserializeArrayFromVector( deserialize: ( diff --git a/yarn-project/foundation/src/serialize/index.ts b/yarn-project/foundation/src/serialize/index.ts index 12312961a1a..355994a86ab 100644 --- a/yarn-project/foundation/src/serialize/index.ts +++ b/yarn-project/foundation/src/serialize/index.ts @@ -1,5 +1,3 @@ export * from './free_funcs.js'; -export * from './deserializer.js'; -export * from './serializer.js'; export * from './buffer_reader.js'; export * from './types.js'; diff --git a/yarn-project/foundation/src/serialize/serialize.test.ts b/yarn-project/foundation/src/serialize/serialize.test.ts deleted file mode 100644 index 32d0851069d..00000000000 --- a/yarn-project/foundation/src/serialize/serialize.test.ts +++ /dev/null @@ -1,89 +0,0 @@ -import { randomBytes } from '../crypto/index.js'; -import { Fr } from '../fields/fields.js'; -import { - deserializeArrayFromVector, - deserializeBufferFromVector, - deserializeField, - deserializeUInt32, - serializeBufferArrayToVector, - serializeBufferToVector, -} from './index.js'; - -describe('serialize', () => { - it('serialize buffer to vector and deserialize it back', () => { - const data = randomBytes(32); - const vector = serializeBufferToVector(data); - expect(vector.length).toBe(36); - - const recovered = deserializeBufferFromVector(vector); - expect(recovered.elem).toEqual(data); - expect(recovered.adv).toEqual(4 + 32); - - const paddedVector = Buffer.concat([randomBytes(10), vector, randomBytes(20)]); - const recovered2 = deserializeBufferFromVector(paddedVector, 10); - expect(recovered2.elem).toEqual(data); - expect(recovered2.adv).toEqual(4 + 32); - }); - - it('deserialize uint32', () => { - const uintBuf = Buffer.alloc(4); - uintBuf.writeUInt32BE(19, 0); - - const recovered = deserializeUInt32(uintBuf); - expect(recovered.elem).toBe(19); - expect(recovered.adv).toBe(4); - - const paddedBuf = Buffer.concat([randomBytes(10), uintBuf, randomBytes(20)]); - const recovered2 = deserializeUInt32(paddedBuf, 10); - expect(recovered2.elem).toBe(19); - expect(recovered2.adv).toBe(4); - }); - - it('deserialize field', () => { - const field = Fr.random(); - - const recovered = deserializeField(field.toBuffer()); - expect(recovered.elem).toEqual(field); - expect(recovered.adv).toBe(32); - - const paddedBuf = Buffer.concat([randomBytes(10), field.toBuffer(), randomBytes(20)]); - const recovered2 = deserializeField(paddedBuf, 10); - expect(recovered2.elem).toEqual(field); - expect(recovered2.adv).toBe(32); - }); - - it('serialize buffer array to vector and deserialize it back', () => { - // Array of uint32 - const uintArr = [7, 13, 16]; - const uintBufArr = uintArr.map(num => { - const uintBuf = Buffer.alloc(4); - uintBuf.writeUInt32BE(num, 0); - return uintBuf; - }); - const uintArrVec = serializeBufferArrayToVector(uintBufArr); - expect(uintArrVec.length).toBe(4 + 4 * 3); - - const recoveredUintArr = deserializeArrayFromVector(deserializeUInt32, uintArrVec); - expect(recoveredUintArr.elem).toEqual(uintArr); - expect(recoveredUintArr.adv).toEqual(4 + 4 * 3); - - const paddedUintArrVec = Buffer.concat([randomBytes(10), uintArrVec, randomBytes(20)]); - const recoveredUintArr2 = deserializeArrayFromVector(deserializeUInt32, paddedUintArrVec, 10); - expect(recoveredUintArr2.elem).toEqual(uintArr); - expect(recoveredUintArr2.adv).toEqual(4 + 4 * 3); - - // Array of field - const fieldArr = [Fr.random(), Fr.random(), Fr.random()]; - const fieldArrVec = serializeBufferArrayToVector(fieldArr.map(fr => fr.toBuffer())); - expect(fieldArrVec.length).toBe(4 + 32 * 3); - - const recoveredFieldArr = deserializeArrayFromVector(deserializeField, fieldArrVec); - expect(recoveredFieldArr.elem).toEqual(fieldArr); - expect(recoveredFieldArr.adv).toEqual(4 + 32 * 3); - - const paddedFieldVec = Buffer.concat([randomBytes(10), fieldArrVec, randomBytes(20)]); - const recoveredFieldArr2 = deserializeArrayFromVector(deserializeField, paddedFieldVec, 10); - expect(recoveredFieldArr2.elem).toEqual(fieldArr); - expect(recoveredFieldArr2.adv).toEqual(4 + 32 * 3); - }); -}); diff --git a/yarn-project/foundation/src/serialize/serializer.ts b/yarn-project/foundation/src/serialize/serializer.ts deleted file mode 100644 index 90eae6e171e..00000000000 --- a/yarn-project/foundation/src/serialize/serializer.ts +++ /dev/null @@ -1,128 +0,0 @@ -import { - boolToByte, - numToInt32BE, - numToUInt32BE, - serializeBigInt, - serializeBufferToVector, - serializeDate, -} from './free_funcs.js'; -import { serializeBufferArrayToVector } from './index.js'; - -/** - * The Serializer class provides a convenient and efficient way to serialize various data types into binary format. - * It supports serialization of primitive types (like boolean, signed/unsigned integers, BigInt), as well as more complex types (like Buffer, Date, and custom structures with their own 'toBuffer()' methods). - * The class maintains an internal buffer array that accumulates serialized data, allowing for easy concatenation and retrieval of the final serialized Buffer. - * This can be useful in various applications such as network communication, file storage, or other scenarios where binary data representation is needed. - */ -export class Serializer { - private buf: Buffer[] = []; - - constructor() {} - - /** - * Serialize a boolean value into a Buffer and append it to the internal buffer array. - * The serialized boolean will be stored as a single byte, where true is represented as 1 and false as 0. - * This method updates the Serializer's internal state and does not return any values. - * - * @param bool - The boolean value to be serialized. - */ - public bool(bool: boolean) { - this.buf.push(boolToByte(bool)); - } - - /** - * Encodes a given unsigned 32-bit integer into a Buffer and appends it to the internal buffer array. - * The provided number should be within the range of 0 and 2^32 - 1, inclusive. - * Throws an error if the input value is out of range or not a valid number. - * - * @param num - The unsigned 32-bit integer to be encoded and appended. - */ - public uInt32(num: number) { - this.buf.push(numToUInt32BE(num)); - } - - /** - * Serialize a signed 32-bit integer (int32) into the internal buffer. - * The number should be within the range of -2147483648 to 2147483647 inclusive. - * Throws an error if the input is not within the valid int32 range. - * - * @param num - The signed 32-bit integer to serialize. - */ - public int32(num: number) { - this.buf.push(numToInt32BE(num)); - } - - /** - * Serializes a BigInt into a Buffer and appends it to the internal buffer array. - * The given 'num' is treated as a signed integer and is serialized using - * little-endian byte order. This method is useful for efficiently storing - * large integer values that may not fit within the range of a standard number. - * - * @param num - The BigInt value to serialize. - */ - public bigInt(num: bigint) { - this.buf.push(serializeBigInt(num)); - } - - /** - * The given buffer is of variable length. Prefixes the buffer with its length. - * @param buf - The buffer to serialize as a variable-length vector. - */ - public vector(buf: Buffer) { - this.buf.push(serializeBufferToVector(buf)); - } - - /** - * Directly serializes a buffer that maybe of fixed, or variable length. - * It is assumed the corresponding deserialize function will handle variable length data, thus the length - * does not need to be prefixed here. - * If serializing a raw, variable length buffer, use vector(). - * @param buf - The buffer to serialize as a fixed-length array. - */ - public buffer(buf: Buffer) { - this.buf.push(buf); - } - - /** - * Serialize a string by first converting it to a buffer and then encoding its length as a prefix. - * The serialized string can be deserialized by reading the prefixed length and extracting the corresponding data. - * This method is useful for serializing strings of variable length in a consistent format. - * - * @param str - The input string to be serialized. - */ - public string(str: string) { - this.vector(Buffer.from(str)); - } - - /** - * Serialize a given Date instance into a Buffer and append it to the internal buffer list. - * The serialized date is stored as an 8-byte BigInt representing the number of milliseconds since the Unix epoch. - * This function facilitates serialization of JavaScript's built-in Date objects for subsequent data transmission or storage. - * - * @param date - The Date instance to be serialized. - */ - public date(date: Date) { - this.buf.push(serializeDate(date)); - } - - /** - * Returns the serialized Buffer object that was created by calling various serialization methods on this Serializer instance. - * The resulting buffer can be used for sending or storing serialized data in binary format. - * - * @returns A Buffer containing the serialized data. - */ - public getBuffer() { - return Buffer.concat(this.buf); - } - - /** - * Serializes an array of elements, where each element has a 'toBuffer()' method, into a single Buffer. - * The resulting buffer is prefixed with its length (number of elements), allowing for easy deserialization. - * This method is useful for serializing arrays of custom classes or data structures that have their own serialization logic. - * - * @param arr - The array of elements to be serialized. Each element must have a 'toBuffer()' method for serialization. - */ - public serializeArray(arr: T[]) { - this.buf.push(serializeBufferArrayToVector(arr.map((e: any) => e.toBuffer()))); - } -} diff --git a/yarn-project/types/src/logs/function_l2_logs.ts b/yarn-project/types/src/logs/function_l2_logs.ts index 9c68ea68182..b504490537e 100644 --- a/yarn-project/types/src/logs/function_l2_logs.ts +++ b/yarn-project/types/src/logs/function_l2_logs.ts @@ -1,6 +1,6 @@ import { sha256 } from '@aztec/foundation/crypto'; import { Point } from '@aztec/foundation/fields'; -import { BufferReader, serializeBufferToVector } from '@aztec/foundation/serialize'; +import { BufferReader, prefixBufferWithLength } from '@aztec/foundation/serialize'; import { randomBytes } from 'crypto'; @@ -22,8 +22,8 @@ export class FunctionL2Logs { * the resulting buffer is prefixed with 4 bytes for its total length. */ public toBuffer(): Buffer { - const serializedLogs = this.logs.map(buffer => serializeBufferToVector(buffer)); - return serializeBufferToVector(Buffer.concat(serializedLogs)); + const serializedLogs = this.logs.map(buffer => prefixBufferWithLength(buffer)); + return prefixBufferWithLength(Buffer.concat(serializedLogs)); } /** diff --git a/yarn-project/types/src/logs/l2_block_l2_logs.ts b/yarn-project/types/src/logs/l2_block_l2_logs.ts index 8f4824a4fde..9c8702bfb02 100644 --- a/yarn-project/types/src/logs/l2_block_l2_logs.ts +++ b/yarn-project/types/src/logs/l2_block_l2_logs.ts @@ -1,4 +1,4 @@ -import { BufferReader, serializeBufferToVector } from '@aztec/foundation/serialize'; +import { BufferReader, prefixBufferWithLength } from '@aztec/foundation/serialize'; import isEqual from 'lodash.isequal'; @@ -22,7 +22,7 @@ export class L2BlockL2Logs { public toBuffer(): Buffer { const serializedTxLogs = this.txLogs.map(logs => logs.toBuffer()); // Concatenate all serialized function logs into a single buffer and prefix it with 4 bytes for its total length. - return serializeBufferToVector(Buffer.concat(serializedTxLogs)); + return prefixBufferWithLength(Buffer.concat(serializedTxLogs)); } /** diff --git a/yarn-project/types/src/logs/tx_l2_logs.ts b/yarn-project/types/src/logs/tx_l2_logs.ts index e6b1a40b79f..98fb94d1be6 100644 --- a/yarn-project/types/src/logs/tx_l2_logs.ts +++ b/yarn-project/types/src/logs/tx_l2_logs.ts @@ -1,4 +1,4 @@ -import { BufferReader, serializeBufferToVector } from '@aztec/foundation/serialize'; +import { BufferReader, prefixBufferWithLength } from '@aztec/foundation/serialize'; import { FunctionL2Logs } from './function_l2_logs.js'; @@ -20,7 +20,7 @@ export class TxL2Logs { public toBuffer(): Buffer { const serializedFunctionLogs = this.functionLogs.map(logs => logs.toBuffer()); // Concatenate all serialized function logs into a single buffer and prefix it with 4 bytes for its total length. - return serializeBufferToVector(Buffer.concat(serializedFunctionLogs)); + return prefixBufferWithLength(Buffer.concat(serializedFunctionLogs)); } /** diff --git a/yarn-project/types/src/logs/unencrypted_l2_log.ts b/yarn-project/types/src/logs/unencrypted_l2_log.ts index 3ea2986ba2d..e4707f3ad34 100644 --- a/yarn-project/types/src/logs/unencrypted_l2_log.ts +++ b/yarn-project/types/src/logs/unencrypted_l2_log.ts @@ -1,5 +1,5 @@ import { AztecAddress, FunctionSelector } from '@aztec/circuits.js'; -import { BufferReader, serializeBufferToVector } from '@aztec/foundation/serialize'; +import { BufferReader, prefixBufferWithLength } from '@aztec/foundation/serialize'; import { randomBytes } from 'crypto'; @@ -35,7 +35,7 @@ export class UnencryptedL2Log { return Buffer.concat([ this.contractAddress.toBuffer(), this.selector.toBuffer(), - serializeBufferToVector(this.data), + prefixBufferWithLength(this.data), ]); }