From 9762dbe5554d894ddc3c19cacd4e97942a4e9300 Mon Sep 17 00:00:00 2001 From: Alex Gherghisan Date: Mon, 4 Mar 2024 09:17:25 +0000 Subject: [PATCH] feat: store partial notes in pxe --- .../aztec-nr/aztec/src/note/lifecycle.nr | 18 +-- .../aztec-nr/aztec/src/note/note_header.nr | 4 - .../aztec-nr/aztec/src/note/note_interface.nr | 7 +- .../aztec/src/state_vars/private_set.nr | 6 +- .../contracts/token_contract/src/main.nr | 25 +---- .../token_contract/src/types/balances_map.nr | 13 ++- .../token_contract/src/types/token_note.nr | 37 ++++++- .../aztec.js/src/wallet/base_wallet.ts | 9 ++ .../circuit-types/src/interfaces/pxe.ts | 15 +++ yarn-project/circuit-types/src/stats/stats.ts | 4 + .../src/e2e_partial_token_notes.test.ts | 81 +++++++------- .../pxe/src/database/kv_pxe_database.ts | 39 +++++++ .../pxe/src/database/partial_note_dao.ts | 50 +++++++++ yarn-project/pxe/src/database/pxe_database.ts | 19 ++++ .../src/note_processor/chopped_note_error.ts | 20 ++++ .../pxe/src/note_processor/note_processor.ts | 103 +++++++++++++++++- .../src/note_processor/produce_note_dao.ts | 37 ++++--- .../pxe/src/pxe_service/pxe_service.ts | 51 +++++++++ .../pxe/src/synchronizer/synchronizer.ts | 34 +++++- 19 files changed, 464 insertions(+), 108 deletions(-) create mode 100644 yarn-project/pxe/src/database/partial_note_dao.ts create mode 100644 yarn-project/pxe/src/note_processor/chopped_note_error.ts diff --git a/noir-projects/aztec-nr/aztec/src/note/lifecycle.nr b/noir-projects/aztec-nr/aztec/src/note/lifecycle.nr index 7553db9e6121..0f59c0e68bd6 100644 --- a/noir-projects/aztec-nr/aztec/src/note/lifecycle.nr +++ b/noir-projects/aztec-nr/aztec/src/note/lifecycle.nr @@ -1,6 +1,6 @@ use crate::context::{PrivateContext, PublicContext}; use crate::note::{ - note_header::NoteHeader, note_interface::NoteInterface, + note_header::NoteHeader, note_interface::{NoteInterface, CompletableNoteInterface}, utils::{compute_note_hash_for_insertion, compute_note_hash_for_consumption, compute_partial_note_hash} }; use crate::oracle::notes::{notify_created_note, notify_nullified_note}; @@ -43,7 +43,7 @@ pub fn create_partial_note( storage_slot: Field, note: &mut Note, broadcast: bool -) -> Field where Note: NoteInterface { +) -> Field where Note: NoteInterface + CompletableNoteInterface { let contract_address = (*context).this_address(); let mut header = NoteHeader { contract_address, storage_slot, nonce: 0, is_transient: true, partial_note_hash: 0 }; @@ -57,7 +57,7 @@ pub fn create_partial_note( Note::set_header(note, header); if broadcast { - Note::broadcast(*note, context, storage_slot); + CompletableNoteInterface::broadcast_partial_data(*note, context, storage_slot); } partial_note_hash @@ -66,8 +66,9 @@ pub fn create_partial_note( pub fn complete_partial_note_from_public( context: &mut PublicContext, partial_note_hash: Field, - note: &mut Note -) where Note: NoteInterface { + note: &mut Note, + broadcast: bool +) where Note: NoteInterface + CompletableNoteInterface { let contract_address = (*context).this_address(); let header = NoteHeader { contract_address, storage_slot: 0, nonce: 0, is_transient: true, partial_note_hash }; @@ -76,10 +77,11 @@ pub fn complete_partial_note_from_public( // As `is_transient` is true, this will compute the inner note hsah let note_hash = compute_note_hash_for_insertion(*note); - // No need to broadcast. - // Whoever created the partial note "should" already have the missing info ncessary to complete the note in their PXE. - // Also broadcsating would require the note's storage slot. context.push_new_note_hash(note_hash); + + if broadcast { + CompletableNoteInterface::broadcast_completion_from_public(*note, context); + } } pub fn create_note_hash_from_public( diff --git a/noir-projects/aztec-nr/aztec/src/note/note_header.nr b/noir-projects/aztec-nr/aztec/src/note/note_header.nr index 6755f385ea5b..9675b897b176 100644 --- a/noir-projects/aztec-nr/aztec/src/note/note_header.nr +++ b/noir-projects/aztec-nr/aztec/src/note/note_header.nr @@ -23,8 +23,4 @@ impl NoteHeader { pub fn new(contract_address: AztecAddress, nonce: Field, storage_slot: Field) -> Self { NoteHeader { contract_address, nonce, storage_slot, is_transient: false, partial_note_hash: 0 } } - - pub fn is_partial(self) -> bool { - self.partial_note_hash != 0 - } } diff --git a/noir-projects/aztec-nr/aztec/src/note/note_interface.nr b/noir-projects/aztec-nr/aztec/src/note/note_interface.nr index 4389d067487d..0ad8d8891c72 100644 --- a/noir-projects/aztec-nr/aztec/src/note/note_interface.nr +++ b/noir-projects/aztec-nr/aztec/src/note/note_interface.nr @@ -1,4 +1,4 @@ -use crate::context::PrivateContext; +use crate::context::{PrivateContext, PublicContext}; use crate::note::note_header::NoteHeader; // docs:start:note_interface @@ -24,3 +24,8 @@ trait NoteInterface { fn get_note_type_id() -> Field; } // docs:end:note_interface + +trait CompletableNoteInterface { + fn broadcast_partial_data(self, context: &mut PrivateContext, slot: Field) -> (); + fn broadcast_completion_from_public(self, context: &mut PublicContext) -> (); +} diff --git a/noir-projects/aztec-nr/aztec/src/state_vars/private_set.nr b/noir-projects/aztec-nr/aztec/src/state_vars/private_set.nr index 3c5bf5848e67..78d29a783e94 100644 --- a/noir-projects/aztec-nr/aztec/src/state_vars/private_set.nr +++ b/noir-projects/aztec-nr/aztec/src/state_vars/private_set.nr @@ -7,8 +7,8 @@ use crate::context::{PrivateContext, PublicContext, Context}; use crate::note::{ lifecycle::{create_note, create_partial_note, create_note_hash_from_public, destroy_note}, note_getter::{get_notes, view_notes}, note_getter_options::NoteGetterOptions, - note_header::NoteHeader, note_interface::NoteInterface, note_viewer_options::NoteViewerOptions, - utils::compute_note_hash_for_consumption + note_header::NoteHeader, note_interface::{NoteInterface, CompletableNoteInterface}, + note_viewer_options::NoteViewerOptions, utils::compute_note_hash_for_consumption }; use crate::state_vars::storage::Storage; @@ -43,7 +43,7 @@ impl PrivateSet { self, note: &mut Note, broadcast: bool - ) -> Field where Note: NoteInterface { + ) -> Field where Note: NoteInterface + CompletableNoteInterface { create_partial_note( self.context.private.unwrap(), self.storage_slot, diff --git a/noir-projects/noir-contracts/contracts/token_contract/src/main.nr b/noir-projects/noir-contracts/contracts/token_contract/src/main.nr index 4034a3925efa..c255ff6083d7 100644 --- a/noir-projects/noir-contracts/contracts/token_contract/src/main.nr +++ b/noir-projects/noir-contracts/contracts/token_contract/src/main.nr @@ -399,11 +399,8 @@ contract Token { storage.balances.sub(from, U128::from_integer(amount)); - let partial_note_0 = storage.balances.add_partial(from); - let partial_note_1 = storage.balances.add_partial(to); - - let partial_note_hash_0 = partial_note_0.get_header().partial_note_hash; - let partial_note_hash_1 = partial_note_1.get_header().partial_note_hash; + let partial_note_hash_0 = storage.balances.add_partial(from); + let partial_note_hash_1 = storage.balances.add_partial(to); let hash = compute_partial_notes_pair_hash( partial_note_hash_0, @@ -418,18 +415,6 @@ contract Token { [hash.to_field()] ); - // HACK: emit private data as unencrypted logs until PXE can track partial notes - // (the partial note is emitted as an encrypted log but the PXE drops it because its not in the tree) - emit_unencrypted_log_from_private( - &mut context, - [partial_note_0.randomness, partial_note_1.randomness] - ); - - emit_unencrypted_log_from_private( - &mut context, - [partial_note_0.get_header().storage_slot, partial_note_1.get_header().storage_slot] - ); - // emit as unencrypted for e2e testing as return values are not accessible there emit_unencrypted_log_from_private(&mut context, [partial_note_hash_0, partial_note_hash_1]); @@ -455,12 +440,10 @@ contract Token { context.push_new_nullifier(hash.to_field(), 0); let mut token_note_0 = TokenNote::new_from_public(U128::from_integer(amounts[0])); - complete_partial_note_from_public(&mut context, partial_note_hashes[0], &mut token_note_0); + complete_partial_note_from_public(&mut context, partial_note_hashes[0], &mut token_note_0, true); let mut token_note_1 = TokenNote::new_from_public(U128::from_integer(amounts[1])); - complete_partial_note_from_public(&mut context, partial_note_hashes[1], &mut token_note_1); - - emit_unencrypted_log(&mut context, amounts); + complete_partial_note_from_public(&mut context, partial_note_hashes[1], &mut token_note_1, true); } } // docs:end:token_all diff --git a/noir-projects/noir-contracts/contracts/token_contract/src/types/balances_map.nr b/noir-projects/noir-contracts/contracts/token_contract/src/types/balances_map.nr index d978cd8a92da..b86c48c97008 100644 --- a/noir-projects/noir-contracts/contracts/token_contract/src/types/balances_map.nr +++ b/noir-projects/noir-contracts/contracts/token_contract/src/types/balances_map.nr @@ -5,7 +5,8 @@ use dep::aztec::{ state_vars::{PrivateSet, Map}, note::{ note_getter::view_notes, note_getter_options::{NoteGetterOptions, SortOrder}, - note_viewer_options::NoteViewerOptions, note_header::NoteHeader, note_interface::NoteInterface + note_viewer_options::NoteViewerOptions, note_header::NoteHeader, + note_interface::{NoteInterface, CompletableNoteInterface} } }; use crate::types::token_note::{TokenNote, OwnedNote}; @@ -53,11 +54,13 @@ impl BalancesMap { balance } - pub fn add_partial(self: Self, owner: AztecAddress) -> T where T: NoteInterface + OwnedNote { + pub fn add_partial( + self: Self, + owner: AztecAddress + ) -> Field where T: NoteInterface + CompletableNoteInterface + OwnedNote { let mut addend_note = T::new(U128::empty(), owner); - let _partial_note_hash = self.map.at(owner).insert_partial(&mut addend_note, true); - // TODO once PXE tracks partial notes, just return the hash instead of the whole note - addend_note + let partial_note_hash = self.map.at(owner).insert_partial(&mut addend_note, true); + partial_note_hash } pub fn add( diff --git a/noir-projects/noir-contracts/contracts/token_contract/src/types/token_note.nr b/noir-projects/noir-contracts/contracts/token_contract/src/types/token_note.nr index 8eefda0ce788..7583d4e1c78f 100644 --- a/noir-projects/noir-contracts/contracts/token_contract/src/types/token_note.nr +++ b/noir-projects/noir-contracts/contracts/token_contract/src/types/token_note.nr @@ -1,7 +1,11 @@ use dep::aztec::{ - protocol_types::address::AztecAddress, - note::{note_header::NoteHeader, note_interface::NoteInterface, utils::compute_note_hash_for_consumption}, - context::PrivateContext, log::emit_encrypted_log, hash::pedersen_hash + protocol_types::{address::AztecAddress, traits::is_empty}, + note::{ + note_header::NoteHeader, note_interface::{NoteInterface, CompletableNoteInterface}, + utils::compute_note_hash_for_consumption +}, + context::{PrivateContext, PublicContext}, log::{emit_encrypted_log, emit_unencrypted_log}, + hash::pedersen_hash }; use dep::aztec::oracle::{rand::rand, nullifier_key::get_nullifier_secret_key, get_public_key::get_public_key}; use dep::std::option::Option; @@ -88,7 +92,7 @@ impl NoteInterface for TokenNote { // Broadcasts the note as an encrypted log on L1. fn broadcast(self, context: &mut PrivateContext, slot: Field) { // We only bother inserting the note if non-empty to save funds on gas. - if !(self.amount == U128::from_integer(0)) | self.header.is_partial() { + if !is_empty(self.amount) { let encryption_pub_key = get_public_key(self.owner); emit_encrypted_log( context, @@ -127,6 +131,31 @@ impl OwnedNote for TokenNote { } } +impl CompletableNoteInterface for TokenNote { + fn broadcast_partial_data(self, context: &mut PrivateContext, slot: Field) { + let encryption_pub_key = get_public_key(self.owner); + + emit_encrypted_log( + context, + (*context).this_address(), + slot, + TokenNote::get_note_type_id(), + encryption_pub_key, + self.serialize_content(), + ); + } + + fn broadcast_completion_from_public(self, context: &mut PublicContext) { + if !is_empty(self.amount) { + // TODO change this with a well-structured Event so the PXE can listen for them + emit_unencrypted_log( + context, + [TokenNote::get_note_type_id(), self.amount.to_field()] + ); + } + } +} + impl TokenNote { pub fn new_from_public(amount: U128) -> Self { Self { amount, owner: AztecAddress::empty(), randomness: 0, header: NoteHeader::empty() } diff --git a/yarn-project/aztec.js/src/wallet/base_wallet.ts b/yarn-project/aztec.js/src/wallet/base_wallet.ts index b84b83711812..02ad6b83ac31 100644 --- a/yarn-project/aztec.js/src/wallet/base_wallet.ts +++ b/yarn-project/aztec.js/src/wallet/base_wallet.ts @@ -126,4 +126,13 @@ export abstract class BaseWallet implements Wallet { isContractClassPubliclyRegistered(id: Fr): Promise { return this.pxe.isContractClassPubliclyRegistered(id); } + + completePartialNote( + contractAddress: AztecAddress, + noteTypeId: Fr, + patches: [number | Fr, Fr][], + txHash: TxHash, + ): Promise { + return this.pxe.completePartialNote(contractAddress, noteTypeId, patches, txHash); + } } diff --git a/yarn-project/circuit-types/src/interfaces/pxe.ts b/yarn-project/circuit-types/src/interfaces/pxe.ts index e32101b0971a..55017ac2f930 100644 --- a/yarn-project/circuit-types/src/interfaces/pxe.ts +++ b/yarn-project/circuit-types/src/interfaces/pxe.ts @@ -287,5 +287,20 @@ export interface PXE { * @param id - Identifier of the class. */ isContractClassPubliclyRegistered(id: Fr): Promise; + + /** + * Attempts to complete a partial note. + * + * @param contractAddress - The contract whose note is to be completed. + * @param noteTypeId - The type of the note to be completed. + * @param patches - The patches to apply to the serialized partial note data + * @param completionTxHash - The hash of the transaction that completed the partial note. + */ + completePartialNote( + contractAddress: AztecAddress, + noteTypeId: Fr, + patches: [number | Fr, Fr][], + completionTxHash: TxHash, + ): Promise; } // docs:end:pxe-interface diff --git a/yarn-project/circuit-types/src/stats/stats.ts b/yarn-project/circuit-types/src/stats/stats.ts index 8f87e3339053..99d7774ce36d 100644 --- a/yarn-project/circuit-types/src/stats/stats.ts +++ b/yarn-project/circuit-types/src/stats/stats.ts @@ -111,8 +111,12 @@ export type NoteProcessorStats = { seen: number; /** How many notes had decryption deferred due to a missing contract */ deferred: number; + /** How many partial notes we've discovered so far */ + partial: number; /** How many notes were successfully decrypted. */ decrypted: number; + /** How many partial notes were completed */ + completed: number; /** How many notes failed processing. */ failed: number; /** How many blocks were spanned. */ diff --git a/yarn-project/end-to-end/src/e2e_partial_token_notes.test.ts b/yarn-project/end-to-end/src/e2e_partial_token_notes.test.ts index 5fdae1211080..e8f9aac4f6f4 100644 --- a/yarn-project/end-to-end/src/e2e_partial_token_notes.test.ts +++ b/yarn-project/end-to-end/src/e2e_partial_token_notes.test.ts @@ -33,16 +33,19 @@ describe('e2e_partial_token_notes', () => { await addPendingShieldNoteToPXE(ctx.wallet, tokenContract.address, 1000n, secretHash, tx.txHash); await tokenContract.methods.redeem_shield(ctx.wallet.getAddress(), 1000n, secret).send().wait(); - }); + }, 50_000); it('splits a balance in two partial notes', async () => { - const splitData = await splitAmount(1000n); + const partialNoteHashes = await createPartialNotes(1000n); + + await expect(tokenContract.methods.balance_of_private(ctx.wallets[0].getAddress()).view()).resolves.toEqual(0n); + const completeNotesTx = await tokenContract.methods - .complete_partial_notes(splitData.partialNotes, [400, 600]) + .complete_partial_notes(partialNoteHashes, [400, 600]) .send() .wait(); - await addPartialNotes([400, 600], completeNotesTx.txHash, splitData); + await completePartialNotes([400, 600], completeNotesTx.txHash); await expect(tokenContract.methods.balance_of_private(ctx.wallets[0].getAddress()).view()).resolves.toEqual(400n); await expect(tokenContract.methods.balance_of_private(ctx.wallets[1].getAddress()).view()).resolves.toEqual(600n); @@ -67,75 +70,69 @@ describe('e2e_partial_token_notes', () => { }); it('partial notes are completable only once', async () => { - const splitData = await splitAmount(10n); - await tokenContract.methods.complete_partial_notes(splitData.partialNotes, [5n, 5n]).send().wait(); + const partialNoteHashes = await createPartialNotes(10n); + await tokenContract.methods.complete_partial_notes(partialNoteHashes, [5n, 5n]).send().wait(); await expect( - tokenContract.methods.complete_partial_notes(splitData.partialNotes, [5n, 5n]).send().wait(), + tokenContract.methods.complete_partial_notes(partialNoteHashes, [5n, 5n]).send().wait(), ).rejects.toThrow(/was dropped/); }); it('partial notes are be constrained to original total amount', async () => { - const splitData = await splitAmount(10n); + const partialNoteHashes = await createPartialNotes(10n); await expect( - tokenContract.methods.complete_partial_notes(splitData.partialNotes, [5n, 6n]).send().wait(), + tokenContract.methods.complete_partial_notes(partialNoteHashes, [5n, 6n]).send().wait(), ).rejects.toThrow(/Partial notes not authorized/); }); it('partial notes can only be completed by original creator', async () => { - const splitData = await splitAmount(10n); + const partialNoteHashes = await createPartialNotes(10n); await expect( tokenContract .withWallet(ctx.wallets[1]) - .methods.complete_partial_notes(splitData.partialNotes, [5n, 5n]) + .methods.complete_partial_notes(partialNoteHashes, [5n, 5n]) .send() .wait(), ).rejects.toThrow(/Partial notes not authorized/); }); - const splitAmount = async (totalAmount: bigint | number) => { + const createPartialNotes = async (totalAmount: bigint | number) => { const { txHash } = await tokenContract.methods .split_to_partial_notes(ctx.wallets[0].getAddress(), ctx.wallets[1].getAddress(), totalAmount, 0) .send() .wait(); const logs = await ctx.pxe.getUnencryptedLogs({ txHash }); + const partialNoteHashes = BufferReader.asReader(logs.logs[0].log.data).readArray(2, Fr); - const randomness = BufferReader.asReader(logs.logs[0].log.data).readArray(2, Fr); - const storageSlots = BufferReader.asReader(logs.logs[1].log.data).readArray(2, Fr); - const partialNotes = BufferReader.asReader(logs.logs[2].log.data).readArray(2, Fr); - - // TODO remove randomness, storageSlots and noteId once PXE tracks partial notes automatically - return { randomness, storageSlots, partialNotes, noteId: new Fr(8411110710111078111116101n) }; + return partialNoteHashes; }; - // TODO remove this once PXE tracks partial notes automatically - const addPartialNotes = async ( - amounts: number[], - completedTxHash: TxHash, - partialNoteData: Awaited>, - ) => { - await ctx.pxe.addNote( - new ExtendedNote( - new Note([new Fr(amounts[0]), ctx.wallets[0].getAddress().toField(), partialNoteData.randomness[0]]), - ctx.wallets[0].getAddress(), - tokenContract.address, - partialNoteData.storageSlots[0], - partialNoteData.noteId, - completedTxHash, - ), + const completePartialNotes = async (expectedAmounts: number[], completionTxHash: TxHash) => { + // TODO use a dedicated event selector for these completion events once implemented + const logs = await ctx.pxe.getUnencryptedLogs({ txHash: completionTxHash }); + const completedEvent0 = BufferReader.asReader(logs.logs[0].log.data).readArray(2, Fr); + const completedEvent1 = BufferReader.asReader(logs.logs[1].log.data).readArray(2, Fr); + + // constant value taken from the Noir contract + const tokenNoteId = new Fr(8411110710111078111116101n); + + expect(completedEvent0).toEqual([tokenNoteId, new Fr(expectedAmounts[0])]); + expect(completedEvent1).toEqual([tokenNoteId, new Fr(expectedAmounts[1])]); + + await ctx.pxe.completePartialNote( + tokenContract.address, + tokenNoteId, + [[0, new Fr(expectedAmounts[0])]], + completionTxHash, ); - await ctx.wallets[1].addNote( - new ExtendedNote( - new Note([new Fr(amounts[1]), ctx.wallets[1].getAddress().toField(), partialNoteData.randomness[1]]), - ctx.wallets[1].getAddress(), - tokenContract.address, - partialNoteData.storageSlots[1], - partialNoteData.noteId, - completedTxHash, - ), + await ctx.pxe.completePartialNote( + tokenContract.address, + tokenNoteId, + [[0, new Fr(expectedAmounts[1])]], + completionTxHash, ); }; }); diff --git a/yarn-project/pxe/src/database/kv_pxe_database.ts b/yarn-project/pxe/src/database/kv_pxe_database.ts index c63b913ea778..e4e8d3f3431c 100644 --- a/yarn-project/pxe/src/database/kv_pxe_database.ts +++ b/yarn-project/pxe/src/database/kv_pxe_database.ts @@ -9,6 +9,7 @@ import { ContractInstanceWithAddress, SerializableContractInstance } from '@azte import { DeferredNoteDao } from './deferred_note_dao.js'; import { NoteDao } from './note_dao.js'; +import { PartialNoteDao } from './partial_note_dao.js'; import { PxeDatabase } from './pxe_database.js'; /** @@ -34,6 +35,8 @@ export class KVPxeDatabase implements PxeDatabase { #nullifiedNotesByOwner: AztecMultiMap; #deferredNotes: AztecArray; #deferredNotesByContract: AztecMultiMap; + #partialNotes: AztecMap; + #partialNotesByContract: AztecMultiMap; #syncedBlockPerPublicKey: AztecMap; #contractArtifacts: AztecMap; #contractInstances: AztecMap; @@ -71,6 +74,9 @@ export class KVPxeDatabase implements PxeDatabase { this.#deferredNotes = db.openArray('deferred_notes'); this.#deferredNotesByContract = db.openMultiMap('deferred_notes_by_contract'); + + this.#partialNotes = db.openMap('partial_notes'); + this.#partialNotesByContract = db.openMultiMap('partial_notes_by_contract'); } public async addContractArtifact(id: Fr, contract: ContractArtifact): Promise { @@ -162,6 +168,39 @@ export class KVPxeDatabase implements PxeDatabase { return Promise.resolve(notes); } + addPartialNotes(partialNotes: PartialNoteDao[]): Promise { + return this.db.transaction(() => { + for (const note of partialNotes) { + const key = note.siloedNoteHash.toString(); + void this.#partialNotes.set(key, note.toBuffer()); + void this.#partialNotesByContract.set(note.contractAddress.toString(), key); + } + }); + } + + getPartialNotesByContract(contractAddress: AztecAddress): Promise { + const noteIds = this.#partialNotesByContract.getValues(contractAddress.toString()); + const notes: PartialNoteDao[] = []; + for (const noteId of noteIds) { + const serializedNote = this.#partialNotes.get(noteId); + if (!serializedNote) { + continue; + } + + const note = PartialNoteDao.fromBuffer(serializedNote); + notes.push(note); + } + return Promise.resolve(notes); + } + + removePartialNotes(noteIds: string[]): Promise { + return this.db.transaction(() => { + for (const noteId of noteIds) { + void this.#partialNotes.delete(noteId); + } + }); + } + /** * Removes all deferred notes for a given contract address. * @param contractAddress - the contract address to remove deferred notes for diff --git a/yarn-project/pxe/src/database/partial_note_dao.ts b/yarn-project/pxe/src/database/partial_note_dao.ts new file mode 100644 index 000000000000..2b4ae49c2d43 --- /dev/null +++ b/yarn-project/pxe/src/database/partial_note_dao.ts @@ -0,0 +1,50 @@ +import { Note, TxHash } from '@aztec/circuit-types'; +import { AztecAddress, Fr, Point, PublicKey } from '@aztec/circuits.js'; +import { BufferReader, serializeToBuffer } from '@aztec/foundation/serialize'; + +/** + * A note that is intended for us, but it's incomplete. + * So keep the state that we need to complete it later. + */ +export class PartialNoteDao { + constructor( + /** The public key associated with this note */ + public publicKey: PublicKey, + /** The note as emitted from the Noir contract. */ + public note: Note, + /** The contract address this note is created in. */ + public contractAddress: AztecAddress, + /** The specific storage location of the note on the contract. */ + public storageSlot: Fr, + /** The type ID of the note on the contract. */ + public noteTypeId: Fr, + /** The hash of the tx the note was created in. Equal to the first nullifier */ + public txHash: TxHash, + /** The siloed note hash */ + public siloedNoteHash: Fr, + ) {} + + toBuffer(): Buffer { + return serializeToBuffer( + this.publicKey.toBuffer(), + this.note.toBuffer(), + this.contractAddress.toBuffer(), + this.storageSlot.toBuffer(), + this.noteTypeId.toBuffer(), + this.txHash.toBuffer(), + this.siloedNoteHash, + ); + } + static fromBuffer(buffer: Buffer | BufferReader) { + const reader = BufferReader.asReader(buffer); + return new PartialNoteDao( + reader.readObject(Point), + reader.readObject(Note), + reader.readObject(AztecAddress), + reader.readObject(Fr), + reader.readObject(Fr), + reader.readObject(TxHash), + reader.readObject(Fr), + ); + } +} diff --git a/yarn-project/pxe/src/database/pxe_database.ts b/yarn-project/pxe/src/database/pxe_database.ts index f97847dfe97a..b0ed1e8a77f3 100644 --- a/yarn-project/pxe/src/database/pxe_database.ts +++ b/yarn-project/pxe/src/database/pxe_database.ts @@ -7,6 +7,7 @@ import { ContractArtifactDatabase } from './contracts/contract_artifact_db.js'; import { ContractInstanceDatabase } from './contracts/contract_instance_db.js'; import { DeferredNoteDao } from './deferred_note_dao.js'; import { NoteDao } from './note_dao.js'; +import { PartialNoteDao } from './partial_note_dao.js'; /** * A database interface that provides methods for retrieving, adding, and removing transactional data related to Aztec @@ -75,6 +76,24 @@ export interface PxeDatabase extends ContractDatabase, ContractArtifactDatabase, */ getDeferredNotesByContract(contractAddress: AztecAddress): Promise; + /** + * Add partial notes to the database + * @param partialNotes - An array of partial notes. + */ + addPartialNotes(partialNotes: PartialNoteDao[]): Promise; + + /** + * Add partial notes to the database + * @param contractAddress - The contract address to get the partial notes for. + */ + getPartialNotesByContract(contractAddress: AztecAddress): Promise; + + /** + * Removes partial notes from the database + * @param noteIds - An array of note ids to remove. + */ + removePartialNotes(noteIds: string[]): Promise; + /** * Remove deferred notes for a given contract address. * @param contractAddress - The contract address to remove the deferred notes for. diff --git a/yarn-project/pxe/src/note_processor/chopped_note_error.ts b/yarn-project/pxe/src/note_processor/chopped_note_error.ts new file mode 100644 index 000000000000..0a162492c8c4 --- /dev/null +++ b/yarn-project/pxe/src/note_processor/chopped_note_error.ts @@ -0,0 +1,20 @@ +import { Fr } from '@aztec/foundation/fields'; + +export class ChoppedNoteError extends Error { + constructor(public readonly siloedNoteHash: Fr) { + const errorString = `We decrypted a log, but couldn't find a corresponding note in the tree. +This might be because the note was nullified in the same tx which created it or it was a partial note. +In that case, everything is fine. To check whether this note was nullified in the same transaction, look back through +the logs for a notification +'important: chopped commitment for siloed inner hash note +${siloedNoteHash.toString()}'. +If you can see that notification or this note is meant to be completed at a later date +(it implements CompletableNoteInterface in the contract) then everything's fine. +If that's not the case, and you can't find such a notification, something has gone wrong. +There could be a problem with the way you've defined a custom note, or with the way you're +serializing / deserializing / hashing / encrypting / decrypting that note. +Please see the following github issue to track an improvement that we're working on: +https://github.com/AztecProtocol/aztec-packages/issues/1641`; + super(errorString); + } +} diff --git a/yarn-project/pxe/src/note_processor/note_processor.ts b/yarn-project/pxe/src/note_processor/note_processor.ts index 82a02a36151b..bd62ac55d696 100644 --- a/yarn-project/pxe/src/note_processor/note_processor.ts +++ b/yarn-project/pxe/src/note_processor/note_processor.ts @@ -1,6 +1,14 @@ -import { AztecNode, KeyStore, L1NotePayload, L2BlockContext, L2BlockL2Logs, TaggedNote } from '@aztec/circuit-types'; +import { + AztecNode, + KeyStore, + L1NotePayload, + L2BlockContext, + L2BlockL2Logs, + TaggedNote, + TxEffect, +} from '@aztec/circuit-types'; import { NoteProcessorStats } from '@aztec/circuit-types/stats'; -import { INITIAL_L2_BLOCK_NUM, MAX_NEW_NOTE_HASHES_PER_TX, PublicKey } from '@aztec/circuits.js'; +import { AztecAddress, INITIAL_L2_BLOCK_NUM, MAX_NEW_NOTE_HASHES_PER_TX, PublicKey } from '@aztec/circuits.js'; import { Grumpkin } from '@aztec/circuits.js/barretenberg'; import { Fr } from '@aztec/foundation/fields'; import { createDebugLogger } from '@aztec/foundation/log'; @@ -10,7 +18,9 @@ import { ContractNotFoundError } from '@aztec/simulator'; import { DeferredNoteDao } from '../database/deferred_note_dao.js'; import { PxeDatabase } from '../database/index.js'; import { NoteDao } from '../database/note_dao.js'; +import { PartialNoteDao } from '../database/partial_note_dao.js'; import { getAcirSimulator } from '../simulator/index.js'; +import { ChoppedNoteError } from './chopped_note_error.js'; import { produceNoteDao } from './produce_note_dao.js'; /** @@ -36,7 +46,16 @@ export class NoteProcessor { public readonly timer: Timer = new Timer(); /** Stats accumulated for this processor. */ - public readonly stats: NoteProcessorStats = { seen: 0, decrypted: 0, deferred: 0, failed: 0, blocks: 0, txs: 0 }; + public readonly stats: NoteProcessorStats = { + seen: 0, + decrypted: 0, + deferred: 0, + partial: 0, + completed: 0, + failed: 0, + blocks: 0, + txs: 0, + }; constructor( /** @@ -98,6 +117,7 @@ export class NoteProcessor { const blocksAndNotes: ProcessedData[] = []; // Keep track of notes that we couldn't process because the contract was not found. const deferredNoteDaos: DeferredNoteDao[] = []; + const partialNoteDaos: PartialNoteDao[] = []; // Iterate over both blocks and encrypted logs. for (let blockIndex = 0; blockIndex < encryptedL2BlockLogs.length; ++blockIndex) { @@ -157,6 +177,19 @@ export class NoteProcessor { dataStartIndexForTx, ); deferredNoteDaos.push(deferredNoteDao); + } else if (e instanceof ChoppedNoteError) { + this.stats.partial++; + this.log.warn(e.message); + const partialNoteDao = new PartialNoteDao( + this.publicKey, + payload.note, + payload.contractAddress, + payload.storageSlot, + payload.noteTypeId, + txHash, + e.siloedNoteHash, + ); + partialNoteDaos.push(partialNoteDao); } else { this.stats.failed++; this.log.warn(`Could not process note because of "${e}". Discarding note...`); @@ -175,6 +208,7 @@ export class NoteProcessor { await this.processBlocksAndNotes(blocksAndNotes); await this.processDeferredNotes(deferredNoteDaos); + await this.processPartialNotes(partialNoteDaos); const syncedToBlock = l2BlockContexts[l2BlockContexts.length - 1].block.number; await this.db.setSynchedBlockNumberForPublicKey(this.publicKey, syncedToBlock); @@ -235,6 +269,19 @@ export class NoteProcessor { } } + private async processPartialNotes(partialNoteDaos: PartialNoteDao[]): Promise { + if (partialNoteDaos.length) { + await this.db.addPartialNotes(partialNoteDaos); + partialNoteDaos.forEach(noteDao => { + this.log( + `Partial note for contract ${noteDao.contractAddress} at slot ${ + noteDao.storageSlot + } in tx ${noteDao.txHash.toString()}`, + ); + }); + } + } + /** * Retry decoding the given deferred notes because we now have the contract code. * @@ -272,4 +319,54 @@ export class NoteProcessor { return noteDaos; } + + public async completePartialNotes( + contractAddress: AztecAddress, + noteTypeId: Fr, + patches: [Fr | number, Fr][], + tx: TxEffect, + dataStartIndexForTx: number, + ): Promise { + const excludedIndices: Set = new Set(); + const noteDaos: NoteDao[] = []; + const completedNoteIds: string[] = []; + const partialNotes = await this.db.getPartialNotesByContract(contractAddress); + for (const partialNote of partialNotes) { + if (!partialNote.publicKey.equals(this.publicKey)) { + continue; + } + + if (!partialNote.noteTypeId.equals(noteTypeId)) { + continue; + } + + const { note, storageSlot } = partialNote; + for (const patch of patches) { + note.items[new Fr(patch[0]).toNumber()] = patch[1]; + } + + const payload = new L1NotePayload(note, contractAddress, storageSlot, noteTypeId); + try { + const noteDao = await produceNoteDao( + this.simulator, + this.publicKey, + payload, + tx.txHash, + tx.noteHashes, + dataStartIndexForTx, + excludedIndices, + ); + + noteDaos.push(noteDao); + completedNoteIds.push(partialNote.siloedNoteHash.toString()); + this.log.info(`Completed partial note for contract ${contractAddress} at slot ${storageSlot}`); + this.stats.completed++; + } catch (e) { + this.log.warn(`Could not complete partial note because of "${e}". Skipping note...`); + } + } + + await this.db.removePartialNotes(completedNoteIds); + await this.db.addNotes(noteDaos); + } } diff --git a/yarn-project/pxe/src/note_processor/produce_note_dao.ts b/yarn-project/pxe/src/note_processor/produce_note_dao.ts index 95a5d8b85ad2..c23e562e7f24 100644 --- a/yarn-project/pxe/src/note_processor/produce_note_dao.ts +++ b/yarn-project/pxe/src/note_processor/produce_note_dao.ts @@ -4,6 +4,7 @@ import { computeCommitmentNonce, siloNullifier } from '@aztec/circuits.js/hash'; import { AcirSimulator } from '@aztec/simulator'; import { NoteDao } from '../database/note_dao.js'; +import { ChoppedNoteError } from './chopped_note_error.js'; /** * Decodes a note from a transaction that we know was intended for us. @@ -95,6 +96,12 @@ async function findNoteIndexAndNullifier( const expectedNonce = computeCommitmentNonce(firstNullifier, commitmentIndex); ({ nonSiloedNoteHash, siloedNoteHash, uniqueSiloedNoteHash, nonSiloedNullifier } = await simulator.computeNoteHashAndNullifier(contractAddress, expectedNonce, storageSlot, noteTypeId, note)); + if (commitment.equals(siloedNoteHash)) { + // TODO(https://github.com/AztecProtocol/aztec-packages/issues/1386) + // Remove this once notes added from public also include nonces. + nonce = Fr.ZERO; + break; + } if (commitment.equals(uniqueSiloedNoteHash)) { nonce = expectedNonce; break; @@ -102,25 +109,23 @@ async function findNoteIndexAndNullifier( } if (!nonce) { - let errorString; if (siloedNoteHash == undefined) { - errorString = 'Cannot find a matching commitment for the note.'; + throw new Error('Cannot find a matching commitment for the note.'); } else { - errorString = `We decrypted a log, but couldn't find a corresponding note in the tree. -This might be because the note was nullified in the same tx which created it. -In that case, everything is fine. To check whether this is the case, look back through -the logs for a notification -'important: chopped commitment for siloed inner hash note -${siloedNoteHash.toString()}'. -If you can see that notification. Everything's fine. -If that's not the case, and you can't find such a notification, something has gone wrong. -There could be a problem with the way you've defined a custom note, or with the way you're -serializing / deserializing / hashing / encrypting / decrypting that note. -Please see the following github issue to track an improvement that we're working on: -https://github.com/AztecProtocol/aztec-packages/issues/1641`; + throw new ChoppedNoteError(siloedNoteHash); } - - throw new Error(errorString); + } else if (nonce.isZero()) { + // TODO(https://github.com/AztecProtocol/aztec-packages/issues/1386) + // this note was inserted from public so its nonce is 0 + // recompute the note hash and nullifier with nonce 0, this is needed because the "nonSiloedNullifier" is + // actually computed from the note's unique hash (ie note content hash + storage slot + contract address + nonce) + ({ nonSiloedNoteHash, nonSiloedNullifier } = await simulator.computeNoteHashAndNullifier( + contractAddress, + Fr.ZERO, + storageSlot, + noteTypeId, + note, + )); } return { diff --git a/yarn-project/pxe/src/pxe_service/pxe_service.ts b/yarn-project/pxe/src/pxe_service/pxe_service.ts index d0de596f50fb..0e02ab4c2824 100644 --- a/yarn-project/pxe/src/pxe_service/pxe_service.ts +++ b/yarn-project/pxe/src/pxe_service/pxe_service.ts @@ -21,6 +21,7 @@ import { TxHash, TxL2Logs, TxReceipt, + TxStatus, getNewContractPublicFunctions, isNoirCallStackUnresolved, } from '@aztec/circuit-types'; @@ -31,6 +32,7 @@ import { CompleteAddress, FunctionData, GrumpkinPrivateKey, + MAX_NEW_NOTE_HASHES_PER_TX, MAX_NON_REVERTIBLE_PUBLIC_CALL_STACK_LENGTH_PER_TX, MAX_REVERTIBLE_PUBLIC_CALL_STACK_LENGTH_PER_TX, PartialAddress, @@ -234,6 +236,55 @@ export class PXEService implements PXE { } } + /** + * Attempts to complete a partial note. + * + * @param contractAddress - The contract whose note is to be completed. + * @param noteTypeId - The type of the note to be completed. + * @param patches - The patches to apply to the serialized partial note data + * @param completionTxHash - The hash of the transaction that completed the partial note. + */ + public async completePartialNote( + contractAddress: AztecAddress, + noteTypeId: Fr, + patches: [number | Fr, Fr][], + completionTxHash: TxHash, + ) { + const txReceipt = await this.getTxReceipt(completionTxHash); + if (txReceipt.status !== TxStatus.MINED || typeof txReceipt.blockNumber !== 'number') { + throw new Error(`Tx ${completionTxHash} not mined`); + } + + const block = await this.getBlock(txReceipt.blockNumber); + if (!block) { + throw new Error(`Block ${txReceipt.blockNumber} not found`); + } + + // up to which leaf has this block written to in the note hash tree + const blockEndLeafIndex = block.header.state.partial.noteHashTree.nextAvailableLeafIndex; + // how many txs this block had, including empty txs + const blockSize = block.body.encryptedLogs.txLogs.length; + let completionTxStartLeafIndex = 0; + let completionTx: TxEffect; + + // find the tx in the block and the start leaf index + for (const [i, tx] of block.getTxs().entries()) { + completionTxStartLeafIndex = blockEndLeafIndex - (blockSize - i) * MAX_NEW_NOTE_HASHES_PER_TX; + if (tx.txHash.equals(completionTxHash)) { + completionTx = tx; + break; + } + } + + return this.synchronizer.processPartialNotes( + contractAddress, + noteTypeId, + patches, + completionTx!, + completionTxStartLeafIndex, + ); + } + private async addArtifactsAndInstancesFromDeployedContracts(contracts: DeployedContract[]) { for (const contract of contracts) { const artifact = contract.artifact; diff --git a/yarn-project/pxe/src/synchronizer/synchronizer.ts b/yarn-project/pxe/src/synchronizer/synchronizer.ts index 7ae66192402b..9cff048b4ae7 100644 --- a/yarn-project/pxe/src/synchronizer/synchronizer.ts +++ b/yarn-project/pxe/src/synchronizer/synchronizer.ts @@ -1,4 +1,12 @@ -import { AztecNode, KeyStore, L2BlockContext, L2BlockL2Logs, MerkleTreeId, TxHash } from '@aztec/circuit-types'; +import { + AztecNode, + KeyStore, + L2BlockContext, + L2BlockL2Logs, + MerkleTreeId, + TxEffect, + TxHash, +} from '@aztec/circuit-types'; import { NoteProcessorCaughtUpStats } from '@aztec/circuit-types/stats'; import { AztecAddress, Fr, INITIAL_L2_BLOCK_NUM, PublicKey } from '@aztec/circuits.js'; import { SerialQueue } from '@aztec/foundation/fifo'; @@ -324,6 +332,30 @@ export class Synchronizer { return this.jobQueue.put(() => this.#reprocessDeferredNotesForContract(contractAddress)); } + public processPartialNotes( + contractAddress: AztecAddress, + noteTypeId: Fr, + patches: [number | Fr, Fr][], + tx: TxEffect, + dataStartIndexForTx: number, + ): Promise { + return this.jobQueue.put(() => + this.#processPartialNotes(contractAddress, noteTypeId, patches, tx, dataStartIndexForTx), + ); + } + + async #processPartialNotes( + contractAddress: AztecAddress, + noteTypeId: Fr, + patches: [number | Fr, Fr][], + tx: TxEffect, + dataStartIndexForTx: number, + ) { + for (const processor of this.noteProcessors) { + await processor.completePartialNotes(contractAddress, noteTypeId, patches, tx, dataStartIndexForTx); + } + } + async #reprocessDeferredNotesForContract(contractAddress: AztecAddress): Promise { const deferredNotes = await this.db.getDeferredNotesByContract(contractAddress);