feat: Benchmarks (#2605)

Initial version of benchmarks for end-to-end tests. - Adds support for structured logging. All log methods now accept a string as message (requiring the caller to use `util.format` if needed) and an optional object. The object is passed directly to each entry when logging to JSONL. - Exports stats associated to publishing a rollup to L1 as first use case. Stats are collected using viem and exported with the structured logging format. Adds a first `bench-` e2e test that runs multiple rollup sizes to collect stats. - Collects benchmark data as JSONL from all e2e tests and uploads it to a new S3 bucket `aztec-ci-artifacts` (private for now) using the name of the job and the commit identifier. Benchmark data is any entry that has an `eventName` associated. - Once all e2e tests are finished, aggregates benchmark data from e2e tests named `bench*`, filtering by the metrics requested in #1663, and reuploading it to S3 and adding it to the PR as a comment. The JSONL files extracted from each test look like: ```jsonl {"blockNumber":1,"calldataGas":104732,"calldataSize":22340,"encryptedLogCount":2,"encryptedLogSize":516,"eventName":"rollup-published-to-l1","gasPrice":"1500000008","gasUsed":"422583","level":"info","message":"Published L2 block to L1 rollup contract","namespace":"aztec:sequencer:publisher","timestamp":"2023-09-29T20:14:30.534Z","transactionHash":"0x47724f679d813ec6bdb87002d8ed994abd8f11382eadc81bb41b3de0d7087db7","txCount":4,"unencryptedLogCount":0,"unencryptedLogSize":28} {"blockNumber":2,"calldataGas":93852,"calldataSize":21828,"encryptedLogCount":0,"encryptedLogSize":24,"eventName":"rollup-published-to-l1","gasPrice":"1500000008","gasUsed":"370654","level":"info","message":"Published L2 block to L1 rollup contract","namespace":"aztec:sequencer:publisher","timestamp":"2023-09-29T20:14:35.246Z","transactionHash":"0x1f9ad018f0a7a8f4414a7f789db7baa9b885e524c10a43e2e00d864b4f700721","txCount":4,"unencryptedLogCount":0,"unencryptedLogSize":24} {"blockNumber":3,"calldataGas":94016,"calldataSize":21860,"encryptedLogCount":0,"encryptedLogSize":24,"eventName":"rollup-published-to-l1","gasPrice":"1500000008","gasUsed":"373541","level":"info","message":"Published L2 block to L1 rollup contract","namespace":"aztec:sequencer:publisher","timestamp":"2023-09-29T20:14:41.191Z","transactionHash":"0xf919ea0083420d3bf45263c28797b024f6447ba046ea3d2fba7c116abbac54a7","txCount":4,"unencryptedLogCount":0,"unencryptedLogSize":28} {"blockNumber":4,"calldataGas":93680,"calldataSize":21860,"encryptedLogCount":0,"encryptedLogSize":24,"eventName":"rollup-published-to-l1","gasPrice":"1500000008","gasUsed":"373205","level":"info","message":"Published L2 block to L1 rollup contract","namespace":"aztec:sequencer:publisher","timestamp":"2023-09-29T20:14:46.089Z","transactionHash":"0x34f386ead9e72cbef2ed8b79cc1cffc4540ec9317811d7381d0311e980c90493","txCount":4,"unencryptedLogCount":0,"unencryptedLogSize":28} {"blockNumber":5,"calldataGas":186888,"calldataSize":43044,"encryptedLogCount":0,"encryptedLogSize":68,"eventName":"rollup-published-to-l1","gasPrice":"1500000008","gasUsed":"730533","level":"info","message":"Published L2 block to L1 rollup contract","namespace":"aztec:sequencer:publisher","timestamp":"2023-09-29T20:14:59.600Z","transactionHash":"0xa4ebaeb35dee76bdf8ff3313ede44cf2c8553108d724285eac8c7f1ac47adfdb","txCount":8,"unencryptedLogCount":0,"unencryptedLogSize":100} {"blockNumber":1,"calldataGas":104720,"calldataSize":22340,"encryptedLogCount":2,"encryptedLogSize":516,"eventName":"rollup-published-to-l1","gasPrice":"1500000008","gasUsed":"422571","level":"info","message":"Published L2 block to L1 rollup contract","namespace":"aztec:sequencer:publisher","timestamp":"2023-09-29T20:15:08.321Z","transactionHash":"0x5fd0ea8e8f0bddcd62d178751154f98d10ef234f69ed44f265628eac2b6e3110","txCount":4,"unencryptedLogCount":0,"unencryptedLogSize":28} {"blockNumber":2,"calldataGas":93900,"calldataSize":21828,"encryptedLogCount":0,"encryptedLogSize":24,"eventName":"rollup-published-to-l1","gasPrice":"1500000008","gasUsed":"370702","level":"info","message":"Published L2 block to L1 rollup contract","namespace":"aztec:sequencer:publisher","timestamp":"2023-09-29T20:15:12.976Z","transactionHash":"0x4c30b46ae0ba3f90622843fe363a70300ae8c71e25ad3487e6d43650326d41b1","txCount":4,"unencryptedLogCount":0,"unencryptedLogSize":24} {"blockNumber":3,"calldataGas":94028,"calldataSize":21860,"encryptedLogCount":0,"encryptedLogSize":24,"eventName":"rollup-published-to-l1","gasPrice":"1500000008","gasUsed":"373553","level":"info","message":"Published L2 block to L1 rollup contract","namespace":"aztec:sequencer:publisher","timestamp":"2023-09-29T20:15:16.832Z","transactionHash":"0x7641b6ad965ab2bacc4d6c860de2eb8a08b0e8fd4e01e99f10c5541c20c996a5","txCount":4,"unencryptedLogCount":0,"unencryptedLogSize":28} {"blockNumber":4,"calldataGas":93692,"calldataSize":21860,"encryptedLogCount":0,"encryptedLogSize":24,"eventName":"rollup-published-to-l1","gasPrice":"1500000008","gasUsed":"373217","level":"info","message":"Published L2 block to L1 rollup contract","namespace":"aztec:sequencer:publisher","timestamp":"2023-09-29T20:15:21.969Z","transactionHash":"0x7c13aa9ebd8ec280f99b4dbcc2505cfca08034ec8d22fcf850ffb17fd5493228","txCount":4,"unencryptedLogCount":0,"unencryptedLogSize":28} {"blockNumber":5,"calldataGas":365728,"calldataSize":85348,"encryptedLogCount":0,"encryptedLogSize":132,"eventName":"rollup-published-to-l1","gasPrice":"1500000008","gasUsed":"1444415","level":"info","message":"Published L2 block to L1 rollup contract","namespace":"aztec:sequencer:publisher","timestamp":"2023-09-29T20:15:45.668Z","transactionHash":"0x0ec877bc7a6c30a7879fffe5c9ffd2d72df28f5e46971519def7385790bd1deb","txCount":16,"unencryptedLogCount":0,"unencryptedLogSize":196} ``` Sample S3 structure after running this PR: ``` aztec-ci-artifacts ├── benchmarks-v1 [public] │ └── pulls │ └── 2605.json └── logs-v1 [private] └── pulls └── 2605 ├── archiver.jsonl ├── benchmarks_publish_rollup.jsonl ├── e2e_2_pxes.jsonl ├── e2e_account_contracts.jsonl ├── e2e_card_game.jsonl ├── e2e_cross_chain_messaging.jsonl ├── e2e_deploy_contract.jsonl ├── e2e_escrow_contract.jsonl ├── e2e_multiple_accounts_1_enc_key.jsonl ├── e2e_nested_contract.jsonl ├── e2e_non_contract_account.jsonl ├── e2e_ordering.jsonl ├── e2e_p2p_network.jsonl ├── e2e_pending_commitments_contract.jsonl ├── e2e_private_token_contract.jsonl ├── e2e_public_cross_chain_messaging.jsonl ├── e2e_public_token_contract.jsonl ├── e2e_public_to_private_messaging.jsonl ├── guides_writing_an_account_contract.jsonl ├── multi-transfer.jsonl ├── private.jsonl ├── uniswap_trade_on_l1_from_l2.jsonl └── unknown.jsonl ``` And the aggregated benchmark file has one entry for each metric, and then one entry for each of the `ROLLUP_SIZES` configured: ```json { "l1_rollup_calldata_gas": { "8": 186888, "32": 723612 }, "l1_rollup_calldata_size_in_bytes": { "8": 43044, "32": 169956 }, "l1_rollup_execution_gas": { "8": 730533, "32": 3092494 }, "timestamp": "2023-10-02T17:44:19.237Z" } ``` Note that the `AZTEC_GITHUB_TOKEN` in CCI at the moment doesn't have rights for commenting. I'll review this with Adam once he's back.
AztecProtocol · Oct 3, 2023 · 37d9f9c · 37d9f9c
1 parent 5eafa3d
commit 37d9f9c
Show file tree

Hide file tree

Showing 46 changed files with 839 additions and 166 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -530,6 +530,7 @@ jobs:
       - run:
           name: "Test"
           command: cond_run_script end-to-end ./scripts/run_tests_local e2e_2_pxes.test.ts
+          environment: { DEBUG: 'aztec:*' }
 
   e2e-multiple-accounts-1-enc-key:
     machine:
@@ -541,6 +542,7 @@ jobs:
       - run:
           name: "Test"
           command: cond_run_script end-to-end ./scripts/run_tests_local e2e_multiple_accounts_1_enc_key.test.ts
+          environment: { DEBUG: 'aztec:*' }
 
   e2e-deploy-contract:
     machine:
@@ -552,6 +554,7 @@ jobs:
       - run:
           name: "Test"
           command: cond_run_script end-to-end ./scripts/run_tests_local e2e_deploy_contract.test.ts
+          environment: { DEBUG: 'aztec:*' }
 
   e2e-lending-contract:
     machine:
@@ -563,6 +566,7 @@ jobs:
       - run:
           name: "Test"
           command: cond_run_script end-to-end ./scripts/run_tests_local e2e_lending_contract.test.ts
+          environment: { DEBUG: 'aztec:*' }
 
   e2e-token-contract:
     machine:
@@ -574,6 +578,7 @@ jobs:
       - run:
           name: "Test"
           command: cond_run_script end-to-end ./scripts/run_tests_local e2e_token_contract.test.ts
+          environment: { DEBUG: 'aztec:*' }
 
   e2e-private-airdrop:
     machine:
@@ -585,6 +590,7 @@ jobs:
       - run:
           name: "Test"
           command: cond_run_script end-to-end ./scripts/run_tests_local e2e_private_airdrop.test.ts
+          environment: { DEBUG: 'aztec:*' }
 
   e2e-private-token-contract:
     machine:
@@ -596,6 +602,7 @@ jobs:
       - run:
           name: "Test"
           command: cond_run_script end-to-end ./scripts/run_tests_local e2e_private_token_contract.test.ts
+          environment: { DEBUG: 'aztec:*' }
 
   e2e-sandbox-example:
     machine:
@@ -607,6 +614,7 @@ jobs:
       - run:
           name: "Test"
           command: cond_run_script end-to-end ./scripts/run_tests_local e2e_sandbox_example.test.ts ./scripts/docker-compose-e2e-sandbox.yml
+          environment: { DEBUG: 'aztec:*' }
 
   e2e-multi-transfer-contract:
     machine:
@@ -618,6 +626,7 @@ jobs:
       - run:
           name: "Test"
           command: cond_run_script end-to-end ./scripts/run_tests_local e2e_multi_transfer.test.ts
+          environment: { DEBUG: 'aztec:*' }
 
   e2e-block-building:
     machine:
@@ -629,6 +638,7 @@ jobs:
       - run:
           name: "Test"
           command: cond_run_script end-to-end ./scripts/run_tests_local e2e_block_building.test.ts
+          environment: { DEBUG: 'aztec:*' }
 
   e2e-nested-contract:
     machine:
@@ -640,6 +650,7 @@ jobs:
       - run:
           name: "Test"
           command: cond_run_script end-to-end ./scripts/run_tests_local e2e_nested_contract.test.ts
+          environment: { DEBUG: 'aztec:*' }
 
   e2e-non-contract-account:
     machine:
@@ -651,6 +662,7 @@ jobs:
       - run:
           name: "Test"
           command: cond_run_script end-to-end ./scripts/run_tests_local e2e_non_contract_account.test.ts
+          environment: { DEBUG: 'aztec:*' }
 
   e2e-cross-chain-messaging:
     machine:
@@ -662,6 +674,7 @@ jobs:
       - run:
           name: "Test"
           command: cond_run_script end-to-end ./scripts/run_tests_local e2e_cross_chain_messaging.test.ts
+          environment: { DEBUG: 'aztec:*' }
 
   e2e-public-cross-chain-messaging:
     machine:
@@ -673,6 +686,7 @@ jobs:
       - run:
           name: "Test"
           command: cond_run_script end-to-end ./scripts/run_tests_local e2e_public_cross_chain_messaging.test.ts
+          environment: { DEBUG: 'aztec:*' }
 
   e2e-public-to-private-messaging:
     machine:
@@ -684,6 +698,7 @@ jobs:
       - run:
           name: "Test"
           command: cond_run_script end-to-end ./scripts/run_tests_local e2e_public_to_private_messaging.test.ts
+          environment: { DEBUG: 'aztec:*' }
 
   e2e-account-contracts:
     machine:
@@ -695,6 +710,7 @@ jobs:
       - run:
           name: "Test"
           command: cond_run_script end-to-end ./scripts/run_tests_local e2e_account_contracts.test.ts
+          environment: { DEBUG: 'aztec:*' }
 
   e2e-escrow-contract:
     machine:
@@ -706,6 +722,7 @@ jobs:
       - run:
           name: "Test"
           command: cond_run_script end-to-end ./scripts/run_tests_local e2e_escrow_contract.test.ts
+          environment: { DEBUG: 'aztec:*' }
 
   e2e-pending-commitments-contract:
     machine:
@@ -717,6 +734,7 @@ jobs:
       - run:
           name: "Test"
           command: cond_run_script end-to-end ./scripts/run_tests_local e2e_pending_commitments_contract.test.ts
+          environment: { DEBUG: 'aztec:*' }
 
   e2e-ordering:
     machine:
@@ -728,6 +746,7 @@ jobs:
       - run:
           name: "Test"
           command: cond_run_script end-to-end ./scripts/run_tests_local e2e_ordering.test.ts
+          environment: { DEBUG: 'aztec:*' }
 
   uniswap-trade-on-l1-from-l2:
     machine:
@@ -739,6 +758,7 @@ jobs:
       - run:
           name: "Test"
           command: cond_run_script end-to-end ./scripts/run_tests_local uniswap_trade_on_l1_from_l2.test.ts
+          environment: { DEBUG: 'aztec:*' }
 
   integration-archiver-l1-to-l2:
     machine:
@@ -750,6 +770,7 @@ jobs:
       - run:
           name: "Test"
           command: cond_run_script end-to-end ./scripts/run_tests_local integration_archiver_l1_to_l2.test.ts
+          environment: { DEBUG: 'aztec:*' }
 
   integration-l1-publisher:
     machine:
@@ -761,6 +782,7 @@ jobs:
       - run:
           name: "Test"
           command: cond_run_script end-to-end ./scripts/run_tests_local integration_l1_publisher.test.ts
+          environment: { DEBUG: 'aztec:*' }
 
   e2e-public-token-contract:
     machine:
@@ -772,6 +794,7 @@ jobs:
       - run:
           name: "Test"
           command: cond_run_script end-to-end ./scripts/run_tests_local e2e_public_token_contract.test.ts
+          environment: { DEBUG: 'aztec:*' }
 
   e2e-cli:
     machine:
@@ -783,6 +806,7 @@ jobs:
       - run:
           name: "Test"
           command: cond_run_script end-to-end ./scripts/run_tests_local e2e_cli.test.ts
+          environment: { DEBUG: 'aztec:*' }
 
   e2e-p2p:
     machine:
@@ -794,6 +818,7 @@ jobs:
       - run:
           name: "Test"
           command: cond_run_script end-to-end ./scripts/run_tests_local e2e_p2p_network.test.ts
+          environment: { DEBUG: 'aztec:*' }
 
   e2e-browser-sandbox:
     machine:
@@ -805,6 +830,7 @@ jobs:
       - run:
           name: "Test"
           command: cond_run_script end-to-end ./scripts/run_tests_local e2e_aztec_js_browser.test.ts ./scripts/docker-compose-e2e-sandbox-browser.yml
+          environment: { DEBUG: 'aztec:*' }
 
   e2e-card-game:
     machine:
@@ -816,6 +842,7 @@ jobs:
       - run:
           name: "Test"
           command: cond_run_script end-to-end ./scripts/run_tests_local e2e_card_game.test.ts
+          environment: { DEBUG: 'aztec:*' }
 
   pxe-sandbox:
     machine:
@@ -827,6 +854,7 @@ jobs:
       - run:
           name: "Test"
           command: cond_run_script end-to-end ./scripts/run_tests_local pxe_sandbox.test.ts ./scripts/docker-compose-e2e-sandbox.yml
+          environment: { DEBUG: 'aztec:*' }
 
   cli-docs-sandbox:
     machine:
@@ -838,6 +866,7 @@ jobs:
       - run:
           name: "Test"
           command: cond_run_script end-to-end ./scripts/run_tests_local cli_docs_sandbox.test.ts ./scripts/docker-compose-e2e-sandbox.yml
+          environment: { DEBUG: 'aztec:*' }
 
   guides-writing-an-account-contract:
     machine:
@@ -849,6 +878,7 @@ jobs:
       - run:
           name: "Test"
           command: cond_run_script end-to-end ./scripts/run_tests_local guides/writing_an_account_contract.test.ts
+          environment: { DEBUG: 'aztec:*' }
 
   guides-dapp-testing:
     machine:
@@ -860,6 +890,7 @@ jobs:
       - run:
           name: "Test"
           command: cond_run_script end-to-end ./scripts/run_tests_local guides/dapp_testing.test.ts ./scripts/docker-compose-e2e-sandbox.yml
+          environment: { DEBUG: 'aztec:*' }
 
   guides-sample-dapp:
     machine:
@@ -871,6 +902,7 @@ jobs:
       - run:
           name: "Test"
           command: cond_run_script end-to-end ./scripts/run_tests_local sample-dapp ./scripts/docker-compose-e2e-sandbox.yml
+          environment: { DEBUG: 'aztec:*' }
 
   guides-up-quick-start:
     machine:
@@ -882,6 +914,7 @@ jobs:
       - run:
           name: "Test"
           command: cond_run_script end-to-end ./scripts/run_tests_local guides/up_quick_start.test.ts ./scripts/docker-compose-e2e-sandbox.yml
+          environment: { DEBUG: 'aztec:*' }
 
   e2e-canary-test:
     machine:
@@ -894,6 +927,18 @@ jobs:
           name: "Test"
           command: cond_run_script canary-build ./scripts/run_tests uniswap_trade_on_l1_from_l2.test.ts canary-build ./scripts/docker-compose-e2e-sandbox.yml
 
+  bench-publish-rollup:
+    machine:
+      image: ubuntu-2204:2023.07.2
+    resource_class: large
+    steps:
+      - *checkout
+      - *setup_env
+      - run:
+          name: "Benchmark"
+          command: cond_run_script end-to-end ./scripts/run_tests_local benchmarks/bench_publish_rollup.test.ts
+          environment: { DEBUG: 'aztec:benchmarks:*,aztec:sequencer' }
+
   build-docs:
     machine:
       image: ubuntu-2204:2023.07.2
@@ -926,6 +971,16 @@ jobs:
           name: "Noop"
           command: echo Noop
 
+  bench-summary:
+    machine:
+      image: ubuntu-2204:2023.07.2
+    steps:
+      - *checkout
+      - *setup_env
+      - run:
+          name: "Assemble benchmark summary from uploaded logs"
+          command: ./scripts/ci/assemble_e2e_benchmark.sh
+
   deploy-npm:
     machine:
       image: ubuntu-2204:2023.07.2
@@ -1220,6 +1275,7 @@ workflows:
       - guides-dapp-testing: *e2e_test
       - guides-sample-dapp: *e2e_test
       - guides-up-quick-start: *e2e_test
+      - bench-publish-rollup: *e2e_test
 
       - e2e-end:
           requires:
@@ -1257,6 +1313,12 @@ workflows:
             - guides-dapp-testing
             - guides-sample-dapp
             - guides-up-quick-start
+            - bench-publish-rollup
+          <<: *defaults
+
+      - bench-summary:
+          requires:
+            - e2e-end
           <<: *defaults
 
         # Deployment and Canary tests