Skip to content

Commit

Permalink
Merge remote-tracking branch 'origin/master' into das-client-metrics
Browse files Browse the repository at this point in the history
Minor conflict in das/dasRpcClient.go around addition of linter
suppression.
  • Loading branch information
Tristan-Wilson committed Sep 23, 2024
2 parents 32c63ed + ac929a7 commit 69505b5
Show file tree
Hide file tree
Showing 140 changed files with 2,345 additions and 1,166 deletions.
26 changes: 0 additions & 26 deletions .github/workflows/submodule-pin-check.sh

This file was deleted.

23 changes: 22 additions & 1 deletion .github/workflows/submodule-pin-check.yml
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,32 @@ jobs:
with:
fetch-depth: 0
submodules: true
ref: "${{ github.event.pull_request.merge_commit_sha }}"

- name: Check all submodules are ancestors of origin/HEAD or configured branch
run: |
status_state="pending"
if ${{ github.workspace }}/.github/workflows/submodule-pin-check.sh; then
declare -Ar exceptions=(
[contracts]=origin/develop
[nitro-testnode]=origin/master
#TODO Rachel to check these are the intended branches.
[arbitrator/langs/c]=origin/vm-storage-cache
[arbitrator/tools/wasmer]=origin/adopt-v4.2.8
)
divergent=0
for mod in `git submodule --quiet foreach 'echo $name'`; do
branch=origin/HEAD
if [[ -v exceptions[$mod] ]]; then
branch=${exceptions[$mod]}
fi
if ! git -C $mod merge-base --is-ancestor HEAD $branch; then
echo $mod diverges from $branch
divergent=1
fi
done
if [ $divergent -eq 0 ]; then
status_state="success"
else
resp="$(curl -sSL --fail-with-body \
Expand Down
39 changes: 14 additions & 25 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,14 @@ ifneq ($(origin GOLANG_LDFLAGS),undefined)
GOLANG_PARAMS = -ldflags="-extldflags '-ldl' $(GOLANG_LDFLAGS)"
endif

UNAME_S := $(shell uname -s)

# In Mac OSX, there are a lot of warnings emitted if these environment variables aren't set.
ifeq ($(UNAME_S), Darwin)
export MACOSX_DEPLOYMENT_TARGET := $(shell sw_vers -productVersion)
export CGO_LDFLAGS := -Wl,-no_warn_duplicate_libraries
endif

precompile_names = AddressTable Aggregator BLS Debug FunctionTable GasInfo Info osTest Owner RetryableTx Statistics Sys
precompiles = $(patsubst %,./solgen/generated/%.go, $(precompile_names))

Expand All @@ -52,7 +60,7 @@ replay_wasm=$(output_latest)/replay.wasm
arb_brotli_files = $(wildcard arbitrator/brotli/src/*.* arbitrator/brotli/src/*/*.* arbitrator/brotli/*.toml arbitrator/brotli/*.rs) .make/cbrotli-lib .make/cbrotli-wasm

arbitrator_generated_header=$(output_root)/include/arbitrator.h
arbitrator_wasm_libs=$(patsubst %, $(output_root)/machines/latest/%.wasm, forward wasi_stub host_io soft-float arbcompress user_host program_exec)
arbitrator_wasm_libs=$(patsubst %, $(output_root)/machines/latest/%.wasm, wasi_stub host_io soft-float arbcompress user_host program_exec)
arbitrator_stylus_lib=$(output_root)/lib/libstylus.a
prover_bin=$(output_root)/bin/prover
arbitrator_jit=$(output_root)/bin/jit
Expand All @@ -67,29 +75,22 @@ arbitrator_test_wasms=$(patsubst %.wat,%.wasm, $(arbitrator_tests_wat)) $(patsub
arbitrator_tests_link_info = $(shell cat $(arbitrator_cases)/link.txt | xargs)
arbitrator_tests_link_deps = $(patsubst %,$(arbitrator_cases)/%.wasm, $(arbitrator_tests_link_info))

arbitrator_tests_forward_wats = $(wildcard $(arbitrator_cases)/forward/*.wat)
arbitrator_tests_forward_deps = $(arbitrator_tests_forward_wats:wat=wasm)

WASI_SYSROOT?=/opt/wasi-sdk/wasi-sysroot

arbitrator_wasm_lib_flags=$(patsubst %, -l %, $(arbitrator_wasm_libs))

rust_arbutil_files = $(wildcard arbitrator/arbutil/src/*.* arbitrator/arbutil/src/*/*.* arbitrator/arbutil/*.toml arbitrator/caller-env/src/*.* arbitrator/caller-env/src/*/*.* arbitrator/caller-env/*.toml) .make/cbrotli-lib

prover_direct_includes = $(patsubst %,$(output_latest)/%.wasm, forward forward_stub)
prover_dir = arbitrator/prover/
rust_prover_files = $(wildcard $(prover_dir)/src/*.* $(prover_dir)/src/*/*.* $(prover_dir)/*.toml $(prover_dir)/*.rs) $(rust_arbutil_files) $(prover_direct_includes) $(arb_brotli_files)

wasm_lib = arbitrator/wasm-libraries
wasm_lib_cargo = $(wasm_lib)/.cargo/config.toml
wasm_lib_deps = $(wildcard $(wasm_lib)/$(1)/*.toml $(wasm_lib)/$(1)/src/*.rs $(wasm_lib)/$(1)/*.rs) $(wasm_lib_cargo) $(rust_arbutil_files) $(arb_brotli_files) .make/machines
wasm_lib_go_abi = $(call wasm_lib_deps,go-abi)
wasm_lib_forward = $(call wasm_lib_deps,forward)
wasm_lib_user_host_trait = $(call wasm_lib_deps,user-host-trait)
wasm_lib_user_host = $(call wasm_lib_deps,user-host) $(wasm_lib_user_host_trait)

forward_dir = $(wasm_lib)/forward

stylus_files = $(wildcard $(stylus_dir)/*.toml $(stylus_dir)/src/*.rs) $(wasm_lib_user_host_trait) $(rust_prover_files)

jit_dir = arbitrator/jit
Expand Down Expand Up @@ -273,7 +274,6 @@ clean:
rm -f arbitrator/wasm-libraries/soft-float/*.o
rm -f arbitrator/wasm-libraries/soft-float/SoftFloat/build/Wasm-Clang/*.o
rm -f arbitrator/wasm-libraries/soft-float/SoftFloat/build/Wasm-Clang/*.a
rm -f arbitrator/wasm-libraries/forward/*.wat
rm -rf arbitrator/stylus/tests/*/target/ arbitrator/stylus/tests/*/*.wasm
@rm -rf contracts/build contracts/cache solgen/go/
@rm -f .make/*
Expand Down Expand Up @@ -391,7 +391,7 @@ $(output_latest)/host_io.wasm: $(DEP_PREDICATE) $(call wasm_lib_deps,host-io) $(
cargo build --manifest-path arbitrator/wasm-libraries/Cargo.toml --release --target wasm32-wasi --config $(wasm_lib_cargo) --package host-io
install arbitrator/wasm-libraries/$(wasm32_wasi)/host_io.wasm $@

$(output_latest)/user_host.wasm: $(DEP_PREDICATE) $(wasm_lib_user_host) $(rust_prover_files) $(output_latest)/forward_stub.wasm .make/machines
$(output_latest)/user_host.wasm: $(DEP_PREDICATE) $(wasm_lib_user_host) $(rust_prover_files) .make/machines
cargo build --manifest-path arbitrator/wasm-libraries/Cargo.toml --release --target wasm32-wasi --config $(wasm_lib_cargo) --package user-host
install arbitrator/wasm-libraries/$(wasm32_wasi)/user_host.wasm $@

Expand All @@ -407,17 +407,9 @@ $(output_latest)/arbcompress.wasm: $(DEP_PREDICATE) $(call wasm_lib_deps,brotli)
cargo build --manifest-path arbitrator/wasm-libraries/Cargo.toml --release --target wasm32-wasi --config $(wasm_lib_cargo) --package arbcompress
install arbitrator/wasm-libraries/$(wasm32_wasi)/arbcompress.wasm $@

$(output_latest)/forward.wasm: $(DEP_PREDICATE) $(wasm_lib_forward) .make/machines
cargo run --manifest-path $(forward_dir)/Cargo.toml -- --path $(forward_dir)/forward.wat
wat2wasm $(wasm_lib)/forward/forward.wat -o $@

$(output_latest)/forward_stub.wasm: $(DEP_PREDICATE) $(wasm_lib_forward) .make/machines
cargo run --manifest-path $(forward_dir)/Cargo.toml -- --path $(forward_dir)/forward_stub.wat --stub
wat2wasm $(wasm_lib)/forward/forward_stub.wat -o $@

$(output_latest)/machine.wavm.br: $(DEP_PREDICATE) $(prover_bin) $(arbitrator_wasm_libs) $(replay_wasm)
$(prover_bin) $(replay_wasm) --generate-binaries $(output_latest) \
$(patsubst %,-l $(output_latest)/%.wasm, forward soft-float wasi_stub host_io user_host arbcompress program_exec)
$(prover_bin) $(replay_wasm) --generate-binaries $(output_latest) --with-forwarder \
$(patsubst %,-l $(output_latest)/%.wasm, soft-float wasi_stub host_io user_host arbcompress program_exec)

$(arbitrator_cases)/%.wasm: $(arbitrator_cases)/%.wat
wat2wasm $< -o $@
Expand Down Expand Up @@ -484,10 +476,10 @@ target/testdata/preimages.bin:
python3 scripts/create-test-preimages.py $@

contracts/test/prover/proofs/rust-%.json: $(arbitrator_cases)/rust/$(wasm32_wasi)/%.wasm $(prover_bin) $(arbitrator_wasm_libs) target/testdata/preimages.bin
$(prover_bin) $< $(arbitrator_wasm_lib_flags) -o $@ -b --allow-hostapi --require-success --inbox-add-stub-headers --inbox $(arbitrator_cases)/rust/data/msg0.bin --inbox $(arbitrator_cases)/rust/data/msg1.bin --delayed-inbox $(arbitrator_cases)/rust/data/msg0.bin --delayed-inbox $(arbitrator_cases)/rust/data/msg1.bin --preimages target/testdata/preimages.bin
$(prover_bin) $< $(arbitrator_wasm_lib_flags) -o $@ -b --allow-hostapi --require-success --inbox-add-stub-headers --inbox $(arbitrator_cases)/rust/data/msg0.bin --inbox $(arbitrator_cases)/rust/data/msg1.bin --delayed-inbox $(arbitrator_cases)/rust/data/msg0.bin --delayed-inbox $(arbitrator_cases)/rust/data/msg1.bin --preimages target/testdata/preimages.bin --with-forwarder

contracts/test/prover/proofs/go.json: $(arbitrator_cases)/go/testcase.wasm $(prover_bin) $(arbitrator_wasm_libs) target/testdata/preimages.bin $(arbitrator_tests_link_deps) $(arbitrator_cases)/user.wasm
$(prover_bin) $< $(arbitrator_wasm_lib_flags) -o $@ -b --require-success --preimages target/testdata/preimages.bin --stylus-modules $(arbitrator_cases)/user.wasm
$(prover_bin) $< $(arbitrator_wasm_lib_flags) -o $@ -b --require-success --preimages target/testdata/preimages.bin --stylus-modules $(arbitrator_cases)/user.wasm --with-forwarder

# avoid testing user.wasm in onestepproofs. It can only run as stylus program.
contracts/test/prover/proofs/user.json:
Expand All @@ -500,9 +492,6 @@ contracts/test/prover/proofs/read-inboxmsg-10.json:
contracts/test/prover/proofs/global-state.json:
echo "[]" > $@

contracts/test/prover/proofs/forward-test.json: $(arbitrator_cases)/forward-test.wasm $(arbitrator_tests_forward_deps) $(prover_bin)
$(prover_bin) $< -o $@ --allow-hostapi $(patsubst %,-l %, $(arbitrator_tests_forward_deps))

contracts/test/prover/proofs/link.json: $(arbitrator_cases)/link.wasm $(arbitrator_tests_link_deps) $(prover_bin)
$(prover_bin) $< -o $@ --allow-hostapi --stylus-modules $(arbitrator_tests_link_deps) --require-success

Expand Down
64 changes: 64 additions & 0 deletions arbitrator/arbutil/src/hostios.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
use std::fmt::Display;

pub enum ParamType {
I32,
I64,
}

impl Display for ParamType {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
use ParamType::*;
match self {
I32 => write!(f, "i32"),
I64 => write!(f, "i64"),
}
}
}

use ParamType::*;

/// order matters!
pub const HOSTIOS: [(&str, &[ParamType], &[ParamType]); 42] = [
("read_args", &[I32], &[]),
("write_result", &[I32, I32], &[]),
("exit_early", &[I32], &[]),
("storage_load_bytes32", &[I32, I32], &[]),
("storage_cache_bytes32", &[I32, I32], &[]),
("storage_flush_cache", &[I32], &[]),
("transient_load_bytes32", &[I32, I32], &[]),
("transient_store_bytes32", &[I32, I32], &[]),
("call_contract", &[I32, I32, I32, I32, I64, I32], &[I32]),
("delegate_call_contract", &[I32, I32, I32, I64, I32], &[I32]),
("static_call_contract", &[I32, I32, I32, I64, I32], &[I32]),
("create1", &[I32, I32, I32, I32, I32], &[]),
("create2", &[I32, I32, I32, I32, I32, I32], &[]),
("read_return_data", &[I32, I32, I32], &[I32]),
("return_data_size", &[], &[I32]),
("emit_log", &[I32, I32, I32], &[]),
("account_balance", &[I32, I32], &[]),
("account_code", &[I32, I32, I32, I32], &[I32]),
("account_code_size", &[I32], &[I32]),
("account_codehash", &[I32, I32], &[]),
("evm_gas_left", &[], &[I64]),
("evm_ink_left", &[], &[I64]),
("block_basefee", &[I32], &[]),
("chainid", &[], &[I64]),
("block_coinbase", &[I32], &[]),
("block_gas_limit", &[], &[I64]),
("block_number", &[], &[I64]),
("block_timestamp", &[], &[I64]),
("contract_address", &[I32], &[]),
("math_div", &[I32, I32], &[]),
("math_mod", &[I32, I32], &[]),
("math_pow", &[I32, I32], &[]),
("math_add_mod", &[I32, I32, I32], &[]),
("math_mul_mod", &[I32, I32, I32], &[]),
("msg_reentrant", &[], &[I32]),
("msg_sender", &[I32], &[]),
("msg_value", &[I32], &[]),
("native_keccak256", &[I32, I32, I32], &[]),
("tx_gas_price", &[I32], &[]),
("tx_ink_price", &[], &[I32]),
("tx_origin", &[I32], &[]),
("pay_for_memory_grow", &[I32], &[]),
];
1 change: 1 addition & 0 deletions arbitrator/arbutil/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ pub mod color;
pub mod crypto;
pub mod evm;
pub mod format;
pub mod hostios;
pub mod math;
pub mod operator;
pub mod pricing;
Expand Down
34 changes: 19 additions & 15 deletions arbitrator/jit/src/wavmio.rs
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,6 @@ use crate::{
};
use arbutil::{Color, PreimageType};
use caller_env::{GuestPtr, MemAccess};
use sha2::Sha256;
use sha3::{Digest, Keccak256};
use std::{
io,
io::{BufReader, BufWriter, ErrorKind},
Expand Down Expand Up @@ -170,19 +168,25 @@ pub fn resolve_preimage_impl(
error!("Missing requested preimage for hash {hash_hex} in {name}")
};

// Check if preimage rehashes to the provided hash. Exclude blob preimages
let calculated_hash: [u8; 32] = match preimage_type {
PreimageType::Keccak256 => Keccak256::digest(preimage).into(),
PreimageType::Sha2_256 => Sha256::digest(preimage).into(),
PreimageType::EthVersionedHash => *hash,
};
if calculated_hash != *hash {
error!(
"Calculated hash {} of preimage {} does not match provided hash {}",
hex::encode(calculated_hash),
hex::encode(preimage),
hex::encode(*hash)
);
#[cfg(debug_assertions)]
{
use sha2::Sha256;
use sha3::{Digest, Keccak256};

// Check if preimage rehashes to the provided hash. Exclude blob preimages
let calculated_hash: [u8; 32] = match preimage_type {
PreimageType::Keccak256 => Keccak256::digest(preimage).into(),
PreimageType::Sha2_256 => Sha256::digest(preimage).into(),
PreimageType::EthVersionedHash => *hash,
};
if calculated_hash != *hash {
error!(
"Calculated hash {} of preimage {} does not match provided hash {}",
hex::encode(calculated_hash),
hex::encode(preimage),
hex::encode(*hash)
);
}
}

if offset % 32 != 0 {
Expand Down
4 changes: 4 additions & 0 deletions arbitrator/prover/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,10 @@ enum-iterator = "2.0.1"
criterion = { version = "0.5.0", features = ["html_reports"] }
rand = "0.8.4"

[build-dependencies]
wasmer = { path = "../tools/wasmer/lib/api" }
arbutil = { path = "../arbutil/" }

[[bench]]
name = "merkle_bench"
harness = false
Expand Down
89 changes: 89 additions & 0 deletions arbitrator/prover/build.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
use arbutil::hostios::HOSTIOS;
use std::{env, fmt::Write, fs, path::Path};

pub fn gen_forwarder(out_path: &Path) {
let mut wat = String::new();
macro_rules! wln {
($($text:tt)*) => {
writeln!(wat, $($text)*).unwrap();
};
}
let s = " ";

wln!("(module");

macro_rules! group {
($list:expr, $kind:expr) => {
(!$list.is_empty())
.then(|| {
format!(
" ({} {})",
$kind,
$list
.iter()
.map(|x| x.to_string())
.collect::<Vec<_>>()
.join(" ")
)
})
.unwrap_or_default()
};
}

wln!("{s};; symbols to re-export");
for (name, ins, outs) in HOSTIOS {
let params = group!(ins, "param");
let result = group!(outs, "result");
wln!(r#"{s}(import "user_host" "{name}" (func $_{name}{params}{result}))"#);
}
wln!();

wln!("{s};; reserved offsets for future user_host imports");
for i in HOSTIOS.len()..512 {
wln!("{s}(func $reserved_{i} unreachable)");
}
wln!();

wln!(
"{s};; allows user_host to request a trap\n\
{s}(global $trap (mut i32) (i32.const 0))\n\
{s}(func $check\n\
{s}{s}global.get $trap ;; see if set\n\
{s}{s}(global.set $trap (i32.const 0)) ;; reset the flag\n\
{s}{s}(if (then (unreachable)))\n\
{s})\n\
{s}(func (export \"forward__set_trap\")\n\
{s}{s}(global.set $trap (i32.const 1))\n\
{s})\n"
);

wln!("{s};; user linkage");
for (name, ins, outs) in HOSTIOS {
let params = group!(ins, "param");
let result = group!(outs, "result");
wln!("{s}(func (export \"vm_hooks__{name}\"){params}{result}");

for i in 0..ins.len() {
wln!("{s}{s}local.get {i}");
}

wln!(
"{s}{s}call $_{name}\n\
{s}{s}call $check\n\
{s})"
);
}

wln!(")");
eprintln!("{}", &wat);

let wasm = wasmer::wat2wasm(wat.as_bytes()).unwrap();

fs::write(out_path, wasm.as_ref()).unwrap();
}

fn main() {
let out_dir = env::var("OUT_DIR").unwrap();
let forwarder_path = Path::new(&out_dir).join("forwarder.wasm");
gen_forwarder(&forwarder_path);
}
Loading

0 comments on commit 69505b5

Please sign in to comment.