diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..995f0ce --- /dev/null +++ b/Makefile @@ -0,0 +1,7 @@ +.PHONY: test +test: + MIX_ENV=test mix test + +.PHONY: format +format: + MIX_ENV=test mix format \ No newline at end of file diff --git a/README.md b/README.md index 59ca229..d251d44 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,16 @@ -# ProofOfReserves +# Proof Of Reserves River's Proof of Reserves implementation in Elixir. This implementation is based on BitMEX's Proof of Reseres Python implementation, found [here](https://github.com/BitMEX/proof-of-reserves-liabilities). This library is used by River to generate its Proof of Liabilities tree and to allow users to download and verify the Proof of Liabilities. +See River's Proof of Reserves [here](https://river.com/reserves). + ## Verifying River's Proof of Reserves -This library comes with a `verify_liabilities.exs` script that will verify River's Proof of Reserves and the balances of any accounts you provide. To run the script, run the following command: +This library comes with a `verify_liabilities.exs` script that will verify River's Proof of Reserves and the balances of any accounts you provide. -### 2. Fetch the Proof of Reserves Data +### 1. Fetch the Proof of Reserves Data Go to River's [Proof of Reserves](https://river.com/reserves) page. Log in with the email address you used to sign up for River. @@ -16,7 +18,7 @@ Click "Verify Liabilities" and select the "Verify on your computer" option. This Click "Continue" and you will be prompted to run the setup script also seen in the next step. -### 1. Setup the Project +### 2. Setup the Project In your terminal, run this command to clone the repository and install the dependencies. ```bash @@ -27,11 +29,11 @@ cd proof-of-reserves This script will install Erlang/Elixir and the project dependencies. It will then compile the library. -Back on the River [Proof of Reserves](https://river.com/reserves) page, click "Continue" and you will see the verification command, also shown in the step below. +Back in River's Proof of Reserves flow, click "Continue" and you will see the verification command, also shown in the step below. ### 2. Run the Verification Script -You will need to replace a few variables in the command below to run the script successfully. These values can all be found in the River "Verify Liabilities" flow. If you followed the steps above, you will now see the command in the final screen. The command on the River page will already have your email and account string(s) filled in. You will need to fill in the `` with the path to the CSV file you downloaded in Step 1. +You will need to replace a few variables in the command below to run the script successfully. These values can all be found in the River Proof of Reserves flow. If you followed the steps above, you will now see the command in the final screen. The command on the River page will already have your email and account string(s) filled in. You will need to fill in the `` with the path to the CSV file you downloaded in Step 1. - `` with the email address you used to sign up for River. - `` in the format `:`. @@ -45,7 +47,7 @@ If the verification is successful, it will output your balance and a summary of If you click continue on the River page, you can check that these balances are correct. -## Installation As a Library +## Installation as a Library This section is for developers who want to use this library in their own projects. diff --git a/data/.gitignore b/data/.gitignore new file mode 100644 index 0000000..16f2dc5 --- /dev/null +++ b/data/.gitignore @@ -0,0 +1 @@ +*.csv \ No newline at end of file diff --git a/lib/proof_of_reserves.ex b/lib/proof_of_reserves.ex index 592dc7a..60c1d15 100644 --- a/lib/proof_of_reserves.ex +++ b/lib/proof_of_reserves.ex @@ -75,48 +75,48 @@ defmodule ProofOfReserves do # ACCOUNT BALANCE CALCULATIONS @doc """ - find_account_leaves finds all leaves that belong to a particular + find_balances_for_accounts finds all leaves that belong to a particular account using the attestation_key """ - def find_account_leaves(leaves, block_height, account_id, account_subkey) do - attestation_key = Util.calculate_attestation_key(account_subkey, block_height, account_id) - find_account_leaves(leaves, attestation_key) - end + @spec find_balances_for_accounts( + list(MerkleSumTree.Node.t()), + non_neg_integer(), + list(%{ + account_id: non_neg_integer(), + account_subkey: binary() + }) + ) :: + list(%{ + account_id: non_neg_integer(), + balance: non_neg_integer(), + attestation_key: binary() + }) + def find_balances_for_accounts(leaves, block_height, accounts) do + account_balances = + Enum.map(accounts, fn %{account_id: account_id, account_subkey: subkey} -> + %{ + account_id: account_id, + balance: 0, + attestation_key: Util.calculate_attestation_key(subkey, block_height, account_id) + } + end) - def find_account_leaves(leaves, attestation_key) do leaves + # enumerate the leaves with their index since index is used in identifying th leaf hash |> Enum.with_index() - |> Enum.filter(fn {%{value: value, hash: hash}, idx} -> - Util.leaf_hash(value, attestation_key, idx) == hash + # reduce over the leaves and sum account balances for each account + |> Enum.reduce(account_balances, fn {%{value: value, hash: hash}, idx}, account_balances -> + # only one match will occur per this map + Enum.map(account_balances, fn %{balance: balance, attestation_key: attestation_key} = + account_balance -> + if Util.leaf_hash(value, attestation_key, idx) == hash do + # if the leaf hash matches, we add the value to the balance + Map.put(account_balance, :balance, balance + value) + else + account_balance + end + end) end) - |> Enum.map(fn {node, _} -> node end) - end - - @doc """ - get_account_balance calculates the balance of an account by summing the - values of all leaves that belong to the account in a given tree. - """ - @spec get_account_balance( - list(list(MerkleSumTree.Node.t())), - non_neg_integer(), - non_neg_integer(), - String.t() - ) :: non_neg_integer() - def get_account_balance(tree, block_height, account_id, account_subkey) do - attestation_key = Util.calculate_attestation_key(account_subkey, block_height, account_id) - get_account_balance(tree, attestation_key) - end - - @doc """ - get_account_balance calculates the balance of an account by summing the - values of all leaves that belong to the account in a given tree. - """ - @spec get_account_balance(list(list(MerkleSumTree.Node.t())), String.t()) :: non_neg_integer() - def get_account_balance(tree, attestation_key) do - tree - |> MerkleSumTree.get_leaves() - |> find_account_leaves(attestation_key) - |> Enum.reduce(0, fn node, acc -> acc + node.value end) end @doc """ diff --git a/lib/util.ex b/lib/util.ex index ddf0dda..7af3f8b 100644 --- a/lib/util.ex +++ b/lib/util.ex @@ -143,7 +143,7 @@ defmodule ProofOfReserves.Util do end @doc """ - calculate_attestation_key generates the sub nonce for the liability in a specific attestation. + calculate_attestation_key generates the attestation key for the account in a specific attestation. attestation_key = sha256(account_subkey || block_height || account_id) block_height and account_id must be 8-byte ints. """ @@ -165,4 +165,8 @@ defmodule ProofOfReserves.Util do msg = int_to_little(value, 8) <> int_to_little(leaf_index, 8) sha256hmac(attestation_key, msg) end + + def sats_to_btc(sats) do + sats / 100_000_000 + end end diff --git a/scripts/setup.sh b/scripts/setup.sh old mode 100644 new mode 100755 index 4131aad..60a1a97 --- a/scripts/setup.sh +++ b/scripts/setup.sh @@ -1,4 +1,3 @@ -#!/bin/bash set -euo pipefail # This script is used to setup the project, install erlang, elixir, and mix dependencies. @@ -10,14 +9,16 @@ function install_asdf_deps() { # MacOS Darwin*) echo "Setting up project for MacOS" - brew install coreutils curl git + brew install coreutils curl git openssl ;; # Linux Linux*) echo "Setting up project for Linux" - # Currently supporting APT & DNF only. - for candidate in apt-get dnf + required_packages="curl git automake autoconf libncurses-dev unzip gcc build-essential autoconf m4 libncurses-dev libwxgtk3.2-dev libwxgtk-webview3.2-dev libgl1-mesa-dev libglu1-mesa-dev libpng-dev libssh-dev unixodbc-dev xsltproc fop libxml2-utils openjdk-17-jdk" + + # Currently supporting APT, APK, and DNF only. + for candidate in apt-get dnf do if [ -x "$(command -v ${candidate})" ] then @@ -31,7 +32,8 @@ function install_asdf_deps() { exit +1 fi - sudo "${package_manager}" install -y curl git automake autoconf libncurses-dev + # Not all systems will have sudo + "${package_manager}" install -y $required_packages ;; *) >&2 echo "Unsupported OS: $(uname -s)" @@ -41,12 +43,13 @@ function install_asdf_deps() { } function install_asdf() { + echo "Installing asdf (version manager for erlang and elixir)..." ASDF_DIR="$HOME/.asdf" - # Avoid reinstalling if ASDF directory exists. + # Avoid reinstalling if asdf directory exists. if [ -d "$ASDF_DIR" ] then - echo "Existing ASDF directory found, skipping installation..." + echo "Existing asdf directory found, skipping installation..." return fi @@ -55,18 +58,28 @@ function install_asdf() { ASDF_ENV="$ASDF_DIR/asdf.sh" # Add asdf to your shell and source RC file to include ASDF path additions. + if [ -z "${SHELL+X}" ]; then + SHELL=$(echo $0) + fi + case $SHELL in */bash) - echo -e "\nsource $ASDF_ENV" >> ~/.bashrc + echo "\nsource $ASDF_ENV" >> ~/.bashrc source "$ASDF_ENV" + + echo "Sourcing asdf $ASDF_ENV" ;; */zsh) - echo -e "\nsource $ASDF_ENV" >> ~/.zshrc + echo "\nsource $ASDF_ENV" >> ~/.zshrc source "$ASDF_ENV" + + echo "Sourcing asdf $ASDF_ENV" ;; */fish) - echo -e "\nsource $HOME/.asdf/asdf.fish" >> ~/.config/fish/config.fish + echo "\nsource $HOME/.asdf/asdf.fish" >> ~/.config/fish/config.fish # TODO: Reload to adjust path? + + echo "Sourcing asdf $ASDF_ENV" ;; *) >&2 echo "Unsupported shell: $SHELL" @@ -78,31 +91,63 @@ function install_asdf() { } function install_erlang_elixir() { + echo "Installing erlang and elixir..." # Install erlang & elixir using asdf. # The plugins will be added to asdf and the versions will be installed according to the .tool-versions file. - asdf plugin add erlang - asdf plugin add elixir - asdf install + ASDF=$(which asdf) + if [ -f "$HOME/.asdf/asdf.sh" ]; then + echo "Sourcing asdf" + source "$HOME/.asdf/asdf.sh" + fi + + if [ -z "${ASDF+X}" ]; then + echo "asdf not found, please install asdf first" + exit +4 + fi + + echo "Adding asdf plugins: $ASDF" + $ASDF plugin add erlang || true + $ASDF plugin add elixir || true + $ASDF install } function install_mix_deps() { - # Fetch and compile the Elixir project dependencies. - mix deps.get && mix deps.compile + echo "Installing and compiling elixir dependencies..." + # Fetch and compile the Elixir project dependencies. + # Exclude dev and test dependencies to speed up the process. + MIX_ENV=prod mix deps.get && mix deps.compile } function compile_project() { + echo "Compiling elixir tool..." # Compile the Elixir project. - mix compile + MIX_ENV=prod mix compile } # This function is the main entry point for the setup script. # It installs the dependencies, sets up the project, and compiles the project. function setup_project() { + echo "Installing dependencies (elixir, erlang, etc.)..." + # Install dependencies for asdf and Elixir. + # Openssl is required for elixir's crypto library. install_asdf_deps - install_asdf - install_erlang_elixir + if ! command -v elixir &> /dev/null; then + echo "Elixir is not installed, searching for asdf to install elixir..." + if ! command -v asdf &> /dev/null; then + echo "asdf not found, installing dependencies for asdf..." + install_asdf + else + echo "asdf found, installing elixir..." + fi + install_erlang_elixir + else + echo "Elixir found, skipping installation..." + fi + install_mix_deps compile_project + + echo "Setup complete! You are ready to run the verification script." } setup_project diff --git a/test/proof_of_reserves_test.exs b/test/proof_of_reserves_test.exs index 154a73a..dae901c 100644 --- a/test/proof_of_reserves_test.exs +++ b/test/proof_of_reserves_test.exs @@ -211,6 +211,8 @@ defmodule ProofOfReservesTest do acct_key = Util.hex_to_bin!("abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234") + balance = 2 + liabilities = [ Liabilities.fake_liability(1), # create a liability with a different account_id so @@ -218,7 +220,7 @@ defmodule ProofOfReservesTest do ProofOfReserves.Liability.new( acct_id, acct_key, - 2 + balance ), Liabilities.fake_liability(3), Liabilities.fake_liability(4), @@ -233,13 +235,14 @@ defmodule ProofOfReservesTest do ) leaves = ProofOfReserves.MerkleSumTree.get_leaves(tree) - my_leaves = ProofOfReserves.find_account_leaves(leaves, block_height, acct_id, acct_key) - assert length(my_leaves) == 2 - attestation_key = Util.calculate_attestation_key(acct_key, block_height, acct_id) + [%{account_id: res_acct_id, balance: res_balance}] = + ProofOfReserves.find_balances_for_accounts(leaves, block_height, [ + %{account_id: acct_id, account_subkey: acct_key} + ]) - my_balance = ProofOfReserves.get_account_balance(tree, attestation_key) - assert my_balance == 2 + assert res_acct_id == acct_id + assert res_balance == balance end test "test larger balance", %{ @@ -278,14 +281,14 @@ defmodule ProofOfReservesTest do ) leaves = ProofOfReserves.MerkleSumTree.get_leaves(tree) - my_leaves = ProofOfReserves.find_account_leaves(leaves, block_height, acct_id, acct_key) - # we can't predict the exact number of leaves, but we know it's at least 3 - assert length(my_leaves) >= 3 - attestation_key = Util.calculate_attestation_key(acct_key, block_height, acct_id) + [%{account_id: res_acct_id, balance: res_balance}] = + ProofOfReserves.find_balances_for_accounts(leaves, block_height, [ + %{account_id: acct_id, account_subkey: acct_key} + ]) - my_balance = ProofOfReserves.get_account_balance(tree, attestation_key) - assert my_balance == balance + assert res_acct_id == acct_id + assert res_balance == balance end end end diff --git a/verify_liabilities.exs b/verify_liabilities.exs index 3da0d12..3166916 100644 --- a/verify_liabilities.exs +++ b/verify_liabilities.exs @@ -1,57 +1,63 @@ defmodule VerifyLiabilities do @moduledoc """ Verify liabilities for a given Proof of Reserves file. + You must pass set at least the following arguments: + --email + --file + --account : + + Multiple accounts can be provided by repeating the --account flag. """ - def run(args) do - args - |> parse_args() - |> case do - {:error, err} -> - IO.puts("error parsing args: #{err}") - {:ok, %{email: _email, filename: _filename, accounts: _accounts} = data} -> - data - |> verify_balances() - |> case do - {:error, err} -> - IO.puts("error verifying balances: #{err}") + def run(args) do + # leading newline to separate from terminal prompt + IO.puts("\nRunning River Proof of Liabilities verifier...\n") - {block_height, balances} when is_list(balances) -> - print_results(block_height, balances) - end + with {_, {:ok, %{filename: filename, accounts: accounts}}} <- {:parse_args, parse_args(args)}, + {_, {:ok, block_height, tree}} <- {:verify_tree, verify_tree(filename)}, + {_, {:ok, balances}} <- {:verify_balances, verify_balances(accounts, block_height, tree)} do + print_results(block_height, tree, balances) + else + {step, {:error, err}} -> + IO.puts("error in #{step}: #{err}") end end # Parse command line arguments. # First, we read in the email, filename and accounts. Then, we parse the accounts to ensure they are in the correct format. defp parse_args(args) do - # TODO - args |> Enum.chunk_every(2) |> Enum.reduce(%{}, fn # Only accept one email and one file ["--email", email], acc -> Map.put_new(acc, :email, email) + ["--file", filename], acc -> Map.put_new(acc, :filename, filename) + # accept multiple accounts, add to a list ["--account", account], acc -> Map.update(acc, :account_strs, [account], &[account | &1]) + # ignore other args - _, acc -> acc + _, acc -> + acc end) |> case do - %{account: []} -> + %{account_strs: []} -> {:error, "at least one account must be provided"} + %{email: email} when is_nil(email) or length(email) == 0 -> {:error, "email is required"} + %{filename: filename} when is_nil(filename) or length(filename) == 0 -> {:error, "filename is required"} %{email: email, filename: filename, account_strs: account_strs} = args - when is_binary(email) and is_binary(filename) and is_list(account_strs) -> - handle_args(args) + when is_binary(email) and is_binary(filename) and is_list(account_strs) -> + handle_args(args) + _ -> {:error, "invalid arguments"} end @@ -67,8 +73,11 @@ defmodule VerifyLiabilities do end end) |> case do - {:error, error} -> {:error, error} - accounts when is_list(accounts) -> {:ok, %{email: email, filename: filename, accounts: accounts}} + {:error, error} -> + {:error, error} + + accounts when is_list(accounts) -> + {:ok, %{email: email, filename: filename, accounts: accounts}} end end @@ -83,14 +92,42 @@ defmodule VerifyLiabilities do # Account ID and subkey are used to find account's leaves in the liabilities tree. # Account UID is used to print the account's balances in the final output. - {:ok, %{ - account_id: account_id, - account_uid: account_uid, - account_subkey: subkey - }} + {:ok, + %{ + account_id: account_id, + account_uid: account_uid, + account_subkey: subkey + }} _ -> - {:error, "invalid account format: #{account} does not match format :"} + {:error, + "invalid account format: #{account} does not match format :"} + end + end + + @doc """ + Verify the tree is valid. + We open the file and parse the liabilities. + Then, we check if the tree is valid. + """ + @spec verify_tree(String.t()) :: + {:ok, non_neg_integer(), list(list(ProofOfReserves.MerkleSumTree.Node.t()))} + | {:error, String.t()} + def verify_tree(filename) do + IO.puts("Verifying validity of Merkle Tree...") + + with {:ok, stream} <- open_file(filename) do + {block_height, tree} = ProofOfReserves.Liabilities.parse_liabilities(stream) + + if ProofOfReserves.MerkleSumTree.verify_tree?(tree) do + IO.puts("✅ Merkle Tree is valid! ✅") + {:ok, block_height, tree} + else + {:error, "tree is invalid"} + end + else + {:error, err} -> + {:error, err} end end @@ -99,35 +136,46 @@ defmodule VerifyLiabilities do We open the file and parse the liabilities. Then, for each account, we get the balance from the tree and return the results. """ - @spec verify_balances(%{filename: String.t(), accounts: list(%{ - account_uid: String.t(), - account_subkey: binary() - })} | {:error, String.t()}) :: list(%{ - account_uid: String.t(), - balance: integer() - }) | {:error, String.t()} - def verify_balances(%{filename: filename, accounts: accounts}) do - filename - |> open_file() - # Handle the case where the file open fails - |> case do - {:ok, stream} -> - {block_height, tree} = ProofOfReserves.Liabilities.parse_liabilities(stream) - - # TODO: preferably do it one by one. - balances = - Enum.map(accounts, fn %{account_uid: account_uid,account_id: account_id, account_subkey: account_subkey} -> - balance = ProofOfReserves.get_account_balance(tree, block_height, account_id, account_subkey) - %{account_uid: account_uid, balance: balance} - end) - - {block_height, balances} - # - {:error, err} -> - {:error, err} - end + @spec verify_balances( + %{ + accounts: + list(%{ + account_uid: String.t(), + account_subkey: binary() + }) + }, + non_neg_integer(), + list(list(ProofOfReserves.MerkleSumTree.Node.t())) + ) :: + {:ok, + list(%{ + account_uid: String.t(), + balance: integer() + })} + | {:error, String.t()} + def verify_balances(accounts, block_height, tree) do + account_uids = + Map.new(accounts, fn %{account_id: id, account_uid: account_uid} -> {id, account_uid} end) + + IO.puts("Verifying balances in Merkle Tree...") + + balances = + tree + |> ProofOfReserves.MerkleSumTree.get_leaves() + |> ProofOfReserves.find_balances_for_accounts(block_height, accounts) + |> Enum.map(fn %{balance: balance, account_id: account_id} -> + %{ + account_uid: Map.fetch!(account_uids, account_id), + balance: balance + } + end) + + {:ok, balances} end + # Open the file and return a stream. + # If the file cannot be opened, return an error. + @spec open_file(String.t()) :: {:ok, Stream.t()} | {:error, String.t()} defp open_file(filename) do try do {:ok, File.stream!(filename)} @@ -136,24 +184,74 @@ defmodule VerifyLiabilities do end end + # Check if the system has UTF-8 support (for emojis) + @spec utf8_support?() :: boolean() + defp utf8_support?() do + case System.get_env("LANG") || System.get_env("LC_CTYPE") do + nil -> false + locale -> String.contains?(locale, "UTF-8") + end + end + @doc """ Print the results to the console in a nice format. """ - @spec print_results(non_neg_integer(), list(%{ - account_uid: String.t(), - balance: integer() - })) :: :ok - def print_results(block_height,account_balances) do + @spec print_results( + non_neg_integer(), + list(list(ProofOfReserves.MerkleSumTree.Node.t())), + list(%{ + account_uid: String.t(), + balance: integer() + }) + ) :: :ok + def print_results(block_height, tree, account_balances) do + use_emoji? = utf8_support?() + + {:ok, root} = ProofOfReserves.MerkleSumTree.get_tree_root(tree) + IO.puts("=================================") IO.puts("River Proof of Liabilities Report") - IO.puts("================================") + IO.puts("=================================") + IO.puts("\n") + IO.puts("=============SUMMARY================") + IO.puts("Liabilities Root Hash: #{ProofOfReserves.Util.bin_to_hex!(root.hash)}") + IO.puts("Total Liabilities: #{ProofOfReserves.Util.sats_to_btc(root.value)} BTC") IO.puts("Block Height: #{block_height}") IO.puts("Verified at: #{DateTime.utc_now()}") - IO.puts("Accounts:") - # TODO: print in a nicer format + IO.puts("====================================") + IO.puts("\n") + IO.puts("============ACCOUNTS================") + for %{account_uid: account_uid, balance: balance} <- account_balances do - IO.puts(" #{account_uid}: #{balance}") + check = + cond do + not use_emoji? -> "" + balance == 0 -> " ℹ️ " + true -> " ✅" + end + + balance = + balance + |> ProofOfReserves.Util.sats_to_btc() + |> :erlang.float_to_binary(decimals: 8) + |> String.pad_leading(12) + + IO.puts(" ----------------------------------") + IO.puts(" | #{account_uid}: #{balance} BTC#{check} |") + IO.puts(" ----------------------------------") + end + + IO.puts("====================================") + + if Enum.any?(account_balances, fn %{balance: balance} -> balance == 0 end) do + msg = + " Accounts with zero balance either had no balance as of this Proof or were not found in the tree. + If you believe your balance was not zero as of this Proof, please double check the account ID and + attestation key you provided." + + msg = if use_emoji?, do: "ℹ️" <> msg, else: msg + + IO.puts(msg) end - IO.puts("================================") end end