From 8aebb53799287156bc31670af0b4fbf5483fbb95 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Thu, 9 Jul 2020 10:41:06 +0300 Subject: [PATCH] Add reorged docker compose (#1579) --- .circleci/config.yml | 88 ++++++++- Makefile | 35 +++- docker-compose.yml | 58 +++++- nginx.conf | 24 +++ priv/cabbage/Makefile | 6 + priv/cabbage/apps/itest/lib/account.ex | 19 +- priv/cabbage/apps/itest/lib/client.ex | 83 +++++++-- priv/cabbage/apps/itest/lib/gas.ex | 5 + .../apps/itest/lib/plasma_framework.ex | 2 + priv/cabbage/apps/itest/lib/poller.ex | 20 +- priv/cabbage/apps/itest/lib/reorg.ex | 173 ++++++++++++++++++ .../apps/itest/test/features/deposits.feature | 2 + .../test/features/in_flight_exits.feature | 2 +- .../test/itest/configuration_api_test.exs | 1 + .../apps/itest/test/itest/deposits_test.exs | 44 +++-- .../itest/test/itest/transactions_test.exs | 8 +- .../watcher_info_api_pagination_test.exs | 1 + priv/cabbage/config/config.exs | 5 +- priv/cabbage/docker-compose-reorg.yml | 108 +++++++++++ priv/cabbage/nginx.conf | 39 ++++ snapshot_reorg.env | 2 + 21 files changed, 667 insertions(+), 58 deletions(-) create mode 100644 nginx.conf create mode 100644 priv/cabbage/apps/itest/lib/reorg.ex create mode 100644 priv/cabbage/docker-compose-reorg.yml create mode 100644 priv/cabbage/nginx.conf create mode 100644 snapshot_reorg.env diff --git a/.circleci/config.yml b/.circleci/config.yml index 2e037464f7..815267bb64 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -160,9 +160,12 @@ jobs: - priv - data - snapshots.env + - snapshot_reorg.env + - nginx.conf - contract_addresses_template.env - localchain_contract_addresses.env + audit_deps: executor: builder environment: @@ -281,7 +284,7 @@ jobs: # if mix failed, then coveralls_report won't run, so signal done here and return original exit status (retval=$? && curl -k https://coveralls.io/webhook?repo_token=$COVERALLS_REPO_TOKEN -d "payload[build_num]=$CIRCLE_WORKFLOW_WORKSPACE_ID&payload[status]=done" && exit $retval) fi - + watcher_info_coveralls_and_integration_tests: executor: builder_pg_geth environment: @@ -526,6 +529,82 @@ jobs: cd priv/perf mix do credo, format --check-formatted --dry-run + test_docker_compose_reorg: + machine: + image: ubuntu-1604:201903-01 + environment: + REORG: true + steps: + - checkout + - run: + name: Setup data dir + command: | + [ -d data1 ] || mkdir data1 && chmod 777 data1 + [ -d data2 ] || mkdir data2 && chmod 777 data2 + [ -d data ] || mkdir data && chmod 777 data + - run: make docker-child_chain + - run: make docker-watcher + - run: make docker-watcher_info + - run: + name: Start daemon services + command: | + cd priv/cabbage + make start_daemon_services_reorg || (START_RESULT=$?; docker-compose logs; exit $START_RESULT;) + - run: + name: Print docker states + command: | + docker image ls + docker-compose ps + - restore_cache: + key: v2-asdf-install + - run: + name: Install Erlang and Elixir + command: | + [ -d ~/.asdf-vm ] || git clone https://github.com/asdf-vm/asdf.git ~/.asdf-vm --branch v0.7.4 + echo 'source ~/.asdf-vm/asdf.sh' >> $BASH_ENV + source $BASH_ENV + asdf plugin-add erlang || asdf plugin-update erlang + asdf plugin-add elixir || asdf plugin-update elixir + asdf install erlang 22.3 + asdf install elixir 1.10.2 + asdf global elixir 1.10.2 + asdf global erlang 22.3 + no_output_timeout: 2400 + - save_cache: + key: v2-asdf-install + paths: + - ~/.asdf + - ~/.asdf-vm + - run: make install-hex-rebar + - run: sh .circleci/status.sh + - restore_cache: + key: v2-mix-specs-cache-{{ .Branch }}-{{ checksum "mix.lock" }} + - run: + name: Print watcher logs + command: docker-compose -f docker-compose.yml -f ./priv/cabbage/docker-compose-reorg.yml -f ./priv/cabbage/docker-compose-cabbage.yml logs --follow watcher + background: true + - run: + name: Print watcher_info logs + command: docker-compose -f docker-compose.yml -f ./priv/cabbage/docker-compose-reorg.yml -f ./priv/cabbage/docker-compose-cabbage.yml logs --follow watcher_info + background: true + - run: + name: Print childchain logs + command: docker-compose -f docker-compose.yml -f ./priv/cabbage/docker-compose-reorg.yml -f ./priv/cabbage/docker-compose-cabbage.yml logs --follow childchain + background: true + - run: + name: Print reorg logs + command: docker-compose -f docker-compose.yml -f ./priv/cabbage/docker-compose-reorg.yml -f ./priv/cabbage/docker-compose-cabbage.yml logs --follow | grep "reorg" + background: true + - run: + name: Run specs + command: | + cd priv/cabbage + make install + make generate_api_code + mix deps.get + mix test --only reorg --trace + no_output_timeout: 30m + test_barebone_release: machine: image: ubuntu-1604:201903-01 @@ -653,7 +732,7 @@ jobs: steps: - checkout - run: make docker-watcher WATCHER_IMAGE_NAME=$WATCHER_IMAGE_NAME - - run: + - run: name: "cp release" command: | mkdir current_release/ @@ -671,7 +750,7 @@ jobs: steps: - checkout - run: make docker-watcher_info WATCHER_INFO_IMAGE_NAME=$WATCHER_INFO_IMAGE_NAME - - run: + - run: name: "cp release" command: | mkdir current_release/ @@ -804,6 +883,9 @@ workflows: - test_docker_compose_release: requires: [build] filters: *all_branches_and_tags + - test_docker_compose_reorg: + requires: [build] + filters: *all_branches_and_tags - audit_deps: requires: [build] filters: *all_branches_and_tags diff --git a/Makefile b/Makefile index e54ed87905..01bb7e636a 100644 --- a/Makefile +++ b/Makefile @@ -219,6 +219,37 @@ init-contracts: clean-contracts PLASMA_FRAMEWORK_TX_HASH=$$PLASMA_FRAMEWORK_TX_HASH PLASMA_FRAMEWORK=$$PLASMA_FRAMEWORK \ PAYMENT_EIP712_LIBMOCK=$$PAYMENT_EIP712_LIBMOCK MERKLE_WRAPPER=$$MERKLE_WRAPPER ERC20_MINTABLE=$$ERC20_MINTABLE +init-contracts-reorg: clean-contracts + mkdir data1/ || true && \ + mkdir data2/ || true && \ + mkdir data/ || true && \ + URL=$$(grep "SNAPSHOT" snapshot_reorg.env | cut -d'=' -f2-) && \ + curl -o data1/snapshot.tar.gz $$URL && \ + cd data1 && \ + tar --strip-components 1 -zxvf snapshot.tar.gz data/geth && \ + tar --exclude=data/* -xvzf snapshot.tar.gz && \ + mv snapshot.tar.gz ../data2/snapshot.tar.gz && \ + cd ../data2 && \ + tar --strip-components 1 -zxvf snapshot.tar.gz data/geth && \ + tar --exclude=data/* -xvzf snapshot.tar.gz && \ + mv snapshot.tar.gz ../data/snapshot.tar.gz && \ + cd ../data && \ + tar --strip-components 1 -zxvf snapshot.tar.gz data/geth && \ + tar --exclude=data/* -xvzf snapshot.tar.gz && \ + AUTHORITY_ADDRESS=$$(cat plasma-contracts/build/authority_address) && \ + ETH_VAULT=$$(cat plasma-contracts/build/eth_vault) && \ + ERC20_VAULT=$$(cat plasma-contracts/build/erc20_vault) && \ + PAYMENT_EXIT_GAME=$$(cat plasma-contracts/build/payment_exit_game) && \ + PLASMA_FRAMEWORK_TX_HASH=$$(cat plasma-contracts/build/plasma_framework_tx_hash) && \ + PLASMA_FRAMEWORK=$$(cat plasma-contracts/build/plasma_framework) && \ + PAYMENT_EIP712_LIBMOCK=$$(cat plasma-contracts/build/paymentEip712LibMock) && \ + MERKLE_WRAPPER=$$(cat plasma-contracts/build/merkleWrapper) && \ + ERC20_MINTABLE=$$(cat plasma-contracts/build/erc20Mintable) && \ + sh ../bin/generate-localchain-env AUTHORITY_ADDRESS=$$AUTHORITY_ADDRESS ETH_VAULT=$$ETH_VAULT \ + ERC20_VAULT=$$ERC20_VAULT PAYMENT_EXIT_GAME=$$PAYMENT_EXIT_GAME \ + PLASMA_FRAMEWORK_TX_HASH=$$PLASMA_FRAMEWORK_TX_HASH PLASMA_FRAMEWORK=$$PLASMA_FRAMEWORK \ + PAYMENT_EIP712_LIBMOCK=$$PAYMENT_EIP712_LIBMOCK MERKLE_WRAPPER=$$MERKLE_WRAPPER ERC20_MINTABLE=$$ERC20_MINTABLE + .PHONY: init-contracts # @@ -227,6 +258,8 @@ init-contracts: clean-contracts init_test: init-contracts +init_test_reorg: init-contracts-reorg + test: mix test --include test --exclude common --exclude watcher --exclude watcher_info --exclude child_chain @@ -377,7 +410,7 @@ docker-remote-childchain: ### start-services: SNAPSHOT=SNAPSHOT_MIX_EXIT_PERIOD_SECONDS_120 make init_test && \ - docker-compose up geth postgres + docker-compose up geth nginx postgres start-child_chain: . ${OVERRIDING_VARIABLES} && \ diff --git a/docker-compose.yml b/docker-compose.yml index 4a5ae849dc..03cb7e389d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,6 +14,9 @@ services: interval: 5s timeout: 3s retries: 5 + networks: + chain_net: + ipv4_address: 172.27.0.106 geth: image: ethereum/client-go:v1.9.12 @@ -47,8 +50,8 @@ services: --mine \ --allow-insecure-unlock ports: - - "8545:8545" - - "8546:8546" + - "8555:8545" + - "8556:8546" expose: - "8546" - "8545" @@ -59,6 +62,27 @@ services: interval: 5s timeout: 3s retries: 5 + networks: + chain_net: + ipv4_address: 172.27.0.101 + + nginx: + image: nginx:latest + container_name: nginx + volumes: + - ./nginx.conf:/etc/nginx/nginx.conf + ports: + - 8545:80 + - 8546:81 + - 443:443 + healthcheck: + test: curl nginx:80 + interval: 5s + timeout: 3s + retries: 5 + networks: + chain_net: + ipv4_address: 172.27.0.102 childchain: image: omisego/child_chain:latest @@ -68,7 +92,7 @@ services: - ./localchain_contract_addresses.env environment: - ETHEREUM_NETWORK=LOCALCHAIN - - ETHEREUM_RPC_URL=http://geth:8545 + - ETHEREUM_RPC_URL=http://172.27.0.102:80 - APP_ENV=local_docker_development - DD_HOSTNAME=datadog - DD_DISABLED=true @@ -97,8 +121,11 @@ services: retries: 5 start_period: 30s depends_on: - geth: + nginx: condition: service_healthy + networks: + chain_net: + ipv4_address: 172.27.0.103 watcher: image: omisego/watcher:latest @@ -108,8 +135,8 @@ services: - ./localchain_contract_addresses.env environment: - ETHEREUM_NETWORK=LOCALCHAIN - - ETHEREUM_RPC_URL=http://geth:8545 - - CHILD_CHAIN_URL=http://childchain:9656 + - ETHEREUM_RPC_URL=http://172.27.0.102:80 + - CHILD_CHAIN_URL=http://172.27.0.103:9656 - PORT=7434 - APP_ENV=local_docker_development - DD_HOSTNAME=datadog @@ -139,6 +166,9 @@ services: depends_on: childchain: condition: service_healthy + networks: + chain_net: + ipv4_address: 172.27.0.104 watcher_info: image: omisego/watcher_info:latest @@ -148,9 +178,9 @@ services: - ./localchain_contract_addresses.env environment: - ETHEREUM_NETWORK=LOCALCHAIN - - ETHEREUM_RPC_URL=http://geth:8545 - - CHILD_CHAIN_URL=http://childchain:9656 - - DATABASE_URL=postgresql://omisego_dev:omisego_dev@postgres:5432/omisego_dev + - ETHEREUM_RPC_URL=http://172.27.0.102:80 + - CHILD_CHAIN_URL=http://172.27.0.103:9656 + - DATABASE_URL=postgresql://omisego_dev:omisego_dev@172.27.0.106:5432/omisego_dev - PORT=7534 - APP_ENV=local_docker_development - DD_HOSTNAME=datadog @@ -181,3 +211,13 @@ services: condition: service_healthy postgres: condition: service_healthy + networks: + chain_net: + ipv4_address: 172.27.0.105 + +networks: + chain_net: + driver: bridge + ipam: + config: + - subnet: 172.27.0.0/24 diff --git a/nginx.conf b/nginx.conf new file mode 100644 index 0000000000..9775fb7319 --- /dev/null +++ b/nginx.conf @@ -0,0 +1,24 @@ +events {} +http { + server { + listen 80; + + location / { + proxy_pass http://172.27.0.101:8545; + } + } + + server { + listen 81; + + location / { + proxy_pass http://172.27.0.101:8546; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_connect_timeout 7d; + proxy_send_timeout 7d; + proxy_read_timeout 7d; + } + } +} \ No newline at end of file diff --git a/priv/cabbage/Makefile b/priv/cabbage/Makefile index 16a7abc2db..6b68081950 100644 --- a/priv/cabbage/Makefile +++ b/priv/cabbage/Makefile @@ -14,6 +14,12 @@ start_daemon_services: cd priv/cabbage/ && \ docker-compose -f ../../docker-compose.yml -f docker-compose-cabbage.yml up -d +start_daemon_services_reorg: + cd ../../ && \ + make init_test_reorg && \ + cd priv/cabbage/ && \ + docker-compose -f ../../docker-compose.yml -f docker-compose-reorg.yml -f docker-compose-cabbage.yml up -d + generate-security_critical_api_specs: priv/openapitools/openapi-generator-cli generate -i ../../apps/omg_watcher_rpc/priv/swagger/security_critical_api_specs.yaml -g elixir -o apps/watcher_security_critical_api diff --git a/priv/cabbage/apps/itest/lib/account.ex b/priv/cabbage/apps/itest/lib/account.ex index 1c80a6f6d7..61ec0d975f 100644 --- a/priv/cabbage/apps/itest/lib/account.ex +++ b/priv/cabbage/apps/itest/lib/account.ex @@ -17,6 +17,7 @@ defmodule Itest.Account do Maintaining used accounts state so that we're able to run tests multiple times. """ + alias Itest.Reorg alias Itest.Transactions.Encoding import Itest.Poller, only: [wait_on_receipt_confirmed: 1] @@ -24,7 +25,7 @@ defmodule Itest.Account do def take_accounts(number_of_accounts) do 1..number_of_accounts |> Task.async_stream(fn _ -> account() end, - timeout: 60_000, + timeout: 120_000, on_timeout: :kill_task, max_concurrency: System.schedulers_online() * 2 ) @@ -46,7 +47,11 @@ defmodule Itest.Account do wait_on_receipt_confirmed(receipt_hash) - {:ok, true} = Ethereumex.HttpClient.request("personal_unlockAccount", [addr, "dev.period", 0], []) + if Application.get_env(:cabbage, :reorg) do + Reorg.unlock_account(addr, passphrase) + else + {:ok, true} = Ethereumex.HttpClient.request("personal_unlockAccount", [addr, passphrase, 0], []) + end {addr, account_priv_enc} end @@ -79,9 +84,13 @@ defmodule Itest.Account do {:ok, address} end + defp hash(message), do: ExthCrypto.Hash.hash(message, ExthCrypto.Hash.kec()) + defp create_account_from_secret(secret, passphrase) do - Ethereumex.HttpClient.request("personal_importRawKey", [secret, passphrase], []) + if Application.get_env(:cabbage, :reorg) do + Reorg.create_account_from_secret(secret, passphrase) + else + Ethereumex.HttpClient.request("personal_importRawKey", [secret, passphrase], []) + end end - - defp hash(message), do: ExthCrypto.Hash.hash(message, ExthCrypto.Hash.kec()) end diff --git a/priv/cabbage/apps/itest/lib/client.ex b/priv/cabbage/apps/itest/lib/client.ex index 669bc8fb63..d8e2253893 100644 --- a/priv/cabbage/apps/itest/lib/client.ex +++ b/priv/cabbage/apps/itest/lib/client.ex @@ -56,7 +56,7 @@ defmodule Itest.Client do {:ok, receipt_hash} end - def create_transaction(amount_in_wei, input_address, output_address, currency \\ Currency.ether()) do + def create_transaction(amount_in_wei, input_address, output_address, currency \\ Currency.ether(), tries \\ 120) do transaction = %CreateTransactionsBodySchema{ owner: input_address, payments: [ @@ -71,18 +71,8 @@ defmodule Itest.Client do {:ok, response} = Transaction.create_transaction(WatcherInfo.new(), transaction) - %{ - "result" => "complete", - "transactions" => [ - %{ - "sign_hash" => sign_hash, - "typed_data" => typed_data, - "txbytes" => txbytes - } - ] - } = Jason.decode!(response.body)["data"] - - {:ok, [sign_hash, typed_data, txbytes]} + result = Jason.decode!(response.body)["data"] + process_transaction_result(result, amount_in_wei, input_address, output_address, currency, tries) end def submit_transaction(typed_data, sign_hash, private_keys) do @@ -161,6 +151,73 @@ defmodule Itest.Client do do_wait_until_tx_sync_to_watcher(tx_id, @default_retry_attempts) end + def wait_until_block_number(block_number) do + {:ok, current_block_number} = get_latest_block_number() + + if current_block_number >= block_number do + :ok + else + Process.sleep(1_000) + + wait_until_block_number(block_number) + end + end + + def get_latest_block_number() do + case Ethereumex.HttpClient.eth_get_block_by_number("latest", false) do + {:ok, %{"number" => "0x" <> number_hex}} -> + {return, ""} = Integer.parse(number_hex, 16) + {:ok, return} + + _other -> + Process.sleep(1_000) + get_latest_block_number() + end + end + + defp process_transaction_result( + %{ + "result" => "complete", + "transactions" => [ + %{ + "sign_hash" => sign_hash, + "typed_data" => typed_data, + "txbytes" => txbytes + } + ] + }, + _amount_in_wei, + _input_address, + _output_address, + _currency, + _tries + ) do + {:ok, [sign_hash, typed_data, txbytes]} + end + + defp process_transaction_result( + %{"code" => "create:client_error", "messages" => %{"code" => "operation:service_unavailable"}} = result, + _amount_in_wei, + _input_address, + _output_address, + _currency, + 0 + ) do + result + end + + defp process_transaction_result( + %{"code" => "create:client_error", "messages" => %{"code" => "operation:service_unavailable"}}, + amount_in_wei, + input_address, + output_address, + currency, + tries + ) do + Process.sleep(1_000) + create_transaction(amount_in_wei, input_address, output_address, currency, tries - 1) + end + defp do_wait_until_tx_sync_to_watcher(_tx_id, 0), do: :wait_until_tx_sync_failed defp do_wait_until_tx_sync_to_watcher(tx_id, retry) do diff --git a/priv/cabbage/apps/itest/lib/gas.ex b/priv/cabbage/apps/itest/lib/gas.ex index a99b59ab57..1c058230c7 100644 --- a/priv/cabbage/apps/itest/lib/gas.ex +++ b/priv/cabbage/apps/itest/lib/gas.ex @@ -36,6 +36,11 @@ defmodule Itest.Gas do {{:ok, nil}, {:ok, nil}} -> 0 + + # reorg + {{:ok, nil}, {:ok, %{"blockHash" => nil, "blockNumber" => nil}}} -> + Logger.info("transaction #{receipt_hash} is in reorg") + 0 end end end diff --git a/priv/cabbage/apps/itest/lib/plasma_framework.ex b/priv/cabbage/apps/itest/lib/plasma_framework.ex index 0c6c5fb0a8..a7b2672b16 100644 --- a/priv/cabbage/apps/itest/lib/plasma_framework.ex +++ b/priv/cabbage/apps/itest/lib/plasma_framework.ex @@ -53,6 +53,7 @@ defmodule Itest.PlasmaFramework do def exit_game_contract_address(tx_type) do data = ABI.encode("exitGames(uint256)", [tx_type]) + {:ok, result} = Ethereumex.HttpClient.eth_call(%{to: address(), data: Encoding.to_hex(data)}) result @@ -64,6 +65,7 @@ defmodule Itest.PlasmaFramework do defp get_vault(id) do data = ABI.encode("vaults(uint256)", [id]) + {:ok, result} = Ethereumex.HttpClient.eth_call(%{to: address(), data: Encoding.to_hex(data)}) result diff --git a/priv/cabbage/apps/itest/lib/poller.ex b/priv/cabbage/apps/itest/lib/poller.ex index c8da3c9489..43c67c88c2 100644 --- a/priv/cabbage/apps/itest/lib/poller.ex +++ b/priv/cabbage/apps/itest/lib/poller.ex @@ -28,7 +28,7 @@ defmodule Itest.Poller do alias WatcherSecurityCriticalAPI.Api.Status @sleep_retry_sec 1_000 - @retry_count 30 + @retry_count 240 def pull_for_utxo_until_recognized_deposit(account, amount, currency, blknum) do payload = %AddressBodySchema1{address: account} @@ -38,6 +38,13 @@ defmodule Itest.Poller do def pull_api_until_successful(module, function, connection, payload \\ nil), do: pull_api_until_successful(module, function, connection, payload, @retry_count) + def get_transaction_block_number(receipt_hash) do + {:ok, %{"blockNumber" => "0x" <> number_hex}} = get_transaction_receipt(receipt_hash) + {number, ""} = Integer.parse(number_hex, 16) + + {:ok, number} + end + @doc """ API:: If we're trying to transact with UTXOs that were not recognized *yet* """ @@ -96,6 +103,7 @@ defmodule Itest.Poller do ####################################################################################################### ### PRIVATE ####################################################################################################### + defp pull_api_until_successful(module, function, connection, payload, 0), do: Jason.decode!(apply(module, function, [connection, payload]))["data"] @@ -138,11 +146,7 @@ defmodule Itest.Poller do Process.sleep(@sleep_retry_sec) do_wait_on_receipt_status(receipt_hash, expected_status, counter - 1) - {:error, :closed} -> - Process.sleep(@sleep_retry_sec) - do_wait_on_receipt_status(receipt_hash, expected_status, counter - 1) - - {:error, :socket_closed_remotely} -> + {:error, _} -> Process.sleep(@sleep_retry_sec) do_wait_on_receipt_status(receipt_hash, expected_status, counter - 1) @@ -295,6 +299,10 @@ defmodule Itest.Poller do Process.sleep(@sleep_retry_sec) submit_typed(typed_data_signed, counter - 1) + %{"messages" => %{"code" => "operation:service_unavailable"}} -> + Process.sleep(@sleep_retry_sec) + submit_typed(typed_data_signed, counter - 1) + %{"txhash" => _} -> SubmitTransactionResponse.to_struct(decoded_response) end diff --git a/priv/cabbage/apps/itest/lib/reorg.ex b/priv/cabbage/apps/itest/lib/reorg.ex new file mode 100644 index 0000000000..5ffd5029c8 --- /dev/null +++ b/priv/cabbage/apps/itest/lib/reorg.ex @@ -0,0 +1,173 @@ +# Copyright 2019-2020 OmiseGO Pte Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +defmodule Itest.Reorg do + @moduledoc """ + Chain reorg triggering logic. + """ + + alias Itest.Client + + require Logger + + @node1 "geth-1" + @node2 "geth-2" + + @rpc_nodes ["http://localhost:9000", "http://localhost:9001"] + + def execute_in_reorg(func) do + if Application.get_env(:cabbage, :reorg) do + wait_for_nodes_to_be_in_sync() + + {:ok, block_before_reorg} = Client.get_latest_block_number() + + pause_container!(@node1) + unpause_container!(@node2) + + :ok = Client.wait_until_block_number(block_before_reorg + 4) + + func.() + + {:ok, block_on_the_first_node1} = Client.get_latest_block_number() + + :ok = Client.wait_until_block_number(block_on_the_first_node1 + 2) + + {:ok, block_on_the_first_node2} = Client.get_latest_block_number() + + pause_container!(@node2) + unpause_container!(@node1) + + :ok = Client.wait_until_block_number(block_before_reorg + 4) + + response = func.() + + :ok = Client.wait_until_block_number(block_on_the_first_node2 + 2) + + unpause_container!(@node2) + unpause_container!(@node1) + + wait_for_nodes_to_be_in_sync() + + response + else + func.() + end + end + + def create_account_from_secret(secret, passphrase) do + result = + Enum.map(@rpc_nodes, fn rpc_node -> + with_retries(fn -> + Ethereumex.HttpClient.request("personal_importRawKey", [secret, passphrase], url: rpc_node) + end) + end) + + List.first(result) + end + + def unlock_account(addr, passphrase) do + Enum.each(@rpc_nodes, fn rpc_node -> + {:ok, true} = + with_retries(fn -> + Ethereumex.HttpClient.request("personal_unlockAccount", [addr, passphrase, 0], url: rpc_node) + end) + end) + end + + def wait_until_peer_count(peer_count) do + _ = Logger.info("Waiting for peer count to equal to #{peer_count}") + + Enum.each(@rpc_nodes, fn node -> do_wait_until_peer_count(node, peer_count) end) + end + + defp wait_for_nodes_to_be_in_sync() do + wait_until_peer_count(1) && Enum.each(@rpc_nodes, fn rpc_node -> wait_until_synced(rpc_node) end) && + wait_until_peer_count(1) + end + + defp wait_until_synced(node) do + case Ethereumex.HttpClient.request("eth_syncing", [], url: node) do + {:ok, false} -> + :ok + + _other -> + Process.sleep(1_000) + wait_until_synced(node) + end + end + + defp do_wait_until_peer_count(node, peer_count) do + case Ethereumex.HttpClient.request("net_peerCount", [], url: node) do + {:ok, "0x" <> number_hex} -> + {count, ""} = Integer.parse(number_hex, 16) + + if count >= peer_count do + :ok + else + Process.sleep(1_000) + do_wait_until_peer_count(node, peer_count) + end + + _other -> + Process.sleep(1_000) + do_wait_until_peer_count(node, peer_count) + end + end + + defp pause_container!(container) do + pause_container_url = "http+unix://%2Fvar%2Frun%2Fdocker.sock/containers/#{container}/pause" + + pause_response = post_request!(pause_container_url) + + Logger.info("Chain reorg: pause response - #{inspect(pause_response)}") + + # the pause operation is not instant, let's wait for 2s + Process.sleep(2_000) + + 204 = pause_response.status_code + end + + defp unpause_container!(container) do + unpause_container_url = "http+unix://%2Fvar%2Frun%2Fdocker.sock/containers/#{container}/unpause" + + unpause_response = post_request!(unpause_container_url) + + # the unpause operation is not instant, let's wait for 2s + Process.sleep(2_000) + + Logger.info("Chain reorg: unpause response - #{inspect(unpause_response)}") + end + + defp with_retries(func, total_time \\ 510, current_time \\ 0) do + case func.() do + {:ok, _} = result -> + result + + result -> + if current_time < total_time do + Process.sleep(1_000) + with_retries(func, total_time, current_time + 1) + else + result + end + end + end + + defp post_request!(url) do + HTTPoison.post!(url, "", [{"content-type", "application/json"}], + timeout: 60_000, + recv_timeout: 60_000 + ) + end +end diff --git a/priv/cabbage/apps/itest/test/features/deposits.feature b/priv/cabbage/apps/itest/test/features/deposits.feature index 878d0a0866..6d13ae0732 100644 --- a/priv/cabbage/apps/itest/test/features/deposits.feature +++ b/priv/cabbage/apps/itest/test/features/deposits.feature @@ -4,10 +4,12 @@ Feature: Deposits When Alice deposits "1" ETH to the root chain And Alice deposits "1" ETH to the root chain Then Alice should have "2" ETH on the child chain + Then Alice should have the root chain balance changed by "-2" ETH Scenario: Alice sends Bob funds When Alice deposits "1" ETH to the root chain And Alice deposits "1" ETH to the root chain Then Alice should have "2" ETH on the child chain + Then Alice should have the root chain balance changed by "-2" ETH When Alice sends Bob "1" ETH on the child chain Then Bob should have "1" ETH on the child chain \ No newline at end of file diff --git a/priv/cabbage/apps/itest/test/features/in_flight_exits.feature b/priv/cabbage/apps/itest/test/features/in_flight_exits.feature index d40da0679b..e0fff4d775 100644 --- a/priv/cabbage/apps/itest/test/features/in_flight_exits.feature +++ b/priv/cabbage/apps/itest/test/features/in_flight_exits.feature @@ -14,7 +14,7 @@ Feature: In Flight Exits And Bob starts a piggybacked in flight exit using his most recently prepared in flight exit data And Alice fully challenges Bobs most recent invalid in flight exit Then "Alice" can processes its own most recent in flight exit - + Scenario: Standard exit invalidated with an In Flight Exit Given "Alice" deposits "10" ETH to the root chain Then "Alice" should have "10" ETH on the child chain after finality margin diff --git a/priv/cabbage/apps/itest/test/itest/configuration_api_test.exs b/priv/cabbage/apps/itest/test/itest/configuration_api_test.exs index 70581b7c01..c116706bbc 100644 --- a/priv/cabbage/apps/itest/test/itest/configuration_api_test.exs +++ b/priv/cabbage/apps/itest/test/itest/configuration_api_test.exs @@ -13,6 +13,7 @@ # limitations under the License. defmodule ConfigurationRetrievalTests do use Cabbage.Feature, async: true, file: "configuration_api.feature" + alias Itest.Transactions.Encoding require Logger diff --git a/priv/cabbage/apps/itest/test/itest/deposits_test.exs b/priv/cabbage/apps/itest/test/itest/deposits_test.exs index b6715a45e0..cbd4b44d8c 100644 --- a/priv/cabbage/apps/itest/test/itest/deposits_test.exs +++ b/priv/cabbage/apps/itest/test/itest/deposits_test.exs @@ -13,12 +13,15 @@ # limitations under the License. defmodule DepositsTests do use Cabbage.Feature, async: true, file: "deposits.feature" + @moduletag :reorg require Logger alias Itest.Account alias Itest.ApiModel.WatcherSecurityCriticalConfiguration alias Itest.Client + alias Itest.Poller + alias Itest.Reorg alias Itest.Transactions.Currency setup do @@ -33,9 +36,11 @@ defmodule DepositsTests do initial_balance = Itest.Poller.root_chain_get_balance(alice_account) {:ok, receipt_hash} = - amount - |> Currency.to_wei() - |> Client.deposit(alice_account, Itest.PlasmaFramework.vault(Currency.ether())) + Reorg.execute_in_reorg(fn -> + amount + |> Currency.to_wei() + |> Client.deposit(alice_account, Itest.PlasmaFramework.vault(Currency.ether())) + end) gas_used = Client.get_gas_used(receipt_hash) @@ -44,17 +49,17 @@ defmodule DepositsTests do {current_gas, current_gas + gas_used} end) - balance_after_deposit = Itest.Poller.root_chain_get_balance(alice_account) + state = + new_state + |> Map.put_new(:alice_initial_balance, initial_balance) + |> Map.put(:deposit_transaction_hash, receipt_hash) - state = Map.put_new(new_state, :alice_ethereum_balance, balance_after_deposit) - {:ok, Map.put_new(state, :alice_initial_balance, initial_balance)} + {:ok, state} end defthen ~r/^Alice should have "(?[^"]+)" ETH on the child chain$/, %{amount: amount}, - %{alice_account: alice_account} = state do - geth_block_every = 1 - + %{alice_account: alice_account, deposit_transaction_hash: deposit_transaction_hash} = state do {:ok, response} = WatcherSecurityCriticalAPI.Api.Configuration.configuration_get(WatcherSecurityCriticalAPI.Connection.new()) @@ -62,13 +67,9 @@ defmodule DepositsTests do WatcherSecurityCriticalConfiguration.to_struct(Jason.decode!(response.body)["data"]) finality_margin_blocks = watcher_security_critical_config.deposit_finality_margin - to_miliseconds = 1000 - finality_margin_blocks - |> Kernel.*(geth_block_every) - |> Kernel.*(to_miliseconds) - |> Kernel.round() - |> Process.sleep() + {:ok, number} = Poller.get_transaction_block_number(deposit_transaction_hash) + :ok = Client.wait_until_block_number(number + finality_margin_blocks) expecting_amount = Currency.to_wei(amount) @@ -95,6 +96,19 @@ defmodule DepositsTests do {:ok, state} end + defthen ~r/^Alice should have the root chain balance changed by "(?[^"]+)" ETH$/, + %{amount: amount}, + %{ + alice_account: alice_account, + alice_initial_balance: alice_initial_balance, + gas: gas + } = state do + balance_after_deposit = Itest.Poller.root_chain_get_balance(alice_account) + assert_equal(Currency.to_wei(amount), balance_after_deposit + gas - alice_initial_balance, "For #{alice_account}.") + + {:ok, state} + end + defthen ~r/^Bob should have "(?[^"]+)" ETH on the child chain$/, %{amount: amount}, %{bob_account: bob_account} = state do diff --git a/priv/cabbage/apps/itest/test/itest/transactions_test.exs b/priv/cabbage/apps/itest/test/itest/transactions_test.exs index 83d608cd5b..40755112da 100644 --- a/priv/cabbage/apps/itest/test/itest/transactions_test.exs +++ b/priv/cabbage/apps/itest/test/itest/transactions_test.exs @@ -63,7 +63,7 @@ defmodule TransactionsTests do data = [{key, balance_after_deposit} | data] Map.put(state, String.to_atom("alice_data_#{index}"), data) end, - timeout: 60_000, + timeout: 240_000, on_timeout: :kill_task, max_concurrency: @num_accounts ) @@ -91,7 +91,7 @@ defmodule TransactionsTests do # Alice needs to sign 2 inputs of 1 Eth, 1 for Bob and 1 for the fees _ = Client.submit_transaction(typed_data, sign_hash, [alice_pkey, alice_pkey]) end, - timeout: 60_000, + timeout: 240_000, on_timeout: :kill_task, max_concurrency: @num_accounts ) @@ -114,7 +114,7 @@ defmodule TransactionsTests do assert_equal(Currency.to_wei(amount), balance, "For #{alice_account} #{index}.") end, - timeout: 60_000, + timeout: 240_000, on_timeout: :kill_task, max_concurrency: @num_accounts ) @@ -134,7 +134,7 @@ defmodule TransactionsTests do assert_equal(Currency.to_wei(amount), balance, "For #{bob_account} #{index}.") end, - timeout: 60_000, + timeout: 240_000, on_timeout: :kill_task, max_concurrency: @num_accounts ) diff --git a/priv/cabbage/apps/itest/test/itest/watcher_info_api_pagination_test.exs b/priv/cabbage/apps/itest/test/itest/watcher_info_api_pagination_test.exs index 22a0ae8328..f6ceb66497 100644 --- a/priv/cabbage/apps/itest/test/itest/watcher_info_api_pagination_test.exs +++ b/priv/cabbage/apps/itest/test/itest/watcher_info_api_pagination_test.exs @@ -29,6 +29,7 @@ defmodule WatcherInfoApiTest do accounts = Account.take_accounts(2) alice_account = Enum.at(accounts, 0) bob_account = Enum.at(accounts, 1) + %{alice_account: alice_account, bob_account: bob_account} end diff --git a/priv/cabbage/config/config.exs b/priv/cabbage/config/config.exs index 1d5cec21d6..459ad4ea5b 100644 --- a/priv/cabbage/config/config.exs +++ b/priv/cabbage/config/config.exs @@ -2,7 +2,10 @@ use Mix.Config config :ethereumex, url: "http://localhost:8545", - http_options: [timeout: 68_000, recv_timeout: 60_000] + http_options: [timeout: 60_000, recv_timeout: 60_000] + +config :cabbage, + reorg: System.get_env("REORG") config :tesla, adapter: Tesla.Adapter.Hackney diff --git a/priv/cabbage/docker-compose-reorg.yml b/priv/cabbage/docker-compose-reorg.yml new file mode 100644 index 0000000000..0861ed37a0 --- /dev/null +++ b/priv/cabbage/docker-compose-reorg.yml @@ -0,0 +1,108 @@ +version: "2.3" +services: + geth: + entrypoint: ["echo", "clique geth is disabled for reorgs"] + + geth-1: + image: ethereum/client-go:v1.9.15 + container_name: geth-1 + environment: + - ACCOUNT=0x6de4b3b9c28e9c3e84c2b2d3a875c947a84de68d + - BOOTNODES=enode://b655cc3e5b72ab9beb8a8536a3c3ae92fbeb79feb1ebd7f95d72be72554ca586428bd48a54eb9c2bcaae455cc674299b6dd3df3c6556a493dfd50070f1a448aa@172.25.0.103:30303 + - INIT=false + entrypoint: /bin/sh -c ". data/geth/command" + expose: + - 8545 + - 8546 + - 30303 + ports: + - 9000:8545 + volumes: + - ./data1:/data + - ./data/ethash:/root/.ethash + + healthcheck: + test: curl geth-1:8545 + interval: 5s + timeout: 3s + retries: 5 + networks: + chain: + ipv4_address: 172.25.0.102 + + geth-2: + image: ethereum/client-go:v1.9.15 + container_name: geth-2 + depends_on: + - geth-1 + environment: + - ACCOUNT=0xc0f780dfc35075979b0def588d999225b7ecc56f + - BOOTNODES=enode://4574f825d67bf570b9216e704a5b761d05d5015c458e2c9dd4b30abb2fe8c881400c2074a126df94690c4c9fb72ee046e6e3ac2bb73dede42fce66cb0a963b36@172.25.0.102:30303 + - INIT=false + entrypoint: /bin/sh -c ". data/geth/command" + expose: + - 8546 + - 8545 + - 30303 + ports: + - 9001:8545 + volumes: + - ./data2:/data + - ./data/ethash:/root/.ethash + healthcheck: + test: curl geth-2:8545 + interval: 5s + timeout: 3s + retries: 5 + networks: + chain: + ipv4_address: 172.25.0.103 + + nginx: + depends_on: + geth-1: + condition: service_healthy + geth-2: + condition: service_healthy + volumes: + - ./priv/cabbage/nginx.conf:/etc/nginx/nginx.conf + networks: + chain: + ipv4_address: 172.25.0.104 + + postgres: + networks: + chain: + ipv4_address: 172.25.0.110 + + childchain: + environment: + - ETHEREUM_RPC_URL=http://172.25.0.104:80 + networks: + chain: + ipv4_address: 172.25.0.105 + + watcher: + environment: + - ETHEREUM_RPC_URL=http://172.25.0.104:80 + - CHILD_CHAIN_URL=http://172.25.0.105:9656 + networks: + chain: + ipv4_address: 172.25.0.106 + + watcher_info: + environment: + - ETHEREUM_RPC_URL=http://172.25.0.104:80 + - CHILD_CHAIN_URL=http://172.25.0.105:9656 + - DATABASE_URL=postgresql://omisego_dev:omisego_dev@172.25.0.110:5432/omisego_dev + networks: + chain: + ipv4_address: 172.25.0.107 + + +networks: + chain: + driver: bridge + ipam: + config: + - subnet: 172.25.0.0/24 diff --git a/priv/cabbage/nginx.conf b/priv/cabbage/nginx.conf new file mode 100644 index 0000000000..75a4310d27 --- /dev/null +++ b/priv/cabbage/nginx.conf @@ -0,0 +1,39 @@ +events {} +http { + upstream geth { + server 172.25.0.102:8545; + server 172.25.0.103:8545; + } + + upstream websocket { + server 172.25.0.102:8546; + server 172.25.0.103:8546; + } + + server { + listen 80; + + location / { + proxy_pass http://geth; + proxy_next_upstream non_idempotent invalid_header error timeout http_500 http_502 http_504 http_403 http_404; + proxy_next_upstream_tries 4; + fastcgi_read_timeout 10; + proxy_read_timeout 10; + } + } + + server { + listen 81; + + location / { + proxy_pass http://websocket; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_next_upstream non_idempotent invalid_header error timeout http_500 http_502 http_504 http_403 http_404; + proxy_connect_timeout 7d; + proxy_send_timeout 7d; + proxy_read_timeout 7d; + } + } +} \ No newline at end of file diff --git a/snapshot_reorg.env b/snapshot_reorg.env new file mode 100644 index 0000000000..8e43c9843b --- /dev/null +++ b/snapshot_reorg.env @@ -0,0 +1,2 @@ +SNAPSHOT=https://storage.googleapis.com/circleci-docker-artifacts/data-elixir-omg-tester-plasma-deployer-dev-10cafba-MIN_EXIT_PERIOD-120-PLASMA_CONTRACTS_SHA-a69c763f239b81c5eb46b0bbdef3459f764360dc-reorg.tar.gz +CONTRACT_SHA=a69c763f239b81c5eb46b0bbdef3459f764360dc \ No newline at end of file