From b057f25c7ac82d25d061a2c80b30e7086290050c Mon Sep 17 00:00:00 2001
From: unnawut <unnawut@omg.network>
Date: Fri, 17 Jul 2020 00:18:44 +0700
Subject: [PATCH 01/39] feat: change block formation log from debug to info

---
 apps/omg/lib/omg/state.ex | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/apps/omg/lib/omg/state.ex b/apps/omg/lib/omg/state.ex
index ce60c85504..a31f9e2e2b 100644
--- a/apps/omg/lib/omg/state.ex
+++ b/apps/omg/lib/omg/state.ex
@@ -259,10 +259,10 @@ defmodule OMG.State do
     - pushes the new block to subscribers of `"blocks"` internal event bus topic
   """
   def handle_cast(:form_block, state) do
-    _ = Logger.debug("Forming new block...")
+    _ = Logger.info("Forming new block...")
     state = Core.claim_fees(state)
     {:ok, {%Block{number: blknum} = block, db_updates}, new_state} = Core.form_block(state)
-    _ = Logger.debug("Formed new block ##{blknum}")
+    _ = Logger.info("Formed new block ##{blknum}")
 
     # persistence is required to be here, since propagating the block onwards requires restartability including the
     # new block

From f63798590ec16e4417d422e1beb8e58ac36b5763 Mon Sep 17 00:00:00 2001
From: unnawut <unnawut@omg.network>
Date: Fri, 17 Jul 2020 00:19:41 +0700
Subject: [PATCH 02/39] feat: BlockQueue publishes :block_submitting and
 :block_submitted events

---
 .../lib/omg_child_chain/block_queue.ex        | 24 +++++++++++++++++--
 1 file changed, 22 insertions(+), 2 deletions(-)

diff --git a/apps/omg_child_chain/lib/omg_child_chain/block_queue.ex b/apps/omg_child_chain/lib/omg_child_chain/block_queue.ex
index 6f8a2dfa09..89f9bd0f1c 100644
--- a/apps/omg_child_chain/lib/omg_child_chain/block_queue.ex
+++ b/apps/omg_child_chain/lib/omg_child_chain/block_queue.ex
@@ -205,6 +205,7 @@ defmodule OMG.ChildChain.BlockQueue do
 
   defp submit(submission) do
     _ = Logger.info("Submitting: #{inspect(submission)}")
+    _ = publish_block_submitting_event(submission.num)
 
     submit_result = Eth.submit_block(submission.hash, submission.nonce, submission.gas_price)
     newest_mined_blknum = RootChain.get_mined_child_block()
@@ -217,8 +218,9 @@ defmodule OMG.ChildChain.BlockQueue do
           _ = log_eth_node_error()
           error
 
-        {:ok, txhash} ->
-          _ = GasAnalyzer.enqueue(txhash)
+        {:ok, root_chain_txhash} ->
+          _ = publish_block_submitted_event(submission.num)
+          _ = GasAnalyzer.enqueue(root_chain_txhash)
           _ = Balance.check()
           :ok
 
@@ -229,6 +231,24 @@ defmodule OMG.ChildChain.BlockQueue do
     :ok = final_result
   end
 
+  # Publishes each time a block is being submitted, regardless of failing or being successful.
+  # This differs from `:enqueue_block` which publishes only once when the block is formed.
+  # For example, when a block is re-submitted 3 times before it got accepted, there would be
+  # 1 x `:enqueue_block` and 3 x `:block_submitting` events published.
+  defp publish_block_submitting_event(blknum) do
+    {:child_chain, "blocks"}
+    |> OMG.Bus.Event.new(:block_submitting, blknum)
+    |> OMG.Bus.direct_local_broadcast()
+  end
+
+  # Publishes when a block is successfully submitted. Only 1 `block_submitted` event will ever
+  # be published for each block.
+  defp publish_block_submitted_event(blknum) do
+    {:child_chain, "blocks"}
+    |> OMG.Bus.Event.new(:block_submitted, blknum)
+    |> OMG.Bus.direct_local_broadcast()
+  end
+
   defp log_init_error(fields) do
     fields = Keyword.update!(fields, :known_hashes, fn hashes -> Enum.map(hashes, &Encoding.to_hex/1) end)
 

From 5c771dd29cb2a11ca0d8754093a64524cdb2fcb7 Mon Sep 17 00:00:00 2001
From: unnawut <unnawut@omg.network>
Date: Fri, 17 Jul 2020 00:20:10 +0700
Subject: [PATCH 03/39] feat: BlockQueue.Monitor that raises/clears
 :block_submission_stalled alarm

---
 .../omg_child_chain/block_queue/monitor.ex    | 160 ++++++++++++++++++
 .../block_queue/monitor/alarm_handler.ex      |  48 ++++++
 2 files changed, 208 insertions(+)
 create mode 100644 apps/omg_child_chain/lib/omg_child_chain/block_queue/monitor.ex
 create mode 100644 apps/omg_child_chain/lib/omg_child_chain/block_queue/monitor/alarm_handler.ex

diff --git a/apps/omg_child_chain/lib/omg_child_chain/block_queue/monitor.ex b/apps/omg_child_chain/lib/omg_child_chain/block_queue/monitor.ex
new file mode 100644
index 0000000000..0716f194e2
--- /dev/null
+++ b/apps/omg_child_chain/lib/omg_child_chain/block_queue/monitor.ex
@@ -0,0 +1,160 @@
+# Copyright 2019-2020 OmiseGO Pte Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+defmodule OMG.ChildChain.BlockQueue.Monitor do
+  @moduledoc """
+  Listens to block events and raises :block_submission_stalled alarm when a pending block
+  doesn't get successfully submitted within the specified time threshold.
+  """
+
+  use GenServer
+  require Logger
+  alias OMG.Eth.EthereumHeight
+
+  defstruct pending_blocks: [],
+            stall_threshold_in_rootchain_blocks: 2,
+            alarm_module: nil,
+            alarm_raised: false
+
+  @typep blknum() :: pos_integer()
+
+  @type t() :: %__MODULE__{
+          pending_blocks: [{blknum :: blknum(), first_submit_height :: pos_integer()}],
+          stall_threshold_in_rootchain_blocks: pos_integer(),
+          alarm_module: module(),
+          alarm_raised: boolean()
+        }
+
+  #
+  # GenServer APIs
+  #
+
+  def start_link(args) do
+    GenServer.start_link(__MODULE__, args, name: __MODULE__)
+  end
+
+  #
+  # GenServer behaviors
+  #
+
+  def init(opts) do
+    _ = Logger.info("Starting #{__MODULE__}")
+    _ = install_alarm_handler()
+    event_bus = Keyword.fetch!(opts, :event_bus)
+    check_interval_ms = Keyword.fetch!(opts, :check_interval_ms)
+
+    state = %__MODULE__{
+      pending_blocks: [],
+      stall_threshold_in_rootchain_blocks: Keyword.fetch!(opts, :stall_threshold_in_rootchain_blocks),
+      alarm_module: Keyword.fetch!(opts, :alarm_module),
+      alarm_raised: false
+    }
+
+    :ok = event_bus.subscribe({:child_chain, "blocks"}, link: true)
+    :ok = event_bus.subscribe({:root_chain, "ethereum_new_height"}, link: true)
+    {:ok, _} = :timer.send_interval(check_interval_ms, self(), :check_stall)
+    {:ok, state}
+  end
+
+  def handle_info(:check_stall, state) do
+    root_chain_height = EthereumHeight.get()
+
+    stalled_blocks =
+      Enum.filter(state.pending_blocks, fn {_blknum, first_submit_height} ->
+        root_chain_height - first_submit_height >= state.stall_threshold_in_rootchain_blocks
+      end)
+
+    _ = log_stalled_blocks(stalled_blocks, root_chain_height)
+    _ = trigger_alarm(state.alarm_module, state.alarm_raised, stalled_blocks)
+
+    {:ok, state}
+  end
+
+  # Listens for a block being submitted and add it to monitoring if it hasn't been tracked
+  def handle_info({:internal_event_bus, :block_submitting, blknum}, state) do
+    pending_blocks = add_new_blknum(state.pending_blocks, blknum)
+    {:noreply, %{state | pending_blocks: pending_blocks}}
+  end
+
+  # Listens for a block that got submitted and drop it from monitoring
+  def handle_info({:internal_event_bus, :block_submitted, blknum}, state) do
+    pending_blocks = remove_blknum(state.pending_blocks, blknum)
+    {:noreply, %{state | pending_blocks: pending_blocks}}
+  end
+
+  #
+  # Handle incoming alarms
+  #
+  # These functions are called by the AlarmHandler so that this monitor process can update
+  # its internal state according to the raised alarms.
+  #
+  def handle_cast({:set_alarm, :block_submission_stalled}, state) do
+    {:noreply, %{state | alarm_raised: true}}
+  end
+
+  def handle_cast({:clear_alarm, :block_submission_stalled}, state) do
+    {:noreply, %{state | alarm_raised: false}}
+  end
+
+  #
+  # Private functions
+  #
+
+  # Add the blknum to tracking only if it is not already tracked
+  defp add_new_blknum(pending_blocks, blknum) do
+    case Enum.any?(pending_blocks, fn {pending_blknum, _} -> pending_blknum == blknum end) do
+      true -> pending_blocks
+      false -> [{blknum, EthereumHeight.get()} | pending_blocks]
+    end
+  end
+
+  defp remove_blknum(pending_blocks, blknum) do
+    Enum.reject(pending_blocks, fn {pending_blknum, _} -> pending_blknum == blknum end)
+  end
+
+  #
+  # Alarms management
+  #
+
+  defp install_alarm_handler() do
+    case Enum.member?(:gen_event.which_handlers(:alarm_handler), __MODULE__.AlarmHandler) do
+      true -> :ok
+      _ -> :alarm_handler.add_alarm_handler(__MODULE__.AlarmHandler)
+    end
+  end
+
+  @spec trigger_alarm(module(), boolean(), [blknum()]) :: :ok
+  defp trigger_alarm(_alarm_module, false, []), do: :ok
+
+  defp trigger_alarm(alarm_module, false, _stalled_blocks) do
+    alarm_module.set(alarm_module.block_submission_stalled(__MODULE__))
+  end
+
+  defp trigger_alarm(alarm_module, true, []) do
+    alarm_module.clear(alarm_module.block_submission_stalled(__MODULE__))
+  end
+
+  defp trigger_alarm(_alarm_module, true, _stalled_blocks), do: :ok
+
+  #
+  # Logging
+  #
+  defp log_stalled_blocks([], _), do: :ok
+
+  defp log_stalled_blocks(stalled_blocks, root_chain_height) do
+    Logger.warn(
+      "#{__MODULE__}: Stalled blocks: #{inspect(stalled_blocks)}. Current height: #{inspect(root_chain_height)}"
+    )
+  end
+end
diff --git a/apps/omg_child_chain/lib/omg_child_chain/block_queue/monitor/alarm_handler.ex b/apps/omg_child_chain/lib/omg_child_chain/block_queue/monitor/alarm_handler.ex
new file mode 100644
index 0000000000..0feb51a823
--- /dev/null
+++ b/apps/omg_child_chain/lib/omg_child_chain/block_queue/monitor/alarm_handler.ex
@@ -0,0 +1,48 @@
+# Copyright 2019-2020 OmiseGO Pte Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+defmodule OMG.ChildChain.BlockQueue.Monitor.AlarmHandler do
+  @moduledoc """
+  Listens for :block_submission_stalled alarms and reflect the alarm's state back to the monitor.
+  """
+  require Logger
+
+  # The alarm reporter and monitor happen to be the same module here because we are just
+  # reflecting the alarm's state back to the reporter.
+  @reporter OMG.ChildChain.BlockQueue.Monitor
+  @monitor OMG.ChildChain.BlockQueue.Monitor
+
+  def init(_args) do
+    {:ok, %{}}
+  end
+
+  def handle_call(_request, state), do: {:ok, :ok, state}
+
+  def handle_event({:set_alarm, {:block_submission_stalled, %{reporter: @reporter}}}, state) do
+    _ = Logger.warn(":block_submission_stalled alarm raised.")
+    :ok = GenServer.cast(@monitor, {:set_alarm, :block_submission_stalled})
+    {:ok, state}
+  end
+
+  def handle_event({:clear_alarm, {:block_submission_stalled, %{reporter: @reporter}}}, state) do
+    _ = Logger.warn(":block_submission_stalled alarm cleared.")
+    :ok = GenServer.cast(@monitor, {:clear_alarm, :block_submission_stalled})
+    {:ok, state}
+  end
+
+  def handle_event(event, state) do
+    _ = Logger.info("#{__MODULE__} got event: #{inspect(event)}. Ignoring.")
+    {:ok, state}
+  end
+end

From b8a8c815b38215c9e45aba58333bcec35ed6fc20 Mon Sep 17 00:00:00 2001
From: unnawut <unnawut@omg.network>
Date: Fri, 17 Jul 2020 00:34:20 +0700
Subject: [PATCH 04/39] refactor: tracks root chain height directly from the
 bus

---
 .../omg_child_chain/block_queue/monitor.ex    | 24 ++++++++++++-------
 1 file changed, 15 insertions(+), 9 deletions(-)

diff --git a/apps/omg_child_chain/lib/omg_child_chain/block_queue/monitor.ex b/apps/omg_child_chain/lib/omg_child_chain/block_queue/monitor.ex
index 0716f194e2..1f02f09830 100644
--- a/apps/omg_child_chain/lib/omg_child_chain/block_queue/monitor.ex
+++ b/apps/omg_child_chain/lib/omg_child_chain/block_queue/monitor.ex
@@ -17,13 +17,12 @@ defmodule OMG.ChildChain.BlockQueue.Monitor do
   Listens to block events and raises :block_submission_stalled alarm when a pending block
   doesn't get successfully submitted within the specified time threshold.
   """
-
   use GenServer
   require Logger
-  alias OMG.Eth.EthereumHeight
 
   defstruct pending_blocks: [],
-            stall_threshold_in_rootchain_blocks: 2,
+            root_chain_height: 0,
+            stall_threshold_in_root_chain_blocks: 2,
             alarm_module: nil,
             alarm_raised: false
 
@@ -31,7 +30,8 @@ defmodule OMG.ChildChain.BlockQueue.Monitor do
 
   @type t() :: %__MODULE__{
           pending_blocks: [{blknum :: blknum(), first_submit_height :: pos_integer()}],
-          stall_threshold_in_rootchain_blocks: pos_integer(),
+          root_chain_height: non_neg_integer(),
+          stall_threshold_in_root_chain_blocks: pos_integer(),
           alarm_module: module(),
           alarm_raised: boolean()
         }
@@ -56,31 +56,35 @@ defmodule OMG.ChildChain.BlockQueue.Monitor do
 
     state = %__MODULE__{
       pending_blocks: [],
-      stall_threshold_in_rootchain_blocks: Keyword.fetch!(opts, :stall_threshold_in_rootchain_blocks),
+      stall_threshold_in_root_chain_blocks: Keyword.fetch!(opts, :stall_threshold_in_root_chain_blocks),
       alarm_module: Keyword.fetch!(opts, :alarm_module),
       alarm_raised: false
     }
 
     :ok = event_bus.subscribe({:child_chain, "blocks"}, link: true)
     :ok = event_bus.subscribe({:root_chain, "ethereum_new_height"}, link: true)
+
     {:ok, _} = :timer.send_interval(check_interval_ms, self(), :check_stall)
     {:ok, state}
   end
 
   def handle_info(:check_stall, state) do
-    root_chain_height = EthereumHeight.get()
-
     stalled_blocks =
       Enum.filter(state.pending_blocks, fn {_blknum, first_submit_height} ->
-        root_chain_height - first_submit_height >= state.stall_threshold_in_rootchain_blocks
+        state.root_chain_height - first_submit_height >= state.stall_threshold_in_root_chain_blocks
       end)
 
-    _ = log_stalled_blocks(stalled_blocks, root_chain_height)
+    _ = log_stalled_blocks(stalled_blocks, state.root_chain_height)
     _ = trigger_alarm(state.alarm_module, state.alarm_raised, stalled_blocks)
 
     {:ok, state}
   end
 
+  # Keeps track of the latest root chain height
+  def handle_info({:internal_event_bus, :ethereum_new_height, new_height}, state) do
+    {:noreply, %{state | root_chain_height: new_height}}
+  end
+
   # Listens for a block being submitted and add it to monitoring if it hasn't been tracked
   def handle_info({:internal_event_bus, :block_submitting, blknum}, state) do
     pending_blocks = add_new_blknum(state.pending_blocks, blknum)
@@ -112,6 +116,7 @@ defmodule OMG.ChildChain.BlockQueue.Monitor do
   #
 
   # Add the blknum to tracking only if it is not already tracked
+  @spec add_new_blknum([{blknum(), any()}], blknum()) :: :ok
   defp add_new_blknum(pending_blocks, blknum) do
     case Enum.any?(pending_blocks, fn {pending_blknum, _} -> pending_blknum == blknum end) do
       true -> pending_blocks
@@ -119,6 +124,7 @@ defmodule OMG.ChildChain.BlockQueue.Monitor do
     end
   end
 
+  @spec add_new_blknum([{blknum(), any()}], blknum()) :: :ok
   defp remove_blknum(pending_blocks, blknum) do
     Enum.reject(pending_blocks, fn {pending_blknum, _} -> pending_blknum == blknum end)
   end

From 17c76c48e7a5e8623a5c9cef076b2bff1e44f40d Mon Sep 17 00:00:00 2001
From: unnawut <unnawut@omg.network>
Date: Fri, 17 Jul 2020 00:37:04 +0700
Subject: [PATCH 05/39] refactor: revert variable naming

---
 apps/omg_child_chain/lib/omg_child_chain/block_queue.ex | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/apps/omg_child_chain/lib/omg_child_chain/block_queue.ex b/apps/omg_child_chain/lib/omg_child_chain/block_queue.ex
index 89f9bd0f1c..90959bf8af 100644
--- a/apps/omg_child_chain/lib/omg_child_chain/block_queue.ex
+++ b/apps/omg_child_chain/lib/omg_child_chain/block_queue.ex
@@ -218,9 +218,9 @@ defmodule OMG.ChildChain.BlockQueue do
           _ = log_eth_node_error()
           error
 
-        {:ok, root_chain_txhash} ->
+        {:ok, txhash} ->
           _ = publish_block_submitted_event(submission.num)
-          _ = GasAnalyzer.enqueue(root_chain_txhash)
+          _ = GasAnalyzer.enqueue(txhash)
           _ = Balance.check()
           :ok
 

From a55a7e5d3308f44f4b1e5ca75e23a1270d23f2c3 Mon Sep 17 00:00:00 2001
From: unnawut <unnawut@omg.network>
Date: Fri, 17 Jul 2020 00:38:44 +0700
Subject: [PATCH 06/39] fix: remove remaining EtherereumHeight.get()

---
 .../lib/omg_child_chain/block_queue/monitor.ex            | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/apps/omg_child_chain/lib/omg_child_chain/block_queue/monitor.ex b/apps/omg_child_chain/lib/omg_child_chain/block_queue/monitor.ex
index 1f02f09830..ab2d8da0ca 100644
--- a/apps/omg_child_chain/lib/omg_child_chain/block_queue/monitor.ex
+++ b/apps/omg_child_chain/lib/omg_child_chain/block_queue/monitor.ex
@@ -87,7 +87,7 @@ defmodule OMG.ChildChain.BlockQueue.Monitor do
 
   # Listens for a block being submitted and add it to monitoring if it hasn't been tracked
   def handle_info({:internal_event_bus, :block_submitting, blknum}, state) do
-    pending_blocks = add_new_blknum(state.pending_blocks, blknum)
+    pending_blocks = add_new_blknum(state.pending_blocks, blknum, state.root_chain_height)
     {:noreply, %{state | pending_blocks: pending_blocks}}
   end
 
@@ -116,11 +116,11 @@ defmodule OMG.ChildChain.BlockQueue.Monitor do
   #
 
   # Add the blknum to tracking only if it is not already tracked
-  @spec add_new_blknum([{blknum(), any()}], blknum()) :: :ok
-  defp add_new_blknum(pending_blocks, blknum) do
+  @spec add_new_blknum([{blknum(), any()}], blknum(), non_neg_integer()) :: :ok
+  defp add_new_blknum(pending_blocks, blknum, root_chain_height) do
     case Enum.any?(pending_blocks, fn {pending_blknum, _} -> pending_blknum == blknum end) do
       true -> pending_blocks
-      false -> [{blknum, EthereumHeight.get()} | pending_blocks]
+      false -> [{blknum, root_chain_height} | pending_blocks]
     end
   end
 

From ec310b36dfc8bdaa40a6209560775e04e8a9df7a Mon Sep 17 00:00:00 2001
From: unnawut <unnawut@omg.network>
Date: Fri, 17 Jul 2020 01:09:14 +0700
Subject: [PATCH 07/39] fix: typespec

---
 apps/omg_child_chain/lib/omg_child_chain/block_queue/monitor.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/apps/omg_child_chain/lib/omg_child_chain/block_queue/monitor.ex b/apps/omg_child_chain/lib/omg_child_chain/block_queue/monitor.ex
index ab2d8da0ca..46dde16388 100644
--- a/apps/omg_child_chain/lib/omg_child_chain/block_queue/monitor.ex
+++ b/apps/omg_child_chain/lib/omg_child_chain/block_queue/monitor.ex
@@ -124,7 +124,7 @@ defmodule OMG.ChildChain.BlockQueue.Monitor do
     end
   end
 
-  @spec add_new_blknum([{blknum(), any()}], blknum()) :: :ok
+  @spec remove_blknum([{blknum(), any()}], blknum()) :: :ok
   defp remove_blknum(pending_blocks, blknum) do
     Enum.reject(pending_blocks, fn {pending_blknum, _} -> pending_blknum == blknum end)
   end

From fb97563772503f079e48fda59f0fa39fb7745d78 Mon Sep 17 00:00:00 2001
From: unnawut <unnawut@omg.network>
Date: Fri, 17 Jul 2020 01:12:24 +0700
Subject: [PATCH 08/39] feat: add telemetry and statsd metric for
 blknum_submitting and blknum_submitted

---
 .../lib/omg_child_chain/block_queue.ex             |  4 ++++
 .../lib/omg_child_chain/block_queue/measure.ex     | 10 ++++++++++
 apps/omg_status/lib/omg_status/metric/event.ex     | 14 +++++++++++++-
 3 files changed, 27 insertions(+), 1 deletion(-)

diff --git a/apps/omg_child_chain/lib/omg_child_chain/block_queue.ex b/apps/omg_child_chain/lib/omg_child_chain/block_queue.ex
index 90959bf8af..bf5fd84222 100644
--- a/apps/omg_child_chain/lib/omg_child_chain/block_queue.ex
+++ b/apps/omg_child_chain/lib/omg_child_chain/block_queue.ex
@@ -236,6 +236,8 @@ defmodule OMG.ChildChain.BlockQueue do
   # For example, when a block is re-submitted 3 times before it got accepted, there would be
   # 1 x `:enqueue_block` and 3 x `:block_submitting` events published.
   defp publish_block_submitting_event(blknum) do
+    _ = :telemetry.execute([:blknum_submitting, __MODULE__], blknum)
+
     {:child_chain, "blocks"}
     |> OMG.Bus.Event.new(:block_submitting, blknum)
     |> OMG.Bus.direct_local_broadcast()
@@ -244,6 +246,8 @@ defmodule OMG.ChildChain.BlockQueue do
   # Publishes when a block is successfully submitted. Only 1 `block_submitted` event will ever
   # be published for each block.
   defp publish_block_submitted_event(blknum) do
+    _ = :telemetry.execute([:blknum_submitted, __MODULE__], blknum)
+
     {:child_chain, "blocks"}
     |> OMG.Bus.Event.new(:block_submitted, blknum)
     |> OMG.Bus.direct_local_broadcast()
diff --git a/apps/omg_child_chain/lib/omg_child_chain/block_queue/measure.ex b/apps/omg_child_chain/lib/omg_child_chain/block_queue/measure.ex
index 6629db99f9..eaaf9744de 100644
--- a/apps/omg_child_chain/lib/omg_child_chain/block_queue/measure.ex
+++ b/apps/omg_child_chain/lib/omg_child_chain/block_queue/measure.ex
@@ -31,6 +31,8 @@ defmodule OMG.ChildChain.BlockQueue.Measure do
 
   @supported_events [
     [:process, BlockQueue],
+    [:blknum_submitting, BlockQueue],
+    [:blknum_submitted, BlockQueue],
     [:gas, GasAnalyzer],
     [:authority_balance, Balance]
   ]
@@ -54,4 +56,12 @@ defmodule OMG.ChildChain.BlockQueue.Measure do
     gwei = div(authority_balance, 1_000_000_000)
     _ = Datadog.gauge(name(:authority_balance), gwei)
   end
+
+  def handle_event([:blknum_submitting, BlockQueue], blknum, _, _config) do
+    _ = Datadog.gauge(name(:block_queue_blknum_submitting), blknum)
+  end
+
+  def handle_event([:blknum_submitted, BlockQueue], blknum, _, _config) do
+    _ = Datadog.gauge(name(:block_queue_blknum_submitted), blknum)
+  end
 end
diff --git a/apps/omg_status/lib/omg_status/metric/event.ex b/apps/omg_status/lib/omg_status/metric/event.ex
index 5f956a0ae0..933204a832 100644
--- a/apps/omg_status/lib/omg_status/metric/event.ex
+++ b/apps/omg_status/lib/omg_status/metric/event.ex
@@ -31,7 +31,9 @@ defmodule OMG.Status.Metric.Event do
     :piggyback,
     :piggyback_challenges_processor,
     :piggyback_processor,
-    :block_queue
+    :block_queue,
+    :block_queue_blknum_submitting,
+    :block_queue_blknum_submitted,
   ]
 
   @doc """
@@ -64,6 +66,16 @@ defmodule OMG.Status.Metric.Event do
   """
   def name(:block_submission), do: "block_submission_gas"
 
+  @doc """
+  Child Chain BlockQueue blknum being submitted
+  """
+  def name(:block_queue_blknum_submitting), do: "block_queue_blknum_submitting"
+
+  @doc """
+  Child Chain BlockQueue blknum submitted
+  """
+  def name(:block_queue_blknum_submitted), do: "block_queue_blknum_submitted"
+
   @doc """
   Child Chain authority address balance
   """

From 244f608ce26d9eb24db1f51c7d9c3bd3b4132c0f Mon Sep 17 00:00:00 2001
From: unnawut <unnawut@omg.network>
Date: Fri, 17 Jul 2020 01:13:00 +0700
Subject: [PATCH 09/39] feat: add telemetry and statsd metric for
 blocks_submitting and blocks_stalled

---
 .../lib/omg_child_chain/block_queue/measure.ex       | 10 ++++++++++
 .../lib/omg_child_chain/block_queue/monitor.ex       |  2 ++
 apps/omg_status/lib/omg_status/metric/event.ex       | 12 ++++++++++++
 3 files changed, 24 insertions(+)

diff --git a/apps/omg_child_chain/lib/omg_child_chain/block_queue/measure.ex b/apps/omg_child_chain/lib/omg_child_chain/block_queue/measure.ex
index eaaf9744de..8076ac5c5d 100644
--- a/apps/omg_child_chain/lib/omg_child_chain/block_queue/measure.ex
+++ b/apps/omg_child_chain/lib/omg_child_chain/block_queue/measure.ex
@@ -33,6 +33,8 @@ defmodule OMG.ChildChain.BlockQueue.Measure do
     [:process, BlockQueue],
     [:blknum_submitting, BlockQueue],
     [:blknum_submitted, BlockQueue],
+    [:blocks_submitting, BlockQueue.Monitor],
+    [:blocks_stalled, BlockQueue.Monitor],
     [:gas, GasAnalyzer],
     [:authority_balance, Balance]
   ]
@@ -64,4 +66,12 @@ defmodule OMG.ChildChain.BlockQueue.Measure do
   def handle_event([:blknum_submitted, BlockQueue], blknum, _, _config) do
     _ = Datadog.gauge(name(:block_queue_blknum_submitted), blknum)
   end
+
+  def handle_event([:blocks_submitting, BlockQueue.Monitor], _, %{blocks: blocks}, _config) do
+    _ = Datadog.gauge(name(:block_queue_num_blocks_submitting), length(blocks))
+  end
+
+  def handle_event([:blocks_stalled, BlockQueue.Monitor], _, %{blocks: blocks}, _config) do
+    _ = Datadog.gauge(name(:block_queue_num_blocks_stalled), length(blocks))
+  end
 end
diff --git a/apps/omg_child_chain/lib/omg_child_chain/block_queue/monitor.ex b/apps/omg_child_chain/lib/omg_child_chain/block_queue/monitor.ex
index 46dde16388..9cb219069f 100644
--- a/apps/omg_child_chain/lib/omg_child_chain/block_queue/monitor.ex
+++ b/apps/omg_child_chain/lib/omg_child_chain/block_queue/monitor.ex
@@ -74,6 +74,8 @@ defmodule OMG.ChildChain.BlockQueue.Monitor do
         state.root_chain_height - first_submit_height >= state.stall_threshold_in_root_chain_blocks
       end)
 
+    _ = :telemetry.execute([:blocks_submitting, __MODULE__], %{blocks: state.pending_blocks})
+    _ = :telemetry.execute([:blocks_stalled, __MODULE__], %{blocks: stalled_blocks})
     _ = log_stalled_blocks(stalled_blocks, state.root_chain_height)
     _ = trigger_alarm(state.alarm_module, state.alarm_raised, stalled_blocks)
 
diff --git a/apps/omg_status/lib/omg_status/metric/event.ex b/apps/omg_status/lib/omg_status/metric/event.ex
index 933204a832..d97d4e857e 100644
--- a/apps/omg_status/lib/omg_status/metric/event.ex
+++ b/apps/omg_status/lib/omg_status/metric/event.ex
@@ -34,6 +34,8 @@ defmodule OMG.Status.Metric.Event do
     :block_queue,
     :block_queue_blknum_submitting,
     :block_queue_blknum_submitted,
+    :block_queue_num_blocks_submitting,
+    :block_queue_num_blocks_stalled
   ]
 
   @doc """
@@ -76,6 +78,16 @@ defmodule OMG.Status.Metric.Event do
   """
   def name(:block_queue_blknum_submitted), do: "block_queue_blknum_submitted"
 
+  @doc """
+  Child Chain BlockQueue's number of blocks currently being submitted
+  """
+  def name(:block_queue_num_blocks_submitting), do: "block_queue_num_blocks_submitting"
+
+  @doc """
+  Child Chain BlockQueue's number of blocks currently being submitted and stalled
+  """
+  def name(:block_queue_num_blocks_stalled), do: "block_queue_num_blocks_stalled"
+
   @doc """
   Child Chain authority address balance
   """

From 782c3e3320833ea4ec4806032205731b91866de2 Mon Sep 17 00:00:00 2001
From: unnawut <unnawut@omg.network>
Date: Fri, 17 Jul 2020 01:16:12 +0700
Subject: [PATCH 10/39] fix: rename :block_submission to :block_submission_gas

---
 apps/omg_child_chain/lib/omg_child_chain/block_queue/measure.ex | 2 +-
 apps/omg_status/lib/omg_status/metric/event.ex                  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/apps/omg_child_chain/lib/omg_child_chain/block_queue/measure.ex b/apps/omg_child_chain/lib/omg_child_chain/block_queue/measure.ex
index 8076ac5c5d..ddb50f5a0f 100644
--- a/apps/omg_child_chain/lib/omg_child_chain/block_queue/measure.ex
+++ b/apps/omg_child_chain/lib/omg_child_chain/block_queue/measure.ex
@@ -51,7 +51,7 @@ defmodule OMG.ChildChain.BlockQueue.Measure do
 
   def handle_event([:gas, GasAnalyzer], %{gas: gas}, _, _config) do
     gwei = div(gas, 1_000_000_000)
-    _ = Datadog.gauge(name(:block_submission), gwei)
+    _ = Datadog.gauge(name(:block_submission_gas), gwei)
   end
 
   def handle_event([:authority_balance, Balance], %{authority_balance: authority_balance}, _, _config) do
diff --git a/apps/omg_status/lib/omg_status/metric/event.ex b/apps/omg_status/lib/omg_status/metric/event.ex
index d97d4e857e..8f3ee536d3 100644
--- a/apps/omg_status/lib/omg_status/metric/event.ex
+++ b/apps/omg_status/lib/omg_status/metric/event.ex
@@ -66,7 +66,7 @@ defmodule OMG.Status.Metric.Event do
   @doc """
   Child Chain Block queue gas usage metric
   """
-  def name(:block_submission), do: "block_submission_gas"
+  def name(:block_submission_gas), do: "block_submission_gas"
 
   @doc """
   Child Chain BlockQueue blknum being submitted

From de16c8e511d83af9cc08c76aa6f653d09c47944f Mon Sep 17 00:00:00 2001
From: unnawut <unnawut@omg.network>
Date: Fri, 17 Jul 2020 01:17:09 +0700
Subject: [PATCH 11/39] feat: add statsd metric for :block_submission_success

---
 .../lib/omg_child_chain/block_queue/measure.ex               | 1 +
 apps/omg_status/lib/omg_status/metric/event.ex               | 5 +++++
 2 files changed, 6 insertions(+)

diff --git a/apps/omg_child_chain/lib/omg_child_chain/block_queue/measure.ex b/apps/omg_child_chain/lib/omg_child_chain/block_queue/measure.ex
index ddb50f5a0f..a09c6d0391 100644
--- a/apps/omg_child_chain/lib/omg_child_chain/block_queue/measure.ex
+++ b/apps/omg_child_chain/lib/omg_child_chain/block_queue/measure.ex
@@ -65,6 +65,7 @@ defmodule OMG.ChildChain.BlockQueue.Measure do
 
   def handle_event([:blknum_submitted, BlockQueue], blknum, _, _config) do
     _ = Datadog.gauge(name(:block_queue_blknum_submitted), blknum)
+    _ = Datadog.increment(name(:block_submission_success), 1)
   end
 
   def handle_event([:blocks_submitting, BlockQueue.Monitor], _, %{blocks: blocks}, _config) do
diff --git a/apps/omg_status/lib/omg_status/metric/event.ex b/apps/omg_status/lib/omg_status/metric/event.ex
index 8f3ee536d3..e48b3178dd 100644
--- a/apps/omg_status/lib/omg_status/metric/event.ex
+++ b/apps/omg_status/lib/omg_status/metric/event.ex
@@ -63,6 +63,11 @@ defmodule OMG.Status.Metric.Event do
   """
   def name(:block_transactions), do: "block_transactions"
 
+  @doc """
+  Childchain Block successfully submitted
+  """
+  def name(:block_submission_success), do: "block_submission_success"
+
   @doc """
   Child Chain Block queue gas usage metric
   """

From c5da3eabc39e295db93cd73a403d179a971dfb5e Mon Sep 17 00:00:00 2001
From: unnawut <unnawut@omg.network>
Date: Fri, 17 Jul 2020 01:17:23 +0700
Subject: [PATCH 12/39] fix: remove non services

---
 apps/omg_status/lib/omg_status/metric/event.ex | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/apps/omg_status/lib/omg_status/metric/event.ex b/apps/omg_status/lib/omg_status/metric/event.ex
index e48b3178dd..0264621ef6 100644
--- a/apps/omg_status/lib/omg_status/metric/event.ex
+++ b/apps/omg_status/lib/omg_status/metric/event.ex
@@ -31,11 +31,7 @@ defmodule OMG.Status.Metric.Event do
     :piggyback,
     :piggyback_challenges_processor,
     :piggyback_processor,
-    :block_queue,
-    :block_queue_blknum_submitting,
-    :block_queue_blknum_submitted,
-    :block_queue_num_blocks_submitting,
-    :block_queue_num_blocks_stalled
+    :block_queue
   ]
 
   @doc """

From c4e437ffd753ca6c5f52a66e0b5492f4ee8b53c6 Mon Sep 17 00:00:00 2001
From: unnawut <unnawut@omg.network>
Date: Fri, 17 Jul 2020 01:23:57 +0700
Subject: [PATCH 13/39] docs: clearer explanation of block queue metrics

---
 .../lib/omg_child_chain/block_queue/measure.ex                | 2 ++
 apps/omg_status/lib/omg_status/metric/event.ex                | 4 ++--
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/apps/omg_child_chain/lib/omg_child_chain/block_queue/measure.ex b/apps/omg_child_chain/lib/omg_child_chain/block_queue/measure.ex
index a09c6d0391..75d51a6818 100644
--- a/apps/omg_child_chain/lib/omg_child_chain/block_queue/measure.ex
+++ b/apps/omg_child_chain/lib/omg_child_chain/block_queue/measure.ex
@@ -31,8 +31,10 @@ defmodule OMG.ChildChain.BlockQueue.Measure do
 
   @supported_events [
     [:process, BlockQueue],
+    # Events triggered when a block is submitting or submitted
     [:blknum_submitting, BlockQueue],
     [:blknum_submitted, BlockQueue],
+    # Events providing collections of blocks being submitted or stalled
     [:blocks_submitting, BlockQueue.Monitor],
     [:blocks_stalled, BlockQueue.Monitor],
     [:gas, GasAnalyzer],
diff --git a/apps/omg_status/lib/omg_status/metric/event.ex b/apps/omg_status/lib/omg_status/metric/event.ex
index 0264621ef6..04943e281d 100644
--- a/apps/omg_status/lib/omg_status/metric/event.ex
+++ b/apps/omg_status/lib/omg_status/metric/event.ex
@@ -70,12 +70,12 @@ defmodule OMG.Status.Metric.Event do
   def name(:block_submission_gas), do: "block_submission_gas"
 
   @doc """
-  Child Chain BlockQueue blknum being submitted
+  Child Chain BlockQueue's blknum of the block being submitted
   """
   def name(:block_queue_blknum_submitting), do: "block_queue_blknum_submitting"
 
   @doc """
-  Child Chain BlockQueue blknum submitted
+  Child Chain BlockQueue's blknum of the block submitted
   """
   def name(:block_queue_blknum_submitted), do: "block_queue_blknum_submitted"
 

From b87a7126234512805f802c56fa438bf8388e4e2c Mon Sep 17 00:00:00 2001
From: unnawut <unnawut@omg.network>
Date: Fri, 17 Jul 2020 01:31:20 +0700
Subject: [PATCH 14/39] docs: more explanation on event publishing

---
 apps/omg_child_chain/lib/omg_child_chain/block_queue.ex | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/apps/omg_child_chain/lib/omg_child_chain/block_queue.ex b/apps/omg_child_chain/lib/omg_child_chain/block_queue.ex
index bf5fd84222..7a033a2f69 100644
--- a/apps/omg_child_chain/lib/omg_child_chain/block_queue.ex
+++ b/apps/omg_child_chain/lib/omg_child_chain/block_queue.ex
@@ -235,6 +235,9 @@ defmodule OMG.ChildChain.BlockQueue do
   # This differs from `:enqueue_block` which publishes only once when the block is formed.
   # For example, when a block is re-submitted 3 times before it got accepted, there would be
   # 1 x `:enqueue_block` and 3 x `:block_submitting` events published.
+  #
+  # The telemetry event is emitted for raw metrics propagation. The bus event is published so
+  # a consumer, potentially a GenServer or other processes can perform extra operations on the data.
   defp publish_block_submitting_event(blknum) do
     _ = :telemetry.execute([:blknum_submitting, __MODULE__], blknum)
 
@@ -245,6 +248,9 @@ defmodule OMG.ChildChain.BlockQueue do
 
   # Publishes when a block is successfully submitted. Only 1 `block_submitted` event will ever
   # be published for each block.
+  #
+  # The telemetry event is emitted for raw metrics propagation. The bus event is published so
+  # a consumer, potentially a GenServer or other processes can perform extra operations on the data.
   defp publish_block_submitted_event(blknum) do
     _ = :telemetry.execute([:blknum_submitted, __MODULE__], blknum)
 

From 03b983b1a40db49c75aae662effbc57e0d5d2262 Mon Sep 17 00:00:00 2001
From: unnawut <unnawut@omg.network>
Date: Fri, 17 Jul 2020 01:35:54 +0700
Subject: [PATCH 15/39] feat: add :block_submission_attempt statsd metric

---
 .../lib/omg_child_chain/block_queue/measure.ex               | 1 +
 apps/omg_status/lib/omg_status/metric/event.ex               | 5 +++++
 2 files changed, 6 insertions(+)

diff --git a/apps/omg_child_chain/lib/omg_child_chain/block_queue/measure.ex b/apps/omg_child_chain/lib/omg_child_chain/block_queue/measure.ex
index 75d51a6818..d62c3ba68d 100644
--- a/apps/omg_child_chain/lib/omg_child_chain/block_queue/measure.ex
+++ b/apps/omg_child_chain/lib/omg_child_chain/block_queue/measure.ex
@@ -63,6 +63,7 @@ defmodule OMG.ChildChain.BlockQueue.Measure do
 
   def handle_event([:blknum_submitting, BlockQueue], blknum, _, _config) do
     _ = Datadog.gauge(name(:block_queue_blknum_submitting), blknum)
+    _ = Datadog.increment(name(:block_submission_attempt), 1)
   end
 
   def handle_event([:blknum_submitted, BlockQueue], blknum, _, _config) do
diff --git a/apps/omg_status/lib/omg_status/metric/event.ex b/apps/omg_status/lib/omg_status/metric/event.ex
index 04943e281d..599d966965 100644
--- a/apps/omg_status/lib/omg_status/metric/event.ex
+++ b/apps/omg_status/lib/omg_status/metric/event.ex
@@ -59,6 +59,11 @@ defmodule OMG.Status.Metric.Event do
   """
   def name(:block_transactions), do: "block_transactions"
 
+  @doc """
+  Childchain Block submission attempted
+  """
+  def name(:block_submission_attempt), do: "block_submission_attempt"
+
   @doc """
   Childchain Block successfully submitted
   """

From 8773021aa618a6fb0d4f4cde23b3225e288a4097 Mon Sep 17 00:00:00 2001
From: unnawut <unnawut@omg.network>
Date: Fri, 17 Jul 2020 01:39:13 +0700
Subject: [PATCH 16/39] fix: dialyzer

---
 .../lib/omg_child_chain/block_queue/monitor.ex             | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/apps/omg_child_chain/lib/omg_child_chain/block_queue/monitor.ex b/apps/omg_child_chain/lib/omg_child_chain/block_queue/monitor.ex
index 9cb219069f..b4c9e33ce8 100644
--- a/apps/omg_child_chain/lib/omg_child_chain/block_queue/monitor.ex
+++ b/apps/omg_child_chain/lib/omg_child_chain/block_queue/monitor.ex
@@ -27,9 +27,10 @@ defmodule OMG.ChildChain.BlockQueue.Monitor do
             alarm_raised: false
 
   @typep blknum() :: pos_integer()
+  @typep pending_block() :: {blknum :: blknum(), first_submit_height :: pos_integer()}
 
   @type t() :: %__MODULE__{
-          pending_blocks: [{blknum :: blknum(), first_submit_height :: pos_integer()}],
+          pending_blocks: [pending_block()],
           root_chain_height: non_neg_integer(),
           stall_threshold_in_root_chain_blocks: pos_integer(),
           alarm_module: module(),
@@ -118,7 +119,7 @@ defmodule OMG.ChildChain.BlockQueue.Monitor do
   #
 
   # Add the blknum to tracking only if it is not already tracked
-  @spec add_new_blknum([{blknum(), any()}], blknum(), non_neg_integer()) :: :ok
+  @spec add_new_blknum([{blknum(), any()}], blknum(), non_neg_integer()) :: [pending_block()]
   defp add_new_blknum(pending_blocks, blknum, root_chain_height) do
     case Enum.any?(pending_blocks, fn {pending_blknum, _} -> pending_blknum == blknum end) do
       true -> pending_blocks
@@ -126,7 +127,7 @@ defmodule OMG.ChildChain.BlockQueue.Monitor do
     end
   end
 
-  @spec remove_blknum([{blknum(), any()}], blknum()) :: :ok
+  @spec remove_blknum([{blknum(), any()}], blknum()) :: [pending_block()]
   defp remove_blknum(pending_blocks, blknum) do
     Enum.reject(pending_blocks, fn {pending_blknum, _} -> pending_blknum == blknum end)
   end

From 0f127a8787ada8faf115083f79c8e17ebb5e770d Mon Sep 17 00:00:00 2001
From: unnawut <unnawut@omg.network>
Date: Fri, 17 Jul 2020 01:50:45 +0700
Subject: [PATCH 17/39] feat: block submission stall config and supervisor
 setup

---
 .../lib/omg_child_chain/block_queue/monitor.ex       |  2 +-
 .../lib/omg_child_chain/sync_supervisor.ex           |  9 +++++++++
 apps/omg_eth/lib/omg_eth/configuration.ex            | 12 ++++++++++++
 config/config.exs                                    |  2 ++
 4 files changed, 24 insertions(+), 1 deletion(-)

diff --git a/apps/omg_child_chain/lib/omg_child_chain/block_queue/monitor.ex b/apps/omg_child_chain/lib/omg_child_chain/block_queue/monitor.ex
index b4c9e33ce8..c530521de5 100644
--- a/apps/omg_child_chain/lib/omg_child_chain/block_queue/monitor.ex
+++ b/apps/omg_child_chain/lib/omg_child_chain/block_queue/monitor.ex
@@ -22,7 +22,7 @@ defmodule OMG.ChildChain.BlockQueue.Monitor do
 
   defstruct pending_blocks: [],
             root_chain_height: 0,
-            stall_threshold_in_root_chain_blocks: 2,
+            stall_threshold_in_root_chain_blocks: 4,
             alarm_module: nil,
             alarm_raised: false
 
diff --git a/apps/omg_child_chain/lib/omg_child_chain/sync_supervisor.ex b/apps/omg_child_chain/lib/omg_child_chain/sync_supervisor.ex
index 1b0c9bb5c5..a8b7d66bc0 100644
--- a/apps/omg_child_chain/lib/omg_child_chain/sync_supervisor.ex
+++ b/apps/omg_child_chain/lib/omg_child_chain/sync_supervisor.ex
@@ -70,6 +70,8 @@ defmodule OMG.ChildChain.SyncSupervisor do
     child_block_interval = OMG.Eth.Configuration.child_block_interval()
     contracts = OMG.Eth.Configuration.contracts()
     authority_address = OMG.Eth.Configuration.authority_address()
+    block_submission_stall_check_interval_ms = OMG.Eth.Configuration.block_submission_stall_check_interval_ms()
+    stall_threshold_in_root_chain_blocks = OMG.Eth.Configuration.block_submission_stall_threshold_in_root_chain_blocks()
 
     [
       {GasAnalyzer, []},
@@ -84,6 +86,13 @@ defmodule OMG.ChildChain.SyncSupervisor do
          block_submit_max_gas_price: block_submit_max_gas_price,
          child_block_interval: child_block_interval
        ]},
+      {BlockQueue.Monitor,
+       [
+         check_interval_ms: block_submission_stall_check_interval_ms,
+         stall_threshold_in_root_chain_blocks: stall_threshold_in_root_chain_blocks,
+         event_bus_module: OMG.Bus,
+         alarm_module: Alarm,
+       ]},
       {RootChainCoordinator,
        CoordinatorSetup.coordinator_setup(
          metrics_collection_interval,
diff --git a/apps/omg_eth/lib/omg_eth/configuration.ex b/apps/omg_eth/lib/omg_eth/configuration.ex
index 98a33f9c88..09759d25c1 100644
--- a/apps/omg_eth/lib/omg_eth/configuration.ex
+++ b/apps/omg_eth/lib/omg_eth/configuration.ex
@@ -66,11 +66,23 @@ defmodule OMG.Eth.Configuration do
     Application.fetch_env!(@app, :eth_node)
   end
 
+  @spec ethereum_events_check_interval_ms() :: pos_integer | no_return
   def ethereum_events_check_interval_ms() do
     Application.fetch_env!(@app, :ethereum_events_check_interval_ms)
   end
 
+  @spec ethereum_stalled_sync_threshold_ms() :: pos_integer | no_return
   def ethereum_stalled_sync_threshold_ms() do
     Application.fetch_env!(@app, :ethereum_stalled_sync_threshold_ms)
   end
+
+  @spec block_submission_stall_check_interval_ms() :: pos_integer | no_return
+  def block_submission_stall_check_interval_ms() do
+    Application.fetch_env!(@app, :block_submission_stall_check_interval_ms)
+  end
+
+  @spec block_submission_stall_threshold_in_root_chain_blocks() :: pos_integer | no_return
+  def block_submission_stall_threshold_in_root_chain_blocks() do
+    Application.fetch_env!(@app, :block_submission_stall_threshold_in_root_chain_blocks)
+  end
 end
diff --git a/config/config.exs b/config/config.exs
index ba7a920807..70de745ffc 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -106,6 +106,8 @@ config :omg_eth,
   ethereum_block_time_seconds: 15,
   ethereum_events_check_interval_ms: ethereum_events_check_interval_ms,
   ethereum_stalled_sync_threshold_ms: 20_000,
+  block_submission_stall_check_interval_ms: 5_000,
+  block_submission_stall_threshold_in_root_chain_blocks: 4,
   node_logging_in_debug: false
 
 config :omg_status,

From 04a575489d2ae69f2a47c8d6c406b8b8fd5539a3 Mon Sep 17 00:00:00 2001
From: unnawut <unnawut@omg.network>
Date: Fri, 17 Jul 2020 13:39:34 +0700
Subject: [PATCH 18/39] format: formatting and naming alignment

---
 .../lib/omg_child_chain/block_queue/monitor.ex       |  3 ++-
 .../lib/omg_child_chain/sync_supervisor.ex           | 10 +++++-----
 apps/omg_eth/lib/omg_eth/configuration.ex            | 12 ++++++------
 config/config.exs                                    |  4 ++--
 4 files changed, 15 insertions(+), 14 deletions(-)

diff --git a/apps/omg_child_chain/lib/omg_child_chain/block_queue/monitor.ex b/apps/omg_child_chain/lib/omg_child_chain/block_queue/monitor.ex
index c530521de5..681dc10609 100644
--- a/apps/omg_child_chain/lib/omg_child_chain/block_queue/monitor.ex
+++ b/apps/omg_child_chain/lib/omg_child_chain/block_queue/monitor.ex
@@ -163,7 +163,8 @@ defmodule OMG.ChildChain.BlockQueue.Monitor do
 
   defp log_stalled_blocks(stalled_blocks, root_chain_height) do
     Logger.warn(
-      "#{__MODULE__}: Stalled blocks: #{inspect(stalled_blocks)}. Current height: #{inspect(root_chain_height)}"
+      "#{__MODULE__}: Stalled blocks: #{inspect(stalled_blocks)}. " <>
+        "Current height: #{inspect(root_chain_height)}"
     )
   end
 end
diff --git a/apps/omg_child_chain/lib/omg_child_chain/sync_supervisor.ex b/apps/omg_child_chain/lib/omg_child_chain/sync_supervisor.ex
index a8b7d66bc0..2eb325463c 100644
--- a/apps/omg_child_chain/lib/omg_child_chain/sync_supervisor.ex
+++ b/apps/omg_child_chain/lib/omg_child_chain/sync_supervisor.ex
@@ -70,8 +70,8 @@ defmodule OMG.ChildChain.SyncSupervisor do
     child_block_interval = OMG.Eth.Configuration.child_block_interval()
     contracts = OMG.Eth.Configuration.contracts()
     authority_address = OMG.Eth.Configuration.authority_address()
-    block_submission_stall_check_interval_ms = OMG.Eth.Configuration.block_submission_stall_check_interval_ms()
-    stall_threshold_in_root_chain_blocks = OMG.Eth.Configuration.block_submission_stall_threshold_in_root_chain_blocks()
+    block_stalled_submission_check_interval_ms = OMG.Eth.Configuration.block_stalled_submission_check_interval_ms()
+    block_stalled_submission_threshold_in_root_chain_blocks = OMG.Eth.Configuration.block_stalled_submission_threshold_in_root_chain_blocks()
 
     [
       {GasAnalyzer, []},
@@ -88,10 +88,10 @@ defmodule OMG.ChildChain.SyncSupervisor do
        ]},
       {BlockQueue.Monitor,
        [
-         check_interval_ms: block_submission_stall_check_interval_ms,
-         stall_threshold_in_root_chain_blocks: stall_threshold_in_root_chain_blocks,
+         check_interval_ms: block_stalled_submission_check_interval_ms,
+         stall_threshold_in_root_chain_blocks: block_stalled_submission_threshold_in_root_chain_blocks,
          event_bus_module: OMG.Bus,
-         alarm_module: Alarm,
+         alarm_module: Alarm
        ]},
       {RootChainCoordinator,
        CoordinatorSetup.coordinator_setup(
diff --git a/apps/omg_eth/lib/omg_eth/configuration.ex b/apps/omg_eth/lib/omg_eth/configuration.ex
index 09759d25c1..43896c9367 100644
--- a/apps/omg_eth/lib/omg_eth/configuration.ex
+++ b/apps/omg_eth/lib/omg_eth/configuration.ex
@@ -76,13 +76,13 @@ defmodule OMG.Eth.Configuration do
     Application.fetch_env!(@app, :ethereum_stalled_sync_threshold_ms)
   end
 
-  @spec block_submission_stall_check_interval_ms() :: pos_integer | no_return
-  def block_submission_stall_check_interval_ms() do
-    Application.fetch_env!(@app, :block_submission_stall_check_interval_ms)
+  @spec block_stalled_submission_check_interval_ms() :: pos_integer | no_return
+  def block_stalled_submission_check_interval_ms() do
+    Application.fetch_env!(@app, :block_stalled_submission_check_interval_ms)
   end
 
-  @spec block_submission_stall_threshold_in_root_chain_blocks() :: pos_integer | no_return
-  def block_submission_stall_threshold_in_root_chain_blocks() do
-    Application.fetch_env!(@app, :block_submission_stall_threshold_in_root_chain_blocks)
+  @spec block_stalled_submission_threshold_in_root_chain_blocks() :: pos_integer | no_return
+  def block_stalled_submission_threshold_in_root_chain_blocks() do
+    Application.fetch_env!(@app, :block_stalled_submission_threshold_in_root_chain_blocks)
   end
 end
diff --git a/config/config.exs b/config/config.exs
index 70de745ffc..3bdf3599e8 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -106,8 +106,8 @@ config :omg_eth,
   ethereum_block_time_seconds: 15,
   ethereum_events_check_interval_ms: ethereum_events_check_interval_ms,
   ethereum_stalled_sync_threshold_ms: 20_000,
-  block_submission_stall_check_interval_ms: 5_000,
-  block_submission_stall_threshold_in_root_chain_blocks: 4,
+  block_stalled_submission_check_interval_ms: 5_000,
+  block_stalled_submission_threshold_in_root_chain_blocks: 4,
   node_logging_in_debug: false
 
 config :omg_status,

From d1594d3a42fd8b570a0c257aa4a90378b2fc273a Mon Sep 17 00:00:00 2001
From: unnawut <unnawut@omg.network>
Date: Fri, 17 Jul 2020 15:23:47 +0700
Subject: [PATCH 19/39] test: add BlockQueue.Monitor test

---
 .../omg_child_chain/block_queue/monitor.ex    |   2 +-
 .../lib/omg_child_chain/sync_supervisor.ex    |   4 +-
 .../block_queue/monitor_test.exs              | 204 ++++++++++++++++++
 3 files changed, 208 insertions(+), 2 deletions(-)
 create mode 100644 apps/omg_child_chain/test/omg_child_chain/block_queue/monitor_test.exs

diff --git a/apps/omg_child_chain/lib/omg_child_chain/block_queue/monitor.ex b/apps/omg_child_chain/lib/omg_child_chain/block_queue/monitor.ex
index 681dc10609..b6932e01a8 100644
--- a/apps/omg_child_chain/lib/omg_child_chain/block_queue/monitor.ex
+++ b/apps/omg_child_chain/lib/omg_child_chain/block_queue/monitor.ex
@@ -80,7 +80,7 @@ defmodule OMG.ChildChain.BlockQueue.Monitor do
     _ = log_stalled_blocks(stalled_blocks, state.root_chain_height)
     _ = trigger_alarm(state.alarm_module, state.alarm_raised, stalled_blocks)
 
-    {:ok, state}
+    {:noreply, state}
   end
 
   # Keeps track of the latest root chain height
diff --git a/apps/omg_child_chain/lib/omg_child_chain/sync_supervisor.ex b/apps/omg_child_chain/lib/omg_child_chain/sync_supervisor.ex
index 2eb325463c..ac43b59c72 100644
--- a/apps/omg_child_chain/lib/omg_child_chain/sync_supervisor.ex
+++ b/apps/omg_child_chain/lib/omg_child_chain/sync_supervisor.ex
@@ -20,6 +20,7 @@ defmodule OMG.ChildChain.SyncSupervisor do
   use Supervisor
   use OMG.Utils.LoggerExt
 
+  alias OMG.Bus
   alias OMG.ChildChain.BlockQueue
   alias OMG.ChildChain.BlockQueue.Balance
   alias OMG.ChildChain.BlockQueue.GasAnalyzer
@@ -31,6 +32,7 @@ defmodule OMG.ChildChain.SyncSupervisor do
   alias OMG.EthereumEventListener
   alias OMG.RootChainCoordinator
   alias OMG.State
+  alias OMG.Status.Alert.Alarm
 
   @events_bucket :events_bucket
   def start_link(args) do
@@ -90,7 +92,7 @@ defmodule OMG.ChildChain.SyncSupervisor do
        [
          check_interval_ms: block_stalled_submission_check_interval_ms,
          stall_threshold_in_root_chain_blocks: block_stalled_submission_threshold_in_root_chain_blocks,
-         event_bus_module: OMG.Bus,
+         event_bus_module: Bus,
          alarm_module: Alarm
        ]},
       {RootChainCoordinator,
diff --git a/apps/omg_child_chain/test/omg_child_chain/block_queue/monitor_test.exs b/apps/omg_child_chain/test/omg_child_chain/block_queue/monitor_test.exs
new file mode 100644
index 0000000000..28844bad8a
--- /dev/null
+++ b/apps/omg_child_chain/test/omg_child_chain/block_queue/monitor_test.exs
@@ -0,0 +1,204 @@
+# Copyright 2019-2020 OmiseGO Pte Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+defmodule OMG.ChildChain.BlockQueue.MonitorTest do
+  use ExUnit.Case, async: true
+  import ExUnit.CaptureLog, only: [capture_log: 1]
+  alias OMG.ChildChain.BlockQueue.Monitor
+
+  setup_all do
+    {:ok, apps} = Application.ensure_all_started(:omg_status)
+
+    on_exit(fn ->
+      apps |> Enum.reverse() |> Enum.each(&Application.stop/1)
+    end)
+
+    :ok
+  end
+
+  setup do
+    {:ok, alarm} = __MODULE__.Alarm.start(self())
+    stall_threshold_blocks = 10
+    check_interval_ms = 50
+
+    {:ok, monitor} =
+      Monitor.start_link(
+        alarm_module: __MODULE__.Alarm,
+        event_bus: __MODULE__.BusMock,
+        stall_threshold_in_root_chain_blocks: stall_threshold_blocks,
+        check_interval_ms: check_interval_ms
+      )
+
+    :ok = on_exit(fn ->
+      _ = Process.exit(alarm, :test_cleanup)
+      _ = Process.exit(monitor, :test_cleanup)
+      _ = Process.sleep(10)
+    end)
+
+    {:ok, %{
+      alarm: alarm,
+      monitor: monitor,
+      stall_threshold_blocks: stall_threshold_blocks,
+      check_interval_ms: check_interval_ms
+    }}
+  end
+
+  test "does not raise :block_submission_stalled alarm when block is below stall threshold", context do
+    height = 1
+    _ = send(context.monitor, {:internal_event_bus, :ethereum_new_height, height})
+    _ = send(context.monitor, {:internal_event_bus, :block_submitting, 1000})
+
+    capture_log(fn ->
+      _ = send(context.monitor, {:internal_event_bus, :ethereum_new_height, height + context.stall_threshold_blocks - 1})
+      refute_receive(:got_raise_alarm)
+    end)
+  end
+
+  test "raises :block_submission_stalled alarm when blocks is at stall threshold", context do
+    height = 1
+    _ = send(context.monitor, {:internal_event_bus, :ethereum_new_height, height})
+    _ = send(context.monitor, {:internal_event_bus, :block_submitting, 1000})
+
+    capture_log(fn ->
+      _ = send(context.monitor, {:internal_event_bus, :ethereum_new_height, height + context.stall_threshold_blocks})
+      assert_receive(:got_raise_alarm)
+    end)
+  end
+
+  test "raises :block_submission_stalled alarm when blocks is above stall threshold", context do
+    height = 1
+    _ = send(context.monitor, {:internal_event_bus, :ethereum_new_height, height})
+    _ = send(context.monitor, {:internal_event_bus, :block_submitting, 1000})
+
+    capture_log(fn ->
+      _ = send(context.monitor, {:internal_event_bus, :ethereum_new_height, height + context.stall_threshold_blocks + 1})
+      assert_receive(:got_raise_alarm)
+    end)
+  end
+
+  test "does not raise :block_submission_stalled alarm when it is already raised", context do
+    height = 1
+    _ = send(context.monitor, {:internal_event_bus, :ethereum_new_height, height})
+    _ = send(context.monitor, {:internal_event_bus, :block_submitting, 1000})
+
+    stalled_height = height + context.stall_threshold_blocks + 1
+
+    capture_log(fn ->
+      _ = send(context.monitor, {:internal_event_bus, :ethereum_new_height, stalled_height})
+      assert_receive(:got_raise_alarm)
+    end)
+
+    # Because we're bypassing the actual alarm mechanism with a custom one for testing, the monitor's
+    # alarm_raised state does not get triggered, so we manually set `alarm_raised: true` here so that
+    # the alarm can be cleared
+    _ = GenServer.cast(context.monitor, {:set_alarm, :block_submission_stalled})
+
+    _ = send(context.monitor, {:internal_event_bus, :ethereum_new_height, stalled_height + 1})
+    _ = send(context.monitor, {:internal_event_bus, :block_submitting, 2000})
+
+    next_stalled_height = stalled_height + context.stall_threshold_blocks + 1
+
+    capture_log(fn ->
+      _ = send(context.monitor, {:internal_event_bus, :ethereum_new_height, next_stalled_height})
+      # We wait for 10x the check interval to make sure it really does not get raised again.
+      refute_receive(:got_raise_alarm, context.check_interval_ms * 10)
+    end)
+  end
+
+  test "clears :block_submission_stalled alarm when the stalled block no longer stalls", context do
+    height = 1
+    _ = send(context.monitor, {:internal_event_bus, :ethereum_new_height, height})
+    _ = send(context.monitor, {:internal_event_bus, :block_submitting, 1000})
+
+    capture_log(fn ->
+      _ = send(context.monitor, {:internal_event_bus, :ethereum_new_height, height + context.stall_threshold_blocks + 1})
+      assert_receive(:got_raise_alarm)
+    end)
+
+    # Because we're bypassing the actual alarm mechanism with a custom one for testing, the monitor's
+    # alarm_raised state does not get triggered, so we manually set `alarm_raised: true` here so that
+    # the alarm can be cleared
+    _ = GenServer.cast(context.monitor, {:set_alarm, :block_submission_stalled})
+
+    _ = send(context.monitor, {:internal_event_bus, :block_submitted, 1000})
+    assert_receive(:got_clear_alarm)
+  end
+
+  test "does not clear :block_submission_stalled alarm when some but not all stalled blocks got submitted", context do
+    _ = send(context.monitor, {:internal_event_bus, :ethereum_new_height, 100})
+    _ = send(context.monitor, {:internal_event_bus, :block_submitting, 1000})
+
+    _ = send(context.monitor, {:internal_event_bus, :ethereum_new_height, 200})
+    _ = send(context.monitor, {:internal_event_bus, :block_submitting, 2000})
+
+    capture_log(fn ->
+      # At height 300, both block #1000 and #2000 are stalled, and the alarm should've been raised
+      _ = send(context.monitor, {:internal_event_bus, :ethereum_new_height, 300})
+      assert_receive(:got_raise_alarm)
+
+      # Because we're bypassing the actual alarm mechanism with a custom one for testing, the monitor's
+      # alarm_raised state does not get triggered, so we manually set `alarm_raised: true` here so that
+      # the alarm can be cleared.
+      _ = GenServer.cast(context.monitor, {:set_alarm, :block_submission_stalled})
+
+      # Now we tell the monitor that block #1000 is submitted, leaving #2000 still stalled
+      _ = send(context.monitor, {:internal_event_bus, :block_submitted, 1000})
+
+      # Because #2000 is still stalled, the alarm must not be cleared. We wait for 10x the check interval
+      # to make sure it really does not get cleared.
+      refute_receive(:got_clear_alarm, context.check_interval_ms * 10)
+    end)
+  end
+
+  defmodule Alarm do
+    @moduledoc """
+    Mocks `OMG.Status.Alert.Alarm` so we can observe it for test assertions.
+    """
+    use GenServer
+
+    def start(listener) do
+      GenServer.start(__MODULE__, [listener], name: __MODULE__)
+    end
+
+    def init([listener]) do
+      {:ok, %{listener: listener}}
+    end
+
+    def block_submission_stalled(reporter) do
+      {:block_submission_stalled, %{node: Node.self(), reporter: reporter}}
+    end
+
+    def set({:block_submission_stalled, _details}) do
+      GenServer.call(__MODULE__, :got_raise_alarm)
+    end
+
+    def clear({:block_submission_stalled, _details}) do
+      GenServer.call(__MODULE__, :got_clear_alarm)
+    end
+
+    def handle_call(:got_raise_alarm, _, state) do
+      {:reply, send(state.listener, :got_raise_alarm), state}
+    end
+
+    def handle_call(:got_clear_alarm, _, state) do
+      {:reply, send(state.listener, :got_clear_alarm), state}
+    end
+  end
+
+  defmodule BusMock do
+    def subscribe(_, _) do
+      :ok
+    end
+  end
+end

From 0916252ad8d169476f46d0121810d1d717d471ec Mon Sep 17 00:00:00 2001
From: unnawut <unnawut@omg.network>
Date: Fri, 17 Jul 2020 15:28:27 +0700
Subject: [PATCH 20/39] refactor: shorten config name

---
 .../lib/omg_child_chain/block_queue/monitor.ex            | 8 ++++----
 .../lib/omg_child_chain/sync_supervisor.ex                | 4 ++--
 apps/omg_eth/lib/omg_eth/configuration.ex                 | 6 +++---
 config/config.exs                                         | 2 +-
 4 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/apps/omg_child_chain/lib/omg_child_chain/block_queue/monitor.ex b/apps/omg_child_chain/lib/omg_child_chain/block_queue/monitor.ex
index b6932e01a8..58afea33a5 100644
--- a/apps/omg_child_chain/lib/omg_child_chain/block_queue/monitor.ex
+++ b/apps/omg_child_chain/lib/omg_child_chain/block_queue/monitor.ex
@@ -22,7 +22,7 @@ defmodule OMG.ChildChain.BlockQueue.Monitor do
 
   defstruct pending_blocks: [],
             root_chain_height: 0,
-            stall_threshold_in_root_chain_blocks: 4,
+            stall_threshold_blocks: 4,
             alarm_module: nil,
             alarm_raised: false
 
@@ -32,7 +32,7 @@ defmodule OMG.ChildChain.BlockQueue.Monitor do
   @type t() :: %__MODULE__{
           pending_blocks: [pending_block()],
           root_chain_height: non_neg_integer(),
-          stall_threshold_in_root_chain_blocks: pos_integer(),
+          stall_threshold_blocks: pos_integer(),
           alarm_module: module(),
           alarm_raised: boolean()
         }
@@ -57,7 +57,7 @@ defmodule OMG.ChildChain.BlockQueue.Monitor do
 
     state = %__MODULE__{
       pending_blocks: [],
-      stall_threshold_in_root_chain_blocks: Keyword.fetch!(opts, :stall_threshold_in_root_chain_blocks),
+      stall_threshold_blocks: Keyword.fetch!(opts, :stall_threshold_blocks),
       alarm_module: Keyword.fetch!(opts, :alarm_module),
       alarm_raised: false
     }
@@ -72,7 +72,7 @@ defmodule OMG.ChildChain.BlockQueue.Monitor do
   def handle_info(:check_stall, state) do
     stalled_blocks =
       Enum.filter(state.pending_blocks, fn {_blknum, first_submit_height} ->
-        state.root_chain_height - first_submit_height >= state.stall_threshold_in_root_chain_blocks
+        state.root_chain_height - first_submit_height >= state.stall_threshold_blocks
       end)
 
     _ = :telemetry.execute([:blocks_submitting, __MODULE__], %{blocks: state.pending_blocks})
diff --git a/apps/omg_child_chain/lib/omg_child_chain/sync_supervisor.ex b/apps/omg_child_chain/lib/omg_child_chain/sync_supervisor.ex
index ac43b59c72..645352b09c 100644
--- a/apps/omg_child_chain/lib/omg_child_chain/sync_supervisor.ex
+++ b/apps/omg_child_chain/lib/omg_child_chain/sync_supervisor.ex
@@ -73,7 +73,7 @@ defmodule OMG.ChildChain.SyncSupervisor do
     contracts = OMG.Eth.Configuration.contracts()
     authority_address = OMG.Eth.Configuration.authority_address()
     block_stalled_submission_check_interval_ms = OMG.Eth.Configuration.block_stalled_submission_check_interval_ms()
-    block_stalled_submission_threshold_in_root_chain_blocks = OMG.Eth.Configuration.block_stalled_submission_threshold_in_root_chain_blocks()
+    block_stalled_submission_threshold_blocks = OMG.Eth.Configuration.block_stalled_submission_threshold_blocks()
 
     [
       {GasAnalyzer, []},
@@ -91,7 +91,7 @@ defmodule OMG.ChildChain.SyncSupervisor do
       {BlockQueue.Monitor,
        [
          check_interval_ms: block_stalled_submission_check_interval_ms,
-         stall_threshold_in_root_chain_blocks: block_stalled_submission_threshold_in_root_chain_blocks,
+         stall_threshold_blocks: block_stalled_submission_threshold_blocks,
          event_bus_module: Bus,
          alarm_module: Alarm
        ]},
diff --git a/apps/omg_eth/lib/omg_eth/configuration.ex b/apps/omg_eth/lib/omg_eth/configuration.ex
index 43896c9367..43e9383abb 100644
--- a/apps/omg_eth/lib/omg_eth/configuration.ex
+++ b/apps/omg_eth/lib/omg_eth/configuration.ex
@@ -81,8 +81,8 @@ defmodule OMG.Eth.Configuration do
     Application.fetch_env!(@app, :block_stalled_submission_check_interval_ms)
   end
 
-  @spec block_stalled_submission_threshold_in_root_chain_blocks() :: pos_integer | no_return
-  def block_stalled_submission_threshold_in_root_chain_blocks() do
-    Application.fetch_env!(@app, :block_stalled_submission_threshold_in_root_chain_blocks)
+  @spec block_stalled_submission_threshold_blocks() :: pos_integer | no_return
+  def block_stalled_submission_threshold_blocks() do
+    Application.fetch_env!(@app, :block_stalled_submission_threshold_blocks)
   end
 end
diff --git a/config/config.exs b/config/config.exs
index 3bdf3599e8..8daf67f21e 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -107,7 +107,7 @@ config :omg_eth,
   ethereum_events_check_interval_ms: ethereum_events_check_interval_ms,
   ethereum_stalled_sync_threshold_ms: 20_000,
   block_stalled_submission_check_interval_ms: 5_000,
-  block_stalled_submission_threshold_in_root_chain_blocks: 4,
+  block_stalled_submission_threshold_blocks: 4,
   node_logging_in_debug: false
 
 config :omg_status,

From 44c5ebc39c822aed424501b56125ba4e07ca24b8 Mon Sep 17 00:00:00 2001
From: unnawut <unnawut@omg.network>
Date: Fri, 17 Jul 2020 16:09:14 +0700
Subject: [PATCH 21/39] test: fix tests

---
 .../block_queue/monitor_test.exs              | 142 ++++++++----------
 1 file changed, 65 insertions(+), 77 deletions(-)

diff --git a/apps/omg_child_chain/test/omg_child_chain/block_queue/monitor_test.exs b/apps/omg_child_chain/test/omg_child_chain/block_queue/monitor_test.exs
index 28844bad8a..5af63d2a92 100644
--- a/apps/omg_child_chain/test/omg_child_chain/block_queue/monitor_test.exs
+++ b/apps/omg_child_chain/test/omg_child_chain/block_queue/monitor_test.exs
@@ -30,133 +30,121 @@ defmodule OMG.ChildChain.BlockQueue.MonitorTest do
   setup do
     {:ok, alarm} = __MODULE__.Alarm.start(self())
     stall_threshold_blocks = 10
-    check_interval_ms = 50
+    check_interval_ms = 10
 
     {:ok, monitor} =
       Monitor.start_link(
         alarm_module: __MODULE__.Alarm,
         event_bus: __MODULE__.BusMock,
-        stall_threshold_in_root_chain_blocks: stall_threshold_blocks,
+        stall_threshold_blocks: stall_threshold_blocks,
         check_interval_ms: check_interval_ms
       )
 
-    :ok = on_exit(fn ->
-      _ = Process.exit(alarm, :test_cleanup)
-      _ = Process.exit(monitor, :test_cleanup)
-      _ = Process.sleep(10)
-    end)
-
-    {:ok, %{
-      alarm: alarm,
-      monitor: monitor,
-      stall_threshold_blocks: stall_threshold_blocks,
-      check_interval_ms: check_interval_ms
-    }}
+    :ok =
+      on_exit(fn ->
+        _ = Process.exit(alarm, :test_cleanup)
+        _ = Process.exit(monitor, :test_cleanup)
+        _ = Process.sleep(10)
+      end)
+
+    {:ok,
+     %{
+       alarm: alarm,
+       monitor: monitor,
+       stall_threshold_blocks: stall_threshold_blocks,
+       check_interval_ms: check_interval_ms
+     }}
   end
 
   test "does not raise :block_submission_stalled alarm when block is below stall threshold", context do
-    height = 1
-    _ = send(context.monitor, {:internal_event_bus, :ethereum_new_height, height})
-    _ = send(context.monitor, {:internal_event_bus, :block_submitting, 1000})
-
     capture_log(fn ->
-      _ = send(context.monitor, {:internal_event_bus, :ethereum_new_height, height + context.stall_threshold_blocks - 1})
-      refute_receive(:got_raise_alarm)
+      # Inform the monitor of a pending block
+      _ = send(context.monitor, {:internal_event_bus, :block_submitting, 1000})
+
+      # Push the height just below the stalling height
+      _ = send(context.monitor, {:internal_event_bus, :ethereum_new_height, context.stall_threshold_blocks - 1})
+
+      # Wait for 10x the check interval to make sure it really does not get raised again.
+      refute_receive(:got_raise_alarm, context.check_interval_ms * 10)
     end)
   end
 
   test "raises :block_submission_stalled alarm when blocks is at stall threshold", context do
-    height = 1
-    _ = send(context.monitor, {:internal_event_bus, :ethereum_new_height, height})
-    _ = send(context.monitor, {:internal_event_bus, :block_submitting, 1000})
-
     capture_log(fn ->
-      _ = send(context.monitor, {:internal_event_bus, :ethereum_new_height, height + context.stall_threshold_blocks})
+      # Inform the monitor of a pending block
+      _ = send(context.monitor, {:internal_event_bus, :block_submitting, 1000})
+
+      # Push the height to the stalling height
+      _ = send(context.monitor, {:internal_event_bus, :ethereum_new_height, context.stall_threshold_blocks})
+
       assert_receive(:got_raise_alarm)
     end)
   end
 
   test "raises :block_submission_stalled alarm when blocks is above stall threshold", context do
-    height = 1
-    _ = send(context.monitor, {:internal_event_bus, :ethereum_new_height, height})
-    _ = send(context.monitor, {:internal_event_bus, :block_submitting, 1000})
-
     capture_log(fn ->
-      _ = send(context.monitor, {:internal_event_bus, :ethereum_new_height, height + context.stall_threshold_blocks + 1})
+      # Inform the monitor of a pending block
+      _ = send(context.monitor, {:internal_event_bus, :block_submitting, 1000})
+
+      # Push the height pass the stalling height
+      _ = send(context.monitor, {:internal_event_bus, :ethereum_new_height, context.stall_threshold_blocks + 1})
+
       assert_receive(:got_raise_alarm)
     end)
   end
 
   test "does not raise :block_submission_stalled alarm when it is already raised", context do
-    height = 1
-    _ = send(context.monitor, {:internal_event_bus, :ethereum_new_height, height})
-    _ = send(context.monitor, {:internal_event_bus, :block_submitting, 1000})
-
-    stalled_height = height + context.stall_threshold_blocks + 1
+    # Set the monitor in a raised state
+    :sys.replace_state(context.monitor, fn state -> %{state | alarm_raised: true} end)
 
     capture_log(fn ->
-      _ = send(context.monitor, {:internal_event_bus, :ethereum_new_height, stalled_height})
-      assert_receive(:got_raise_alarm)
-    end)
-
-    # Because we're bypassing the actual alarm mechanism with a custom one for testing, the monitor's
-    # alarm_raised state does not get triggered, so we manually set `alarm_raised: true` here so that
-    # the alarm can be cleared
-    _ = GenServer.cast(context.monitor, {:set_alarm, :block_submission_stalled})
+      # Inform the monitor of a pending block
+      _ = send(context.monitor, {:internal_event_bus, :block_submitting, 1000})
 
-    _ = send(context.monitor, {:internal_event_bus, :ethereum_new_height, stalled_height + 1})
-    _ = send(context.monitor, {:internal_event_bus, :block_submitting, 2000})
+      # Push the height pass the stalling height
+      _ = send(context.monitor, {:internal_event_bus, :ethereum_new_height, context.stall_threshold_blocks})
 
-    next_stalled_height = stalled_height + context.stall_threshold_blocks + 1
-
-    capture_log(fn ->
-      _ = send(context.monitor, {:internal_event_bus, :ethereum_new_height, next_stalled_height})
-      # We wait for 10x the check interval to make sure it really does not get raised again.
+      # Wait for 10x the check interval to make sure it really does not get raised again.
       refute_receive(:got_raise_alarm, context.check_interval_ms * 10)
     end)
   end
 
   test "clears :block_submission_stalled alarm when the stalled block no longer stalls", context do
-    height = 1
-    _ = send(context.monitor, {:internal_event_bus, :ethereum_new_height, height})
-    _ = send(context.monitor, {:internal_event_bus, :block_submitting, 1000})
+    # Set the monitor in a raised state
+    :sys.replace_state(context.monitor, fn state -> %{state | alarm_raised: true} end)
 
     capture_log(fn ->
-      _ = send(context.monitor, {:internal_event_bus, :ethereum_new_height, height + context.stall_threshold_blocks + 1})
-      assert_receive(:got_raise_alarm)
-    end)
+      # Inform the monitor of a pending block
+      _ = send(context.monitor, {:internal_event_bus, :block_submitting, 1000})
+
+      # Push the height pass the stalling height
+      _ = send(context.monitor, {:internal_event_bus, :ethereum_new_height, context.stall_threshold_blocks})
 
-    # Because we're bypassing the actual alarm mechanism with a custom one for testing, the monitor's
-    # alarm_raised state does not get triggered, so we manually set `alarm_raised: true` here so that
-    # the alarm can be cleared
-    _ = GenServer.cast(context.monitor, {:set_alarm, :block_submission_stalled})
+      # Now we inform the monitor that block #1000 is submitted
+      _ = send(context.monitor, {:internal_event_bus, :block_submitted, 1000})
 
-    _ = send(context.monitor, {:internal_event_bus, :block_submitted, 1000})
-    assert_receive(:got_clear_alarm)
+      # Expecting the alarm to be cleared
+      assert_receive(:got_clear_alarm)
+    end)
   end
 
   test "does not clear :block_submission_stalled alarm when some but not all stalled blocks got submitted", context do
-    _ = send(context.monitor, {:internal_event_bus, :ethereum_new_height, 100})
-    _ = send(context.monitor, {:internal_event_bus, :block_submitting, 1000})
-
-    _ = send(context.monitor, {:internal_event_bus, :ethereum_new_height, 200})
-    _ = send(context.monitor, {:internal_event_bus, :block_submitting, 2000})
+    # Set the monitor in a raised state
+    :sys.replace_state(context.monitor, fn state -> %{state | alarm_raised: true} end)
 
     capture_log(fn ->
-      # At height 300, both block #1000 and #2000 are stalled, and the alarm should've been raised
-      _ = send(context.monitor, {:internal_event_bus, :ethereum_new_height, 300})
-      assert_receive(:got_raise_alarm)
+      # Inform the monitor of two pending blocks
+      _ = send(context.monitor, {:internal_event_bus, :block_submitting, 1000})
+      _ = send(context.monitor, {:internal_event_bus, :block_submitting, 2000})
 
-      # Because we're bypassing the actual alarm mechanism with a custom one for testing, the monitor's
-      # alarm_raised state does not get triggered, so we manually set `alarm_raised: true` here so that
-      # the alarm can be cleared.
-      _ = GenServer.cast(context.monitor, {:set_alarm, :block_submission_stalled})
+      # Push the height pass the stalling height
+      _ = send(context.monitor, {:internal_event_bus, :ethereum_new_height, context.stall_threshold_blocks})
 
-      # Now we tell the monitor that block #1000 is submitted, leaving #2000 still stalled
+      # Now we inform the monitor that block #1000 is submitted, leaving #2000 still stalled
       _ = send(context.monitor, {:internal_event_bus, :block_submitted, 1000})
 
-      # Because #2000 is still stalled, the alarm must not be cleared. We wait for 10x the check interval
-      # to make sure it really does not get cleared.
+      # Because #2000 is still stalled, the alarm must not be cleared.
+      # Wait for 10x the check interval to make sure it really does not get cleared.
       refute_receive(:got_clear_alarm, context.check_interval_ms * 10)
     end)
   end

From 14c6f372dc3b07819e2c8deac1fb28591d4e629b Mon Sep 17 00:00:00 2001
From: unnawut <unnawut@omg.network>
Date: Fri, 17 Jul 2020 17:21:50 +0700
Subject: [PATCH 22/39] fix: broken service startup

---
 .../lib/omg_child_chain/block_queue/measure.ex        | 11 ++++++-----
 .../block_queue/{monitor.ex => submission_monitor.ex} |  4 ++--
 .../lib/omg_child_chain/sync_supervisor.ex            |  3 ++-
 .../{monitor_test.exs => submission_monitor_test.exs} |  6 +++---
 4 files changed, 13 insertions(+), 11 deletions(-)
 rename apps/omg_child_chain/lib/omg_child_chain/block_queue/{monitor.ex => submission_monitor.ex} (98%)
 rename apps/omg_child_chain/test/omg_child_chain/block_queue/{monitor_test.exs => submission_monitor_test.exs} (97%)

diff --git a/apps/omg_child_chain/lib/omg_child_chain/block_queue/measure.ex b/apps/omg_child_chain/lib/omg_child_chain/block_queue/measure.ex
index d62c3ba68d..9140b57b84 100644
--- a/apps/omg_child_chain/lib/omg_child_chain/block_queue/measure.ex
+++ b/apps/omg_child_chain/lib/omg_child_chain/block_queue/measure.ex
@@ -27,6 +27,7 @@ defmodule OMG.ChildChain.BlockQueue.Measure do
   alias OMG.ChildChain.BlockQueue
   alias OMG.ChildChain.BlockQueue.Balance
   alias OMG.ChildChain.BlockQueue.GasAnalyzer
+  alias OMG.ChildChain.BlockQueue.SubmissionMonitor
   alias OMG.Status.Metric.Datadog
 
   @supported_events [
@@ -35,14 +36,14 @@ defmodule OMG.ChildChain.BlockQueue.Measure do
     [:blknum_submitting, BlockQueue],
     [:blknum_submitted, BlockQueue],
     # Events providing collections of blocks being submitted or stalled
-    [:blocks_submitting, BlockQueue.Monitor],
-    [:blocks_stalled, BlockQueue.Monitor],
+    [:blocks_submitting, SubmissionMonitor],
+    [:blocks_stalled, SubmissionMonitor],
     [:gas, GasAnalyzer],
     [:authority_balance, Balance]
   ]
   def supported_events(), do: @supported_events
 
-  def handle_event([:process, BlockQueue], _, _state, _config) do
+  def handle_event([:process, BlockQueue], _measurements, _metadata, _config) do
     value =
       self()
       |> Process.info(:message_queue_len)
@@ -71,11 +72,11 @@ defmodule OMG.ChildChain.BlockQueue.Measure do
     _ = Datadog.increment(name(:block_submission_success), 1)
   end
 
-  def handle_event([:blocks_submitting, BlockQueue.Monitor], _, %{blocks: blocks}, _config) do
+  def handle_event([:blocks_submitting, SubmissionMonitor], %{blocks: blocks}, _, _config) do
     _ = Datadog.gauge(name(:block_queue_num_blocks_submitting), length(blocks))
   end
 
-  def handle_event([:blocks_stalled, BlockQueue.Monitor], _, %{blocks: blocks}, _config) do
+  def handle_event([:blocks_stalled, SubmissionMonitor], %{blocks: blocks}, _, _config) do
     _ = Datadog.gauge(name(:block_queue_num_blocks_stalled), length(blocks))
   end
 end
diff --git a/apps/omg_child_chain/lib/omg_child_chain/block_queue/monitor.ex b/apps/omg_child_chain/lib/omg_child_chain/block_queue/submission_monitor.ex
similarity index 98%
rename from apps/omg_child_chain/lib/omg_child_chain/block_queue/monitor.ex
rename to apps/omg_child_chain/lib/omg_child_chain/block_queue/submission_monitor.ex
index 58afea33a5..ea2e47c3fb 100644
--- a/apps/omg_child_chain/lib/omg_child_chain/block_queue/monitor.ex
+++ b/apps/omg_child_chain/lib/omg_child_chain/block_queue/submission_monitor.ex
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-defmodule OMG.ChildChain.BlockQueue.Monitor do
+defmodule OMG.ChildChain.BlockQueue.SubmissionMonitor do
   @moduledoc """
   Listens to block events and raises :block_submission_stalled alarm when a pending block
   doesn't get successfully submitted within the specified time threshold.
@@ -52,7 +52,7 @@ defmodule OMG.ChildChain.BlockQueue.Monitor do
   def init(opts) do
     _ = Logger.info("Starting #{__MODULE__}")
     _ = install_alarm_handler()
-    event_bus = Keyword.fetch!(opts, :event_bus)
+    event_bus = Keyword.fetch!(opts, :event_bus_module)
     check_interval_ms = Keyword.fetch!(opts, :check_interval_ms)
 
     state = %__MODULE__{
diff --git a/apps/omg_child_chain/lib/omg_child_chain/sync_supervisor.ex b/apps/omg_child_chain/lib/omg_child_chain/sync_supervisor.ex
index 645352b09c..6c1169176b 100644
--- a/apps/omg_child_chain/lib/omg_child_chain/sync_supervisor.ex
+++ b/apps/omg_child_chain/lib/omg_child_chain/sync_supervisor.ex
@@ -24,6 +24,7 @@ defmodule OMG.ChildChain.SyncSupervisor do
   alias OMG.ChildChain.BlockQueue
   alias OMG.ChildChain.BlockQueue.Balance
   alias OMG.ChildChain.BlockQueue.GasAnalyzer
+  alias OMG.ChildChain.BlockQueue.SubmissionMonitor
   alias OMG.ChildChain.ChildManager
   alias OMG.ChildChain.Configuration
   alias OMG.ChildChain.CoordinatorSetup
@@ -88,7 +89,7 @@ defmodule OMG.ChildChain.SyncSupervisor do
          block_submit_max_gas_price: block_submit_max_gas_price,
          child_block_interval: child_block_interval
        ]},
-      {BlockQueue.Monitor,
+      {SubmissionMonitor,
        [
          check_interval_ms: block_stalled_submission_check_interval_ms,
          stall_threshold_blocks: block_stalled_submission_threshold_blocks,
diff --git a/apps/omg_child_chain/test/omg_child_chain/block_queue/monitor_test.exs b/apps/omg_child_chain/test/omg_child_chain/block_queue/submission_monitor_test.exs
similarity index 97%
rename from apps/omg_child_chain/test/omg_child_chain/block_queue/monitor_test.exs
rename to apps/omg_child_chain/test/omg_child_chain/block_queue/submission_monitor_test.exs
index 5af63d2a92..9f439378db 100644
--- a/apps/omg_child_chain/test/omg_child_chain/block_queue/monitor_test.exs
+++ b/apps/omg_child_chain/test/omg_child_chain/block_queue/submission_monitor_test.exs
@@ -12,10 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-defmodule OMG.ChildChain.BlockQueue.MonitorTest do
+defmodule OMG.ChildChain.BlockQueue.SubmissionMonitorTest do
   use ExUnit.Case, async: true
   import ExUnit.CaptureLog, only: [capture_log: 1]
-  alias OMG.ChildChain.BlockQueue.Monitor
+  alias OMG.ChildChain.BlockQueue.SubmissionMonitor
 
   setup_all do
     {:ok, apps} = Application.ensure_all_started(:omg_status)
@@ -33,7 +33,7 @@ defmodule OMG.ChildChain.BlockQueue.MonitorTest do
     check_interval_ms = 10
 
     {:ok, monitor} =
-      Monitor.start_link(
+      SubmissionMonitor.start_link(
         alarm_module: __MODULE__.Alarm,
         event_bus: __MODULE__.BusMock,
         stall_threshold_blocks: stall_threshold_blocks,

From d4079fc65fb062c22aebc5e8d75e5c5f9897e85c Mon Sep 17 00:00:00 2001
From: unnawut <unnawut@omg.network>
Date: Fri, 17 Jul 2020 17:36:01 +0700
Subject: [PATCH 23/39] feat: configurable block submit stall from env var

---
 .../set_block_submit_stall_threshold.ex       | 52 +++++++++++++++++++
 .../lib/omg_child_chain/sync_supervisor.ex    |  7 ++-
 .../set_block_submit_stall_threshold_test.exs | 38 ++++++++++++++
 config/config.exs                             |  4 +-
 docs/deployment_configuration.md              |  3 +-
 5 files changed, 97 insertions(+), 7 deletions(-)
 create mode 100644 apps/omg_child_chain/lib/omg_child_chain/release_tasks/set_block_submit_stall_threshold.ex
 create mode 100644 apps/omg_child_chain/test/omg_child_chain/release_tasks/set_block_submit_stall_threshold_test.exs

diff --git a/apps/omg_child_chain/lib/omg_child_chain/release_tasks/set_block_submit_stall_threshold.ex b/apps/omg_child_chain/lib/omg_child_chain/release_tasks/set_block_submit_stall_threshold.ex
new file mode 100644
index 0000000000..f3a26df52d
--- /dev/null
+++ b/apps/omg_child_chain/lib/omg_child_chain/release_tasks/set_block_submit_stall_threshold.ex
@@ -0,0 +1,52 @@
+# Copyright 2019-2019 OmiseGO Pte Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+defmodule OMG.ChildChain.ReleaseTasks.SetBlockSubmitStallThreshold do
+  @moduledoc false
+  @behaviour Config.Provider
+  require Logger
+
+  @app :omg_child_chain
+  @config_key :block_submit_stall_threshold_blocks
+  @env_var_name "BLOCK_SUBMIT_STALL_THRESHOLD_BLOCKS"
+
+  def init(args) do
+    args
+  end
+
+  def load(config, _args) do
+    _ = on_load()
+    stall_threshold_blocks = stall_threshold_blocks()
+    Config.Reader.merge(config, omg_child_chain: [block_submit_stall_threshold_blocks: stall_threshold_blocks])
+  end
+
+  defp stall_threshold_blocks() do
+    stall_threshold_blocks =
+      @env_var_name
+      |> System.get_env()
+      |> validate_integer(Application.get_env(@app, @config_key))
+
+    _ = Logger.info("CONFIGURATION: App: #{@app} Key: #{@config_key} Value: #{inspect(stall_threshold_blocks)}.")
+
+    stall_threshold_blocks
+  end
+
+  defp validate_integer(value, _default) when is_binary(value), do: String.to_integer(value)
+  defp validate_integer(_, default), do: default
+
+  defp on_load() do
+    _ = Application.ensure_all_started(:logger)
+    _ = Application.load(@app)
+  end
+end
diff --git a/apps/omg_child_chain/lib/omg_child_chain/sync_supervisor.ex b/apps/omg_child_chain/lib/omg_child_chain/sync_supervisor.ex
index 6c1169176b..218958eb8e 100644
--- a/apps/omg_child_chain/lib/omg_child_chain/sync_supervisor.ex
+++ b/apps/omg_child_chain/lib/omg_child_chain/sync_supervisor.ex
@@ -67,14 +67,13 @@ defmodule OMG.ChildChain.SyncSupervisor do
     submission_finality_margin = Configuration.submission_finality_margin()
     block_submit_every_nth = Configuration.block_submit_every_nth()
     block_submit_max_gas_price = Configuration.block_submit_max_gas_price()
+    block_submit_stall_threshold_blocks = OMG.ChildChain.Configuration.block_submit_stall_threshold_blocks()
     ethereum_events_check_interval_ms = OMG.Configuration.ethereum_events_check_interval_ms()
     coordinator_eth_height_check_interval_ms = OMG.Configuration.coordinator_eth_height_check_interval_ms()
     deposit_finality_margin = OMG.Configuration.deposit_finality_margin()
     child_block_interval = OMG.Eth.Configuration.child_block_interval()
     contracts = OMG.Eth.Configuration.contracts()
     authority_address = OMG.Eth.Configuration.authority_address()
-    block_stalled_submission_check_interval_ms = OMG.Eth.Configuration.block_stalled_submission_check_interval_ms()
-    block_stalled_submission_threshold_blocks = OMG.Eth.Configuration.block_stalled_submission_threshold_blocks()
 
     [
       {GasAnalyzer, []},
@@ -91,8 +90,8 @@ defmodule OMG.ChildChain.SyncSupervisor do
        ]},
       {SubmissionMonitor,
        [
-         check_interval_ms: block_stalled_submission_check_interval_ms,
-         stall_threshold_blocks: block_stalled_submission_threshold_blocks,
+         check_interval_ms: ethereum_events_check_interval_ms,
+         stall_threshold_blocks: block_submit_stall_threshold_blocks,
          event_bus_module: Bus,
          alarm_module: Alarm
        ]},
diff --git a/apps/omg_child_chain/test/omg_child_chain/release_tasks/set_block_submit_stall_threshold_test.exs b/apps/omg_child_chain/test/omg_child_chain/release_tasks/set_block_submit_stall_threshold_test.exs
new file mode 100644
index 0000000000..6163d4258a
--- /dev/null
+++ b/apps/omg_child_chain/test/omg_child_chain/release_tasks/set_block_submit_stall_threshold_test.exs
@@ -0,0 +1,38 @@
+# Copyright 2019-2020 OmiseGO Pte Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+defmodule OMG.Eth.ReleaseTasks.SetBlockSubmitStallThresholdTest do
+  use ExUnit.Case, async: true
+  alias OMG.Eth.ReleaseTasks.SetBlockSubmitStallThreshold
+
+  @app :omg_eth
+  @env_key "BLOCK_SUBMIT_STALL_THRESHOLD_BLOCKS"
+  @config_key :block_submit_stall_threshold_blocks
+
+  test "that interval is set when the env var is present" do
+    :ok = System.put_env(@env_key, "9999")
+    config = SetBlockSubmitStallThreshold.load([], [])
+    block_submit_stall_threshold_blocks = config |> Keyword.fetch!(@app) |> Keyword.fetch!(@config_key)
+    assert block_submit_stall_threshold_blocks == 9999
+    :ok = System.delete_env(@env_key)
+  end
+
+  test "that the default config is used when the env var is not set" do
+    old_config = Application.get_env(@app, @config_key)
+    :ok = System.delete_env(@env_key)
+    config = SetBlockSubmitStallThreshold.load([], [])
+    block_submit_stall_threshold_blocks = config |> Keyword.fetch!(@app) |> Keyword.fetch!(@config_key)
+    assert block_submit_stall_threshold_blocks == old_config
+  end
+end
diff --git a/config/config.exs b/config/config.exs
index 8daf67f21e..ad287526b7 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -46,6 +46,8 @@ config :omg_child_chain,
   block_queue_eth_height_check_interval_ms: 6_000,
   block_submit_every_nth: 1,
   block_submit_max_gas_price: 20_000_000_000,
+  block_submit_stall_check_interval_ms: 5_000,
+  block_submit_stall_threshold_blocks: 4,
   metrics_collection_interval: 60_000,
   fee_adapter_check_interval_ms: 10_000,
   fee_buffer_duration_ms: 30_000,
@@ -106,8 +108,6 @@ config :omg_eth,
   ethereum_block_time_seconds: 15,
   ethereum_events_check_interval_ms: ethereum_events_check_interval_ms,
   ethereum_stalled_sync_threshold_ms: 20_000,
-  block_stalled_submission_check_interval_ms: 5_000,
-  block_stalled_submission_threshold_blocks: 4,
   node_logging_in_debug: false
 
 config :omg_status,
diff --git a/docs/deployment_configuration.md b/docs/deployment_configuration.md
index 71b055b0e3..dd975776c6 100644
--- a/docs/deployment_configuration.md
+++ b/docs/deployment_configuration.md
@@ -22,7 +22,8 @@
 
 ***Child Chain only***
 
-- "BLOCK_SUBMIT_MAX_GAS_PRICE" - The maximum gas price to use for block submission. The first block submission after application boot will use the max price. The gas price gradually adjusts on subsequent blocks to reach the current optimum price . Defaults to `20000000000` (20 Gwei).
+- "BLOCK_SUBMIT_MAX_GAS_PRICE" - The maximum gas price to use for block submission. The first block submission after application boot will use the max price. The gas price gradually adjusts on subsequent blocks to reach the current optimum price. Defaults to `20000000000` (20 Gwei).
+- "BLOCK_SUBMIT_STALL_THRESHOLD_BLOCKS" - The number of root chain blocks passed until a child chain block pending submission is considered stalled. Defaults to `4` root chain blocks.
 - "FEE_ADAPTER" - The adapter to use to populate the fee specs. Either `file` or `feed` (case-insensitive). Defaults to `file` with an empty fee specs.
 - "FEE_CLAIMER_ADDRESS" - 20-bytes HEX-encoded string of Ethereum address of Fee Claimer.
 - "FEE_BUFFER_DURATION_MS" - Buffer period during which a fee is still valid after being updated.

From 047bd98fec636229ca07ab9ca8bf01b25afaf218 Mon Sep 17 00:00:00 2001
From: unnawut <unnawut@omg.network>
Date: Fri, 17 Jul 2020 17:51:53 +0700
Subject: [PATCH 24/39] fix: move config namespace

---
 .../lib/omg_child_chain/configuration.ex               |  5 +++++
 apps/omg_eth/lib/omg_eth/configuration.ex              | 10 ----------
 2 files changed, 5 insertions(+), 10 deletions(-)

diff --git a/apps/omg_child_chain/lib/omg_child_chain/configuration.ex b/apps/omg_child_chain/lib/omg_child_chain/configuration.ex
index 6f9a78d611..64c8109dd0 100644
--- a/apps/omg_child_chain/lib/omg_child_chain/configuration.ex
+++ b/apps/omg_child_chain/lib/omg_child_chain/configuration.ex
@@ -43,6 +43,11 @@ defmodule OMG.ChildChain.Configuration do
     Application.fetch_env!(@app, :block_submit_max_gas_price)
   end
 
+  @spec block_stalled_submission_threshold_blocks() :: pos_integer() | no_return()
+  def block_stalled_submission_threshold_blocks() do
+    Application.fetch_env!(@app, :block_stalled_submission_threshold_blocks)
+  end
+
   @doc """
   Prepares options Keyword for the FeeServer process
   """
diff --git a/apps/omg_eth/lib/omg_eth/configuration.ex b/apps/omg_eth/lib/omg_eth/configuration.ex
index 43e9383abb..84c30d5a95 100644
--- a/apps/omg_eth/lib/omg_eth/configuration.ex
+++ b/apps/omg_eth/lib/omg_eth/configuration.ex
@@ -75,14 +75,4 @@ defmodule OMG.Eth.Configuration do
   def ethereum_stalled_sync_threshold_ms() do
     Application.fetch_env!(@app, :ethereum_stalled_sync_threshold_ms)
   end
-
-  @spec block_stalled_submission_check_interval_ms() :: pos_integer | no_return
-  def block_stalled_submission_check_interval_ms() do
-    Application.fetch_env!(@app, :block_stalled_submission_check_interval_ms)
-  end
-
-  @spec block_stalled_submission_threshold_blocks() :: pos_integer | no_return
-  def block_stalled_submission_threshold_blocks() do
-    Application.fetch_env!(@app, :block_stalled_submission_threshold_blocks)
-  end
 end

From d8d3bf330b9b239e87221c9d1a9455eda558a6c0 Mon Sep 17 00:00:00 2001
From: unnawut <unnawut@omg.network>
Date: Fri, 17 Jul 2020 17:52:06 +0700
Subject: [PATCH 25/39] refactor: brackets everywhere

---
 .../lib/omg_child_chain/configuration.ex           | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/apps/omg_child_chain/lib/omg_child_chain/configuration.ex b/apps/omg_child_chain/lib/omg_child_chain/configuration.ex
index 64c8109dd0..23c43ba5e4 100644
--- a/apps/omg_child_chain/lib/omg_child_chain/configuration.ex
+++ b/apps/omg_child_chain/lib/omg_child_chain/configuration.ex
@@ -18,27 +18,27 @@ defmodule OMG.ChildChain.Configuration do
   """
   @app :omg_child_chain
 
-  @spec metrics_collection_interval() :: no_return | pos_integer()
+  @spec metrics_collection_interval() :: no_return() | pos_integer()
   def metrics_collection_interval() do
     Application.fetch_env!(@app, :metrics_collection_interval)
   end
 
-  @spec block_queue_eth_height_check_interval_ms() :: no_return | pos_integer()
+  @spec block_queue_eth_height_check_interval_ms() :: no_return() | pos_integer()
   def block_queue_eth_height_check_interval_ms() do
     Application.fetch_env!(@app, :block_queue_eth_height_check_interval_ms)
   end
 
-  @spec submission_finality_margin() :: no_return | pos_integer()
+  @spec submission_finality_margin() :: no_return() | pos_integer()
   def submission_finality_margin() do
     Application.fetch_env!(@app, :submission_finality_margin)
   end
 
-  @spec block_submit_every_nth() :: no_return | pos_integer()
+  @spec block_submit_every_nth() :: no_return() | pos_integer()
   def block_submit_every_nth() do
     Application.fetch_env!(@app, :block_submit_every_nth)
   end
 
-  @spec block_submit_max_gas_price() :: no_return | pos_integer()
+  @spec block_submit_max_gas_price() :: no_return() | pos_integer()
   def block_submit_max_gas_price() do
     Application.fetch_env!(@app, :block_submit_max_gas_price)
   end
@@ -51,7 +51,7 @@ defmodule OMG.ChildChain.Configuration do
   @doc """
   Prepares options Keyword for the FeeServer process
   """
-  @spec fee_server_opts() :: no_return | Keyword.t()
+  @spec fee_server_opts() :: no_return() | Keyword.t()
   def fee_server_opts() do
     fee_server_opts = [
       fee_adapter_check_interval_ms: Application.fetch_env!(@app, :fee_adapter_check_interval_ms),
@@ -63,7 +63,7 @@ defmodule OMG.ChildChain.Configuration do
     Keyword.merge(fee_server_opts, fee_adapter: adapter, fee_adapter_opts: adapter_opts)
   end
 
-  @spec fee_adapter_opts() :: no_return | tuple()
+  @spec fee_adapter_opts() :: no_return() | tuple()
   defp fee_adapter_opts() do
     Application.fetch_env!(@app, :fee_adapter)
   end

From 69fe633e80ca9bd02b011a7415c2cf766723f50b Mon Sep 17 00:00:00 2001
From: unnawut <unnawut@omg.network>
Date: Fri, 17 Jul 2020 17:53:23 +0700
Subject: [PATCH 26/39] fix: config naming

---
 apps/omg_child_chain/lib/omg_child_chain/configuration.ex | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/apps/omg_child_chain/lib/omg_child_chain/configuration.ex b/apps/omg_child_chain/lib/omg_child_chain/configuration.ex
index 23c43ba5e4..a47b004b6b 100644
--- a/apps/omg_child_chain/lib/omg_child_chain/configuration.ex
+++ b/apps/omg_child_chain/lib/omg_child_chain/configuration.ex
@@ -43,9 +43,9 @@ defmodule OMG.ChildChain.Configuration do
     Application.fetch_env!(@app, :block_submit_max_gas_price)
   end
 
-  @spec block_stalled_submission_threshold_blocks() :: pos_integer() | no_return()
-  def block_stalled_submission_threshold_blocks() do
-    Application.fetch_env!(@app, :block_stalled_submission_threshold_blocks)
+  @spec block_submit_stall_threshold_blocks() :: pos_integer() | no_return()
+  def block_submit_stall_threshold_blocks() do
+    Application.fetch_env!(@app, :block_submit_stall_threshold_blocks)
   end
 
   @doc """

From 2a20535f9f20d2cecd541bc88cd6738a02606e13 Mon Sep 17 00:00:00 2001
From: unnawut <unnawut@omg.network>
Date: Fri, 17 Jul 2020 18:00:03 +0700
Subject: [PATCH 27/39] fix: wrong namespace

---
 .../release_tasks/set_block_submit_stall_threshold_test.exs   | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/apps/omg_child_chain/test/omg_child_chain/release_tasks/set_block_submit_stall_threshold_test.exs b/apps/omg_child_chain/test/omg_child_chain/release_tasks/set_block_submit_stall_threshold_test.exs
index 6163d4258a..274604e3a5 100644
--- a/apps/omg_child_chain/test/omg_child_chain/release_tasks/set_block_submit_stall_threshold_test.exs
+++ b/apps/omg_child_chain/test/omg_child_chain/release_tasks/set_block_submit_stall_threshold_test.exs
@@ -12,9 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-defmodule OMG.Eth.ReleaseTasks.SetBlockSubmitStallThresholdTest do
+defmodule OMG.ChildChain.ReleaseTasks.SetBlockSubmitStallThresholdTest do
   use ExUnit.Case, async: true
-  alias OMG.Eth.ReleaseTasks.SetBlockSubmitStallThreshold
+  alias OMG.ChildChain.ReleaseTasks.SetBlockSubmitStallThreshold
 
   @app :omg_eth
   @env_key "BLOCK_SUBMIT_STALL_THRESHOLD_BLOCKS"

From 79b2b22e3a73a53a973696a15a741f98e15f506b Mon Sep 17 00:00:00 2001
From: unnawut <unnawut@omg.network>
Date: Fri, 17 Jul 2020 18:06:15 +0700
Subject: [PATCH 28/39] test: fix app name

---
 .../release_tasks/set_block_submit_stall_threshold_test.exs     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/apps/omg_child_chain/test/omg_child_chain/release_tasks/set_block_submit_stall_threshold_test.exs b/apps/omg_child_chain/test/omg_child_chain/release_tasks/set_block_submit_stall_threshold_test.exs
index 274604e3a5..c02b634ed2 100644
--- a/apps/omg_child_chain/test/omg_child_chain/release_tasks/set_block_submit_stall_threshold_test.exs
+++ b/apps/omg_child_chain/test/omg_child_chain/release_tasks/set_block_submit_stall_threshold_test.exs
@@ -16,7 +16,7 @@ defmodule OMG.ChildChain.ReleaseTasks.SetBlockSubmitStallThresholdTest do
   use ExUnit.Case, async: true
   alias OMG.ChildChain.ReleaseTasks.SetBlockSubmitStallThreshold
 
-  @app :omg_eth
+  @app :omg_child_chain
   @env_key "BLOCK_SUBMIT_STALL_THRESHOLD_BLOCKS"
   @config_key :block_submit_stall_threshold_blocks
 

From e452b969513751d37c9ebdd6dca65e5c1d45e6f6 Mon Sep 17 00:00:00 2001
From: unnawut <unnawut@omg.network>
Date: Fri, 17 Jul 2020 18:23:15 +0700
Subject: [PATCH 29/39] test: fix init

---
 .../omg_child_chain/block_queue/submission_monitor_test.exs     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/apps/omg_child_chain/test/omg_child_chain/block_queue/submission_monitor_test.exs b/apps/omg_child_chain/test/omg_child_chain/block_queue/submission_monitor_test.exs
index 9f439378db..7b2ba284d1 100644
--- a/apps/omg_child_chain/test/omg_child_chain/block_queue/submission_monitor_test.exs
+++ b/apps/omg_child_chain/test/omg_child_chain/block_queue/submission_monitor_test.exs
@@ -35,7 +35,7 @@ defmodule OMG.ChildChain.BlockQueue.SubmissionMonitorTest do
     {:ok, monitor} =
       SubmissionMonitor.start_link(
         alarm_module: __MODULE__.Alarm,
-        event_bus: __MODULE__.BusMock,
+        event_bus_module: __MODULE__.BusMock,
         stall_threshold_blocks: stall_threshold_blocks,
         check_interval_ms: check_interval_ms
       )

From 7ce931d78557106aeb1b848f475590eb2a07089b Mon Sep 17 00:00:00 2001
From: unnawut <unnawut@omg.network>
Date: Fri, 17 Jul 2020 20:39:46 +0700
Subject: [PATCH 30/39] refactor: use :block_submit_stalled everywhere

---
 .../block_queue/monitor/alarm_handler.ex      | 14 ++++++-------
 .../block_queue/submission_monitor.ex         | 10 +++++-----
 .../block_queue/submission_monitor_test.exs   | 20 +++++++++----------
 apps/omg_status/lib/omg_status/alert/alarm.ex | 10 ++++++++--
 4 files changed, 30 insertions(+), 24 deletions(-)

diff --git a/apps/omg_child_chain/lib/omg_child_chain/block_queue/monitor/alarm_handler.ex b/apps/omg_child_chain/lib/omg_child_chain/block_queue/monitor/alarm_handler.ex
index 0feb51a823..b8c6aadddd 100644
--- a/apps/omg_child_chain/lib/omg_child_chain/block_queue/monitor/alarm_handler.ex
+++ b/apps/omg_child_chain/lib/omg_child_chain/block_queue/monitor/alarm_handler.ex
@@ -14,7 +14,7 @@
 
 defmodule OMG.ChildChain.BlockQueue.Monitor.AlarmHandler do
   @moduledoc """
-  Listens for :block_submission_stalled alarms and reflect the alarm's state back to the monitor.
+  Listens for :block_submit_stalled alarms and reflect the alarm's state back to the monitor.
   """
   require Logger
 
@@ -29,15 +29,15 @@ defmodule OMG.ChildChain.BlockQueue.Monitor.AlarmHandler do
 
   def handle_call(_request, state), do: {:ok, :ok, state}
 
-  def handle_event({:set_alarm, {:block_submission_stalled, %{reporter: @reporter}}}, state) do
-    _ = Logger.warn(":block_submission_stalled alarm raised.")
-    :ok = GenServer.cast(@monitor, {:set_alarm, :block_submission_stalled})
+  def handle_event({:set_alarm, {:block_submit_stalled, %{reporter: @reporter}}}, state) do
+    _ = Logger.warn(":block_submit_stalled alarm raised.")
+    :ok = GenServer.cast(@monitor, {:set_alarm, :block_submit_stalled})
     {:ok, state}
   end
 
-  def handle_event({:clear_alarm, {:block_submission_stalled, %{reporter: @reporter}}}, state) do
-    _ = Logger.warn(":block_submission_stalled alarm cleared.")
-    :ok = GenServer.cast(@monitor, {:clear_alarm, :block_submission_stalled})
+  def handle_event({:clear_alarm, {:block_submit_stalled, %{reporter: @reporter}}}, state) do
+    _ = Logger.warn(":block_submit_stalled alarm cleared.")
+    :ok = GenServer.cast(@monitor, {:clear_alarm, :block_submit_stalled})
     {:ok, state}
   end
 
diff --git a/apps/omg_child_chain/lib/omg_child_chain/block_queue/submission_monitor.ex b/apps/omg_child_chain/lib/omg_child_chain/block_queue/submission_monitor.ex
index ea2e47c3fb..35d1d99139 100644
--- a/apps/omg_child_chain/lib/omg_child_chain/block_queue/submission_monitor.ex
+++ b/apps/omg_child_chain/lib/omg_child_chain/block_queue/submission_monitor.ex
@@ -14,7 +14,7 @@
 
 defmodule OMG.ChildChain.BlockQueue.SubmissionMonitor do
   @moduledoc """
-  Listens to block events and raises :block_submission_stalled alarm when a pending block
+  Listens to block events and raises :block_submit_stalled alarm when a pending block
   doesn't get successfully submitted within the specified time threshold.
   """
   use GenServer
@@ -106,11 +106,11 @@ defmodule OMG.ChildChain.BlockQueue.SubmissionMonitor do
   # These functions are called by the AlarmHandler so that this monitor process can update
   # its internal state according to the raised alarms.
   #
-  def handle_cast({:set_alarm, :block_submission_stalled}, state) do
+  def handle_cast({:set_alarm, :block_submit_stalled}, state) do
     {:noreply, %{state | alarm_raised: true}}
   end
 
-  def handle_cast({:clear_alarm, :block_submission_stalled}, state) do
+  def handle_cast({:clear_alarm, :block_submit_stalled}, state) do
     {:noreply, %{state | alarm_raised: false}}
   end
 
@@ -147,11 +147,11 @@ defmodule OMG.ChildChain.BlockQueue.SubmissionMonitor do
   defp trigger_alarm(_alarm_module, false, []), do: :ok
 
   defp trigger_alarm(alarm_module, false, _stalled_blocks) do
-    alarm_module.set(alarm_module.block_submission_stalled(__MODULE__))
+    alarm_module.set(alarm_module.block_submit_stalled(__MODULE__))
   end
 
   defp trigger_alarm(alarm_module, true, []) do
-    alarm_module.clear(alarm_module.block_submission_stalled(__MODULE__))
+    alarm_module.clear(alarm_module.block_submit_stalled(__MODULE__))
   end
 
   defp trigger_alarm(_alarm_module, true, _stalled_blocks), do: :ok
diff --git a/apps/omg_child_chain/test/omg_child_chain/block_queue/submission_monitor_test.exs b/apps/omg_child_chain/test/omg_child_chain/block_queue/submission_monitor_test.exs
index 7b2ba284d1..52764f43ed 100644
--- a/apps/omg_child_chain/test/omg_child_chain/block_queue/submission_monitor_test.exs
+++ b/apps/omg_child_chain/test/omg_child_chain/block_queue/submission_monitor_test.exs
@@ -56,7 +56,7 @@ defmodule OMG.ChildChain.BlockQueue.SubmissionMonitorTest do
      }}
   end
 
-  test "does not raise :block_submission_stalled alarm when block is below stall threshold", context do
+  test "does not raise :block_submit_stalled alarm when block is below stall threshold", context do
     capture_log(fn ->
       # Inform the monitor of a pending block
       _ = send(context.monitor, {:internal_event_bus, :block_submitting, 1000})
@@ -69,7 +69,7 @@ defmodule OMG.ChildChain.BlockQueue.SubmissionMonitorTest do
     end)
   end
 
-  test "raises :block_submission_stalled alarm when blocks is at stall threshold", context do
+  test "raises :block_submit_stalled alarm when blocks is at stall threshold", context do
     capture_log(fn ->
       # Inform the monitor of a pending block
       _ = send(context.monitor, {:internal_event_bus, :block_submitting, 1000})
@@ -81,7 +81,7 @@ defmodule OMG.ChildChain.BlockQueue.SubmissionMonitorTest do
     end)
   end
 
-  test "raises :block_submission_stalled alarm when blocks is above stall threshold", context do
+  test "raises :block_submit_stalled alarm when blocks is above stall threshold", context do
     capture_log(fn ->
       # Inform the monitor of a pending block
       _ = send(context.monitor, {:internal_event_bus, :block_submitting, 1000})
@@ -93,7 +93,7 @@ defmodule OMG.ChildChain.BlockQueue.SubmissionMonitorTest do
     end)
   end
 
-  test "does not raise :block_submission_stalled alarm when it is already raised", context do
+  test "does not raise :block_submit_stalled alarm when it is already raised", context do
     # Set the monitor in a raised state
     :sys.replace_state(context.monitor, fn state -> %{state | alarm_raised: true} end)
 
@@ -109,7 +109,7 @@ defmodule OMG.ChildChain.BlockQueue.SubmissionMonitorTest do
     end)
   end
 
-  test "clears :block_submission_stalled alarm when the stalled block no longer stalls", context do
+  test "clears :block_submit_stalled alarm when the stalled block no longer stalls", context do
     # Set the monitor in a raised state
     :sys.replace_state(context.monitor, fn state -> %{state | alarm_raised: true} end)
 
@@ -128,7 +128,7 @@ defmodule OMG.ChildChain.BlockQueue.SubmissionMonitorTest do
     end)
   end
 
-  test "does not clear :block_submission_stalled alarm when some but not all stalled blocks got submitted", context do
+  test "does not clear :block_submit_stalled alarm when some but not all stalled blocks got submitted", context do
     # Set the monitor in a raised state
     :sys.replace_state(context.monitor, fn state -> %{state | alarm_raised: true} end)
 
@@ -163,15 +163,15 @@ defmodule OMG.ChildChain.BlockQueue.SubmissionMonitorTest do
       {:ok, %{listener: listener}}
     end
 
-    def block_submission_stalled(reporter) do
-      {:block_submission_stalled, %{node: Node.self(), reporter: reporter}}
+    def block_submit_stalled(reporter) do
+      {:block_submit_stalled, %{node: Node.self(), reporter: reporter}}
     end
 
-    def set({:block_submission_stalled, _details}) do
+    def set({:block_submit_stalled, _details}) do
       GenServer.call(__MODULE__, :got_raise_alarm)
     end
 
-    def clear({:block_submission_stalled, _details}) do
+    def clear({:block_submit_stalled, _details}) do
       GenServer.call(__MODULE__, :got_clear_alarm)
     end
 
diff --git a/apps/omg_status/lib/omg_status/alert/alarm.ex b/apps/omg_status/lib/omg_status/alert/alarm.ex
index 7ac9cddae3..dbf34a5db8 100644
--- a/apps/omg_status/lib/omg_status/alert/alarm.ex
+++ b/apps/omg_status/lib/omg_status/alert/alarm.ex
@@ -33,7 +33,8 @@ defmodule OMG.Status.Alert.Alarm do
            | :invalid_fee_source
            | :statsd_client_connection
            | :main_supervisor_halted
-           | :system_memory_too_high, alarm_detail}
+           | :system_memory_too_high
+           | :block_submit_stalled, alarm_detail}
 
   def alarm_types(),
     do: [
@@ -43,7 +44,8 @@ defmodule OMG.Status.Alert.Alarm do
       :invalid_fee_source,
       :statsd_client_connection,
       :main_supervisor_halted,
-      :system_memory_too_high
+      :system_memory_too_high,
+      :block_submit_stalled
     ]
 
   @spec statsd_client_connection(module()) :: {:statsd_client_connection, alarm_detail}
@@ -74,6 +76,10 @@ defmodule OMG.Status.Alert.Alarm do
   def system_memory_too_high(reporter),
     do: {:system_memory_too_high, %{node: Node.self(), reporter: reporter}}
 
+  @spec block_submit_stalled(module()) :: {:block_submit_stalled, alarm_detail}
+  def block_submit_stalled(reporter),
+    do: {:block_submit_stalled, %{node: Node.self(), reporter: reporter}}
+
   @spec set(alarms()) :: :ok | :duplicate
   def set(alarm), do: do_raise(alarm)
 

From 91f4d4b7d681e81243106a26efa237e792b7855a Mon Sep 17 00:00:00 2001
From: unnawut <unnawut@omg.network>
Date: Fri, 17 Jul 2020 21:23:18 +0700
Subject: [PATCH 31/39] fix: ignore unrelated events

---
 apps/omg_child_chain/lib/omg_child_chain/block_queue.ex      | 5 +++++
 .../lib/omg_child_chain/block_queue/submission_monitor.ex    | 5 +++++
 2 files changed, 10 insertions(+)

diff --git a/apps/omg_child_chain/lib/omg_child_chain/block_queue.ex b/apps/omg_child_chain/lib/omg_child_chain/block_queue.ex
index 7a033a2f69..df5158e711 100644
--- a/apps/omg_child_chain/lib/omg_child_chain/block_queue.ex
+++ b/apps/omg_child_chain/lib/omg_child_chain/block_queue.ex
@@ -194,6 +194,11 @@ defmodule OMG.ChildChain.BlockQueue do
     {:noreply, state1}
   end
 
+  @doc false
+  # Ignore unrelated events
+  def handle_info({:internal_event_bus, :block_submitting, _}, state), do: {:noreply, state}
+  def handle_info({:internal_event_bus, :block_submitted, _}, state), do: {:noreply, state}
+
   # private (server)
 
   @spec submit_blocks(Core.t()) :: :ok
diff --git a/apps/omg_child_chain/lib/omg_child_chain/block_queue/submission_monitor.ex b/apps/omg_child_chain/lib/omg_child_chain/block_queue/submission_monitor.ex
index 35d1d99139..f41b764901 100644
--- a/apps/omg_child_chain/lib/omg_child_chain/block_queue/submission_monitor.ex
+++ b/apps/omg_child_chain/lib/omg_child_chain/block_queue/submission_monitor.ex
@@ -100,6 +100,11 @@ defmodule OMG.ChildChain.BlockQueue.SubmissionMonitor do
     {:noreply, %{state | pending_blocks: pending_blocks}}
   end
 
+  # Ignore unrelated events
+  def handle_info({:internal_event_bus, :enqueue_block, _}, state) do
+    {:noreply, state}
+  end
+
   #
   # Handle incoming alarms
   #

From bd4b2c103bedebf06f8d75174f456a98d626a1b7 Mon Sep 17 00:00:00 2001
From: unnawut <unnawut@omg.network>
Date: Sat, 18 Jul 2020 00:08:23 +0700
Subject: [PATCH 32/39] fix: telemetry deprecation

---
 apps/omg_child_chain/lib/omg_child_chain/block_queue.ex      | 4 ++--
 .../lib/omg_child_chain/block_queue/measure.ex               | 4 ++--
 apps/omg_status/lib/omg_status/metric/event.ex               | 5 -----
 3 files changed, 4 insertions(+), 9 deletions(-)

diff --git a/apps/omg_child_chain/lib/omg_child_chain/block_queue.ex b/apps/omg_child_chain/lib/omg_child_chain/block_queue.ex
index df5158e711..a756db5cf3 100644
--- a/apps/omg_child_chain/lib/omg_child_chain/block_queue.ex
+++ b/apps/omg_child_chain/lib/omg_child_chain/block_queue.ex
@@ -244,7 +244,7 @@ defmodule OMG.ChildChain.BlockQueue do
   # The telemetry event is emitted for raw metrics propagation. The bus event is published so
   # a consumer, potentially a GenServer or other processes can perform extra operations on the data.
   defp publish_block_submitting_event(blknum) do
-    _ = :telemetry.execute([:blknum_submitting, __MODULE__], blknum)
+    _ = :telemetry.execute([:blknum_submitting, __MODULE__], %{blknum: blknum})
 
     {:child_chain, "blocks"}
     |> OMG.Bus.Event.new(:block_submitting, blknum)
@@ -257,7 +257,7 @@ defmodule OMG.ChildChain.BlockQueue do
   # The telemetry event is emitted for raw metrics propagation. The bus event is published so
   # a consumer, potentially a GenServer or other processes can perform extra operations on the data.
   defp publish_block_submitted_event(blknum) do
-    _ = :telemetry.execute([:blknum_submitted, __MODULE__], blknum)
+    _ = :telemetry.execute([:blknum_submitted, __MODULE__], %{blknum: blknum})
 
     {:child_chain, "blocks"}
     |> OMG.Bus.Event.new(:block_submitted, blknum)
diff --git a/apps/omg_child_chain/lib/omg_child_chain/block_queue/measure.ex b/apps/omg_child_chain/lib/omg_child_chain/block_queue/measure.ex
index 9140b57b84..6944a65d8e 100644
--- a/apps/omg_child_chain/lib/omg_child_chain/block_queue/measure.ex
+++ b/apps/omg_child_chain/lib/omg_child_chain/block_queue/measure.ex
@@ -62,12 +62,12 @@ defmodule OMG.ChildChain.BlockQueue.Measure do
     _ = Datadog.gauge(name(:authority_balance), gwei)
   end
 
-  def handle_event([:blknum_submitting, BlockQueue], blknum, _, _config) do
+  def handle_event([:blknum_submitting, BlockQueue], %{blknum: blknum}, _, _config) do
     _ = Datadog.gauge(name(:block_queue_blknum_submitting), blknum)
     _ = Datadog.increment(name(:block_submission_attempt), 1)
   end
 
-  def handle_event([:blknum_submitted, BlockQueue], blknum, _, _config) do
+  def handle_event([:blknum_submitted, BlockQueue], %{blknum: blknum}, _, _config) do
     _ = Datadog.gauge(name(:block_queue_blknum_submitted), blknum)
     _ = Datadog.increment(name(:block_submission_success), 1)
   end
diff --git a/apps/omg_status/lib/omg_status/metric/event.ex b/apps/omg_status/lib/omg_status/metric/event.ex
index 599d966965..84f632845c 100644
--- a/apps/omg_status/lib/omg_status/metric/event.ex
+++ b/apps/omg_status/lib/omg_status/metric/event.ex
@@ -84,11 +84,6 @@ defmodule OMG.Status.Metric.Event do
   """
   def name(:block_queue_blknum_submitted), do: "block_queue_blknum_submitted"
 
-  @doc """
-  Child Chain BlockQueue's number of blocks currently being submitted
-  """
-  def name(:block_queue_num_blocks_submitting), do: "block_queue_num_blocks_submitting"
-
   @doc """
   Child Chain BlockQueue's number of blocks currently being submitted and stalled
   """

From aaacf5cd2cc539b4a896daa6fe7916376ec3ec8c Mon Sep 17 00:00:00 2001
From: unnawut <unnawut@omg.network>
Date: Sat, 18 Jul 2020 00:09:57 +0700
Subject: [PATCH 33/39] fix: remove misleading metric
 :block_queue_num_blocks_submitting due to the way block submission sampling
 works

---
 .../lib/omg_child_chain/block_queue/measure.ex                | 4 ----
 .../lib/omg_child_chain/block_queue/submission_monitor.ex     | 1 -
 2 files changed, 5 deletions(-)

diff --git a/apps/omg_child_chain/lib/omg_child_chain/block_queue/measure.ex b/apps/omg_child_chain/lib/omg_child_chain/block_queue/measure.ex
index 6944a65d8e..e61f33688b 100644
--- a/apps/omg_child_chain/lib/omg_child_chain/block_queue/measure.ex
+++ b/apps/omg_child_chain/lib/omg_child_chain/block_queue/measure.ex
@@ -72,10 +72,6 @@ defmodule OMG.ChildChain.BlockQueue.Measure do
     _ = Datadog.increment(name(:block_submission_success), 1)
   end
 
-  def handle_event([:blocks_submitting, SubmissionMonitor], %{blocks: blocks}, _, _config) do
-    _ = Datadog.gauge(name(:block_queue_num_blocks_submitting), length(blocks))
-  end
-
   def handle_event([:blocks_stalled, SubmissionMonitor], %{blocks: blocks}, _, _config) do
     _ = Datadog.gauge(name(:block_queue_num_blocks_stalled), length(blocks))
   end
diff --git a/apps/omg_child_chain/lib/omg_child_chain/block_queue/submission_monitor.ex b/apps/omg_child_chain/lib/omg_child_chain/block_queue/submission_monitor.ex
index f41b764901..c3910b131c 100644
--- a/apps/omg_child_chain/lib/omg_child_chain/block_queue/submission_monitor.ex
+++ b/apps/omg_child_chain/lib/omg_child_chain/block_queue/submission_monitor.ex
@@ -75,7 +75,6 @@ defmodule OMG.ChildChain.BlockQueue.SubmissionMonitor do
         state.root_chain_height - first_submit_height >= state.stall_threshold_blocks
       end)
 
-    _ = :telemetry.execute([:blocks_submitting, __MODULE__], %{blocks: state.pending_blocks})
     _ = :telemetry.execute([:blocks_stalled, __MODULE__], %{blocks: stalled_blocks})
     _ = log_stalled_blocks(stalled_blocks, state.root_chain_height)
     _ = trigger_alarm(state.alarm_module, state.alarm_raised, stalled_blocks)

From 5faa0a20b8c5414250264b856638ac1378f92faf Mon Sep 17 00:00:00 2001
From: unnawut <unnawut@omg.network>
Date: Sat, 18 Jul 2020 00:10:39 +0700
Subject: [PATCH 34/39] fix: remove dup geth argument

---
 docker-compose.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/docker-compose.yml b/docker-compose.yml
index 03cb7e389d..5b069b0502 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -41,7 +41,6 @@ services:
             --datadir data/ \
             --syncmode 'full' \
             --networkid 1337 \
-            --gasprice '1' \
             --keystore=./data/geth/keystore/ \
             --password /tmp/geth-blank-password \
             --unlock "0,1" \

From 365d0297b683a9c1f1f9f2bb8182bed5da9db78e Mon Sep 17 00:00:00 2001
From: unnawut <unnawut@omg.network>
Date: Sat, 18 Jul 2020 02:07:56 +0700
Subject: [PATCH 35/39] fix: remove unused block_submit_stall_check_interval_ms

---
 config/config.exs | 1 -
 1 file changed, 1 deletion(-)

diff --git a/config/config.exs b/config/config.exs
index ad287526b7..68f912291a 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -46,7 +46,6 @@ config :omg_child_chain,
   block_queue_eth_height_check_interval_ms: 6_000,
   block_submit_every_nth: 1,
   block_submit_max_gas_price: 20_000_000_000,
-  block_submit_stall_check_interval_ms: 5_000,
   block_submit_stall_threshold_blocks: 4,
   metrics_collection_interval: 60_000,
   fee_adapter_check_interval_ms: 10_000,

From d6543e558077400d15eff550cd9de70fa9e1c0ad Mon Sep 17 00:00:00 2001
From: unnawut <unnawut@omg.network>
Date: Mon, 17 Aug 2020 15:11:49 +0700
Subject: [PATCH 36/39] refactor: add alarm function naming

---
 .../lib/omg_child_chain/block_queue/submission_monitor.ex       | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/apps/omg_child_chain/lib/omg_child_chain/block_queue/submission_monitor.ex b/apps/omg_child_chain/lib/omg_child_chain/block_queue/submission_monitor.ex
index c3910b131c..7470a359a1 100644
--- a/apps/omg_child_chain/lib/omg_child_chain/block_queue/submission_monitor.ex
+++ b/apps/omg_child_chain/lib/omg_child_chain/block_queue/submission_monitor.ex
@@ -147,7 +147,7 @@ defmodule OMG.ChildChain.BlockQueue.SubmissionMonitor do
     end
   end
 
-  @spec trigger_alarm(module(), boolean(), [blknum()]) :: :ok
+  @spec trigger_alarm(alarm :: module(), alarm_raised :: boolean(), stalled_blocks :: [blknum()]) :: :ok
   defp trigger_alarm(_alarm_module, false, []), do: :ok
 
   defp trigger_alarm(alarm_module, false, _stalled_blocks) do

From d676020cda4efa555e2e376f811cc063e5a78020 Mon Sep 17 00:00:00 2001
From: unnawut <unnawut@omg.network>
Date: Mon, 17 Aug 2020 15:14:24 +0700
Subject: [PATCH 37/39] refactor: reduce alarm function pattern matching cases

---
 .../lib/omg_child_chain/block_queue/submission_monitor.ex   | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/apps/omg_child_chain/lib/omg_child_chain/block_queue/submission_monitor.ex b/apps/omg_child_chain/lib/omg_child_chain/block_queue/submission_monitor.ex
index 7470a359a1..dcca892066 100644
--- a/apps/omg_child_chain/lib/omg_child_chain/block_queue/submission_monitor.ex
+++ b/apps/omg_child_chain/lib/omg_child_chain/block_queue/submission_monitor.ex
@@ -148,9 +148,7 @@ defmodule OMG.ChildChain.BlockQueue.SubmissionMonitor do
   end
 
   @spec trigger_alarm(alarm :: module(), alarm_raised :: boolean(), stalled_blocks :: [blknum()]) :: :ok
-  defp trigger_alarm(_alarm_module, false, []), do: :ok
-
-  defp trigger_alarm(alarm_module, false, _stalled_blocks) do
+  defp trigger_alarm(alarm_module, false, stalled_blocks) when length(stalled_blocks) >= 1 do
     alarm_module.set(alarm_module.block_submit_stalled(__MODULE__))
   end
 
@@ -158,7 +156,7 @@ defmodule OMG.ChildChain.BlockQueue.SubmissionMonitor do
     alarm_module.clear(alarm_module.block_submit_stalled(__MODULE__))
   end
 
-  defp trigger_alarm(_alarm_module, true, _stalled_blocks), do: :ok
+  defp trigger_alarm(_alarm_module, _alarm_raised, _stalled_blocks), do: :ok
 
   #
   # Logging

From 02a541fec265c5a0e132ab81ee4469894d6243a8 Mon Sep 17 00:00:00 2001
From: unnawut <unnawut@omg.network>
Date: Tue, 25 Aug 2020 18:43:16 +0700
Subject: [PATCH 38/39] refactor: move release task to releases.exs

---
 .../set_block_submit_stall_threshold.ex       | 52 -------------------
 config/releases.exs                           |  3 ++
 2 files changed, 3 insertions(+), 52 deletions(-)
 delete mode 100644 apps/omg_child_chain/lib/omg_child_chain/release_tasks/set_block_submit_stall_threshold.ex

diff --git a/apps/omg_child_chain/lib/omg_child_chain/release_tasks/set_block_submit_stall_threshold.ex b/apps/omg_child_chain/lib/omg_child_chain/release_tasks/set_block_submit_stall_threshold.ex
deleted file mode 100644
index f3a26df52d..0000000000
--- a/apps/omg_child_chain/lib/omg_child_chain/release_tasks/set_block_submit_stall_threshold.ex
+++ /dev/null
@@ -1,52 +0,0 @@
-# Copyright 2019-2019 OmiseGO Pte Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-defmodule OMG.ChildChain.ReleaseTasks.SetBlockSubmitStallThreshold do
-  @moduledoc false
-  @behaviour Config.Provider
-  require Logger
-
-  @app :omg_child_chain
-  @config_key :block_submit_stall_threshold_blocks
-  @env_var_name "BLOCK_SUBMIT_STALL_THRESHOLD_BLOCKS"
-
-  def init(args) do
-    args
-  end
-
-  def load(config, _args) do
-    _ = on_load()
-    stall_threshold_blocks = stall_threshold_blocks()
-    Config.Reader.merge(config, omg_child_chain: [block_submit_stall_threshold_blocks: stall_threshold_blocks])
-  end
-
-  defp stall_threshold_blocks() do
-    stall_threshold_blocks =
-      @env_var_name
-      |> System.get_env()
-      |> validate_integer(Application.get_env(@app, @config_key))
-
-    _ = Logger.info("CONFIGURATION: App: #{@app} Key: #{@config_key} Value: #{inspect(stall_threshold_blocks)}.")
-
-    stall_threshold_blocks
-  end
-
-  defp validate_integer(value, _default) when is_binary(value), do: String.to_integer(value)
-  defp validate_integer(_, default), do: default
-
-  defp on_load() do
-    _ = Application.ensure_all_started(:logger)
-    _ = Application.load(@app)
-  end
-end
diff --git a/config/releases.exs b/config/releases.exs
index 412baea707..d309b4fb88 100644
--- a/config/releases.exs
+++ b/config/releases.exs
@@ -5,6 +5,9 @@ import Config
 #
 # See https://hexdocs.pm/mix/1.9.0/Mix.Tasks.Release.html#module-runtime-configuration
 
+config :omg_child_chain,
+  block_submit_stall_threshold_blocks: String.to_integer(System.get_env("BLOCK_SUBMIT_STALL_THRESHOLD_BLOCKS") || "4")
+
 config :omg_watcher_info, OMG.WatcherInfo.DB.Repo,
   # Have at most `:pool_size` DB connections on standby and serving DB queries.
   pool_size: String.to_integer(System.get_env("WATCHER_INFO_DB_POOL_SIZE") || "10"),

From 1fb5f4df063eb34ee60eba364490906b046f2a1c Mon Sep 17 00:00:00 2001
From: unnawut <unnawut@omg.network>
Date: Tue, 25 Aug 2020 19:08:01 +0700
Subject: [PATCH 39/39] fix: remove obsolete tests

---
 .../set_block_submit_stall_threshold_test.exs | 38 -------------------
 1 file changed, 38 deletions(-)
 delete mode 100644 apps/omg_child_chain/test/omg_child_chain/release_tasks/set_block_submit_stall_threshold_test.exs

diff --git a/apps/omg_child_chain/test/omg_child_chain/release_tasks/set_block_submit_stall_threshold_test.exs b/apps/omg_child_chain/test/omg_child_chain/release_tasks/set_block_submit_stall_threshold_test.exs
deleted file mode 100644
index c02b634ed2..0000000000
--- a/apps/omg_child_chain/test/omg_child_chain/release_tasks/set_block_submit_stall_threshold_test.exs
+++ /dev/null
@@ -1,38 +0,0 @@
-# Copyright 2019-2020 OmiseGO Pte Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-defmodule OMG.ChildChain.ReleaseTasks.SetBlockSubmitStallThresholdTest do
-  use ExUnit.Case, async: true
-  alias OMG.ChildChain.ReleaseTasks.SetBlockSubmitStallThreshold
-
-  @app :omg_child_chain
-  @env_key "BLOCK_SUBMIT_STALL_THRESHOLD_BLOCKS"
-  @config_key :block_submit_stall_threshold_blocks
-
-  test "that interval is set when the env var is present" do
-    :ok = System.put_env(@env_key, "9999")
-    config = SetBlockSubmitStallThreshold.load([], [])
-    block_submit_stall_threshold_blocks = config |> Keyword.fetch!(@app) |> Keyword.fetch!(@config_key)
-    assert block_submit_stall_threshold_blocks == 9999
-    :ok = System.delete_env(@env_key)
-  end
-
-  test "that the default config is used when the env var is not set" do
-    old_config = Application.get_env(@app, @config_key)
-    :ok = System.delete_env(@env_key)
-    config = SetBlockSubmitStallThreshold.load([], [])
-    block_submit_stall_threshold_blocks = config |> Keyword.fetch!(@app) |> Keyword.fetch!(@config_key)
-    assert block_submit_stall_threshold_blocks == old_config
-  end
-end