Wait until transactions are confirmed

This commit adds an additional waiting time before pending transactions
are checked for their status.

src/monitoring_service/service.py

@@ -170,14 +170,30 @@ def _process_new_blocks(self, last_block: BlockNumber) -> None:
             handle_event(event, self.context)
             self.context.db.remove_scheduled_event(scheduled_event)
 
-        # check pending transactions
-        # this is done here so we don't have to block waiting for receipts in the state machine
+        # Check pending transactions
+        # This is done here so we don't have to block waiting for receipts in the state machine
+        self._check_pending_transactions()
+
+    def _check_pending_transactions(self) -> None:
+        """ Checks if pending transaction have been mined and confirmed.
+
+        In theory it's not necessary to check all pending transactions, but only the one with the
+        smallest nonce, and continue from there when this one is mined and confirmed. However,
+        as it is not expected that this list becomes to big this isn't optimized currently.
+        """
+
         for tx_hash in self.context.db.get_waiting_transactions():
             receipt = self.web3.eth.getTransactionReceipt(tx_hash)
-
             if receipt is not None:
-                self.context.db.remove_waiting_transaction(tx_hash)
+                tx_block = receipt.get("blockNumber")
+                if tx_block is None:
+                    return
 
+                confirmation_block = tx_block + self.context.required_confirmations
+                if self.context.last_known_block < confirmation_block:
+                    return
+
+                self.context.db.remove_waiting_transaction(tx_hash)
                 if receipt["status"] == 1:
                     log.info(
                         "Transaction was mined successfully",

tests/monitoring/monitoring_service/test_end_to_end.py

@@ -1,10 +1,13 @@
+from unittest.mock import Mock, patch
+
 import gevent
 from eth_utils import decode_hex, encode_hex, to_checksum_address
 from request_collector.server import RequestCollector
 from web3 import Web3
 
 from monitoring_service.service import MonitoringService, handle_event
 from monitoring_service.states import HashedBalanceProof
+from raiden.tests.utils.factories import make_transaction_hash
 from raiden.utils.typing import Address, BlockNumber, ChainID, Nonce, TokenAmount
 from raiden_contracts.constants import (
     CONTRACT_MONITORING_SERVICE,
@@ -135,6 +138,27 @@ def test_first_allowed_monitoring(
     assert [e.event for e in query()] == [MonitoringServiceEvent.NEW_BALANCE_PROOF_RECEIVED]
 
 
+def test_check_pending_transactions(web3, wait_for_blocks, monitoring_service: MonitoringService):
+    required_confirmations = 3
+
+    monitoring_service.context.required_confirmations = required_confirmations
+    monitoring_service.database.add_waiting_transaction(waiting_tx_hash=make_transaction_hash())
+
+    for tx_status in (0, 1):
+        tx_receipt = {"blockNumber": web3.eth.blockNumber, "status": tx_status}
+        with patch.object(web3.eth, "getTransactionReceipt", Mock(return_value=tx_receipt)):
+            with patch.object(
+                monitoring_service.database, "remove_waiting_transaction"
+            ) as remove_mock:
+
+                for should_call in (False, False, False, True):
+                    monitoring_service.context.last_known_block = web3.eth.blockNumber
+                    monitoring_service._check_pending_transactions()  # pylint: disable=protected-access # noqa
+
+                    assert remove_mock.called == should_call
+                    wait_for_blocks(1)
+
+
 def test_e2e(  # pylint: disable=too-many-arguments,too-many-locals
     web3,
     monitoring_service_contract,