From e55bc233abba44459aa201e13bf1a70485fa8e53 Mon Sep 17 00:00:00 2001 From: Kirill Fedoseev Date: Sat, 31 Jul 2021 00:35:17 +0400 Subject: [PATCH 1/3] Add detection of unsynced node state --- oracle/src/confirmRelay.js | 2 +- oracle/src/sender.js | 2 +- oracle/src/services/HttpListProvider.js | 14 ++++++ oracle/src/services/SafeEthLogsProvider.js | 54 ++++++++++------------ oracle/src/services/web3.js | 4 +- oracle/src/utils/constants.js | 1 + oracle/src/watcher.js | 31 ++++++++++++- 7 files changed, 73 insertions(+), 35 deletions(-) diff --git a/oracle/src/confirmRelay.js b/oracle/src/confirmRelay.js index 9a71f62ed..647909512 100644 --- a/oracle/src/confirmRelay.js +++ b/oracle/src/confirmRelay.js @@ -49,7 +49,7 @@ async function initialize() { try { const checkHttps = checkHTTPS(ORACLE_ALLOW_HTTP_FOR_RPC, logger) - web3.currentProvider.subProvider.urls.forEach(checkHttps(chain)) + web3.currentProvider.urls.forEach(checkHttps(chain)) attached = await isAttached() if (attached) { diff --git a/oracle/src/sender.js b/oracle/src/sender.js index 9c431dc46..7484c9d21 100644 --- a/oracle/src/sender.js +++ b/oracle/src/sender.js @@ -40,7 +40,7 @@ async function initialize() { try { const checkHttps = checkHTTPS(process.env.ORACLE_ALLOW_HTTP_FOR_RPC, logger) - web3.currentProvider.subProvider.urls.forEach(checkHttps(config.id)) + web3.currentProvider.urls.forEach(checkHttps(config.id)) GasPrice.start(config.id) diff --git a/oracle/src/services/HttpListProvider.js b/oracle/src/services/HttpListProvider.js index 03695bba7..781e597ac 100644 --- a/oracle/src/services/HttpListProvider.js +++ b/oracle/src/services/HttpListProvider.js @@ -41,6 +41,20 @@ function HttpListProvider(urls, options = {}) { }) } +HttpListProvider.prototype.switchToFallbackRPC = function() { + if (this.urls.length < 2) { + return + } + + const prevIndex = this.currentIndex + const newIndex = (prevIndex + 1) % this.urls.length + this.logger.info( + { index: newIndex, oldURL: this.urls[prevIndex], newURL: this.urls[newIndex] }, + 'Switching to fallback JSON-RPC URL' + ) + this.currentIndex = newIndex +} + HttpListProvider.prototype.send = async function send(payload, callback) { // if fallback URL is being used for too long, switch back to the primary URL if (this.currentIndex > 0 && Date.now() - this.lastTimeUsedPrimary > FALLBACK_RPC_URL_SWITCH_TIMEOUT) { diff --git a/oracle/src/services/SafeEthLogsProvider.js b/oracle/src/services/SafeEthLogsProvider.js index e473fc32c..c55c6f3ad 100644 --- a/oracle/src/services/SafeEthLogsProvider.js +++ b/oracle/src/services/SafeEthLogsProvider.js @@ -1,39 +1,35 @@ const { hexToNumber, isHexStrict } = require('web3').utils -const { onInjected } = require('./injectedLogger') - function SafeEthLogsProvider(provider) { - this.subProvider = provider - onInjected(logger => { - this.logger = logger.child({ module: 'SafeEthLogsProvider' }) - }) -} - -SafeEthLogsProvider.prototype.send = function send(payload, callback) { - if (payload.method === 'eth_getLogs' && isHexStrict(payload.params[0].toBlock)) { - this.logger.debug('Modifying eth_getLogs request to include batch eth_blockNumber request') + const oldSend = provider.send.bind(provider) + const newSend = function(payload, callback) { + if (payload.method === 'eth_getLogs' && isHexStrict(payload.params[0].toBlock)) { + this.logger.debug('Modifying eth_getLogs request to include batch eth_blockNumber request') - const newPayload = [payload, { jsonrpc: '2.0', id: payload.id + 1, method: 'eth_blockNumber', params: [] }] - this.subProvider.send(newPayload, (err, res) => { - if (err) { - callback(err, null) - } else { - const rawLogs = res.find(({ id }) => id === payload.id) - const rawBlockNumber = res.find(({ id }) => id === payload.id + 1) - const blockNumber = hexToNumber(rawBlockNumber.result) - const toBlock = hexToNumber(payload.params[0].toBlock) - - if (blockNumber < toBlock) { - this.logger.warn({ toBlock, blockNumber }, 'Returned block number is less than the specified toBlock') - callback(new Error('block number too low'), null) + const newPayload = [payload, { jsonrpc: '2.0', id: payload.id + 1, method: 'eth_blockNumber', params: [] }] + oldSend(newPayload, (err, res) => { + if (err) { + callback(err, null) } else { - callback(null, rawLogs) + const rawLogs = res.find(({ id }) => id === payload.id) + const rawBlockNumber = res.find(({ id }) => id === payload.id + 1) + const blockNumber = hexToNumber(rawBlockNumber.result) + const toBlock = hexToNumber(payload.params[0].toBlock) + + if (blockNumber < toBlock) { + this.logger.warn({ toBlock, blockNumber }, 'Returned block number is less than the specified toBlock') + callback(new Error('block number too low'), null) + } else { + callback(null, rawLogs) + } } - } - }) - } else { - this.subProvider.send(payload, callback) + }) + } else { + oldSend(payload, callback) + } } + provider.send = newSend.bind(provider) + return provider } module.exports = { diff --git a/oracle/src/services/web3.js b/oracle/src/services/web3.js index 99088180a..0d0c49df0 100644 --- a/oracle/src/services/web3.js +++ b/oracle/src/services/web3.js @@ -38,10 +38,10 @@ const foreignOptions = { retry: RETRY_CONFIG } -const homeProvider = new SafeEthLogsProvider(new HttpListProvider(homeUrls, homeOptions)) +const homeProvider = SafeEthLogsProvider(new HttpListProvider(homeUrls, homeOptions)) const web3Home = new Web3(homeProvider) -const foreignProvider = new SafeEthLogsProvider(new HttpListProvider(foreignUrls, foreignOptions)) +const foreignProvider = SafeEthLogsProvider(new HttpListProvider(foreignUrls, foreignOptions)) const web3Foreign = new Web3(foreignProvider) let web3ForeignArchive = null diff --git a/oracle/src/utils/constants.js b/oracle/src/utils/constants.js index 63c8ef98c..10a6470b7 100644 --- a/oracle/src/utils/constants.js +++ b/oracle/src/utils/constants.js @@ -25,6 +25,7 @@ module.exports = { }, DEFAULT_TRANSACTION_RESEND_INTERVAL: 20 * 60 * 1000, FALLBACK_RPC_URL_SWITCH_TIMEOUT: 60 * 60 * 1000, + BLOCK_NUMBER_PROGRESS_ITERATIONS_LIMIT: 10, SENDER_QUEUE_MAX_PRIORITY: 10, SENDER_QUEUE_SEND_PRIORITY: 5, SENDER_QUEUE_CHECK_STATUS_PRIORITY: 1, diff --git a/oracle/src/watcher.js b/oracle/src/watcher.js index 2252190dc..ceb0fcdca 100644 --- a/oracle/src/watcher.js +++ b/oracle/src/watcher.js @@ -6,7 +6,7 @@ const logger = require('./services/logger') const { getShutdownFlag } = require('./services/shutdownState') const { getBlockNumber, getRequiredBlockConfirmations, getEvents } = require('./tx/web3') const { checkHTTPS, watchdog } = require('./utils/utils') -const { EXIT_CODES } = require('./utils/constants') +const { EXIT_CODES, BLOCK_NUMBER_PROGRESS_ITERATIONS_LIMIT } = require('./utils/constants') if (process.argv.length < 3) { logger.error('Please check the number of arguments, config file was not provided') @@ -29,12 +29,14 @@ const { getTokensState } = require('./utils/tokenState') const { web3, bridgeContract, eventContract, startBlock, pollingInterval, chain } = config.main const lastBlockRedisKey = `${config.id}:lastProcessedBlock` let lastProcessedBlock = Math.max(startBlock - 1, 0) +let lastSeenBlockNumber = 0 +let sameBlockNumberCounter = 0 async function initialize() { try { const checkHttps = checkHTTPS(process.env.ORACLE_ALLOW_HTTP_FOR_RPC, logger) - web3.currentProvider.subProvider.urls.forEach(checkHttps(chain)) + web3.currentProvider.urls.forEach(checkHttps(chain)) await getLastProcessedBlock() connectWatcherToQueue({ @@ -117,6 +119,31 @@ async function getLastBlockToProcess(web3, bridgeContract) { getBlockNumber(web3), getRequiredBlockConfirmations(bridgeContract) ]) + + if (lastBlockNumber < lastSeenBlockNumber) { + logger.warn( + { lastBlockNumber, lastSeenBlockNumber }, + 'Received block number less than already seen block. Switching to fallback RPC.' + ) + } else if (lastBlockNumber === lastSeenBlockNumber) { + sameBlockNumberCounter++ + if (sameBlockNumberCounter > 1) { + logger.info({ sameBlockNumberCounter }, 'Received the same block number for the more than twice') + if (sameBlockNumberCounter >= BLOCK_NUMBER_PROGRESS_ITERATIONS_LIMIT) { + sameBlockNumberCounter = 0 + logger.info( + { n: BLOCK_NUMBER_PROGRESS_ITERATIONS_LIMIT }, + 'Received the same block number for too many times. Probably node is not synced anymore' + ) + if (web3.currentProvider.switchToFallbackRPC) { + web3.currentProvider.switchToFallbackRPC() + } + } + } + } else { + sameBlockNumberCounter = 0 + lastSeenBlockNumber = lastBlockNumber + } return lastBlockNumber - requiredBlockConfirmations } From 7511cbbe864dd1759c75059da54c3bad41bb3fd5 Mon Sep 17 00:00:00 2001 From: Kirill Fedoseev Date: Sat, 31 Jul 2021 09:54:31 +0400 Subject: [PATCH 2/3] Fix docker login via curl --- .github/workflows/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 8b74601d1..9b71e9a56 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -77,7 +77,7 @@ jobs: - name: Rebuild and push updated images run: | function check_if_image_exists() { - curl -fsSlL -H 'Authorization: bearer ${{ github.token }}' "https://${DOCKER_REGISTRY}/v2/${DOCKER_REPO}/tokenbridge-e2e-$1/manifests/$2" > /dev/null + curl -fsSlL "https://${{ github.actor }}:${{ github.token }}@${DOCKER_REGISTRY}/v2/${DOCKER_REPO}/tokenbridge-e2e-$1/manifests/$2" > /dev/null } updated=() if ! check_if_image_exists e2e ${E2E_TAG}; then updated+=("e2e"); fi @@ -104,7 +104,7 @@ jobs: - name: Rebuild and push molecule runner e2e image run: | function check_if_image_exists() { - curl -fsSlL -H 'Authorization: bearer ${{ github.token }}' "https://${DOCKER_REGISTRY}/v2/${DOCKER_REPO}/tokenbridge-e2e-$1/manifests/$2" > /dev/null + curl -fsSlL "https://${{ github.actor }}:${{ github.token }}@${DOCKER_REGISTRY}/v2/${DOCKER_REPO}/tokenbridge-e2e-$1/manifests/$2" > /dev/null } if check_if_image_exists molecule_runner ${MOLECULE_RUNNER_TAG}; then echo "Image already exists" From 53f040965faa1d9a7df3933afd988ef5682807a3 Mon Sep 17 00:00:00 2001 From: Kirill Fedoseev Date: Sat, 31 Jul 2021 10:44:43 +0400 Subject: [PATCH 3/3] Extend logging --- oracle/src/watcher.js | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/oracle/src/watcher.js b/oracle/src/watcher.js index ceb0fcdca..9be726eed 100644 --- a/oracle/src/watcher.js +++ b/oracle/src/watcher.js @@ -121,23 +121,20 @@ async function getLastBlockToProcess(web3, bridgeContract) { ]) if (lastBlockNumber < lastSeenBlockNumber) { - logger.warn( - { lastBlockNumber, lastSeenBlockNumber }, - 'Received block number less than already seen block. Switching to fallback RPC.' - ) + sameBlockNumberCounter = 0 + logger.warn({ lastBlockNumber, lastSeenBlockNumber }, 'Received block number less than already seen block') + web3.currentProvider.switchToFallbackRPC() } else if (lastBlockNumber === lastSeenBlockNumber) { sameBlockNumberCounter++ if (sameBlockNumberCounter > 1) { - logger.info({ sameBlockNumberCounter }, 'Received the same block number for the more than twice') + logger.info({ lastBlockNumber, sameBlockNumberCounter }, 'Received the same block number more than twice') if (sameBlockNumberCounter >= BLOCK_NUMBER_PROGRESS_ITERATIONS_LIMIT) { sameBlockNumberCounter = 0 - logger.info( - { n: BLOCK_NUMBER_PROGRESS_ITERATIONS_LIMIT }, + logger.warn( + { lastBlockNumber, n: BLOCK_NUMBER_PROGRESS_ITERATIONS_LIMIT }, 'Received the same block number for too many times. Probably node is not synced anymore' ) - if (web3.currentProvider.switchToFallbackRPC) { - web3.currentProvider.switchToFallbackRPC() - } + web3.currentProvider.switchToFallbackRPC() } } } else {