Merge pull request #505 from near/handle-timeouts

Implement timeout handling at sendJsonRpc level
near · Feb 24, 2021 · 43d344c · 43d344c
2 parents 6d4e688 + c08f4f6
commit 43d344c
Show file tree

Hide file tree

Showing 4 changed files with 105 additions and 98 deletions.
diff --git a/lib/account.js b/lib/account.js
diff --git a/lib/providers/json-rpc-provider.js b/lib/providers/json-rpc-provider.js
diff --git a/src/account.ts b/src/account.ts
@@ -39,14 +39,11 @@ const DEFAULT_FUNC_CALL_GAS = new BN('30000000000000');
 // Default number of retries with different nonce before giving up on a transaction.
 const TX_NONCE_RETRY_NUMBER = 12;
 
-// Default number of retries before giving up on a transaction.
-const TX_STATUS_RETRY_NUMBER = 12;
-
 // Default wait until next retry in millis.
-const TX_STATUS_RETRY_WAIT = 500;
+const TX_NONCE_RETRY_WAIT = 500;
 
 // Exponential back off for waiting to retry.
-const TX_STATUS_RETRY_WAIT_BACKOFF = 1.5;
+const TX_NONCE_RETRY_WAIT_BACKOFF = 1.5;
 
 export interface AccountState {
     amount: string;
@@ -151,32 +148,12 @@ export class Account {
 
         let txHash, signedTx;
         // TODO: TX_NONCE (different constants for different uses of exponentialBackoff?)
-        const result = await exponentialBackoff(TX_STATUS_RETRY_WAIT, TX_NONCE_RETRY_NUMBER, TX_STATUS_RETRY_WAIT_BACKOFF, async () => {
+        const result = await exponentialBackoff(TX_NONCE_RETRY_WAIT, TX_NONCE_RETRY_NUMBER, TX_NONCE_RETRY_WAIT_BACKOFF, async () => {
             [txHash, signedTx] = await this.signTransaction(receiverId, actions);
             const publicKey = signedTx.transaction.publicKey;
 
             try {
-                const result = await exponentialBackoff(TX_STATUS_RETRY_WAIT, TX_STATUS_RETRY_NUMBER, TX_STATUS_RETRY_WAIT_BACKOFF, async () => {
-                    try {
-                        return await this.connection.provider.sendTransaction(signedTx);
-                    } catch (error) {
-                        // TODO: Somehow getting still:
-                        // Error: send_tx_commit has timed out.
-                        if (error.type === 'TimeoutError') {
-                            console.warn(`Retrying transaction ${receiverId}:${baseEncode(txHash)} as it has timed out`);
-                            return null;
-                        }
-
-                        throw error;
-                    }
-                });
-                if (!result) {
-                    throw new TypedError(
-                        `Exceeded ${TX_STATUS_RETRY_NUMBER} attempts for transaction ${baseEncode(txHash)}.`,
-                        'RetriesExceeded',
-                        new ErrorContext(baseEncode(txHash)));
-                }
-                return result;
+                return await this.connection.provider.sendTransaction(signedTx);
             } catch (error) {
                 if (error.message.match(/Transaction nonce \d+ must be larger than nonce of the used access key \d+/)) {
                     console.warn(`Retrying transaction ${receiverId}:${baseEncode(txHash)} with new nonce.`);

diff --git a/src/providers/json-rpc-provider.ts b/src/providers/json-rpc-provider.ts
@@ -8,11 +8,21 @@ import { Network } from '../utils/network';
 import { ConnectionInfo, fetchJson } from '../utils/web';
 import { TypedError, ErrorContext } from '../utils/errors';
 import { baseEncode } from 'borsh';
+import exponentialBackoff from '../utils/exponential-backoff';
 import { parseRpcError } from '../utils/rpc_errors';
 import { SignedTransaction } from '../transaction';
 
 export { TypedError, ErrorContext };
 
+// Default number of retries before giving up on a request.
+const REQUEST_RETRY_NUMBER = 12;
+
+// Default wait until next retry in millis.
+const REQUEST_RETRY_WAIT = 500;
+
+// Exponential back off for waiting to retry.
+const REQUEST_RETRY_WAIT_BACKOFF = 1.5;
+
 /// Keep ids unique across all connections.
 let _nextId = 123;
 
@@ -152,32 +162,49 @@ export class JsonRpcProvider extends Provider {
      * @param params Parameters to the method
      */
     async sendJsonRpc(method: string, params: object): Promise<any> {
-        const request = {
-            method,
-            params,
-            id: (_nextId++),
-            jsonrpc: '2.0'
-        };
-        const response = await fetchJson(this.connection, JSON.stringify(request));
-        if (response.error) {
-            if (typeof response.error.data === 'object') {
-                if (typeof response.error.data.error_message === 'string' && typeof response.error.data.error_type === 'string') {
-                    // if error data has error_message and error_type properties, we consider that node returned an error in the old format
-                    throw new TypedError(response.error.data.error_message, response.error.data.error_type);
-                } else {
-                    throw parseRpcError(response.error.data);
+        const result = await exponentialBackoff(REQUEST_RETRY_WAIT, REQUEST_RETRY_NUMBER, REQUEST_RETRY_WAIT_BACKOFF, async () => {
+            try {
+                const request = {
+                    method,
+                    params,
+                    id: (_nextId++),
+                    jsonrpc: '2.0'
+                };
+                const response = await fetchJson(this.connection, JSON.stringify(request));
+                if (response.error) {
+                    if (typeof response.error.data === 'object') {
+                        if (typeof response.error.data.error_message === 'string' && typeof response.error.data.error_type === 'string') {
+                            // if error data has error_message and error_type properties, we consider that node returned an error in the old format
+                            throw new TypedError(response.error.data.error_message, response.error.data.error_type);
+                        }
+
+                        throw parseRpcError(response.error.data);
+                    } else {
+                        const errorMessage = `[${response.error.code}] ${response.error.message}: ${response.error.data}`;
+                        // NOTE: All this hackery is happening because structured errors not implemented
+                        // TODO: Fix when https://github.com/nearprotocol/nearcore/issues/1839 gets resolved
+                        if (response.error.data === 'Timeout' || errorMessage.includes('Timeout error')
+                                || errorMessage.includes('query has timed out')) {
+                            throw new TypedError(errorMessage, 'TimeoutError');
+                        }
+
+                        throw new TypedError(errorMessage);
+                    }
                 }
-            } else {
-                const errorMessage = `[${response.error.code}] ${response.error.message}: ${response.error.data}`;
-                // NOTE: All this hackery is happening because structured errors not implemented
-                // TODO: Fix when https://github.com/nearprotocol/nearcore/issues/1839 gets resolved
-                if (response.error.data === 'Timeout' || errorMessage.includes('Timeout error')) {
-                    throw new TypedError('send_tx_commit has timed out.', 'TimeoutError');
-                } else {
-                    throw new TypedError(errorMessage);
+                return response.result;
+            } catch (error) {
+                if (error.type === 'TimeoutError') {
+                    console.warn(`Retrying request to ${method} as it has timed out`, params);
+                    return null;
                 }
+
+                throw error;
             }
+        });
+        if (!result) {
+            throw new TypedError(
+                `Exceeded ${REQUEST_RETRY_NUMBER} attempts for request to ${method}.`, 'RetriesExceeded');
         }
-        return response.result;
+        return result;
     }
 }