feat: retry upon RPC connection failure (#71)

## Description


jira: https://forbole.atlassian.net/browse/BDU-488

## Checklist
- [x] Targeted PR against correct branch.
- [ ] Linked to Github issue with discussion and accepted design OR link to spec that describes this work.
- [ ] Wrote unit tests.  
- [x] Re-reviewed `Files changed` in the Github PR explorer.

CHANGELOG.md

@@ -1,3 +1,7 @@
+## Unreleased
+### Changes
+- ([\#71](https://github.com/forbole/juno/pull/71)) Retry RPC client connection upon failure instead of panic
+
 ## v3.3.0
 ### Changes
 - ([\#67](https://github.com/forbole/juno/pull/67)) Added support for concurrent transaction handling

cmd/start/cmd.go

@@ -1,7 +1,6 @@
 package start
 
 import (
-	"fmt"
 	"os"
 	"os/signal"
 	"sync"
@@ -125,16 +124,13 @@ func enqueueMissingBlocks(exportQueue types.HeightQueue, ctx *parser.Context) {
 	cfg := config.Cfg.Parser
 
 	// Get the latest height
-	latestBlockHeight, err := ctx.Node.LatestHeight()
-	if err != nil {
-		panic(fmt.Errorf("failed to get last block from RPCConfig client: %s", err))
-	}
+	latestBlockHeight := mustGetLatestHeight(ctx)
 
 	if cfg.FastSync {
 		ctx.Logger.Info("fast sync is enabled, ignoring all previous blocks", "latest_block_height", latestBlockHeight)
 		for _, module := range ctx.Modules {
 			if mod, ok := module.(modules.FastSyncModule); ok {
-				err = mod.DownloadState(latestBlockHeight)
+				err := mod.DownloadState(latestBlockHeight)
 				if err != nil {
 					ctx.Logger.Error("error while performing fast sync",
 						"err", err,
@@ -155,17 +151,11 @@ func enqueueMissingBlocks(exportQueue types.HeightQueue, ctx *parser.Context) {
 
 // enqueueNewBlocks enqueues new block heights onto the provided queue.
 func enqueueNewBlocks(exportQueue types.HeightQueue, ctx *parser.Context) {
-	currHeight, err := ctx.Node.LatestHeight()
-	if err != nil {
-		panic(fmt.Errorf("failed to get last block from RPCConfig client: %s", err))
-	}
+	currHeight := mustGetLatestHeight(ctx)
 
 	// Enqueue upcoming heights
 	for {
-		latestBlockHeight, err := ctx.Node.LatestHeight()
-		if err != nil {
-			panic(fmt.Errorf("failed to get last block from RPCConfig client: %s", err))
-		}
+		latestBlockHeight := mustGetLatestHeight(ctx)
 
 		// Enqueue all heights from the current height up to the latest height
 		for ; currHeight <= latestBlockHeight; currHeight++ {
@@ -176,6 +166,27 @@ func enqueueNewBlocks(exportQueue types.HeightQueue, ctx *parser.Context) {
 	}
 }
 
+// mustGetLatestHeight tries getting the latest height from the RPC client.
+// If after 50 tries no latest height can be found, it returns 0.
+func mustGetLatestHeight(ctx *parser.Context) int64 {
+	avgBlockTime := config.Cfg.Parser.AvgBlockTime
+
+	for retryCount := 0; retryCount < 50; retryCount++ {
+		latestBlockHeight, err := ctx.Node.LatestHeight()
+		if err == nil {
+			return latestBlockHeight
+		}
+
+		ctx.Logger.Error("failed to get last block from RPCConfig client",
+			"err", err,
+			"retry interval", avgBlockTime,
+			"retry count", retryCount)
+		time.Sleep(avgBlockTime)
+	}
+
+	return 0
+}
+
 // trapSignal will listen for any OS signal and invoke Done on the main
 // WaitGroup allowing the main process to gracefully exit.
 func trapSignal(ctx *parser.Context) {