e2e: avoid starting nodes from the future (#6835) (#6838)

(cherry picked from commit a4cc8317da47ba92d1a97460a953d0e687a46cfb)

Co-authored-by: Sam Kleinman <garen@tychoish.com>

test/e2e/runner/start.go

@@ -25,6 +25,7 @@ func Start(testnet *e2e.Testnet) error {
 		}
 		return false
 	})
+
 	sort.SliceStable(nodeQueue, func(i, j int) bool {
 		return nodeQueue[i].StartAt < nodeQueue[j].StartAt
 	})
@@ -49,9 +50,15 @@ func Start(testnet *e2e.Testnet) error {
 		logger.Info(fmt.Sprintf("Node %v up on http://127.0.0.1:%v", node.Name, node.ProxyPort))
 	}
 
+	networkHeight := testnet.InitialHeight
+
 	// Wait for initial height
-	logger.Info(fmt.Sprintf("Waiting for initial height %v...", testnet.InitialHeight))
-	block, blockID, err := waitForHeight(testnet, testnet.InitialHeight)
+	logger.Info("Waiting for initial height",
+		"height", networkHeight,
+		"nodes", len(testnet.Nodes)-len(nodeQueue),
+		"pending", len(nodeQueue))
+
+	block, blockID, err := waitForHeight(testnet, networkHeight)
 	if err != nil {
 		return err
 	}
@@ -66,8 +73,26 @@ func Start(testnet *e2e.Testnet) error {
 		}
 	}
 
-	// Start up remaining nodes
 	for _, node := range nodeQueue {
+		if node.StartAt > networkHeight {
+			// if we're starting a node that's ahead of
+			// the last known height of the network, then
+			// we should make sure that the rest of the
+			// network has reached at least the height
+			// that this node will start at before we
+			// start the node.
+
+			networkHeight = node.StartAt
+
+			logger.Info("Waiting for network to advance before starting catch up node",
+				"node", node.Name,
+				"height", networkHeight)
+
+			if _, _, err := waitForHeight(testnet, networkHeight); err != nil {
+				return err
+			}
+		}
+
 		logger.Info(fmt.Sprintf("Starting node %v at height %v...", node.Name, node.StartAt))
 		if _, _, err := waitForHeight(testnet, node.StartAt); err != nil {
 			return err