sync reprocess full batch on close, check GER, and fix minor bugs (#3374

)

* sync reprocess full batch on close
* reduce the of live of batch cache from 1hour to 5min
* fix reprocess, check virtualBatch before reseting state
* #3376 - check GlobalExitRoot before working on a batch
* update same data as updateWIPBatch do in the cache of batches

config/config_test.go

@@ -56,6 +56,10 @@ func Test_Defaults(t *testing.T) {
 			path:          "Synchronizer.L2Synchronization.AcceptEmptyClosedBatches",
 			expectedValue: false,
 		},
+		{
+			path:          "Synchronizer.L2Synchronization.ReprocessFullBatchOnClose",
+			expectedValue: true,
+		},
 		{
 			path:          "Sequencer.DeletePoolTxsL1BlockConfirmations",
 			expectedValue: uint64(100),

config/default.go

@@ -118,6 +118,7 @@ L1SynchronizationMode = "sequential"
 			ApplyAfterNumRollupReceived = 10
 	[Synchronizer.L2Synchronization]
 		AcceptEmptyClosedBatches = false
+		ReprocessFullBatchOnClose = true
 
 [Sequencer]
 DeletePoolTxsL1BlockConfirmations = 100

docs/config-file/node-config-doc.html

@@ -28,7 +28,7 @@
 </pre></div> </div><div id=Synchronizer_L1ParallelSynchronization_TimeOutMainLoop_ex2 class="jumbotron examples"><div class=highlight><pre><span></span><span class=s2>&quot;300ms&quot;</span>
 </pre></div> </div> <hr> <div class=breadcrumbs> <!-- None --><!-- None --><!-- None --><a href=#Synchronizer.L1ParallelSynchronization.RollupInfoRetriesSpacing onclick="anchorLink('Synchronizer.L1ParallelSynchronization.RollupInfoRetriesSpacing')">Synchronizer.L1ParallelSynchronization.RollupInfoRetriesSpacing=</a> </div> <span class="badge badge-success default-value">Default: "5s"</span><span class="badge badge-dark value-type">Type: string</span><br> <span class=description><p>RollupInfoRetriesSpacing is the minimum time between retries to request rollup info (it will sleep for fulfill this time) to avoid spamming L1</p> </span> <br> <div class="badge badge-secondary">Examples:</div> <br><div id=Synchronizer_L1ParallelSynchronization_RollupInfoRetriesSpacing_ex1 class="jumbotron examples"><div class=highlight><pre><span></span><span class=s2>&quot;1m&quot;</span>
 </pre></div> </div><div id=Synchronizer_L1ParallelSynchronization_RollupInfoRetriesSpacing_ex2 class="jumbotron examples"><div class=highlight><pre><span></span><span class=s2>&quot;300ms&quot;</span>
-</pre></div> </div> <hr> <div class=breadcrumbs> <!-- None --><!-- None --><!-- None --><a href=#Synchronizer.L1ParallelSynchronization.FallbackToSequentialModeOnSynchronized onclick="anchorLink('Synchronizer.L1ParallelSynchronization.FallbackToSequentialModeOnSynchronized')">Synchronizer.L1ParallelSynchronization.FallbackToSequentialModeOnSynchronized=</a> </div> <span class="badge badge-success default-value">Default: false</span><span class="badge badge-dark value-type">Type: boolean</span><br> <span class=description><p>FallbackToSequentialModeOnSynchronized if true switch to sequential mode if the system is synchronized</p> </span> <hr> </div> </div> </div> </div> <div class=accordion id=accordionSynchronizer_L2Synchronization> <div class=card> <div class=card-header id=headingSynchronizer_L2Synchronization> <h2 class=mb-0> <button class="btn btn-link property-name-button" type=button data-toggle=collapse data-target=#Synchronizer_L2Synchronization aria-expanded aria-controls=Synchronizer_L2Synchronization onclick="setAnchor('#Synchronizer_L2Synchronization')"><span class=property-name> <div class=breadcrumbs>[<a href=#Synchronizer onclick="anchorLink('Synchronizer')">Synchronizer</a> . <a href=#Synchronizer_L2Synchronization onclick="anchorLink('Synchronizer_L2Synchronization')">L2Synchronization</a>] </div></span></button> </h2> L2Synchronization Configuration for L2 synchronization </div> <div id=Synchronizer_L2Synchronization class="collapse property-definition-div" aria-labelledby=headingSynchronizer_L2Synchronization data-parent=#accordionSynchronizer_L2Synchronization> <div class="card-body pl-5"> <div class=breadcrumbs> <!-- None --><!-- None --><!-- None --><a href=#Synchronizer.L2Synchronization.AcceptEmptyClosedBatches onclick="anchorLink('Synchronizer.L2Synchronization.AcceptEmptyClosedBatches')">Synchronizer.L2Synchronization.AcceptEmptyClosedBatches=</a> </div> <span class="badge badge-success default-value">Default: false</span><span class="badge badge-dark value-type">Type: boolean</span><br> <span class=description><p>AcceptEmptyClosedBatches is a flag to enable or disable the acceptance of empty batches.<br> if true, the synchronizer will accept empty batches and process them.</p> </span> <hr> </div> </div> </div> </div> </div> </div> </div> </div> <div class=accordion id=accordionSequencer> <div class=card> <div class=card-header id=headingSequencer> <h2 class=mb-0> <button class="btn btn-link property-name-button" type=button data-toggle=collapse data-target=#Sequencer aria-expanded aria-controls=Sequencer onclick="setAnchor('#Sequencer')"><span class=property-name> <div class=breadcrumbs>[<a href=#Sequencer onclick="anchorLink('Sequencer')">Sequencer</a>] </div></span></button> </h2> Configuration of the sequencer service </div> <div id=Sequencer class="collapse property-definition-div" aria-labelledby=headingSequencer data-parent=#accordionSequencer> <div class="card-body pl-5"> <div class=breadcrumbs> <!-- None --><!-- None --><a href=#Sequencer.DeletePoolTxsL1BlockConfirmations onclick="anchorLink('Sequencer.DeletePoolTxsL1BlockConfirmations')">Sequencer.DeletePoolTxsL1BlockConfirmations=</a> </div> <span class="badge badge-success default-value">Default: 100</span><span class="badge badge-dark value-type">Type: integer</span><br> <span class=description><p>DeletePoolTxsL1BlockConfirmations is blocks amount after which txs will be deleted from the pool</p> </span> <hr> <div class=breadcrumbs> <!-- None --><!-- None --><a href=#Sequencer.DeletePoolTxsCheckInterval onclick="anchorLink('Sequencer.DeletePoolTxsCheckInterval')">Sequencer.DeletePoolTxsCheckInterval=</a> </div> <span class="badge badge-success default-value">Default: "12h0m0s"</span><span class="badge badge-dark value-type">Type: string</span><br> <span class=description><p>DeletePoolTxsCheckInterval is frequency with which txs will be checked for deleting</p> </span> <br> <div class="badge badge-secondary">Examples:</div> <br><div id=Sequencer_DeletePoolTxsCheckInterval_ex1 class="jumbotron examples"><div class=highlight><pre><span></span><span class=s2>&quot;1m&quot;</span>
+</pre></div> </div> <hr> <div class=breadcrumbs> <!-- None --><!-- None --><!-- None --><a href=#Synchronizer.L1ParallelSynchronization.FallbackToSequentialModeOnSynchronized onclick="anchorLink('Synchronizer.L1ParallelSynchronization.FallbackToSequentialModeOnSynchronized')">Synchronizer.L1ParallelSynchronization.FallbackToSequentialModeOnSynchronized=</a> </div> <span class="badge badge-success default-value">Default: false</span><span class="badge badge-dark value-type">Type: boolean</span><br> <span class=description><p>FallbackToSequentialModeOnSynchronized if true switch to sequential mode if the system is synchronized</p> </span> <hr> </div> </div> </div> </div> <div class=accordion id=accordionSynchronizer_L2Synchronization> <div class=card> <div class=card-header id=headingSynchronizer_L2Synchronization> <h2 class=mb-0> <button class="btn btn-link property-name-button" type=button data-toggle=collapse data-target=#Synchronizer_L2Synchronization aria-expanded aria-controls=Synchronizer_L2Synchronization onclick="setAnchor('#Synchronizer_L2Synchronization')"><span class=property-name> <div class=breadcrumbs>[<a href=#Synchronizer onclick="anchorLink('Synchronizer')">Synchronizer</a> . <a href=#Synchronizer_L2Synchronization onclick="anchorLink('Synchronizer_L2Synchronization')">L2Synchronization</a>] </div></span></button> </h2> L2Synchronization Configuration for L2 synchronization </div> <div id=Synchronizer_L2Synchronization class="collapse property-definition-div" aria-labelledby=headingSynchronizer_L2Synchronization data-parent=#accordionSynchronizer_L2Synchronization> <div class="card-body pl-5"> <div class=breadcrumbs> <!-- None --><!-- None --><!-- None --><a href=#Synchronizer.L2Synchronization.AcceptEmptyClosedBatches onclick="anchorLink('Synchronizer.L2Synchronization.AcceptEmptyClosedBatches')">Synchronizer.L2Synchronization.AcceptEmptyClosedBatches=</a> </div> <span class="badge badge-success default-value">Default: false</span><span class="badge badge-dark value-type">Type: boolean</span><br> <span class=description><p>AcceptEmptyClosedBatches is a flag to enable or disable the acceptance of empty batches.<br> if true, the synchronizer will accept empty batches and process them.</p> </span> <hr> <div class=breadcrumbs> <!-- None --><!-- None --><!-- None --><a href=#Synchronizer.L2Synchronization.ReprocessFullBatchOnClose onclick="anchorLink('Synchronizer.L2Synchronization.ReprocessFullBatchOnClose')">Synchronizer.L2Synchronization.ReprocessFullBatchOnClose=</a> </div> <span class="badge badge-success default-value">Default: true</span><span class="badge badge-dark value-type">Type: boolean</span><br> <span class=description><p>ReprocessFullBatchOnClose if is true when a batch is closed is force to reprocess again</p> </span> <hr> </div> </div> </div> </div> </div> </div> </div> </div> <div class=accordion id=accordionSequencer> <div class=card> <div class=card-header id=headingSequencer> <h2 class=mb-0> <button class="btn btn-link property-name-button" type=button data-toggle=collapse data-target=#Sequencer aria-expanded aria-controls=Sequencer onclick="setAnchor('#Sequencer')"><span class=property-name> <div class=breadcrumbs>[<a href=#Sequencer onclick="anchorLink('Sequencer')">Sequencer</a>] </div></span></button> </h2> Configuration of the sequencer service </div> <div id=Sequencer class="collapse property-definition-div" aria-labelledby=headingSequencer data-parent=#accordionSequencer> <div class="card-body pl-5"> <div class=breadcrumbs> <!-- None --><!-- None --><a href=#Sequencer.DeletePoolTxsL1BlockConfirmations onclick="anchorLink('Sequencer.DeletePoolTxsL1BlockConfirmations')">Sequencer.DeletePoolTxsL1BlockConfirmations=</a> </div> <span class="badge badge-success default-value">Default: 100</span><span class="badge badge-dark value-type">Type: integer</span><br> <span class=description><p>DeletePoolTxsL1BlockConfirmations is blocks amount after which txs will be deleted from the pool</p> </span> <hr> <div class=breadcrumbs> <!-- None --><!-- None --><a href=#Sequencer.DeletePoolTxsCheckInterval onclick="anchorLink('Sequencer.DeletePoolTxsCheckInterval')">Sequencer.DeletePoolTxsCheckInterval=</a> </div> <span class="badge badge-success default-value">Default: "12h0m0s"</span><span class="badge badge-dark value-type">Type: string</span><br> <span class=description><p>DeletePoolTxsCheckInterval is frequency with which txs will be checked for deleting</p> </span> <br> <div class="badge badge-secondary">Examples:</div> <br><div id=Sequencer_DeletePoolTxsCheckInterval_ex1 class="jumbotron examples"><div class=highlight><pre><span></span><span class=s2>&quot;1m&quot;</span>
 </pre></div> </div><div id=Sequencer_DeletePoolTxsCheckInterval_ex2 class="jumbotron examples"><div class=highlight><pre><span></span><span class=s2>&quot;300ms&quot;</span>
 </pre></div> </div> <hr> <div class=breadcrumbs> <!-- None --><!-- None --><a href=#Sequencer.TxLifetimeCheckInterval onclick="anchorLink('Sequencer.TxLifetimeCheckInterval')">Sequencer.TxLifetimeCheckInterval=</a> </div> <span class="badge badge-success default-value">Default: "10m0s"</span><span class="badge badge-dark value-type">Type: string</span><br> <span class=description><p>TxLifetimeCheckInterval is the time the sequencer waits to check txs lifetime</p> </span> <br> <div class="badge badge-secondary">Examples:</div> <br><div id=Sequencer_TxLifetimeCheckInterval_ex1 class="jumbotron examples"><div class=highlight><pre><span></span><span class=s2>&quot;1m&quot;</span>
 </pre></div> </div><div id=Sequencer_TxLifetimeCheckInterval_ex2 class="jumbotron examples"><div class=highlight><pre><span></span><span class=s2>&quot;300ms&quot;</span>

docs/config-file/node-config-doc.md

@@ -1683,9 +1683,10 @@ FallbackToSequentialModeOnSynchronized=false
 **Type:** : `object`
 **Description:** L2Synchronization Configuration for L2 synchronization
 
-| Property                                                                                | Pattern | Type    | Deprecated | Definition | Title/Description                                                                                                                                                   |
-| --------------------------------------------------------------------------------------- | ------- | ------- | ---------- | ---------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| - [AcceptEmptyClosedBatches](#Synchronizer_L2Synchronization_AcceptEmptyClosedBatches ) | No      | boolean | No         | -          | AcceptEmptyClosedBatches is a flag to enable or disable the acceptance of empty batches.<br />if true, the synchronizer will accept empty batches and process them. |
+| Property                                                                                  | Pattern | Type    | Deprecated | Definition | Title/Description                                                                                                                                                   |
+| ----------------------------------------------------------------------------------------- | ------- | ------- | ---------- | ---------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| - [AcceptEmptyClosedBatches](#Synchronizer_L2Synchronization_AcceptEmptyClosedBatches )   | No      | boolean | No         | -          | AcceptEmptyClosedBatches is a flag to enable or disable the acceptance of empty batches.<br />if true, the synchronizer will accept empty batches and process them. |
+| - [ReprocessFullBatchOnClose](#Synchronizer_L2Synchronization_ReprocessFullBatchOnClose ) | No      | boolean | No         | -          | ReprocessFullBatchOnClose if is true when a batch is closed is force to reprocess again                                                                             |
 
 #### <a name="Synchronizer_L2Synchronization_AcceptEmptyClosedBatches"></a>9.6.1. `Synchronizer.L2Synchronization.AcceptEmptyClosedBatches`
 
@@ -1702,6 +1703,20 @@ if true, the synchronizer will accept empty batches and process them.
 AcceptEmptyClosedBatches=false
 ```
 
+#### <a name="Synchronizer_L2Synchronization_ReprocessFullBatchOnClose"></a>9.6.2. `Synchronizer.L2Synchronization.ReprocessFullBatchOnClose`
+
+**Type:** : `boolean`
+
+**Default:** `true`
+
+**Description:** ReprocessFullBatchOnClose if is true when a batch is closed is force to reprocess again
+
+**Example setting the default value** (true):
+```
+[Synchronizer.L2Synchronization]
+ReprocessFullBatchOnClose=true
+```
+
 ## <a name="Sequencer"></a>10. `[Sequencer]`
 
 **Type:** : `object`

docs/config-file/node-config-schema.json

@@ -631,6 +631,11 @@
 							"type": "boolean",
 							"description": "AcceptEmptyClosedBatches is a flag to enable or disable the acceptance of empty batches.\nif true, the synchronizer will accept empty batches and process them.",
 							"default": false
+						},
+						"ReprocessFullBatchOnClose": {
+							"type": "boolean",
+							"description": "ReprocessFullBatchOnClose if is true when a batch is closed is force to reprocess again",
+							"default": true
 						}
 					},
 					"additionalProperties": false,

state/batch.go

@@ -173,7 +173,7 @@ func (s *State) OpenBatch(ctx context.Context, processingContext ProcessingConte
 		return err
 	}
 	if prevTimestamp.Unix() > processingContext.Timestamp.Unix() {
-		return ErrTimestampGE
+		return fmt.Errorf(" oldBatch(%d) tstamp=%d > openingBatch(%d)=%d err: %w", lastBatchNum, prevTimestamp.Unix(), processingContext.BatchNumber, processingContext.Timestamp.Unix(), ErrTimestampGE)
 	}
 	return s.OpenBatchInStorage(ctx, processingContext, dbTx)
 }

state/pgstatestorage/l2block.go

@@ -197,7 +197,8 @@ func (p *PostgresStorage) AddL2Block(ctx context.Context, batchNumber uint64, l2
 		}
 		uncles = string(unclesBytes)
 	}
-
+	l2blockNumber := l2Block.Number().Uint64()
+	log.Debugf("[AddL2Block] adding L2 block %d", l2blockNumber)
 	if _, err := e.Exec(ctx, addL2BlockSQL,
 		l2Block.Number().Uint64(), l2Block.Hash().String(), header, uncles,
 		l2Block.ParentHash().String(), l2Block.Root().String(),