Sort subkey lookups be vertexid

Since data is ordered by VertexID on disk, with this simple trick we can
make much better use of the various rocksdb caches.

Computing the state root of the full mainnet state is down to 4 hours
(from 9) on my laptop.

nimbus/db/aristo/aristo_compute.nim

@@ -132,14 +132,15 @@ proc computeKeyImpl(
     db: AristoDbRef,
     rvid: RootedVertexID,
     batch: var WriteBatch,
-    vtxl: (VertexRef, int),
+    vtx: VertexRef,
+    level: int,
     skipLayers: static bool,
 ): Result[(HashKey, int), AristoError] =
   # The bloom filter available used only when creating the key cache from an
   # empty state
 
   # Top-most level of all the verticies this hash computation depends on
-  var (vtx, level) = vtxl
+  var level = level
 
   # TODO this is the same code as when serializing NodeRef, without the NodeRef
   var writer = initRlpWriter()
@@ -160,9 +161,12 @@ proc computeKeyImpl(
                     if keyvtxl[0][0].isValid:
                       (keyvtxl[0][0], keyvtxl[1])
                     else:
-                      let vtxl = (keyvtxl[0][1], keyvtxl[1])
                       ?db.computeKeyImpl(
-                        (stoID.vid, stoID.vid), batch, vtxl, skipLayers = skipLayers
+                        (stoID.vid, stoID.vid),
+                        batch,
+                        keyvtxl[0][1],
+                        keyvtxl[1],
+                        skipLayers = skipLayers,
                       )
                 level = maxLevel(level, sl)
                 skey
@@ -185,24 +189,52 @@ proc computeKeyImpl(
       for n, subvid in vtx.pairs:
         keyvtxs[n] = ?db.getKey((rvid.root, subvid), skipLayers)
 
-      template writeBranch(w: var RlpWriter): HashKey =
-        w.encodeBranch(vtx):
-          if subvid.isValid:
-            batch.enter(n)
-            let (bkey, bl) =
-              if keyvtxs[n][0][0].isValid:
-                (keyvtxs[n][0][0], keyvtxs[n][1])
-              else:
+      block allKeys:
+        while true:
+          var min = 17
+          var n = 0'u8
+          for i, a in keyvtxs.mpairs:
+            if not vtx.bVid(uint8 i).isValid:
+              n += 1
+              continue
+            if a[0][0].isValid:
+              n += 1
+              continue
+            if a[0][1].vType == Leaf:
+              batch.enter(n)
+              (a[0][0], a[1]) =
                 ?db.computeKeyImpl(
-                  (rvid.root, subvid),
+                  (rvid.root, vtx.bVid(uint8 i)),
                   batch,
-                  (keyvtxs[n][0][1], keyvtxs[n][1]),
+                  a[0][1],
+                  a[1],
                   skipLayers = skipLayers,
                 )
+              batch.leave(n)
+              n += 1
+            else:
+              if min == 17 or a[0][1].startVid < keyvtxs[min][0][1].startVid:
+                min = i
+
+          if min != 17:
+            batch.enter(n)
+            (keyvtxs[min][0][0], keyvtxs[min][1]) =
+              ?db.computeKeyImpl(
+                (rvid.root, vtx.bVid(uint8 min)),
+                batch,
+                keyvtxs[min][0][1],
+                keyvtxs[min][1],
+                skipLayers = skipLayers,
+              )
             batch.leave(n)
+          else:
+            break allKeys
 
-            level = maxLevel(level, bl)
-            bkey
+      template writeBranch(w: var RlpWriter): HashKey =
+        w.encodeBranch(vtx):
+          if subvid.isValid:
+            level = maxLevel(level, keyvtxs[n][1])
+            keyvtxs[n][0][0]
           else:
             VOID_HASH_KEY
 
@@ -237,7 +269,7 @@ proc computeKeyImpl(
     return ok(keyvtx[0])
 
   var batch: WriteBatch
-  let res = computeKeyImpl(db, rvid, batch, (keyvtx[1], level), skipLayers = skipLayers)
+  let res = computeKeyImpl(db, rvid, batch, keyvtx[1], level, skipLayers = skipLayers)
   if res.isOk:
     ?batch.flush(db)
 
@@ -263,6 +295,7 @@ proc computeKey*(
 proc computeKeys*(db: AristoDbRef, root: VertexID): Result[void, AristoError] =
   ## Ensure that key cache is topped up with the latest state root
   discard db.computeKeyImpl((root, root), skipLayers = true)
+
   ok()
 
 # ------------------------------------------------------------------------------

nimbus/db/aristo/aristo_layers.nim

@@ -69,11 +69,6 @@ func layersGetVtx*(db: AristoDbRef; rvid: RootedVertexID): Opt[(VertexRef, int)]
 
   Opt.none((VertexRef, int))
 
-func layersGetVtxOrVoid*(db: AristoDbRef; rvid: RootedVertexID): VertexRef =
-  ## Simplified version of `layersGetVtx()`
-  db.layersGetVtx(rvid).valueOr((VertexRef(nil), 0))[0]
-
-
 func layersGetKey*(db: AristoDbRef; rvid: RootedVertexID): Opt[(HashKey, int)] =
   ## Find a hash key on the cache layers. An `ok()` result might contain a void
   ## hash key if it is stored on the cache that way.