Extract db-related methods from baseTrie

src/baseTrie.js

@@ -1,20 +1,23 @@
 const assert = require('assert')
-const level = require('level-mem')
 const async = require('async')
 const rlp = require('rlp')
 const ethUtil = require('ethereumjs-util')
 const semaphore = require('semaphore')
+const DB = require('./db')
 const TrieNode = require('./trieNode')
 const ReadStream = require('./readStream')
 const PrioritizedTaskExecutor = require('./prioritizedTaskExecutor')
-const { callTogether, asyncFirstSeries } = require('./util/async')
+const { callTogether } = require('./util/async')
 const { stringToNibbles, matchingNibbleLength, doKeysMatch } = require('./util/nibbles')
 
 /**
- * Use `require('merkel-patricia-tree')` for the base interface. In Ethereum applications stick with the Secure Trie Overlay `require('merkel-patricia-tree/secure')`. The API for the raw and the secure interface are about the same
+ * Use `require('merkel-patricia-tree')` for the base interface. In Ethereum applications
+ * stick with the Secure Trie Overlay `require('merkel-patricia-tree/secure')`.
+ * The API for the raw and the secure interface are about the same
  * @class Trie
  * @public
- * @param {Object} [db] An instance of [levelup](https://github.com/rvagg/node-levelup/) or a compatible API. If the db is `null` or left undefined, then the trie will be stored in memory via [memdown](https://github.com/rvagg/memdown)
+ * @param {Object} [db] An instance of [levelup](https://github.com/rvagg/node-levelup/), a compatible API or an instance of `DB`.
+ * If the db is `null` or left undefined, then the trie will be stored in memory via [memdown](https://github.com/rvagg/memdown)
  * @param {Buffer|String} [root] A hex `String` or `Buffer` for the root of a previously stored trie
  * @prop {Buffer} root The current root of the `trie`
  * @prop {Boolean} isCheckpoint  determines if you are saving to a checkpoint or directly to the db
@@ -26,11 +29,7 @@ module.exports = class Trie {
     this.EMPTY_TRIE_ROOT = ethUtil.SHA3_RLP
     this.sem = semaphore(1)
 
-    // setup dbs
-    this.db = db || level()
-
-    this._getDBs = [this.db]
-    this._putDBs = [this.db]
+    this.db = db || new DB()
 
     Object.defineProperty(this, 'root', {
       set(value) {
@@ -49,8 +48,6 @@ module.exports = class Trie {
     })
 
     this.root = root
-
-    this.putRaw = this._putRaw
   }
 
   /**
@@ -133,38 +130,12 @@ module.exports = class Trie {
     })
   }
 
-  /**
-   * Retrieves a raw value in the underlying db
-   * @method getRaw
-   * @memberof Trie
-   * @param {Buffer} key
-   * @param {Function} callback A callback `Function`, which is given the arguments `err` - for errors that may have occured and `value` - the found value in a `Buffer` or if no value was found `null`.
-   */
-  getRaw (key, cb) {
-    key = ethUtil.toBuffer(key)
-
-    function dbGet (db, cb2) {
-      db.get(key, {
-        keyEncoding: 'binary',
-        valueEncoding: 'binary'
-      }, (err, foundNode) => {
-        if (err || !foundNode) {
-          cb2(null, null)
-        } else {
-          cb2(null, foundNode)
-        }
-      })
-    }
-
-    asyncFirstSeries(this._getDBs, dbGet, cb)
-  }
-
   // retrieves a node from dbs by hash
   _lookupNode (node, cb) {
     if (TrieNode.isRawNode(node)) {
       cb(new TrieNode(node))
     } else {
-      this.getRaw(node, (err, value) => {
+      this.db.get(node, (err, value) => {
         if (err) {
           throw err
         }
@@ -178,60 +149,11 @@ module.exports = class Trie {
     }
   }
 
-  /**
-   * Writes a value directly to the underlining db
-   * @method putRaw
-   * @memberof Trie
-   * @param {Buffer|String} key The key as a `Buffer` or `String`
-   * @param {Buffer} value The value to be stored
-   * @param {Function} callback A callback `Function`, which is given the argument `err` - for errors that may have occured
-   */
-  // TODO: remove the proxy method when changing the caching
-  _putRaw (key, val, cb) {
-    function dbPut (db, cb2) {
-      db.put(key, val, {
-        keyEncoding: 'binary',
-        valueEncoding: 'binary'
-      }, cb2)
-    }
-
-    async.each(this._putDBs, dbPut, cb)
-  }
-
-  /**
-   * Removes a raw value in the underlying db
-   * @method delRaw
-   * @memberof Trie
-   * @param {Buffer|String} key
-   * @param {Function} callback A callback `Function`, which is given the argument `err` - for errors that may have occured
-   */
-  delRaw (key, cb) {
-    function del (db, cb2) {
-      db.del(key, {
-        keyEncoding: 'binary'
-      }, cb2)
-    }
-
-    async.each(this._putDBs, del, cb)
-  }
-
   // writes a single node to dbs
   _putNode (node, cb) {
     const hash = node.hash()
     const serialized = node.serialize()
-    this._putRaw(hash, serialized, cb)
-  }
-
-  // writes many nodes to db
-  _batchNodes (opStack, cb) {
-    function dbBatch (db, cb) {
-      db.batch(opStack, {
-        keyEncoding: 'binary',
-        valueEncoding: 'binary'
-      }, cb)
-    }
-
-    async.each(this._putDBs, dbBatch, cb)
+    this.db.put(hash, serialized, cb)
   }
 
   /**
@@ -560,7 +482,7 @@ module.exports = class Trie {
       this.root = lastRoot
     }
 
-    this._batchNodes(opStack, cb)
+    this.db.batch(opStack, cb)
   }
 
   _deleteNode (key, stack, cb) {
@@ -732,7 +654,8 @@ module.exports = class Trie {
   // creates a new trie backed by the same db
   // and starting at the same root
   copy () {
-    return new Trie(this.db, this.root)
+    const db = this.db.copy()
+    return new Trie(db, this.root)
   }
 
   /**

src/checkpoint-trie.js → src/checkpointTrie.js

@@ -1,15 +1,20 @@
 const async = require('async')
-const level = require('level-mem')
 const WriteStream = require('level-ws')
 const BaseTrie = require('./baseTrie')
 const proof = require('./proof.js')
 const ScratchReadStream = require('./scratchReadStream')
+const DB = require('./db')
+const ScratchDB = require('./scratch')
 const { callTogether } = require('./util/async')
 
 module.exports = class CheckpointTrie extends BaseTrie {
   constructor (...args) {
     super(...args)
+    // Reference to main DB instance
+    this._mainDB = this.db
+    // DB instance used for checkpoints
     this._scratch = null
+    // Roots of trie at the moment of checkpoint
     this._checkpoints = []
   }
 
@@ -71,7 +76,8 @@ module.exports = class CheckpointTrie extends BaseTrie {
 
   /**
    * Reverts the trie to the state it was at when `checkpoint` was first called.
-   * If during a nested checkpoint, only sets parent as current checkpoint.
+   * If during a nested checkpoint, sets root to most recent checkpoint, and sets
+   * parent checkpoint as current.
    * @method revert
    * @param {Function} cb the callback
    */
@@ -97,9 +103,10 @@ module.exports = class CheckpointTrie extends BaseTrie {
    * @method copy
    */
   copy () {
-    const trie = new CheckpointTrie(this.db, this.root)
+    const db = this._mainDB.copy()
+    const trie = new CheckpointTrie(db, this.root)
     trie._scratch = this._scratch
-    // trie._checkpoints = this._checkpoints.slice()
+    trie._checkpoints = this._checkpoints.slice()
     return trie
   }
 
@@ -111,60 +118,37 @@ module.exports = class CheckpointTrie extends BaseTrie {
   createScratchReadStream (scratch) {
     const trie = this.copy()
     scratch = scratch || this._scratch
+    scratch = new DB(scratch._db)
     // Only read from the scratch
-    trie._getDBs = [scratch]
+    trie.db = scratch
     trie._scratch = scratch
     return new ScratchReadStream(trie)
   }
 
-  /**
-   * Puts kv-pair directly to db, ignoring checkpoints.
-   * @private
-   */
-  _overridePutRaw (key, val, cb) {
-    const dbPut = (db, cb2) => {
-      db.put(key, val, {
-        keyEncoding: 'binary',
-        valueEncoding: 'binary'
-      }, cb2)
-    }
-    async.each(this.__putDBs, dbPut, cb)
-  }
-
   /**
    * Enter into checkpoint mode.
    * @private
    */
   _enterCpMode () {
-    this._scratch = level()
-    this._getDBs = [this._scratch].concat(this._getDBs)
-    this.__putDBs = this._putDBs
-    this._putDBs = [this._scratch]
-    this._putRaw = this.putRaw
-    this.putRaw = this._overridePutRaw
+    this._scratch = new ScratchDB(this._mainDB)
+    this.db = this._scratch
   }
 
   /**
    * Exit from checkpoint mode.
    * @private
    */
   _exitCpMode (commitState, cb) {
-    var scratch = this._scratch
+    const scratch = this._scratch
     this._scratch = null
-    this._getDBs = this._getDBs.slice(1)
-    this._putDBs = this.__putDBs
-    this.putRaw = this._putRaw
+    this.db = this._mainDB
 
-    const flushScratch = (db, cb) => {
+    if (commitState) {
       this.createScratchReadStream(scratch)
-        .pipe(WriteStream(db))
+        .pipe(WriteStream(this.db))
         .on('close', cb)
-    }
-
-    if (commitState) {
-      async.map(this._putDBs, flushScratch, cb)
     } else {
-      cb()
+      async.nextTick(cb)
     }
   }
 }

src/db.js

@@ -0,0 +1,70 @@
+const level = require('level-mem')
+const ethUtil = require('ethereumjs-util')
+
+const ENCODING_OPTS = { keyEncoding: 'binary', valueEncoding: 'binary' }
+
+/**
+ * DB is a thin wrapper around the underlying levelup db,
+ * which validates inputs and sets encoding type.
+ */
+module.exports = class DB {
+  constructor (db) {
+    this._db = db || level()
+  }
+
+  /**
+   * Retrieves a raw value from db.
+   * @param {Buffer|String} key
+   * @param {Function} cb A callback `Function`, which is given the arguments
+   * `err` - for errors that may have occured
+   * and `value` - the found value in a `Buffer` or if no value was found `null`.
+   */
+  get (key, cb) {
+    key = ethUtil.toBuffer(key)
+    this._db.get(key, ENCODING_OPTS, (err, v) => {
+      if (err || !v) {
+        cb(null, null)
+      } else {
+        cb(null, v)
+      }
+    })
+  }
+
+  /**
+   * Writes a value directly to db.
+   * @param {Buffer|String} key The key as a `Buffer` or `String`
+   * @param {Buffer} value The value to be stored
+   * @param {Function} cb A callback `Function`, which is given the argument
+   * `err` - for errors that may have occured
+   */
+  put (key, val, cb) {
+    this._db.put(key, val, ENCODING_OPTS, cb)
+  }
+
+  /**
+   * Removes a raw value in the underlying db.
+   * @param {Buffer|String} key
+   * @param {Function} cb A callback `Function`, which is given the argument
+   * `err` - for errors that may have occured
+   */
+  del (key, cb) {
+    this._db.del(key, ENCODING_OPTS, cb)
+  }
+
+  /**
+   * Performs a batch operation on db.
+   * @param {Array} opStack A stack of levelup operations
+   * @param {Function} cb A callback `Function`, which is given the argument
+   * `err` - for errors that may have occured
+   */
+  batch (opStack, cb) {
+    this._db.batch(opStack, ENCODING_OPTS, cb)
+  }
+
+  /**
+   * Returns a copy of DB.
+   */
+  copy () {
+    return new DB(this._db)
+  }
+}

src/index.js

@@ -1 +1 @@
-module.exports = require('./checkpoint-trie')
+module.exports = require('./checkpointTrie')

src/scratch.js

@@ -0,0 +1,36 @@
+const DB = require('./db')
+const { asyncFirstSeries } = require('./util/async')
+
+const ENCODING_OPTS = { keyEncoding: 'binary', valueEncoding: 'binary' }
+
+/**
+ * An in-memory wrap over `DB` with a backend DB
+ * which will be queried when a key is not found
+ * in the in-memory scratch. This class is used to implement
+ * checkpointing functionality in CheckpointTrie.
+ */
+module.exports = class ScratchDB extends DB {
+  constructor (db) {
+    super()
+    this._backend = db._db
+  }
+
+  /**
+   * Similar to `DB.get`, but first searches in-memory
+   * scratch DB, if key not found, searches backend DB.
+   */
+  get (key, cb) {
+    const getDBs = this._backend ? [this._db, this._backend] : [this._db]
+    const dbGet = (db, cb2) => {
+      db.get(key, ENCODING_OPTS, (err, v) => {
+        if (err || !v) {
+          cb2(null, null)
+        } else {
+          cb2(null, v)
+        }
+      })
+    }
+
+    asyncFirstSeries(getDBs, dbGet, cb)
+  }
+}