Fix progress reporting & initialblockdownload

.travis.yml

@@ -52,9 +52,9 @@ before_script:
     - if [ -n "$OSX_SDK" -a -f depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then tar -C depends/SDKs -xf depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz; fi
     - make $MAKEJOBS -C depends HOST=$HOST $DEP_OPTS
 script:
-    - if [ "$RUN_TESTS" = "true" -a "$TRAVIS_REPO_SLUG" = "ElementsProject/elements" -a "$TRAVIS_PULL_REQUEST" = "false" ]; then while read LINE; do travis_retry gpg --keyserver hkp://pool.sks-keyservers.net --recv-keys $LINE; done < contrib/verify-commits/trusted-keys; fi
-    - if [ "$RUN_TESTS" = "true" -a "$TRAVIS_REPO_SLUG" = "ElementsProject/elements" -a "$TRAVIS_PULL_REQUEST" = "false" ]; then git fetch --unshallow; fi
-    - if [ "$RUN_TESTS" = "true" -a "$TRAVIS_REPO_SLUG" = "ElementsProject/elements" -a "$TRAVIS_PULL_REQUEST" = "false" ]; then contrib/verify-commits/verify-commits.sh; fi
+    - if [ "$RUN_TESTS" = "true" -a "$TRAVIS_REPO_SLUG" = "blockstream/liquid" -a "$TRAVIS_PULL_REQUEST" = "false" ]; then while read LINE; do travis_retry gpg --keyserver hkp://pool.sks-keyservers.net --recv-keys $LINE; done < contrib/verify-commits/trusted-keys; fi
+    - if [ "$RUN_TESTS" = "true" -a "$TRAVIS_REPO_SLUG" = "blockstream/liquid" -a "$TRAVIS_PULL_REQUEST" = "false" ]; then git fetch --unshallow; fi
+    - if [ "$RUN_TESTS" = "true" -a "$TRAVIS_REPO_SLUG" = "blockstream/liquid" -a "$TRAVIS_PULL_REQUEST" = "false" ]; then contrib/verify-commits/verify-commits.sh; fi
     - export TRAVIS_COMMIT_LOG=`git log --format=fuller -1`
     - if [ -n "$USE_SHELL" ]; then export CONFIG_SHELL="$USE_SHELL"; fi
     - OUTDIR=$BASE_OUTDIR/$TRAVIS_PULL_REQUEST/$TRAVIS_JOB_NUMBER-$HOST
@@ -64,7 +64,7 @@ script:
     - mkdir build && cd build
     - ../configure --cache-file=config.cache $BITCOIN_CONFIG_ALL $BITCOIN_CONFIG || ( cat config.log && false)
     - make distdir VERSION=$HOST
-    - cd elements-$HOST
+    - cd liquid-$HOST
     - ./configure --cache-file=../config.cache $BITCOIN_CONFIG_ALL $BITCOIN_CONFIG || ( cat config.log && false)
     - make $MAKEJOBS $GOAL || ( echo "Build failure. Verbose build follows." && make $GOAL V=1 ; false )
     - export LD_LIBRARY_PATH=$TRAVIS_BUILD_DIR/depends/$HOST/lib

contrib/devtools/check-doc.py

@@ -23,7 +23,7 @@
 # list unsupported, deprecated and duplicate args as they need no documentation
 SET_DOC_OPTIONAL = set(['-rpcssl', '-benchmark', '-h', '-help', '-socks', '-tor', '-debugnet', '-whitelistalwaysrelay', '-prematurewitness', '-walletprematurewitness', '-promiscuousmempoolflags', '-blockminsize'])
 
-SET_DOC_OPTIONAL.update(['-con_fpowallowmindifficultyblocks', '-con_fpownoretargeting', '-con_nsubsidyhalvinginterval', '-con_bip34height', '-con_bip65height', '-con_bip66height', '-con_npowtargettimespan', '-con_npowtargetspacing', '-con_nrulechangeactivationthreshold', '-con_nminerconfirmationwindow', '-con_powlimit', '-con_parentpowlimit', '-con_bip34hash', '-con_nminimumchainwork', '-con_defaultassumevalid', '-parentgenesisblockhash', '-ndefaultport', '-npruneafterheight', '-fdefaultconsistencychecks', '-frequirestandard', '-fmineblocksondemand', '-mainchainrpccookiefile', '-testnet', '-ct_bits', '-ct_exponent', '-anyonecanspendaremine', '-fminingrequirespeers', '-fmineblocksondemand', '-con_mandatorycoinbase']])
+SET_DOC_OPTIONAL.update(['-con_fpowallowmindifficultyblocks', '-con_fpownoretargeting', '-con_nsubsidyhalvinginterval', '-con_bip34height', '-con_bip65height', '-con_bip66height', '-con_npowtargettimespan', '-con_npowtargetspacing', '-con_nrulechangeactivationthreshold', '-con_nminerconfirmationwindow', '-con_powlimit', '-con_parentpowlimit', '-con_bip34hash', '-con_nminimumchainwork', '-con_defaultassumevalid', '-parentgenesisblockhash', '-ndefaultport', '-npruneafterheight', '-fdefaultconsistencychecks', '-frequirestandard', '-fmineblocksondemand', '-mainchainrpccookiefile', '-testnet', '-ct_bits', '-ct_exponent', '-anyonecanspendaremine', '-fminingrequirespeers', '-fmineblocksondemand', '-con_mandatorycoinbase', '-initialfreecoins', '-pak', '-extprvkeyprefix'])
 
 def main():
   used = check_output(CMD_GREP_ARGS, shell=True)

qa/pull-tester/rpc-tests.py

@@ -166,6 +166,7 @@
     'p2p-leaktests.py',
     'pak_tests.py',
     'signed_blockchain.py',
+    'progress.py',
 ]
 if ENABLE_ZMQ:
     testScripts.append('zmq_test.py')

qa/rpc-tests/blockchain.py

@@ -112,7 +112,8 @@ def _test_getblockchaininfo(self):
         assert_equal(res['headers'], 200)
         assert_equal(res['bestblockhash'], besthash)
         assert isinstance(res['mediantime'], int)
-        assert_equal(res['verificationprogress'], 1)
+        # see progress.py test
+        #assert_equal(res['verificationprogress'], 1)
         assert_equal(res['pruned'], False)
         assert 'pruneheight' not in res
         assert 'softforks' not in res

qa/rpc-tests/confidential_transactions.py

@@ -475,7 +475,7 @@ def run_test(self):
         unblinded = self.nodes[0].validateaddress(self.nodes[0].getnewaddress())["unconfidential"]
         self.nodes[0].sendtoaddress(unblinded, self.nodes[0].getbalance()["bitcoin"], "", "", True)
         # Make tx with blinded destination and change outputs only
-        self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), self.nodes[0].getbalance()["bitcoin"]/2)
+        self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), round(self.nodes[0].getbalance()["bitcoin"]/2, 8))
         # Send back again, this transaction should have 3 outputs, all unblinded
         txid = self.nodes[0].sendtoaddress(unblinded, self.nodes[0].getbalance()["bitcoin"], "", "", True)
         outputs = self.nodes[0].getrawtransaction(txid, 1)["vout"]

qa/rpc-tests/progress.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env python3
+# Copyright (c) 2015-2016 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#
+# Test progress code
+#
+
+import time
+
+from test_framework.test_framework import BitcoinTestFramework
+from test_framework.util import (
+    start_nodes,
+    Decimal,
+)
+
+def assert_close(f1, f2):
+    assert(abs(Decimal(f1)-f2) < 0.1)
+
+class ProgressTest(BitcoinTestFramework):
+    def __init__(self):
+        super().__init__()
+        self.setup_clean_chain = True
+        self.num_nodes = 2
+        self.extra_args = [["-debug", "-con_npowtargetspacing=1", "-maxtimeadjustment=0"]] * self.num_nodes
+
+    def setup_network(self):
+        self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, extra_args=self.extra_args)
+        self.is_network_split = True
+        self.starttime = int(time.time())
+
+    def setmocktime(self, ntime):
+        for node in self.nodes:
+            node.setmocktime(self.starttime + ntime)
+
+    def run_test(self):
+        node1 = self.nodes[0]
+        node2 = self.nodes[1]
+        self.setmocktime(0)
+
+        blocks = []
+        for i in range(10):
+            self.setmocktime(i)
+            blocks.extend(node1.generate(1))
+
+        self.setmocktime(19)
+        assert_close(0.5, node1.getblockchaininfo()["verificationprogress"])
+
+        assert(node2.getblockchaininfo()["initialblockdownload"])
+
+        self.setmocktime(10)
+        for i in range(10):
+            node2.submitblock(node1.getblock(blocks[i], False))
+            progress = node2.getblockchaininfo()["verificationprogress"]
+            assert_close(i/10.0, progress)
+
+        assert(not node2.getblockchaininfo()["initialblockdownload"])
+
+if __name__ == '__main__':
+    ProgressTest().main()

src/qt/clientmodel.cpp

@@ -130,7 +130,7 @@ double ClientModel::getVerificationProgress(const CBlockIndex *tipIn) const
         LOCK(cs_main);
         tip = chainActive.Tip();
     }
-    return GuessVerificationProgress(Params().TxData(), tip);
+    return GuessVerificationProgress(tip, Params().GetConsensus().nPowTargetSpacing);
 }
 
 void ClientModel::updateTimer()

src/rpc/blockchain.cpp

@@ -19,6 +19,7 @@
 #include "sync.h"
 #include "txmempool.h"
 #include "util.h"
+#include "utiltime.h"
 #include "utilstrencodings.h"
 #include "hash.h"
 
@@ -1048,6 +1049,7 @@ UniValue getblockchaininfo(const JSONRPCRequest& request)
             "  \"bestblockhash\": \"...\", (string) the hash of the currently best block\n"
             "  \"mediantime\": xxxxxx,     (numeric) median time for the current best block\n"
             "  \"verificationprogress\": xxxx, (numeric) estimate of verification progress [0..1]\n"
+            "  \"initialblockdownload\": xx,  (boolean) whether or not the initial block download is still ongoing\n"
             "  \"pruned\": xx,             (boolean) if the blocks are subject to pruning\n"
             "  \"pruneheight\": xxxxxx,    (numeric) lowest-height complete block stored\n"
             "  \"bip9_softforks\": {          (object) status of BIP9 softforks in progress\n"
@@ -1074,10 +1076,11 @@ UniValue getblockchaininfo(const JSONRPCRequest& request)
     obj.push_back(Pair("headers",               pindexBestHeader ? pindexBestHeader->nHeight : -1));
     obj.push_back(Pair("bestblockhash",         tip->GetBlockHash().GetHex()));
     obj.push_back(Pair("mediantime",            (int64_t)tip->GetMedianTimePast()));
-    obj.push_back(Pair("verificationprogress",  GuessVerificationProgress(Params().TxData(), tip)));
+    obj.push_back(Pair("verificationprogress",  GuessVerificationProgress(tip, Params().GetConsensus().nPowTargetSpacing)));
     obj.push_back(Pair("pruned",                fPruneMode));
     obj.push_back(Pair("signblock_asm", ScriptToAsmStr(tip->proof.challenge)));
     obj.push_back(Pair("signblock_hex", HexStr(tip->proof.challenge.begin(), tip->proof.challenge.end())));
+    obj.push_back(Pair("initialblockdownload",  IsInitialBlockDownload()));
 
     const Consensus::Params& consensusParams = Params().GetConsensus();
     UniValue bip9_softforks(UniValue::VOBJ);

src/validation.cpp

@@ -3042,7 +3042,8 @@ void static UpdateTip(CBlockIndex *pindexNew, const CChainParams& chainParams) {
       chainActive.Tip()->GetBlockHash().ToString(), chainActive.Height(), chainActive.Tip()->nVersion,
       log(chainActive.Tip()->nChainWork.getdouble())/log(2.0), (unsigned long)chainActive.Tip()->nChainTx,
       DateTimeStrFormat("%Y-%m-%d %H:%M:%S", chainActive.Tip()->GetBlockTime()),
-      GuessVerificationProgress(chainParams.TxData(), chainActive.Tip()), pcoinsTip->DynamicMemoryUsage() * (1.0 / (1<<20)), pcoinsTip->GetCacheSize());
+      GuessVerificationProgress(chainActive.Tip(), chainParams.GetConsensus().nPowTargetSpacing),
+      pcoinsTip->DynamicMemoryUsage() * (1.0 / (1<<20)), pcoinsTip->GetCacheSize());
     if (!warningMessages.empty())
         LogPrintf(" warning='%s'", boost::algorithm::join(warningMessages, ", "));
     LogPrintf("\n");
@@ -4609,10 +4610,10 @@ bool static LoadBlockIndexDB(const CChainParams& chainparams)
 
     PruneBlockIndexCandidates();
 
-    LogPrintf("%s: hashBestChain=%s height=%d date=%s progress=%f\n", __func__,
+    LogPrintf("%s: hashBestChain=%s height=%d date=%s progress=%.3f\n", __func__,
         chainActive.Tip()->GetBlockHash().ToString(), chainActive.Height(),
         DateTimeStrFormat("%Y-%m-%d %H:%M:%S", chainActive.Tip()->GetBlockTime()),
-        GuessVerificationProgress(chainparams.TxData(), chainActive.Tip()));
+        GuessVerificationProgress(chainActive.Tip(), chainparams.GetConsensus().nPowTargetSpacing));
 
     return true;
 }
@@ -5317,22 +5318,21 @@ void DumpMempool(void)
     }
 }
 
-//! Guess how far we are in the verification process at the given block index
-double GuessVerificationProgress(const ChainTxData& data, CBlockIndex *pindex) {
-    if (pindex == NULL)
+// Guess how far we are in the verification process at the given block index.
+// Since we have signed fixed-interval blocks, estimating progress is a very easy.
+// We can extrapolate the last block time to the current time to estimate how many more blocks
+// we expect.
+double GuessVerificationProgress(CBlockIndex *pindex, int64_t blockInterval) {
+    if (pindex == NULL || pindex->nHeight < 1)
         return 0.0;
 
-    int64_t nNow = time(NULL);
-
-    double fTxTotal;
-
-    if (pindex->nChainTx <= data.nTxCount) {
-        fTxTotal = data.nTxCount + (nNow - data.nTime) * data.dTxRate;
-    } else {
-        fTxTotal = pindex->nChainTx + (nNow - pindex->GetBlockTime()) * data.dTxRate;
-    }
-
-    return pindex->nChainTx / fTxTotal;
+    int64_t nNow = GetTime();
+    int64_t moreBlocksExpected = (nNow - pindex->GetBlockTime()) / blockInterval;
+    double progress = (pindex->nHeight + 0.0) / (pindex->nHeight + moreBlocksExpected);
+    // Round to 3 digits to avoid 0.999999 when finished.
+    progress = ceil(progress * 1000.0) / 1000.0;
+    // Avoid higher than one if last block is newer than current time.
+    return std::min(1.0, progress);
 }
 
 class CMainCleanup

src/validation.h

@@ -287,7 +287,7 @@ bool ActivateBestChain(CValidationState& state, const CChainParams& chainparams,
 CAmount GetBlockSubsidy(int nHeight, const Consensus::Params& consensusParams);
 
 /** Guess verification progress (as a fraction between 0.0=genesis and 1.0=current tip). */
-double GuessVerificationProgress(const ChainTxData& data, CBlockIndex* pindex);
+double GuessVerificationProgress(CBlockIndex* pindex, int64_t blockInterval);
 
 /**
  * Prune block and undo files (blk???.dat and undo???.dat) so that the disk space used is less than a user-defined target.

src/wallet/wallet.cpp

@@ -1692,12 +1692,14 @@ CBlockIndex* CWallet::ScanForWalletTransactions(CBlockIndex* pindexStart, bool f
             pindex = chainActive.Next(pindex);
 
         ShowProgress(_("Rescanning..."), 0); // show rescan progress in GUI as dialog or on splashscreen, if -rescan on startup
-        double dProgressStart = GuessVerificationProgress(chainParams.TxData(), pindex);
-        double dProgressTip = GuessVerificationProgress(chainParams.TxData(), chainActive.Tip());
+        double dProgressStart = GuessVerificationProgress(pindex, chainParams.GetConsensus().nPowTargetSpacing);
+        double dProgressTip = GuessVerificationProgress(chainActive.Tip(), chainParams.GetConsensus().nPowTargetSpacing);
         while (pindex)
         {
-            if (pindex->nHeight % 100 == 0 && dProgressTip - dProgressStart > 0.0)
-                ShowProgress(_("Rescanning..."), std::max(1, std::min(99, (int)((GuessVerificationProgress(chainParams.TxData(), pindex) - dProgressStart) / (dProgressTip - dProgressStart) * 100))));
+            if (pindex->nHeight % 100 == 0 && dProgressTip - dProgressStart > 0.0) {
+                double progress = GuessVerificationProgress(pindex, chainParams.GetConsensus().nPowTargetSpacing);
+                ShowProgress(_("Rescanning..."), std::max(1, std::min(99, (int)((progress - dProgressStart) / (dProgressTip - dProgressStart) * 100))));
+            }
 
             CBlock block;
             if (ReadBlockFromDisk(block, pindex, Params().GetConsensus())) {
@@ -1713,7 +1715,8 @@ CBlockIndex* CWallet::ScanForWalletTransactions(CBlockIndex* pindexStart, bool f
             pindex = chainActive.Next(pindex);
             if (GetTime() >= nNow + 60) {
                 nNow = GetTime();
-                LogPrintf("Still rescanning. At block %d. Progress=%f\n", pindex->nHeight, GuessVerificationProgress(chainParams.TxData(), pindex));
+                double progress = GuessVerificationProgress(pindex, chainParams.GetConsensus().nPowTargetSpacing);
+                LogPrintf("Still rescanning. At block %d. Progress=%f\n", pindex->nHeight, progress);
             }
         }
         ShowProgress(_("Rescanning..."), 100); // hide progress dialog in GUI