From 80282ccdcbf025cdd13fe1cd7fa0382bc892cef1 Mon Sep 17 00:00:00 2001 From: Steven Roose Date: Thu, 6 Dec 2018 19:30:35 +0100 Subject: [PATCH 1/2] Fix Travis build --- .travis.yml | 8 ++++---- contrib/devtools/check-doc.py | 2 +- qa/rpc-tests/confidential_transactions.py | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 879909033..50fafd242 100644 --- a/.travis.yml +++ b/.travis.yml @@ -52,9 +52,9 @@ before_script: - if [ -n "$OSX_SDK" -a -f depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then tar -C depends/SDKs -xf depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz; fi - make $MAKEJOBS -C depends HOST=$HOST $DEP_OPTS script: - - if [ "$RUN_TESTS" = "true" -a "$TRAVIS_REPO_SLUG" = "ElementsProject/elements" -a "$TRAVIS_PULL_REQUEST" = "false" ]; then while read LINE; do travis_retry gpg --keyserver hkp://pool.sks-keyservers.net --recv-keys $LINE; done < contrib/verify-commits/trusted-keys; fi - - if [ "$RUN_TESTS" = "true" -a "$TRAVIS_REPO_SLUG" = "ElementsProject/elements" -a "$TRAVIS_PULL_REQUEST" = "false" ]; then git fetch --unshallow; fi - - if [ "$RUN_TESTS" = "true" -a "$TRAVIS_REPO_SLUG" = "ElementsProject/elements" -a "$TRAVIS_PULL_REQUEST" = "false" ]; then contrib/verify-commits/verify-commits.sh; fi + - if [ "$RUN_TESTS" = "true" -a "$TRAVIS_REPO_SLUG" = "blockstream/liquid" -a "$TRAVIS_PULL_REQUEST" = "false" ]; then while read LINE; do travis_retry gpg --keyserver hkp://pool.sks-keyservers.net --recv-keys $LINE; done < contrib/verify-commits/trusted-keys; fi + - if [ "$RUN_TESTS" = "true" -a "$TRAVIS_REPO_SLUG" = "blockstream/liquid" -a "$TRAVIS_PULL_REQUEST" = "false" ]; then git fetch --unshallow; fi + - if [ "$RUN_TESTS" = "true" -a "$TRAVIS_REPO_SLUG" = "blockstream/liquid" -a "$TRAVIS_PULL_REQUEST" = "false" ]; then contrib/verify-commits/verify-commits.sh; fi - export TRAVIS_COMMIT_LOG=`git log --format=fuller -1` - if [ -n "$USE_SHELL" ]; then export CONFIG_SHELL="$USE_SHELL"; fi - OUTDIR=$BASE_OUTDIR/$TRAVIS_PULL_REQUEST/$TRAVIS_JOB_NUMBER-$HOST @@ -64,7 +64,7 @@ script: - mkdir build && cd build - ../configure --cache-file=config.cache $BITCOIN_CONFIG_ALL $BITCOIN_CONFIG || ( cat config.log && false) - make distdir VERSION=$HOST - - cd elements-$HOST + - cd liquid-$HOST - ./configure --cache-file=../config.cache $BITCOIN_CONFIG_ALL $BITCOIN_CONFIG || ( cat config.log && false) - make $MAKEJOBS $GOAL || ( echo "Build failure. Verbose build follows." && make $GOAL V=1 ; false ) - export LD_LIBRARY_PATH=$TRAVIS_BUILD_DIR/depends/$HOST/lib diff --git a/contrib/devtools/check-doc.py b/contrib/devtools/check-doc.py index 05b4ed194..8f3f82877 100755 --- a/contrib/devtools/check-doc.py +++ b/contrib/devtools/check-doc.py @@ -23,7 +23,7 @@ # list unsupported, deprecated and duplicate args as they need no documentation SET_DOC_OPTIONAL = set(['-rpcssl', '-benchmark', '-h', '-help', '-socks', '-tor', '-debugnet', '-whitelistalwaysrelay', '-prematurewitness', '-walletprematurewitness', '-promiscuousmempoolflags', '-blockminsize']) -SET_DOC_OPTIONAL.update(['-con_fpowallowmindifficultyblocks', '-con_fpownoretargeting', '-con_nsubsidyhalvinginterval', '-con_bip34height', '-con_bip65height', '-con_bip66height', '-con_npowtargettimespan', '-con_npowtargetspacing', '-con_nrulechangeactivationthreshold', '-con_nminerconfirmationwindow', '-con_powlimit', '-con_parentpowlimit', '-con_bip34hash', '-con_nminimumchainwork', '-con_defaultassumevalid', '-parentgenesisblockhash', '-ndefaultport', '-npruneafterheight', '-fdefaultconsistencychecks', '-frequirestandard', '-fmineblocksondemand', '-mainchainrpccookiefile', '-testnet', '-ct_bits', '-ct_exponent', '-anyonecanspendaremine', '-fminingrequirespeers', '-fmineblocksondemand', '-con_mandatorycoinbase']]) +SET_DOC_OPTIONAL.update(['-con_fpowallowmindifficultyblocks', '-con_fpownoretargeting', '-con_nsubsidyhalvinginterval', '-con_bip34height', '-con_bip65height', '-con_bip66height', '-con_npowtargettimespan', '-con_npowtargetspacing', '-con_nrulechangeactivationthreshold', '-con_nminerconfirmationwindow', '-con_powlimit', '-con_parentpowlimit', '-con_bip34hash', '-con_nminimumchainwork', '-con_defaultassumevalid', '-parentgenesisblockhash', '-ndefaultport', '-npruneafterheight', '-fdefaultconsistencychecks', '-frequirestandard', '-fmineblocksondemand', '-mainchainrpccookiefile', '-testnet', '-ct_bits', '-ct_exponent', '-anyonecanspendaremine', '-fminingrequirespeers', '-fmineblocksondemand', '-con_mandatorycoinbase', '-initialfreecoins', '-pak', '-extprvkeyprefix']) def main(): used = check_output(CMD_GREP_ARGS, shell=True) diff --git a/qa/rpc-tests/confidential_transactions.py b/qa/rpc-tests/confidential_transactions.py index 95baedea0..d7c99a586 100755 --- a/qa/rpc-tests/confidential_transactions.py +++ b/qa/rpc-tests/confidential_transactions.py @@ -475,7 +475,7 @@ def run_test(self): unblinded = self.nodes[0].validateaddress(self.nodes[0].getnewaddress())["unconfidential"] self.nodes[0].sendtoaddress(unblinded, self.nodes[0].getbalance()["bitcoin"], "", "", True) # Make tx with blinded destination and change outputs only - self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), self.nodes[0].getbalance()["bitcoin"]/2) + self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), round(self.nodes[0].getbalance()["bitcoin"]/2, 8)) # Send back again, this transaction should have 3 outputs, all unblinded txid = self.nodes[0].sendtoaddress(unblinded, self.nodes[0].getbalance()["bitcoin"], "", "", True) outputs = self.nodes[0].getrawtransaction(txid, 1)["vout"] From ee1d706a99cc6eb7cd65c3eec43d41689e49bc34 Mon Sep 17 00:00:00 2001 From: Steven Roose Date: Tue, 20 Nov 2018 17:59:12 +0000 Subject: [PATCH 2/2] Fix progress reporting issue --- qa/pull-tester/rpc-tests.py | 1 + qa/rpc-tests/blockchain.py | 3 +- qa/rpc-tests/progress.py | 61 +++++++++++++++++++++++++++++++++++++ src/qt/clientmodel.cpp | 2 +- src/rpc/blockchain.cpp | 5 ++- src/validation.cpp | 34 ++++++++++----------- src/validation.h | 2 +- src/wallet/wallet.cpp | 13 +++++--- 8 files changed, 95 insertions(+), 26 deletions(-) create mode 100755 qa/rpc-tests/progress.py diff --git a/qa/pull-tester/rpc-tests.py b/qa/pull-tester/rpc-tests.py index 9a9ff0212..6269a6bb5 100755 --- a/qa/pull-tester/rpc-tests.py +++ b/qa/pull-tester/rpc-tests.py @@ -166,6 +166,7 @@ 'p2p-leaktests.py', 'pak_tests.py', 'signed_blockchain.py', + 'progress.py', ] if ENABLE_ZMQ: testScripts.append('zmq_test.py') diff --git a/qa/rpc-tests/blockchain.py b/qa/rpc-tests/blockchain.py index d9d46cda4..efa6b45c7 100755 --- a/qa/rpc-tests/blockchain.py +++ b/qa/rpc-tests/blockchain.py @@ -112,7 +112,8 @@ def _test_getblockchaininfo(self): assert_equal(res['headers'], 200) assert_equal(res['bestblockhash'], besthash) assert isinstance(res['mediantime'], int) - assert_equal(res['verificationprogress'], 1) + # see progress.py test + #assert_equal(res['verificationprogress'], 1) assert_equal(res['pruned'], False) assert 'pruneheight' not in res assert 'softforks' not in res diff --git a/qa/rpc-tests/progress.py b/qa/rpc-tests/progress.py new file mode 100755 index 000000000..196d42a60 --- /dev/null +++ b/qa/rpc-tests/progress.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 +# Copyright (c) 2015-2016 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +# +# Test progress code +# + +import time + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + start_nodes, + Decimal, +) + +def assert_close(f1, f2): + assert(abs(Decimal(f1)-f2) < 0.1) + +class ProgressTest(BitcoinTestFramework): + def __init__(self): + super().__init__() + self.setup_clean_chain = True + self.num_nodes = 2 + self.extra_args = [["-debug", "-con_npowtargetspacing=1", "-maxtimeadjustment=0"]] * self.num_nodes + + def setup_network(self): + self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, extra_args=self.extra_args) + self.is_network_split = True + self.starttime = int(time.time()) + + def setmocktime(self, ntime): + for node in self.nodes: + node.setmocktime(self.starttime + ntime) + + def run_test(self): + node1 = self.nodes[0] + node2 = self.nodes[1] + self.setmocktime(0) + + blocks = [] + for i in range(10): + self.setmocktime(i) + blocks.extend(node1.generate(1)) + + self.setmocktime(19) + assert_close(0.5, node1.getblockchaininfo()["verificationprogress"]) + + assert(node2.getblockchaininfo()["initialblockdownload"]) + + self.setmocktime(10) + for i in range(10): + node2.submitblock(node1.getblock(blocks[i], False)) + progress = node2.getblockchaininfo()["verificationprogress"] + assert_close(i/10.0, progress) + + assert(not node2.getblockchaininfo()["initialblockdownload"]) + +if __name__ == '__main__': + ProgressTest().main() diff --git a/src/qt/clientmodel.cpp b/src/qt/clientmodel.cpp index bb10e4942..76896e1cd 100644 --- a/src/qt/clientmodel.cpp +++ b/src/qt/clientmodel.cpp @@ -130,7 +130,7 @@ double ClientModel::getVerificationProgress(const CBlockIndex *tipIn) const LOCK(cs_main); tip = chainActive.Tip(); } - return GuessVerificationProgress(Params().TxData(), tip); + return GuessVerificationProgress(tip, Params().GetConsensus().nPowTargetSpacing); } void ClientModel::updateTimer() diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index bf28d40cd..e86a05d1c 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -19,6 +19,7 @@ #include "sync.h" #include "txmempool.h" #include "util.h" +#include "utiltime.h" #include "utilstrencodings.h" #include "hash.h" @@ -1048,6 +1049,7 @@ UniValue getblockchaininfo(const JSONRPCRequest& request) " \"bestblockhash\": \"...\", (string) the hash of the currently best block\n" " \"mediantime\": xxxxxx, (numeric) median time for the current best block\n" " \"verificationprogress\": xxxx, (numeric) estimate of verification progress [0..1]\n" + " \"initialblockdownload\": xx, (boolean) whether or not the initial block download is still ongoing\n" " \"pruned\": xx, (boolean) if the blocks are subject to pruning\n" " \"pruneheight\": xxxxxx, (numeric) lowest-height complete block stored\n" " \"bip9_softforks\": { (object) status of BIP9 softforks in progress\n" @@ -1074,10 +1076,11 @@ UniValue getblockchaininfo(const JSONRPCRequest& request) obj.push_back(Pair("headers", pindexBestHeader ? pindexBestHeader->nHeight : -1)); obj.push_back(Pair("bestblockhash", tip->GetBlockHash().GetHex())); obj.push_back(Pair("mediantime", (int64_t)tip->GetMedianTimePast())); - obj.push_back(Pair("verificationprogress", GuessVerificationProgress(Params().TxData(), tip))); + obj.push_back(Pair("verificationprogress", GuessVerificationProgress(tip, Params().GetConsensus().nPowTargetSpacing))); obj.push_back(Pair("pruned", fPruneMode)); obj.push_back(Pair("signblock_asm", ScriptToAsmStr(tip->proof.challenge))); obj.push_back(Pair("signblock_hex", HexStr(tip->proof.challenge.begin(), tip->proof.challenge.end()))); + obj.push_back(Pair("initialblockdownload", IsInitialBlockDownload())); const Consensus::Params& consensusParams = Params().GetConsensus(); UniValue bip9_softforks(UniValue::VOBJ); diff --git a/src/validation.cpp b/src/validation.cpp index 4e8accc07..3b1d4fce1 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -3042,7 +3042,8 @@ void static UpdateTip(CBlockIndex *pindexNew, const CChainParams& chainParams) { chainActive.Tip()->GetBlockHash().ToString(), chainActive.Height(), chainActive.Tip()->nVersion, log(chainActive.Tip()->nChainWork.getdouble())/log(2.0), (unsigned long)chainActive.Tip()->nChainTx, DateTimeStrFormat("%Y-%m-%d %H:%M:%S", chainActive.Tip()->GetBlockTime()), - GuessVerificationProgress(chainParams.TxData(), chainActive.Tip()), pcoinsTip->DynamicMemoryUsage() * (1.0 / (1<<20)), pcoinsTip->GetCacheSize()); + GuessVerificationProgress(chainActive.Tip(), chainParams.GetConsensus().nPowTargetSpacing), + pcoinsTip->DynamicMemoryUsage() * (1.0 / (1<<20)), pcoinsTip->GetCacheSize()); if (!warningMessages.empty()) LogPrintf(" warning='%s'", boost::algorithm::join(warningMessages, ", ")); LogPrintf("\n"); @@ -4609,10 +4610,10 @@ bool static LoadBlockIndexDB(const CChainParams& chainparams) PruneBlockIndexCandidates(); - LogPrintf("%s: hashBestChain=%s height=%d date=%s progress=%f\n", __func__, + LogPrintf("%s: hashBestChain=%s height=%d date=%s progress=%.3f\n", __func__, chainActive.Tip()->GetBlockHash().ToString(), chainActive.Height(), DateTimeStrFormat("%Y-%m-%d %H:%M:%S", chainActive.Tip()->GetBlockTime()), - GuessVerificationProgress(chainparams.TxData(), chainActive.Tip())); + GuessVerificationProgress(chainActive.Tip(), chainparams.GetConsensus().nPowTargetSpacing)); return true; } @@ -5317,22 +5318,21 @@ void DumpMempool(void) } } -//! Guess how far we are in the verification process at the given block index -double GuessVerificationProgress(const ChainTxData& data, CBlockIndex *pindex) { - if (pindex == NULL) +// Guess how far we are in the verification process at the given block index. +// Since we have signed fixed-interval blocks, estimating progress is a very easy. +// We can extrapolate the last block time to the current time to estimate how many more blocks +// we expect. +double GuessVerificationProgress(CBlockIndex *pindex, int64_t blockInterval) { + if (pindex == NULL || pindex->nHeight < 1) return 0.0; - int64_t nNow = time(NULL); - - double fTxTotal; - - if (pindex->nChainTx <= data.nTxCount) { - fTxTotal = data.nTxCount + (nNow - data.nTime) * data.dTxRate; - } else { - fTxTotal = pindex->nChainTx + (nNow - pindex->GetBlockTime()) * data.dTxRate; - } - - return pindex->nChainTx / fTxTotal; + int64_t nNow = GetTime(); + int64_t moreBlocksExpected = (nNow - pindex->GetBlockTime()) / blockInterval; + double progress = (pindex->nHeight + 0.0) / (pindex->nHeight + moreBlocksExpected); + // Round to 3 digits to avoid 0.999999 when finished. + progress = ceil(progress * 1000.0) / 1000.0; + // Avoid higher than one if last block is newer than current time. + return std::min(1.0, progress); } class CMainCleanup diff --git a/src/validation.h b/src/validation.h index 63b3ef5cf..96203b140 100644 --- a/src/validation.h +++ b/src/validation.h @@ -287,7 +287,7 @@ bool ActivateBestChain(CValidationState& state, const CChainParams& chainparams, CAmount GetBlockSubsidy(int nHeight, const Consensus::Params& consensusParams); /** Guess verification progress (as a fraction between 0.0=genesis and 1.0=current tip). */ -double GuessVerificationProgress(const ChainTxData& data, CBlockIndex* pindex); +double GuessVerificationProgress(CBlockIndex* pindex, int64_t blockInterval); /** * Prune block and undo files (blk???.dat and undo???.dat) so that the disk space used is less than a user-defined target. diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 36096395b..d79c6166a 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -1692,12 +1692,14 @@ CBlockIndex* CWallet::ScanForWalletTransactions(CBlockIndex* pindexStart, bool f pindex = chainActive.Next(pindex); ShowProgress(_("Rescanning..."), 0); // show rescan progress in GUI as dialog or on splashscreen, if -rescan on startup - double dProgressStart = GuessVerificationProgress(chainParams.TxData(), pindex); - double dProgressTip = GuessVerificationProgress(chainParams.TxData(), chainActive.Tip()); + double dProgressStart = GuessVerificationProgress(pindex, chainParams.GetConsensus().nPowTargetSpacing); + double dProgressTip = GuessVerificationProgress(chainActive.Tip(), chainParams.GetConsensus().nPowTargetSpacing); while (pindex) { - if (pindex->nHeight % 100 == 0 && dProgressTip - dProgressStart > 0.0) - ShowProgress(_("Rescanning..."), std::max(1, std::min(99, (int)((GuessVerificationProgress(chainParams.TxData(), pindex) - dProgressStart) / (dProgressTip - dProgressStart) * 100)))); + if (pindex->nHeight % 100 == 0 && dProgressTip - dProgressStart > 0.0) { + double progress = GuessVerificationProgress(pindex, chainParams.GetConsensus().nPowTargetSpacing); + ShowProgress(_("Rescanning..."), std::max(1, std::min(99, (int)((progress - dProgressStart) / (dProgressTip - dProgressStart) * 100)))); + } CBlock block; if (ReadBlockFromDisk(block, pindex, Params().GetConsensus())) { @@ -1713,7 +1715,8 @@ CBlockIndex* CWallet::ScanForWalletTransactions(CBlockIndex* pindexStart, bool f pindex = chainActive.Next(pindex); if (GetTime() >= nNow + 60) { nNow = GetTime(); - LogPrintf("Still rescanning. At block %d. Progress=%f\n", pindex->nHeight, GuessVerificationProgress(chainParams.TxData(), pindex)); + double progress = GuessVerificationProgress(pindex, chainParams.GetConsensus().nPowTargetSpacing); + LogPrintf("Still rescanning. At block %d. Progress=%f\n", pindex->nHeight, progress); } } ShowProgress(_("Rescanning..."), 100); // hide progress dialog in GUI