Quorum crashed on heavy loading with clique consensus.

[pccr10001]

System information

Geth version: geth version

Geth
Version: 1.8.12-stable
Quorum Version: 2.2.1
Architecture: amd64
Protocol Versions: [63 62]
Network Id: 1337
Go Version: go1.10.7
Operating System: linux
GOPATH=
GOROOT=/usr/local/go

OS & Version: Ubuntu 16.04

Branch, Commit Hash or Release: git status

Expected behaviour

Everything works.

Actual behaviour

Some nodes crashed with map concurrent fatal.

Signers: 5
Total nodes: 6
Crash nodes: 2

Steps to reproduce the behaviour

Use JMeter to perform load tests with 100 threads and 10 loops.

Web3Sampler.java

    public SampleResult runTest(JavaSamplerContext javaSamplerContext) {
        SampleResult result = new SampleResult();

        Function add = new Function(FUNC_ADD,
                Collections.emptyList(),
                Collections.singletonList(new TypeReference<Uint256>() {
                }));
        String data = FunctionEncoder.encode(add);
        RawTransactionManager rtm = new RawTransactionManager(quorum, Credentials.create(getRandomPrivateKey()),(byte)10);
        String hash="";
        result.sampleStart();
        try {
            hash = rtm.sendTransaction(BigInteger.ZERO, BigInteger.valueOf(0xE00000), contractAddress, data, BigInteger.ZERO).getTransactionHash();
            result.setResponseMessage("operation successful");
            result.setResponseCodeOK();
            result.setResponseOK();
            result.setDataType(SampleResult.TEXT);
            result.setResponseData(hash,"UTF-8");
        } catch (IOException e) {
            e.printStackTrace();
            result.setResponseCode("500");
            result.setResponseMessage("An exception occured: "+e.getMessage());
        }
        finally {
            result.sampleEnd();
        }

        return result;
    }
}

Counter.sol

pragma solidity ^0.5.1;

contract counter{
    uint256 public counter=0;
    uint256 public lastModify=0;
    uint256 public firstModify=0;
    
    
    function add() public returns (uint256) {
        if(counter==0)
            firstModify=now/1000000;
            
        counter++;
        lastModify=now/1000000;
    }
    
    function reset() public {
        counter=0;
        lastModify=0;
        firstModify=0;
    }
}

{
  "alloc": {
    "6613b20bb920f1c1c310effa68b1ef976a96f539": {
      "balance": "1000000000000000000000000000"
    },
    "e8dcf37e70d0a39d6ee8b3ffc8a38e8a30810e81": {
      "balance": "1000000000000000000000000000"
    },
    "26e9dfbd0ba6b43c135ff3c6ebf7bb77180a26cd": {
      "balance": "1000000000000000000000000000"
    },
    "7996a601d2e4719acb304b4f1ac28123b1690985": {
      "balance": "1000000000000000000000000000"
    },
    "2649b87632dc38cd87e730ebdd666f6aae50af17": {
      "balance": "1000000000000000000000000000"
    },
    "219d3fb8056179aa49cf9c632577ca0877b00c57": {
      "balance": "1000000000000000000000000000"
    }
  },
  "coinbase": "0x0000000000000000000000000000000000000000",
  "difficulty": "0x1",
  "gasLimit": "0xE0000000",
  "nonce": "0x0",
  "parentHash": "0x0000000000000000000000000000000000000000000000000000000000000000",
  "timestamp": "0x5c8b6809",
  "config":{
    "chainId": 10,
    "eip150Block": 1,
    "eip150Hash": "0x0000000000000000000000000000000000000000000000000000000000000000",
    "eip155Block": 1,
    "eip158Block": 1,
    "byzantiumBlock": 1,
    "constantinopleBlock": 1,
    "isQuorum": true,
    "clique": {
      "period": 0,
      "epoch": 30000
    }
  },
  "extraData": "0x0000000000000000000000000000000000000000000000000000000000000000219d3fb8056179aa49cf9c632577ca0877b00c572649b87632dc38cd87e730ebdd666f6aae50af177996a601d2e4719acb304b4f1ac28123b169098526e9dfbd0ba6b43c135ff3c6ebf7bb77180a26cde8dcf37e70d0a39d6ee8b3ffc8a38e8a30810e810000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000",
  "mixhash": "0x0000000000000000000000000000000000000000000000000000000000000000"
}

Backtrace

INFO [03-15|08:55:41.085] Imported new chain segment               blocks=1 txs=29 mgas=0.928 elapsed=17.695ms  mgasps=52.426  number=147 hash=5e8bb8…97dda9 cache=369.88kB
INFO [03-15|08:55:41.088] Commit new mining work                   number=148 txs=42 uncles=1 elapsed=8.281ms
INFO [03-15|08:55:41.088] Successfully sealed new block            number=148 hash=719770…68a53a
fatal error: concurrent map iteration and map write

goroutine 137 [running]:
runtime.throw(0x10723a6, 0x26)
        /usr/local/go/src/runtime/panic.go:616 +0x81 fp=0xc44ad7d280 sp=0xc44ad7d260 pc=0x440811
runtime.mapiternext(0xc44ad7d608)
        /usr/local/go/src/runtime/hashmap.go:747 +0x55c fp=0xc44ad7d310 sp=0xc44ad7d280 pc=0x41e4cc
github.com/ethereum/go-ethereum/core/state.(*StateDB).Copy(0xc4227790a0, 0x0)
        /work/quorum/build/_workspace/src/github.com/ethereum/go-ethereum/core/state/statedb.go:495 +0x53a fp=0xc44ad7d6d8 sp=0xc44ad7d310 pc=0x71203a
github.com/ethereum/go-ethereum/miner.(*worker).updateSnapshot(0xc4202f9080)
        /work/quorum/build/_workspace/src/github.com/ethereum/go-ethereum/miner/worker.go:569 +0x100 fp=0xc44ad7d748 sp=0xc44ad7d6d8 pc=0xa99c20
github.com/ethereum/go-ethereum/miner.(*worker).commitNewWork(0xc4202f9080)
        /work/quorum/build/_workspace/src/github.com/ethereum/go-ethereum/miner/worker.go:541 +0x11c3 fp=0xc44ad7dd20 sp=0xc44ad7d748 pc=0xa98b23
github.com/ethereum/go-ethereum/miner.(*worker).update(0xc4202f9080)
        /work/quorum/build/_workspace/src/github.com/ethereum/go-ethereum/miner/worker.go:271 +0x7c1 fp=0xc44ad7dfd8 sp=0xc44ad7dd20 pc=0xa95ea1
runtime.goexit()
        /usr/local/go/src/runtime/asm_amd64.s:2361 +0x1 fp=0xc44ad7dfe0 sp=0xc44ad7dfd8 pc=0x46ff71
created by github.com/ethereum/go-ethereum/miner.newWorker
        /work/quorum/build/_workspace/src/github.com/ethereum/go-ethereum/miner/worker.go:164 +0x4f9

goroutine 1 [chan receive, 1 minutes]:
github.com/ethereum/go-ethereum/node.(*Node).Wait(0xc420227900)
        /work/quorum/build/_workspace/src/github.com/ethereum/go-ethereum/node/node.go:466 +0x7f
main.geth(0xc420132160, 0x18, 0xc4204e3d58)
        /work/quorum/build/_workspace/src/github.com/ethereum/go-ethereum/cmd/geth/main.go:264 +0x92
github.com/ethereum/go-ethereum/vendor/gopkg.in/urfave/cli%2ev1.HandleAction(0xea5260, 0x10e55e8, 0xc420132160, 0xc420132160, 0xc4204e3f48)
        /work/quorum/build/_workspace/src/github.com/ethereum/go-ethereum/vendor/gopkg.in/urfave/cli.v1/app.go:490 +0xc8
github.com/ethereum/go-ethereum/vendor/gopkg.in/urfave/cli%2ev1.(*App).Run(0xc4200fe4e0, 0xc4200301c0, 0x1c, 0x1c, 0x0, 0x0)
        /work/quorum/build/_workspace/src/github.com/ethereum/go-ethereum/vendor/gopkg.in/urfave/cli.v1/app.go:264 +0x5ee
main.main()
        /work/quorum/build/_workspace/src/github.com/ethereum/go-ethereum/cmd/geth/main.go:248 +0x55

goroutine 6 [chan receive, 1 minutes]:
github.com/ethereum/go-ethereum/vendor/github.com/rjeczalik/notify.(*nonrecursiveTree).dispatch(0xc4200322a0, 0xc4200321e0)
        /work/quorum/build/_workspace/src/github.com/ethereum/go-ethereum/vendor/github.com/rjeczalik/notify/tree_nonrecursive.go:36 +0xb6
created by github.com/ethereum/go-ethereum/vendor/github.com/rjeczalik/notify.newNonrecursiveTree
        /work/quorum/build/_workspace/src/github.com/ethereum/go-ethereum/vendor/github.com/rjeczalik/notify/tree_nonrecursive.go:29 +0xf2

goroutine 7 [chan receive, 1 minutes]:
github.com/ethereum/go-ethereum/vendor/github.com/rjeczalik/notify.(*nonrecursiveTree).internal(0xc4200322a0, 0xc420032240)
        /work/quorum/build/_workspace/src/github.com/ethereum/go-ethereum/vendor/github.com/rjeczalik/notify/tree_nonrecursive.go:81 +0x7b
created by github.com/ethereum/go-ethereum/vendor/github.com/rjeczalik/notify.newNonrecursiveTree
        /work/quorum/build/_workspace/src/github.com/ethereum/go-ethereum/vendor/github.com/rjeczalik/notify/tree_nonrecursive.go:30 +0x11e

goroutine 8 [syscall, 1 minutes]:
os/signal.signal_recv(0x0)
        /usr/local/go/src/runtime/sigqueue.go:139 +0xa6
os/signal.loop()
        /usr/local/go/src/os/signal/signal_unix.go:22 +0x22
created by os/signal.init.0
        /usr/local/go/src/os/signal/signal_unix.go:28 +0x41

goroutine 9 [chan receive]:
github.com/ethereum/go-ethereum/core.(*txSenderCacher).cache(0xc420231440)
        /work/quorum/build/_workspace/src/github.com/ethereum/go-ethereum/core/tx_cacher.go:63 +0x97
created by github.com/ethereum/go-ethereum/core.newTxSenderCacher
        /work/quorum/build/_workspace/src/github.com/ethereum/go-ethereum/core/tx_cacher.go:55 +0xa4

and go on...

Thanks.

[jbhurat]

Hi @pccr10001, I tried to reproduce the issue you are having by updating the 7 nodes example, but so far I am unable to...I tried to mimic your setup as much as I could. I am running public transactions and for 100 threads and 10 loops, I had all success. Can you provide more information on your setup like, are you executing private contract and private transactions. If yes, how many nodes are party to the transaction.

[pccr10001]

Hi @jbhurat .

I didn't use private contract and private transaction.

This is one of my startup command.
GETH_ARGS="--datadir /qdata/dd --gcmode=archive --rpcport 62103 --port 62203 --identity master-3 --syncmode full --mine --rpc --rpcaddr 0.0.0.0 --rpcvhosts=* --rpcapi admin,db,eth,debug,miner,net,shh,txpool,personal,web3,quorum,clique,raft,istanbul --unlock 0 --password /qdata/passwords.txt --networkid 10 --bootnodes enode://....."

And the memory is enough for geth. (64G)

Thank you.

[pccr10001]

I followed this PR on go-ethereum #16754 and my geth didn't crash again, but it is getting very slow on syncing block.

[jbhurat]

Hi @pccr10001, looks like the issue occurs when setting config.clique.period to 0. Is there any reason you are using block period as 0. Can you try with value 5 for config.clique.period. When configuring the genesis file using puppeth the default block period is 15.

[pccr10001]

Hi @jbhurat
I just want to make clique engine to seal blocks when transactions been sent to save disk space, so I set period to 0.
Is there any better advice for this situation?

[jbhurat]

Hi @pccr10001, this PR ethereum/go-ethereum#17173 seems to have fixed this issue. I no longer see fatal error: concurrent map iteration and map write error after applying the fix

[jbhurat]

We recently merged go-ethereum version 1.8.18 changes in Quorum and there are a lot of code changes around this and noticed fatal error: concurrent map iteration and map write error as well, but at a different place

[jbhurat]

Hi @pccr10001, as mentioned in the above comment ethereum/go-ethereum#17173 fixes this issue, as the code around this has changed quite a bit, the below patch works fine.

diff --git a/miner/worker.go b/miner/worker.go
index 0d2eb420a..93b897d26 100644
--- a/miner/worker.go
+++ b/miner/worker.go
@@ -605,7 +605,9 @@ func (w *worker) resultLoop() {
                        allReceipts := mergeReceipts(work.receipts, work.privateReceipts)

                        // Commit block and state to database.
+                       w.mu.Lock()
                        stat, err := w.chain.WriteBlockWithState(block, allReceipts, work.state, nil)
+                       w.mu.Unlock()
                        if err != nil {
                                log.Error("Failed writWriteBlockAndStating block to chain", "err", err)
                                continue

Can you please let me know if you still see this after making the above changes.

[pccr10001]

Sorry for a late reply.
I will try it again.
Thank you.

[hello2mao]

+1

Use ethereum/go-ethereum#17173 to fix this.
But the block syncing is sometimes slow, and I don't know why.

@pccr10001 Hope you can try it.

[jbhurat]

Hi @pccr10001, you can also use #684 to test this. Do let me know if you see any issues

[pccr10001]

HI @jbhurat
I tested PR#653 and it's still crash due to concurrent map iteration and map write, but it got better (1 node crashed) than previous version(2 or more nodes crashed) without this patch.
I think it still somewhere need to lock the state db, and I will keep testing.

Thanks

[jbhurat]

Hi @pccr10001, with PR #684 I am no longer able to reproduce concurrent map iteration and map write panic. Are you still using the above clique configuration? Can you also paste the error stack here

[jbhurat]

Hi @pccr10001, any updates on the above comment

[pccr10001]

Hello @jbhurat I still keep trying on testing the environment caused crash. sorry for late reply.

[pccr10001]

Hi @jbhurat

I still got crashed.

error report:

fatal error: concurrent map read and map write

goroutine 148 [running]:
runtime.throw(0x10ab396, 0x21)
        /usr/local/go/src/runtime/panic.go:616 +0x81 fp=0xc42a660c80 sp=0xc42a660c60 pc=0x440811
runtime.mapaccess2(0xf65d60, 0xc4292645d0, 0xc42a660d88, 0xc6658caab400868e, 0x9cd16be37ffe4bb6)
        /usr/local/go/src/runtime/hashmap.go:409 +0x225 fp=0xc42a660cc8 sp=0xc42a660c80 pc=0x41d195
github.com/ethereum/go-ethereum/core/state.(*stateObject).GetCommittedState(0xc428c269a0, 0x123c2a0, 0xc4203339b0, 0x1af85fc1fbac358a, 0xf209d74f347dae39, 0xc6658caab400868e, 0x9cd16be37ffe4bb6, 0x0, 0x0, 0x0, ...)
        /work/quorum/build/_workspace/src/github.com/ethereum/go-ethereum/core/state/state_object.go:182 +0xce fp=0xc42a660dc0 sp=0xc42a660cc8 pc=0x7150ce
github.com/ethereum/go-ethereum/core/state.(*stateObject).GetState(0xc428c269a0, 0x123c2a0, 0xc4203339b0, 0x1af85fc1fbac358a, 0xf209d74f347dae39, 0xc6658caab400868e, 0x9cd16be37ffe4bb6, 0x0, 0x0, 0x0, ...)
        /work/quorum/build/_workspace/src/github.com/ethereum/go-ethereum/core/state/state_object.go:176 +0x15a fp=0xc42a660ea8 sp=0xc42a660dc0 pc=0x714f9a
github.com/ethereum/go-ethereum/core/state.(*stateObject).SetState(0xc428c269a0, 0x123c2a0, 0xc4203339b0, 0x1af85fc1fbac358a, 0xf209d74f347dae39, 0xc6658caab400868e, 0x9cd16be37ffe4bb6, 0x4446383739313541, 0x3942344138453136, 0x3332393031423341, ...)
        /work/quorum/build/_workspace/src/github.com/ethereum/go-ethereum/core/state/state_object.go:206 +0x7d fp=0xc42a661020 sp=0xc42a660ea8 pc=0x71546d
github.com/ethereum/go-ethereum/core/state.(*StateDB).SetState(0xc42341d6c0, 0xde5dabd82be16fb, 0x5dee6a249637d29b, 0xfbac358a18c15334, 0x347dae391af85fc1, 0xb400868ef209d74f, 0x7ffe4bb6c6658caa, 0x393135419cd16be3, 0x3845313644463837, 0x3142334139423441, ...)
        /work/quorum/build/_workspace/src/github.com/ethereum/go-ethereum/core/state/statedb.go:368 +0xb6 fp=0xc42a661088 sp=0xc42a661020 pc=0x718e46
github.com/ethereum/go-ethereum/core/vm.opSstore(0xc429f350a0, 0xc428c0ca80, 0xc422478480, 0xc42926c6c0, 0xc42926c6e0, 0x2bc, 0x1388, 0x32, 0x61a8, 0xc428a0ea80)
        /work/quorum/build/_workspace/src/github.com/ethereum/go-ethereum/core/vm/instructions.go:647 +0x1e3 fp=0xc42a661188 sp=0xc42a661088 pc=0x90fd83
github.com/ethereum/go-ethereum/core/vm.(*EVMInterpreter).Run(0xc428c0ca80, 0xc422478480, 0x0, 0x0, 0x0, 0x41ca00, 0x0, 0x0, 0x0, 0x0, ...)
        /work/quorum/build/_workspace/src/github.com/ethereum/go-ethereum/core/vm/interpreter.go:263 +0x5d6 fp=0xc42a661350 sp=0xc42a661188 pc=0x914636
github.com/ethereum/go-ethereum/core/vm.run(0xc428a0ea80, 0xc422478480, 0x0, 0x0, 0x0, 0x5f14200, 0x0, 0x0, 0x0, 0x0, ...)
        /work/quorum/build/_workspace/src/github.com/ethereum/go-ethereum/core/vm/evm.go:76 +0x1f5 fp=0xc42a6613f0 sp=0xc42a661350 pc=0x900e15
github.com/ethereum/go-ethereum/core/vm.(*EVM).create(0xc428a0ea80, 0x122dc60, 0xc42926e2a0, 0xc42a661700, 0x5f1424c, 0xc42806b060, 0xde5dabd82be16fb, 0x5dee6a249637d29b, 0xc418c15334, 0x99cdaa, ...)
        /work/quorum/build/_workspace/src/github.com/ethereum/go-ethereum/core/vm/evm.go:488 +0x6d6 fp=0xc42a661608 sp=0xc42a6613f0 pc=0x904286
github.com/ethereum/go-ethereum/core/vm.(*EVM).Create(0xc428a0ea80, 0x122dc60, 0xc42926e2a0, 0xc429144c00, 0x12cb, 0x12cb, 0x5f1424c, 0xc42806b060, 0xc42026e020, 0x11230b8, ...)
        /work/quorum/build/_workspace/src/github.com/ethereum/go-ethereum/core/vm/evm.go:546 +0x257 fp=0xc42a661748 sp=0xc42a661608 pc=0x904e17
github.com/ethereum/go-ethereum/core.(*StateTransition).TransitionDb(0xc425a09b90, 0x123e040, 0xc42467b560, 0xc429f35028, 0xc425a09b90, 0xc42467b560, 0xc42467b560, 0x105e840)
        /work/quorum/build/_workspace/src/github.com/ethereum/go-ethereum/core/state_transition.go:241 +0x9cd fp=0xc42a6619e8 sp=0xc42a661748 pc=0x99d94d
github.com/ethereum/go-ethereum/core.ApplyMessage(0xc428a0ea80, 0x123e040, 0xc42467b560, 0xc429f35028, 0x6763f60703594a1, 0x85511a3, 0xc42926c560, 0x77b892d3097239f9, 0xfef9cebfff411081, 0xc2f32b3e, ...)
        /work/quorum/build/_workspace/src/github.com/ethereum/go-ethereum/core/state_transition.go:140 +0x5b fp=0xc42a661a38 sp=0xc42a6619e8 pc=0x99ca1b
github.com/ethereum/go-ethereum/core.ApplyTransaction(0xc420414000, 0xc42029a000, 0xc42a667410, 0xc429f35028, 0xc42341d6c0, 0xc42341d790, 0xc42925cb40, 0xc429d174d0, 0xc42925cd10, 0x0, ...)
        /work/quorum/build/_workspace/src/github.com/ethereum/go-ethereum/core/state_processor.go:120 +0x4ea fp=0xc42a664aa0 sp=0xc42a661a38 pc=0x99b98a
github.com/ethereum/go-ethereum/miner.(*worker).commitTransaction(0xc4200316c0, 0xc429d174d0, 0x77b892d3097239f9, 0xfef9cebfff411081, 0x238d7047c2f32b3e, 0x85511a3, 0x0, 0x0, 0x2, 0x0)
        /work/quorum/build/_workspace/src/github.com/ethereum/go-ethereum/miner/worker.go:754 +0x1b4 fp=0xc42a667400 sp=0xc42a664aa0 pc=0xb17734
github.com/ethereum/go-ethereum/miner.(*worker).commitTransactions(0xc4200316c0, 0xc4292644e0, 0x77b892d3097239f9, 0xfef9cebfff411081, 0xc2f32b3e, 0x0, 0xbf2d1492a1fd9d97)
        /work/quorum/build/_workspace/src/github.com/ethereum/go-ethereum/miner/worker.go:831 +0x3ce fp=0xc42a667848 sp=0xc42a667400 pc=0xb1819e
github.com/ethereum/go-ethereum/miner.(*worker).commitNewWork(0xc4200316c0, 0x0, 0x1d4e800, 0x5cd3dacb)
        /work/quorum/build/_workspace/src/github.com/ethereum/go-ethereum/miner/worker.go:999 +0xd86 fp=0xc42a667c10 sp=0xc42a667848 pc=0xb19d96
github.com/ethereum/go-ethereum/miner.(*worker).mainLoop(0xc4200316c0)
        /work/quorum/build/_workspace/src/github.com/ethereum/go-ethereum/miner/worker.go:493 +0x462 fp=0xc42a667fd8 sp=0xc42a667c10 pc=0xb13f42
runtime.goexit()
        /usr/local/go/src/runtime/asm_amd64.s:2361 +0x1 fp=0xc42a667fe0 sp=0xc42a667fd8 pc=0x46ff71
created by github.com/ethereum/go-ethereum/miner.newWorker
        /work/quorum/build/_workspace/src/github.com/ethereum/go-ethereum/miner/worker.go:228 +0x61b

goroutine 1 [chan receive, 2 minutes]:
github.com/ethereum/go-ethereum/node.(*Node).Wait(0xc420249180)
        /work/quorum/build/_workspace/src/github.com/ethereum/go-ethereum/node/node.go:466 +0x7f
main.geth(0xc420328f20, 0x0, 0x0)
        /work/quorum/build/_workspace/src/github.com/ethereum/go-ethereum/cmd/geth/main.go:279 +0x12e
github.com/ethereum/go-ethereum/vendor/gopkg.in/urfave/cli%2ev1.HandleAction(0xedba20, 0x11250f8, 0xc420328f20, 0xc420328f20, 0xc4204fff48)
        /work/quorum/build/_workspace/src/github.com/ethereum/go-ethereum/vendor/gopkg.in/urfave/cli.v1/app.go:490 +0xc8
github.com/ethereum/go-ethereum/vendor/gopkg.in/urfave/cli%2ev1.(*App).Run(0xc420110680, 0xc420102000, 0x1c, 0x1c, 0x0, 0x0)
        /work/quorum/build/_workspace/src/github.com/ethereum/go-ethereum/vendor/gopkg.in/urfave/cli.v1/app.go:264 +0x5ee
main.main()
        /work/quorum/build/_workspace/src/github.com/ethereum/go-ethereum/cmd/geth/main.go:260 +0x55

and

fatal error: concurrent map iteration and map write

goroutine 141 [running]:
runtime.throw(0x10b0fac, 0x26)
        /usr/local/go/src/runtime/panic.go:616 +0x81 fp=0xc460d65338 sp=0xc460d65318 pc=0x440811
runtime.mapiternext(0xc460d65540)
        /usr/local/go/src/runtime/hashmap.go:747 +0x55c fp=0xc460d653c8 sp=0xc460d65338 pc=0x41e4cc
github.com/ethereum/go-ethereum/core/state.(*StateDB).Commit(0xc428a485b0, 0xc460d65901, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
        /work/quorum/build/_workspace/src/github.com/ethereum/go-ethereum/core/state/statedb.go:637 +0x209 fp=0xc460d65610 sp=0xc460d653c8 pc=0x71b5e9
github.com/ethereum/go-ethereum/core.(*BlockChain).WriteBlockWithState(0xc4385fa000, 0xc45b73dcb0, 0xc420414600, 0x222, 0x222, 0xc428a485b0, 0x0, 0x200, 0x0, 0x0)
        /work/quorum/build/_workspace/src/github.com/ethereum/go-ethereum/core/blockchain.go:949 +0x5e7 fp=0xc460d65a50 sp=0xc460d65610 pc=0x9826f7
github.com/ethereum/go-ethereum/miner.(*worker).resultLoop(0xc420031880)
        /work/quorum/build/_workspace/src/github.com/ethereum/go-ethereum/miner/worker.go:608 +0x842 fp=0xc460d65fd8 sp=0xc460d65a50 pc=0xb15562
runtime.goexit()
        /usr/local/go/src/runtime/asm_amd64.s:2361 +0x1 fp=0xc460d65fe0 sp=0xc460d65fd8 pc=0x46ff71
created by github.com/ethereum/go-ethereum/miner.newWorker
        /work/quorum/build/_workspace/src/github.com/ethereum/go-ethereum/miner/worker.go:230 +0x66c

goroutine 1 [chan receive, 16 minutes]:
github.com/ethereum/go-ethereum/node.(*Node).Wait(0xc420229900)
        /work/quorum/build/_workspace/src/github.com/ethereum/go-ethereum/node/node.go:466 +0x7f
main.geth(0xc420134160, 0x0, 0x0)
        /work/quorum/build/_workspace/src/github.com/ethereum/go-ethereum/cmd/geth/main.go:279 +0x12e
github.com/ethereum/go-ethereum/vendor/gopkg.in/urfave/cli%2ev1.HandleAction(0xedba20, 0x11250f8, 0xc420134160, 0xc420134160, 0xc4204eff48)
        /work/quorum/build/_workspace/src/github.com/ethereum/go-ethereum/vendor/gopkg.in/urfave/cli.v1/app.go:490 +0xc8
github.com/ethereum/go-ethereum/vendor/gopkg.in/urfave/cli%2ev1.(*App).Run(0xc420362b60, 0xc4200301c0, 0x1c, 0x1c, 0x0, 0x0)
        /work/quorum/build/_workspace/src/github.com/ethereum/go-ethereum/vendor/gopkg.in/urfave/cli.v1/app.go:264 +0x5ee
main.main()
        /work/quorum/build/_workspace/src/github.com/ethereum/go-ethereum/cmd/geth/main.go:260 +0x55

goroutine 6 [chan receive, 16 minutes]:
github.com/ethereum/go-ethereum/vendor/github.com/rjeczalik/notify.(*nonrecursiveTree).dispatch(0xc4200ca1e0, 0xc4200ca120)
        /work/quorum/build/_workspace/src/github.com/ethereum/go-ethereum/vendor/github.com/rjeczalik/notify/tree_nonrecursive.go:36 +0xb6
created by github.com/ethereum/go-ethereum/vendor/github.com/rjeczalik/notify.newNonrecursiveTree
        /work/quorum/build/_workspace/src/github.com/ethereum/go-ethereum/vendor/github.com/rjeczalik/notify/tree_nonrecursive.go:29 +0xf2

goroutine 7 [chan receive, 16 minutes]:
github.com/ethereum/go-ethereum/vendor/github.com/rjeczalik/notify.(*nonrecursiveTree).internal(0xc4200ca1e0, 0xc4200ca180)
        /work/quorum/build/_workspace/src/github.com/ethereum/go-ethereum/vendor/github.com/rjeczalik/notify/tree_nonrecursive.go:81 +0x7b
created by github.com/ethereum/go-ethereum/vendor/github.com/rjeczalik/notify.newNonrecursiveTree
        /work/quorum/build/_workspace/src/github.com/ethereum/go-ethereum/vendor/github.com/rjeczalik/notify/tree_nonrecursive.go:30 +0x11e

goroutine 8 [syscall, 16 minutes]:
os/signal.signal_recv(0x0)
        /usr/local/go/src/runtime/sigqueue.go:139 +0xa6
os/signal.loop()
        /usr/local/go/src/os/signal/signal_unix.go:22 +0x22
created by os/signal.init.0
        /usr/local/go/src/os/signal/signal_unix.go:28 +0x41

goroutine 9 [chan receive]:
github.com/ethereum/go-ethereum/metrics.(*meterArbiter).tick(0x1d48d60)
        /work/quorum/build/_workspace/src/github.com/ethereum/go-ethereum/metrics/meter.go:289 +0x78
created by github.com/ethereum/go-ethereum/metrics.NewMeterForced
        /work/quorum/build/_workspace/src/github.com/ethereum/go-ethereum/metrics/meter.go:70 +0xfc

goroutine 10 [select]:
github.com/ethereum/go-ethereum/consensus/ethash.(*Ethash).remote(0xc420256000, 0x0, 0x0, 0x0, 0x0)
        /work/quorum/build/_workspace/src/github.com/ethereum/go-ethereum/consensus/ethash/sealer.go:307 +0x5fb
created by github.com/ethereum/go-ethereum/consensus/ethash.New
        /work/quorum/build/_workspace/src/github.com/ethereum/go-ethereum/consensus/ethash/ethash.go:496 +0x387

and sometimes got BAD BLOCK Error

########## BAD BLOCK #########
Chain config: {ChainID: 10 Homestead: <nil> DAO: <nil> DAOSupport: false EIP150: 1 EIP155: 1 EIP158: 1 Byzantium: 1 IsQuorum: true Constantinople: 1 Engine: clique}

Number: 32
Hash: 0x258b27bd193f9cbf483dd015a7de68f64be144299026b0fbc94d92775802db5f
        &{[] 1 227525 [0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0] [] [61 242 88 18 152 49 62 192 157 148 61 232 221 99 245 75 101 164 65 132 130 196 122 45 241 156 36 240 249 25 197 16] [87 244 176 142 11 60 197 169 203 200 136 99 157 190 33 113 177 64 135 34] 227525}
        &{[] 1 1651682 [0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0] [] [51 45 98 168 108 131 142 43 149 158 83 88 146 123 210 46 82 73 54 91 3 86 160 2 136 104 188 223 178 76 136 163] [9 20 10 187 25 234 218 17 254 15 44 107 249 119 170 178 255 191 221 252] 1424157}


Error: invalid merkle root (remote: ef615cb8bbc1d028253affeaf5f282304c96c42a22001c6b2fd090a8d2cc70c6 local: 53228d809a9cd20d8af3131e85adea4d928599cc0197a80719aedeba993ff561)
##############################

I tested it with block period = 0 and = 1, without and differents.

[jbhurat]

Hi @pccr10001, I haven't been able to reproduce the issue after apply PR #684 with the above genesis configuration. Are you still testing using the same configuration?

[pccr10001]

Hi @jbhurat
I reinstalled everything and built them again, No crashed anymore.
Thank you for your help.