From 8698fbabf6c7811e8eec6e84512510e9c9a4eb45 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Sat, 24 Nov 2018 23:22:25 +0100 Subject: [PATCH 01/22] cmd/puppeth: implement chainspec converters --- cmd/puppeth/genesis.go | 386 ++++++++++++++--------- cmd/puppeth/module_dashboard.go | 2 +- cmd/puppeth/puppeth.go | 32 ++ cmd/puppeth/puppeth_test.go | 91 ++++++ cmd/puppeth/testdata/stureby_aleth.json | 112 +++++++ cmd/puppeth/testdata/stureby_geth.json | 47 +++ cmd/puppeth/testdata/stureby_parity.json | 181 +++++++++++ cmd/puppeth/wizard.go | 22 ++ cmd/puppeth/wizard_genesis.go | 67 +++- 9 files changed, 779 insertions(+), 161 deletions(-) create mode 100644 cmd/puppeth/puppeth_test.go create mode 100644 cmd/puppeth/testdata/stureby_aleth.json create mode 100644 cmd/puppeth/testdata/stureby_geth.json create mode 100644 cmd/puppeth/testdata/stureby_parity.json diff --git a/cmd/puppeth/genesis.go b/cmd/puppeth/genesis.go index 5f39a889d1..83212a126b 100644 --- a/cmd/puppeth/genesis.go +++ b/cmd/puppeth/genesis.go @@ -20,35 +20,41 @@ import ( "encoding/binary" "errors" "math" + "math/big" + "strings" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" + math2 "github.com/ethereum/go-ethereum/common/math" "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/params" ) -// cppEthereumGenesisSpec represents the genesis specification format used by the +// alethGenesisSpec represents the genesis specification format used by the // C++ Ethereum implementation. -type cppEthereumGenesisSpec struct { +type alethGenesisSpec struct { SealEngine string `json:"sealEngine"` Params struct { - AccountStartNonce hexutil.Uint64 `json:"accountStartNonce"` - HomesteadForkBlock hexutil.Uint64 `json:"homesteadForkBlock"` - EIP150ForkBlock hexutil.Uint64 `json:"EIP150ForkBlock"` - EIP158ForkBlock hexutil.Uint64 `json:"EIP158ForkBlock"` - ByzantiumForkBlock hexutil.Uint64 `json:"byzantiumForkBlock"` - ConstantinopleForkBlock hexutil.Uint64 `json:"constantinopleForkBlock"` - NetworkID hexutil.Uint64 `json:"networkID"` - ChainID hexutil.Uint64 `json:"chainID"` - MaximumExtraDataSize hexutil.Uint64 `json:"maximumExtraDataSize"` - MinGasLimit hexutil.Uint64 `json:"minGasLimit"` - MaxGasLimit hexutil.Uint64 `json:"maxGasLimit"` - GasLimitBoundDivisor hexutil.Uint64 `json:"gasLimitBoundDivisor"` - MinimumDifficulty *hexutil.Big `json:"minimumDifficulty"` - DifficultyBoundDivisor *hexutil.Big `json:"difficultyBoundDivisor"` - DurationLimit *hexutil.Big `json:"durationLimit"` - BlockReward *hexutil.Big `json:"blockReward"` + AccountStartNonce math2.HexOrDecimal64 `json:"accountStartNonce"` + MaximumExtraDataSize hexutil.Uint64 `json:"maximumExtraDataSize"` + HomesteadForkBlock hexutil.Uint64 `json:"homesteadForkBlock"` + DaoHardforkBlock math2.HexOrDecimal64 `json:"daoHardforkBlock"` + EIP150ForkBlock hexutil.Uint64 `json:"EIP150ForkBlock"` + EIP158ForkBlock hexutil.Uint64 `json:"EIP158ForkBlock"` + ByzantiumForkBlock hexutil.Uint64 `json:"byzantiumForkBlock"` + ConstantinopleForkBlock hexutil.Uint64 `json:"constantinopleForkBlock"` + MinGasLimit hexutil.Uint64 `json:"minGasLimit"` + MaxGasLimit hexutil.Uint64 `json:"maxGasLimit"` + TieBreakingGas bool `json:"tieBreakingGas"` + GasLimitBoundDivisor math2.HexOrDecimal64 `json:"gasLimitBoundDivisor"` + MinimumDifficulty *hexutil.Big `json:"minimumDifficulty"` + DifficultyBoundDivisor *math2.HexOrDecimal256 `json:"difficultyBoundDivisor"` + DurationLimit *math2.HexOrDecimal256 `json:"durationLimit"` + BlockReward *hexutil.Big `json:"blockReward"` + NetworkID hexutil.Uint64 `json:"networkID"` + ChainID hexutil.Uint64 `json:"chainID"` + AllowFutureBlocks bool `json:"allowFutureBlocks""` } `json:"params"` Genesis struct { @@ -62,57 +68,68 @@ type cppEthereumGenesisSpec struct { GasLimit hexutil.Uint64 `json:"gasLimit"` } `json:"genesis"` - Accounts map[common.Address]*cppEthereumGenesisSpecAccount `json:"accounts"` + Accounts map[common.UnprefixedAddress]*alethGenesisSpecAccount `json:"accounts"` } -// cppEthereumGenesisSpecAccount is the prefunded genesis account and/or precompiled +// alethGenesisSpecAccount is the prefunded genesis account and/or precompiled // contract definition. -type cppEthereumGenesisSpecAccount struct { - Balance *hexutil.Big `json:"balance"` - Nonce uint64 `json:"nonce,omitempty"` - Precompiled *cppEthereumGenesisSpecBuiltin `json:"precompiled,omitempty"` +type alethGenesisSpecAccount struct { + Balance *math2.HexOrDecimal256 `json:"balance"` + Nonce uint64 `json:"nonce,omitempty"` + Precompiled *alethGenesisSpecBuiltin `json:"precompiled,omitempty"` } -// cppEthereumGenesisSpecBuiltin is the precompiled contract definition. -type cppEthereumGenesisSpecBuiltin struct { - Name string `json:"name,omitempty"` - StartingBlock hexutil.Uint64 `json:"startingBlock,omitempty"` - Linear *cppEthereumGenesisSpecLinearPricing `json:"linear,omitempty"` +// alethGenesisSpecBuiltin is the precompiled contract definition. +type alethGenesisSpecBuiltin struct { + Name string `json:"name,omitempty"` + StartingBlock hexutil.Uint64 `json:"startingBlock,omitempty"` + Linear *alethGenesisSpecLinearPricing `json:"linear,omitempty"` } -type cppEthereumGenesisSpecLinearPricing struct { +type alethGenesisSpecLinearPricing struct { Base uint64 `json:"base"` Word uint64 `json:"word"` } -// newCppEthereumGenesisSpec converts a go-ethereum genesis block into a Parity specific +// newAlethGenesisSpec converts a go-ethereum genesis block into a Aleth-specific // chain specification format. -func newCppEthereumGenesisSpec(network string, genesis *core.Genesis) (*cppEthereumGenesisSpec, error) { - // Only ethash is currently supported between go-ethereum and cpp-ethereum +func newAlethGenesisSpec(network string, genesis *core.Genesis) (*alethGenesisSpec, error) { + // Only ethash is currently supported between go-ethereum and aleth if genesis.Config.Ethash == nil { return nil, errors.New("unsupported consensus engine") } - // Reconstruct the chain spec in Parity's format - spec := &cppEthereumGenesisSpec{ + // Reconstruct the chain spec in Aleth format + spec := &alethGenesisSpec{ SealEngine: "Ethash", } + // Some defaults spec.Params.AccountStartNonce = 0 + spec.Params.TieBreakingGas = false + spec.Params.AllowFutureBlocks = false + spec.Params.DaoHardforkBlock = 0 + spec.Params.HomesteadForkBlock = (hexutil.Uint64)(genesis.Config.HomesteadBlock.Uint64()) spec.Params.EIP150ForkBlock = (hexutil.Uint64)(genesis.Config.EIP150Block.Uint64()) spec.Params.EIP158ForkBlock = (hexutil.Uint64)(genesis.Config.EIP158Block.Uint64()) - spec.Params.ByzantiumForkBlock = (hexutil.Uint64)(genesis.Config.ByzantiumBlock.Uint64()) - spec.Params.ConstantinopleForkBlock = (hexutil.Uint64)(math.MaxUint64) + + // Byzantium + if num := genesis.Config.ByzantiumBlock; num != nil { + spec.setByzantium(num) + } + // Constantinople + if num := genesis.Config.ConstantinopleBlock; num != nil { + spec.setConstantinople(num) + } spec.Params.NetworkID = (hexutil.Uint64)(genesis.Config.ChainID.Uint64()) spec.Params.ChainID = (hexutil.Uint64)(genesis.Config.ChainID.Uint64()) - spec.Params.MaximumExtraDataSize = (hexutil.Uint64)(params.MaximumExtraDataSize) spec.Params.MinGasLimit = (hexutil.Uint64)(params.MinGasLimit) - spec.Params.MaxGasLimit = (hexutil.Uint64)(math.MaxUint64) + spec.Params.MaxGasLimit = (hexutil.Uint64)(math.MaxInt64) spec.Params.MinimumDifficulty = (*hexutil.Big)(params.MinimumDifficulty) - spec.Params.DifficultyBoundDivisor = (*hexutil.Big)(params.DifficultyBoundDivisor) - spec.Params.GasLimitBoundDivisor = (hexutil.Uint64)(params.GasLimitBoundDivisor) - spec.Params.DurationLimit = (*hexutil.Big)(params.DurationLimit) + spec.Params.DifficultyBoundDivisor = (*math2.HexOrDecimal256)(params.DifficultyBoundDivisor) + spec.Params.GasLimitBoundDivisor = (math2.HexOrDecimal64)(params.GasLimitBoundDivisor) + spec.Params.DurationLimit = (*math2.HexOrDecimal256)(params.DurationLimit) spec.Params.BlockReward = (*hexutil.Big)(ethash.FrontierBlockReward) spec.Genesis.Nonce = (hexutil.Bytes)(make([]byte, 8)) @@ -126,77 +143,104 @@ func newCppEthereumGenesisSpec(network string, genesis *core.Genesis) (*cppEther spec.Genesis.ExtraData = (hexutil.Bytes)(genesis.ExtraData) spec.Genesis.GasLimit = (hexutil.Uint64)(genesis.GasLimit) - spec.Accounts = make(map[common.Address]*cppEthereumGenesisSpecAccount) for address, account := range genesis.Alloc { - spec.Accounts[address] = &cppEthereumGenesisSpecAccount{ - Balance: (*hexutil.Big)(account.Balance), - Nonce: account.Nonce, - } + spec.setAccount(address, account) } - spec.Accounts[common.BytesToAddress([]byte{1})].Precompiled = &cppEthereumGenesisSpecBuiltin{ - Name: "ecrecover", Linear: &cppEthereumGenesisSpecLinearPricing{Base: 3000}, - } - spec.Accounts[common.BytesToAddress([]byte{2})].Precompiled = &cppEthereumGenesisSpecBuiltin{ - Name: "sha256", Linear: &cppEthereumGenesisSpecLinearPricing{Base: 60, Word: 12}, + + spec.setPrecompile(1, &alethGenesisSpecBuiltin{Name: "ecrecover", + Linear: &alethGenesisSpecLinearPricing{Base: 3000}}) + spec.setPrecompile(2, &alethGenesisSpecBuiltin{Name: "sha256", + Linear: &alethGenesisSpecLinearPricing{Base: 60, Word: 12}}) + spec.setPrecompile(3, &alethGenesisSpecBuiltin{Name: "ripemd160", + Linear: &alethGenesisSpecLinearPricing{Base: 600, Word: 120}}) + spec.setPrecompile(4, &alethGenesisSpecBuiltin{Name: "identity", + Linear: &alethGenesisSpecLinearPricing{Base: 15, Word: 3}}) + if genesis.Config.ByzantiumBlock != nil { + spec.setPrecompile(5, &alethGenesisSpecBuiltin{Name: "modexp", + StartingBlock: (hexutil.Uint64)(genesis.Config.ByzantiumBlock.Uint64())}) + spec.setPrecompile(6, &alethGenesisSpecBuiltin{Name: "alt_bn128_G1_add", + StartingBlock: (hexutil.Uint64)(genesis.Config.ByzantiumBlock.Uint64()), + Linear: &alethGenesisSpecLinearPricing{Base: 500}}) + spec.setPrecompile(7, &alethGenesisSpecBuiltin{Name: "alt_bn128_G1_mul", + StartingBlock: (hexutil.Uint64)(genesis.Config.ByzantiumBlock.Uint64()), + Linear: &alethGenesisSpecLinearPricing{Base: 40000}}) + spec.setPrecompile(8, &alethGenesisSpecBuiltin{Name: "alt_bn128_pairing_product", + StartingBlock: (hexutil.Uint64)(genesis.Config.ByzantiumBlock.Uint64())}) } - spec.Accounts[common.BytesToAddress([]byte{3})].Precompiled = &cppEthereumGenesisSpecBuiltin{ - Name: "ripemd160", Linear: &cppEthereumGenesisSpecLinearPricing{Base: 600, Word: 120}, + return spec, nil +} + +func (spec *alethGenesisSpec) setPrecompile(address byte, data *alethGenesisSpecBuiltin) { + if spec.Accounts == nil { + spec.Accounts = make(map[common.UnprefixedAddress]*alethGenesisSpecAccount) } - spec.Accounts[common.BytesToAddress([]byte{4})].Precompiled = &cppEthereumGenesisSpecBuiltin{ - Name: "identity", Linear: &cppEthereumGenesisSpecLinearPricing{Base: 15, Word: 3}, + spec.Accounts[common.UnprefixedAddress(common.BytesToAddress([]byte{address}))].Precompiled = data +} + +func (spec *alethGenesisSpec) setAccount(address common.Address, account core.GenesisAccount) { + if spec.Accounts == nil { + spec.Accounts = make(map[common.UnprefixedAddress]*alethGenesisSpecAccount) } - if genesis.Config.ByzantiumBlock != nil { - spec.Accounts[common.BytesToAddress([]byte{5})].Precompiled = &cppEthereumGenesisSpecBuiltin{ - Name: "modexp", StartingBlock: (hexutil.Uint64)(genesis.Config.ByzantiumBlock.Uint64()), - } - spec.Accounts[common.BytesToAddress([]byte{6})].Precompiled = &cppEthereumGenesisSpecBuiltin{ - Name: "alt_bn128_G1_add", StartingBlock: (hexutil.Uint64)(genesis.Config.ByzantiumBlock.Uint64()), Linear: &cppEthereumGenesisSpecLinearPricing{Base: 500}, - } - spec.Accounts[common.BytesToAddress([]byte{7})].Precompiled = &cppEthereumGenesisSpecBuiltin{ - Name: "alt_bn128_G1_mul", StartingBlock: (hexutil.Uint64)(genesis.Config.ByzantiumBlock.Uint64()), Linear: &cppEthereumGenesisSpecLinearPricing{Base: 40000}, - } - spec.Accounts[common.BytesToAddress([]byte{8})].Precompiled = &cppEthereumGenesisSpecBuiltin{ - Name: "alt_bn128_pairing_product", StartingBlock: (hexutil.Uint64)(genesis.Config.ByzantiumBlock.Uint64()), - } + + a, exist := spec.Accounts[common.UnprefixedAddress(address)] + if !exist { + a = &alethGenesisSpecAccount{} + spec.Accounts[common.UnprefixedAddress(address)] = a } - return spec, nil + a.Balance = (*math2.HexOrDecimal256)(account.Balance) + a.Nonce = account.Nonce + +} + +func (spec *alethGenesisSpec) setByzantium(num *big.Int) { + spec.Params.ByzantiumForkBlock = hexutil.Uint64(num.Uint64()) +} + +func (spec *alethGenesisSpec) setConstantinople(num *big.Int) { + spec.Params.ConstantinopleForkBlock = hexutil.Uint64(num.Uint64()) } // parityChainSpec is the chain specification format used by Parity. type parityChainSpec struct { - Name string `json:"name"` - Engine struct { + Name string `json:"name"` + Datadir string `json:"dataDir"` + Engine struct { Ethash struct { Params struct { - MinimumDifficulty *hexutil.Big `json:"minimumDifficulty"` - DifficultyBoundDivisor *hexutil.Big `json:"difficultyBoundDivisor"` - DurationLimit *hexutil.Big `json:"durationLimit"` - BlockReward *hexutil.Big `json:"blockReward"` - HomesteadTransition uint64 `json:"homesteadTransition"` - EIP150Transition uint64 `json:"eip150Transition"` - EIP160Transition uint64 `json:"eip160Transition"` - EIP161abcTransition uint64 `json:"eip161abcTransition"` - EIP161dTransition uint64 `json:"eip161dTransition"` - EIP649Reward *hexutil.Big `json:"eip649Reward"` - EIP100bTransition uint64 `json:"eip100bTransition"` - EIP649Transition uint64 `json:"eip649Transition"` + MinimumDifficulty *hexutil.Big `json:"minimumDifficulty"` + DifficultyBoundDivisor *hexutil.Big `json:"difficultyBoundDivisor"` + DurationLimit *hexutil.Big `json:"durationLimit"` + BlockReward map[string]string `json:"blockReward"` + DifficultyBombDelays map[string]string `json:"difficultyBombDelays"` + HomesteadTransition hexutil.Uint64 `json:"homesteadTransition"` + EIP100bTransition hexutil.Uint64 `json:"eip100bTransition"` } `json:"params"` } `json:"Ethash"` } `json:"engine"` Params struct { - MaximumExtraDataSize hexutil.Uint64 `json:"maximumExtraDataSize"` - MinGasLimit hexutil.Uint64 `json:"minGasLimit"` - GasLimitBoundDivisor hexutil.Uint64 `json:"gasLimitBoundDivisor"` - NetworkID hexutil.Uint64 `json:"networkID"` - MaxCodeSize uint64 `json:"maxCodeSize"` - EIP155Transition uint64 `json:"eip155Transition"` - EIP98Transition uint64 `json:"eip98Transition"` - EIP86Transition uint64 `json:"eip86Transition"` - EIP140Transition uint64 `json:"eip140Transition"` - EIP211Transition uint64 `json:"eip211Transition"` - EIP214Transition uint64 `json:"eip214Transition"` - EIP658Transition uint64 `json:"eip658Transition"` + AccountStartNonce hexutil.Uint64 `json:"accountStartNonce"` + MaximumExtraDataSize hexutil.Uint64 `json:"maximumExtraDataSize"` + MinGasLimit hexutil.Uint64 `json:"minGasLimit"` + GasLimitBoundDivisor math2.HexOrDecimal64 `json:"gasLimitBoundDivisor"` + NetworkID hexutil.Uint64 `json:"networkID"` + ChainID hexutil.Uint64 `json:"chainID"` + MaxCodeSize hexutil.Uint64 `json:"maxCodeSize"` + MaxCodeSizeTransition hexutil.Uint64 `json:"maxCodeSizeTransition"` + EIP98Transition hexutil.Uint64 `json:"eip98Transition"` + EIP150Transition hexutil.Uint64 `json:"eip150Transition"` + EIP160Transition hexutil.Uint64 `json:"eip160Transition"` + EIP161abcTransition hexutil.Uint64 `json:"eip161abcTransition"` + EIP161dTransition hexutil.Uint64 `json:"eip161dTransition"` + EIP155Transition hexutil.Uint64 `json:"eip155Transition"` + EIP140Transition hexutil.Uint64 `json:"eip140Transition"` + EIP211Transition hexutil.Uint64 `json:"eip211Transition"` + EIP214Transition hexutil.Uint64 `json:"eip214Transition"` + EIP658Transition hexutil.Uint64 `json:"eip658Transition"` + EIP145Transition hexutil.Uint64 `json:"eip145Transition"` + EIP1014Transition hexutil.Uint64 `json:"eip1014Transition"` + EIP1052Transition hexutil.Uint64 `json:"eip1052Transition"` + EIP1283Transition hexutil.Uint64 `json:"eip1283Transition"` } `json:"params"` Genesis struct { @@ -215,22 +259,22 @@ type parityChainSpec struct { GasLimit hexutil.Uint64 `json:"gasLimit"` } `json:"genesis"` - Nodes []string `json:"nodes"` - Accounts map[common.Address]*parityChainSpecAccount `json:"accounts"` + Nodes []string `json:"nodes"` + Accounts map[common.UnprefixedAddress]*parityChainSpecAccount `json:"accounts"` } // parityChainSpecAccount is the prefunded genesis account and/or precompiled // contract definition. type parityChainSpecAccount struct { - Balance *hexutil.Big `json:"balance"` - Nonce uint64 `json:"nonce,omitempty"` + Balance math2.HexOrDecimal256 `json:"balance"` + Nonce math2.HexOrDecimal64 `json:"nonce,omitempty"` Builtin *parityChainSpecBuiltin `json:"builtin,omitempty"` } // parityChainSpecBuiltin is the precompiled contract definition. type parityChainSpecBuiltin struct { Name string `json:"name,omitempty"` - ActivateAt uint64 `json:"activate_at,omitempty"` + ActivateAt math2.HexOrDecimal64 `json:"activate_at,omitempty"` Pricing *parityChainSpecPricing `json:"pricing,omitempty"` } @@ -265,34 +309,51 @@ func newParityChainSpec(network string, genesis *core.Genesis, bootnodes []strin } // Reconstruct the chain spec in Parity's format spec := &parityChainSpec{ - Name: network, - Nodes: bootnodes, + Name: network, + Nodes: bootnodes, + Datadir: strings.ToLower(network), } + spec.Engine.Ethash.Params.BlockReward = make(map[string]string) + spec.Engine.Ethash.Params.DifficultyBombDelays = make(map[string]string) + // Frontier spec.Engine.Ethash.Params.MinimumDifficulty = (*hexutil.Big)(params.MinimumDifficulty) spec.Engine.Ethash.Params.DifficultyBoundDivisor = (*hexutil.Big)(params.DifficultyBoundDivisor) spec.Engine.Ethash.Params.DurationLimit = (*hexutil.Big)(params.DurationLimit) - spec.Engine.Ethash.Params.BlockReward = (*hexutil.Big)(ethash.FrontierBlockReward) - spec.Engine.Ethash.Params.HomesteadTransition = genesis.Config.HomesteadBlock.Uint64() - spec.Engine.Ethash.Params.EIP150Transition = genesis.Config.EIP150Block.Uint64() - spec.Engine.Ethash.Params.EIP160Transition = genesis.Config.EIP155Block.Uint64() - spec.Engine.Ethash.Params.EIP161abcTransition = genesis.Config.EIP158Block.Uint64() - spec.Engine.Ethash.Params.EIP161dTransition = genesis.Config.EIP158Block.Uint64() - spec.Engine.Ethash.Params.EIP649Reward = (*hexutil.Big)(ethash.ByzantiumBlockReward) - spec.Engine.Ethash.Params.EIP100bTransition = genesis.Config.ByzantiumBlock.Uint64() - spec.Engine.Ethash.Params.EIP649Transition = genesis.Config.ByzantiumBlock.Uint64() + spec.Engine.Ethash.Params.BlockReward["0x0"] = hexutil.EncodeBig(ethash.FrontierBlockReward) + + // Homestead + spec.Engine.Ethash.Params.HomesteadTransition = hexutil.Uint64(genesis.Config.HomesteadBlock.Uint64()) + + // Tangerine Whistle : 150 + // https://github.com/ethereum/EIPs/blob/master/EIPS/eip-608.md + spec.Params.EIP150Transition = hexutil.Uint64(genesis.Config.EIP150Block.Uint64()) + + // Spurious Dragon: 155, 160, 161, 170 + // https://github.com/ethereum/EIPs/blob/master/EIPS/eip-607.md + spec.Params.EIP155Transition = hexutil.Uint64(genesis.Config.EIP155Block.Uint64()) + spec.Params.EIP160Transition = hexutil.Uint64(genesis.Config.EIP155Block.Uint64()) + spec.Params.EIP161abcTransition = hexutil.Uint64(genesis.Config.EIP158Block.Uint64()) + spec.Params.EIP161dTransition = hexutil.Uint64(genesis.Config.EIP158Block.Uint64()) + // Byzantium + if num := genesis.Config.ByzantiumBlock; num != nil { + spec.setByzantium(num) + } + // Constantinople + if num := genesis.Config.ConstantinopleBlock; num != nil { + spec.setConstantinople(num) + } spec.Params.MaximumExtraDataSize = (hexutil.Uint64)(params.MaximumExtraDataSize) spec.Params.MinGasLimit = (hexutil.Uint64)(params.MinGasLimit) - spec.Params.GasLimitBoundDivisor = (hexutil.Uint64)(params.GasLimitBoundDivisor) + spec.Params.GasLimitBoundDivisor = (math2.HexOrDecimal64)(params.GasLimitBoundDivisor) spec.Params.NetworkID = (hexutil.Uint64)(genesis.Config.ChainID.Uint64()) + spec.Params.ChainID = (hexutil.Uint64)(genesis.Config.ChainID.Uint64()) spec.Params.MaxCodeSize = params.MaxCodeSize - spec.Params.EIP155Transition = genesis.Config.EIP155Block.Uint64() - spec.Params.EIP98Transition = math.MaxUint64 - spec.Params.EIP86Transition = math.MaxUint64 - spec.Params.EIP140Transition = genesis.Config.ByzantiumBlock.Uint64() - spec.Params.EIP211Transition = genesis.Config.ByzantiumBlock.Uint64() - spec.Params.EIP214Transition = genesis.Config.ByzantiumBlock.Uint64() - spec.Params.EIP658Transition = genesis.Config.ByzantiumBlock.Uint64() + // geth has it set from zero + spec.Params.MaxCodeSizeTransition = 0 + + // Disable this one + spec.Params.EIP98Transition = math.MaxInt64 spec.Genesis.Seal.Ethereum.Nonce = (hexutil.Bytes)(make([]byte, 8)) binary.LittleEndian.PutUint64(spec.Genesis.Seal.Ethereum.Nonce[:], genesis.Nonce) @@ -305,42 +366,77 @@ func newParityChainSpec(network string, genesis *core.Genesis, bootnodes []strin spec.Genesis.ExtraData = (hexutil.Bytes)(genesis.ExtraData) spec.Genesis.GasLimit = (hexutil.Uint64)(genesis.GasLimit) - spec.Accounts = make(map[common.Address]*parityChainSpecAccount) + spec.Accounts = make(map[common.UnprefixedAddress]*parityChainSpecAccount) for address, account := range genesis.Alloc { - spec.Accounts[address] = &parityChainSpecAccount{ - Balance: (*hexutil.Big)(account.Balance), - Nonce: account.Nonce, + bal := math2.HexOrDecimal256(*account.Balance) + + spec.Accounts[common.UnprefixedAddress(address)] = &parityChainSpecAccount{ + Balance: bal, + Nonce: math2.HexOrDecimal64(account.Nonce), } } - spec.Accounts[common.BytesToAddress([]byte{1})].Builtin = &parityChainSpecBuiltin{ - Name: "ecrecover", Pricing: &parityChainSpecPricing{Linear: &parityChainSpecLinearPricing{Base: 3000}}, - } - spec.Accounts[common.BytesToAddress([]byte{2})].Builtin = &parityChainSpecBuiltin{ + spec.setPrecompile(1, &parityChainSpecBuiltin{Name: "ecrecover", + Pricing: &parityChainSpecPricing{Linear: &parityChainSpecLinearPricing{Base: 3000}}}) + + spec.setPrecompile(2, &parityChainSpecBuiltin{ Name: "sha256", Pricing: &parityChainSpecPricing{Linear: &parityChainSpecLinearPricing{Base: 60, Word: 12}}, - } - spec.Accounts[common.BytesToAddress([]byte{3})].Builtin = &parityChainSpecBuiltin{ + }) + spec.setPrecompile(3, &parityChainSpecBuiltin{ Name: "ripemd160", Pricing: &parityChainSpecPricing{Linear: &parityChainSpecLinearPricing{Base: 600, Word: 120}}, - } - spec.Accounts[common.BytesToAddress([]byte{4})].Builtin = &parityChainSpecBuiltin{ + }) + spec.setPrecompile(4, &parityChainSpecBuiltin{ Name: "identity", Pricing: &parityChainSpecPricing{Linear: &parityChainSpecLinearPricing{Base: 15, Word: 3}}, - } + }) if genesis.Config.ByzantiumBlock != nil { - spec.Accounts[common.BytesToAddress([]byte{5})].Builtin = &parityChainSpecBuiltin{ - Name: "modexp", ActivateAt: genesis.Config.ByzantiumBlock.Uint64(), Pricing: &parityChainSpecPricing{ModExp: &parityChainSpecModExpPricing{Divisor: 20}}, - } - spec.Accounts[common.BytesToAddress([]byte{6})].Builtin = &parityChainSpecBuiltin{ - Name: "alt_bn128_add", ActivateAt: genesis.Config.ByzantiumBlock.Uint64(), Pricing: &parityChainSpecPricing{Linear: &parityChainSpecLinearPricing{Base: 500}}, - } - spec.Accounts[common.BytesToAddress([]byte{7})].Builtin = &parityChainSpecBuiltin{ - Name: "alt_bn128_mul", ActivateAt: genesis.Config.ByzantiumBlock.Uint64(), Pricing: &parityChainSpecPricing{Linear: &parityChainSpecLinearPricing{Base: 40000}}, - } - spec.Accounts[common.BytesToAddress([]byte{8})].Builtin = &parityChainSpecBuiltin{ - Name: "alt_bn128_pairing", ActivateAt: genesis.Config.ByzantiumBlock.Uint64(), Pricing: &parityChainSpecPricing{AltBnPairing: &parityChainSpecAltBnPairingPricing{Base: 100000, Pair: 80000}}, - } + blnum := math2.HexOrDecimal64(genesis.Config.ByzantiumBlock.Uint64()) + spec.setPrecompile(5, &parityChainSpecBuiltin{ + Name: "modexp", ActivateAt: blnum, Pricing: &parityChainSpecPricing{ModExp: &parityChainSpecModExpPricing{Divisor: 20}}, + }) + spec.setPrecompile(6, &parityChainSpecBuiltin{ + Name: "alt_bn128_add", ActivateAt: blnum, Pricing: &parityChainSpecPricing{Linear: &parityChainSpecLinearPricing{Base: 500}}, + }) + spec.setPrecompile(7, &parityChainSpecBuiltin{ + Name: "alt_bn128_mul", ActivateAt: blnum, Pricing: &parityChainSpecPricing{Linear: &parityChainSpecLinearPricing{Base: 40000}}, + }) + spec.setPrecompile(8, &parityChainSpecBuiltin{ + Name: "alt_bn128_pairing", ActivateAt: blnum, Pricing: &parityChainSpecPricing{AltBnPairing: &parityChainSpecAltBnPairingPricing{Base: 100000, Pair: 80000}}, + }) } return spec, nil } +func (spec *parityChainSpec) setPrecompile(address byte, data *parityChainSpecBuiltin) { + if spec.Accounts == nil { + spec.Accounts = make(map[common.UnprefixedAddress]*parityChainSpecAccount) + } + a := common.UnprefixedAddress(common.BytesToAddress([]byte{address})) + if _, exist := spec.Accounts[a]; !exist { + spec.Accounts[a] = &parityChainSpecAccount{} + } + spec.Accounts[a].Builtin = data +} + +func (spec *parityChainSpec) setByzantium(num *big.Int) { + spec.Engine.Ethash.Params.BlockReward[hexutil.EncodeBig(num)] = hexutil.EncodeBig(ethash.ByzantiumBlockReward) + spec.Engine.Ethash.Params.DifficultyBombDelays[hexutil.EncodeBig(num)] = hexutil.EncodeUint64(3000000) + n := hexutil.Uint64(num.Uint64()) + spec.Engine.Ethash.Params.EIP100bTransition = n + spec.Params.EIP140Transition = n + spec.Params.EIP211Transition = n + spec.Params.EIP214Transition = n + spec.Params.EIP658Transition = n +} + +func (spec *parityChainSpec) setConstantinople(num *big.Int) { + spec.Engine.Ethash.Params.BlockReward[hexutil.EncodeBig(num)] = hexutil.EncodeBig(ethash.ConstantinopleBlockReward) + spec.Engine.Ethash.Params.DifficultyBombDelays[hexutil.EncodeBig(num)] = hexutil.EncodeUint64(2000000) + n := hexutil.Uint64(num.Uint64()) + spec.Params.EIP145Transition = n + spec.Params.EIP1014Transition = n + spec.Params.EIP1052Transition = n + spec.Params.EIP1283Transition = n +} + // pyEthereumGenesisSpec represents the genesis specification format used by the // Python Ethereum implementation. type pyEthereumGenesisSpec struct { diff --git a/cmd/puppeth/module_dashboard.go b/cmd/puppeth/module_dashboard.go index d22bd81103..cb3ed6e71c 100644 --- a/cmd/puppeth/module_dashboard.go +++ b/cmd/puppeth/module_dashboard.go @@ -640,7 +640,7 @@ func deployDashboard(client *sshClient, network string, conf *config, config *da files[filepath.Join(workdir, network+".json")] = genesis if conf.Genesis.Config.Ethash != nil { - cppSpec, err := newCppEthereumGenesisSpec(network, conf.Genesis) + cppSpec, err := newAlethGenesisSpec(network, conf.Genesis) if err != nil { return nil, err } diff --git a/cmd/puppeth/puppeth.go b/cmd/puppeth/puppeth.go index df3fff51b0..3af1751eae 100644 --- a/cmd/puppeth/puppeth.go +++ b/cmd/puppeth/puppeth.go @@ -18,11 +18,15 @@ package main import ( + "encoding/json" + "io/ioutil" "math/rand" "os" "strings" "time" + "github.com/ethereum/go-ethereum/cmd/utils" + "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/log" "gopkg.in/urfave/cli.v1" ) @@ -43,6 +47,14 @@ func main() { Usage: "log level to emit to the screen", }, } + app.Commands = []cli.Command{ + cli.Command{ + Action: utils.MigrateFlags(convert), + Name: "convert", + Usage: "Convert from geth genesis into chainspecs for other nodes.", + ArgsUsage: "", + }, + } app.Action = func(c *cli.Context) error { // Set up the logger to print everything and the random generator log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(c.Int("loglevel")), log.StreamHandler(os.Stdout, log.TerminalFormat(true)))) @@ -58,3 +70,23 @@ func main() { } app.Run(os.Args) } + +func convert(ctx *cli.Context) error { + // Ensure we have a source genesis + log.Root().SetHandler(log.LvlFilterHandler(log.LvlInfo, log.StreamHandler(os.Stdout, log.TerminalFormat(true)))) + if len(ctx.Args()) != 1 { + utils.Fatalf("No geth genesis provided") + } + blob, err := ioutil.ReadFile(ctx.Args().First()) + if err != nil { + utils.Fatalf("Could not read file: %v", err) + } + + var genesis core.Genesis + if err := json.Unmarshal(blob, &genesis); err != nil { + utils.Fatalf("Failed parsing genesis: %v", err) + } + basename := strings.TrimRight(ctx.Args().First(), ".json") + convertGenesis(&genesis, basename, basename, []string{}) + return nil +} diff --git a/cmd/puppeth/puppeth_test.go b/cmd/puppeth/puppeth_test.go new file mode 100644 index 0000000000..ae0752d549 --- /dev/null +++ b/cmd/puppeth/puppeth_test.go @@ -0,0 +1,91 @@ +package main + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "reflect" + "strings" + "testing" + + "github.com/davecgh/go-spew/spew" + "github.com/ethereum/go-ethereum/core" +) + +func TestConverter_AlethStureby(t *testing.T) { + blob, err := ioutil.ReadFile("testdata/stureby_geth.json") + if err != nil { + t.Fatalf("could not read file: %v", err) + } + var genesis core.Genesis + if err := json.Unmarshal(blob, &genesis); err != nil { + t.Fatalf("failed parsing genesis: %v", err) + } + spec, err := newAlethGenesisSpec("stureby", &genesis) + if err != nil { + t.Fatalf("failed creating chainspec: %v", err) + } + + expBlob, err := ioutil.ReadFile("testdata/stureby_aleth.json") + if err != nil { + t.Fatalf("could not read file: %v", err) + } + expspec := &alethGenesisSpec{} + if err := json.Unmarshal(expBlob, expspec); err != nil { + t.Fatalf("failed parsing genesis: %v", err) + } + if !reflect.DeepEqual(expspec, spec) { + t.Errorf("chainspec mismatch") + c := spew.ConfigState{ + DisablePointerAddresses: true, + SortKeys: true, + } + exp := strings.Split(c.Sdump(expspec), "\n") + got := strings.Split(c.Sdump(spec), "\n") + for i := 0; i < len(exp) && i < len(got); i++ { + if exp[i] != got[i] { + fmt.Printf("got: %v\nexp: %v\n", exp[i], got[i]) + } + } + } +} + +func TestConverter_ParityStureby(t *testing.T) { + blob, err := ioutil.ReadFile("testdata/stureby_geth.json") + if err != nil { + t.Fatalf("could not read file: %v", err) + } + var genesis core.Genesis + if err := json.Unmarshal(blob, &genesis); err != nil { + t.Fatalf("failed parsing genesis: %v", err) + } + spec, err := newParityChainSpec("Stureby", &genesis, []string{}) + if err != nil { + t.Fatalf("failed creating chainspec: %v", err) + } + + expBlob, err := ioutil.ReadFile("testdata/stureby_parity.json") + if err != nil { + t.Fatalf("could not read file: %v", err) + } + expspec := &parityChainSpec{} + if err := json.Unmarshal(expBlob, expspec); err != nil { + t.Fatalf("failed parsing genesis: %v", err) + } + expspec.Nodes = []string{} + + if !reflect.DeepEqual(expspec, spec) { + t.Errorf("chainspec mismatch") + c := spew.ConfigState{ + DisablePointerAddresses: true, + SortKeys: true, + } + exp := strings.Split(c.Sdump(expspec), "\n") + got := strings.Split(c.Sdump(spec), "\n") + for i := 0; i < len(exp) && i < len(got); i++ { + if exp[i] != got[i] { + fmt.Printf("got: %v\nexp: %v\n", exp[i], got[i]) + } + } + } +} diff --git a/cmd/puppeth/testdata/stureby_aleth.json b/cmd/puppeth/testdata/stureby_aleth.json new file mode 100644 index 0000000000..1ef1d8ae18 --- /dev/null +++ b/cmd/puppeth/testdata/stureby_aleth.json @@ -0,0 +1,112 @@ +{ + "sealEngine":"Ethash", + "params":{ + "accountStartNonce":"0x00", + "maximumExtraDataSize":"0x20", + "homesteadForkBlock":"0x2710", + "daoHardforkBlock":"0x00", + "EIP150ForkBlock":"0x3a98", + "EIP158ForkBlock":"0x59d8", + "byzantiumForkBlock":"0x7530", + "constantinopleForkBlock":"0x9c40", + "minGasLimit":"0x1388", + "maxGasLimit":"0x7fffffffffffffff", + "tieBreakingGas":false, + "gasLimitBoundDivisor":"0x0400", + "minimumDifficulty":"0x20000", + "difficultyBoundDivisor":"0x0800", + "durationLimit":"0x0d", + "blockReward":"0x4563918244F40000", + "networkID":"0x4cb2e", + "chainID":"0x4cb2e", + "allowFutureBlocks":false + }, + "genesis":{ + "nonce":"0x0000000000000000", + "difficulty":"0x20000", + "mixHash":"0x0000000000000000000000000000000000000000000000000000000000000000", + "author":"0x0000000000000000000000000000000000000000", + "timestamp":"0x59a4e76d", + "parentHash":"0x0000000000000000000000000000000000000000000000000000000000000000", + "extraData":"0x0000000000000000000000000000000000000000000000000000000b4dc0ffee", + "gasLimit":"0x47b760" + }, + "accounts":{ + "0000000000000000000000000000000000000001":{ + "balance":"1", + "precompiled":{ + "name":"ecrecover", + "linear":{ + "base":3000, + "word":0 + } + } + }, + "0000000000000000000000000000000000000002":{ + "balance":"1", + "precompiled":{ + "name":"sha256", + "linear":{ + "base":60, + "word":12 + } + } + }, + "0000000000000000000000000000000000000003":{ + "balance":"1", + "precompiled":{ + "name":"ripemd160", + "linear":{ + "base":600, + "word":120 + } + } + }, + "0000000000000000000000000000000000000004":{ + "balance":"1", + "precompiled":{ + "name":"identity", + "linear":{ + "base":15, + "word":3 + } + } + }, + "0000000000000000000000000000000000000005":{ + "balance":"1", + "precompiled":{ + "name":"modexp", + "startingBlock":"0x7530" + } + }, + "0000000000000000000000000000000000000006":{ + "balance":"1", + "precompiled":{ + "name":"alt_bn128_G1_add", + "startingBlock":"0x7530", + "linear":{ + "base":500, + "word":0 + } + } + }, + "0000000000000000000000000000000000000007":{ + "balance":"1", + "precompiled":{ + "name":"alt_bn128_G1_mul", + "startingBlock":"0x7530", + "linear":{ + "base":40000, + "word":0 + } + } + }, + "0000000000000000000000000000000000000008":{ + "balance":"1", + "precompiled":{ + "name":"alt_bn128_pairing_product", + "startingBlock":"0x7530" + } + } + } +} diff --git a/cmd/puppeth/testdata/stureby_geth.json b/cmd/puppeth/testdata/stureby_geth.json new file mode 100644 index 0000000000..c8c3b3c956 --- /dev/null +++ b/cmd/puppeth/testdata/stureby_geth.json @@ -0,0 +1,47 @@ +{ + "config": { + "ethash":{}, + "chainId": 314158, + "homesteadBlock": 10000, + "eip150Block": 15000, + "eip150Hash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "eip155Block": 23000, + "eip158Block": 23000, + "byzantiumBlock": 30000, + "constantinopleBlock": 40000 + }, + "nonce": "0x0", + "timestamp": "0x59a4e76d", + "parentHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "extraData": "0x0000000000000000000000000000000000000000000000000000000b4dc0ffee", + "gasLimit": "0x47b760", + "difficulty": "0x20000", + "mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "coinbase": "0x0000000000000000000000000000000000000000", + "alloc": { + "0000000000000000000000000000000000000001": { + "balance": "0x01" + }, + "0000000000000000000000000000000000000002": { + "balance": "0x01" + }, + "0000000000000000000000000000000000000003": { + "balance": "0x01" + }, + "0000000000000000000000000000000000000004": { + "balance": "0x01" + }, + "0000000000000000000000000000000000000005": { + "balance": "0x01" + }, + "0000000000000000000000000000000000000006": { + "balance": "0x01" + }, + "0000000000000000000000000000000000000007": { + "balance": "0x01" + }, + "0000000000000000000000000000000000000008": { + "balance": "0x01" + } + } +} diff --git a/cmd/puppeth/testdata/stureby_parity.json b/cmd/puppeth/testdata/stureby_parity.json new file mode 100644 index 0000000000..f3fa8386a0 --- /dev/null +++ b/cmd/puppeth/testdata/stureby_parity.json @@ -0,0 +1,181 @@ +{ + "name":"Stureby", + "dataDir":"stureby", + "engine":{ + "Ethash":{ + "params":{ + "minimumDifficulty":"0x20000", + "difficultyBoundDivisor":"0x800", + "durationLimit":"0xd", + "blockReward":{ + "0x0":"0x4563918244f40000", + "0x7530":"0x29a2241af62c0000", + "0x9c40":"0x1bc16d674ec80000" + }, + "homesteadTransition":"0x2710", + "eip100bTransition":"0x7530", + "difficultyBombDelays":{ + "0x7530":"0x2dc6c0", + "0x9c40":"0x1e8480" + } + } + } + }, + "params":{ + "accountStartNonce":"0x0", + "maximumExtraDataSize":"0x20", + "gasLimitBoundDivisor":"0x400", + "minGasLimit":"0x1388", + "networkID":"0x4cb2e", + "chainID":"0x4cb2e", + "maxCodeSize":"0x6000", + "maxCodeSizeTransition":"0x0", + "eip98Transition": "0x7fffffffffffffff", + "eip150Transition":"0x3a98", + "eip160Transition":"0x59d8", + "eip161abcTransition":"0x59d8", + "eip161dTransition":"0x59d8", + "eip155Transition":"0x59d8", + "eip140Transition":"0x7530", + "eip211Transition":"0x7530", + "eip214Transition":"0x7530", + "eip658Transition":"0x7530", + "eip145Transition":"0x9c40", + "eip1014Transition":"0x9c40", + "eip1052Transition":"0x9c40", + "eip1283Transition":"0x9c40" + }, + "genesis":{ + "seal":{ + "ethereum":{ + "nonce":"0x0000000000000000", + "mixHash":"0x0000000000000000000000000000000000000000000000000000000000000000" + } + }, + "difficulty":"0x20000", + "author":"0x0000000000000000000000000000000000000000", + "timestamp":"0x59a4e76d", + "parentHash":"0x0000000000000000000000000000000000000000000000000000000000000000", + "extraData":"0x0000000000000000000000000000000000000000000000000000000b4dc0ffee", + "gasLimit":"0x47b760" + }, + "nodes":[ + "enode://dfa7aca3f5b635fbfe7d0b20575f25e40d9e27b4bfbb3cf74364a42023ad9f25c1a4383bcc8cced86ee511a7d03415345a4df05be37f1dff040e4c780699f1c0@168.61.153.255:31303", + "enode://ef441b20dd70aeabf0eac35c3b8a2854e5ce04db0e30be9152ea9fd129359dcbb3f803993303ff5781c755dfd7223f3fe43505f583cccb740949407677412ba9@40.74.91.252:31303", + "enode://953b5ea1c8987cf46008232a0160324fd00d41320ecf00e23af86ec8f5396b19eb57ddab37c78141be56f62e9077de4f4dfa0747fa768ed8c8531bbfb1046237@40.70.214.166:31303", + "enode://276e613dd4b277a66591e565711e6c8bb107f0905248a9f8f8228c1a87992e156e5114bb9937c02824a9d9d25f76340442cf86e2028bf5293cae19904fb2b98e@35.178.251.52:30303", + "enode://064c820d41e52ed7d426ac64b60506c2998235bedc7e67cb497c6faf7bb4fc54fe56fc82d0add3180b747c0c4f40a1108a6f84d7d0629ed606d504528e61cc57@3.8.5.3:30303", + "enode://90069fdabcc5e684fa5d59430bebbb12755d9362dfe5006a1485b13d71a78a3812d36e74dd7d88e50b51add01e097ea80f16263aeaa4f0230db6c79e2a97e7ca@217.29.191.142:30303", + "enode://0aac74b7fd28726275e466acb5e03bc88a95927e9951eb66b5efb239b2f798ada0690853b2f2823fe4efa408f0f3d4dd258430bc952a5ff70677b8625b3e3b14@40.115.33.57:40404", + "enode://0b96415a10f835106d83e090a0528eed5e7887e5c802a6d084e9f1993a9d0fc713781e6e4101f6365e9b91259712f291acc0a9e6e667e22023050d602c36fbe2@40.115.33.57:40414" + ], + "accounts":{ + "0000000000000000000000000000000000000001":{ + "balance":"1", + "nonce":"0", + "builtin":{ + "name":"ecrecover", + "pricing":{ + "linear":{ + "base":3000, + "word":0 + } + } + } + }, + "0000000000000000000000000000000000000002":{ + "balance":"1", + "nonce":"0", + "builtin":{ + "name":"sha256", + "pricing":{ + "linear":{ + "base":60, + "word":12 + } + } + } + }, + "0000000000000000000000000000000000000003":{ + "balance":"1", + "nonce":"0", + "builtin":{ + "name":"ripemd160", + "pricing":{ + "linear":{ + "base":600, + "word":120 + } + } + } + }, + "0000000000000000000000000000000000000004":{ + "balance":"1", + "nonce":"0", + "builtin":{ + "name":"identity", + "pricing":{ + "linear":{ + "base":15, + "word":3 + } + } + } + }, + "0000000000000000000000000000000000000005":{ + "balance":"1", + "nonce":"0", + "builtin":{ + "name":"modexp", + "activate_at":"0x7530", + "pricing":{ + "modexp":{ + "divisor":20 + } + } + } + }, + "0000000000000000000000000000000000000006":{ + "balance":"1", + "nonce":"0", + "builtin":{ + "name":"alt_bn128_add", + "activate_at":"0x7530", + "pricing":{ + "linear":{ + "base":500, + "word":0 + } + } + } + }, + "0000000000000000000000000000000000000007":{ + "balance":"1", + "nonce":"0", + "builtin":{ + "name":"alt_bn128_mul", + "activate_at":"0x7530", + "pricing":{ + "linear":{ + "base":40000, + "word":0 + } + } + } + }, + "0000000000000000000000000000000000000008":{ + "balance":"1", + "nonce":"0", + "builtin":{ + "name":"alt_bn128_pairing", + "activate_at":"0x7530", + "pricing":{ + "alt_bn128_pairing":{ + "base":100000, + "pair":80000 + } + } + } + } + } +} diff --git a/cmd/puppeth/wizard.go b/cmd/puppeth/wizard.go index b88a61de7d..42fc125e58 100644 --- a/cmd/puppeth/wizard.go +++ b/cmd/puppeth/wizard.go @@ -104,6 +104,28 @@ func (w *wizard) readString() string { } } +// readYesNo reads a yes or no from stdin, returning boolean true for yes +func (w *wizard) readYesNo(def bool) bool { + for { + fmt.Printf("> ") + text, err := w.in.ReadString('\n') + if err != nil { + log.Crit("Failed to read user input", "err", err) + } + text = strings.ToLower(strings.TrimSpace(text)) + if text == "y" || text == "yes" { + return true + } + if text == "n" || text == "no" { + return false + } + if len(text) == 0 { + return def + } + fmt.Println("Valid answers: y, yes, n, no or leave empty for default") + } +} + // readDefaultString reads a single line from stdin, trimming if from spaces. If // an empty line is entered, the default value is returned. func (w *wizard) readDefaultString(def string) string { diff --git a/cmd/puppeth/wizard_genesis.go b/cmd/puppeth/wizard_genesis.go index 6c4cd571fb..681a387dff 100644 --- a/cmd/puppeth/wizard_genesis.go +++ b/cmd/puppeth/wizard_genesis.go @@ -114,9 +114,13 @@ func (w *wizard) makeGenesis() { } break } - // Add a batch of precompile balances to avoid them getting deleted - for i := int64(0); i < 256; i++ { - genesis.Alloc[common.BigToAddress(big.NewInt(i))] = core.GenesisAccount{Balance: big.NewInt(1)} + fmt.Println() + fmt.Println("Should the precompile-addresses (0x1 .. 0xff) be pre-funded with 1 wei? (advisable yes)") + if w.readYesNo(true) { + // Add a batch of precompile balances to avoid them getting deleted + for i := int64(0); i < 256; i++ { + genesis.Alloc[common.BigToAddress(big.NewInt(i))] = core.GenesisAccount{Balance: big.NewInt(1)} + } } // Query the user for some custom extras fmt.Println() @@ -136,47 +140,57 @@ func (w *wizard) manageGenesis() { // Figure out whether to modify or export the genesis fmt.Println() fmt.Println(" 1. Modify existing fork rules") - fmt.Println(" 2. Export genesis configuration") + fmt.Println(" 2. Export genesis configurations") fmt.Println(" 3. Remove genesis configuration") choice := w.read() - switch { - case choice == "1": + switch choice { + case "1": // Fork rule updating requested, iterate over each fork fmt.Println() fmt.Printf("Which block should Homestead come into effect? (default = %v)\n", w.conf.Genesis.Config.HomesteadBlock) w.conf.Genesis.Config.HomesteadBlock = w.readDefaultBigInt(w.conf.Genesis.Config.HomesteadBlock) fmt.Println() - fmt.Printf("Which block should EIP150 come into effect? (default = %v)\n", w.conf.Genesis.Config.EIP150Block) + fmt.Printf("Which block should EIP150 (Tangerine Whistle) come into effect? (default = %v)\n", w.conf.Genesis.Config.EIP150Block) w.conf.Genesis.Config.EIP150Block = w.readDefaultBigInt(w.conf.Genesis.Config.EIP150Block) fmt.Println() - fmt.Printf("Which block should EIP155 come into effect? (default = %v)\n", w.conf.Genesis.Config.EIP155Block) + fmt.Printf("Which block should EIP155 (Spurious Dragon) come into effect? (default = %v)\n", w.conf.Genesis.Config.EIP155Block) w.conf.Genesis.Config.EIP155Block = w.readDefaultBigInt(w.conf.Genesis.Config.EIP155Block) fmt.Println() - fmt.Printf("Which block should EIP158 come into effect? (default = %v)\n", w.conf.Genesis.Config.EIP158Block) + fmt.Printf("Which block should EIP158/161 (also Spurious Dragon) come into effect? (default = %v)\n", w.conf.Genesis.Config.EIP158Block) w.conf.Genesis.Config.EIP158Block = w.readDefaultBigInt(w.conf.Genesis.Config.EIP158Block) fmt.Println() fmt.Printf("Which block should Byzantium come into effect? (default = %v)\n", w.conf.Genesis.Config.ByzantiumBlock) w.conf.Genesis.Config.ByzantiumBlock = w.readDefaultBigInt(w.conf.Genesis.Config.ByzantiumBlock) + fmt.Println() + fmt.Printf("Which block should Constantinople come into effect? (default = %v)\n", w.conf.Genesis.Config.ByzantiumBlock) + w.conf.Genesis.Config.ConstantinopleBlock = w.readDefaultBigInt(w.conf.Genesis.Config.ConstantinopleBlock) + out, _ := json.MarshalIndent(w.conf.Genesis.Config, "", " ") fmt.Printf("Chain configuration updated:\n\n%s\n", out) - case choice == "2": + case "2": // Save whatever genesis configuration we currently have fmt.Println() - fmt.Printf("Which file to save the genesis into? (default = %s.json)\n", w.network) + fmt.Printf("Which base filename to save the genesis specifications into? (default = %s)\n", w.network) out, _ := json.MarshalIndent(w.conf.Genesis, "", " ") - if err := ioutil.WriteFile(w.readDefaultString(fmt.Sprintf("%s.json", w.network)), out, 0644); err != nil { + basename := w.readDefaultString(fmt.Sprintf("%s.json", w.network)) + + gethJson := fmt.Sprintf("%s.json", basename) + if err := ioutil.WriteFile((gethJson), out, 0644); err != nil { log.Error("Failed to save genesis file", "err", err) } - log.Info("Exported existing genesis block") + log.Info("Saved geth genesis as %v", gethJson) + if err := convertGenesis(w.conf.Genesis, basename, w.network, w.conf.bootnodes); err != nil { + log.Error("Conversion failed", "err", err) + } - case choice == "3": + case "3": // Make sure we don't have any services running if len(w.conf.servers()) > 0 { log.Error("Genesis reset requires all services and servers torn down") @@ -186,8 +200,31 @@ func (w *wizard) manageGenesis() { w.conf.Genesis = nil w.conf.flush() - default: log.Error("That's not something I can do") } } + +func saveGenesis(basename, client string, spec interface{}) { + filename := fmt.Sprintf("%s-%s.json", basename, client) + out, _ := json.Marshal(spec) + if err := ioutil.WriteFile(filename, out, 0644); err != nil { + log.Error("failed to save genesis file", "client", client, "err", err) + } + log.Info("saved chainspec", "client", client, "filename", filename) +} + +func convertGenesis(genesis *core.Genesis, basename string, network string, bootnodes []string) error { + if spec, err := newAlethGenesisSpec(network, genesis); err == nil { + saveGenesis(basename, "aleth", spec) + } else { + log.Error("failed to create chain spec", "client", "aleth", "err", err) + } + if spec, err := newParityChainSpec(network, genesis, []string{}); err == nil { + saveGenesis(basename, "parity", spec) + } else { + log.Error("failed to create chain spec", "client", "parity", "err", err) + } + saveGenesis(basename, "harmony", genesis) + return nil +} From d4415f5e40fbd155adb3e85e4eae5c7db7832694 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Mon, 3 Dec 2018 16:50:59 +0200 Subject: [PATCH 02/22] cmd/puppeth: chain import/export via wizard, minor polishes --- .../{puppeth_test.go => genesis_test.go} | 22 ++- cmd/puppeth/puppeth.go | 43 ++---- cmd/puppeth/wizard.go | 56 +++++--- cmd/puppeth/wizard_dashboard.go | 4 +- cmd/puppeth/wizard_ethstats.go | 6 +- cmd/puppeth/wizard_explorer.go | 2 +- cmd/puppeth/wizard_faucet.go | 8 +- cmd/puppeth/wizard_genesis.go | 127 +++++++++++++----- cmd/puppeth/wizard_intro.go | 16 ++- cmd/puppeth/wizard_nginx.go | 4 +- cmd/puppeth/wizard_node.go | 4 +- cmd/puppeth/wizard_wallet.go | 2 +- 12 files changed, 189 insertions(+), 105 deletions(-) rename cmd/puppeth/{puppeth_test.go => genesis_test.go} (70%) diff --git a/cmd/puppeth/puppeth_test.go b/cmd/puppeth/genesis_test.go similarity index 70% rename from cmd/puppeth/puppeth_test.go rename to cmd/puppeth/genesis_test.go index ae0752d549..83e7383605 100644 --- a/cmd/puppeth/puppeth_test.go +++ b/cmd/puppeth/genesis_test.go @@ -1,3 +1,19 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + package main import ( @@ -12,7 +28,8 @@ import ( "github.com/ethereum/go-ethereum/core" ) -func TestConverter_AlethStureby(t *testing.T) { +// Tests the go-ethereum to Aleth chainspec conversion for the Stureby testnet. +func TestAlethSturebyConverter(t *testing.T) { blob, err := ioutil.ReadFile("testdata/stureby_geth.json") if err != nil { t.Fatalf("could not read file: %v", err) @@ -50,7 +67,8 @@ func TestConverter_AlethStureby(t *testing.T) { } } -func TestConverter_ParityStureby(t *testing.T) { +// Tests the go-ethereum to Parity chainspec conversion for the Stureby testnet. +func TestParitySturebyConverter(t *testing.T) { blob, err := ioutil.ReadFile("testdata/stureby_geth.json") if err != nil { t.Fatalf("could not read file: %v", err) diff --git a/cmd/puppeth/puppeth.go b/cmd/puppeth/puppeth.go index 3af1751eae..c3de5f9360 100644 --- a/cmd/puppeth/puppeth.go +++ b/cmd/puppeth/puppeth.go @@ -18,15 +18,11 @@ package main import ( - "encoding/json" - "io/ioutil" "math/rand" "os" "strings" "time" - "github.com/ethereum/go-ethereum/cmd/utils" - "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/log" "gopkg.in/urfave/cli.v1" ) @@ -47,46 +43,23 @@ func main() { Usage: "log level to emit to the screen", }, } - app.Commands = []cli.Command{ - cli.Command{ - Action: utils.MigrateFlags(convert), - Name: "convert", - Usage: "Convert from geth genesis into chainspecs for other nodes.", - ArgsUsage: "", - }, - } - app.Action = func(c *cli.Context) error { + app.Before = func(c *cli.Context) error { // Set up the logger to print everything and the random generator log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(c.Int("loglevel")), log.StreamHandler(os.Stdout, log.TerminalFormat(true)))) rand.Seed(time.Now().UnixNano()) - network := c.String("network") - if strings.Contains(network, " ") || strings.Contains(network, "-") || strings.ToLower(network) != network { - log.Crit("No spaces, hyphens or capital letters allowed in network name") - } - // Start the wizard and relinquish control - makeWizard(c.String("network")).run() return nil } + app.Action = runWizard app.Run(os.Args) } -func convert(ctx *cli.Context) error { - // Ensure we have a source genesis - log.Root().SetHandler(log.LvlFilterHandler(log.LvlInfo, log.StreamHandler(os.Stdout, log.TerminalFormat(true)))) - if len(ctx.Args()) != 1 { - utils.Fatalf("No geth genesis provided") - } - blob, err := ioutil.ReadFile(ctx.Args().First()) - if err != nil { - utils.Fatalf("Could not read file: %v", err) - } - - var genesis core.Genesis - if err := json.Unmarshal(blob, &genesis); err != nil { - utils.Fatalf("Failed parsing genesis: %v", err) +// runWizard start the wizard and relinquish control to it. +func runWizard(c *cli.Context) error { + network := c.String("network") + if strings.Contains(network, " ") || strings.Contains(network, "-") || strings.ToLower(network) != network { + log.Crit("No spaces, hyphens or capital letters allowed in network name") } - basename := strings.TrimRight(ctx.Args().First(), ".json") - convertGenesis(&genesis, basename, basename, []string{}) + makeWizard(c.String("network")).run() return nil } diff --git a/cmd/puppeth/wizard.go b/cmd/puppeth/wizard.go index 42fc125e58..83536506c4 100644 --- a/cmd/puppeth/wizard.go +++ b/cmd/puppeth/wizard.go @@ -23,6 +23,7 @@ import ( "io/ioutil" "math/big" "net" + "net/url" "os" "path/filepath" "sort" @@ -104,40 +105,59 @@ func (w *wizard) readString() string { } } -// readYesNo reads a yes or no from stdin, returning boolean true for yes -func (w *wizard) readYesNo(def bool) bool { +// readDefaultString reads a single line from stdin, trimming if from spaces. If +// an empty line is entered, the default value is returned. +func (w *wizard) readDefaultString(def string) string { + fmt.Printf("> ") + text, err := w.in.ReadString('\n') + if err != nil { + log.Crit("Failed to read user input", "err", err) + } + if text = strings.TrimSpace(text); text != "" { + return text + } + return def +} + +// readDefaultYesNo reads a single line from stdin, trimming if from spaces and +// interpreting it as a 'yes' or a 'no'. If an empty line is entered, the default +// value is returned. +func (w *wizard) readDefaultYesNo(def bool) bool { for { fmt.Printf("> ") text, err := w.in.ReadString('\n') if err != nil { log.Crit("Failed to read user input", "err", err) } - text = strings.ToLower(strings.TrimSpace(text)) + if text = strings.ToLower(strings.TrimSpace(text)); text == "" { + return def + } if text == "y" || text == "yes" { return true } if text == "n" || text == "no" { return false } - if len(text) == 0 { - return def - } - fmt.Println("Valid answers: y, yes, n, no or leave empty for default") + log.Error("Invalid input, expected 'y', 'yes', 'n', 'no' or empty") } } -// readDefaultString reads a single line from stdin, trimming if from spaces. If -// an empty line is entered, the default value is returned. -func (w *wizard) readDefaultString(def string) string { - fmt.Printf("> ") - text, err := w.in.ReadString('\n') - if err != nil { - log.Crit("Failed to read user input", "err", err) - } - if text = strings.TrimSpace(text); text != "" { - return text +// readURL reads a single line from stdin, trimming if from spaces and trying to +// interpret it as a URL (http, https or file). +func (w *wizard) readURL() *url.URL { + for { + fmt.Printf("> ") + text, err := w.in.ReadString('\n') + if err != nil { + log.Crit("Failed to read user input", "err", err) + } + uri, err := url.Parse(strings.TrimSpace(text)) + if err != nil { + log.Error("Invalid input, expected URL", "err", err) + continue + } + return uri } - return def } // readInt reads a single line from stdin, trimming if from spaces, enforcing it diff --git a/cmd/puppeth/wizard_dashboard.go b/cmd/puppeth/wizard_dashboard.go index 1a01631ff4..8a8370845b 100644 --- a/cmd/puppeth/wizard_dashboard.go +++ b/cmd/puppeth/wizard_dashboard.go @@ -137,14 +137,14 @@ func (w *wizard) deployDashboard() { if w.conf.ethstats != "" { fmt.Println() fmt.Println("Include ethstats secret on dashboard (y/n)? (default = yes)") - infos.trusted = w.readDefaultString("y") == "y" + infos.trusted = w.readDefaultYesNo(true) } // Try to deploy the dashboard container on the host nocache := false if existed { fmt.Println() fmt.Printf("Should the dashboard be built from scratch (y/n)? (default = no)\n") - nocache = w.readDefaultString("n") != "n" + nocache = w.readDefaultYesNo(false) } if out, err := deployDashboard(client, w.network, &w.conf, infos, nocache); err != nil { log.Error("Failed to deploy dashboard container", "err", err) diff --git a/cmd/puppeth/wizard_ethstats.go b/cmd/puppeth/wizard_ethstats.go index fb2529c267..58ff3efbe9 100644 --- a/cmd/puppeth/wizard_ethstats.go +++ b/cmd/puppeth/wizard_ethstats.go @@ -67,11 +67,11 @@ func (w *wizard) deployEthstats() { if existed { fmt.Println() fmt.Printf("Keep existing IP %v blacklist (y/n)? (default = yes)\n", infos.banned) - if w.readDefaultString("y") != "y" { + if !w.readDefaultYesNo(true) { // The user might want to clear the entire list, although generally probably not fmt.Println() fmt.Printf("Clear out blacklist and start over (y/n)? (default = no)\n") - if w.readDefaultString("n") != "n" { + if w.readDefaultYesNo(false) { infos.banned = nil } // Offer the user to explicitly add/remove certain IP addresses @@ -106,7 +106,7 @@ func (w *wizard) deployEthstats() { if existed { fmt.Println() fmt.Printf("Should the ethstats be built from scratch (y/n)? (default = no)\n") - nocache = w.readDefaultString("n") != "n" + nocache = w.readDefaultYesNo(false) } trusted := make([]string, 0, len(w.servers)) for _, client := range w.servers { diff --git a/cmd/puppeth/wizard_explorer.go b/cmd/puppeth/wizard_explorer.go index 413511c1c3..a128fb9fb5 100644 --- a/cmd/puppeth/wizard_explorer.go +++ b/cmd/puppeth/wizard_explorer.go @@ -100,7 +100,7 @@ func (w *wizard) deployExplorer() { if existed { fmt.Println() fmt.Printf("Should the explorer be built from scratch (y/n)? (default = no)\n") - nocache = w.readDefaultString("n") != "n" + nocache = w.readDefaultYesNo(false) } if out, err := deployExplorer(client, w.network, chain, infos, nocache); err != nil { log.Error("Failed to deploy explorer container", "err", err) diff --git a/cmd/puppeth/wizard_faucet.go b/cmd/puppeth/wizard_faucet.go index 6f08408947..9068c1d30b 100644 --- a/cmd/puppeth/wizard_faucet.go +++ b/cmd/puppeth/wizard_faucet.go @@ -81,7 +81,7 @@ func (w *wizard) deployFaucet() { if infos.captchaToken != "" { fmt.Println() fmt.Println("Reuse previous reCaptcha API authorization (y/n)? (default = yes)") - if w.readDefaultString("y") != "y" { + if !w.readDefaultYesNo(true) { infos.captchaToken, infos.captchaSecret = "", "" } } @@ -89,7 +89,7 @@ func (w *wizard) deployFaucet() { // No previous authorization (or old one discarded) fmt.Println() fmt.Println("Enable reCaptcha protection against robots (y/n)? (default = no)") - if w.readDefaultString("n") == "n" { + if !w.readDefaultYesNo(false) { log.Warn("Users will be able to requests funds via automated scripts") } else { // Captcha protection explicitly requested, read the site and secret keys @@ -132,7 +132,7 @@ func (w *wizard) deployFaucet() { } else { fmt.Println() fmt.Printf("Reuse previous (%s) funding account (y/n)? (default = yes)\n", key.Address.Hex()) - if w.readDefaultString("y") != "y" { + if !w.readDefaultYesNo(true) { infos.node.keyJSON, infos.node.keyPass = "", "" } } @@ -166,7 +166,7 @@ func (w *wizard) deployFaucet() { if existed { fmt.Println() fmt.Printf("Should the faucet be built from scratch (y/n)? (default = no)\n") - nocache = w.readDefaultString("n") != "n" + nocache = w.readDefaultYesNo(false) } if out, err := deployFaucet(client, w.network, w.conf.bootnodes, infos, nocache); err != nil { log.Error("Failed to deploy faucet container", "err", err) diff --git a/cmd/puppeth/wizard_genesis.go b/cmd/puppeth/wizard_genesis.go index 681a387dff..95da5bd4f3 100644 --- a/cmd/puppeth/wizard_genesis.go +++ b/cmd/puppeth/wizard_genesis.go @@ -20,9 +20,13 @@ import ( "bytes" "encoding/json" "fmt" + "io" "io/ioutil" "math/big" "math/rand" + "net/http" + "os" + "path/filepath" "time" "github.com/ethereum/go-ethereum/common" @@ -40,11 +44,12 @@ func (w *wizard) makeGenesis() { Difficulty: big.NewInt(524288), Alloc: make(core.GenesisAlloc), Config: ¶ms.ChainConfig{ - HomesteadBlock: big.NewInt(1), - EIP150Block: big.NewInt(2), - EIP155Block: big.NewInt(3), - EIP158Block: big.NewInt(3), - ByzantiumBlock: big.NewInt(4), + HomesteadBlock: big.NewInt(1), + EIP150Block: big.NewInt(2), + EIP155Block: big.NewInt(3), + EIP158Block: big.NewInt(3), + ByzantiumBlock: big.NewInt(4), + ConstantinopleBlock: big.NewInt(5), }, } // Figure out which consensus engine to choose @@ -116,7 +121,7 @@ func (w *wizard) makeGenesis() { } fmt.Println() fmt.Println("Should the precompile-addresses (0x1 .. 0xff) be pre-funded with 1 wei? (advisable yes)") - if w.readYesNo(true) { + if w.readDefaultYesNo(true) { // Add a batch of precompile balances to avoid them getting deleted for i := int64(0); i < 256; i++ { genesis.Alloc[common.BigToAddress(big.NewInt(i))] = core.GenesisAccount{Balance: big.NewInt(1)} @@ -134,6 +139,53 @@ func (w *wizard) makeGenesis() { w.conf.flush() } +// importGenesis imports a Geth genesis spec into puppeth. +func (w *wizard) importGenesis() { + // Request the genesis JSON spec URL from the user + fmt.Println() + fmt.Println("Where's the genesis file? (local file or http/https url)") + url := w.readURL() + + // Convert the various allowed URLs to a reader stream + var reader io.Reader + + switch url.Scheme { + case "http", "https": + // Remote web URL, retrieve it via an HTTP client + res, err := http.Get(url.String()) + if err != nil { + log.Error("Failed to retrieve remote genesis", "err", err) + return + } + defer res.Body.Close() + reader = res.Body + + case "": + // Schemaless URL, interpret as a local file + file, err := os.Open(url.String()) + if err != nil { + log.Error("Failed to open local genesis", "err", err) + return + } + defer file.Close() + reader = file + + default: + log.Error("Unsupported genesis URL scheme", "scheme", url.Scheme) + return + } + // Parse the genesis file and inject it successful + var genesis core.Genesis + if err := json.NewDecoder(reader).Decode(&genesis); err != nil { + log.Error("Invalid genesis spec: %v", err) + return + } + log.Info("Imported genesis block") + + w.conf.Genesis = &genesis + w.conf.flush() +} + // manageGenesis permits the modification of chain configuration parameters in // a genesis config and the export of the entire genesis spec. func (w *wizard) manageGenesis() { @@ -168,7 +220,7 @@ func (w *wizard) manageGenesis() { w.conf.Genesis.Config.ByzantiumBlock = w.readDefaultBigInt(w.conf.Genesis.Config.ByzantiumBlock) fmt.Println() - fmt.Printf("Which block should Constantinople come into effect? (default = %v)\n", w.conf.Genesis.Config.ByzantiumBlock) + fmt.Printf("Which block should Constantinople come into effect? (default = %v)\n", w.conf.Genesis.Config.ConstantinopleBlock) w.conf.Genesis.Config.ConstantinopleBlock = w.readDefaultBigInt(w.conf.Genesis.Config.ConstantinopleBlock) out, _ := json.MarshalIndent(w.conf.Genesis.Config, "", " ") @@ -177,18 +229,38 @@ func (w *wizard) manageGenesis() { case "2": // Save whatever genesis configuration we currently have fmt.Println() - fmt.Printf("Which base filename to save the genesis specifications into? (default = %s)\n", w.network) + fmt.Printf("Which folder to save the genesis specs into? (default = current)\n") + fmt.Printf(" Will create %s.json, %s-aleth.json, %s-harmony.json, %s-parity.json\n", w.network, w.network, w.network, w.network) + + folder := w.readDefaultString(".") + if err := os.MkdirAll(folder, 0755); err != nil { + log.Error("Failed to create spec folder", "folder", folder, "err", err) + return + } out, _ := json.MarshalIndent(w.conf.Genesis, "", " ") - basename := w.readDefaultString(fmt.Sprintf("%s.json", w.network)) - gethJson := fmt.Sprintf("%s.json", basename) + // Export the native genesis spec used by puppeth and Geth + gethJson := filepath.Join(folder, fmt.Sprintf("%s.json", w.network)) if err := ioutil.WriteFile((gethJson), out, 0644); err != nil { log.Error("Failed to save genesis file", "err", err) + return } - log.Info("Saved geth genesis as %v", gethJson) - if err := convertGenesis(w.conf.Genesis, basename, w.network, w.conf.bootnodes); err != nil { - log.Error("Conversion failed", "err", err) + log.Info("Saved native genesis chain spec", "path", gethJson) + + // Export the genesis spec used by Aleth (formerly C++ Ethereum) + if spec, err := newAlethGenesisSpec(w.network, w.conf.Genesis); err != nil { + log.Error("Failed to create Aleth chain spec", "err", err) + } else { + saveGenesis(folder, w.network, "aleth", spec) } + // Export the genesis spec used by Parity + if spec, err := newParityChainSpec(w.network, w.conf.Genesis, []string{}); err != nil { + log.Error("Failed to create Parity chain spec", "err", err) + } else { + saveGenesis(folder, w.network, "parity", spec) + } + // Export the genesis spec used by Harmony (formerly EthereumJ + saveGenesis(folder, w.network, "harmony", w.conf.Genesis) case "3": // Make sure we don't have any services running @@ -202,29 +274,18 @@ func (w *wizard) manageGenesis() { w.conf.flush() default: log.Error("That's not something I can do") + return } } -func saveGenesis(basename, client string, spec interface{}) { - filename := fmt.Sprintf("%s-%s.json", basename, client) - out, _ := json.Marshal(spec) - if err := ioutil.WriteFile(filename, out, 0644); err != nil { - log.Error("failed to save genesis file", "client", client, "err", err) - } - log.Info("saved chainspec", "client", client, "filename", filename) -} +// saveGenesis JSON encodes an arbitrary genesis spec into a pre-defined file. +func saveGenesis(folder, network, client string, spec interface{}) { + path := filepath.Join(folder, fmt.Sprintf("%s-%s.json", network, client)) -func convertGenesis(genesis *core.Genesis, basename string, network string, bootnodes []string) error { - if spec, err := newAlethGenesisSpec(network, genesis); err == nil { - saveGenesis(basename, "aleth", spec) - } else { - log.Error("failed to create chain spec", "client", "aleth", "err", err) - } - if spec, err := newParityChainSpec(network, genesis, []string{}); err == nil { - saveGenesis(basename, "parity", spec) - } else { - log.Error("failed to create chain spec", "client", "parity", "err", err) + out, _ := json.Marshal(spec) + if err := ioutil.WriteFile(path, out, 0644); err != nil { + log.Error("Failed to save genesis file", "client", client, "err", err) + return } - saveGenesis(basename, "harmony", genesis) - return nil + log.Info("Saved genesis chain spec", "client", client, "path", path) } diff --git a/cmd/puppeth/wizard_intro.go b/cmd/puppeth/wizard_intro.go index 3db9a1087e..75fb04b76f 100644 --- a/cmd/puppeth/wizard_intro.go +++ b/cmd/puppeth/wizard_intro.go @@ -131,7 +131,20 @@ func (w *wizard) run() { case choice == "2": if w.conf.Genesis == nil { - w.makeGenesis() + fmt.Println() + fmt.Println("What would you like to do? (default = create)") + fmt.Println(" 1. Create new genesis from scratch") + fmt.Println(" 2. Import already existing genesis") + + choice := w.read() + switch { + case choice == "" || choice == "1": + w.makeGenesis() + case choice == "2": + w.importGenesis() + default: + log.Error("That's not something I can do") + } } else { w.manageGenesis() } @@ -149,7 +162,6 @@ func (w *wizard) run() { } else { w.manageComponents() } - default: log.Error("That's not something I can do") } diff --git a/cmd/puppeth/wizard_nginx.go b/cmd/puppeth/wizard_nginx.go index 4eeae93a0b..8397b7fd57 100644 --- a/cmd/puppeth/wizard_nginx.go +++ b/cmd/puppeth/wizard_nginx.go @@ -41,12 +41,12 @@ func (w *wizard) ensureVirtualHost(client *sshClient, port int, def string) (str // Reverse proxy is not running, offer to deploy a new one fmt.Println() fmt.Println("Allow sharing the port with other services (y/n)? (default = yes)") - if w.readDefaultString("y") == "y" { + if w.readDefaultYesNo(true) { nocache := false if proxy != nil { fmt.Println() fmt.Printf("Should the reverse-proxy be rebuilt from scratch (y/n)? (default = no)\n") - nocache = w.readDefaultString("n") != "n" + nocache = w.readDefaultYesNo(false) } if out, err := deployNginx(client, w.network, port, nocache); err != nil { log.Error("Failed to deploy reverse-proxy", "err", err) diff --git a/cmd/puppeth/wizard_node.go b/cmd/puppeth/wizard_node.go index 49b10a023a..e37297f6d6 100644 --- a/cmd/puppeth/wizard_node.go +++ b/cmd/puppeth/wizard_node.go @@ -126,7 +126,7 @@ func (w *wizard) deployNode(boot bool) { } else { fmt.Println() fmt.Printf("Reuse previous (%s) signing account (y/n)? (default = yes)\n", key.Address.Hex()) - if w.readDefaultString("y") != "y" { + if !w.readDefaultYesNo(true) { infos.keyJSON, infos.keyPass = "", "" } } @@ -165,7 +165,7 @@ func (w *wizard) deployNode(boot bool) { if existed { fmt.Println() fmt.Printf("Should the node be built from scratch (y/n)? (default = no)\n") - nocache = w.readDefaultString("n") != "n" + nocache = w.readDefaultYesNo(false) } if out, err := deployNode(client, w.network, w.conf.bootnodes, infos, nocache); err != nil { log.Error("Failed to deploy Ethereum node container", "err", err) diff --git a/cmd/puppeth/wizard_wallet.go b/cmd/puppeth/wizard_wallet.go index 7624d11e20..ca1ea5bd25 100644 --- a/cmd/puppeth/wizard_wallet.go +++ b/cmd/puppeth/wizard_wallet.go @@ -96,7 +96,7 @@ func (w *wizard) deployWallet() { if existed { fmt.Println() fmt.Printf("Should the wallet be built from scratch (y/n)? (default = no)\n") - nocache = w.readDefaultString("n") != "n" + nocache = w.readDefaultYesNo(false) } if out, err := deployWallet(client, w.network, w.conf.bootnodes, infos, nocache); err != nil { log.Error("Failed to deploy wallet container", "err", err) From 92639b676aaa7f262d19081256c43cdb78aa0efd Mon Sep 17 00:00:00 2001 From: Vedhavyas Singareddi Date: Tue, 4 Dec 2018 15:27:55 +0100 Subject: [PATCH 03/22] Add packing for dynamic array and slice types (#18051) * added tests for new abi encoding features (#4) * added tests from bytes32[][] and string[] * added offset to other types * formatting * Abi/dynamic types (#5) * Revert "Abi/dynamic types (#5)" (#6) This reverts commit dabca31d797623d43bd780721fc0ad461d24be53. * Abi/dynamic types (#7) * some cleanup * Apply suggestions from code review apply suggestions Co-Authored-By: vedhavyas * added better formatting (#8) * review chnages * better comments --- accounts/abi/argument.go | 19 +++++------- accounts/abi/pack_test.go | 62 ++++++++++++++++++++++++++++++++++++++- accounts/abi/type.go | 58 ++++++++++++++++++++++++++++++------ 3 files changed, 117 insertions(+), 22 deletions(-) diff --git a/accounts/abi/argument.go b/accounts/abi/argument.go index 93b513c346..f544c80dbb 100644 --- a/accounts/abi/argument.go +++ b/accounts/abi/argument.go @@ -243,11 +243,7 @@ func (arguments Arguments) Pack(args ...interface{}) ([]byte, error) { // input offset is the bytes offset for packed output inputOffset := 0 for _, abiArg := range abiArgs { - if abiArg.Type.T == ArrayTy { - inputOffset += 32 * abiArg.Type.Size - } else { - inputOffset += 32 - } + inputOffset += getDynamicTypeOffset(abiArg.Type) } var ret []byte for i, a := range args { @@ -257,14 +253,13 @@ func (arguments Arguments) Pack(args ...interface{}) ([]byte, error) { if err != nil { return nil, err } - // check for a slice type (string, bytes, slice) - if input.Type.requiresLengthPrefix() { - // calculate the offset - offset := inputOffset + len(variableInput) + // check for dynamic types + if isDynamicType(input.Type) { // set the offset - ret = append(ret, packNum(reflect.ValueOf(offset))...) - // Append the packed output to the variable input. The variable input - // will be appended at the end of the input. + ret = append(ret, packNum(reflect.ValueOf(inputOffset))...) + // calculate next offset + inputOffset += len(packed) + // append to variable input variableInput = append(variableInput, packed...) } else { // append the packed value to the input diff --git a/accounts/abi/pack_test.go b/accounts/abi/pack_test.go index 58a5b7a581..ddd2b7362d 100644 --- a/accounts/abi/pack_test.go +++ b/accounts/abi/pack_test.go @@ -324,6 +324,66 @@ func TestPack(t *testing.T) { "foobar", common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000006666f6f6261720000000000000000000000000000000000000000000000000000"), }, + { + "string[]", + []string{"hello", "foobar"}, + common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000002" + // len(array) = 2 + "0000000000000000000000000000000000000000000000000000000000000040" + // offset 64 to i = 0 + "0000000000000000000000000000000000000000000000000000000000000080" + // offset 128 to i = 1 + "0000000000000000000000000000000000000000000000000000000000000005" + // len(str[0]) = 5 + "68656c6c6f000000000000000000000000000000000000000000000000000000" + // str[0] + "0000000000000000000000000000000000000000000000000000000000000006" + // len(str[1]) = 6 + "666f6f6261720000000000000000000000000000000000000000000000000000"), // str[1] + }, + { + "string[2]", + []string{"hello", "foobar"}, + common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000040" + // offset to i = 0 + "0000000000000000000000000000000000000000000000000000000000000080" + // offset to i = 1 + "0000000000000000000000000000000000000000000000000000000000000005" + // len(str[0]) = 5 + "68656c6c6f000000000000000000000000000000000000000000000000000000" + // str[0] + "0000000000000000000000000000000000000000000000000000000000000006" + // len(str[1]) = 6 + "666f6f6261720000000000000000000000000000000000000000000000000000"), // str[1] + }, + { + "bytes32[][]", + [][]common.Hash{{{1}, {2}}, {{3}, {4}, {5}}}, + common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000002" + // len(array) = 2 + "0000000000000000000000000000000000000000000000000000000000000040" + // offset 64 to i = 0 + "00000000000000000000000000000000000000000000000000000000000000a0" + // offset 160 to i = 1 + "0000000000000000000000000000000000000000000000000000000000000002" + // len(array[0]) = 2 + "0100000000000000000000000000000000000000000000000000000000000000" + // array[0][0] + "0200000000000000000000000000000000000000000000000000000000000000" + // array[0][1] + "0000000000000000000000000000000000000000000000000000000000000003" + // len(array[1]) = 3 + "0300000000000000000000000000000000000000000000000000000000000000" + // array[1][0] + "0400000000000000000000000000000000000000000000000000000000000000" + // array[1][1] + "0500000000000000000000000000000000000000000000000000000000000000"), // array[1][2] + }, + + { + "bytes32[][2]", + [][]common.Hash{{{1}, {2}}, {{3}, {4}, {5}}}, + common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000040" + // offset 64 to i = 0 + "00000000000000000000000000000000000000000000000000000000000000a0" + // offset 160 to i = 1 + "0000000000000000000000000000000000000000000000000000000000000002" + // len(array[0]) = 2 + "0100000000000000000000000000000000000000000000000000000000000000" + // array[0][0] + "0200000000000000000000000000000000000000000000000000000000000000" + // array[0][1] + "0000000000000000000000000000000000000000000000000000000000000003" + // len(array[1]) = 3 + "0300000000000000000000000000000000000000000000000000000000000000" + // array[1][0] + "0400000000000000000000000000000000000000000000000000000000000000" + // array[1][1] + "0500000000000000000000000000000000000000000000000000000000000000"), // array[1][2] + }, + + { + "bytes32[3][2]", + [][]common.Hash{{{1}, {2}, {3}}, {{3}, {4}, {5}}}, + common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000" + // array[0][0] + "0200000000000000000000000000000000000000000000000000000000000000" + // array[0][1] + "0300000000000000000000000000000000000000000000000000000000000000" + // array[0][2] + "0300000000000000000000000000000000000000000000000000000000000000" + // array[1][0] + "0400000000000000000000000000000000000000000000000000000000000000" + // array[1][1] + "0500000000000000000000000000000000000000000000000000000000000000"), // array[1][2] + }, } { typ, err := NewType(test.typ) if err != nil { @@ -336,7 +396,7 @@ func TestPack(t *testing.T) { } if !bytes.Equal(output, test.output) { - t.Errorf("%d failed. Expected bytes: '%x' Got: '%x'", i, test.output, output) + t.Errorf("input %d for typ: %v failed. Expected bytes: '%x' Got: '%x'", i, typ.String(), test.output, output) } } } diff --git a/accounts/abi/type.go b/accounts/abi/type.go index dce89d2b4e..6bfaabf5a5 100644 --- a/accounts/abi/type.go +++ b/accounts/abi/type.go @@ -183,23 +183,39 @@ func (t Type) pack(v reflect.Value) ([]byte, error) { return nil, err } - if t.T == SliceTy || t.T == ArrayTy { - var packed []byte + switch t.T { + case SliceTy, ArrayTy: + var ret []byte + if t.requiresLengthPrefix() { + // append length + ret = append(ret, packNum(reflect.ValueOf(v.Len()))...) + } + + // calculate offset if any + offset := 0 + offsetReq := isDynamicType(*t.Elem) + if offsetReq { + offset = getDynamicTypeOffset(*t.Elem) * v.Len() + } + var tail []byte for i := 0; i < v.Len(); i++ { val, err := t.Elem.pack(v.Index(i)) if err != nil { return nil, err } - packed = append(packed, val...) - } - if t.T == SliceTy { - return packBytesSlice(packed, v.Len()), nil - } else if t.T == ArrayTy { - return packed, nil + if !offsetReq { + ret = append(ret, val...) + continue + } + ret = append(ret, packNum(reflect.ValueOf(offset))...) + offset += len(val) + tail = append(tail, val...) } + return append(ret, tail...), nil + default: + return packElement(t, v), nil } - return packElement(t, v), nil } // requireLengthPrefix returns whether the type requires any sort of length @@ -207,3 +223,27 @@ func (t Type) pack(v reflect.Value) ([]byte, error) { func (t Type) requiresLengthPrefix() bool { return t.T == StringTy || t.T == BytesTy || t.T == SliceTy } + +// isDynamicType returns true if the type is dynamic. +// StringTy, BytesTy, and SliceTy(irrespective of slice element type) are dynamic types +// ArrayTy is considered dynamic if and only if the Array element is a dynamic type. +// This function recursively checks the type for slice and array elements. +func isDynamicType(t Type) bool { + // dynamic types + // array is also a dynamic type if the array type is dynamic + return t.T == StringTy || t.T == BytesTy || t.T == SliceTy || (t.T == ArrayTy && isDynamicType(*t.Elem)) +} + +// getDynamicTypeOffset returns the offset for the type. +// See `isDynamicType` to know which types are considered dynamic. +// If the type t is an array and element type is not a dynamic type, then we consider it a static type and +// return 32 * size of array since length prefix is not required. +// If t is a dynamic type or element type(for slices and arrays) is dynamic, then we simply return 32 as offset. +func getDynamicTypeOffset(t Type) int { + // if it is an array and there are no dynamic types + // then the array is static type + if t.T == ArrayTy && !isDynamicType(*t.Elem) { + return 32 * t.Size + } + return 32 +} From b98d2e9a1c11f797502a3afa711e470a41cbc91d Mon Sep 17 00:00:00 2001 From: holisticode Date: Tue, 4 Dec 2018 12:29:51 -0500 Subject: [PATCH 04/22] swarm/network/stream: Debug log instead of Warn for retrieval failure (#18246) --- swarm/network/stream/delivery.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/swarm/network/stream/delivery.go b/swarm/network/stream/delivery.go index a5c9fa7e86..c73298d9aa 100644 --- a/swarm/network/stream/delivery.go +++ b/swarm/network/stream/delivery.go @@ -39,6 +39,7 @@ const ( var ( processReceivedChunksCount = metrics.NewRegisteredCounter("network.stream.received_chunks.count", nil) handleRetrieveRequestMsgCount = metrics.NewRegisteredCounter("network.stream.handle_retrieve_request_msg.count", nil) + retrieveChunkFail = metrics.NewRegisteredCounter("network.stream.retrieve_chunks_fail.count", nil) requestFromPeersCount = metrics.NewRegisteredCounter("network.stream.request_from_peers.count", nil) requestFromPeersEachCount = metrics.NewRegisteredCounter("network.stream.request_from_peers_each.count", nil) @@ -169,7 +170,8 @@ func (d *Delivery) handleRetrieveRequestMsg(ctx context.Context, sp *Peer, req * go func() { chunk, err := d.chunkStore.Get(ctx, req.Addr) if err != nil { - log.Warn("ChunkStore.Get can not retrieve chunk", "peer", sp.ID().String(), "addr", req.Addr, "hopcount", req.HopCount, "err", err) + retrieveChunkFail.Inc(1) + log.Debug("ChunkStore.Get can not retrieve chunk", "peer", sp.ID().String(), "addr", req.Addr, "hopcount", req.HopCount, "err", err) return } if req.SkipCheck { From 3ac633ba8423851a85cb909a1f1f0f3c0d2c5351 Mon Sep 17 00:00:00 2001 From: Elad Date: Wed, 5 Dec 2018 15:50:55 +0530 Subject: [PATCH 05/22] swarm/api/http: add resetting timer metrics to requests (#18249) --- swarm/api/http/middleware.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/swarm/api/http/middleware.go b/swarm/api/http/middleware.go index f5f70138b3..115a008566 100644 --- a/swarm/api/http/middleware.go +++ b/swarm/api/http/middleware.go @@ -5,6 +5,7 @@ import ( "net/http" "runtime/debug" "strings" + "time" "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/swarm/api" @@ -73,9 +74,13 @@ func ParseURI(h http.Handler) http.Handler { func InitLoggingResponseWriter(h http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + startTime := time.Now() + defer metrics.GetOrRegisterResettingTimer(fmt.Sprintf("http.request.%s.time", r.Method), nil).UpdateSince(startTime) + writer := newLoggingResponseWriter(w) h.ServeHTTP(writer, r) log.Info("request served", "ruid", GetRUID(r.Context()), "code", writer.statusCode) + metrics.GetOrRegisterResettingTimer(fmt.Sprintf("http.request.%s.%d.time", r.Method, writer.statusCode), nil).UpdateSince(startTime) }) } From eac3fdb21873e813024527a490842b44b71cd8c0 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Mon, 3 Dec 2018 15:11:31 +0100 Subject: [PATCH 06/22] cmd/swarm/swarm-smoke: timeout flag --- cmd/swarm/swarm-smoke/main.go | 7 +++++++ cmd/swarm/swarm-smoke/upload_and_sync.go | 19 +++++++++++++++++-- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/cmd/swarm/swarm-smoke/main.go b/cmd/swarm/swarm-smoke/main.go index 845998dc12..06ec20a079 100644 --- a/cmd/swarm/swarm-smoke/main.go +++ b/cmd/swarm/swarm-smoke/main.go @@ -35,6 +35,7 @@ var ( from int to int verbosity int + timeout int ) func main() { @@ -91,6 +92,12 @@ func main() { Usage: "verbosity", Destination: &verbosity, }, + cli.IntFlag{ + Name: "timeout", + Value: 120, + Usage: "timeout in seconds after which kill the process", + Destination: &timeout, + }, } app.Commands = []cli.Command{ diff --git a/cmd/swarm/swarm-smoke/upload_and_sync.go b/cmd/swarm/swarm-smoke/upload_and_sync.go index 3843457dc4..21be544fe3 100644 --- a/cmd/swarm/swarm-smoke/upload_and_sync.go +++ b/cmd/swarm/swarm-smoke/upload_and_sync.go @@ -40,11 +40,11 @@ import ( func generateEndpoints(scheme string, cluster string, app string, from int, to int) { if cluster == "prod" { - for port := from; port <= to; port++ { + for port := from; port < to; port++ { endpoints = append(endpoints, fmt.Sprintf("%s://%v.swarm-gateways.net", scheme, port)) } } else { - for port := from; port <= to; port++ { + for port := from; port < to; port++ { endpoints = append(endpoints, fmt.Sprintf("%s://%s-%v-%s.stg.swarm-gateways.net", scheme, app, port, cluster)) } } @@ -58,6 +58,21 @@ func cliUploadAndSync(c *cli.Context) error { log.PrintOrigins(true) log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(verbosity), log.StreamHandler(os.Stdout, log.TerminalFormat(true)))) + errc := make(chan error) + go func() { + errc <- uploadAndSync(c) + }() + + select { + case err := <-errc: + return err + case <-time.After(time.Duration(timeout) * time.Second): + return fmt.Errorf("timeout after %v sec", timeout) + } + +} + +func uploadAndSync(c *cli.Context) error { defer func(now time.Time) { log.Info("total time", "time", time.Since(now), "kb", filesize) }(time.Now()) generateEndpoints(scheme, cluster, appName, from, to) From e176fe5bbd5b3076995e32869a38f5aaf98e4b56 Mon Sep 17 00:00:00 2001 From: Elad Nachmias Date: Tue, 4 Dec 2018 12:27:00 +0530 Subject: [PATCH 07/22] added influxdb flags --- cmd/swarm/swarm-smoke/main.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cmd/swarm/swarm-smoke/main.go b/cmd/swarm/swarm-smoke/main.go index 06ec20a079..c52e9edc95 100644 --- a/cmd/swarm/swarm-smoke/main.go +++ b/cmd/swarm/swarm-smoke/main.go @@ -20,6 +20,8 @@ import ( "os" "sort" + swarmmetrics "github.com/ethereum/go-ethereum/swarm/metrics" + "github.com/ethereum/go-ethereum/log" cli "gopkg.in/urfave/cli.v1" @@ -100,6 +102,9 @@ func main() { }, } + app.Flags = append(app.Flags, swarmmetrics.Flags[0]) + app.Flags = append(app.Flags, swarmmetrics.Flags[1:]...) + app.Commands = []cli.Command{ { Name: "upload_and_sync", From 8be02faefaf3c9e0d9aeb929e62ebd0a4cf15624 Mon Sep 17 00:00:00 2001 From: Elad Nachmias Date: Tue, 4 Dec 2018 13:28:32 +0530 Subject: [PATCH 08/22] correct metrics flags and add metrics init on smoke run --- cmd/swarm/swarm-smoke/main.go | 39 +++++++++++++++++++++++++++++++++-- swarm/metrics/flags.go | 32 +++++++++++++++------------- 2 files changed, 55 insertions(+), 16 deletions(-) diff --git a/cmd/swarm/swarm-smoke/main.go b/cmd/swarm/swarm-smoke/main.go index c52e9edc95..48b762c076 100644 --- a/cmd/swarm/swarm-smoke/main.go +++ b/cmd/swarm/swarm-smoke/main.go @@ -18,8 +18,13 @@ package main import ( "os" + "runtime" "sort" + "time" + "github.com/ethereum/go-ethereum/cmd/utils" + gethmetrics "github.com/ethereum/go-ethereum/metrics" + "github.com/ethereum/go-ethereum/metrics/influxdb" swarmmetrics "github.com/ethereum/go-ethereum/swarm/metrics" "github.com/ethereum/go-ethereum/log" @@ -102,8 +107,14 @@ func main() { }, } - app.Flags = append(app.Flags, swarmmetrics.Flags[0]) - app.Flags = append(app.Flags, swarmmetrics.Flags[1:]...) + app.Flags = append(app.Flags, []cli.Flag{ + utils.MetricsEnabledFlag, + swarmmetrics.MetricsInfluxDBEndpointFlag, + swarmmetrics.MetricsInfluxDBDatabaseFlag, + swarmmetrics.MetricsInfluxDBUsernameFlag, + swarmmetrics.MetricsInfluxDBPasswordFlag, + swarmmetrics.MetricsInfluxDBHostTagFlag, + }...) app.Commands = []cli.Command{ { @@ -122,6 +133,11 @@ func main() { sort.Sort(cli.FlagsByName(app.Flags)) sort.Sort(cli.CommandsByName(app.Commands)) + app.Before = func(ctx *cli.Context) error { + runtime.GOMAXPROCS(runtime.NumCPU()) + setupMetrics(ctx) + return nil + } err := app.Run(os.Args) if err != nil { @@ -129,3 +145,22 @@ func main() { os.Exit(1) } } + +func setupMetrics(ctx *cli.Context) { + if gethmetrics.Enabled { + var ( + endpoint = ctx.GlobalString(swarmmetrics.MetricsInfluxDBEndpointFlag.Name) + database = ctx.GlobalString(swarmmetrics.MetricsInfluxDBDatabaseFlag.Name) + username = ctx.GlobalString(swarmmetrics.MetricsInfluxDBUsernameFlag.Name) + password = ctx.GlobalString(swarmmetrics.MetricsInfluxDBPasswordFlag.Name) + hosttag = ctx.GlobalString(swarmmetrics.MetricsInfluxDBHostTagFlag.Name) + ) + + // Start system runtime metrics collection + go gethmetrics.CollectProcessMetrics(2 * time.Second) + + go influxdb.InfluxDBWithTags(gethmetrics.DefaultRegistry, 10*time.Second, endpoint, database, username, password, "swarm.", map[string]string{ + "host": hosttag, + }) + } +} diff --git a/swarm/metrics/flags.go b/swarm/metrics/flags.go index 79490fd360..7c12120a60 100644 --- a/swarm/metrics/flags.go +++ b/swarm/metrics/flags.go @@ -27,26 +27,26 @@ import ( ) var ( - metricsEnableInfluxDBExportFlag = cli.BoolFlag{ + MetricsEnableInfluxDBExportFlag = cli.BoolFlag{ Name: "metrics.influxdb.export", Usage: "Enable metrics export/push to an external InfluxDB database", } - metricsInfluxDBEndpointFlag = cli.StringFlag{ + MetricsInfluxDBEndpointFlag = cli.StringFlag{ Name: "metrics.influxdb.endpoint", Usage: "Metrics InfluxDB endpoint", Value: "http://127.0.0.1:8086", } - metricsInfluxDBDatabaseFlag = cli.StringFlag{ + MetricsInfluxDBDatabaseFlag = cli.StringFlag{ Name: "metrics.influxdb.database", Usage: "Metrics InfluxDB database", Value: "metrics", } - metricsInfluxDBUsernameFlag = cli.StringFlag{ + MetricsInfluxDBUsernameFlag = cli.StringFlag{ Name: "metrics.influxdb.username", Usage: "Metrics InfluxDB username", Value: "", } - metricsInfluxDBPasswordFlag = cli.StringFlag{ + MetricsInfluxDBPasswordFlag = cli.StringFlag{ Name: "metrics.influxdb.password", Usage: "Metrics InfluxDB password", Value: "", @@ -55,7 +55,7 @@ var ( // It is used so that we can group all nodes and average a measurement across all of them, but also so // that we can select a specific node and inspect its measurements. // https://docs.influxdata.com/influxdb/v1.4/concepts/key_concepts/#tag-key - metricsInfluxDBHostTagFlag = cli.StringFlag{ + MetricsInfluxDBHostTagFlag = cli.StringFlag{ Name: "metrics.influxdb.host.tag", Usage: "Metrics InfluxDB `host` tag attached to all measurements", Value: "localhost", @@ -65,20 +65,24 @@ var ( // Flags holds all command-line flags required for metrics collection. var Flags = []cli.Flag{ utils.MetricsEnabledFlag, - metricsEnableInfluxDBExportFlag, - metricsInfluxDBEndpointFlag, metricsInfluxDBDatabaseFlag, metricsInfluxDBUsernameFlag, metricsInfluxDBPasswordFlag, metricsInfluxDBHostTagFlag, + MetricsEnableInfluxDBExportFlag, + MetricsInfluxDBEndpointFlag, + MetricsInfluxDBDatabaseFlag, + MetricsInfluxDBUsernameFlag, + MetricsInfluxDBPasswordFlag, + MetricsInfluxDBHostTagFlag, } func Setup(ctx *cli.Context) { if gethmetrics.Enabled { log.Info("Enabling swarm metrics collection") var ( - enableExport = ctx.GlobalBool(metricsEnableInfluxDBExportFlag.Name) - endpoint = ctx.GlobalString(metricsInfluxDBEndpointFlag.Name) - database = ctx.GlobalString(metricsInfluxDBDatabaseFlag.Name) - username = ctx.GlobalString(metricsInfluxDBUsernameFlag.Name) - password = ctx.GlobalString(metricsInfluxDBPasswordFlag.Name) - hosttag = ctx.GlobalString(metricsInfluxDBHostTagFlag.Name) + enableExport = ctx.GlobalBool(MetricsEnableInfluxDBExportFlag.Name) + endpoint = ctx.GlobalString(MetricsInfluxDBEndpointFlag.Name) + database = ctx.GlobalString(MetricsInfluxDBDatabaseFlag.Name) + username = ctx.GlobalString(MetricsInfluxDBUsernameFlag.Name) + password = ctx.GlobalString(MetricsInfluxDBPasswordFlag.Name) + hosttag = ctx.GlobalString(MetricsInfluxDBHostTagFlag.Name) ) // Start system runtime metrics collection From 238195adbaddbdb116c945421929dfcb3606dfee Mon Sep 17 00:00:00 2001 From: Elad Nachmias Date: Tue, 4 Dec 2018 14:57:03 +0530 Subject: [PATCH 09/22] add metrics to upload and sync test --- cmd/swarm/swarm-smoke/main.go | 6 ++++++ cmd/swarm/swarm-smoke/upload_and_sync.go | 9 ++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/cmd/swarm/swarm-smoke/main.go b/cmd/swarm/swarm-smoke/main.go index 48b762c076..e14d3a20f9 100644 --- a/cmd/swarm/swarm-smoke/main.go +++ b/cmd/swarm/swarm-smoke/main.go @@ -44,6 +44,12 @@ var ( verbosity int timeout int ) +var ( + smokeUploadAndSyncCount = gethmetrics.NewRegisteredCounter("swarm-smoke.upload-and-sync.count", nil) + smokeUploadAndSyncFailCount = gethmetrics.NewRegisteredCounter("swarm-smoke.upload-and-sync.fail.count", nil) + smokeUploadAndSyncRunTime = gethmetrics.NewRegisteredCounter("swarm-smoke.upload-and-sync.time", nil) + smokeUploadAndSyncTimeout = gethmetrics.NewRegisteredCounter("swarm-smoke.upload-and-sync.timeout", nil) +) func main() { diff --git a/cmd/swarm/swarm-smoke/upload_and_sync.go b/cmd/swarm/swarm-smoke/upload_and_sync.go index 21be544fe3..4b7de7cbe2 100644 --- a/cmd/swarm/swarm-smoke/upload_and_sync.go +++ b/cmd/swarm/swarm-smoke/upload_and_sync.go @@ -55,6 +55,7 @@ func generateEndpoints(scheme string, cluster string, app string, from int, to i } func cliUploadAndSync(c *cli.Context) error { + smokeUploadAndSyncCount.Inc(1) log.PrintOrigins(true) log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(verbosity), log.StreamHandler(os.Stdout, log.TerminalFormat(true)))) @@ -65,15 +66,21 @@ func cliUploadAndSync(c *cli.Context) error { select { case err := <-errc: + if err != nil { + smokeUploadAndSyncFailCount.Inc(1) + } return err case <-time.After(time.Duration(timeout) * time.Second): + smokeUploadAndSyncTimeout.Inc(1) return fmt.Errorf("timeout after %v sec", timeout) } } func uploadAndSync(c *cli.Context) error { - defer func(now time.Time) { log.Info("total time", "time", time.Since(now), "kb", filesize) }(time.Now()) + defer func(now time.Time) { + log.Info("total time", "time", time.Since(now), "kb", filesize) + }(time.Now()) generateEndpoints(scheme, cluster, appName, from, to) From 5f932263de4a3ce60bf6fa7b7d2d0f9adf4bbf56 Mon Sep 17 00:00:00 2001 From: Elad Nachmias Date: Tue, 4 Dec 2018 17:28:44 +0530 Subject: [PATCH 10/22] add feed smoke metrics --- cmd/swarm/swarm-smoke/feed_upload_and_sync.go | 22 +++++++++++++++++-- cmd/swarm/swarm-smoke/main.go | 4 ++++ 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/cmd/swarm/swarm-smoke/feed_upload_and_sync.go b/cmd/swarm/swarm-smoke/feed_upload_and_sync.go index 7ec1528263..dec354c06b 100644 --- a/cmd/swarm/swarm-smoke/feed_upload_and_sync.go +++ b/cmd/swarm/swarm-smoke/feed_upload_and_sync.go @@ -26,11 +26,29 @@ const ( feedRandomDataLength = 8 ) -// TODO: retrieve with manifest + extract repeating code func cliFeedUploadAndSync(c *cli.Context) error { - + feedUploadAndSyncCount.Inc(1) log.Root().SetHandler(log.CallerFileHandler(log.LvlFilterHandler(log.Lvl(verbosity), log.StreamHandler(colorable.NewColorableStderr(), log.TerminalFormat(true))))) + errc := make(chan error) + go func() { + errc <- feedUploadAndSync(c) + }() + + select { + case err := <-errc: + if err != nil { + feedUploadAndSyncFailCount.Inc(1) + } + return err + case <-time.After(time.Duration(timeout) * time.Second): + feedUploadAndSyncTimeout.Inc(1) + return fmt.Errorf("timeout after %v sec", timeout) + } +} + +// TODO: retrieve with manifest + extract repeating code +func feedUploadAndSync(c *cli.Context) error { defer func(now time.Time) { log.Info("total time", "time", time.Since(now), "size (kb)", filesize) }(time.Now()) generateEndpoints(scheme, cluster, appName, from, to) diff --git a/cmd/swarm/swarm-smoke/main.go b/cmd/swarm/swarm-smoke/main.go index e14d3a20f9..df307ef40d 100644 --- a/cmd/swarm/swarm-smoke/main.go +++ b/cmd/swarm/swarm-smoke/main.go @@ -45,6 +45,10 @@ var ( timeout int ) var ( + feedUploadAndSyncCount = gethmetrics.NewRegisteredCounter("swarm-smoke.feed-and-sync.count", nil) + feedUploadAndSyncFailCount = gethmetrics.NewRegisteredCounter("swarm-smoke.feed-and-sync.fail.count", nil) + feedUploadAndSyncRunTime = gethmetrics.NewRegisteredCounter("swarm-smoke.feed-and-sync.time", nil) + feedUploadAndSyncTimeout = gethmetrics.NewRegisteredCounter("swarm-smoke.feed-and-sync.timeout", nil) smokeUploadAndSyncCount = gethmetrics.NewRegisteredCounter("swarm-smoke.upload-and-sync.count", nil) smokeUploadAndSyncFailCount = gethmetrics.NewRegisteredCounter("swarm-smoke.upload-and-sync.fail.count", nil) smokeUploadAndSyncRunTime = gethmetrics.NewRegisteredCounter("swarm-smoke.upload-and-sync.time", nil) From f9c231cf7251c17157c5e006bbdb7cbd8689f8dc Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Tue, 4 Dec 2018 13:58:36 +0100 Subject: [PATCH 11/22] cmd/swarm/swarm-smoke: fix metrics collection --- cmd/swarm/swarm-smoke/main.go | 32 ++++++++++++------------ cmd/swarm/swarm-smoke/upload_and_sync.go | 16 ++++++++---- 2 files changed, 27 insertions(+), 21 deletions(-) diff --git a/cmd/swarm/swarm-smoke/main.go b/cmd/swarm/swarm-smoke/main.go index df307ef40d..bf4a6e6e61 100644 --- a/cmd/swarm/swarm-smoke/main.go +++ b/cmd/swarm/swarm-smoke/main.go @@ -17,8 +17,8 @@ package main import ( + "fmt" "os" - "runtime" "sort" "time" @@ -32,6 +32,10 @@ import ( cli "gopkg.in/urfave/cli.v1" ) +const ( + collectionInterval = 5 * time.Second +) + var ( endpoints []string includeLocalhost bool @@ -45,14 +49,9 @@ var ( timeout int ) var ( - feedUploadAndSyncCount = gethmetrics.NewRegisteredCounter("swarm-smoke.feed-and-sync.count", nil) - feedUploadAndSyncFailCount = gethmetrics.NewRegisteredCounter("swarm-smoke.feed-and-sync.fail.count", nil) - feedUploadAndSyncRunTime = gethmetrics.NewRegisteredCounter("swarm-smoke.feed-and-sync.time", nil) - feedUploadAndSyncTimeout = gethmetrics.NewRegisteredCounter("swarm-smoke.feed-and-sync.timeout", nil) - smokeUploadAndSyncCount = gethmetrics.NewRegisteredCounter("swarm-smoke.upload-and-sync.count", nil) - smokeUploadAndSyncFailCount = gethmetrics.NewRegisteredCounter("swarm-smoke.upload-and-sync.fail.count", nil) - smokeUploadAndSyncRunTime = gethmetrics.NewRegisteredCounter("swarm-smoke.upload-and-sync.time", nil) - smokeUploadAndSyncTimeout = gethmetrics.NewRegisteredCounter("swarm-smoke.upload-and-sync.timeout", nil) + feedUploadAndSyncCount = gethmetrics.NewRegisteredCounter("feed-and-sync", nil) + feedUploadAndSyncFailCount = gethmetrics.NewRegisteredCounter("feed-and-sync.fail", nil) + feedUploadAndSyncTimeout = gethmetrics.NewRegisteredCounter("feed-and-sync.timeout", nil) ) func main() { @@ -141,11 +140,14 @@ func main() { }, } + // wait for metrics reporter to push latest measurements + defer func() { + time.Sleep(collectionInterval + 1*time.Second) + }() + sort.Sort(cli.FlagsByName(app.Flags)) sort.Sort(cli.CommandsByName(app.Commands)) app.Before = func(ctx *cli.Context) error { - runtime.GOMAXPROCS(runtime.NumCPU()) - setupMetrics(ctx) return nil } @@ -166,11 +168,9 @@ func setupMetrics(ctx *cli.Context) { hosttag = ctx.GlobalString(swarmmetrics.MetricsInfluxDBHostTagFlag.Name) ) - // Start system runtime metrics collection - go gethmetrics.CollectProcessMetrics(2 * time.Second) - - go influxdb.InfluxDBWithTags(gethmetrics.DefaultRegistry, 10*time.Second, endpoint, database, username, password, "swarm.", map[string]string{ - "host": hosttag, + go influxdb.InfluxDBWithTags(gethmetrics.DefaultRegistry, collectionInterval, endpoint, database, username, password, "swarm-smoke.", map[string]string{ + "host": hosttag, + "filesize": fmt.Sprintf("%v", filesize), }) } } diff --git a/cmd/swarm/swarm-smoke/upload_and_sync.go b/cmd/swarm/swarm-smoke/upload_and_sync.go index 4b7de7cbe2..9910a82c1e 100644 --- a/cmd/swarm/swarm-smoke/upload_and_sync.go +++ b/cmd/swarm/swarm-smoke/upload_and_sync.go @@ -34,6 +34,7 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/pborman/uuid" + metrics "github.com/rcrowley/go-metrics" cli "gopkg.in/urfave/cli.v1" ) @@ -55,10 +56,13 @@ func generateEndpoints(scheme string, cluster string, app string, from int, to i } func cliUploadAndSync(c *cli.Context) error { - smokeUploadAndSyncCount.Inc(1) log.PrintOrigins(true) log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(verbosity), log.StreamHandler(os.Stdout, log.TerminalFormat(true)))) + setupMetrics(c) + + metrics.GetOrRegisterCounter("upload-and-sync", nil).Inc(1) + errc := make(chan error) go func() { errc <- uploadAndSync(c) @@ -67,19 +71,21 @@ func cliUploadAndSync(c *cli.Context) error { select { case err := <-errc: if err != nil { - smokeUploadAndSyncFailCount.Inc(1) + metrics.GetOrRegisterCounter("upload-and-sync.fail", nil).Inc(1) } return err case <-time.After(time.Duration(timeout) * time.Second): - smokeUploadAndSyncTimeout.Inc(1) + metrics.GetOrRegisterCounter("upload-and-sync.timeout", nil).Inc(1) return fmt.Errorf("timeout after %v sec", timeout) } - } func uploadAndSync(c *cli.Context) error { defer func(now time.Time) { - log.Info("total time", "time", time.Since(now), "kb", filesize) + totalTime := time.Since(now) + + log.Info("total time", "time", totalTime, "kb", filesize) + metrics.GetOrRegisterCounter("upload-and-sync.time", nil).Inc(int64(totalTime)) }(time.Now()) generateEndpoints(scheme, cluster, appName, from, to) From fe3da49a2b85e7ae4682486340df5e8eb9aa1e2b Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Tue, 4 Dec 2018 15:03:21 +0100 Subject: [PATCH 12/22] cmd/swarm: use correct import --- cmd/swarm/swarm-smoke/upload_and_sync.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/swarm/swarm-smoke/upload_and_sync.go b/cmd/swarm/swarm-smoke/upload_and_sync.go index 9910a82c1e..30ed7371ee 100644 --- a/cmd/swarm/swarm-smoke/upload_and_sync.go +++ b/cmd/swarm/swarm-smoke/upload_and_sync.go @@ -33,8 +33,8 @@ import ( "time" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" "github.com/pborman/uuid" - metrics "github.com/rcrowley/go-metrics" cli "gopkg.in/urfave/cli.v1" ) From f7b0db6563338cd87fef2339d02d21a4245bae98 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Tue, 4 Dec 2018 15:53:25 +0100 Subject: [PATCH 13/22] add git commit --- cmd/swarm/swarm-smoke/main.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cmd/swarm/swarm-smoke/main.go b/cmd/swarm/swarm-smoke/main.go index bf4a6e6e61..b9252be4f6 100644 --- a/cmd/swarm/swarm-smoke/main.go +++ b/cmd/swarm/swarm-smoke/main.go @@ -32,6 +32,10 @@ import ( cli "gopkg.in/urfave/cli.v1" ) +var ( + gitCommit string // Git SHA1 commit hash of the release (set via linker flags) +) + const ( collectionInterval = 5 * time.Second ) @@ -170,6 +174,7 @@ func setupMetrics(ctx *cli.Context) { go influxdb.InfluxDBWithTags(gethmetrics.DefaultRegistry, collectionInterval, endpoint, database, username, password, "swarm-smoke.", map[string]string{ "host": hosttag, + "version": gitCommit, "filesize": fmt.Sprintf("%v", filesize), }) } From f6cd8232fbc697a921698d1156dcafc376ffff7d Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Tue, 4 Dec 2018 17:01:54 +0100 Subject: [PATCH 14/22] cmd/swarm: fix wait for reporter --- cmd/swarm/swarm-smoke/main.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/cmd/swarm/swarm-smoke/main.go b/cmd/swarm/swarm-smoke/main.go index b9252be4f6..21f41af65c 100644 --- a/cmd/swarm/swarm-smoke/main.go +++ b/cmd/swarm/swarm-smoke/main.go @@ -144,11 +144,6 @@ func main() { }, } - // wait for metrics reporter to push latest measurements - defer func() { - time.Sleep(collectionInterval + 1*time.Second) - }() - sort.Sort(cli.FlagsByName(app.Flags)) sort.Sort(cli.CommandsByName(app.Commands)) app.Before = func(ctx *cli.Context) error { @@ -158,8 +153,15 @@ func main() { err := app.Run(os.Args) if err != nil { log.Error(err.Error()) + + // wait for metrics reporter to push latest measurements + time.Sleep(collectionInterval + 1*time.Second) + os.Exit(1) } + + // wait for metrics reporter to push latest measurements + time.Sleep(collectionInterval + 1*time.Second) } func setupMetrics(ctx *cli.Context) { From 77cc885b8b61a9fc821bad724d595ac6bbc44550 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Tue, 4 Dec 2018 17:34:14 +0100 Subject: [PATCH 15/22] measure upload time --- cmd/swarm/swarm-smoke/upload_and_sync.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cmd/swarm/swarm-smoke/upload_and_sync.go b/cmd/swarm/swarm-smoke/upload_and_sync.go index 30ed7371ee..f41d80d947 100644 --- a/cmd/swarm/swarm-smoke/upload_and_sync.go +++ b/cmd/swarm/swarm-smoke/upload_and_sync.go @@ -85,7 +85,7 @@ func uploadAndSync(c *cli.Context) error { totalTime := time.Since(now) log.Info("total time", "time", totalTime, "kb", filesize) - metrics.GetOrRegisterCounter("upload-and-sync.time", nil).Inc(int64(totalTime)) + metrics.GetOrRegisterCounter("upload-and-sync.total-time", nil).Inc(int64(totalTime)) }(time.Now()) generateEndpoints(scheme, cluster, appName, from, to) @@ -95,11 +95,13 @@ func uploadAndSync(c *cli.Context) error { f, cleanup := generateRandomFile(filesize * 1000) defer cleanup() + t1 := time.Now() hash, err := upload(f, endpoints[0]) if err != nil { log.Error(err.Error()) return err } + metrics.GetOrRegisterCounter("upload-and-sync.upload-time", nil).Inc(int64(time.Since(t1))) fhash, err := digest(f) if err != nil { From 891f92b965f0f49e72e22e6f7d1ad3ae5ea4a560 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Tue, 4 Dec 2018 18:21:06 +0100 Subject: [PATCH 16/22] emit metrics only once --- cmd/swarm/swarm-smoke/main.go | 19 +++++----------- cmd/swarm/swarm-smoke/upload_and_sync.go | 2 -- metrics/influxdb/influxdb.go | 28 ++++++++++++++++++++++++ 3 files changed, 34 insertions(+), 15 deletions(-) diff --git a/cmd/swarm/swarm-smoke/main.go b/cmd/swarm/swarm-smoke/main.go index 21f41af65c..f1e50372ec 100644 --- a/cmd/swarm/swarm-smoke/main.go +++ b/cmd/swarm/swarm-smoke/main.go @@ -20,7 +20,6 @@ import ( "fmt" "os" "sort" - "time" "github.com/ethereum/go-ethereum/cmd/utils" gethmetrics "github.com/ethereum/go-ethereum/metrics" @@ -36,10 +35,6 @@ var ( gitCommit string // Git SHA1 commit hash of the release (set via linker flags) ) -const ( - collectionInterval = 5 * time.Second -) - var ( endpoints []string includeLocalhost bool @@ -149,22 +144,20 @@ func main() { app.Before = func(ctx *cli.Context) error { return nil } + app.After = func(ctx *cli.Context) error { + emitMetrics(ctx) + return nil + } err := app.Run(os.Args) if err != nil { log.Error(err.Error()) - // wait for metrics reporter to push latest measurements - time.Sleep(collectionInterval + 1*time.Second) - os.Exit(1) } - - // wait for metrics reporter to push latest measurements - time.Sleep(collectionInterval + 1*time.Second) } -func setupMetrics(ctx *cli.Context) { +func emitMetrics(ctx *cli.Context) { if gethmetrics.Enabled { var ( endpoint = ctx.GlobalString(swarmmetrics.MetricsInfluxDBEndpointFlag.Name) @@ -174,7 +167,7 @@ func setupMetrics(ctx *cli.Context) { hosttag = ctx.GlobalString(swarmmetrics.MetricsInfluxDBHostTagFlag.Name) ) - go influxdb.InfluxDBWithTags(gethmetrics.DefaultRegistry, collectionInterval, endpoint, database, username, password, "swarm-smoke.", map[string]string{ + influxdb.InfluxDBWithTagsOnce(gethmetrics.DefaultRegistry, endpoint, database, username, password, "swarm-smoke.", map[string]string{ "host": hosttag, "version": gitCommit, "filesize": fmt.Sprintf("%v", filesize), diff --git a/cmd/swarm/swarm-smoke/upload_and_sync.go b/cmd/swarm/swarm-smoke/upload_and_sync.go index f41d80d947..e5699dd50a 100644 --- a/cmd/swarm/swarm-smoke/upload_and_sync.go +++ b/cmd/swarm/swarm-smoke/upload_and_sync.go @@ -59,8 +59,6 @@ func cliUploadAndSync(c *cli.Context) error { log.PrintOrigins(true) log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(verbosity), log.StreamHandler(os.Stdout, log.TerminalFormat(true)))) - setupMetrics(c) - metrics.GetOrRegisterCounter("upload-and-sync", nil).Inc(1) errc := make(chan error) diff --git a/metrics/influxdb/influxdb.go b/metrics/influxdb/influxdb.go index 31a5c21b5f..5f99bb0146 100644 --- a/metrics/influxdb/influxdb.go +++ b/metrics/influxdb/influxdb.go @@ -58,6 +58,34 @@ func InfluxDBWithTags(r metrics.Registry, d time.Duration, url, database, userna rep.run() } +// InfluxDBWithTagsOnce runs once an InfluxDB reporter and post the given metrics.Registry with the specified tags +func InfluxDBWithTagsOnce(r metrics.Registry, url, database, username, password, namespace string, tags map[string]string) { + u, err := uurl.Parse(url) + if err != nil { + log.Warn("Unable to parse InfluxDB", "url", url, "err", err) + return + } + + rep := &reporter{ + reg: r, + url: *u, + database: database, + username: username, + password: password, + namespace: namespace, + tags: tags, + cache: make(map[string]int64), + } + if err := rep.makeClient(); err != nil { + log.Warn("Unable to make InfluxDB client", "err", err) + return + } + + if err := rep.send(); err != nil { + log.Warn("Unable to send to InfluxDB", "err", err) + } +} + func (r *reporter) makeClient() (err error) { r.client, err = client.NewClient(client.Config{ URL: r.url, From 24b6b95972f8bae16608811033da8580a584cfc8 Mon Sep 17 00:00:00 2001 From: Elad Nachmias Date: Wed, 5 Dec 2018 12:32:16 +0530 Subject: [PATCH 17/22] address PR comments --- cmd/swarm/swarm-smoke/main.go | 11 +++-------- metrics/influxdb/influxdb.go | 13 +++++++------ 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/cmd/swarm/swarm-smoke/main.go b/cmd/swarm/swarm-smoke/main.go index f1e50372ec..ea419e9fac 100644 --- a/cmd/swarm/swarm-smoke/main.go +++ b/cmd/swarm/swarm-smoke/main.go @@ -141,12 +141,8 @@ func main() { sort.Sort(cli.FlagsByName(app.Flags)) sort.Sort(cli.CommandsByName(app.Commands)) - app.Before = func(ctx *cli.Context) error { - return nil - } app.After = func(ctx *cli.Context) error { - emitMetrics(ctx) - return nil + return emitMetrics(ctx) } err := app.Run(os.Args) @@ -157,7 +153,7 @@ func main() { } } -func emitMetrics(ctx *cli.Context) { +func emitMetrics(ctx *cli.Context) error { if gethmetrics.Enabled { var ( endpoint = ctx.GlobalString(swarmmetrics.MetricsInfluxDBEndpointFlag.Name) @@ -166,8 +162,7 @@ func emitMetrics(ctx *cli.Context) { password = ctx.GlobalString(swarmmetrics.MetricsInfluxDBPasswordFlag.Name) hosttag = ctx.GlobalString(swarmmetrics.MetricsInfluxDBHostTagFlag.Name) ) - - influxdb.InfluxDBWithTagsOnce(gethmetrics.DefaultRegistry, endpoint, database, username, password, "swarm-smoke.", map[string]string{ + return influxdb.InfluxDBWithTagsOnce(gethmetrics.DefaultRegistry, endpoint, database, username, password, "swarm-smoke.", map[string]string{ "host": hosttag, "version": gitCommit, "filesize": fmt.Sprintf("%v", filesize), diff --git a/metrics/influxdb/influxdb.go b/metrics/influxdb/influxdb.go index 5f99bb0146..1c2b04bf82 100644 --- a/metrics/influxdb/influxdb.go +++ b/metrics/influxdb/influxdb.go @@ -1,6 +1,7 @@ package influxdb import ( + "errors" "fmt" uurl "net/url" "time" @@ -59,11 +60,10 @@ func InfluxDBWithTags(r metrics.Registry, d time.Duration, url, database, userna } // InfluxDBWithTagsOnce runs once an InfluxDB reporter and post the given metrics.Registry with the specified tags -func InfluxDBWithTagsOnce(r metrics.Registry, url, database, username, password, namespace string, tags map[string]string) { +func InfluxDBWithTagsOnce(r metrics.Registry, url, database, username, password, namespace string, tags map[string]string) error { u, err := uurl.Parse(url) if err != nil { - log.Warn("Unable to parse InfluxDB", "url", url, "err", err) - return + return errors.New(fmt.Sprintf("Unable to parse InfluxDB. url: %s, err: %v", url, err)) } rep := &reporter{ @@ -77,13 +77,14 @@ func InfluxDBWithTagsOnce(r metrics.Registry, url, database, username, password, cache: make(map[string]int64), } if err := rep.makeClient(); err != nil { - log.Warn("Unable to make InfluxDB client", "err", err) - return + return errors.New(fmt.Sprintf("Unable to make InfluxDB client. err: %v", err)) } if err := rep.send(); err != nil { - log.Warn("Unable to send to InfluxDB", "err", err) + return errors.New(fmt.Sprintf("Unable to send to InfluxDB. err: %v", err)) } + + return nil } func (r *reporter) makeClient() (err error) { From a2dfa31334a180e61e6fbafa0851d2778b82b86c Mon Sep 17 00:00:00 2001 From: Elad Nachmias Date: Wed, 5 Dec 2018 12:36:50 +0530 Subject: [PATCH 18/22] return nil on no error --- cmd/swarm/swarm-smoke/main.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmd/swarm/swarm-smoke/main.go b/cmd/swarm/swarm-smoke/main.go index ea419e9fac..f6e489df8a 100644 --- a/cmd/swarm/swarm-smoke/main.go +++ b/cmd/swarm/swarm-smoke/main.go @@ -168,4 +168,6 @@ func emitMetrics(ctx *cli.Context) error { "filesize": fmt.Sprintf("%v", filesize), }) } + + return nil } From de8a8afa229be6b0a0d5228222f6e02e48920941 Mon Sep 17 00:00:00 2001 From: Elad Nachmias Date: Wed, 5 Dec 2018 16:47:20 +0530 Subject: [PATCH 19/22] fix linter errors --- metrics/influxdb/influxdb.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/metrics/influxdb/influxdb.go b/metrics/influxdb/influxdb.go index 1c2b04bf82..c4ef927234 100644 --- a/metrics/influxdb/influxdb.go +++ b/metrics/influxdb/influxdb.go @@ -1,7 +1,6 @@ package influxdb import ( - "errors" "fmt" uurl "net/url" "time" @@ -63,7 +62,7 @@ func InfluxDBWithTags(r metrics.Registry, d time.Duration, url, database, userna func InfluxDBWithTagsOnce(r metrics.Registry, url, database, username, password, namespace string, tags map[string]string) error { u, err := uurl.Parse(url) if err != nil { - return errors.New(fmt.Sprintf("Unable to parse InfluxDB. url: %s, err: %v", url, err)) + return fmt.Errorf("Unable to parse InfluxDB. url: %s, err: %v", url, err) } rep := &reporter{ @@ -77,11 +76,11 @@ func InfluxDBWithTagsOnce(r metrics.Registry, url, database, username, password, cache: make(map[string]int64), } if err := rep.makeClient(); err != nil { - return errors.New(fmt.Sprintf("Unable to make InfluxDB client. err: %v", err)) + return fmt.Errorf("Unable to make InfluxDB client. err: %v", err) } if err := rep.send(); err != nil { - return errors.New(fmt.Sprintf("Unable to send to InfluxDB. err: %v", err)) + return fmt.Errorf("Unable to send to InfluxDB. err: %v", err) } return nil From 972e699479c3a1876c95dd8e9d31428c058ddd30 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Wed, 5 Dec 2018 12:34:49 +0100 Subject: [PATCH 20/22] cmd/swarm-smoke: inline metrics --- cmd/swarm/swarm-smoke/feed_upload_and_sync.go | 7 ++++--- cmd/swarm/swarm-smoke/main.go | 5 ----- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/cmd/swarm/swarm-smoke/feed_upload_and_sync.go b/cmd/swarm/swarm-smoke/feed_upload_and_sync.go index dec354c06b..9d01c29a6f 100644 --- a/cmd/swarm/swarm-smoke/feed_upload_and_sync.go +++ b/cmd/swarm/swarm-smoke/feed_upload_and_sync.go @@ -16,6 +16,7 @@ import ( "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/swarm/storage/feed" colorable "github.com/mattn/go-colorable" "github.com/pborman/uuid" @@ -27,7 +28,7 @@ const ( ) func cliFeedUploadAndSync(c *cli.Context) error { - feedUploadAndSyncCount.Inc(1) + metrics.GetOrRegisterCounter("feed-and-sync", nil).Inc(1) log.Root().SetHandler(log.CallerFileHandler(log.LvlFilterHandler(log.Lvl(verbosity), log.StreamHandler(colorable.NewColorableStderr(), log.TerminalFormat(true))))) errc := make(chan error) @@ -38,11 +39,11 @@ func cliFeedUploadAndSync(c *cli.Context) error { select { case err := <-errc: if err != nil { - feedUploadAndSyncFailCount.Inc(1) + metrics.GetOrRegisterCounter("feed-and-sync.fail", nil).Inc(1) } return err case <-time.After(time.Duration(timeout) * time.Second): - feedUploadAndSyncTimeout.Inc(1) + metrics.GetOrRegisterCounter("feed-and-sync.timeout", nil).Inc(1) return fmt.Errorf("timeout after %v sec", timeout) } } diff --git a/cmd/swarm/swarm-smoke/main.go b/cmd/swarm/swarm-smoke/main.go index f6e489df8a..bf0d4762ed 100644 --- a/cmd/swarm/swarm-smoke/main.go +++ b/cmd/swarm/swarm-smoke/main.go @@ -47,11 +47,6 @@ var ( verbosity int timeout int ) -var ( - feedUploadAndSyncCount = gethmetrics.NewRegisteredCounter("feed-and-sync", nil) - feedUploadAndSyncFailCount = gethmetrics.NewRegisteredCounter("feed-and-sync.fail", nil) - feedUploadAndSyncTimeout = gethmetrics.NewRegisteredCounter("feed-and-sync.timeout", nil) -) func main() { From 72a99531e492692f2ba7df1b579c91db3b3215a1 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Wed, 5 Dec 2018 14:47:51 +0100 Subject: [PATCH 21/22] swarm/storage: measure lazy chunk reader chunk get time --- swarm/storage/chunker.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/swarm/storage/chunker.go b/swarm/storage/chunker.go index 40292e88f9..cbe65372a1 100644 --- a/swarm/storage/chunker.go +++ b/swarm/storage/chunker.go @@ -22,6 +22,7 @@ import ( "fmt" "io" "sync" + "time" "github.com/ethereum/go-ethereum/metrics" ch "github.com/ethereum/go-ethereum/swarm/chunk" @@ -410,10 +411,14 @@ func (r *LazyChunkReader) Size(ctx context.Context, quitC chan bool) (n int64, e log.Debug("lazychunkreader.size", "addr", r.addr) if r.chunkData == nil { + + startTime := time.Now() chunkData, err := r.getter.Get(cctx, Reference(r.addr)) if err != nil { + metrics.GetOrRegisterResettingTimer("lcr.getter.get.err", nil).UpdateSince(startTime) return 0, err } + metrics.GetOrRegisterResettingTimer("lcr.getter.get", nil).UpdateSince(startTime) r.chunkData = chunkData s := r.chunkData.Size() log.Debug("lazychunkreader.size", "key", r.addr, "size", s) @@ -542,8 +547,10 @@ func (r *LazyChunkReader) join(b []byte, off int64, eoff int64, depth int, treeS wg.Add(1) go func(j int64) { childAddress := chunkData[8+j*r.hashSize : 8+(j+1)*r.hashSize] + startTime := time.Now() chunkData, err := r.getter.Get(r.ctx, Reference(childAddress)) if err != nil { + metrics.GetOrRegisterResettingTimer("lcr.getter.get.err", nil).UpdateSince(startTime) log.Debug("lazychunkreader.join", "key", fmt.Sprintf("%x", childAddress), "err", err) select { case errC <- fmt.Errorf("chunk %v-%v not found; key: %s", off, off+treeSize, fmt.Sprintf("%x", childAddress)): @@ -551,6 +558,7 @@ func (r *LazyChunkReader) join(b []byte, off int64, eoff int64, depth int, treeS } return } + metrics.GetOrRegisterResettingTimer("lcr.getter.get", nil).UpdateSince(startTime) if l := len(chunkData); l < 9 { select { case errC <- fmt.Errorf("chunk %v-%v incomplete; key: %s, data length %v", off, off+treeSize, fmt.Sprintf("%x", childAddress), l): From 159d9af395bfb2c9f4382019bac3938d0cd040ae Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Wed, 5 Dec 2018 17:00:35 +0100 Subject: [PATCH 22/22] swarm/api: log and measure the same value for request time --- swarm/api/http/middleware.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/swarm/api/http/middleware.go b/swarm/api/http/middleware.go index 115a008566..3800212c21 100644 --- a/swarm/api/http/middleware.go +++ b/swarm/api/http/middleware.go @@ -74,13 +74,15 @@ func ParseURI(h http.Handler) http.Handler { func InitLoggingResponseWriter(h http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - startTime := time.Now() - defer metrics.GetOrRegisterResettingTimer(fmt.Sprintf("http.request.%s.time", r.Method), nil).UpdateSince(startTime) + tn := time.Now() writer := newLoggingResponseWriter(w) h.ServeHTTP(writer, r) - log.Info("request served", "ruid", GetRUID(r.Context()), "code", writer.statusCode) - metrics.GetOrRegisterResettingTimer(fmt.Sprintf("http.request.%s.%d.time", r.Method, writer.statusCode), nil).UpdateSince(startTime) + + ts := time.Since(tn) + log.Info("request served", "ruid", GetRUID(r.Context()), "code", writer.statusCode, "time", ts*time.Millisecond) + metrics.GetOrRegisterResettingTimer(fmt.Sprintf("http.request.%s.time", r.Method), nil).Update(ts) + metrics.GetOrRegisterResettingTimer(fmt.Sprintf("http.request.%s.%d.time", r.Method, writer.statusCode), nil).Update(ts) }) }