From 3a5b3573caff92c4a83515eb643d1c0b0cded9a6 Mon Sep 17 00:00:00 2001 From: HuangYi Date: Tue, 4 Jul 2023 16:58:10 +0800 Subject: [PATCH 1/5] Problem: zero-copy is unsafe to use with current sdk behavior Solution: - set to false by default, and unsafty comment --- CHANGELOG.md | 1 + integration_tests/configs/default.jsonnet | 1 - store/config/config.go | 2 +- store/config/toml.go | 1 + 4 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2b301a3ad1..9dab58ec54 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -55,6 +55,7 @@ - [#1064](https://github.com/crypto-org-chain/cronos/pull/1064) Simplify memiavl snapshot switching. - [#1067](https://github.com/crypto-org-chain/cronos/pull/1067) memiavl: only export state-sync snapshots on an exist snapshot - [#1082](https://github.com/crypto-org-chain/cronos/pull/1082) Make memiavl setup code reusable. +- [#]() memiavl disable zero-copy by default. *April 13, 2023* diff --git a/integration_tests/configs/default.jsonnet b/integration_tests/configs/default.jsonnet index a496636cce..dff9ae7e64 100644 --- a/integration_tests/configs/default.jsonnet +++ b/integration_tests/configs/default.jsonnet @@ -29,7 +29,6 @@ 'app-config': { memiavl: { enable: true, - 'zero-copy': true, 'snapshot-interval': 5, }, store: { diff --git a/store/config/config.go b/store/config/config.go index 554ea46f2c..66dfd01f0c 100644 --- a/store/config/config.go +++ b/store/config/config.go @@ -9,6 +9,7 @@ type MemIAVLConfig struct { Enable bool `mapstructure:"enable"` // ZeroCopy defines if the memiavl should return slices pointing to mmap-ed buffers directly (zero-copy), // the zero-copied slices must not be retained beyond current block's execution. + // zero-copy is unsafe to use with current sdk's address cache. ZeroCopy bool `mapstructure:"zero-copy"` // AsyncCommitBuffer defines the size of asynchronous commit queue, this greatly improve block catching-up // performance, -1 means synchronous commit. @@ -26,7 +27,6 @@ func DefaultMemIAVLConfig() MemIAVLConfig { return MemIAVLConfig{ CacheSize: DefaultCacheSize, SnapshotInterval: memiavl.DefaultSnapshotInterval, - ZeroCopy: true, SnapshotKeepRecent: 1, } } diff --git a/store/config/toml.go b/store/config/toml.go index 12738e1f1e..e81a6c0262 100644 --- a/store/config/toml.go +++ b/store/config/toml.go @@ -13,6 +13,7 @@ enable = {{ .MemIAVL.Enable }} # ZeroCopy defines if the memiavl should return slices pointing to mmap-ed buffers directly (zero-copy), # the zero-copied slices must not be retained beyond current block's execution. +# zero-copy is unsafe to use with current sdk's address cache. zero-copy = {{ .MemIAVL.ZeroCopy }} # AsyncCommitBuffer defines the size of asynchronous commit queue, this greatly improve block catching-up From e690f760851a4bb1f4cd43002fa3fa15017ac190 Mon Sep 17 00:00:00 2001 From: yihuang Date: Tue, 4 Jul 2023 17:01:08 +0800 Subject: [PATCH 2/5] Update CHANGELOG.md Signed-off-by: yihuang --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9dab58ec54..f06d8fd4da 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -55,7 +55,7 @@ - [#1064](https://github.com/crypto-org-chain/cronos/pull/1064) Simplify memiavl snapshot switching. - [#1067](https://github.com/crypto-org-chain/cronos/pull/1067) memiavl: only export state-sync snapshots on an exist snapshot - [#1082](https://github.com/crypto-org-chain/cronos/pull/1082) Make memiavl setup code reusable. -- [#]() memiavl disable zero-copy by default. +- [#1092](https://github.com/crypto-org-chain/cronos/pull/1092) memiavl disable zero-copy by default. *April 13, 2023* From 4e5b36746528b30788831a13b86c088329e7c25b Mon Sep 17 00:00:00 2001 From: HuangYi Date: Wed, 5 Jul 2023 10:36:51 +0800 Subject: [PATCH 3/5] disable address cache --- CHANGELOG.md | 2 +- go.mod | 4 ++-- go.sum | 4 ++-- gomod2nix.toml | 4 ++-- store/config/config.go | 2 +- store/config/toml.go | 2 +- store/go.mod | 4 ++-- store/go.sum | 4 ++-- store/setup.go | 7 +++++++ 9 files changed, 20 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f06d8fd4da..2ec4e3565b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -55,7 +55,7 @@ - [#1064](https://github.com/crypto-org-chain/cronos/pull/1064) Simplify memiavl snapshot switching. - [#1067](https://github.com/crypto-org-chain/cronos/pull/1067) memiavl: only export state-sync snapshots on an exist snapshot - [#1082](https://github.com/crypto-org-chain/cronos/pull/1082) Make memiavl setup code reusable. -- [#1092](https://github.com/crypto-org-chain/cronos/pull/1092) memiavl disable zero-copy by default. +- [#1092](https://github.com/crypto-org-chain/cronos/pull/1092) memiavl disable sdk address cache if zero-copy enabled, and disable zero-copy by default. *April 13, 2023* diff --git a/go.mod b/go.mod index 2048e4eb6b..abc9a654ee 100644 --- a/go.mod +++ b/go.mod @@ -6,13 +6,14 @@ require ( cosmossdk.io/errors v1.0.0-beta.7 cosmossdk.io/math v1.0.0-rc.0 github.com/armon/go-metrics v0.4.1 - github.com/cosmos/cosmos-sdk v0.46.14-0.20230619012023-673ee3a54518 + github.com/cosmos/cosmos-sdk v0.46.14-0.20230704091822-d7bae88991b8 github.com/cosmos/gogoproto v1.4.7 github.com/cosmos/ibc-go/v5 v5.2.1 github.com/crypto-org-chain/cronos/store v0.0.2 github.com/crypto-org-chain/cronos/versiondb v0.0.0-00010101000000-000000000000 github.com/ethereum/go-ethereum v1.10.26 github.com/evmos/ethermint v0.0.0-00010101000000-000000000000 + github.com/gogo/protobuf v1.3.3 github.com/golang/protobuf v1.5.3 github.com/gorilla/mux v1.8.0 github.com/grpc-ecosystem/grpc-gateway v1.16.0 @@ -102,7 +103,6 @@ require ( github.com/go-stack/stack v1.8.1 // indirect github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 // indirect github.com/gogo/gateway v1.1.0 // indirect - github.com/gogo/protobuf v1.3.3 // indirect github.com/golang/glog v1.0.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/snappy v0.0.4 // indirect diff --git a/go.sum b/go.sum index d736bf4420..574d95b7ce 100644 --- a/go.sum +++ b/go.sum @@ -412,8 +412,8 @@ github.com/cosmos/btcutil v1.0.5 h1:t+ZFcX77LpKtDBhjucvnOH8C2l2ioGsBNEQ3jef8xFk= github.com/cosmos/btcutil v1.0.5/go.mod h1:IyB7iuqZMJlthe2tkIFL33xPyzbFYP0XVdS8P5lUPis= github.com/cosmos/cosmos-proto v1.0.0-beta.1 h1:iDL5qh++NoXxG8hSy93FdYJut4XfgbShIocllGaXx/0= github.com/cosmos/cosmos-proto v1.0.0-beta.1/go.mod h1:8k2GNZghi5sDRFw/scPL8gMSowT1vDA+5ouxL8GjaUE= -github.com/cosmos/cosmos-sdk v0.46.14-0.20230619012023-673ee3a54518 h1:IBJ1EaG3dyJpJ9FYUlpslCP+Rfcz0feRMBr384cIicg= -github.com/cosmos/cosmos-sdk v0.46.14-0.20230619012023-673ee3a54518/go.mod h1:ABDNny0muxj03rmJcyR0mG7O9QPGktl5e30EVsHvORM= +github.com/cosmos/cosmos-sdk v0.46.14-0.20230704091822-d7bae88991b8 h1:ELkoIZK0mgntg0/4QTs691zhlo86V7JPVzSe+Q+Eq58= +github.com/cosmos/cosmos-sdk v0.46.14-0.20230704091822-d7bae88991b8/go.mod h1:ABDNny0muxj03rmJcyR0mG7O9QPGktl5e30EVsHvORM= github.com/cosmos/go-bip39 v0.0.0-20180819234021-555e2067c45d/go.mod h1:tSxLoYXyBmiFeKpvmq4dzayMdCjCnu8uqmCysIGBT2Y= github.com/cosmos/go-bip39 v1.0.0 h1:pcomnQdrdH22njcAatO0yWojsUnCO3y2tNoV1cb6hHY= github.com/cosmos/go-bip39 v1.0.0/go.mod h1:RNJv0H/pOIVgxw6KS7QeX2a0Uo0aKUlfhZ4xuwvCdJw= diff --git a/gomod2nix.toml b/gomod2nix.toml index cfe870902e..1a74b7b0c7 100644 --- a/gomod2nix.toml +++ b/gomod2nix.toml @@ -137,8 +137,8 @@ schema = 3 version = "v1.0.0-beta.1" hash = "sha256-oATkuj+fM5eBn+ywO+w/tL0AFSIEkx0J3Yz+VhVe0QA=" [mod."github.com/cosmos/cosmos-sdk"] - version = "v0.46.14-0.20230619012023-673ee3a54518" - hash = "sha256-9GDhGWyc/DW81P7uYq7H157Nl5xiERq/SzLtGiptHso=" + version = "v0.46.14-0.20230704091822-d7bae88991b8" + hash = "sha256-6vVIkETNO/d5q8xMQALL97H08IKzsbQEME9iFAK+fGk=" [mod."github.com/cosmos/go-bip39"] version = "v1.0.0" hash = "sha256-Qm2aC2vaS8tjtMUbHmlBSagOSqbduEEDwc51qvQaBmA=" diff --git a/store/config/config.go b/store/config/config.go index 66dfd01f0c..03f4c8ce49 100644 --- a/store/config/config.go +++ b/store/config/config.go @@ -9,7 +9,7 @@ type MemIAVLConfig struct { Enable bool `mapstructure:"enable"` // ZeroCopy defines if the memiavl should return slices pointing to mmap-ed buffers directly (zero-copy), // the zero-copied slices must not be retained beyond current block's execution. - // zero-copy is unsafe to use with current sdk's address cache. + // the sdk address cache will be disabled if zero-copy is enabled. ZeroCopy bool `mapstructure:"zero-copy"` // AsyncCommitBuffer defines the size of asynchronous commit queue, this greatly improve block catching-up // performance, -1 means synchronous commit. diff --git a/store/config/toml.go b/store/config/toml.go index e81a6c0262..2848e226dc 100644 --- a/store/config/toml.go +++ b/store/config/toml.go @@ -13,7 +13,7 @@ enable = {{ .MemIAVL.Enable }} # ZeroCopy defines if the memiavl should return slices pointing to mmap-ed buffers directly (zero-copy), # the zero-copied slices must not be retained beyond current block's execution. -# zero-copy is unsafe to use with current sdk's address cache. +# the sdk address cache will be disabled if zero-copy is enabled. zero-copy = {{ .MemIAVL.ZeroCopy }} # AsyncCommitBuffer defines the size of asynchronous commit queue, this greatly improve block catching-up diff --git a/store/go.mod b/store/go.mod index 54be7343d6..99df35a16f 100644 --- a/store/go.mod +++ b/store/go.mod @@ -5,11 +5,12 @@ go 1.20 require ( cosmossdk.io/errors v1.0.0-beta.7 github.com/confio/ics23/go v0.9.0 - github.com/cosmos/cosmos-sdk v0.46.13 + github.com/cosmos/cosmos-sdk v0.46.14-0.20230704091822-d7bae88991b8 github.com/cosmos/iavl v0.19.6 github.com/crypto-org-chain/cronos/memiavl v0.0.3 github.com/gogo/protobuf v1.3.2 github.com/spf13/cast v1.5.0 + github.com/stretchr/testify v1.8.2 github.com/tendermint/tendermint v0.34.28 github.com/tendermint/tm-db v0.6.7 ) @@ -107,7 +108,6 @@ require ( github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/spf13/viper v1.13.0 // indirect - github.com/stretchr/testify v1.8.2 // indirect github.com/subosito/gotenv v1.4.1 // indirect github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d // indirect github.com/tendermint/go-amino v0.16.0 // indirect diff --git a/store/go.sum b/store/go.sum index 1d6eb5f6b2..a6a4e69ced 100644 --- a/store/go.sum +++ b/store/go.sum @@ -140,8 +140,8 @@ github.com/cosmos/btcutil v1.0.5 h1:t+ZFcX77LpKtDBhjucvnOH8C2l2ioGsBNEQ3jef8xFk= github.com/cosmos/btcutil v1.0.5/go.mod h1:IyB7iuqZMJlthe2tkIFL33xPyzbFYP0XVdS8P5lUPis= github.com/cosmos/cosmos-proto v1.0.0-alpha7 h1:yqYUOHF2jopwZh4dVQp3xgqwftE5/2hkrwIV6vkUbO0= github.com/cosmos/cosmos-proto v1.0.0-alpha7/go.mod h1:dosO4pSAbJF8zWCzCoTWP7nNsjcvSUBQmniFxDg5daw= -github.com/cosmos/cosmos-sdk v0.46.13 h1:LhL6WDBadczqBuCW0t5BHUzGQR3vbujdOYOfU0ORt+o= -github.com/cosmos/cosmos-sdk v0.46.13/go.mod h1:EfY521ATNEla8eJ6oJuZBdgP5+p360s7InnRqX+TWdM= +github.com/cosmos/cosmos-sdk v0.46.14-0.20230704091822-d7bae88991b8 h1:ELkoIZK0mgntg0/4QTs691zhlo86V7JPVzSe+Q+Eq58= +github.com/cosmos/cosmos-sdk v0.46.14-0.20230704091822-d7bae88991b8/go.mod h1:ABDNny0muxj03rmJcyR0mG7O9QPGktl5e30EVsHvORM= github.com/cosmos/go-bip39 v0.0.0-20180819234021-555e2067c45d/go.mod h1:tSxLoYXyBmiFeKpvmq4dzayMdCjCnu8uqmCysIGBT2Y= github.com/cosmos/go-bip39 v1.0.0 h1:pcomnQdrdH22njcAatO0yWojsUnCO3y2tNoV1cb6hHY= github.com/cosmos/go-bip39 v1.0.0/go.mod h1:RNJv0H/pOIVgxw6KS7QeX2a0Uo0aKUlfhZ4xuwvCdJw= diff --git a/store/setup.go b/store/setup.go index 8e211ddb45..2470abef71 100644 --- a/store/setup.go +++ b/store/setup.go @@ -8,6 +8,7 @@ import ( "github.com/cosmos/cosmos-sdk/baseapp" servertypes "github.com/cosmos/cosmos-sdk/server/types" + sdk "github.com/cosmos/cosmos-sdk/types" "github.com/crypto-org-chain/cronos/memiavl" "github.com/crypto-org-chain/cronos/store/rootmulti" @@ -33,6 +34,12 @@ func SetupMemIAVL(logger log.Logger, homePath string, appOpts servertypes.AppOpt SnapshotInterval: cast.ToUint32(appOpts.Get(FlagSnapshotInterval)), CacheSize: cast.ToInt(appOpts.Get(FlagCacheSize)), } + + if opts.ZeroCopy { + // it's unsafe to cache zero-copied byte slices without copying them + sdk.SetAddrCacheEnabled(false) + } + // cms must be overridden before the other options, because they may use the cms, // make sure the cms aren't be overridden by the other options later on. baseAppOptions = append([]func(*baseapp.BaseApp){setMemIAVL(homePath, logger, opts, sdk46Compact)}, baseAppOptions...) From 2b1fd0816868e22f7a8b0af6640ed4539476f282 Mon Sep 17 00:00:00 2001 From: HuangYi Date: Wed, 5 Jul 2023 10:38:41 +0800 Subject: [PATCH 4/5] zero-copy is safe to use again --- integration_tests/configs/default.jsonnet | 1 + 1 file changed, 1 insertion(+) diff --git a/integration_tests/configs/default.jsonnet b/integration_tests/configs/default.jsonnet index dff9ae7e64..a496636cce 100644 --- a/integration_tests/configs/default.jsonnet +++ b/integration_tests/configs/default.jsonnet @@ -29,6 +29,7 @@ 'app-config': { memiavl: { enable: true, + 'zero-copy': true, 'snapshot-interval': 5, }, store: { From 6249278f857dfc716837de7287ca698405c2fcc7 Mon Sep 17 00:00:00 2001 From: HuangYi Date: Wed, 5 Jul 2023 16:38:10 +0800 Subject: [PATCH 5/5] merged --- go.mod | 2 +- go.sum | 4 ++-- gomod2nix.toml | 2 +- store/go.mod | 2 +- store/go.sum | 4 ++-- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index 5d7d807a13..c8ea1849fd 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( cosmossdk.io/errors v1.0.0-beta.7 cosmossdk.io/math v1.0.0-rc.0 github.com/armon/go-metrics v0.4.1 - github.com/cosmos/cosmos-sdk v0.46.14-0.20230704091822-d7bae88991b8 + github.com/cosmos/cosmos-sdk v0.46.14-0.20230705083228-b39cdb2e74ad github.com/cosmos/gogoproto v1.4.8 github.com/cosmos/ibc-go/v6 v6.2.0 github.com/crypto-org-chain/cronos/store v0.0.2 diff --git a/go.sum b/go.sum index 69d39cd6fb..92cb2688c1 100644 --- a/go.sum +++ b/go.sum @@ -412,8 +412,8 @@ github.com/cosmos/btcutil v1.0.5 h1:t+ZFcX77LpKtDBhjucvnOH8C2l2ioGsBNEQ3jef8xFk= github.com/cosmos/btcutil v1.0.5/go.mod h1:IyB7iuqZMJlthe2tkIFL33xPyzbFYP0XVdS8P5lUPis= github.com/cosmos/cosmos-proto v1.0.0-beta.1 h1:iDL5qh++NoXxG8hSy93FdYJut4XfgbShIocllGaXx/0= github.com/cosmos/cosmos-proto v1.0.0-beta.1/go.mod h1:8k2GNZghi5sDRFw/scPL8gMSowT1vDA+5ouxL8GjaUE= -github.com/cosmos/cosmos-sdk v0.46.14-0.20230704091822-d7bae88991b8 h1:ELkoIZK0mgntg0/4QTs691zhlo86V7JPVzSe+Q+Eq58= -github.com/cosmos/cosmos-sdk v0.46.14-0.20230704091822-d7bae88991b8/go.mod h1:ABDNny0muxj03rmJcyR0mG7O9QPGktl5e30EVsHvORM= +github.com/cosmos/cosmos-sdk v0.46.14-0.20230705083228-b39cdb2e74ad h1:8ZpO0zXy80YEFeby5ixmER6ffBYQJwyaZKVNk1ghHrM= +github.com/cosmos/cosmos-sdk v0.46.14-0.20230705083228-b39cdb2e74ad/go.mod h1:ABDNny0muxj03rmJcyR0mG7O9QPGktl5e30EVsHvORM= github.com/cosmos/go-bip39 v0.0.0-20180819234021-555e2067c45d/go.mod h1:tSxLoYXyBmiFeKpvmq4dzayMdCjCnu8uqmCysIGBT2Y= github.com/cosmos/go-bip39 v1.0.0 h1:pcomnQdrdH22njcAatO0yWojsUnCO3y2tNoV1cb6hHY= github.com/cosmos/go-bip39 v1.0.0/go.mod h1:RNJv0H/pOIVgxw6KS7QeX2a0Uo0aKUlfhZ4xuwvCdJw= diff --git a/gomod2nix.toml b/gomod2nix.toml index 83067ac84a..f50a938976 100644 --- a/gomod2nix.toml +++ b/gomod2nix.toml @@ -137,7 +137,7 @@ schema = 3 version = "v1.0.0-beta.1" hash = "sha256-oATkuj+fM5eBn+ywO+w/tL0AFSIEkx0J3Yz+VhVe0QA=" [mod."github.com/cosmos/cosmos-sdk"] - version = "v0.46.14-0.20230704091822-d7bae88991b8" + version = "v0.46.14-0.20230705083228-b39cdb2e74ad" hash = "sha256-6vVIkETNO/d5q8xMQALL97H08IKzsbQEME9iFAK+fGk=" [mod."github.com/cosmos/go-bip39"] version = "v1.0.0" diff --git a/store/go.mod b/store/go.mod index 99df35a16f..f27ae47d90 100644 --- a/store/go.mod +++ b/store/go.mod @@ -5,7 +5,7 @@ go 1.20 require ( cosmossdk.io/errors v1.0.0-beta.7 github.com/confio/ics23/go v0.9.0 - github.com/cosmos/cosmos-sdk v0.46.14-0.20230704091822-d7bae88991b8 + github.com/cosmos/cosmos-sdk v0.46.14-0.20230705083228-b39cdb2e74ad github.com/cosmos/iavl v0.19.6 github.com/crypto-org-chain/cronos/memiavl v0.0.3 github.com/gogo/protobuf v1.3.2 diff --git a/store/go.sum b/store/go.sum index a6a4e69ced..7adf9c731d 100644 --- a/store/go.sum +++ b/store/go.sum @@ -140,8 +140,8 @@ github.com/cosmos/btcutil v1.0.5 h1:t+ZFcX77LpKtDBhjucvnOH8C2l2ioGsBNEQ3jef8xFk= github.com/cosmos/btcutil v1.0.5/go.mod h1:IyB7iuqZMJlthe2tkIFL33xPyzbFYP0XVdS8P5lUPis= github.com/cosmos/cosmos-proto v1.0.0-alpha7 h1:yqYUOHF2jopwZh4dVQp3xgqwftE5/2hkrwIV6vkUbO0= github.com/cosmos/cosmos-proto v1.0.0-alpha7/go.mod h1:dosO4pSAbJF8zWCzCoTWP7nNsjcvSUBQmniFxDg5daw= -github.com/cosmos/cosmos-sdk v0.46.14-0.20230704091822-d7bae88991b8 h1:ELkoIZK0mgntg0/4QTs691zhlo86V7JPVzSe+Q+Eq58= -github.com/cosmos/cosmos-sdk v0.46.14-0.20230704091822-d7bae88991b8/go.mod h1:ABDNny0muxj03rmJcyR0mG7O9QPGktl5e30EVsHvORM= +github.com/cosmos/cosmos-sdk v0.46.14-0.20230705083228-b39cdb2e74ad h1:8ZpO0zXy80YEFeby5ixmER6ffBYQJwyaZKVNk1ghHrM= +github.com/cosmos/cosmos-sdk v0.46.14-0.20230705083228-b39cdb2e74ad/go.mod h1:ABDNny0muxj03rmJcyR0mG7O9QPGktl5e30EVsHvORM= github.com/cosmos/go-bip39 v0.0.0-20180819234021-555e2067c45d/go.mod h1:tSxLoYXyBmiFeKpvmq4dzayMdCjCnu8uqmCysIGBT2Y= github.com/cosmos/go-bip39 v1.0.0 h1:pcomnQdrdH22njcAatO0yWojsUnCO3y2tNoV1cb6hHY= github.com/cosmos/go-bip39 v1.0.0/go.mod h1:RNJv0H/pOIVgxw6KS7QeX2a0Uo0aKUlfhZ4xuwvCdJw=