Skip to content

Commit

Permalink
ethdb/leveldb: disable seek compaction (#20130)
Browse files Browse the repository at this point in the history
* vendor: update leveldb

* ethdb/leveldb: disable seek compaction and add metrics

* vendor: udpate latest levledb

* ethdb/leveldb: fix typo
  • Loading branch information
rjl493456442 authored and karalabe committed Sep 26, 2019
1 parent ead7117 commit df89233
Show file tree
Hide file tree
Showing 15 changed files with 197 additions and 70 deletions.
48 changes: 40 additions & 8 deletions ethdb/leveldb/leveldb.go
Original file line number Diff line number Diff line change
Expand Up @@ -62,14 +62,18 @@ type Database struct {
fn string // filename for reporting
db *leveldb.DB // LevelDB instance

compTimeMeter metrics.Meter // Meter for measuring the total time spent in database compaction
compReadMeter metrics.Meter // Meter for measuring the data read during compaction
compWriteMeter metrics.Meter // Meter for measuring the data written during compaction
writeDelayNMeter metrics.Meter // Meter for measuring the write delay number due to database compaction
writeDelayMeter metrics.Meter // Meter for measuring the write delay duration due to database compaction
diskSizeGauge metrics.Gauge // Gauge for tracking the size of all the levels in the database
diskReadMeter metrics.Meter // Meter for measuring the effective amount of data read
diskWriteMeter metrics.Meter // Meter for measuring the effective amount of data written
compTimeMeter metrics.Meter // Meter for measuring the total time spent in database compaction
compReadMeter metrics.Meter // Meter for measuring the data read during compaction
compWriteMeter metrics.Meter // Meter for measuring the data written during compaction
writeDelayNMeter metrics.Meter // Meter for measuring the write delay number due to database compaction
writeDelayMeter metrics.Meter // Meter for measuring the write delay duration due to database compaction
diskSizeGauge metrics.Gauge // Gauge for tracking the size of all the levels in the database
diskReadMeter metrics.Meter // Meter for measuring the effective amount of data read
diskWriteMeter metrics.Meter // Meter for measuring the effective amount of data written
memCompGauge metrics.Gauge // Gauge for tracking the number of memory compaction
level0CompGauge metrics.Gauge // Gauge for tracking the number of table compaction in level0
nonlevel0CompGauge metrics.Gauge // Gauge for tracking the number of table compaction in non0 level
seekCompGauge metrics.Gauge // Gauge for tracking the number of table compaction caused by read opt

quitLock sync.Mutex // Mutex protecting the quit channel access
quitChan chan chan error // Quit channel to stop the metrics collection before closing the database
Expand All @@ -96,6 +100,7 @@ func New(file string, cache int, handles int, namespace string) (*Database, erro
BlockCacheCapacity: cache / 2 * opt.MiB,
WriteBuffer: cache / 4 * opt.MiB, // Two of these are used internally
Filter: filter.NewBloomFilter(10),
DisableSeeksCompaction: true,
})
if _, corrupted := err.(*errors.ErrCorrupted); corrupted {
db, err = leveldb.RecoverFile(file, nil)
Expand All @@ -118,6 +123,10 @@ func New(file string, cache int, handles int, namespace string) (*Database, erro
ldb.diskWriteMeter = metrics.NewRegisteredMeter(namespace+"disk/write", nil)
ldb.writeDelayMeter = metrics.NewRegisteredMeter(namespace+"compact/writedelay/duration", nil)
ldb.writeDelayNMeter = metrics.NewRegisteredMeter(namespace+"compact/writedelay/counter", nil)
ldb.memCompGauge = metrics.NewRegisteredGauge(namespace+"compact/memory", nil)
ldb.level0CompGauge = metrics.NewRegisteredGauge(namespace+"compact/level0", nil)
ldb.nonlevel0CompGauge = metrics.NewRegisteredGauge(namespace+"compact/nonlevel0", nil)
ldb.seekCompGauge = metrics.NewRegisteredGauge(namespace+"compact/seek", nil)

// Start up the metrics gathering and return
go ldb.meter(metricsGatheringInterval)
Expand Down Expand Up @@ -375,6 +384,29 @@ func (db *Database) meter(refresh time.Duration) {
}
iostats[0], iostats[1] = nRead, nWrite

compCount, err := db.db.GetProperty("leveldb.compcount")
if err != nil {
db.log.Error("Failed to read database iostats", "err", err)
merr = err
continue
}

var (
memComp uint32
level0Comp uint32
nonLevel0Comp uint32
seekComp uint32
)
if n, err := fmt.Sscanf(compCount, "MemComp:%d Level0Comp:%d NonLevel0Comp:%d SeekComp:%d", &memComp, &level0Comp, &nonLevel0Comp, &seekComp); n != 4 || err != nil {
db.log.Error("Compaction count statistic not found")
merr = err
continue
}
db.memCompGauge.Update(int64(memComp))
db.level0CompGauge.Update(int64(level0Comp))
db.nonlevel0CompGauge.Update(int64(nonLevel0Comp))
db.seekCompGauge.Update(int64(seekComp))

// Sleep a bit, then repeat the stats collection
select {
case errc = <-db.quitChan:
Expand Down
5 changes: 5 additions & 0 deletions vendor/github.com/syndtr/goleveldb/leveldb/batch.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

22 changes: 18 additions & 4 deletions vendor/github.com/syndtr/goleveldb/leveldb/db.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

11 changes: 11 additions & 0 deletions vendor/github.com/syndtr/goleveldb/leveldb/db_compaction.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

43 changes: 26 additions & 17 deletions vendor/github.com/syndtr/goleveldb/leveldb/db_iter.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

14 changes: 10 additions & 4 deletions vendor/github.com/syndtr/goleveldb/leveldb/db_transaction.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion vendor/github.com/syndtr/goleveldb/leveldb/filter/bloom.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 4 additions & 0 deletions vendor/github.com/syndtr/goleveldb/leveldb/memdb/memdb.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

21 changes: 20 additions & 1 deletion vendor/github.com/syndtr/goleveldb/leveldb/opt/options.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading

0 comments on commit df89233

Please sign in to comment.