setting up branch

pebble/pebble.go

@@ -13,8 +13,8 @@ import (
 	"github.com/ava-labs/avalanchego/database"
 	"github.com/ava-labs/avalanchego/utils"
 	"github.com/ava-labs/avalanchego/utils/units"
-	"github.com/patrick-ogrady/pebble"
-	"github.com/patrick-ogrady/pebble/bloom"
+	"github.com/cockroachdb/pebble"
+	"github.com/cockroachdb/pebble/bloom"
 	"github.com/prometheus/client_golang/prometheus"
 	"golang.org/x/exp/slices"
 )
@@ -31,8 +31,7 @@ type Database struct {
 	db      *pebble.DB
 	metrics *metrics
 
-	wo           *pebble.WriteOptions
-	maxBatchSize int
+	wo *pebble.WriteOptions
 
 	// We use an atomic bool for most
 	// checks because it is much faster
@@ -43,7 +42,6 @@ type Database struct {
 
 type Config struct {
 	Sync                        bool
-	MaxBatchSize                int // B // TODO: make this an arg once NewBatchWithSize is added to db interface
 	CacheSize                   int // B
 	L0CompactionThreshold       int
 	L0StopWritesThreshold       int
@@ -55,8 +53,7 @@ type Config struct {
 
 func NewDefaultConfig() Config {
 	return Config{
-		Sync:                        false,         // explicitly specified for clarity
-		MaxBatchSize:                1 * units.GiB, // Avoid growing during batch construction (this is reused across batches)
+		Sync:                        false, // explicitly specified for clarity
 		CacheSize:                   2 * units.GiB,
 		L0CompactionThreshold:       2,              // avoid large compaction spikes: https://github.com/cockroachdb/cockroach/blob/a3039fe628f2ab7c5fba31a30ba7bc7c38065230/pkg/storage/pebble.go#L496
 		L0StopWritesThreshold:       1000,           // from cockroachdb: https://github.com/cockroachdb/cockroach/blob/a3039fe628f2ab7c5fba31a30ba7bc7c38065230/pkg/storage/pebble.go#L497
@@ -75,7 +72,7 @@ func New(file string, cfg Config) (database.Database, *prometheus.Registry, erro
 	if cfg.Sync {
 		wo = pebble.Sync
 	}
-	d := &Database{wo: wo, maxBatchSize: cfg.MaxBatchSize, closing: make(chan struct{})}
+	d := &Database{wo: wo, closing: make(chan struct{})}
 	opts := &pebble.Options{
 		Cache:                       pebble.NewCache(int64(cfg.CacheSize)),
 		L0CompactionThreshold:       cfg.L0CompactionThreshold,
@@ -185,7 +182,7 @@ type batch struct {
 // NewBatch creates a write/delete-only buffer that is atomically committed to
 // the database when write is called
 func (db *Database) NewBatch() database.Batch {
-	return &batch{db: db, batch: db.db.NewBatchWithSize(db.maxBatchSize)}
+	return &batch{db: db, batch: db.db.NewBatch()}
 }
 
 // Put the value into the batch for later writing

vilmo/vilmo.go

@@ -489,7 +489,7 @@ type Batch struct {
 
 	pruneableBatch *uint64
 	openWrites     int64 // bytes
-	movingPath     string
+	reuseFile      bool
 	startingCursor int64
 
 	hasher hash.Hash
@@ -642,8 +642,8 @@ func (b *Batch) recycle() (bool, error) {
 
 	// Open old batch for writing
 	b.a.logger.Debug("continuing to build on old batch", zap.Uint64("old", oldestBatch), zap.Uint64("new", b.batch))
-	b.movingPath = filepath.Join(b.a.baseDir, strconv.FormatUint(oldestBatch, 10))
-	f, err := os.OpenFile(b.movingPath, os.O_WRONLY, 0666)
+	oldFile := filepath.Join(b.a.baseDir, strconv.FormatUint(oldestBatch, 10))
+	f, err := os.OpenFile(oldFile, os.O_WRONLY, 0666)
 	if err != nil {
 		return false, err
 	}

vilmo/vilmo_test.go

@@ -513,6 +513,14 @@ func TestVilmoLarge(t *testing.T) {
 	}
 }
 
+func TestMMapReuse(t *testing.T) {
+	require := require.New(t)
+	tdir := t.TempDir()
+	f, err := os.Create(filepath.Join(tdir, "file"))
+	require.NoError(err)
+	f.Write([]byte("hello"))
+}
+
 func BenchmarkVilmo(b *testing.B) {
 	ctx := context.TODO()
 	batches := 10