Implement commitlog rotation as well as writing out, cleaning up, and…

… bootstrapping using snapshot metadata files (#1384)
m3db · Mar 2, 2019 · 54e241a · 54e241a
1 parent 56c0fdd
commit 54e241a
Show file tree

Hide file tree

Showing 96 changed files with 1,874 additions and 2,619 deletions.
diff --git a/kube/bundle.yaml b/kube/bundle.yaml
diff --git a/kube/m3dbnode-configmap.yaml b/kube/m3dbnode-configmap.yaml
@@ -71,7 +71,6 @@ data:
         queue:
             calculationType: fixed
             size: 2097152
-        blockSize: 10m
 
       fs:
         filePathPrefix: /var/lib/m3db

diff --git a/scripts/development/m3_stack/m3dbnode.yml b/scripts/development/m3_stack/m3dbnode.yml
@@ -50,7 +50,6 @@ db:
     queue:
         calculationType: fixed
         size: 2097152
-    blockSize: 10m
 
   fs:
     filePathPrefix: /var/lib/m3db

diff --git a/src/cmd/services/m3dbnode/config/bootstrap.go b/src/cmd/services/m3dbnode/config/bootstrap.go
@@ -77,13 +77,13 @@ type BootstrapFilesystemConfiguration struct {
 
 // BootstrapCommitlogConfiguration specifies config for the commitlog bootstrapper.
 type BootstrapCommitlogConfiguration struct {
-	// ReturnUnfulfilledForCorruptCommitlogFiles controls whether the commitlog bootstrapper
+	// ReturnUnfulfilledForCorruptCommitLogFiles controls whether the commitlog bootstrapper
 	// will return unfulfilled for all shard time ranges when it encounters a corrupt commit
 	// file. Note that regardless of this value, the commitlog bootstrapper will still try and
 	// read all the uncorrupted commitlog files and return as much data as it can, but setting
 	// this to true allows the node to attempt a repair if the peers bootstrapper is configured
 	// after the commitlog bootstrapper.
-	ReturnUnfulfilledForCorruptCommitlogFiles bool `yaml:"returnUnfulfilledForCorruptCommitlogFiles"`
+	ReturnUnfulfilledForCorruptCommitLogFiles bool `yaml:"returnUnfulfilledForCorruptCommitLogFiles"`
 }
 
 // New creates a bootstrap process based on the bootstrap configuration.

diff --git a/src/cmd/services/m3dbnode/config/config.go b/src/cmd/services/m3dbnode/config/config.go
@@ -217,8 +217,9 @@ type CommitLogPolicy struct {
 	// enough for almost all workloads assuming a reasonable batch size is used.
 	QueueChannel *CommitLogQueuePolicy `yaml:"queueChannel"`
 
-	// The commit log block size.
-	BlockSize time.Duration `yaml:"blockSize" validate:"nonzero"`
+	// Deprecated. Left in struct to keep old YAMLs parseable.
+	// TODO(V1): remove
+	DeprecatedBlockSize *time.Duration `yaml:"blockSize"`
 }
 
 // CalculationType is a type of configuration parameter.

diff --git a/src/cmd/services/m3dbnode/config/config_test.go b/src/cmd/services/m3dbnode/config/config_test.go
@@ -105,7 +105,6 @@ db:
       queue:
           calculationType: fixed
           size: 2097152
-      blockSize: 10m
 
   fs:
       filePathPrefix: /var/lib/m3db
@@ -411,7 +410,7 @@ func TestConfiguration(t *testing.T) {
       calculationType: fixed
       size: 2097152
     queueChannel: null
-    blockSize: 10m0s
+    blockSize: null
   repair:
     enabled: false
     interval: 2h0m0s

diff --git a/src/cmd/services/m3dbnode/main/main_index_test.go b/src/cmd/services/m3dbnode/main/main_index_test.go
@@ -353,7 +353,6 @@ db:
         queue:
             calculationType: fixed
             size: 2097152
-        blockSize: 10m
 
     fs:
         filePathPrefix: {{.DataDir}}

diff --git a/src/cmd/services/m3dbnode/main/main_test.go b/src/cmd/services/m3dbnode/main/main_test.go
@@ -498,7 +498,6 @@ db:
         queue:
             calculationType: fixed
             size: 2097152
-        blockSize: 10m
 
     fs:
         filePathPrefix: {{.DataDir}}

diff --git a/src/dbnode/config/m3dbnode-cluster-template.yml b/src/dbnode/config/m3dbnode-cluster-template.yml
@@ -96,7 +96,6 @@ db:
     queue:
         calculationType: fixed
         size: 2097152
-    blockSize: 10m
 
   fs:
     filePathPrefix: /var/lib/m3db
diff --git a/src/dbnode/config/m3dbnode-local-etcd.yml b/src/dbnode/config/m3dbnode-local-etcd.yml
@@ -78,7 +78,6 @@ db:
     queue:
         calculationType: fixed
         size: 2097152
-    blockSize: 10m
 
   fs:
     filePathPrefix: /var/lib/m3db

diff --git a/src/dbnode/config/m3dbnode-local.yml b/src/dbnode/config/m3dbnode-local.yml
@@ -75,7 +75,6 @@ db:
     queue:
         calculationType: fixed
         size: 2097152
-    blockSize: 10m
 
   fs:
     filePathPrefix: /var/lib/m3db

diff --git a/src/dbnode/generated/mocks/generate.go b/src/dbnode/generated/mocks/generate.go
@@ -20,7 +20,7 @@
 
 // mockgen rules for generating mocks for exported interfaces (reflection mode)
 
-//go:generate sh -c "mockgen -package=fs $PACKAGE/src/dbnode/persist/fs DataFileSetWriter,DataFileSetReader,DataFileSetSeeker,IndexFileSetWriter,IndexFileSetReader,IndexSegmentFileSetWriter,IndexSegmentFileSet,IndexSegmentFile | genclean -pkg $PACKAGE/src/dbnode/persist/fs -out $GOPATH/src/$PACKAGE/src/dbnode/persist/fs/fs_mock.go"
+//go:generate sh -c "mockgen -package=fs $PACKAGE/src/dbnode/persist/fs DataFileSetWriter,DataFileSetReader,DataFileSetSeeker,IndexFileSetWriter,IndexFileSetReader,IndexSegmentFileSetWriter,IndexSegmentFileSet,IndexSegmentFile,SnapshotMetadataFileWriter | genclean -pkg $PACKAGE/src/dbnode/persist/fs -out $GOPATH/src/$PACKAGE/src/dbnode/persist/fs/fs_mock.go"
 //go:generate sh -c "mockgen -package=xio $PACKAGE/src/dbnode/x/xio SegmentReader,SegmentReaderPool | genclean -pkg $PACKAGE/src/dbnode/x/xio -out $GOPATH/src/$PACKAGE/src/dbnode/x/xio/io_mock.go"
 //go:generate sh -c "mockgen -package=digest -destination=$GOPATH/src/$PACKAGE/src/dbnode/digest/digest_mock.go $PACKAGE/src/dbnode/digest ReaderWithDigest"
 //go:generate sh -c "mockgen -package=series $PACKAGE/src/dbnode/storage/series DatabaseSeries,QueryableBlockRetriever | genclean -pkg $PACKAGE/src/dbnode/storage/series -out $GOPATH/src/$PACKAGE/src/dbnode/storage/series/series_mock.go"