mydump: dynamic batch size

lightning/mydump/region.go

@@ -50,9 +50,16 @@ func MakeTableRegions(meta *MDTableMeta, columns int, batchSize int64) ([]*Table
 	// Split files into regions
 	filesRegions := make(regionSlice, 0, len(meta.DataFiles))
 
+	// import() step will not be concurrent.
+	// If multiple Batch end times are close, it will result in multiple
+	// Batch import serials. `batchSizeScale` used to implement dynamic
+	// batch size: curBatchSize = batchSize / int64(batchSizeScale-curEngineID)
+	const batchSizeScale = 20
+
 	prevRowIDMax := int64(0)
 	curEngineID := 0
 	curEngineSize := int64(0)
+	curBatchSize := batchSize / batchSizeScale
 	for _, dataFile := range meta.DataFiles {
 		dataFileInfo, err := os.Stat(dataFile)
 		if err != nil {
@@ -75,9 +82,14 @@ func MakeTableRegions(meta *MDTableMeta, columns int, batchSize int64) ([]*Table
 		prevRowIDMax = rowIDMax
 
 		curEngineSize += dataFileSize
-		if curEngineSize > batchSize {
+		if curEngineSize > curBatchSize {
 			curEngineSize = 0
 			curEngineID++
+			if curEngineID < batchSizeScale {
+				curBatchSize = batchSize / int64(batchSizeScale-curEngineID)
+			} else {
+				curBatchSize = batchSize
+			}
 		}
 	}
 

tests/checkpoint_engines/run.sh

@@ -12,7 +12,7 @@ run_lightning
 # Check that we have indeed opened 4 engines
 OPEN_ENGINES_COUNT=$(grep 'open engine' "$TEST_DIR/lightning-checkpoint-engines.log" | wc -l)
 echo "Number of open engines: $OPEN_ENGINES_COUNT"
-[ "$OPEN_ENGINES_COUNT" -eq 4 ]
+[ "$OPEN_ENGINES_COUNT" -eq 5 ]
 
 # Check that everything is correctly imported
 run_sql 'SELECT count(*), sum(c) FROM cpeng.a'
@@ -29,7 +29,7 @@ run_sql 'DROP DATABASE cpeng;'
 
 export GOFAIL_FAILPOINTS='github.com/pingcap/tidb-lightning/lightning/restore/SlowDownImport=sleep(500);github.com/pingcap/tidb-lightning/lightning/restore/FailIfStatusBecomes=return(120)'
 set +e
-for i in $(seq 4); do
+for i in $(seq 5); do
     echo "******** Importing Table Now (step $i/4) ********"
     run_lightning 2> /dev/null
     [ $? -ne 0 ] || exit 1
@@ -52,7 +52,7 @@ check_contains 'sum(c): 46'
 run_sql 'DROP DATABASE cpeng;'
 
 set +e
-for i in $(seq 4); do
+for i in $(seq 5); do
     echo "******** Importing Table Now (step $i/4) ********"
     run_lightning mysql 2> /dev/null
     [ $? -ne 0 ] || exit 1