From 63018e7783bd53e6b109c578643cf67fc74cad5a Mon Sep 17 00:00:00 2001 From: docsir Date: Tue, 23 Nov 2021 11:46:22 +0800 Subject: [PATCH 1/7] add a function for the variable call of dm --- dumpling/export/status.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/dumpling/export/status.go b/dumpling/export/status.go index 6eb1cbeac25fc..0fc27f92a28a6 100644 --- a/dumpling/export/status.go +++ b/dumpling/export/status.go @@ -48,6 +48,23 @@ func (d *Dumper) runLogProgress(tctx *tcontext.Context) { } } +type Midparams struct { + CompletedTables float64 + FinishedBytes float64 + FinishedRows float64 + EstimateTotalRows float64 +} + +func (d *Dumper) GetParameters() (midparams *Midparams) { + conf := d.conf + var mid *Midparams + mid.CompletedTables = ReadCounter(finishedTablesCounter, conf.Labels) + mid.FinishedBytes = ReadGauge(finishedSizeGauge, conf.Labels) + mid.FinishedRows = ReadGauge(finishedRowsGauge, conf.Labels) + mid.EstimateTotalRows = ReadCounter(estimateTotalRowsCounter, conf.Labels) + return mid +} + func calculateTableCount(m DatabaseTables) int { cnt := 0 for _, tables := range m { From 1695c82aadcc30fa7d02b007e3cdd980cbc3d174 Mon Sep 17 00:00:00 2001 From: docsir Date: Tue, 23 Nov 2021 16:18:21 +0800 Subject: [PATCH 2/7] fix the first commit --- dumpling/export/status.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dumpling/export/status.go b/dumpling/export/status.go index 0fc27f92a28a6..8b1ffa865acd7 100644 --- a/dumpling/export/status.go +++ b/dumpling/export/status.go @@ -49,6 +49,7 @@ func (d *Dumper) runLogProgress(tctx *tcontext.Context) { } type Midparams struct { + TotalTables float64 CompletedTables float64 FinishedBytes float64 FinishedRows float64 @@ -58,6 +59,7 @@ type Midparams struct { func (d *Dumper) GetParameters() (midparams *Midparams) { conf := d.conf var mid *Midparams + mid.TotalTables = float64(calculateTableCount(conf.Tables)) mid.CompletedTables = ReadCounter(finishedTablesCounter, conf.Labels) mid.FinishedBytes = ReadGauge(finishedSizeGauge, conf.Labels) mid.FinishedRows = ReadGauge(finishedRowsGauge, conf.Labels) From b15e595064339ee407ea63533683f62b4c3528e8 Mon Sep 17 00:00:00 2001 From: docsir Date: Thu, 25 Nov 2021 17:19:42 +0800 Subject: [PATCH 3/7] fix and add unit test --- dumpling/export/status.go | 2 +- dumpling/export/status_test.go | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 dumpling/export/status_test.go diff --git a/dumpling/export/status.go b/dumpling/export/status.go index 8b1ffa865acd7..8c577b2551e12 100644 --- a/dumpling/export/status.go +++ b/dumpling/export/status.go @@ -58,7 +58,7 @@ type Midparams struct { func (d *Dumper) GetParameters() (midparams *Midparams) { conf := d.conf - var mid *Midparams + mid := &Midparams{} mid.TotalTables = float64(calculateTableCount(conf.Tables)) mid.CompletedTables = ReadCounter(finishedTablesCounter, conf.Labels) mid.FinishedBytes = ReadGauge(finishedSizeGauge, conf.Labels) diff --git a/dumpling/export/status_test.go b/dumpling/export/status_test.go new file mode 100644 index 0000000000000..503e6f75cea48 --- /dev/null +++ b/dumpling/export/status_test.go @@ -0,0 +1,32 @@ +// Copyright 2020 PingCAP, Inc. Licensed under Apache-2.0. + +package export + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestGetParameters(t *testing.T) { + conf := defaultConfigForTest(t) + d := &Dumper{conf: conf} + InitMetricsVector(conf.Labels) + + mid := d.GetParameters() + require.EqualValues(t, float64(0), mid.CompletedTables) + require.EqualValues(t, float64(0), mid.FinishedBytes) + require.EqualValues(t, float64(0), mid.FinishedRows) + require.EqualValues(t, float64(0), mid.EstimateTotalRows) + + AddCounter(finishedTablesCounter, conf.Labels, 10) + AddGauge(finishedSizeGauge, conf.Labels, 20) + AddGauge(finishedRowsGauge, conf.Labels, 30) + AddCounter(estimateTotalRowsCounter, conf.Labels, 40) + + mid = d.GetParameters() + require.EqualValues(t, float64(10), mid.CompletedTables) + require.EqualValues(t, float64(20), mid.FinishedBytes) + require.EqualValues(t, float64(30), mid.FinishedRows) + require.EqualValues(t, float64(40), mid.EstimateTotalRows) +} From 2443821a730a2244dcf74ea3c4cba4115cf18f98 Mon Sep 17 00:00:00 2001 From: docsir Date: Fri, 26 Nov 2021 16:50:21 +0800 Subject: [PATCH 4/7] adopt some suggestions --- dumpling/export/status.go | 24 ++++++++++-------------- dumpling/export/status_test.go | 2 +- 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/dumpling/export/status.go b/dumpling/export/status.go index 8c577b2551e12..b55b0c3f29f7c 100644 --- a/dumpling/export/status.go +++ b/dumpling/export/status.go @@ -15,8 +15,7 @@ import ( const logProgressTick = 2 * time.Minute func (d *Dumper) runLogProgress(tctx *tcontext.Context) { - conf := d.conf - totalTables := float64(calculateTableCount(conf.Tables)) + midd := d.GetParameters() logProgressTicker := time.NewTicker(logProgressTick) lastCheckpoint := time.Now() lastBytes := float64(0) @@ -29,21 +28,16 @@ func (d *Dumper) runLogProgress(tctx *tcontext.Context) { case <-logProgressTicker.C: nanoseconds := float64(time.Since(lastCheckpoint).Nanoseconds()) - completedTables := ReadCounter(finishedTablesCounter, conf.Labels) - finishedBytes := ReadGauge(finishedSizeGauge, conf.Labels) - finishedRows := ReadGauge(finishedRowsGauge, conf.Labels) - estimateTotalRows := ReadCounter(estimateTotalRowsCounter, conf.Labels) - tctx.L().Info("progress", - zap.String("tables", fmt.Sprintf("%.0f/%.0f (%.1f%%)", completedTables, totalTables, completedTables/totalTables*100)), - zap.String("finished rows", fmt.Sprintf("%.0f", finishedRows)), - zap.String("estimate total rows", fmt.Sprintf("%.0f", estimateTotalRows)), - zap.String("finished size", units.HumanSize(finishedBytes)), - zap.Float64("average speed(MiB/s)", (finishedBytes-lastBytes)/(1048576e-9*nanoseconds)), + zap.String("tables", fmt.Sprintf("%.0f/%.0f (%.1f%%)", midd.CompletedTables, midd.TotalTables, midd.CompletedTables/midd.TotalTables*100)), + zap.String("finished rows", fmt.Sprintf("%.0f", midd.FinishedRows)), + zap.String("estimate total rows", fmt.Sprintf("%.0f", midd.EstimateTotalRows)), + zap.String("finished size", units.HumanSize(midd.FinishedBytes)), + zap.Float64("average speed(MiB/s)", (midd.FinishedBytes-lastBytes)/(1048576e-9*nanoseconds)), ) lastCheckpoint = time.Now() - lastBytes = finishedBytes + lastBytes = midd.FinishedBytes } } } @@ -59,7 +53,9 @@ type Midparams struct { func (d *Dumper) GetParameters() (midparams *Midparams) { conf := d.conf mid := &Midparams{} - mid.TotalTables = float64(calculateTableCount(conf.Tables)) + if mid.TotalTables == 0 { + mid.TotalTables = float64(calculateTableCount(conf.Tables)) + } mid.CompletedTables = ReadCounter(finishedTablesCounter, conf.Labels) mid.FinishedBytes = ReadGauge(finishedSizeGauge, conf.Labels) mid.FinishedRows = ReadGauge(finishedRowsGauge, conf.Labels) diff --git a/dumpling/export/status_test.go b/dumpling/export/status_test.go index 503e6f75cea48..a002ac3365ffe 100644 --- a/dumpling/export/status_test.go +++ b/dumpling/export/status_test.go @@ -1,4 +1,4 @@ -// Copyright 2020 PingCAP, Inc. Licensed under Apache-2.0. +// Copyright 2021 PingCAP, Inc. Licensed under Apache-2.0. package export From 2b44493fca5f84b5a945d11861ccaffefdfeb359 Mon Sep 17 00:00:00 2001 From: docsir Date: Mon, 29 Nov 2021 15:52:08 +0800 Subject: [PATCH 5/7] make some changes --- dumpling/export/dump.go | 4 ++++ dumpling/export/status.go | 9 ++------- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/dumpling/export/dump.go b/dumpling/export/dump.go index 39637aa4f1b89..45266376001fd 100755 --- a/dumpling/export/dump.go +++ b/dumpling/export/dump.go @@ -12,6 +12,7 @@ import ( "sort" "strconv" "strings" + "sync/atomic" "time" // import mysql driver @@ -49,6 +50,7 @@ type Dumper struct { tidbPDClientForGC pd.Client selectTiDBTableRegionFunc func(tctx *tcontext.Context, conn *sql.Conn, meta TableMeta) (pkFields []string, pkVals [][]string, err error) + totalTables int64 } // NewDumper returns a new Dumper @@ -158,6 +160,8 @@ func (d *Dumper) Dump() (dumpErr error) { tctx.L().Info("cannot update select table region info for TiDB", zap.Error(err)) } + atomic.StoreInt64(&d.totalTables, int64(calculateTableCount(conf.Tables))) + rebuildConn := func(conn *sql.Conn) (*sql.Conn, error) { // make sure that the lock connection is still alive err1 := conCtrl.PingContext(tctx) diff --git a/dumpling/export/status.go b/dumpling/export/status.go index b55b0c3f29f7c..0865d100727e8 100644 --- a/dumpling/export/status.go +++ b/dumpling/export/status.go @@ -15,7 +15,6 @@ import ( const logProgressTick = 2 * time.Minute func (d *Dumper) runLogProgress(tctx *tcontext.Context) { - midd := d.GetParameters() logProgressTicker := time.NewTicker(logProgressTick) lastCheckpoint := time.Now() lastBytes := float64(0) @@ -27,9 +26,9 @@ func (d *Dumper) runLogProgress(tctx *tcontext.Context) { return case <-logProgressTicker.C: nanoseconds := float64(time.Since(lastCheckpoint).Nanoseconds()) - + midd := d.GetParameters() tctx.L().Info("progress", - zap.String("tables", fmt.Sprintf("%.0f/%.0f (%.1f%%)", midd.CompletedTables, midd.TotalTables, midd.CompletedTables/midd.TotalTables*100)), + zap.String("tables", fmt.Sprintf("%.0f/%.0f (%.1f%%)", midd.CompletedTables, float64(d.totalTables), midd.CompletedTables/float64(d.totalTables)*100)), zap.String("finished rows", fmt.Sprintf("%.0f", midd.FinishedRows)), zap.String("estimate total rows", fmt.Sprintf("%.0f", midd.EstimateTotalRows)), zap.String("finished size", units.HumanSize(midd.FinishedBytes)), @@ -43,7 +42,6 @@ func (d *Dumper) runLogProgress(tctx *tcontext.Context) { } type Midparams struct { - TotalTables float64 CompletedTables float64 FinishedBytes float64 FinishedRows float64 @@ -53,9 +51,6 @@ type Midparams struct { func (d *Dumper) GetParameters() (midparams *Midparams) { conf := d.conf mid := &Midparams{} - if mid.TotalTables == 0 { - mid.TotalTables = float64(calculateTableCount(conf.Tables)) - } mid.CompletedTables = ReadCounter(finishedTablesCounter, conf.Labels) mid.FinishedBytes = ReadGauge(finishedSizeGauge, conf.Labels) mid.FinishedRows = ReadGauge(finishedRowsGauge, conf.Labels) From a868f1ce86c2aefde5256da93edb333070c3ed0c Mon Sep 17 00:00:00 2001 From: docsir Date: Mon, 29 Nov 2021 16:18:21 +0800 Subject: [PATCH 6/7] keep parameter "mid.TotalTables" --- dumpling/export/status.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dumpling/export/status.go b/dumpling/export/status.go index 0865d100727e8..d67a95cfa04a4 100644 --- a/dumpling/export/status.go +++ b/dumpling/export/status.go @@ -46,11 +46,13 @@ type Midparams struct { FinishedBytes float64 FinishedRows float64 EstimateTotalRows float64 + TotalTables float64 } func (d *Dumper) GetParameters() (midparams *Midparams) { conf := d.conf mid := &Midparams{} + mid.TotalTables = float64(d.totalTables) mid.CompletedTables = ReadCounter(finishedTablesCounter, conf.Labels) mid.FinishedBytes = ReadGauge(finishedSizeGauge, conf.Labels) mid.FinishedRows = ReadGauge(finishedRowsGauge, conf.Labels) From 1c92830b40d54156ee2144757b7b4fd6fb24c98a Mon Sep 17 00:00:00 2001 From: docsir Date: Mon, 29 Nov 2021 16:48:12 +0800 Subject: [PATCH 7/7] alter --- dumpling/export/status.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dumpling/export/status.go b/dumpling/export/status.go index d67a95cfa04a4..359eae8ed7b1e 100644 --- a/dumpling/export/status.go +++ b/dumpling/export/status.go @@ -4,6 +4,7 @@ package export import ( "fmt" + "sync/atomic" "time" "github.com/docker/go-units" @@ -46,13 +47,13 @@ type Midparams struct { FinishedBytes float64 FinishedRows float64 EstimateTotalRows float64 - TotalTables float64 + TotalTables int64 } func (d *Dumper) GetParameters() (midparams *Midparams) { conf := d.conf mid := &Midparams{} - mid.TotalTables = float64(d.totalTables) + mid.TotalTables = atomic.LoadInt64(&d.totalTables) mid.CompletedTables = ReadCounter(finishedTablesCounter, conf.Labels) mid.FinishedBytes = ReadGauge(finishedSizeGauge, conf.Labels) mid.FinishedRows = ReadGauge(finishedRowsGauge, conf.Labels)