Merge pull request pingcap#1 from tangenta/fast-demo

Fast demo

.gitignore

@@ -1,2 +1,4 @@
 .DS_Store
 bin/
+
+.idea

Makefile

@@ -15,7 +15,7 @@ endif
 build: bin/dumpling
 
 bin/%: cmd/%/main.go $(wildcard v4/**/*.go)
-	$(GO) build $(GOFLAGS) -o $@ $<
+	$(GO) build $(GOFLAGS) -tags codes -o $@ $<
 
 test:
 	$(GO) list ./... | xargs $(GO) test $(GOFLAGS)

cmd/dumpling/main.go

@@ -24,12 +24,13 @@ import (
 )
 
 var (
-	database string
-	host     string
-	user     string
-	port     int
-	password string
-	threads  int
+	database  string
+	host      string
+	user      string
+	port      int
+	password  string
+	threads   int
+	outputDir string
 )
 
 func init() {
@@ -50,24 +51,33 @@ func init() {
 
 	flag.IntVar(&threads, "threads", 4, "Number of goroutines to use, default 4")
 	flag.IntVar(&threads, "t", 4, "Number of goroutines to use, default 4")
+
+	flag.StringVar(&outputDir, "output", ".", "Output directory")
+	flag.StringVar(&outputDir, "o", ".", "Output directory")
 }
 
 func main() {
 	flag.Parse()
 	println(cli.LongVersion())
 
-	var conf dumpling.Config
+	conf := dumpling.DefaultConfig()
 	conf.Database = database
 	conf.Host = host
 	conf.User = user
 	conf.Port = port
 	conf.Password = password
 	conf.Threads = threads
+	conf.OutputDirPath = outputDir
 
-	err := dumpling.Dump(&conf)
+	err := os.MkdirAll(outputDir, 0755)
+	if err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+	err = dumpling.Dump(conf)
 	if err != nil {
 		fmt.Println(err)
-		os.Exit(-1)
+		os.Exit(1)
 	}
 	return
 }

config.go

@@ -2,6 +2,7 @@ package dumpling
 
 import (
 	"fmt"
+	"github.com/pingcap/dumpling/v4/export"
 )
 
 type Config struct {
@@ -11,8 +12,59 @@ type Config struct {
 	Port     int
 	Password string
 	Threads  int
+
+	LineSplitter  string
+	Logger        export.Logger
+	OutputSize    int
+	OutputDirPath string
+}
+
+func DefaultConfig() *Config {
+	return &Config{
+		Database:      "",
+		Host:          "127.0.0.1",
+		User:          "root",
+		Port:          3306,
+		Password:      "",
+		Threads:       4,
+		LineSplitter:  "\n",
+		Logger:        &DummyLogger{},
+		OutputSize:    export.UnspecifiedSize,
+		OutputDirPath: ".",
+	}
 }
 
 func (conf *Config) getDSN(db string) string {
 	return fmt.Sprintf("%s:%s@tcp(%s:%d)/%s", conf.User, conf.Password, conf.Host, conf.Port, db)
 }
+
+func extractOutputConfig(conf *Config) *export.Config {
+	return &export.Config{
+		LineSplitter:  conf.LineSplitter,
+		Logger:        conf.Logger,
+		OutputSize:    conf.OutputSize,
+		OutputDirPath: conf.OutputDirPath,
+	}
+}
+
+type DummyLogger struct{}
+
+func (d *DummyLogger) Debug(format string, args ...interface{}) {
+	fmt.Printf(format, args...)
+	fmt.Println()
+}
+
+func (d *DummyLogger) Info(format string, args ...interface{}) {
+	fmt.Printf(format, args...)
+	fmt.Println()
+}
+
+func (d *DummyLogger) Warn(format string, args ...interface{}) {
+	fmt.Printf(format, args...)
+	fmt.Println()
+}
+
+func (d *DummyLogger) Error(format string, args ...interface{}) {
+	fmt.Printf(format, args...)
+	fmt.Println()
+}

dump.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"database/sql"
 	"fmt"
+	"github.com/pingcap/dumpling/v4/export"
 	"strings"
 	"sync"
 
@@ -35,7 +36,8 @@ func Dump(conf *Config) error {
 	pool.Close()
 
 	for _, database := range databases {
-		if err := dumpDatabase(context.Background(), conf.getDSN(database), database, &dummyWriter{}); err != nil {
+		fsWriter := NewFileSystemWriter(extractOutputConfig(conf))
+		if err := dumpDatabase(context.Background(), conf.getDSN(database), database, fsWriter); err != nil {
 			return err
 		}
 	}
@@ -87,13 +89,26 @@ func showTables(db *sql.DB) ([]string, error) {
 	return res.data, nil
 }
 
+func showCreateDatabase(db *sql.DB, database string) (string, error) {
+	var oneRow [2]string
+	handleOneRow := func(rows *sql.Rows) error {
+		return rows.Scan(&oneRow[0], &oneRow[1])
+	}
+	query := fmt.Sprintf("SHOW CREATE DATABASE %s", database)
+	err := simpleQuery(db, query, handleOneRow)
+	if err != nil {
+		return "", err
+	}
+	return oneRow[1], nil
+}
+
 func showCreateTable(db *sql.DB, database, table string) (string, error) {
 	var oneRow [2]string
 	handleOneRow := func(rows *sql.Rows) error {
 		return rows.Scan(&oneRow[0], &oneRow[1])
 	}
-	sql := fmt.Sprintf("SHOW CREATE TABLE %s.%s", database, table)
-	err := simpleQuery(db, sql, handleOneRow)
+	query := fmt.Sprintf("SHOW CREATE TABLE %s.%s", database, table)
+	err := simpleQuery(db, query, handleOneRow)
 	if err != nil {
 		return "", err
 	}
@@ -107,7 +122,13 @@ func dumpDatabase(ctx context.Context, dsn, dbName string, writer Writer) error
 	}
 	defer db.Close()
 
-	writer.WriteDatabaseMeta(ctx, dbName)
+	createDatabaseSQL, err := showCreateDatabase(db, dbName)
+	if err != nil {
+		return err
+	}
+	if err := writer.WriteDatabaseMeta(ctx, dbName, createDatabaseSQL); err != nil {
+		return err
+	}
 
 	tables, err := showTables(db)
 	if err != nil {
@@ -125,7 +146,10 @@ func dumpDatabase(ctx context.Context, dsn, dbName string, writer Writer) error
 				res[ith] = err
 				return
 			}
-			writer.WriteTableMeta(ctx, dbName, table, createTableSQL)
+			if err := writer.WriteTableMeta(ctx, dbName, table, createTableSQL); err != nil {
+				res[ith] = err
+				return
+			}
 
 			gRL.getToken()
 			tableIR, err := dumpTable(ctx, db, dbName, table)
@@ -165,7 +189,7 @@ type tableDumper interface {
 	finishTable(ctx context.Context)
 }
 
-func dumpTable(ctx context.Context, db *sql.DB, database, table string) (TableDataIR, error) {
+func dumpTable(ctx context.Context, db *sql.DB, database, table string) (export.TableDataIR, error) {
 	colTypes, err := getColumnTypes(db, table)
 	if err != nil {
 		return nil, err

go.mod

@@ -1,5 +1,8 @@
 module github.com/pingcap/dumpling
 
-go 1.13
+require (
+	github.com/go-sql-driver/mysql v1.4.1
+	github.com/pingcap/check v0.0.0-20191107115940-caf2b9e6ccf4
+)
 
-require github.com/go-sql-driver/mysql v1.4.1
+go 1.13

go.sum

@@ -1,2 +1,71 @@
 github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA=
 github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
+github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8/go.mod h1:B1+S9LNcuMyLH/4HMTViQOJevkGiik3wW2AN9zb2fNQ=
+github.com/pingcap/check v0.0.0-20191107115940-caf2b9e6ccf4 h1:iRtOAQ6FXkY/BGvst3CDfTva4nTqh6CL8WXvanLdbu0=
+github.com/pingcap/check v0.0.0-20191107115940-caf2b9e6ccf4/go.mod h1:PYMCGwN0JHjoqGr3HrZoD+b8Tgx8bKnArhSq8YVzUMc=
+github.com/pingcap/errors v0.11.0 h1:DCJQB8jrHbQ1VVlMFIrbj2ApScNNotVmkSNplu2yUt4=
+github.com/pingcap/errors v0.11.0/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
+github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9 h1:AJD9pZYm72vMgPcQDww9rkZ1DnWfl0pXV3BOWlkYIjA=
+github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8=
+github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
+go.uber.org/atomic v1.5.0 h1:OI5t8sDa1Or+q8AeE+yKeB/SDYioSHAgcVljj9JIETY=
+go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
+go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
+go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4=
+go.uber.org/multierr v1.4.0 h1:f3WCSC2KzAcBXGATIxAB1E2XuCpNU255wNKZ505qi3E=
+go.uber.org/multierr v1.4.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4=
+go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee h1:0mgffUl7nfd+FpvXMVz4IDEaUSmT1ysygQC7qYo7sG4=
+go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA=
+go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
+go.uber.org/zap v1.12.0 h1:dySoUQPFBGj6xwjmBzageVL8jGi8uxc6bEmJQjA06bw=
+go.uber.org/zap v1.12.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191107010934-f79515f33823 h1:akkRBeitX2EZP59KdtKw310CI4WGPCNPyrLbE7WZA8Y=
+golang.org/x/tools v0.0.0-20191107010934-f79515f33823/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
+gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8=
+gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
+gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM=
+honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=