binlog, relay: support binlog sequence number larger than 999999

.github/workflows/binlog-999999.yml

@@ -0,0 +1,100 @@
+name: Test binlog 999999
+
+on:
+  push:
+    branches:
+      - master
+      - release-2.0
+  pull_request:
+    branches:
+      - master
+      - release-2.0
+
+jobs:
+  test-binlog-999999:
+    name: Test binlog 999999
+    runs-on: ubuntu-18.04
+
+    steps:
+      - name: Set up Go 1.13
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.13
+
+      - name: Check out code
+        uses: actions/checkout@v2
+
+      - name: Cache go modules
+        uses: actions/cache@v2
+        with:
+          path: ~/go/pkg/mod
+          key: ${{ runner.os }}-dm-${{ hashFiles('**/go.sum') }}
+
+      - name: Cache tools
+        uses: actions/cache@v2
+        with:
+          path: |
+            **/tools
+          key: ${{ runner.os }}-dm-tools-${{ hashFiles('**/tools/go.sum') }}
+
+      - name: Build DM binary
+        run: make dm_integration_test_build
+
+      - name: Setup CI environment
+        run: |
+          docker-compose -f ./tests/binlog_999999/docker-compose.yml up -d
+          curl http://download.pingcap.org/tidb-enterprise-tools-nightly-linux-amd64.tar.gz | tar xz
+          mv tidb-enterprise-tools-nightly-linux-amd64/bin/sync_diff_inspector bin/
+          curl http://download.pingcap.org/tidb-nightly-linux-amd64.tar.gz | tar xz
+          mv tidb-nightly-linux-amd64/bin/tidb-server bin/
+
+      - name: change binlog sequence number to 999998
+        run: |
+          while ! mysqladmin -h127.0.0.1 -P3306 -p123456 -uroot ping --connect-timeout=1 > /dev/null 2>&1 ; do
+            echo "wait mysql"
+            sleep 1
+          done
+          while ! mysqladmin -h127.0.0.1 -P3307 -p123456 -uroot ping --connect-timeout=1 > /dev/null 2>&1 ; do
+            echo "wait mysql"
+            sleep 1
+          done
+
+          echo "reset master;" | mysql -uroot -h127.0.0.1 -P3306 -p123456
+          echo "reset master;" | mysql -uroot -h127.0.0.1 -P3307 -p123456
+          docker-compose -f ./tests/binlog_999999/docker-compose.yml down
+
+          sudo mv /tmp/mysql1-data/mysql-bin.000001 /tmp/mysql1-data/mysql-bin.999998
+          sudo bash -c "echo './mysql-bin.999998' > /tmp/mysql1-data/mysql-bin.index"
+          sudo mv /tmp/mysql2-data/mysql-bin.000001 /tmp/mysql2-data/mysql-bin.999998
+          sudo bash -c "echo './mysql-bin.999998' > /tmp/mysql2-data/mysql-bin.index"
+          docker-compose -f ./tests/binlog_999999/docker-compose.yml up -d
+
+      - name: Run test cases
+        run: |
+          RESET_MASTER=false make integration_test CASE=incremental_mode
+          echo "show binary logs;" | mysql -uroot -h127.0.0.1 -P3306 -p123456 | grep -q "mysql-bin.1000000"
+          echo "show binary logs;" | mysql -uroot -h127.0.0.1 -P3307 -p123456 | grep -q "mysql-bin.1000000"
+
+      - name: Copy logs to hack permission
+        if: ${{ always() }}
+        run: |
+          mkdir ./logs
+          sudo cp -r -L /tmp/dm_test/incremental_mode/master/log ./logs/master
+          sudo cp -r -L /tmp/dm_test/incremental_mode/worker1/log ./logs/worker1
+          sudo cp -r -L /tmp/dm_test/incremental_mode/worker2/log ./logs/worker2
+          sudo chown -R runner ./logs
+
+      # Update logs as artifact seems not stable, so we set `continue-on-error: true` here.
+      - name: Upload logs
+        continue-on-error: true
+        uses: actions/upload-artifact@v2
+        if: ${{ always() }}
+        with:
+          name: test-binlog-999999
+          path: |
+            ./logs
+
+      # Debug via SSH if previous steps failed
+      - name: Set up tmate session
+        if: ${{ failure() }}
+        uses: mxschmitt/action-tmate@v2

go.mod

@@ -49,3 +49,5 @@ require (
 )
 
 go 1.13
+
+replace github.com/siddontang/go-mysql v1.1.1-0.20200824131207-0c5789dd0bd3 => github.com/lance6716/go-mysql v1.1.1-0.20210303100354-b0e44c2c5623

go.sum

@@ -599,6 +599,8 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/lance6716/go-mysql v1.1.1-0.20210303100354-b0e44c2c5623 h1:HesfLUZvjUaluKJzKetJ6odasdH/WQ/Tc8UO1oAs0WI=
+github.com/lance6716/go-mysql v1.1.1-0.20210303100354-b0e44c2c5623/go.mod h1:+W4RCzesQDI11HvIkaDjS8yM36SpAnGNQ7jmTLn5BnU=
 github.com/leodido/go-urn v1.1.0/go.mod h1:+cyI34gQWZcE1eQU7NVgKkkzdXDQHr1dBMtdAPozLkw=
 github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
 github.com/lib/pq v0.0.0-20181016162627-9eb73efc1fcc/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
@@ -1071,8 +1073,6 @@ github.com/siddontang/go-log v0.0.0-20190221022429-1e957dd83bed h1:KMgQoLJGCq1Io
 github.com/siddontang/go-log v0.0.0-20190221022429-1e957dd83bed/go.mod h1:yFdBgwXP24JziuRl2NMUahT7nGLNOKi1SIiFxMttVD4=
 github.com/siddontang/go-mysql v0.0.0-20200222075837-12e89848f047 h1:boyJ8EgQN/aC3grvx8QUoJrptt7RvneezSJSCbW25a4=
 github.com/siddontang/go-mysql v0.0.0-20200222075837-12e89848f047/go.mod h1:+W4RCzesQDI11HvIkaDjS8yM36SpAnGNQ7jmTLn5BnU=
-github.com/siddontang/go-mysql v1.1.1-0.20200824131207-0c5789dd0bd3 h1:t4iXtvFWrw0yfgf4Bm/KJtf4jOSnuXgs93ML+yLslSc=
-github.com/siddontang/go-mysql v1.1.1-0.20200824131207-0c5789dd0bd3/go.mod h1:+W4RCzesQDI11HvIkaDjS8yM36SpAnGNQ7jmTLn5BnU=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=

pkg/binlog/filename.go

@@ -35,17 +35,17 @@ type Filename struct {
 
 // LessThan checks whether this filename < other filename.
 func (f Filename) LessThan(other Filename) bool {
-	return f.BaseName == other.BaseName && f.Seq < other.Seq
+	return f.BaseName == other.BaseName && f.SeqInt64 < other.SeqInt64
 }
 
 // GreaterThanOrEqualTo checks whether this filename >= other filename.
 func (f Filename) GreaterThanOrEqualTo(other Filename) bool {
-	return f.BaseName == other.BaseName && f.Seq >= other.Seq
+	return f.BaseName == other.BaseName && f.SeqInt64 >= other.SeqInt64
 }
 
 // GreaterThan checks whether this filename > other filename.
 func (f Filename) GreaterThan(other Filename) bool {
-	return f.BaseName == other.BaseName && f.Seq > other.Seq
+	return f.BaseName == other.BaseName && f.SeqInt64 > other.SeqInt64
 }
 
 // ParseFilename parses a string representation binlog filename into a `Filename`.

pkg/binlog/filename_test.go

@@ -26,18 +26,22 @@ func (t *testFilenameSuite) TestFilenameCmp(c *C) {
 	f1 := Filename{
 		BaseName: "mysql-bin",
 		Seq:      "000001",
+		SeqInt64: 1,
 	}
 	f2 := Filename{
 		BaseName: "mysql-bin",
 		Seq:      "000002",
+		SeqInt64: 2,
 	}
 	f3 := Filename{
 		BaseName: "mysql-bin",
 		Seq:      "000001", // == f1
+		SeqInt64: 1,
 	}
 	f4 := Filename{
 		BaseName: "bin-mysq", // diff BaseName
 		Seq:      "000001",
+		SeqInt64: 1,
 	}
 
 	c.Assert(f1.LessThan(f2), IsTrue)

relay/writer/file.go

@@ -241,7 +241,7 @@ func (w *FileWriter) handleRotateEvent(ev *replication.BinlogEvent) (result Resu
 		if err == nil {
 			// update binlog filename if needed
 			nextFile := string(rotateEv.NextLogName)
-			if nextFile > currFile {
+			if gmysql.CompareBinlogFileName(nextFile, currFile) == 1 {
 				// record the next filename, but not create it.
 				// even it's a fake RotateEvent, we still need to record it,
 				// because if we do not specify the filename when creating the writer (like Auto-Position),

tests/binlog_999999/docker-compose.yml

@@ -0,0 +1,43 @@
+version: "3.7"
+
+networks:
+  db-networks:
+    ipam: # for static IP
+      config:
+        - subnet: "172.28.128.0/24"
+
+# re-usable extension fields, https://docs.docker.com/compose/compose-file/#extension-fields.
+x-node: &default-node
+  privileged: true
+  restart: always
+  environment:
+    MYSQL_ROOT_PASSWORD: "123456"
+
+services:
+  mysql1: # upstream MySQL-1
+    <<: *default-node
+    container_name: mysql1
+    hostname: mysql1
+    ports:
+      - "3306:3306"
+    networks:
+      db-networks:
+        ipv4_address: 172.28.128.4
+    image: mysql:5.7.32
+    command: --default-authentication-plugin=mysql_native_password --log-bin=/var/lib/mysql/mysql-bin --server-id=1 --binlog-format=ROW --gtid_mode=ON --enforce-gtid-consistency=true
+    volumes:
+      - /tmp/mysql1-data:/var/lib/mysql
+
+  mysql2: # upstream MySQL-2
+    <<: *default-node
+    container_name: mysql2
+    hostname: mysql2
+    ports:
+      - "3307:3306"
+    networks:
+      db-networks:
+        ipv4_address: 172.28.128.5
+    image: mysql:8.0.23
+    command: --default-authentication-plugin=mysql_native_password --log-bin=/var/lib/mysql/mysql-bin --server-id=1 --binlog-format=ROW --gtid_mode=ON --enforce-gtid-consistency=true
+    volumes:
+      - /tmp/mysql2-data:/var/lib/mysql

tests/incremental_mode/data/db1.increment.sql

@@ -2,6 +2,7 @@ use incremental_mode;
 insert into t1 (id, name) values (3, 'Eddard Stark');
 update t1 set name = 'Arya Stark' where id = 1;
 update t1 set name = 'Catelyn Stark' where name = 'catelyn';
+flush logs;
 
 -- test multi column index with generated column
 alter table t1 add column info json;
@@ -11,6 +12,7 @@ insert into t1 (id, name, info) values (4, 'gentest', '{"id": 123}');
 insert into t1 (id, name, info) values (5, 'gentest', '{"id": 124}');
 update t1 set info = '{"id": 120}' where id = 1;
 update t1 set info = '{"id": 121}' where id = 2;
+flush logs;
 update t1 set info = '{"id": 122}' where id = 3;
 
 -- test genColumnCache is reset after ddl

tests/incremental_mode/data/db2.increment.sql

@@ -1,2 +1,4 @@
 use incremental_mode;
+flush logs;
+flush logs;
 delete from t2 where name = 'Sansa';