test: test number range sharding algorithm (#279)

* test: test number range sharding algorithm
CECTC · Dec 12, 2022 · 1d12859 · 1d12859
1 parent 5a461ec
commit 1d12859
Show file tree

Hide file tree

Showing 12 changed files with 7,547 additions and 23 deletions.
diff --git a/docker/conf/config_shd.yaml → docker/conf/config_shd_mod.yaml b/docker/conf/config_shd.yaml → docker/conf/config_shd_mod.yaml
diff --git a/docker/conf/config_shd_range.yaml b/docker/conf/config_shd_range.yaml
@@ -0,0 +1,104 @@
+probe_port: 9999
+termination_drain_duration: 3s
+app_config:
+  # appid, replace with your own appid
+  svc:
+    distributed_transaction:
+      retry_dead_threshold: 130000
+      rollback_retry_timeout_unlock_enable: true
+      etcd_config:
+        endpoints:
+          - etcd:2379
+
+    listeners:
+      - protocol_type: mysql
+        socket_address:
+          address: 0.0.0.0
+          port: 13306
+        config:
+          users:
+            dksl: "123456"
+          server_version: "8.0.27"
+        executor: redirect
+
+    executors:
+      - name: redirect
+        mode: shd
+        config:
+          transaction_timeout: 60000
+          db_groups:
+            - name: world_0
+              load_balance_algorithm: RandomWeight
+              data_sources:
+                - name: world_0
+                  weight: r10w10
+            - name: world_1
+              load_balance_algorithm: RandomWeight
+              data_sources:
+                - name: world_1
+                  weight: r10w10
+          global_tables:
+            - country
+            - countrylanguage
+            - pt_city_0
+          logic_tables:
+            - db_name: world
+              table_name: city
+              allow_full_scan: true
+              sharding_rule:
+                column: id
+                sharding_algorithm: NumberRange
+                config:
+                  "0": "1-500"
+                  "1": "500-1000"
+                  "2": "1000-1500"
+                  "3": "1500-2000"
+                  "4": "2000-2500"
+                  "5": "2500-3000"
+                  "6": "3000-3500"
+                  "7": "3500-4000"
+                  "8": "4000-4500"
+                  "9": "4500-5000"
+              sequence_generator:
+                # type: segment
+                type: snowflake
+                config:
+                  worker_id: 123
+                  # dsn: root:123456@tcp(dbpack-mysql1:3306)/world?timeout=10s&readTimeout=10s&writeTimeout=10s&parseTime=true&loc=Local&charset=utf8mb4,utf8
+              topology:
+                "0": 0-4
+                "1": 5-9
+          shadow_rules:
+            - table_name: city
+              column: country_code
+              expr: "%s == \"US\""
+              shadow_table_prefix: pt_
+
+    data_source_cluster:
+      - name: world_0
+        capacity: 10
+        max_capacity: 20
+        idle_timeout: 60s
+        dsn: root:123456@tcp(dbpack-mysql1:3306)/world?timeout=10s&readTimeout=10s&writeTimeout=10s&parseTime=true&loc=Local&charset=utf8mb4,utf8
+        ping_interval: 20s
+        ping_times_for_change_status: 3
+        filters:
+          - mysqlDTFilter
+
+      - name: world_1
+        capacity: 10
+        max_capacity: 20
+        idle_timeout: 60s
+        dsn: root:123456@tcp(dbpack-mysql2:3306)/world?timeout=60s&readTimeout=60s&writeTimeout=60s&parseTime=true&loc=Local&charset=utf8mb4,utf8
+        ping_interval: 20s
+        ping_times_for_change_status: 3
+        filters:
+          - mysqlDTFilter
+
+    filters:
+      - name: mysqlDTFilter
+        kind: MysqlDistributedTransaction
+        conf:
+          appid: svc
+          lock_retry_interval: 50ms
+          lock_retry_times: 30
diff --git a/docker/docker-compose-shd.yaml → docker/docker-compose-shd-mod.yaml b/docker/docker-compose-shd.yaml → docker/docker-compose-shd-mod.yaml
@@ -27,7 +27,7 @@ services:
     environment:
       MYSQL_ROOT_PASSWORD: "123456"
     volumes:
-    - ./scripts/world_0.sql:/docker-entrypoint-initdb.d/world_0.sql:rw
+    - ./scripts/mod_world_0.sql:/docker-entrypoint-initdb.d/world_0.sql:rw
     command: ['mysqld', '--character-set-server=utf8mb4', '--collation-server=utf8mb4_unicode_ci']
   mysql2:
     image: mysql:8.0
@@ -39,7 +39,7 @@ services:
     environment:
       MYSQL_ROOT_PASSWORD: "123456"
     volumes:
-      - ./scripts/world_1.sql:/docker-entrypoint-initdb.d/world_1.sql:rw
+      - ./scripts/mod_world_1.sql:/docker-entrypoint-initdb.d/world_1.sql:rw
     command: [ 'mysqld', '--character-set-server=utf8mb4', '--collation-server=utf8mb4_unicode_ci' ]
   dbpack:
     build:
@@ -51,7 +51,7 @@ services:
     ports:
       - "13306:13306"
     volumes:
-      - ./conf/config_shd.yaml:/config.yaml
+      - ./conf/config_shd_mod.yaml:/config.yaml
       - ./scripts/wait-for-mysql.sh:/wait-for-mysql.sh
     depends_on:
       - etcd

diff --git a/docker/docker-compose-shd-range.yaml b/docker/docker-compose-shd-range.yaml
@@ -0,0 +1,63 @@
+version: "2.3"
+services:
+  etcd:
+    image: docker.io/bitnami/etcd:3
+    container_name: etcd
+    environment:
+      - ALLOW_NONE_AUTHENTICATION=yes
+      - ETCD_NAME=etcd
+      - ETCD_INITIAL_ADVERTISE_PEER_URLS=http://etcd:2380
+      - ETCD_LISTEN_PEER_URLS=http://0.0.0.0:2380
+      - ETCD_LISTEN_CLIENT_URLS=http://0.0.0.0:2379
+      - ETCD_ADVERTISE_CLIENT_URLS=http://etcd:2379
+      - ETCD_INITIAL_CLUSTER_TOKEN=etcd-cluster
+      - ETCD_INITIAL_CLUSTER=etcd=http://etcd:2380
+      - ETCD_INITIAL_CLUSTER_STATE=new
+    networks:
+      - local
+    ports:
+      - "2379:2379"
+  mysql1:
+    image: mysql:8.0
+    container_name: dbpack-mysql1
+    networks:
+      - local
+    ports:
+    - "3306:3306"
+    environment:
+      MYSQL_ROOT_PASSWORD: "123456"
+    volumes:
+    - ./scripts/range_world_0.sql:/docker-entrypoint-initdb.d/world_0.sql:rw
+    command: ['mysqld', '--character-set-server=utf8mb4', '--collation-server=utf8mb4_unicode_ci']
+  mysql2:
+    image: mysql:8.0
+    container_name: dbpack-mysql2
+    networks:
+      - local
+    ports:
+      - "3307:3306"
+    environment:
+      MYSQL_ROOT_PASSWORD: "123456"
+    volumes:
+      - ./scripts/range_world_1.sql:/docker-entrypoint-initdb.d/world_1.sql:rw
+    command: [ 'mysqld', '--character-set-server=utf8mb4', '--collation-server=utf8mb4_unicode_ci' ]
+  dbpack:
+    build:
+      context: ${PWD}
+      dockerfile: ./docker/Dockerfile
+    container_name: dbpack
+    networks:
+      - local
+    ports:
+      - "13306:13306"
+    volumes:
+      - ./conf/config_shd_range.yaml:/config.yaml
+      - ./scripts/wait-for-mysql.sh:/wait-for-mysql.sh
+    depends_on:
+      - etcd
+      - mysql1
+      - mysql2
+    command: ["./wait-for-mysql.sh","--","/dbpack", "start", "-c", "config.yaml"]
+networks:
+  local:
+    name: dbpack
diff --git a/docker/scripts/world_0.sql → docker/scripts/mod_world_0.sql b/docker/scripts/world_0.sql → docker/scripts/mod_world_0.sql
@@ -34,7 +34,7 @@ CREATE TABLE `city_0` (
                           `district` char(20) NOT NULL DEFAULT '',
                           `population` int NOT NULL DEFAULT '0',
                           PRIMARY KEY (`id`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
 
 DROP TABLE IF EXISTS `pt_city_0`;
 CREATE TABLE `pt_city_0` (
@@ -44,7 +44,7 @@ CREATE TABLE `pt_city_0` (
                           `district` char(20) NOT NULL DEFAULT '',
                           `population` int NOT NULL DEFAULT '0',
                           PRIMARY KEY (`id`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
 
 -- ----------------------------
 -- Records of city_0
@@ -470,7 +470,7 @@ CREATE TABLE `city_1` (
                           `district` char(20) NOT NULL DEFAULT '',
                           `population` int NOT NULL DEFAULT '0',
                           PRIMARY KEY (`id`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
 
 -- ----------------------------
 -- Records of city_1
@@ -897,7 +897,7 @@ CREATE TABLE `city_2` (
                           `district` char(20) NOT NULL DEFAULT '',
                           `population` int NOT NULL DEFAULT '0',
                           PRIMARY KEY (`id`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
 
 -- ----------------------------
 -- Records of city_2
@@ -1324,7 +1324,7 @@ CREATE TABLE `city_3` (
                           `district` char(20) NOT NULL DEFAULT '',
                           `population` int NOT NULL DEFAULT '0',
                           PRIMARY KEY (`id`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
 
 -- ----------------------------
 -- Records of city_3
@@ -1751,7 +1751,7 @@ CREATE TABLE `city_4` (
                           `district` char(20) NOT NULL DEFAULT '',
                           `population` int NOT NULL DEFAULT '0',
                           PRIMARY KEY (`id`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
 
 -- ----------------------------
 -- Records of city_4
@@ -2188,7 +2188,7 @@ CREATE TABLE `country` (
                            `capital` int DEFAULT NULL,
                            `code2` char(2) NOT NULL DEFAULT '',
                            PRIMARY KEY (`code`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
 
 -- ----------------------------
 -- Records of country
@@ -2447,7 +2447,7 @@ CREATE TABLE `countrylanguage` (
                                    PRIMARY KEY (`country_code`,`language`),
                                    KEY `country_code` (`country_code`),
                                    CONSTRAINT `countrylanguage_ibfk_1` FOREIGN KEY (`country_code`) REFERENCES `country` (`code`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
 
 -- ----------------------------
 -- Records of countrylanguage

diff --git a/docker/scripts/world_1.sql → docker/scripts/mod_world_1.sql b/docker/scripts/world_1.sql → docker/scripts/mod_world_1.sql
@@ -34,7 +34,7 @@ CREATE TABLE `city_5` (
                           `district` char(20) NOT NULL DEFAULT '',
                           `population` int NOT NULL DEFAULT '0',
                           PRIMARY KEY (`id`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
 
 -- ----------------------------
 -- Records of city_5
@@ -461,7 +461,7 @@ CREATE TABLE `city_6` (
                           `district` char(20) NOT NULL DEFAULT '',
                           `population` int NOT NULL DEFAULT '0',
                           PRIMARY KEY (`id`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
 
 -- ----------------------------
 -- Records of city_6
@@ -888,7 +888,7 @@ CREATE TABLE `city_7` (
                           `district` char(20) NOT NULL DEFAULT '',
                           `population` int NOT NULL DEFAULT '0',
                           PRIMARY KEY (`id`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
 
 -- ----------------------------
 -- Records of city_7
@@ -1315,7 +1315,7 @@ CREATE TABLE `city_8` (
                           `district` char(20) NOT NULL DEFAULT '',
                           `population` int NOT NULL DEFAULT '0',
                           PRIMARY KEY (`id`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
 
 -- ----------------------------
 -- Records of city_8
@@ -1742,7 +1742,7 @@ CREATE TABLE `city_9` (
                           `district` char(20) NOT NULL DEFAULT '',
                           `population` int NOT NULL DEFAULT '0',
                           PRIMARY KEY (`id`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
 
 -- ----------------------------
 -- Records of city_9
@@ -2179,7 +2179,7 @@ CREATE TABLE `country` (
                            `capital` int DEFAULT NULL,
                            `code2` char(2) NOT NULL DEFAULT '',
                            PRIMARY KEY (`code`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
 
 -- ----------------------------
 -- Records of country
@@ -2438,7 +2438,7 @@ CREATE TABLE `countrylanguage` (
                                    PRIMARY KEY (`country_code`,`language`),
                                    KEY `country_code` (`country_code`),
                                    CONSTRAINT `countrylanguage_ibfk_1` FOREIGN KEY (`country_code`) REFERENCES `country` (`code`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
 
 -- ----------------------------
 -- Records of countrylanguage