New explain format: VTEXPLAIN

doc/releasenotes/15_0_0_summary.md

@@ -9,6 +9,7 @@
 - [VDiff2](#vdiff2)
 - [Mysql Compatibility](#mysql-compatibility)
 - [Durability Policy](#durability-policy)
+- [New EXPLAIN format](#new-explain-format)
 
 ## Known Issues
 
@@ -184,4 +185,10 @@ This is different from the existing `autocommit` parameter where the query is se
 #### Cross Cell
 
 A new durability policy `cross_cell` is now supported. `cross_cell` durability policy only allows replica tablets from a different cell than the current primary to
-send semi-sync ACKs. This ensures that any committed write exists in at least 2 tablets belonging to different cells.
+send semi-sync ACKs. This ensures that any committed write exists in at least 2 tablets belonging to different cells.
+
+### New EXPLAIN format
+
+#### FORMAT=vtexplain
+
+With this new `explain` format, you can get an output that is very similar to the command line `vtexplain` app, but from a running `vtgate`, through a MySQL query.

go/test/endtoend/vtgate/queries/vtexplain/main_test.go

@@ -0,0 +1,155 @@
+/*
+Copyright 2022 The Vitess Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package vtexplain
+
+import (
+	"context"
+	_ "embed"
+	"flag"
+	"os"
+	"testing"
+
+	"vitess.io/vitess/go/vt/vtgate/planbuilder"
+
+	"github.com/stretchr/testify/require"
+
+	"vitess.io/vitess/go/mysql"
+	"vitess.io/vitess/go/test/endtoend/cluster"
+	"vitess.io/vitess/go/test/endtoend/utils"
+)
+
+var (
+	clusterInstance *cluster.LocalProcessCluster
+	vtParams        mysql.ConnParams
+	shardedKs       = "ks"
+
+	shardedKsShards = []string{"-40", "40-80", "80-c0", "c0-"}
+	Cell            = "test"
+	//go:embed schema.sql
+	shardedSchemaSQL string
+
+	//go:embed vschema.json
+	shardedVSchema string
+)
+
+func TestMain(m *testing.M) {
+	defer cluster.PanicHandler(nil)
+	flag.Parse()
+
+	exitCode := func() int {
+		clusterInstance = cluster.NewCluster(Cell, "localhost")
+		defer clusterInstance.Teardown()
+
+		// Start topo server
+		err := clusterInstance.StartTopo()
+		if err != nil {
+			return 1
+		}
+
+		// Start keyspace
+		sKs := &cluster.Keyspace{
+			Name:      shardedKs,
+			SchemaSQL: shardedSchemaSQL,
+			VSchema:   shardedVSchema,
+		}
+
+		err = clusterInstance.StartKeyspace(*sKs, shardedKsShards, 0, false)
+		if err != nil {
+			return 1
+		}
+
+		// Start vtgate
+		clusterInstance.VtGatePlannerVersion = planbuilder.Gen4
+		err = clusterInstance.StartVtgate()
+		if err != nil {
+			return 1
+		}
+
+		vtParams = mysql.ConnParams{
+			Host: clusterInstance.Hostname,
+			Port: clusterInstance.VtgateMySQLPort,
+		}
+
+		return m.Run()
+	}()
+	os.Exit(exitCode)
+}
+
+func TestVtGateVtExplain(t *testing.T) {
+	conn, err := mysql.Connect(context.Background(), &vtParams)
+	require.NoError(t, err)
+	defer conn.Close()
+
+	utils.AssertContainsError(t, conn,
+		`explain format=vtexplain insert into user (id,lookup,lookup_unique) values (4,'apa','foo'),(5,'apa','bar'),(6,'monkey','nobar')`,
+		"vtexplain will actually run queries")
+
+	expected := `[[INT32(0) VARCHAR("ks") VARCHAR("-40") VARCHAR("begin")]` +
+		` [INT32(0) VARCHAR("ks") VARCHAR("-40") VARCHAR("insert into lookup(lookup, id, keyspace_id) values ('apa', 1, '\x16k@\xb4J\xbaK\xd6'), ('apa', 2, '\x06\xe7\xea\\\"Βp\x8f') on duplicate key update lookup = values(lookup), id = values(id), keyspace_id = values(keyspace_id)")]` +
+		` [INT32(1) VARCHAR("ks") VARCHAR("40-80") VARCHAR("begin")]` +
+		` [INT32(1) VARCHAR("ks") VARCHAR("40-80") VARCHAR("insert into lookup(lookup, id, keyspace_id) values ('monkey', 3, 'N\xb1\x90ɢ\xfa\x16\x9c') on duplicate key update lookup = values(lookup), id = values(id), keyspace_id = values(keyspace_id)")]` +
+		` [INT32(2) VARCHAR("ks") VARCHAR("-40") VARCHAR("commit")]` +
+		` [INT32(3) VARCHAR("ks") VARCHAR("40-80") VARCHAR("commit")]` +
+		` [INT32(4) VARCHAR("ks") VARCHAR("40-80") VARCHAR("begin")]` +
+		` [INT32(4) VARCHAR("ks") VARCHAR("40-80") VARCHAR("insert into lookup_unique(lookup_unique, keyspace_id) values ('monkey', 'N\xb1\x90ɢ\xfa\x16\x9c')")]` +
+		` [INT32(5) VARCHAR("ks") VARCHAR("-40") VARCHAR("begin")]` +
+		` [INT32(5) VARCHAR("ks") VARCHAR("-40") VARCHAR("insert into lookup_unique(lookup_unique, keyspace_id) values ('apa', '\x16k@\xb4J\xbaK\xd6'), ('bandar', '\x06\xe7\xea\\\"Βp\x8f')")]` +
+		` [INT32(6) VARCHAR("ks") VARCHAR("40-80") VARCHAR("commit")]` +
+		` [INT32(7) VARCHAR("ks") VARCHAR("-40") VARCHAR("commit")]` +
+		` [INT32(8) VARCHAR("ks") VARCHAR("40-80") VARCHAR("begin")]` +
+		` [INT32(8) VARCHAR("ks") VARCHAR("40-80") VARCHAR("insert into ` + "`user`" + `(id, lookup, lookup_unique) values (3, 'monkey', 'monkey')")]` +
+		` [INT32(9) VARCHAR("ks") VARCHAR("-40") VARCHAR("begin")]` +
+		` [INT32(9) VARCHAR("ks") VARCHAR("-40") VARCHAR("insert into ` + "`user`" + `(id, lookup, lookup_unique) values (1, 'apa', 'apa'), (2, 'apa', 'bandar')")]]`
+	utils.AssertMatchesNoOrder(t, conn, `explain /*vt+ EXECUTE_DML_QUERIES */ format=vtexplain insert into user (id,lookup,lookup_unique) values (1,'apa','apa'),(2,'apa','bandar'),(3,'monkey','monkey')`, expected)
+
+	expected = `[[INT32(0) VARCHAR("ks") VARCHAR("-40") VARCHAR("select lookup, keyspace_id from lookup where lookup in ('apa')")]` +
+		` [INT32(1) VARCHAR("ks") VARCHAR("-40") VARCHAR("select id from ` + "`user`" + ` where lookup = 'apa'")]]`
+	for _, mode := range []string{"oltp", "olap"} {
+		t.Run(mode, func(t *testing.T) {
+			utils.Exec(t, conn, "set workload = "+mode)
+			utils.AssertMatches(t, conn, `explain format=vtexplain select id from user where lookup = "apa"`, expected)
+		})
+	}
+
+	// transaction explicitly started to no commit in the end.
+	utils.Exec(t, conn, "begin")
+	expected = `[[INT32(0) VARCHAR("ks") VARCHAR("-40") VARCHAR("begin")]` +
+		` [INT32(0) VARCHAR("ks") VARCHAR("-40") VARCHAR("insert into lookup(lookup, id, keyspace_id) values ('apa', 4, '\xd2\xfd\x88g\xd5\\r-\xfe'), ('apa', 5, 'p\xbb\x02<\x81\f\xa8z') on duplicate key update lookup = values(lookup), id = values(id), keyspace_id = values(keyspace_id)")]` +
+		` [INT32(1) VARCHAR("ks") VARCHAR("40-80") VARCHAR("begin")]` +
+		` [INT32(1) VARCHAR("ks") VARCHAR("40-80") VARCHAR("insert into lookup(lookup, id, keyspace_id) values ('monkey', 6, '\xf0\x98H\\n\xc4ľq') on duplicate key update lookup = values(lookup), id = values(id), keyspace_id = values(keyspace_id)")]` +
+		` [INT32(2) VARCHAR("ks") VARCHAR("-40") VARCHAR("commit")]` +
+		` [INT32(3) VARCHAR("ks") VARCHAR("40-80") VARCHAR("commit")]` +
+		` [INT32(4) VARCHAR("ks") VARCHAR("-40") VARCHAR("begin")]` +
+		` [INT32(4) VARCHAR("ks") VARCHAR("-40") VARCHAR("insert into lookup_unique(lookup_unique, keyspace_id) values ('foo', '\xd2\xfd\x88g\xd5\\r-\xfe')")]` +
+		` [INT32(5) VARCHAR("ks") VARCHAR("80-c0") VARCHAR("begin")]` +
+		` [INT32(5) VARCHAR("ks") VARCHAR("80-c0") VARCHAR("insert into lookup_unique(lookup_unique, keyspace_id) values ('bar', 'p\xbb\x02<\x81\f\xa8z')")]` +
+		` [INT32(6) VARCHAR("ks") VARCHAR("c0-") VARCHAR("begin")]` +
+		` [INT32(6) VARCHAR("ks") VARCHAR("c0-") VARCHAR("insert into lookup_unique(lookup_unique, keyspace_id) values ('nobar', '\xf0\x98H\\n\xc4ľq')")]` +
+		` [INT32(7) VARCHAR("ks") VARCHAR("-40") VARCHAR("commit")]` +
+		` [INT32(8) VARCHAR("ks") VARCHAR("80-c0") VARCHAR("commit")]` +
+		` [INT32(9) VARCHAR("ks") VARCHAR("c0-") VARCHAR("commit")]` +
+		` [INT32(10) VARCHAR("ks") VARCHAR("40-80") VARCHAR("begin")]` +
+		` [INT32(10) VARCHAR("ks") VARCHAR("40-80") VARCHAR("insert into ` + "`user`" + `(id, lookup, lookup_unique) values (5, 'apa', 'bar')")]` +
+		` [INT32(11) VARCHAR("ks") VARCHAR("c0-") VARCHAR("begin")]` +
+		` [INT32(11) VARCHAR("ks") VARCHAR("c0-") VARCHAR("insert into ` + "`user`" + `(id, lookup, lookup_unique) values (4, 'apa', 'foo'), (6, 'monkey', 'nobar')")]]`
+
+	utils.AssertMatchesNoOrder(t, conn,
+		`explain /*vt+ EXECUTE_DML_QUERIES */ format=vtexplain insert into user (id,lookup,lookup_unique) values (4,'apa','foo'),(5,'apa','bar'),(6,'monkey','nobar')`,
+		expected)
+
+	utils.Exec(t, conn, "rollback")
+}

go/test/endtoend/vtgate/queries/vtexplain/schema.sql

@@ -0,0 +1,22 @@
+create table user
+(
+    id            bigint,
+    lookup        varchar(128),
+    lookup_unique varchar(128),
+    primary key (id)
+) Engine = InnoDB;
+
+create table lookup
+(
+    lookup      varchar(128),
+    id          bigint,
+    keyspace_id varbinary(100),
+    primary key (id)
+) Engine = InnoDB;
+
+create table lookup_unique
+(
+    lookup_unique varchar(128),
+    keyspace_id   varbinary(100),
+    primary key (lookup_unique)
+) Engine = InnoDB;

go/test/endtoend/vtgate/queries/vtexplain/vschema.json

@@ -0,0 +1,68 @@
+{
+  "sharded": true,
+  "vindexes": {
+    "hash_index": {
+      "type": "hash"
+    },
+    "md5_index": {
+      "type": "unicode_loose_md5"
+    },
+    "lookup_vdx": {
+      "type": "lookup",
+      "params": {
+        "table": "lookup",
+        "from": "lookup,id",
+        "to": "keyspace_id",
+        "autocommit": "true"
+      },
+      "owner": "user"
+    },
+    "lookup_unique_vdx": {
+      "type": "lookup_unique",
+      "params": {
+        "table": "lookup_unique",
+        "from": "lookup_unique",
+        "to": "keyspace_id",
+        "autocommit": "true"
+      },
+      "owner": "user"
+    }
+  },
+  "tables": {
+    "user": {
+      "column_vindexes": [
+        {
+          "column": "id",
+          "name": "hash_index"
+        },
+        {
+          "columns": [
+            "lookup",
+            "id"
+          ],
+          "name": "lookup_vdx"
+        },
+        {
+          "column": "lookup_unique",
+          "name": "lookup_unique_vdx"
+        }
+      ]
+    },
+    "lookup": {
+      "column_vindexes": [
+        {
+          "column": "lookup",
+          "name": "md5_index"
+        }
+      ]
+    },
+    "lookup_unique": {
+      "column_vindexes": [
+        {
+          "column": "lookup_unique",
+          "name": "md5_index"
+        }
+      ]
+    }
+  }
+}

go/vt/schema/online_ddl.go

@@ -276,15 +276,11 @@ func OnlineDDLFromCommentedStatement(stmt sqlparser.Statement) (onlineDDL *Onlin
 
 	directives := comments.Directives()
 	decodeDirective := func(name string) (string, error) {
-		value, ok := directives[name]
+		value, ok := directives.GetString(name, "")
 		if !ok {
 			return "", vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT, "no value found for comment directive %s", name)
 		}
-		unquoted, err := strconv.Unquote(value)
-		if err != nil {
-			return "", err
-		}
-		b, err := hex.DecodeString(unquoted)
+		b, err := hex.DecodeString(value)
 		if err != nil {
 			return "", err
 		}

go/vt/sqlparser/ast.go

@@ -633,6 +633,7 @@ type (
 	ExplainStmt struct {
 		Type      ExplainType
 		Statement Statement
+		Comments  *ParsedComments
 	}
 
 	// ExplainTab represents the Explain table
@@ -1272,6 +1273,11 @@ func (node *AlterTable) SetComments(comments Comments) {
 	node.Comments = comments.Parsed()
 }
 
+// SetComments implements DDLStatement.
+func (node *ExplainStmt) SetComments(comments Comments) {
+	node.Comments = comments.Parsed()
+}
+
 // SetComments implements DDLStatement.
 func (node *CreateTable) SetComments(comments Comments) {
 	node.Comments = comments.Parsed()
@@ -1344,6 +1350,11 @@ func (node *AlterTable) GetParsedComments() *ParsedComments {
 	return node.Comments
 }
 
+// GetParsedComments implements DDLStatement.
+func (node *ExplainStmt) GetParsedComments() *ParsedComments {
+	return node.Comments
+}
+
 // GetParsedComments implements DDLStatement.
 func (node *CreateTable) GetParsedComments() *ParsedComments {
 	return node.Comments
@@ -1924,7 +1935,7 @@ func (c Comments) Parsed() *ParsedComments {
 
 type ParsedComments struct {
 	comments    Comments
-	_directives CommentDirectives
+	_directives *CommentDirectives
 }
 
 // SelectExprs represents SELECT expressions.

go/vt/sqlparser/ast_format.go

@@ -990,7 +990,7 @@ func (node *ExplainStmt) Format(buf *TrackedBuffer) {
 	default:
 		format = "format = " + node.Type.ToString() + " "
 	}
-	buf.astPrintf(node, "explain %s%v", format, node.Statement)
+	buf.astPrintf(node, "explain %v%s%v", node.Comments, format, node.Statement)
 }
 
 // Format formats the node.

go/vt/sqlparser/ast_funcs.go

@@ -1611,6 +1611,8 @@ func (ty ExplainType) ToString() string {
 		return JSONStr
 	case VitessType:
 		return VitessStr
+	case VTExplainType:
+		return VTExplainStr
 	case TraditionalType:
 		return TraditionalStr
 	case AnalyzeType: