sql: v20.2.3: expected expression to have key

[cockroach-teamcity]

This issue was autofiled by Sentry. It represents a crash or reported error on a live cluster with telemetry enabled.

Sentry link: https://sentry.io/organizations/cockroach-labs/issues/2103833235/?referrer=webhooks_plugin

Panic message:

decorrelate_funcs.go:425: expected expression to have key
--
*errutil.leafError: expected expression to have key (1)
decorrelate_funcs.go:425: *withstack.withStack (top exception)
*assert.withAssertionFailure
(check the extra data payloads)

Stacktrace (expand for inline code snippets):

cockroach/pkg/sql/opt/norm/decorrelate_funcs.go

Lines 424 to 426 in 89dd2d1

	if !ok {
	panic(errors.AssertionFailedf("expected expression to have key"))
	}

in pkg/sql/opt/norm.(*CustomFuncs).NonKeyCols
https://github.com/cockroachdb/cockroach/blob/89dd2d14c29787378c434ed54937757ef5d9877c/pkg/sql/opt/xform/explorer.og.go#L383-L385 in pkg/sql/opt/xform.(*explorer).exploreSelect
https://github.com/cockroachdb/cockroach/blob/89dd2d14c29787378c434ed54937757ef5d9877c/pkg/sql/opt/xform/explorer.og.go#L19-L21 in pkg/sql/opt/xform.(*explorer).exploreGroupMember

cockroach/pkg/sql/opt/xform/explorer.go

Lines 177 to 179 in 89dd2d1

	if memberExplored := e.exploreGroupMember(state, member, i); memberExplored {
	// No more rules can ever match this expression, so skip it in

in pkg/sql/opt/xform.(*explorer).exploreGroup

cockroach/pkg/sql/opt/xform/optimizer.go

Lines 462 to 464 in 89dd2d1

	// other expressions in this group.
	if o.shouldExplore(required) && !o.explorer.exploreGroup(grp).fullyExplored {
	fullyOptimized = false

in pkg/sql/opt/xform.(*Optimizer).optimizeGroup

cockroach/pkg/sql/opt/xform/optimizer.go

Lines 249 to 251 in 89dd2d1

	case memo.RelExpr:
	state := o.optimizeGroup(t, required)
	return state.cost, state.fullyOptimized

in pkg/sql/opt/xform.(*Optimizer).optimizeExpr

cockroach/pkg/sql/opt/xform/optimizer.go

Lines 504 to 506 in 89dd2d1

	// Optimize the child with respect to those properties.
	childCost, childOptimized := o.optimizeExpr(member.Child(i), childRequired)

in pkg/sql/opt/xform.(*Optimizer).optimizeGroupMember

cockroach/pkg/sql/opt/xform/optimizer.go

Lines 449 to 451 in 89dd2d1

	// Optimize the group member with respect to the required properties.
	memberOptimized := o.optimizeGroupMember(state, member, required)

in pkg/sql/opt/xform.(*Optimizer).optimizeGroup

cockroach/pkg/sql/opt/xform/optimizer.go

Lines 223 to 225 in 89dd2d1

	rootProps := o.mem.RootProps()
	o.optimizeGroup(root, rootProps)

in pkg/sql/opt/xform.(*Optimizer).Optimize

cockroach/pkg/sql/plan_opt.go

Lines 408 to 410 in 89dd2d1

	opc.log(ctx, "optimizing (no placeholders)")
	if _, err := opc.optimizer.Optimize(); err != nil {
	return nil, err

in pkg/sql.(*optPlanningCtx).buildReusableMemo

cockroach/pkg/sql/plan_opt.go

Lines 125 to 127 in 89dd2d1

	memo, err := opc.buildReusableMemo(ctx)
	if err != nil {

in pkg/sql.(*planner).prepareUsingOptimizer

cockroach/pkg/sql/conn_executor_prepare.go

Lines 251 to 253 in 89dd2d1

	// future.
	flags, err := p.prepareUsingOptimizer(ctx)
	if err != nil {

in pkg/sql.(*connExecutor).populatePrepared

cockroach/pkg/sql/conn_executor_prepare.go

Lines 193 to 195 in 89dd2d1

	p.semaCtx.Annotations = tree.MakeAnnotations(stmt.NumAnnotations)
	flags, err = ex.populatePrepared(ctx, txn, placeholderHints, p)
	return err

in pkg/sql.(*connExecutor).prepare.func1

cockroach/pkg/kv/db.go

Lines 706 to 708 in 89dd2d1

	err := txn.exec(ctx, func(ctx context.Context, txn *Txn) error {
	return retryable(ctx, txn)
	})

in pkg/kv.(*DB).Txn.func1

cockroach/pkg/kv/txn.go

Lines 810 to 812 in 89dd2d1

	}
	err = fn(ctx, txn)

in pkg/kv.(*Txn).exec

cockroach/pkg/kv/db.go

Lines 705 to 707 in 89dd2d1

	txn.SetDebugName("unnamed")
	err := txn.exec(ctx, func(ctx context.Context, txn *Txn) error {
	return retryable(ctx, txn)

in pkg/kv.(*DB).Txn

cockroach/pkg/sql/conn_executor_prepare.go

Lines 205 to 207 in 89dd2d1

	// than bubbling them up to the connExecutor state machine.
	if err := ex.server.cfg.DB.Txn(ctx, prepare); err != nil {
	return nil, err

in pkg/sql.(*connExecutor).prepare

cockroach/pkg/sql/conn_executor_prepare.go

Lines 119 to 121 in 89dd2d1

	// Prepare the query. This completes the typing of placeholders.
	prepared, err := ex.prepare(ctx, stmt, placeholderHints, origin)
	if err != nil {

in pkg/sql.(*connExecutor).addPreparedStmt

cockroach/pkg/sql/conn_executor_prepare.go

Lines 65 to 67 in 89dd2d1

	ps, err := ex.addPreparedStmt(
	ctx,

in pkg/sql.(*connExecutor).execPrepare

cockroach/pkg/sql/conn_executor.go

Lines 1550 to 1552 in 89dd2d1

	stmtCtx := withStatement(ctx, ex.curStmt)
	ev, payload = ex.execPrepare(stmtCtx, tcmd)
	case DescribeStmt:

in pkg/sql.(*connExecutor).execCmd

cockroach/pkg/sql/conn_executor.go

Lines 1390 to 1392 in 89dd2d1

	var err error
	if err = ex.execCmd(ex.Ctx()); err != nil {
	if errors.IsAny(err, io.EOF, errDrainingComplete) {

in pkg/sql.(*connExecutor).run

cockroach/pkg/sql/conn_executor.go

Lines 507 to 509 in 89dd2d1

	}()
	return h.ex.run(ctx, s.pool, reserved, cancel)
	}

in pkg/sql.(*Server).ServeConn

cockroach/pkg/sql/pgwire/conn.go

Lines 625 to 627 in 89dd2d1

	reservedOwned = false // We're about to pass ownership away.
	retErr = sqlServer.ServeConn(ctx, connHandler, reserved, cancelConn)
	}()

in pkg/sql/pgwire.(*conn).processCommandsAsync.func1
/usr/local/go/src/runtime/asm_amd64.s#L1356-L1358 in runtime.goexit

pkg/sql/opt/norm/decorrelate_funcs.go in pkg/sql/opt/norm.(*CustomFuncs).NonKeyCols at line 425
pkg/sql/opt/xform/explorer.og.go in pkg/sql/opt/xform.(*explorer).exploreSelect at line 384
pkg/sql/opt/xform/explorer.og.go in pkg/sql/opt/xform.(*explorer).exploreGroupMember at line 20
pkg/sql/opt/xform/explorer.go in pkg/sql/opt/xform.(*explorer).exploreGroup at line 178
pkg/sql/opt/xform/optimizer.go in pkg/sql/opt/xform.(*Optimizer).optimizeGroup at line 463
pkg/sql/opt/xform/optimizer.go in pkg/sql/opt/xform.(*Optimizer).optimizeExpr at line 250
pkg/sql/opt/xform/optimizer.go in pkg/sql/opt/xform.(*Optimizer).optimizeGroupMember at line 505
pkg/sql/opt/xform/optimizer.go in pkg/sql/opt/xform.(*Optimizer).optimizeGroup at line 450
pkg/sql/opt/xform/optimizer.go in pkg/sql/opt/xform.(*Optimizer).Optimize at line 224
pkg/sql/plan_opt.go in pkg/sql.(*optPlanningCtx).buildReusableMemo at line 409
pkg/sql/plan_opt.go in pkg/sql.(*planner).prepareUsingOptimizer at line 126
pkg/sql/conn_executor_prepare.go in pkg/sql.(*connExecutor).populatePrepared at line 252
pkg/sql/conn_executor_prepare.go in pkg/sql.(*connExecutor).prepare.func1 at line 194
pkg/kv/db.go in pkg/kv.(*DB).Txn.func1 at line 707
pkg/kv/txn.go in pkg/kv.(*Txn).exec at line 811
pkg/kv/db.go in pkg/kv.(*DB).Txn at line 706
pkg/sql/conn_executor_prepare.go in pkg/sql.(*connExecutor).prepare at line 206
pkg/sql/conn_executor_prepare.go in pkg/sql.(*connExecutor).addPreparedStmt at line 120
pkg/sql/conn_executor_prepare.go in pkg/sql.(*connExecutor).execPrepare at line 66
pkg/sql/conn_executor.go in pkg/sql.(*connExecutor).execCmd at line 1551
pkg/sql/conn_executor.go in pkg/sql.(*connExecutor).run at line 1391
pkg/sql/conn_executor.go in pkg/sql.(*Server).ServeConn at line 508
pkg/sql/pgwire/conn.go in pkg/sql/pgwire.(*conn).processCommandsAsync.func1 at line 626
/usr/local/go/src/runtime/asm_amd64.s in runtime.goexit at line 1357

Tag	Value
Cockroach Release	v20.2.3
Cockroach SHA:	89dd2d1
Platform	linux amd64
Distribution	CCL
Environment	v20.2.3
Command	server
Go Version	``
# of CPUs
# of Goroutines

[RaduBerinde]

The rule is SplitDisjunctionAddKey. It is calling NonKeyCols on leftScan. It looks like a copy/paste-o from the other variant, but I'm surprised it wasn't caught by any tests.

[RaduBerinde]

Ah, my bad, leftScan is created by the rule and is a scan that contains the PK.. This is strange. I wonder if maybe the table is virtual and the PK stuff doesn't work as intended.

[RaduBerinde]

Indeed, I was able to repro with:

select * from pg_catalog.pg_attrdef WHERE (adnum = 1 AND adrelid = 1) OR (adbin = 'foo' AND adrelid = 2);

The strange filter is necessary to trick the rule into firing (there is a single index on adrelid).

[RaduBerinde]

Interestingly, this doesn't seem to be a problem on master, where we have changed virtual tables to always present a primary index.

[RaduBerinde]

Actually, looks like this was already fixed (on master and release-20.2) by #58434 - we no longer call NonKeyCols at all. I will add some regression tests.