[backport] api: v2alpha1: Use subquery instead of left join for transaction list. (#6269)

CHANGELOG.md

@@ -2,6 +2,12 @@
 
 See [RELEASE](./RELEASE.md) for workflow instructions.
 
+## Release v1.6.8
+
+### Improvements
+* [#6269](https://github.com/spacemeshos/go-spacemesh/pull/6269) Use subquery instead of left join to speed up 
+  transaction list query
+
 ## Release v1.6.7
 
 ### Improvements

api/grpcserver/v2alpha1/transaction.go

@@ -251,19 +251,8 @@ func toTransactionOperations(filter *spacemeshv2alpha1.TransactionRequest) (buil
 			return builder.Operations{}, err
 		}
 		ops.Filter = append(ops.Filter, builder.Op{
-			Group: []builder.Op{
-				{
-					Field: builder.Address,
-					Token: builder.Eq,
-					Value: addr.Bytes(),
-				},
-				{
-					Field: builder.Principal,
-					Token: builder.Eq,
-					Value: addr.Bytes(),
-				},
-			},
-			GroupOperator: builder.Or,
+			Value:       addr.Bytes(),
+			CustomQuery: "id IN (SELECT tid FROM transactions_results_addresses WHERE address = ?1)",
 		})
 	}
 

api/grpcserver/v2alpha1/transaction_test.go

@@ -120,6 +120,50 @@ func TestTransactionService_List(t *testing.T) {
 		require.Len(t, list.Transactions, len(expectedTxs))
 	})
 
+	t.Run("address/startlayer/endlayer", func(t *testing.T) {
+		address := txsList[0].Principal.String()
+		layer := txsList[0].Layer.Uint32()
+		var expectedTxs []types.TransactionWithResult
+		for _, tx := range txsList {
+			found := false
+			if tx.Transaction.Principal.String() == address &&
+				tx.Layer.Uint32() >= layer && tx.Layer.Uint32() <= layer {
+				found = true
+			}
+
+			for _, addr := range tx.TransactionResult.Addresses {
+				if addr.String() == address &&
+					tx.Layer.Uint32() >= layer && tx.Layer.Uint32() <= layer {
+					found = true
+					break
+				}
+			}
+			if found {
+				expectedTxs = append(expectedTxs, tx)
+			}
+		}
+		list, err := client.List(ctx, &spacemeshv2alpha1.TransactionRequest{
+			Address:    &address,
+			StartLayer: &layer,
+			EndLayer:   &layer,
+			Limit:      100,
+		})
+		require.NoError(t, err)
+		require.Len(t, list.Transactions, len(expectedTxs))
+	})
+
+	t.Run("address/txid", func(t *testing.T) {
+		address := txsList[0].Principal.String()
+		list, err := client.List(ctx, &spacemeshv2alpha1.TransactionRequest{
+			Address: &address,
+			Txid:    [][]byte{txsList[0].ID[:]},
+			Limit:   100,
+		})
+		require.NoError(t, err)
+		require.Len(t, list.Transactions, 1)
+		require.Equal(t, txsList[0].TxHeader.Principal.String(), list.Transactions[0].Tx.Principal)
+	})
+
 	t.Run("tx id", func(t *testing.T) {
 		list, err := client.List(ctx, &spacemeshv2alpha1.TransactionRequest{
 			Txid:  [][]byte{txsList[0].ID[:]},

common/fixture/transaction_results.go

@@ -72,35 +72,32 @@ func (g *TransactionResultGenerator) Next() *types.TransactionWithResult {
 	var tx types.TransactionWithResult
 	g.rng.Read(tx.ID[:])
 
+	rnd := g.rng.Perm(len(g.Addrs))
+	principal := g.Addrs[rnd[0]]
+	tx.Addresses = []types.Address{
+		principal,
+	}
+
 	_, priv, _ := ed25519.GenerateKey(g.rng)
 	var rawTx []byte
 	method := core.MethodSpawn
 
 	if g.rng.Intn(2) == 1 {
-		rawTx = wallet2.Spend(priv, g.Addrs[g.rng.Intn(len(g.Addrs))], 100, types.Nonce(1))
+		dest := g.Addrs[rnd[1]]
+		tx.Addresses = append(tx.Addresses, dest)
+		rawTx = wallet2.Spend(priv, dest, 100, types.Nonce(1))
 		method = core.MethodSpend
 	} else {
 		rawTx = wallet2.SelfSpawn(priv, types.Nonce(1))
 	}
-
 	tx.RawTx = types.NewRawTx(rawTx)
-
 	tx.Block = g.Blocks[g.rng.Intn(len(g.Blocks))]
 	tx.Layer = g.Layers[g.rng.Intn(len(g.Layers))]
 	tx.TxHeader = &types.TxHeader{
 		TemplateAddress: wallet.TemplateAddress,
 		Method:          uint8(method),
-		Principal:       g.Addrs[g.rng.Intn(len(g.Addrs))],
+		Principal:       principal,
 		Nonce:           types.Nonce(1),
 	}
-
-	if lth := g.rng.Intn(len(g.Addrs)); lth > 0 {
-		tx.Addresses = make([]types.Address, lth%10+1)
-
-		g.rng.Shuffle(len(g.Addrs), func(i, j int) {
-			g.Addrs[i], g.Addrs[j] = g.Addrs[j], g.Addrs[i]
-		})
-		copy(tx.Addresses, g.Addrs)
-	}
 	return &tx
 }

sql/builder/builder.go

@@ -59,6 +59,12 @@ type Op struct {
 
 	Group         []Op
 	GroupOperator operator
+
+	// CustomQuery is used to add custom query. If this is set, Field and Token will be ignored.
+	// This is useful for complex queries that can't be expressed with Field and Token.
+	// Value will be used for custom query if it's not nil.
+	// Remember about setting correct bind index for Value.
+	CustomQuery string
 }
 
 type Modifier struct {
@@ -97,6 +103,14 @@ func FilterFrom(operations Operations) string {
 			queryBuilder.WriteString(" and")
 		}
 
+		if len(op.CustomQuery) > 0 {
+			queryBuilder.WriteString(fmt.Sprintf("  %s", op.CustomQuery))
+			if op.Value != nil {
+				bindIndex++
+			}
+			continue
+		}
+
 		if len(op.Group) > 0 {
 			queryBuilder.WriteString(" (")
 			for k, groupOp := range op.Group {

sql/transactions/transactions.go

@@ -419,8 +419,7 @@ func IterateTransactionsOps(
 ) error {
 	var derr error
 	_, err := db.Exec(`select distinct tx, header, layer, block, timestamp, id, result 
-		from transactions
-		left join transactions_results_addresses on id=tid`+builder.FilterFrom(operations),
+		from transactions`+builder.FilterFrom(operations),
 		builder.BindingsFrom(operations),
 		func(stmt *sql.Statement) bool {
 			var txId types.TransactionID