From 86279b7e883e0bd97781d3e24e0c8e03a660b276 Mon Sep 17 00:00:00 2001 From: Matt Topol Date: Wed, 9 Aug 2023 12:05:10 -0400 Subject: [PATCH 1/6] perf(go/adbc/driver/snowflake): Implement concurrency limit --- c/validation/adbc_validation_util.h | 8 ++--- go/adbc/driver/snowflake/connection.go | 7 ++-- go/adbc/driver/snowflake/record_reader.go | 15 ++++++++ go/adbc/go.mod | 4 +-- go/adbc/go.sum | 42 ++++++++++++++++++----- 5 files changed, 58 insertions(+), 18 deletions(-) diff --git a/c/validation/adbc_validation_util.h b/c/validation/adbc_validation_util.h index 7c60e3cd6a..432dd7bbd7 100644 --- a/c/validation/adbc_validation_util.h +++ b/c/validation/adbc_validation_util.h @@ -206,12 +206,8 @@ struct GetObjectsReader { } ~GetObjectsReader() { AdbcGetObjectsDataDelete(get_objects_data_); } - struct AdbcGetObjectsData* operator*() { - return get_objects_data_; - } - struct AdbcGetObjectsData* operator->() { - return get_objects_data_; - } + struct AdbcGetObjectsData* operator*() { return get_objects_data_; } + struct AdbcGetObjectsData* operator->() { return get_objects_data_; } private: struct AdbcGetObjectsData* get_objects_data_; diff --git a/go/adbc/driver/snowflake/connection.go b/go/adbc/driver/snowflake/connection.go index 8f965597c5..3d1b63d508 100644 --- a/go/adbc/driver/snowflake/connection.go +++ b/go/adbc/driver/snowflake/connection.go @@ -33,6 +33,8 @@ import ( "github.com/snowflakedb/gosnowflake" ) +const defaultStatementQueueSize = 200 + type snowflakeConn interface { driver.Conn driver.ConnBeginTx @@ -777,8 +779,9 @@ func (c *cnxn) Rollback(_ context.Context) error { // NewStatement initializes a new statement object tied to this connection func (c *cnxn) NewStatement() (adbc.Statement, error) { return &statement{ - alloc: c.db.alloc, - cnxn: c, + alloc: c.db.alloc, + cnxn: c, + queueSize: defaultStatementQueueSize, }, nil } diff --git a/go/adbc/driver/snowflake/record_reader.go b/go/adbc/driver/snowflake/record_reader.go index b4c8b15a63..73aabd7851 100644 --- a/go/adbc/driver/snowflake/record_reader.go +++ b/go/adbc/driver/snowflake/record_reader.go @@ -21,6 +21,7 @@ import ( "context" "encoding/hex" "math" + "os" "strconv" "strings" "sync/atomic" @@ -36,6 +37,19 @@ import ( "golang.org/x/sync/errgroup" ) +const defaultConcurrentPrefetch = 10 + +var concurrentPrefetch int + +func init() { + concurrentPrefetch = defaultConcurrentPrefetch + if value := os.Getenv("ADBC_SNOWFLAKE_DEFAULT_CONCURRENT_PREFETCH"); value != "" { + if n, err := strconv.Atoi(value); err == nil && n > 0 { + concurrentPrefetch = n + } + } +} + func identCol(_ context.Context, a arrow.Array) (arrow.Array, error) { a.Retain() return a, nil @@ -478,6 +492,7 @@ func newRecordReader(ctx context.Context, alloc memory.Allocator, ld gosnowflake } }() + group.SetLimit(concurrentPrefetch) group.Go(func() error { defer rr.Release() defer r.Close() diff --git a/go/adbc/go.mod b/go/adbc/go.mod index f3d5ce0620..2ba4f44622 100644 --- a/go/adbc/go.mod +++ b/go/adbc/go.mod @@ -24,7 +24,7 @@ require ( github.com/bluele/gcache v0.0.2 github.com/google/uuid v1.3.0 github.com/snowflakedb/gosnowflake v1.6.22 - github.com/stretchr/testify v1.8.2 + github.com/stretchr/testify v1.8.4 golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 golang.org/x/sync v0.3.0 golang.org/x/tools v0.11.0 @@ -41,8 +41,8 @@ require ( github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c // indirect github.com/andybalholm/brotli v1.0.5 // indirect github.com/apache/arrow/go/v12 v12.0.1 // indirect - github.com/apache/thrift v0.16.0 // indirect github.com/aws/aws-sdk-go-v2 v1.19.0 // indirect + github.com/apache/thrift v0.17.0 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 // indirect github.com/aws/aws-sdk-go-v2/credentials v1.13.27 // indirect github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.72 // indirect diff --git a/go/adbc/go.sum b/go/adbc/go.sum index 1793e47215..d1d0a279d9 100644 --- a/go/adbc/go.sum +++ b/go/adbc/go.sum @@ -16,6 +16,7 @@ github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c/go.mod h1:X0 github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs= github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/apache/arrow/go/v12 v12.0.1 h1:JsR2+hzYYjgSUkBSaahpqCetqZMr76djX80fF/DiJbg= +<<<<<<< Updated upstream github.com/apache/arrow/go/v12 v12.0.1/go.mod h1:weuTY7JvTG/HDPtMQxEUp7pU73vkLWMLpY67QwZ/WWw= github.com/apache/arrow/go/v13 v13.0.0-20230713180941-b97597765355 h1:QuXqLb2HzL5EjY99fFp+iG9NagAruvQIbU/2++x+2VY= github.com/apache/arrow/go/v13 v13.0.0-20230713180941-b97597765355/go.mod h1:W69eByFNO0ZR30q1/7Sr9d83zcVZmF2MiP3fFYAWJOc= @@ -23,6 +24,13 @@ github.com/apache/thrift v0.16.0 h1:qEy6UW60iVOlUy+b9ZR0d5WzUWYGOo4HfopoyBaNmoY= github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU= github.com/aws/aws-sdk-go-v2 v1.19.0 h1:klAT+y3pGFBU/qVf1uzwttpBbiuozJYWzNLHioyDJ+k= github.com/aws/aws-sdk-go-v2 v1.19.0/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= +======= +github.com/apache/arrow/go/v13 v13.0.0-20230713180941-b97597765355 h1:QuXqLb2HzL5EjY99fFp+iG9NagAruvQIbU/2++x+2VY= +github.com/apache/arrow/go/v13 v13.0.0-20230713180941-b97597765355/go.mod h1:W69eByFNO0ZR30q1/7Sr9d83zcVZmF2MiP3fFYAWJOc= +github.com/apache/thrift v0.17.0 h1:cMd2aj52n+8VoAtvSvLn4kDC3aZ6IAkBuqWQ2IDu7wo= +github.com/aws/aws-sdk-go-v2 v1.18.0 h1:882kkTpSFhdgYRKVZ/VCgf7sd0ru57p2JCxz4/oN5RY= +github.com/aws/aws-sdk-go-v2 v1.18.0/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= +>>>>>>> Stashed changes github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 h1:dK82zF6kkPeCo8J1e+tGx4JdvDIQzj7ygIoLg8WMuGs= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10/go.mod h1:VeTZetY5KRJLuD/7fkQXMU6Mw7H5m/KP2J5Iy9osMno= github.com/aws/aws-sdk-go-v2/config v1.18.28 h1:TINEaKyh1Td64tqFvn09iYpKiWjmHYrG1fa91q2gnqw= @@ -79,8 +87,12 @@ github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 h1:ZpnhV/YsD2/4cESfV5+Hoeu/iUR3ruzNvZ+yQfO03a0= github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4= +<<<<<<< Updated upstream github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= +======= +github.com/golang-jwt/jwt v3.2.1+incompatible h1:73Z+4BJcrTC+KczS6WvTPvRGOp1WmfEP4Q1lOd9Z/+c= +>>>>>>> Stashed changes github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= @@ -138,17 +150,13 @@ github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVs github.com/snowflakedb/gosnowflake v1.6.23-0.20230717195239-fec38ba82d2a h1:F7fKVj3t12jr3Bopzngsp/PZDm1or8zpk+29NN4YFGk= github.com/snowflakedb/gosnowflake v1.6.23-0.20230717195239-fec38ba82d2a/go.mod h1:KfO4F7bk+aXPUIvBqYxvPhxLlu2/w4TtSC8Rw/yr5Mg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= -github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= -github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= +<<<<<<< Updated upstream golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA= @@ -163,14 +171,22 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50= golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +======= +golang.org/x/crypto v0.10.0 h1:LKqV2xt9+kDzSTfOhx4FrkEBcMrAgHSYgzywV9zcGmM= +golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= +golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 h1:MGwJjxBy0HJshjDNfLsYO8xppfqWlA5ZT9OhtUUhTNw= +golang.org/x/mod v0.11.0 h1:bUO06HqtnRcc/7l71XBe4WcqTZ+3AH1J59zWDDwLKgU= +golang.org/x/mod v0.11.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/net v0.11.0 h1:Gi2tvZIJyBtO9SDr1q9h5hEQCp/4L2RQ+ar0qjx2oNU= +golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ= +>>>>>>> Stashed changes golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +<<<<<<< Updated upstream golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.10.0 h1:3R7pNqamzBraeqj/Tj8qt1aQ2HpmlC+Cx/qL/7hn4/c= @@ -183,6 +199,16 @@ golang.org/x/tools v0.11.0 h1:EMCa6U9S2LtZXLAMoWiR/R8dAQFRqbAitmbJ2UKhoi8= golang.org/x/tools v0.11.0/go.mod h1:anzJrxPjNtfgiYQYirP2CPGzGLxrH2u2QBhn6Bf3qY8= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +======= +golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= +golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.9.0 h1:GRRCnKYhdQrD8kfRAdQ6Zcw1P0OcELxGLKJvtjVMZ28= +golang.org/x/term v0.9.0/go.mod h1:M6DEAAIenWoTxdKrOltXcmDY3rSplQUkrvaDU5FcQyo= +golang.org/x/text v0.10.0 h1:UpjohKhiEgNc0CSauXmwYftY1+LlaC75SJwh0SgCX58= +golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/tools v0.10.0 h1:tvDr/iQoUqNdohiYm0LmmKcBk+q86lb9EprIUFhHHGg= +golang.org/x/tools v0.10.0/go.mod h1:UJwyiVBsOA2uwvK/e5OY3GTpDUJriEd+/YlqAwLPmyM= +>>>>>>> Stashed changes golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= From a7691f54a13eb461b05df693ed0315c760345cea Mon Sep 17 00:00:00 2001 From: Matt Topol Date: Wed, 9 Aug 2023 12:31:22 -0400 Subject: [PATCH 2/6] fix conflicts go.mod --- go/adbc/go.mod | 2 +- go/adbc/go.sum | 49 +++---------------------------------------------- 2 files changed, 4 insertions(+), 47 deletions(-) diff --git a/go/adbc/go.mod b/go/adbc/go.mod index 2ba4f44622..999a8d47f1 100644 --- a/go/adbc/go.mod +++ b/go/adbc/go.mod @@ -41,8 +41,8 @@ require ( github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c // indirect github.com/andybalholm/brotli v1.0.5 // indirect github.com/apache/arrow/go/v12 v12.0.1 // indirect - github.com/aws/aws-sdk-go-v2 v1.19.0 // indirect github.com/apache/thrift v0.17.0 // indirect + github.com/aws/aws-sdk-go-v2 v1.19.0 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 // indirect github.com/aws/aws-sdk-go-v2/credentials v1.13.27 // indirect github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.72 // indirect diff --git a/go/adbc/go.sum b/go/adbc/go.sum index d1d0a279d9..7e47f67f5c 100644 --- a/go/adbc/go.sum +++ b/go/adbc/go.sum @@ -16,21 +16,13 @@ github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c/go.mod h1:X0 github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs= github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/apache/arrow/go/v12 v12.0.1 h1:JsR2+hzYYjgSUkBSaahpqCetqZMr76djX80fF/DiJbg= -<<<<<<< Updated upstream github.com/apache/arrow/go/v12 v12.0.1/go.mod h1:weuTY7JvTG/HDPtMQxEUp7pU73vkLWMLpY67QwZ/WWw= github.com/apache/arrow/go/v13 v13.0.0-20230713180941-b97597765355 h1:QuXqLb2HzL5EjY99fFp+iG9NagAruvQIbU/2++x+2VY= github.com/apache/arrow/go/v13 v13.0.0-20230713180941-b97597765355/go.mod h1:W69eByFNO0ZR30q1/7Sr9d83zcVZmF2MiP3fFYAWJOc= -github.com/apache/thrift v0.16.0 h1:qEy6UW60iVOlUy+b9ZR0d5WzUWYGOo4HfopoyBaNmoY= -github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU= +github.com/apache/thrift v0.17.0 h1:cMd2aj52n+8VoAtvSvLn4kDC3aZ6IAkBuqWQ2IDu7wo= +github.com/apache/thrift v0.17.0/go.mod h1:OLxhMRJxomX+1I/KUw03qoV3mMz16BwaKI+d4fPBx7Q= github.com/aws/aws-sdk-go-v2 v1.19.0 h1:klAT+y3pGFBU/qVf1uzwttpBbiuozJYWzNLHioyDJ+k= github.com/aws/aws-sdk-go-v2 v1.19.0/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= -======= -github.com/apache/arrow/go/v13 v13.0.0-20230713180941-b97597765355 h1:QuXqLb2HzL5EjY99fFp+iG9NagAruvQIbU/2++x+2VY= -github.com/apache/arrow/go/v13 v13.0.0-20230713180941-b97597765355/go.mod h1:W69eByFNO0ZR30q1/7Sr9d83zcVZmF2MiP3fFYAWJOc= -github.com/apache/thrift v0.17.0 h1:cMd2aj52n+8VoAtvSvLn4kDC3aZ6IAkBuqWQ2IDu7wo= -github.com/aws/aws-sdk-go-v2 v1.18.0 h1:882kkTpSFhdgYRKVZ/VCgf7sd0ru57p2JCxz4/oN5RY= -github.com/aws/aws-sdk-go-v2 v1.18.0/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= ->>>>>>> Stashed changes github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 h1:dK82zF6kkPeCo8J1e+tGx4JdvDIQzj7ygIoLg8WMuGs= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10/go.mod h1:VeTZetY5KRJLuD/7fkQXMU6Mw7H5m/KP2J5Iy9osMno= github.com/aws/aws-sdk-go-v2/config v1.18.28 h1:TINEaKyh1Td64tqFvn09iYpKiWjmHYrG1fa91q2gnqw= @@ -87,12 +79,7 @@ github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 h1:ZpnhV/YsD2/4cESfV5+Hoeu/iUR3ruzNvZ+yQfO03a0= github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4= -<<<<<<< Updated upstream github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= -github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= -======= -github.com/golang-jwt/jwt v3.2.1+incompatible h1:73Z+4BJcrTC+KczS6WvTPvRGOp1WmfEP4Q1lOd9Z/+c= ->>>>>>> Stashed changes github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= @@ -153,62 +140,32 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= -<<<<<<< Updated upstream -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA= golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 h1:MGwJjxBy0HJshjDNfLsYO8xppfqWlA5ZT9OhtUUhTNw= golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50= golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -======= -golang.org/x/crypto v0.10.0 h1:LKqV2xt9+kDzSTfOhx4FrkEBcMrAgHSYgzywV9zcGmM= -golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= -golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 h1:MGwJjxBy0HJshjDNfLsYO8xppfqWlA5ZT9OhtUUhTNw= -golang.org/x/mod v0.11.0 h1:bUO06HqtnRcc/7l71XBe4WcqTZ+3AH1J59zWDDwLKgU= -golang.org/x/mod v0.11.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/net v0.11.0 h1:Gi2tvZIJyBtO9SDr1q9h5hEQCp/4L2RQ+ar0qjx2oNU= -golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ= ->>>>>>> Stashed changes golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -<<<<<<< Updated upstream golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.10.0 h1:3R7pNqamzBraeqj/Tj8qt1aQ2HpmlC+Cx/qL/7hn4/c= golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.11.0 h1:EMCa6U9S2LtZXLAMoWiR/R8dAQFRqbAitmbJ2UKhoi8= golang.org/x/tools v0.11.0/go.mod h1:anzJrxPjNtfgiYQYirP2CPGzGLxrH2u2QBhn6Bf3qY8= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -======= -golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= -golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.9.0 h1:GRRCnKYhdQrD8kfRAdQ6Zcw1P0OcELxGLKJvtjVMZ28= -golang.org/x/term v0.9.0/go.mod h1:M6DEAAIenWoTxdKrOltXcmDY3rSplQUkrvaDU5FcQyo= -golang.org/x/text v0.10.0 h1:UpjohKhiEgNc0CSauXmwYftY1+LlaC75SJwh0SgCX58= -golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/tools v0.10.0 h1:tvDr/iQoUqNdohiYm0LmmKcBk+q86lb9EprIUFhHHGg= -golang.org/x/tools v0.10.0/go.mod h1:UJwyiVBsOA2uwvK/e5OY3GTpDUJriEd+/YlqAwLPmyM= ->>>>>>> Stashed changes golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= From 4f30b3ebcb64f9de79562bbbd57402764ec10da3 Mon Sep 17 00:00:00 2001 From: Matt Topol Date: Wed, 9 Aug 2023 12:58:08 -0400 Subject: [PATCH 3/6] updates from feedback --- docs/source/driver/snowflake.rst | 11 ++++++-- go/adbc/driver/snowflake/connection.go | 12 ++++++--- go/adbc/driver/snowflake/record_reader.go | 18 ++----------- go/adbc/driver/snowflake/statement.go | 33 +++++++++++++++++++---- 4 files changed, 47 insertions(+), 27 deletions(-) diff --git a/docs/source/driver/snowflake.rst b/docs/source/driver/snowflake.rst index d5ab24c4e3..9737bf00b6 100644 --- a/docs/source/driver/snowflake.rst +++ b/docs/source/driver/snowflake.rst @@ -207,10 +207,17 @@ In addition, results are potentially fetched in parallel from multiple endpoints A limited number of batches are queued per endpoint, though data is always returned to the client in the order of the endpoints. -The queue size can be changed by setting an option on the :cpp:class:`AdbcStatement`: +To manage the performance of result fetching there are two options to control +buffering and concurrency behavior. These options are only available to be set +on the :cpp:class:`AdbcStatement` object: ``adbc.rpc.result_queue_size`` - The number of batches to queue per endpoint. Defaults to 5. + The number of batches to queue in the record reader. Defaults to 200. + Must be an integer > 0. + +``adbc.snowflake.rpc.prefetch-concurrency`` + The number of concurrent streams being fetched from snowflake at a time. + Defaults to 10. Must be an integer > 0. Transactions ------------ diff --git a/go/adbc/driver/snowflake/connection.go b/go/adbc/driver/snowflake/connection.go index 3d1b63d508..45d74d2d73 100644 --- a/go/adbc/driver/snowflake/connection.go +++ b/go/adbc/driver/snowflake/connection.go @@ -33,7 +33,10 @@ import ( "github.com/snowflakedb/gosnowflake" ) -const defaultStatementQueueSize = 200 +const ( + defaultStatementQueueSize = 200 + defaultPrefectConcurrency = 10 +) type snowflakeConn interface { driver.Conn @@ -779,9 +782,10 @@ func (c *cnxn) Rollback(_ context.Context) error { // NewStatement initializes a new statement object tied to this connection func (c *cnxn) NewStatement() (adbc.Statement, error) { return &statement{ - alloc: c.db.alloc, - cnxn: c, - queueSize: defaultStatementQueueSize, + alloc: c.db.alloc, + cnxn: c, + queueSize: defaultStatementQueueSize, + prefetchConcurrency: defaultPrefectConcurrency, }, nil } diff --git a/go/adbc/driver/snowflake/record_reader.go b/go/adbc/driver/snowflake/record_reader.go index 73aabd7851..db0bf0f89f 100644 --- a/go/adbc/driver/snowflake/record_reader.go +++ b/go/adbc/driver/snowflake/record_reader.go @@ -21,7 +21,6 @@ import ( "context" "encoding/hex" "math" - "os" "strconv" "strings" "sync/atomic" @@ -37,19 +36,6 @@ import ( "golang.org/x/sync/errgroup" ) -const defaultConcurrentPrefetch = 10 - -var concurrentPrefetch int - -func init() { - concurrentPrefetch = defaultConcurrentPrefetch - if value := os.Getenv("ADBC_SNOWFLAKE_DEFAULT_CONCURRENT_PREFETCH"); value != "" { - if n, err := strconv.Atoi(value); err == nil && n > 0 { - concurrentPrefetch = n - } - } -} - func identCol(_ context.Context, a arrow.Array) (arrow.Array, error) { a.Retain() return a, nil @@ -427,7 +413,7 @@ type reader struct { cancelFn context.CancelFunc } -func newRecordReader(ctx context.Context, alloc memory.Allocator, ld gosnowflake.ArrowStreamLoader, bufferSize int) (array.RecordReader, error) { +func newRecordReader(ctx context.Context, alloc memory.Allocator, ld gosnowflake.ArrowStreamLoader, bufferSize, prefetchConcurrency int) (array.RecordReader, error) { batches, err := ld.GetBatches() if err != nil { return nil, errToAdbcErr(adbc.StatusInternal, err) @@ -492,7 +478,7 @@ func newRecordReader(ctx context.Context, alloc memory.Allocator, ld gosnowflake } }() - group.SetLimit(concurrentPrefetch) + group.SetLimit(prefetchConcurrency) group.Go(func() error { defer rr.Release() defer r.Close() diff --git a/go/adbc/driver/snowflake/statement.go b/go/adbc/driver/snowflake/statement.go index be5547809f..bb21b6cdd6 100644 --- a/go/adbc/driver/snowflake/statement.go +++ b/go/adbc/driver/snowflake/statement.go @@ -33,13 +33,15 @@ import ( ) const ( - OptionStatementQueueSize = "adbc.rpc.result_queue_size" + OptionStatementQueueSize = "adbc.rpc.result_queue_size" + OptionStatementPrefetchConcurrency = "adbc.snowflake.rpc.prefetch-concurrency" ) type statement struct { - cnxn *cnxn - alloc memory.Allocator - queueSize int + cnxn *cnxn + alloc memory.Allocator + queueSize int + prefetchConcurrency int query string targetTable string @@ -97,7 +99,28 @@ func (st *statement) SetOption(key string, val string) error { Code: adbc.StatusInvalidArgument, } } + if sz <= 0 { + return adbc.Error{ + Msg: fmt.Sprintf("invalid value ('%d') for option '%s', must be > 0", sz, key), + Code: adbc.StatusInvalidArgument, + } + } st.queueSize = sz + case OptionStatementPrefetchConcurrency: + concurrency, err := strconv.Atoi(val) + if err != nil { + return adbc.Error{ + Msg: fmt.Sprintf("could not parse '%s' as int for option '%s'", val, key), + Code: adbc.StatusInvalidArgument, + } + } + if concurrency <= 0 { + return adbc.Error{ + Msg: fmt.Sprintf("invalid value ('%d') for option '%s', must be > 0", concurrency, key), + Code: adbc.StatusInvalidArgument, + } + } + st.prefetchConcurrency = concurrency default: return adbc.Error{ Msg: fmt.Sprintf("invalid statement option %s=%s", key, val), @@ -436,7 +459,7 @@ func (st *statement) ExecuteQuery(ctx context.Context) (array.RecordReader, int6 return nil, -1, errToAdbcErr(adbc.StatusInternal, err) } - rdr, err := newRecordReader(ctx, st.alloc, loader, st.queueSize) + rdr, err := newRecordReader(ctx, st.alloc, loader, st.queueSize, st.prefetchConcurrency) nrec := loader.TotalRows() return rdr, nrec, err } From d780dcfb195ed36502c6e27505ec411cd156427e Mon Sep 17 00:00:00 2001 From: Matt Topol Date: Wed, 9 Aug 2023 13:10:29 -0400 Subject: [PATCH 4/6] linting --- c/validation/adbc_validation_util.h | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/c/validation/adbc_validation_util.h b/c/validation/adbc_validation_util.h index 432dd7bbd7..5e8913ebf4 100644 --- a/c/validation/adbc_validation_util.h +++ b/c/validation/adbc_validation_util.h @@ -206,8 +206,12 @@ struct GetObjectsReader { } ~GetObjectsReader() { AdbcGetObjectsDataDelete(get_objects_data_); } - struct AdbcGetObjectsData* operator*() { return get_objects_data_; } - struct AdbcGetObjectsData* operator->() { return get_objects_data_; } + struct AdbcGetObjectsData* operator*() { + return get_objects_data_; + } + struct AdbcGetObjectsData* operator->() { + return get_objects_data_; + } private: struct AdbcGetObjectsData* get_objects_data_; From 17dcb2261e01146d641ededed07358fffdb3bd54 Mon Sep 17 00:00:00 2001 From: Matt Topol Date: Thu, 10 Aug 2023 12:36:44 -0400 Subject: [PATCH 5/6] updates from feedback --- c/validation/adbc_validation_util.h | 8 ++++---- docs/source/driver/snowflake.rst | 2 +- go/adbc/driver/snowflake/connection.go | 6 +++--- go/adbc/driver/snowflake/statement.go | 2 +- .../adbc_driver_snowflake/__init__.py | 7 +++++-- 5 files changed, 14 insertions(+), 11 deletions(-) diff --git a/c/validation/adbc_validation_util.h b/c/validation/adbc_validation_util.h index 5e8913ebf4..7c60e3cd6a 100644 --- a/c/validation/adbc_validation_util.h +++ b/c/validation/adbc_validation_util.h @@ -206,11 +206,11 @@ struct GetObjectsReader { } ~GetObjectsReader() { AdbcGetObjectsDataDelete(get_objects_data_); } - struct AdbcGetObjectsData* operator*() { - return get_objects_data_; + struct AdbcGetObjectsData* operator*() { + return get_objects_data_; } - struct AdbcGetObjectsData* operator->() { - return get_objects_data_; + struct AdbcGetObjectsData* operator->() { + return get_objects_data_; } private: diff --git a/docs/source/driver/snowflake.rst b/docs/source/driver/snowflake.rst index 9737bf00b6..3e17b5b060 100644 --- a/docs/source/driver/snowflake.rst +++ b/docs/source/driver/snowflake.rst @@ -215,7 +215,7 @@ on the :cpp:class:`AdbcStatement` object: The number of batches to queue in the record reader. Defaults to 200. Must be an integer > 0. -``adbc.snowflake.rpc.prefetch-concurrency`` +``adbc.snowflake.rpc.prefetch_concurrency`` The number of concurrent streams being fetched from snowflake at a time. Defaults to 10. Must be an integer > 0. diff --git a/go/adbc/driver/snowflake/connection.go b/go/adbc/driver/snowflake/connection.go index 45d74d2d73..b08b94a7bc 100644 --- a/go/adbc/driver/snowflake/connection.go +++ b/go/adbc/driver/snowflake/connection.go @@ -34,8 +34,8 @@ import ( ) const ( - defaultStatementQueueSize = 200 - defaultPrefectConcurrency = 10 + defaultStatementQueueSize = 200 + defaultPrefetchConcurrency = 10 ) type snowflakeConn interface { @@ -785,7 +785,7 @@ func (c *cnxn) NewStatement() (adbc.Statement, error) { alloc: c.db.alloc, cnxn: c, queueSize: defaultStatementQueueSize, - prefetchConcurrency: defaultPrefectConcurrency, + prefetchConcurrency: defaultPrefetchConcurrency, }, nil } diff --git a/go/adbc/driver/snowflake/statement.go b/go/adbc/driver/snowflake/statement.go index bb21b6cdd6..e0d14582d9 100644 --- a/go/adbc/driver/snowflake/statement.go +++ b/go/adbc/driver/snowflake/statement.go @@ -34,7 +34,7 @@ import ( const ( OptionStatementQueueSize = "adbc.rpc.result_queue_size" - OptionStatementPrefetchConcurrency = "adbc.snowflake.rpc.prefetch-concurrency" + OptionStatementPrefetchConcurrency = "adbc.snowflake.rpc.prefetch_concurrency" ) type statement struct { diff --git a/python/adbc_driver_snowflake/adbc_driver_snowflake/__init__.py b/python/adbc_driver_snowflake/adbc_driver_snowflake/__init__.py index daf88084e2..65c6acbcab 100644 --- a/python/adbc_driver_snowflake/adbc_driver_snowflake/__init__.py +++ b/python/adbc_driver_snowflake/adbc_driver_snowflake/__init__.py @@ -90,8 +90,11 @@ class DatabaseOptions(enum.Enum): class StatementOptions(enum.Enum): """Statement options specific to the Snowflake driver.""" - #: The number of rows per batch. Defaults to 1024. - BATCH_ROWS = "adbc.rpc.result_queue_size" + #: The number of batches queued up at a time. Defaults to 200. + RESULT_QUEUE_SIZE = "adbc.rpc.result_queue_size" + #: Number of concurrent streams being prefetched for a result set. + #: Defaults to 10. + PREFETCH_CONCURRENCY = "adbc.snowflake.rpc.prefetch_concurrency" def connect( From 9f30a4161e59da4745d331114261ce2723bc45b0 Mon Sep 17 00:00:00 2001 From: Matt Topol Date: Thu, 10 Aug 2023 15:59:51 -0400 Subject: [PATCH 6/6] fix broken test --- python/adbc_driver_snowflake/tests/test_lowlevel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/adbc_driver_snowflake/tests/test_lowlevel.py b/python/adbc_driver_snowflake/tests/test_lowlevel.py index 18d1e778cb..0d8c9beec7 100644 --- a/python/adbc_driver_snowflake/tests/test_lowlevel.py +++ b/python/adbc_driver_snowflake/tests/test_lowlevel.py @@ -49,7 +49,7 @@ def test_options(snowflake): with adbc_driver_manager.AdbcStatement(snowflake) as stmt: stmt.set_options( **{ - adbc_driver_snowflake.StatementOptions.BATCH_ROWS.value: "1", + adbc_driver_snowflake.StatementOptions.RESULT_QUEUE_SIZE.value: "1", } ) stmt.set_sql_query("SELECT 1")