From ca653492cf2b6596424a2547d44c2f32120b5f4f Mon Sep 17 00:00:00 2001 From: wforget <643348094@qq.com> Date: Fri, 26 Apr 2024 15:58:59 +0800 Subject: [PATCH 1/4] [KYUUBI #6338] Support `kyuubi.server.thrift.resultset.default.fetch.size` conf --- docs/configuration/settings.md | 1 + .../org/apache/kyuubi/config/KyuubiConf.scala | 16 ++++++++++++++++ .../server/KyuubiTBinaryFrontendService.scala | 7 +++++++ .../KyuubiTBinaryFrontendServiceSuite.scala | 13 +++++++++++++ 4 files changed, 37 insertions(+) diff --git a/docs/configuration/settings.md b/docs/configuration/settings.md index c1e418d6249..e922e3dcacb 100644 --- a/docs/configuration/settings.md +++ b/docs/configuration/settings.md @@ -444,6 +444,7 @@ You can configure the Kyuubi properties in `$KYUUBI_HOME/conf/kyuubi-defaults.co | kyuubi.server.name | <undefined> | The name of Kyuubi Server. | string | 1.5.0 | | kyuubi.server.periodicGC.interval | PT30M | How often to trigger a garbage collection. | duration | 1.7.0 | | kyuubi.server.redaction.regex | <undefined> | Regex to decide which Kyuubi contain sensitive information. When this regex matches a property key or value, the value is redacted from the various logs. || 1.6.0 | +| kyuubi.server.thrift.resultset.default.fetch.size | 1000 | The number of rows sent in one Fetch RPC call by the server to the client, if not specified by the client. Respect `hive.server2.thrift.resultset.default.fetch.size` hive conf. | int | 1.9.1 | ### Session diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala index bf547c5ff77..c3d37dd1989 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala @@ -3620,4 +3620,20 @@ object KyuubiConf { .version("1.8.1") .booleanConf .createWithDefault(false) + + private val HIVE_SERVER2_THRIFT_RESULTSET_DEFAULT_FETCH_SIZE: ConfigEntry[Int] = + buildConf("hive.server2.thrift.resultset.default.fetch.size") + .internal + .serverOnly + .intConf + .createWithDefault(1000) + + val KYUUBI_SERVER_THRIFT_RESULTSET_DEFAULT_FETCH_SIZE: ConfigEntry[Int] = + buildConf("kyuubi.server.thrift.resultset.default.fetch.size") + .doc("The number of rows sent in one Fetch RPC call by the server to the client, if not" + + " specified by the client. Respect `hive.server2.thrift.resultset.default.fetch.size`" + + " hive conf.") + .version("1.9.1") + .serverOnly + .fallbackConf(HIVE_SERVER2_THRIFT_RESULTSET_DEFAULT_FETCH_SIZE) } diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/server/KyuubiTBinaryFrontendService.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/server/KyuubiTBinaryFrontendService.scala index b46c1fec4e2..b572c0d8f20 100644 --- a/kyuubi-server/src/main/scala/org/apache/kyuubi/server/KyuubiTBinaryFrontendService.scala +++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/server/KyuubiTBinaryFrontendService.scala @@ -24,6 +24,7 @@ import org.apache.hadoop.conf.Configuration import org.apache.kyuubi.KyuubiSQLException import org.apache.kyuubi.cli.Handle import org.apache.kyuubi.config.KyuubiConf +import org.apache.kyuubi.config.KyuubiConf.KYUUBI_SERVER_THRIFT_RESULTSET_DEFAULT_FETCH_SIZE import org.apache.kyuubi.config.KyuubiReservedKeys._ import org.apache.kyuubi.ha.client.{KyuubiServiceDiscovery, ServiceDiscovery} import org.apache.kyuubi.metrics.MetricsConstants._ @@ -49,6 +50,8 @@ final class KyuubiTBinaryFrontendService( } } + private lazy val defaultFetchSize = conf.get(KYUUBI_SERVER_THRIFT_RESULTSET_DEFAULT_FETCH_SIZE) + override def initialize(conf: KyuubiConf): Unit = synchronized { super.initialize(conf) @@ -94,6 +97,10 @@ final class KyuubiTBinaryFrontendService( respConfiguration.put(KYUUBI_SESSION_ENGINE_LAUNCH_SUPPORT_RESULT, true.toString) + // KYUUBI-6338: Set default fetch size + respConfiguration.put("hive.server2.thrift.resultset.default.fetch.size", + defaultFetchSize.toString) + resp.setSessionHandle(sessionHandle.toTSessionHandle) resp.setConfiguration(respConfiguration) resp.setStatus(OK_STATUS) diff --git a/kyuubi-server/src/test/scala/org/apache/kyuubi/server/KyuubiTBinaryFrontendServiceSuite.scala b/kyuubi-server/src/test/scala/org/apache/kyuubi/server/KyuubiTBinaryFrontendServiceSuite.scala index 9b41fb067c3..6347d7ae722 100644 --- a/kyuubi-server/src/test/scala/org/apache/kyuubi/server/KyuubiTBinaryFrontendServiceSuite.scala +++ b/kyuubi-server/src/test/scala/org/apache/kyuubi/server/KyuubiTBinaryFrontendServiceSuite.scala @@ -30,6 +30,7 @@ import org.apache.kyuubi.shaded.hive.service.rpc.thrift.{TOpenSessionReq, TSessi class KyuubiTBinaryFrontendServiceSuite extends WithKyuubiServer with KyuubiFunSuite { override protected val conf: KyuubiConf = KyuubiConf() + .set(KyuubiConf.KYUUBI_SERVER_THRIFT_RESULTSET_DEFAULT_FETCH_SIZE, 500) test("connection metrics") { val totalConnections = @@ -116,4 +117,16 @@ class KyuubiTBinaryFrontendServiceSuite extends WithKyuubiServer with KyuubiFunS Thread.sleep(3000L) assert(server.backendService.sessionManager.allSessions().size == sessionCount) } + + test("test kyuubi.server.thrift.resultset.default.fetch.size") { + TClientTestUtils.withThriftClient(server.frontendServices.head) { + client => + val req = new TOpenSessionReq() + req.setUsername(Utils.currentUser) + req.setPassword("anonymous") + val resp = client.OpenSession(req) + assertResult("500")( + resp.getConfiguration.get("hive.server2.thrift.resultset.default.fetch.size")) + } + } } From 48b8ec6cd4df81a147b969c516655c3243ca1612 Mon Sep 17 00:00:00 2001 From: wforget <643348094@qq.com> Date: Fri, 26 Apr 2024 17:47:56 +0800 Subject: [PATCH 2/4] fix style and address comments --- .../src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala | 2 ++ .../apache/kyuubi/server/KyuubiTBinaryFrontendService.scala | 5 +++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala index c3d37dd1989..1e19668c11d 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala @@ -3623,6 +3623,8 @@ object KyuubiConf { private val HIVE_SERVER2_THRIFT_RESULTSET_DEFAULT_FETCH_SIZE: ConfigEntry[Int] = buildConf("hive.server2.thrift.resultset.default.fetch.size") + .doc("The number of rows sent in one Fetch RPC call by the server to the client, if not" + + " specified by the client. This is a hive server configuration.") .internal .serverOnly .intConf diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/server/KyuubiTBinaryFrontendService.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/server/KyuubiTBinaryFrontendService.scala index b572c0d8f20..47dbcb78b44 100644 --- a/kyuubi-server/src/main/scala/org/apache/kyuubi/server/KyuubiTBinaryFrontendService.scala +++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/server/KyuubiTBinaryFrontendService.scala @@ -97,8 +97,9 @@ final class KyuubiTBinaryFrontendService( respConfiguration.put(KYUUBI_SESSION_ENGINE_LAUNCH_SUPPORT_RESULT, true.toString) - // KYUUBI-6338: Set default fetch size - respConfiguration.put("hive.server2.thrift.resultset.default.fetch.size", + // After HIVE-23005, hive driver requires this conf + respConfiguration.put( + "hive.server2.thrift.resultset.default.fetch.size", defaultFetchSize.toString) resp.setSessionHandle(sessionHandle.toTSessionHandle) From 5e64ed3a528df0f4e64615b62d3f66b63a9f6a39 Mon Sep 17 00:00:00 2001 From: wforget <643348094@qq.com> Date: Fri, 26 Apr 2024 17:54:07 +0800 Subject: [PATCH 3/4] comments --- .../src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala | 5 +++-- .../apache/kyuubi/server/KyuubiTBinaryFrontendService.scala | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala index 1e19668c11d..38342c5415c 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala @@ -3623,8 +3623,9 @@ object KyuubiConf { private val HIVE_SERVER2_THRIFT_RESULTSET_DEFAULT_FETCH_SIZE: ConfigEntry[Int] = buildConf("hive.server2.thrift.resultset.default.fetch.size") - .doc("The number of rows sent in one Fetch RPC call by the server to the client, if not" + - " specified by the client. This is a hive server configuration.") + .doc("This is a hive server configuration used as a fallback conf" + + s" for `${KYUUBI_SERVER_THRIFT_RESULTSET_DEFAULT_FETCH_SIZE.key}`.") + .version("1.9.1") .internal .serverOnly .intConf diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/server/KyuubiTBinaryFrontendService.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/server/KyuubiTBinaryFrontendService.scala index 47dbcb78b44..f95ea594773 100644 --- a/kyuubi-server/src/main/scala/org/apache/kyuubi/server/KyuubiTBinaryFrontendService.scala +++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/server/KyuubiTBinaryFrontendService.scala @@ -97,7 +97,7 @@ final class KyuubiTBinaryFrontendService( respConfiguration.put(KYUUBI_SESSION_ENGINE_LAUNCH_SUPPORT_RESULT, true.toString) - // After HIVE-23005, hive driver requires this conf + // HIVE-23005(4.0.0), Hive JDBC driver supposes that server always returns this conf respConfiguration.put( "hive.server2.thrift.resultset.default.fetch.size", defaultFetchSize.toString) From acd73a16b5e6925ba02ccbcd843538fd194eb215 Mon Sep 17 00:00:00 2001 From: wforget <643348094@qq.com> Date: Fri, 26 Apr 2024 18:03:21 +0800 Subject: [PATCH 4/4] fix --- .../src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala index 38342c5415c..22d31f7df4e 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala @@ -3624,7 +3624,7 @@ object KyuubiConf { private val HIVE_SERVER2_THRIFT_RESULTSET_DEFAULT_FETCH_SIZE: ConfigEntry[Int] = buildConf("hive.server2.thrift.resultset.default.fetch.size") .doc("This is a hive server configuration used as a fallback conf" + - s" for `${KYUUBI_SERVER_THRIFT_RESULTSET_DEFAULT_FETCH_SIZE.key}`.") + s" for `kyuubi.server.thrift.resultset.default.fetch.size`.") .version("1.9.1") .internal .serverOnly