From 2df635099bcee47e03fb9eccb21aa07f64b03958 Mon Sep 17 00:00:00 2001
From: Lilit0x <baasisek01@gmail.com>
Date: Thu, 21 Sep 2023 20:52:18 +0100
Subject: [PATCH 01/36] feat: open_tsdb support for row insert requests

---
 src/frontend/src/instance/opentsdb.rs |  8 +++---
 src/servers/src/opentsdb.rs           | 37 ++++++++++++++++++++++++++-
 2 files changed, 39 insertions(+), 6 deletions(-)

diff --git a/src/frontend/src/instance/opentsdb.rs b/src/frontend/src/instance/opentsdb.rs
index 1ac8fe029048..e4d8ff14cf04 100644
--- a/src/frontend/src/instance/opentsdb.rs
+++ b/src/frontend/src/instance/opentsdb.rs
@@ -12,13 +12,13 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use api::v1::InsertRequests;
 use async_trait::async_trait;
 use auth::{PermissionChecker, PermissionCheckerRef, PermissionReq};
 use common_error::ext::BoxedError;
 use servers::error as server_error;
 use servers::error::AuthSnafu;
 use servers::opentsdb::codec::DataPoint;
+use servers::opentsdb::data_point_to_grpc_row_insert_requests;
 use servers::query_handler::OpentsdbProtocolHandler;
 use session::context::QueryContextRef;
 use snafu::prelude::*;
@@ -34,11 +34,9 @@ impl OpentsdbProtocolHandler for Instance {
             .check_permission(ctx.current_user(), PermissionReq::Opentsdb)
             .context(AuthSnafu)?;
 
-        let requests = InsertRequests {
-            inserts: vec![data_point.as_grpc_insert()],
-        };
+        let (requests, _) = data_point_to_grpc_row_insert_requests(data_point)?;
         let _ = self
-            .handle_inserts(requests, ctx)
+            .handle_row_inserts(requests, ctx)
             .await
             .map_err(BoxedError::new)
             .with_context(|_| server_error::ExecuteQuerySnafu {
diff --git a/src/servers/src/opentsdb.rs b/src/servers/src/opentsdb.rs
index 7c569cc2ffed..a53f3bc89468 100644
--- a/src/servers/src/opentsdb.rs
+++ b/src/servers/src/opentsdb.rs
@@ -20,6 +20,7 @@ use std::future::Future;
 use std::net::SocketAddr;
 use std::sync::Arc;
 
+use api::v1::RowInsertRequests;
 use async_trait::async_trait;
 use common_runtime::Runtime;
 use common_telemetry::logging::error;
@@ -27,12 +28,14 @@ use futures::StreamExt;
 use tokio::sync::broadcast;
 
 use crate::error::Result;
+use crate::opentsdb::codec::DataPoint;
 use crate::opentsdb::connection::Connection;
 use crate::opentsdb::handler::Handler;
+use crate::prom_store::{FIELD_COLUMN_NAME, TIMESTAMP_COLUMN_NAME};
 use crate::query_handler::OpentsdbProtocolHandlerRef;
+use crate::row_writer::{self, MultiTableData};
 use crate::server::{AbortableStream, BaseTcpServer, Server};
 use crate::shutdown::Shutdown;
-
 pub struct OpentsdbServer {
     base_server: BaseTcpServer,
     query_handler: OpentsdbProtocolHandlerRef,
@@ -123,3 +126,35 @@ impl Server for OpentsdbServer {
         OPENTSDB_SERVER
     }
 }
+
+pub fn data_point_to_grpc_row_insert_requests(
+    data_point: &DataPoint,
+) -> Result<(RowInsertRequests, usize)> {
+    let mut multi_table_data = MultiTableData::new();
+    let table_name = data_point.metric();
+    let tags = data_point.tags();
+    let value = data_point.value();
+    let timestamp = data_point.ts_millis();
+    let num_columns = tags.len() + 1;
+
+    let table_data = multi_table_data.get_or_default_table_data(table_name, num_columns, 0);
+    let mut one_row = table_data.alloc_one_row();
+
+    //tags
+    let kvs = tags.iter().map(|(k, v)| (k.as_str(), v.as_str()));
+    row_writer::write_tags(table_data, kvs, &mut one_row)?;
+
+    // value
+    row_writer::write_f64(table_data, FIELD_COLUMN_NAME, value, &mut one_row)?;
+    // timestamp
+    row_writer::write_ts_millis(
+        table_data,
+        TIMESTAMP_COLUMN_NAME,
+        Some(timestamp),
+        &mut one_row,
+    )?;
+
+    table_data.add_row(one_row);
+
+    Ok(multi_table_data.into_row_insert_requests())
+}

From 67651e2be18ff51e77f8c4510b4f2d44af3bedbe Mon Sep 17 00:00:00 2001
From: Ruihang Xia <waynestxia@gmail.com>
Date: Fri, 22 Sep 2023 09:57:48 +0800
Subject: [PATCH 02/36] refactor: remove SqlStatementExecutor (#2464)

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>
---
 src/datanode/src/lib.rs       | 16 ----------------
 src/datanode/src/tests.rs     | 22 +---------------------
 src/query/src/query_engine.rs |  8 --------
 3 files changed, 1 insertion(+), 45 deletions(-)

diff --git a/src/datanode/src/lib.rs b/src/datanode/src/lib.rs
index 890e5c38ce92..62a9bd9f309a 100644
--- a/src/datanode/src/lib.rs
+++ b/src/datanode/src/lib.rs
@@ -15,8 +15,6 @@
 #![feature(assert_matches)]
 #![feature(trait_upcasting)]
 
-use query::query_engine::SqlStatementExecutor;
-
 pub mod alive_keeper;
 pub mod config;
 pub mod datanode;
@@ -31,17 +29,3 @@ mod store;
 #[cfg(test)]
 #[allow(dead_code)]
 mod tests;
-
-// TODO(ruihang): remove this
-pub struct Instance;
-
-#[async_trait::async_trait]
-impl SqlStatementExecutor for Instance {
-    async fn execute_sql(
-        &self,
-        _stmt: sql::statements::statement::Statement,
-        _query_ctx: session::context::QueryContextRef,
-    ) -> query::error::Result<common_query::Output> {
-        unreachable!()
-    }
-}
diff --git a/src/datanode/src/tests.rs b/src/datanode/src/tests.rs
index 3407390b463a..43a06b34db96 100644
--- a/src/datanode/src/tests.rs
+++ b/src/datanode/src/tests.rs
@@ -23,7 +23,7 @@ use common_meta::heartbeat::handler::{
     HeartbeatResponseHandlerContext, HeartbeatResponseHandlerExecutor,
 };
 use common_meta::heartbeat::mailbox::{HeartbeatMailbox, MessageMeta};
-use common_meta::instruction::{Instruction, InstructionReply, OpenRegion, RegionIdent};
+use common_meta::instruction::{Instruction, OpenRegion, RegionIdent};
 use common_query::prelude::ScalarUdf;
 use common_query::Output;
 use common_runtime::Runtime;
@@ -34,29 +34,9 @@ use query::query_engine::DescribeResult;
 use query::QueryEngine;
 use session::context::QueryContextRef;
 use table::TableRef;
-use tokio::sync::mpsc::{self, Receiver};
 
 use crate::event_listener::NoopRegionServerEventListener;
 use crate::region_server::RegionServer;
-use crate::Instance;
-
-struct HandlerTestGuard {
-    instance: Instance,
-    mailbox: Arc<HeartbeatMailbox>,
-    rx: Receiver<(MessageMeta, InstructionReply)>,
-}
-
-async fn prepare_handler_test(_name: &str) -> HandlerTestGuard {
-    let instance = Instance;
-    let (tx, rx) = mpsc::channel(8);
-    let mailbox = Arc::new(HeartbeatMailbox::new(tx));
-
-    HandlerTestGuard {
-        instance,
-        mailbox,
-        rx,
-    }
-}
 
 pub fn test_message_meta(id: u64, subject: &str, to: &str, from: &str) -> MessageMeta {
     MessageMeta {
diff --git a/src/query/src/query_engine.rs b/src/query/src/query_engine.rs
index b0dde22c36c3..b18834d2c8ec 100644
--- a/src/query/src/query_engine.rs
+++ b/src/query/src/query_engine.rs
@@ -28,7 +28,6 @@ use common_query::prelude::ScalarUdf;
 use common_query::Output;
 use datatypes::schema::Schema;
 use session::context::QueryContextRef;
-use sql::statements::statement::Statement;
 use table::TableRef;
 
 use crate::dataframe::DataFrame;
@@ -40,8 +39,6 @@ pub use crate::query_engine::context::QueryEngineContext;
 pub use crate::query_engine::state::QueryEngineState;
 use crate::region_query::RegionQueryHandlerRef;
 
-pub type SqlStatementExecutorRef = Arc<dyn SqlStatementExecutor>;
-
 /// Describe statement result
 #[derive(Debug)]
 pub struct DescribeResult {
@@ -51,11 +48,6 @@ pub struct DescribeResult {
     pub logical_plan: LogicalPlan,
 }
 
-#[async_trait]
-pub trait SqlStatementExecutor: Send + Sync {
-    async fn execute_sql(&self, stmt: Statement, query_ctx: QueryContextRef) -> Result<Output>;
-}
-
 #[async_trait]
 pub trait QueryEngine: Send + Sync {
     /// Returns the query engine as Any

From 2292c4170ada0d04b0d89f1f85a13f0e5f912c1f Mon Sep 17 00:00:00 2001
From: LinFeng <thewawar@gmail.com>
Date: Fri, 22 Sep 2023 10:07:46 +0800
Subject: [PATCH 03/36] feat: limit grpc message size (#2459)

* feat: add two grpc config options

Those options are for:
* Limit receiving(decoding) message size
* Limit sending(enoding) message size

* test: add integration tests for message size limit
---
 src/client/src/client.rs                | 18 +++++--
 src/common/grpc/src/channel_manager.rs  | 19 ++++---
 src/datanode/src/config.rs              |  9 ++++
 src/datanode/src/server.rs              |  7 ++-
 src/frontend/src/server.rs              |  7 ++-
 src/frontend/src/service_config/grpc.rs |  9 ++++
 src/servers/src/grpc.rs                 | 39 ++++++++++-----
 tests-integration/src/cluster.rs        |  1 +
 tests-integration/src/test_util.rs      | 15 +++++-
 tests-integration/tests/grpc.rs         | 66 ++++++++++++++++++++++++-
 tests-integration/tests/http.rs         |  2 +
 11 files changed, 162 insertions(+), 30 deletions(-)

diff --git a/src/client/src/client.rs b/src/client/src/client.rs
index ada1ae92c56a..c5457e58741f 100644
--- a/src/client/src/client.rs
+++ b/src/client/src/client.rs
@@ -138,8 +138,12 @@ impl Client {
         Ok((addr, channel))
     }
 
-    fn max_grpc_message_size(&self) -> usize {
-        self.inner.channel_manager.config().max_message_size
+    fn max_grpc_recv_message_size(&self) -> usize {
+        self.inner.channel_manager.config().max_recv_message_size
+    }
+
+    fn max_grpc_send_message_size(&self) -> usize {
+        self.inner.channel_manager.config().max_send_message_size
     }
 
     pub(crate) fn make_flight_client(&self) -> Result<FlightClient> {
@@ -147,7 +151,8 @@ impl Client {
         Ok(FlightClient {
             addr,
             client: FlightServiceClient::new(channel)
-                .max_decoding_message_size(self.max_grpc_message_size()),
+                .max_decoding_message_size(self.max_grpc_recv_message_size())
+                .max_encoding_message_size(self.max_grpc_send_message_size()),
         })
     }
 
@@ -155,13 +160,16 @@ impl Client {
         let (_, channel) = self.find_channel()?;
         Ok(DatabaseClient {
             inner: GreptimeDatabaseClient::new(channel)
-                .max_decoding_message_size(self.max_grpc_message_size()),
+                .max_decoding_message_size(self.max_grpc_recv_message_size())
+                .max_encoding_message_size(self.max_grpc_send_message_size()),
         })
     }
 
     pub(crate) fn raw_region_client(&self) -> Result<PbRegionClient<Channel>> {
         let (_, channel) = self.find_channel()?;
-        Ok(PbRegionClient::new(channel).max_decoding_message_size(self.max_grpc_message_size()))
+        Ok(PbRegionClient::new(channel)
+            .max_decoding_message_size(self.max_grpc_recv_message_size())
+            .max_encoding_message_size(self.max_grpc_send_message_size()))
     }
 
     pub fn make_prometheus_gateway_client(&self) -> Result<PrometheusGatewayClient<Channel>> {
diff --git a/src/common/grpc/src/channel_manager.rs b/src/common/grpc/src/channel_manager.rs
index 1102ac0fd303..f52177e2890d 100644
--- a/src/common/grpc/src/channel_manager.rs
+++ b/src/common/grpc/src/channel_manager.rs
@@ -31,7 +31,8 @@ use crate::error::{CreateChannelSnafu, InvalidConfigFilePathSnafu, InvalidTlsCon
 const RECYCLE_CHANNEL_INTERVAL_SECS: u64 = 60;
 pub const DEFAULT_GRPC_REQUEST_TIMEOUT_SECS: u64 = 10;
 pub const DEFAULT_GRPC_CONNECT_TIMEOUT_SECS: u64 = 10;
-pub const DEFAULT_MAX_GRPC_MESSAGE_SIZE: usize = 512 * 1024 * 1024;
+pub const DEFAULT_MAX_GRPC_RECV_MESSAGE_SIZE: usize = 512 * 1024 * 1024;
+pub const DEFAULT_MAX_GRPC_SEND_MESSAGE_SIZE: usize = 512 * 1024 * 1024;
 
 lazy_static! {
     static ref ID: AtomicU64 = AtomicU64::new(0);
@@ -248,9 +249,10 @@ pub struct ChannelConfig {
     pub tcp_keepalive: Option<Duration>,
     pub tcp_nodelay: bool,
     pub client_tls: Option<ClientTlsOption>,
-    // Max gRPC message size
-    // TODO(dennis): make it configurable
-    pub max_message_size: usize,
+    // Max gRPC receiving(decoding) message size
+    pub max_recv_message_size: usize,
+    // Max gRPC sending(encoding) message size
+    pub max_send_message_size: usize,
 }
 
 impl Default for ChannelConfig {
@@ -269,7 +271,8 @@ impl Default for ChannelConfig {
             tcp_keepalive: None,
             tcp_nodelay: true,
             client_tls: None,
-            max_message_size: DEFAULT_MAX_GRPC_MESSAGE_SIZE,
+            max_recv_message_size: DEFAULT_MAX_GRPC_RECV_MESSAGE_SIZE,
+            max_send_message_size: DEFAULT_MAX_GRPC_SEND_MESSAGE_SIZE,
         }
     }
 }
@@ -534,7 +537,8 @@ mod tests {
                 tcp_keepalive: None,
                 tcp_nodelay: true,
                 client_tls: None,
-                max_message_size: DEFAULT_MAX_GRPC_MESSAGE_SIZE,
+                max_recv_message_size: DEFAULT_MAX_GRPC_RECV_MESSAGE_SIZE,
+                max_send_message_size: DEFAULT_MAX_GRPC_SEND_MESSAGE_SIZE,
             },
             default_cfg
         );
@@ -577,7 +581,8 @@ mod tests {
                     client_cert_path: "some_cert_path".to_string(),
                     client_key_path: "some_key_path".to_string(),
                 }),
-                max_message_size: DEFAULT_MAX_GRPC_MESSAGE_SIZE,
+                max_recv_message_size: DEFAULT_MAX_GRPC_RECV_MESSAGE_SIZE,
+                max_send_message_size: DEFAULT_MAX_GRPC_SEND_MESSAGE_SIZE,
             },
             cfg
         );
diff --git a/src/datanode/src/config.rs b/src/datanode/src/config.rs
index 61001f2e39d1..b3835e6e2e84 100644
--- a/src/datanode/src/config.rs
+++ b/src/datanode/src/config.rs
@@ -18,6 +18,9 @@ use std::time::Duration;
 
 use common_base::readable_size::ReadableSize;
 use common_config::WalConfig;
+use common_grpc::channel_manager::{
+    DEFAULT_MAX_GRPC_RECV_MESSAGE_SIZE, DEFAULT_MAX_GRPC_SEND_MESSAGE_SIZE,
+};
 pub use common_procedure::options::ProcedureConfig;
 use common_telemetry::logging::LoggingOptions;
 use file_engine::config::EngineConfig as FileEngineConfig;
@@ -324,6 +327,10 @@ pub struct DatanodeOptions {
     pub rpc_addr: String,
     pub rpc_hostname: Option<String>,
     pub rpc_runtime_size: usize,
+    // Max gRPC receiving(decoding) message size
+    pub rpc_max_recv_message_size: usize,
+    // Max gRPC sending(encoding) message size
+    pub rpc_max_send_message_size: usize,
     pub heartbeat: HeartbeatOptions,
     pub http: HttpOptions,
     pub meta_client: Option<MetaClientOptions>,
@@ -344,6 +351,8 @@ impl Default for DatanodeOptions {
             rpc_addr: "127.0.0.1:3001".to_string(),
             rpc_hostname: None,
             rpc_runtime_size: 8,
+            rpc_max_recv_message_size: DEFAULT_MAX_GRPC_RECV_MESSAGE_SIZE,
+            rpc_max_send_message_size: DEFAULT_MAX_GRPC_SEND_MESSAGE_SIZE,
             http: HttpOptions::default(),
             meta_client: None,
             wal: WalConfig::default(),
diff --git a/src/datanode/src/server.rs b/src/datanode/src/server.rs
index 71c050dc0bbc..1847dc4c992a 100644
--- a/src/datanode/src/server.rs
+++ b/src/datanode/src/server.rs
@@ -16,7 +16,7 @@ use std::net::SocketAddr;
 use std::sync::Arc;
 
 use futures::future;
-use servers::grpc::GrpcServer;
+use servers::grpc::{GrpcServer, GrpcServerConfig};
 use servers::http::{HttpServer, HttpServerBuilder};
 use servers::metrics_handler::MetricsHandler;
 use servers::server::Server;
@@ -39,9 +39,14 @@ impl Services {
         let flight_handler = Some(Arc::new(region_server.clone()) as _);
         let region_server_handler = Some(Arc::new(region_server.clone()) as _);
         let runtime = region_server.runtime();
+        let grpc_config = GrpcServerConfig {
+            max_recv_message_size: opts.rpc_max_recv_message_size,
+            max_send_message_size: opts.rpc_max_send_message_size,
+        };
 
         Ok(Self {
             grpc_server: GrpcServer::new(
+                Some(grpc_config),
                 None,
                 None,
                 flight_handler,
diff --git a/src/frontend/src/server.rs b/src/frontend/src/server.rs
index d96178f1d3ad..5a61c3b48834 100644
--- a/src/frontend/src/server.rs
+++ b/src/frontend/src/server.rs
@@ -22,7 +22,7 @@ use common_runtime::Builder as RuntimeBuilder;
 use common_telemetry::info;
 use servers::configurator::ConfiguratorRef;
 use servers::error::Error::InternalIo;
-use servers::grpc::GrpcServer;
+use servers::grpc::{GrpcServer, GrpcServerConfig};
 use servers::http::HttpServerBuilder;
 use servers::metrics_handler::MetricsHandler;
 use servers::mysql::server::{MysqlServer, MysqlSpawnConfig, MysqlSpawnRef};
@@ -69,7 +69,12 @@ impl Services {
                     .context(error::RuntimeResourceSnafu)?,
             );
 
+            let grpc_config = GrpcServerConfig {
+                max_recv_message_size: opts.max_recv_message_size,
+                max_send_message_size: opts.max_send_message_size,
+            };
             let grpc_server = GrpcServer::new(
+                Some(grpc_config),
                 Some(ServerGrpcQueryHandlerAdaptor::arc(instance.clone())),
                 Some(instance.clone()),
                 None,
diff --git a/src/frontend/src/service_config/grpc.rs b/src/frontend/src/service_config/grpc.rs
index 92d6ea771710..e0a64565015d 100644
--- a/src/frontend/src/service_config/grpc.rs
+++ b/src/frontend/src/service_config/grpc.rs
@@ -12,12 +12,19 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use common_grpc::channel_manager::{
+    DEFAULT_MAX_GRPC_RECV_MESSAGE_SIZE, DEFAULT_MAX_GRPC_SEND_MESSAGE_SIZE,
+};
 use serde::{Deserialize, Serialize};
 
 #[derive(Clone, Debug, Serialize, Deserialize)]
 pub struct GrpcOptions {
     pub addr: String,
     pub runtime_size: usize,
+    // Max gRPC receiving(decoding) message size
+    pub max_recv_message_size: usize,
+    // Max gRPC sending(encoding) message size
+    pub max_send_message_size: usize,
 }
 
 impl Default for GrpcOptions {
@@ -25,6 +32,8 @@ impl Default for GrpcOptions {
         Self {
             addr: "127.0.0.1:4001".to_string(),
             runtime_size: 8,
+            max_recv_message_size: DEFAULT_MAX_GRPC_RECV_MESSAGE_SIZE,
+            max_send_message_size: DEFAULT_MAX_GRPC_SEND_MESSAGE_SIZE,
         }
     }
 }
diff --git a/src/servers/src/grpc.rs b/src/servers/src/grpc.rs
index 5d5582de7faf..69ea1943a6fa 100644
--- a/src/servers/src/grpc.rs
+++ b/src/servers/src/grpc.rs
@@ -33,7 +33,9 @@ use arrow_flight::flight_service_server::FlightService;
 use arrow_flight::flight_service_server::FlightServiceServer;
 use async_trait::async_trait;
 use auth::UserProviderRef;
-use common_grpc::channel_manager::DEFAULT_MAX_GRPC_MESSAGE_SIZE;
+use common_grpc::channel_manager::{
+    DEFAULT_MAX_GRPC_RECV_MESSAGE_SIZE, DEFAULT_MAX_GRPC_SEND_MESSAGE_SIZE,
+};
 use common_runtime::Runtime;
 use common_telemetry::logging::info;
 use common_telemetry::{error, warn};
@@ -82,21 +84,24 @@ pub struct GrpcServer {
 /// Grpc Server configuration
 #[derive(Debug, Clone)]
 pub struct GrpcServerConfig {
-    // Max gRPC message size
-    // TODO(dennis): make it configurable
-    pub max_message_size: usize,
+    // Max gRPC receiving(decoding) message size
+    pub max_recv_message_size: usize,
+    // Max gRPC sending(encoding) message size
+    pub max_send_message_size: usize,
 }
 
 impl Default for GrpcServerConfig {
     fn default() -> Self {
         Self {
-            max_message_size: DEFAULT_MAX_GRPC_MESSAGE_SIZE,
+            max_recv_message_size: DEFAULT_MAX_GRPC_RECV_MESSAGE_SIZE,
+            max_send_message_size: DEFAULT_MAX_GRPC_SEND_MESSAGE_SIZE,
         }
     }
 }
 
 impl GrpcServer {
     pub fn new(
+        config: Option<GrpcServerConfig>,
         query_handler: Option<ServerGrpcQueryHandlerRef>,
         prometheus_handler: Option<PrometheusHandlerRef>,
         flight_handler: Option<FlightCraftRef>,
@@ -110,7 +115,7 @@ impl GrpcServer {
         let region_server_handler = region_server_handler
             .map(|handler| RegionServerRequestHandler::new(handler, runtime.clone()));
         Self {
-            config: GrpcServerConfig::default(),
+            config: config.unwrap_or_default(),
             shutdown_tx: Mutex::new(None),
             user_provider,
             serve_state: Mutex::new(None),
@@ -201,7 +206,8 @@ impl Server for GrpcServer {
     }
 
     async fn start(&self, addr: SocketAddr) -> Result<SocketAddr> {
-        let max_message_size = self.config.max_message_size;
+        let max_recv_message_size = self.config.max_recv_message_size;
+        let max_send_message_size = self.config.max_send_message_size;
         let (tx, rx) = oneshot::channel();
         let (listener, addr) = {
             let mut shutdown_tx = self.shutdown_tx.lock().await;
@@ -227,7 +233,8 @@ impl Server for GrpcServer {
         if let Some(database_handler) = &self.database_handler {
             builder = builder.add_service(
                 GreptimeDatabaseServer::new(DatabaseService::new(database_handler.clone()))
-                    .max_decoding_message_size(max_message_size),
+                    .max_decoding_message_size(max_recv_message_size)
+                    .max_encoding_message_size(max_send_message_size),
             )
         }
         if let Some(prometheus_handler) = &self.prometheus_handler {
@@ -237,18 +244,24 @@ impl Server for GrpcServer {
         if let Some(flight_handler) = &self.flight_handler {
             builder = builder.add_service(
                 FlightServiceServer::new(FlightCraftWrapper(flight_handler.clone()))
-                    .max_decoding_message_size(max_message_size),
+                    .max_decoding_message_size(max_recv_message_size)
+                    .max_encoding_message_size(max_send_message_size),
             )
         } else {
             // TODO(ruihang): this is a temporary workaround before region server is ready.
-            builder = builder.add_service(FlightServiceServer::new(FlightCraftWrapper(
-                self.database_handler.clone().unwrap(),
-            )))
+            builder = builder.add_service(
+                FlightServiceServer::new(FlightCraftWrapper(
+                    self.database_handler.clone().unwrap(),
+                ))
+                .max_decoding_message_size(max_recv_message_size)
+                .max_encoding_message_size(max_send_message_size),
+            )
         }
         if let Some(region_server_handler) = &self.region_server_handler {
             builder = builder.add_service(
                 RegionServer::new(region_server_handler.clone())
-                    .max_decoding_message_size(max_message_size),
+                    .max_decoding_message_size(max_recv_message_size)
+                    .max_encoding_message_size(max_send_message_size),
             );
         }
 
diff --git a/tests-integration/src/cluster.rs b/tests-integration/src/cluster.rs
index 9bdb11f90b00..013eeb681ec6 100644
--- a/tests-integration/src/cluster.rs
+++ b/tests-integration/src/cluster.rs
@@ -272,6 +272,7 @@ async fn create_datanode_client(datanode: &Datanode) -> (String, Client) {
     let flight_handler = Some(Arc::new(datanode.region_server()) as _);
     let region_server_handler = Some(Arc::new(datanode.region_server()) as _);
     let grpc_server = GrpcServer::new(
+        None,
         None,
         None,
         flight_handler,
diff --git a/tests-integration/src/test_util.rs b/tests-integration/src/test_util.rs
index a6c0cf72fb65..e3370836eb2f 100644
--- a/tests-integration/src/test_util.rs
+++ b/tests-integration/src/test_util.rs
@@ -39,7 +39,7 @@ use object_store::test_util::TempFolder;
 use object_store::ObjectStore;
 use secrecy::ExposeSecret;
 use servers::grpc::greptime_handler::GreptimeRequestHandler;
-use servers::grpc::GrpcServer;
+use servers::grpc::{GrpcServer, GrpcServerConfig};
 use servers::http::{HttpOptions, HttpServerBuilder};
 use servers::metrics_handler::MetricsHandler;
 use servers::mysql::server::{MysqlServer, MysqlSpawnConfig, MysqlSpawnRef};
@@ -423,13 +423,22 @@ pub async fn setup_grpc_server(
     store_type: StorageType,
     name: &str,
 ) -> (String, TestGuard, Arc<GrpcServer>) {
-    setup_grpc_server_with_user_provider(store_type, name, None).await
+    setup_grpc_server_with(store_type, name, None, None).await
 }
 
 pub async fn setup_grpc_server_with_user_provider(
     store_type: StorageType,
     name: &str,
     user_provider: Option<UserProviderRef>,
+) -> (String, TestGuard, Arc<GrpcServer>) {
+    setup_grpc_server_with(store_type, name, user_provider, None).await
+}
+
+pub async fn setup_grpc_server_with(
+    store_type: StorageType,
+    name: &str,
+    user_provider: Option<UserProviderRef>,
+    grpc_config: Option<GrpcServerConfig>,
 ) -> (String, TestGuard, Arc<GrpcServer>) {
     let instance = setup_standalone_instance(name, store_type).await;
 
@@ -447,7 +456,9 @@ pub async fn setup_grpc_server_with_user_provider(
         user_provider.clone(),
         runtime.clone(),
     ));
+
     let fe_grpc_server = Arc::new(GrpcServer::new(
+        grpc_config,
         Some(ServerGrpcQueryHandlerAdaptor::arc(fe_instance_ref.clone())),
         Some(fe_instance_ref.clone()),
         Some(flight_handler),
diff --git a/tests-integration/tests/grpc.rs b/tests-integration/tests/grpc.rs
index 7f2f20d87ec8..3d766a5a37f0 100644
--- a/tests-integration/tests/grpc.rs
+++ b/tests-integration/tests/grpc.rs
@@ -24,10 +24,11 @@ use client::{Client, Database, DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
 use common_catalog::consts::{MIN_USER_TABLE_ID, MITO_ENGINE};
 use common_query::Output;
 use common_recordbatch::RecordBatches;
+use servers::grpc::GrpcServerConfig;
 use servers::http::prometheus::{PromData, PromSeries, PrometheusJsonResponse, PrometheusResponse};
 use servers::server::Server;
 use tests_integration::test_util::{
-    setup_grpc_server, setup_grpc_server_with_user_provider, StorageType,
+    setup_grpc_server, setup_grpc_server_with, setup_grpc_server_with_user_provider, StorageType,
 };
 
 #[macro_export]
@@ -64,6 +65,9 @@ macro_rules! grpc_tests {
                 test_auto_create_table,
                 test_insert_and_select,
                 test_dbname,
+                test_grpc_message_size_ok,
+                test_grpc_message_size_limit_recv,
+                test_grpc_message_size_limit_send,
                 test_grpc_auth,
                 test_health_check,
                 test_prom_gateway_query,
@@ -115,6 +119,66 @@ pub async fn test_dbname(store_type: StorageType) {
     guard.remove_all().await;
 }
 
+pub async fn test_grpc_message_size_ok(store_type: StorageType) {
+    let config = GrpcServerConfig {
+        max_recv_message_size: 1024,
+        max_send_message_size: 1024,
+    };
+    let (addr, mut guard, fe_grpc_server) =
+        setup_grpc_server_with(store_type, "auto_create_table", None, Some(config)).await;
+
+    let grpc_client = Client::with_urls(vec![addr]);
+    let db = Database::new_with_dbname(
+        format!("{}-{}", DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME),
+        grpc_client,
+    );
+    db.sql("show tables;").await.unwrap();
+    let _ = fe_grpc_server.shutdown().await;
+    guard.remove_all().await;
+}
+
+pub async fn test_grpc_message_size_limit_send(store_type: StorageType) {
+    let config = GrpcServerConfig {
+        max_recv_message_size: 1024,
+        max_send_message_size: 50,
+    };
+    let (addr, mut guard, fe_grpc_server) =
+        setup_grpc_server_with(store_type, "auto_create_table", None, Some(config)).await;
+
+    let grpc_client = Client::with_urls(vec![addr]);
+    let db = Database::new_with_dbname(
+        format!("{}-{}", DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME),
+        grpc_client,
+    );
+    let err_msg = db.sql("show tables;").await.unwrap_err().to_string();
+    assert!(err_msg.contains("message length too large"), "{}", err_msg);
+    let _ = fe_grpc_server.shutdown().await;
+    guard.remove_all().await;
+}
+
+pub async fn test_grpc_message_size_limit_recv(store_type: StorageType) {
+    let config = GrpcServerConfig {
+        max_recv_message_size: 10,
+        max_send_message_size: 1024,
+    };
+    let (addr, mut guard, fe_grpc_server) =
+        setup_grpc_server_with(store_type, "auto_create_table", None, Some(config)).await;
+
+    let grpc_client = Client::with_urls(vec![addr]);
+    let db = Database::new_with_dbname(
+        format!("{}-{}", DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME),
+        grpc_client,
+    );
+    let err_msg = db.sql("show tables;").await.unwrap_err().to_string();
+    assert!(
+        err_msg.contains("Operation was attempted past the valid range"),
+        "{}",
+        err_msg
+    );
+    let _ = fe_grpc_server.shutdown().await;
+    guard.remove_all().await;
+}
+
 pub async fn test_grpc_auth(store_type: StorageType) {
     let user_provider = user_provider_from_option(
         &"static_user_provider:cmd:greptime_user=greptime_pwd".to_string(),
diff --git a/tests-integration/tests/http.rs b/tests-integration/tests/http.rs
index c186dab8ba7f..0317482da7f4 100644
--- a/tests-integration/tests/http.rs
+++ b/tests-integration/tests/http.rs
@@ -610,6 +610,8 @@ node_id = 0
 require_lease_before_startup = true
 rpc_addr = "127.0.0.1:3001"
 rpc_runtime_size = 8
+rpc_max_recv_message_size = 536870912
+rpc_max_send_message_size = 536870912
 enable_telemetry = true
 
 [heartbeat]

From f9315bdeabd7d956dc6018e3fd7ec1c5a5447694 Mon Sep 17 00:00:00 2001
From: JeremyHi <jiachun_feng@proton.me>
Date: Fri, 22 Sep 2023 10:27:29 +0800
Subject: [PATCH 04/36] chore: remove unused region_stats method form table
 (#2458)

chore: remove unused region_status method form table
---
 src/catalog/src/lib.rs | 67 +-----------------------------------------
 src/table/src/table.rs |  9 ------
 2 files changed, 1 insertion(+), 75 deletions(-)

diff --git a/src/catalog/src/lib.rs b/src/catalog/src/lib.rs
index 3dd45213ed7b..25e1f10d1753 100644
--- a/src/catalog/src/lib.rs
+++ b/src/catalog/src/lib.rs
@@ -20,10 +20,8 @@ use std::any::Any;
 use std::fmt::{Debug, Formatter};
 use std::sync::Arc;
 
-use api::v1::meta::RegionStat;
-use common_telemetry::warn;
 use futures::future::BoxFuture;
-use table::metadata::{TableId, TableType};
+use table::metadata::TableId;
 use table::requests::CreateTableRequest;
 use table::TableRef;
 
@@ -124,66 +122,3 @@ pub struct RegisterSchemaRequest {
     pub catalog: String,
     pub schema: String,
 }
-
-/// The stat of regions in the datanode node.
-/// The number of regions can be got from len of vec.
-///
-/// Ignores any errors occurred during iterating regions. The intention of this method is to
-/// collect region stats that will be carried in Datanode's heartbeat to Metasrv, so it's a
-/// "try our best" job.
-pub async fn datanode_stat(catalog_manager: &CatalogManagerRef) -> (u64, Vec<RegionStat>) {
-    let mut region_number: u64 = 0;
-    let mut region_stats = Vec::new();
-
-    let Ok(catalog_names) = catalog_manager.catalog_names().await else {
-        return (region_number, region_stats);
-    };
-    for catalog_name in catalog_names {
-        let Ok(schema_names) = catalog_manager.schema_names(&catalog_name).await else {
-            continue;
-        };
-        for schema_name in schema_names {
-            let Ok(table_names) = catalog_manager
-                .table_names(&catalog_name, &schema_name)
-                .await
-            else {
-                continue;
-            };
-            for table_name in table_names {
-                let Ok(Some(table)) = catalog_manager
-                    .table(&catalog_name, &schema_name, &table_name)
-                    .await
-                else {
-                    continue;
-                };
-
-                if table.table_type() != TableType::Base {
-                    continue;
-                }
-
-                let table_info = table.table_info();
-                let region_numbers = &table_info.meta.region_numbers;
-                region_number += region_numbers.len() as u64;
-
-                let engine = &table_info.meta.engine;
-
-                match table.region_stats() {
-                    Ok(stats) => {
-                        let stats = stats.into_iter().map(|stat| RegionStat {
-                            region_id: stat.region_id,
-                            approximate_bytes: stat.disk_usage_bytes as i64,
-                            engine: engine.clone(),
-                            ..Default::default()
-                        });
-
-                        region_stats.extend(stats);
-                    }
-                    Err(e) => {
-                        warn!("Failed to get region status, err: {:?}", e);
-                    }
-                };
-            }
-        }
-    }
-    (region_number, region_stats)
-}
diff --git a/src/table/src/table.rs b/src/table/src/table.rs
index 099481a9077b..8af3cfd1ffec 100644
--- a/src/table/src/table.rs
+++ b/src/table/src/table.rs
@@ -30,7 +30,6 @@ use crate::error::{Result, UnsupportedSnafu};
 use crate::metadata::{FilterPushDownType, TableId, TableInfoRef, TableType};
 use crate::requests::{AlterTableRequest, DeleteRequest, InsertRequest};
 use crate::stats::TableStatistics;
-use crate::RegionStat;
 
 pub type AlterContext = anymap::Map<dyn Any + Send + Sync>;
 
@@ -101,14 +100,6 @@ pub trait Table: Send + Sync {
         Ok(())
     }
 
-    /// Get region stats in this table.
-    fn region_stats(&self) -> Result<Vec<RegionStat>> {
-        UnsupportedSnafu {
-            operation: "REGION_STATS",
-        }
-        .fail()?
-    }
-
     /// Return true if contains the region
     fn contains_region(&self, _region: RegionNumber) -> Result<bool> {
         UnsupportedSnafu {

From 37156fc4ff3886632f67cba3c6dd1f18826c204e Mon Sep 17 00:00:00 2001
From: Baasit <baasisek01@gmail.com>
Date: Fri, 22 Sep 2023 03:34:57 +0100
Subject: [PATCH 05/36] feat: support for show full tables (#2410)

* feat: added show tables command

* fix(tests): fixed parser and statement unit tests

* chore: implemeted display trait for table type

* fix: handled no tabletype and error for usopprted command in show databse

* chore: removed full as a show kind, instead as a show option

* chore(tests): fixed failing test and added more tests for show full

* chore: refactored table types to use filters

* fix: changed table_type to tables
---
 src/datatypes/src/vectors/helper.rs | 40 +++++++++++++
 src/query/src/sql.rs                | 81 ++++++++++++++++++++++---
 src/sql/src/parsers/show_parser.rs  | 91 ++++++++++++++++++++++++++++-
 src/sql/src/statements/show.rs      |  1 +
 src/table/src/metadata.rs           | 10 ++++
 5 files changed, 211 insertions(+), 12 deletions(-)

diff --git a/src/datatypes/src/vectors/helper.rs b/src/datatypes/src/vectors/helper.rs
index 37e2cbb47be7..8d3128411259 100644
--- a/src/datatypes/src/vectors/helper.rs
+++ b/src/datatypes/src/vectors/helper.rs
@@ -350,6 +350,15 @@ impl Helper {
         let result = compute::filter(&array, &filter).context(error::ArrowComputeSnafu)?;
         Helper::try_into_vector(result)
     }
+
+    pub fn like_utf8_filter(names: Vec<String>, s: &str) -> Result<(VectorRef, BooleanVector)> {
+        let array = StringArray::from(names);
+        let filter = comparison::like_utf8_scalar(&array, s).context(error::ArrowComputeSnafu)?;
+        let result = compute::filter(&array, &filter).context(error::ArrowComputeSnafu)?;
+        let vector = Helper::try_into_vector(result)?;
+
+        Ok((vector, BooleanVector::from(filter)))
+    }
 }
 
 #[cfg(test)]
@@ -463,6 +472,37 @@ mod tests {
         assert_vector(vec!["greptime", "hello", "public", "world"], &ret);
     }
 
+    #[test]
+    fn test_like_utf8_filter() {
+        fn assert_vector(expected: Vec<&str>, actual: &VectorRef) {
+            let actual = actual.as_any().downcast_ref::<StringVector>().unwrap();
+            assert_eq!(*actual, StringVector::from(expected));
+        }
+
+        fn assert_filter(array: Vec<String>, s: &str, expected_filter: &BooleanVector) {
+            let array = StringArray::from(array);
+            let actual_filter = comparison::like_utf8_scalar(&array, s).unwrap();
+            assert_eq!(BooleanVector::from(actual_filter), *expected_filter);
+        }
+
+        let names: Vec<String> = vec!["greptime", "timeseries", "cloud", "database"]
+            .into_iter()
+            .map(|x| x.to_string())
+            .collect();
+
+        let (table, filter) = Helper::like_utf8_filter(names.clone(), "%ti%").unwrap();
+        assert_vector(vec!["greptime", "timeseries"], &table);
+        assert_filter(names.clone(), "%ti%", &filter);
+
+        let (tables, filter) = Helper::like_utf8_filter(names.clone(), "%lou").unwrap();
+        assert_vector(vec![], &tables);
+        assert_filter(names.clone(), "%lou", &filter);
+
+        let (tables, filter) = Helper::like_utf8_filter(names.clone(), "%d%").unwrap();
+        assert_vector(vec!["cloud", "database"], &tables);
+        assert_filter(names.clone(), "%d%", &filter);
+    }
+
     fn check_try_into_vector(array: impl Array + 'static) {
         let array: ArrayRef = Arc::new(array);
         let vector = Helper::try_into_vector(array.clone()).unwrap();
diff --git a/src/query/src/sql.rs b/src/query/src/sql.rs
index 8a99671a2246..4369d4205c3e 100644
--- a/src/query/src/sql.rs
+++ b/src/query/src/sql.rs
@@ -144,42 +144,86 @@ pub async fn show_tables(
     catalog_manager: CatalogManagerRef,
     query_ctx: QueryContextRef,
 ) -> Result<Output> {
-    let schema = if let Some(database) = stmt.database {
+    let schema_name = if let Some(database) = stmt.database {
         database
     } else {
         query_ctx.current_schema().to_owned()
     };
     // TODO(sunng87): move this function into query_ctx
     let mut tables = catalog_manager
-        .table_names(query_ctx.current_catalog(), &schema)
+        .table_names(query_ctx.current_catalog(), &schema_name)
         .await
         .context(error::CatalogSnafu)?;
 
     // TODO(dennis): Specify the order of the results in schema provider API
     tables.sort();
-    let schema = Arc::new(Schema::new(vec![ColumnSchema::new(
+
+    let table_types: Option<Arc<dyn Vector>> = {
+        if stmt.full {
+            Some(
+                get_table_types(
+                    &tables,
+                    catalog_manager.clone(),
+                    query_ctx.clone(),
+                    &schema_name,
+                )
+                .await?,
+            )
+        } else {
+            None
+        }
+    };
+
+    let mut column_schema = vec![ColumnSchema::new(
         TABLES_COLUMN,
         ConcreteDataType::string_datatype(),
         false,
-    )]));
+    )];
+    if table_types.is_some() {
+        column_schema.push(ColumnSchema::new(
+            "Table_type",
+            ConcreteDataType::string_datatype(),
+            false,
+        ));
+    }
+
+    let schema = Arc::new(Schema::new(column_schema));
+
     match stmt.kind {
         ShowKind::All => {
             let tables = Arc::new(StringVector::from(tables)) as _;
-            let records = RecordBatches::try_from_columns(schema, vec![tables])
+            let mut columns = vec![tables];
+            if let Some(table_types) = table_types {
+                columns.push(table_types)
+            }
+
+            let records = RecordBatches::try_from_columns(schema, columns)
                 .context(error::CreateRecordBatchSnafu)?;
             Ok(Output::RecordBatches(records))
         }
         ShowKind::Where(filter) => {
-            let columns = vec![Arc::new(StringVector::from(tables)) as _];
+            let mut columns = vec![Arc::new(StringVector::from(tables)) as _];
+            if let Some(table_types) = table_types {
+                columns.push(table_types)
+            }
             let record_batch =
                 RecordBatch::new(schema, columns).context(error::CreateRecordBatchSnafu)?;
             let result = execute_show_with_filter(record_batch, Some(filter)).await?;
             Ok(result)
         }
         ShowKind::Like(ident) => {
-            let tables =
-                Helper::like_utf8(tables, &ident.value).context(error::VectorComputationSnafu)?;
-            let records = RecordBatches::try_from_columns(schema, vec![tables])
+            let (tables, filter) = Helper::like_utf8_filter(tables, &ident.value)
+                .context(error::VectorComputationSnafu)?;
+            let mut columns = vec![tables];
+
+            if let Some(table_types) = table_types {
+                let table_types = table_types
+                    .filter(&filter)
+                    .context(error::VectorComputationSnafu)?;
+                columns.push(table_types)
+            }
+
+            let records = RecordBatches::try_from_columns(schema, columns)
                 .context(error::CreateRecordBatchSnafu)?;
             Ok(Output::RecordBatches(records))
         }
@@ -452,6 +496,25 @@ fn parse_file_table_format(options: &HashMap<String, String>) -> Result<Box<dyn
     )
 }
 
+async fn get_table_types(
+    tables: &[String],
+    catalog_manager: CatalogManagerRef,
+    query_ctx: QueryContextRef,
+    schema_name: &str,
+) -> Result<Arc<dyn Vector>> {
+    let mut table_types = Vec::with_capacity(tables.len());
+    for table_name in tables {
+        if let Some(table) = catalog_manager
+            .table(query_ctx.current_catalog(), schema_name, table_name)
+            .await
+            .context(error::CatalogSnafu)?
+        {
+            table_types.push(table.table_type().to_string());
+        }
+    }
+    Ok(Arc::new(StringVector::from(table_types)) as _)
+}
+
 #[cfg(test)]
 mod test {
     use std::sync::Arc;
diff --git a/src/sql/src/parsers/show_parser.rs b/src/sql/src/parsers/show_parser.rs
index 90baef8ff6fc..48a64e86ec5f 100644
--- a/src/sql/src/parsers/show_parser.rs
+++ b/src/sql/src/parsers/show_parser.rs
@@ -30,13 +30,19 @@ impl<'a> ParserContext<'a> {
             self.parse_show_databases()
         } else if self.matches_keyword(Keyword::TABLES) {
             let _ = self.parser.next_token();
-            self.parse_show_tables()
+            self.parse_show_tables(false)
         } else if self.consume_token("CREATE") {
             if self.consume_token("TABLE") {
                 self.parse_show_create_table()
             } else {
                 self.unsupported(self.peek_token_as_string())
             }
+        } else if self.consume_token("FULL") {
+            if self.consume_token("TABLES") {
+                self.parse_show_tables(true)
+            } else {
+                self.unsupported(self.peek_token_as_string())
+            }
         } else {
             self.unsupported(self.peek_token_as_string())
         }
@@ -61,12 +67,13 @@ impl<'a> ParserContext<'a> {
         Ok(Statement::ShowCreateTable(ShowCreateTable { table_name }))
     }
 
-    fn parse_show_tables(&mut self) -> Result<Statement> {
+    fn parse_show_tables(&mut self, full: bool) -> Result<Statement> {
         let database = match self.parser.peek_token().token {
             Token::EOF | Token::SemiColon => {
                 return Ok(Statement::ShowTables(ShowTables {
                     kind: ShowKind::All,
                     database: None,
+                    full,
                 }));
             }
 
@@ -126,7 +133,11 @@ impl<'a> ParserContext<'a> {
             _ => return self.unsupported(self.peek_token_as_string()),
         };
 
-        Ok(Statement::ShowTables(ShowTables { kind, database }))
+        Ok(Statement::ShowTables(ShowTables {
+            kind,
+            database,
+            full,
+        }))
     }
 
     /// Parses `SHOW DATABASES` statement.
@@ -234,6 +245,7 @@ mod tests {
             Statement::ShowTables(ShowTables {
                 kind: ShowKind::All,
                 database: None,
+                full: false
             })
         );
     }
@@ -253,6 +265,7 @@ mod tests {
                     quote_style: None,
                 }),
                 database: None,
+                full: false
             })
         );
 
@@ -269,6 +282,7 @@ mod tests {
                     quote_style: None,
                 }),
                 database: Some(_),
+                full: false
             })
         );
     }
@@ -285,6 +299,7 @@ mod tests {
             Statement::ShowTables(ShowTables {
                 kind: ShowKind::Where(sqlparser::ast::Expr::Like { .. }),
                 database: None,
+                full: false
             })
         );
 
@@ -298,6 +313,76 @@ mod tests {
             Statement::ShowTables(ShowTables {
                 kind: ShowKind::Where(sqlparser::ast::Expr::Like { .. }),
                 database: Some(_),
+                full: false
+            })
+        );
+    }
+
+    #[test]
+    pub fn test_show_full_tables() {
+        let sql = "SHOW FULL TABLES";
+        let stmts = ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}).unwrap();
+        assert_eq!(1, stmts.len());
+        assert_matches!(&stmts[0], Statement::ShowTables { .. });
+        match &stmts[0] {
+            Statement::ShowTables(show) => {
+                assert!(show.full);
+            }
+            _ => {
+                unreachable!();
+            }
+        }
+    }
+
+    #[test]
+    pub fn test_show_full_tables_where() {
+        let sql = "SHOW FULL TABLES IN test_db WHERE Tables LIKE test_table";
+        let stmts = ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}).unwrap();
+        assert_eq!(1, stmts.len());
+
+        assert_matches!(
+            &stmts[0],
+            Statement::ShowTables(ShowTables {
+                kind: ShowKind::Where(sqlparser::ast::Expr::Like { .. }),
+                database: Some(_),
+                full: true
+            })
+        );
+    }
+
+    #[test]
+    pub fn test_show_full_tables_like() {
+        let sql = "SHOW FULL TABLES LIKE test_table";
+        let result = ParserContext::create_with_dialect(sql, &GreptimeDbDialect {});
+        let stmts = result.unwrap();
+        assert_eq!(1, stmts.len());
+
+        assert_matches!(
+            &stmts[0],
+            Statement::ShowTables(ShowTables {
+                kind: ShowKind::Like(sqlparser::ast::Ident {
+                    value: _,
+                    quote_style: None,
+                }),
+                database: None,
+                full: true
+            })
+        );
+
+        let sql = "SHOW FULL TABLES in test_db LIKE test_table";
+        let result = ParserContext::create_with_dialect(sql, &GreptimeDbDialect {});
+        let stmts = result.unwrap();
+        assert_eq!(1, stmts.len());
+
+        assert_matches!(
+            &stmts[0],
+            Statement::ShowTables(ShowTables {
+                kind: ShowKind::Like(sqlparser::ast::Ident {
+                    value: _,
+                    quote_style: None,
+                }),
+                database: Some(_),
+                full: true
             })
         );
     }
diff --git a/src/sql/src/statements/show.rs b/src/sql/src/statements/show.rs
index eed546fba907..b0d712142863 100644
--- a/src/sql/src/statements/show.rs
+++ b/src/sql/src/statements/show.rs
@@ -54,6 +54,7 @@ impl ShowDatabases {
 pub struct ShowTables {
     pub kind: ShowKind,
     pub database: Option<String>,
+    pub full: bool,
 }
 
 /// SQL structure for `SHOW CREATE TABLE`.
diff --git a/src/table/src/metadata.rs b/src/table/src/metadata.rs
index f27d4d1e6145..628b7ce2c29c 100644
--- a/src/table/src/metadata.rs
+++ b/src/table/src/metadata.rs
@@ -80,6 +80,16 @@ pub enum TableType {
     Temporary,
 }
 
+impl std::fmt::Display for TableType {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            TableType::Base => f.write_str("BASE TABLE"),
+            TableType::Temporary => f.write_str("TEMPORARY"),
+            TableType::View => f.write_str("VIEW"),
+        }
+    }
+}
+
 /// Identifier of the table.
 #[derive(Serialize, Deserialize, Clone, Debug, Eq, PartialEq, Default)]
 pub struct TableIdent {

From 8a12762ff60e5bcf1c70f19e260cda299968f5a4 Mon Sep 17 00:00:00 2001
From: Yingwen <realevenyag@gmail.com>
Date: Fri, 22 Sep 2023 11:24:49 +0800
Subject: [PATCH 06/36] feat: update proto and remove create_if_not_exists
 (#2467)

---
 Cargo.lock                              |  2 +-
 Cargo.toml                              |  2 +-
 src/common/meta/src/ddl/create_table.rs |  1 -
 src/file-engine/src/engine.rs           | 15 +++------------
 src/file-engine/src/error.rs            |  7 -------
 src/file-engine/src/region.rs           |  3 ---
 src/meta-srv/src/procedure/tests.rs     |  1 -
 src/mito2/src/error.rs                  |  7 -------
 src/mito2/src/test_util.rs              |  1 -
 src/store-api/src/region_request.rs     |  4 ----
 10 files changed, 5 insertions(+), 38 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 3aa3264286ea..349938a47461 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -4186,7 +4186,7 @@ checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b"
 [[package]]
 name = "greptime-proto"
 version = "0.1.0"
-source = "git+https://github.com/GreptimeTeam/greptime-proto.git?rev=9d3f28d07d29607d0e3c1823f4a4d2bc229d05b9#9d3f28d07d29607d0e3c1823f4a4d2bc229d05b9"
+source = "git+https://github.com/GreptimeTeam/greptime-proto.git?rev=115c1080773be8a819e50b257fece9f839a0c836#115c1080773be8a819e50b257fece9f839a0c836"
 dependencies = [
  "prost",
  "serde",
diff --git a/Cargo.toml b/Cargo.toml
index 8513ed610edd..426b07d468b2 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -78,7 +78,7 @@ datafusion-substrait = { git = "https://github.com/waynexia/arrow-datafusion.git
 derive_builder = "0.12"
 futures = "0.3"
 futures-util = "0.3"
-greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "9d3f28d07d29607d0e3c1823f4a4d2bc229d05b9" }
+greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "115c1080773be8a819e50b257fece9f839a0c836" }
 humantime-serde = "1.1"
 itertools = "0.10"
 lazy_static = "1.4"
diff --git a/src/common/meta/src/ddl/create_table.rs b/src/common/meta/src/ddl/create_table.rs
index 3712ce6ccbbb..87a6cf0dc046 100644
--- a/src/common/meta/src/ddl/create_table.rs
+++ b/src/common/meta/src/ddl/create_table.rs
@@ -161,7 +161,6 @@ impl CreateTableProcedure {
             engine: create_table_expr.engine.to_string(),
             column_defs,
             primary_key,
-            create_if_not_exists: true,
             path: String::new(),
             options: create_table_expr.table_options.clone(),
         })
diff --git a/src/file-engine/src/engine.rs b/src/file-engine/src/engine.rs
index fe853701d204..6a9a97e264da 100644
--- a/src/file-engine/src/engine.rs
+++ b/src/file-engine/src/engine.rs
@@ -33,8 +33,7 @@ use tokio::sync::{Mutex, RwLock};
 
 use crate::config::EngineConfig;
 use crate::error::{
-    RegionExistsSnafu, RegionNotFoundSnafu, Result as EngineResult, UnexpectedEngineSnafu,
-    UnsupportedSnafu,
+    RegionNotFoundSnafu, Result as EngineResult, UnexpectedEngineSnafu, UnsupportedSnafu,
 };
 use crate::region::{FileRegion, FileRegionRef};
 
@@ -168,11 +167,7 @@ impl EngineInner {
         );
 
         if self.exists(region_id).await {
-            return if request.create_if_not_exists {
-                Ok(Output::AffectedRows(0))
-            } else {
-                RegionExistsSnafu { region_id }.fail()
-            };
+            return Ok(Output::AffectedRows(0));
         }
 
         info!("Try to create region, region_id: {}", region_id);
@@ -180,11 +175,7 @@ impl EngineInner {
         let _lock = self.region_mutex.lock().await;
         // Check again after acquiring the lock
         if self.exists(region_id).await {
-            return if request.create_if_not_exists {
-                Ok(Output::AffectedRows(0))
-            } else {
-                RegionExistsSnafu { region_id }.fail()
-            };
+            return Ok(Output::AffectedRows(0));
         }
 
         let res = FileRegion::create(region_id, request, &self.object_store).await;
diff --git a/src/file-engine/src/error.rs b/src/file-engine/src/error.rs
index 57e2d26b11b9..87cf49278315 100644
--- a/src/file-engine/src/error.rs
+++ b/src/file-engine/src/error.rs
@@ -40,12 +40,6 @@ pub enum Error {
         location: Location,
     },
 
-    #[snafu(display("Region {} already exists", region_id))]
-    RegionExists {
-        region_id: RegionId,
-        location: Location,
-    },
-
     #[snafu(display("Region not found, region_id: {}", region_id))]
     RegionNotFound {
         region_id: RegionId,
@@ -191,7 +185,6 @@ impl ErrorExt for Error {
             | CreateDefault { .. }
             | MissingColumnNoDefault { .. } => StatusCode::InvalidArguments,
 
-            RegionExists { .. } => StatusCode::RegionAlreadyExists,
             RegionNotFound { .. } => StatusCode::RegionNotFound,
 
             BuildBackend { source, .. } => source.status_code(),
diff --git a/src/file-engine/src/region.rs b/src/file-engine/src/region.rs
index 6340bbc81ed1..218a5e49d5ed 100644
--- a/src/file-engine/src/region.rs
+++ b/src/file-engine/src/region.rs
@@ -112,7 +112,6 @@ mod tests {
             engine: "file".to_string(),
             column_metadatas: new_test_column_metadata(),
             primary_key: vec![1],
-            create_if_not_exists: true,
             options: new_test_options(),
             region_dir: "create_region_dir/".to_string(),
         };
@@ -151,7 +150,6 @@ mod tests {
             engine: "file".to_string(),
             column_metadatas: new_test_column_metadata(),
             primary_key: vec![1],
-            create_if_not_exists: true,
             options: new_test_options(),
             region_dir: region_dir.clone(),
         };
@@ -189,7 +187,6 @@ mod tests {
             engine: "file".to_string(),
             column_metadatas: new_test_column_metadata(),
             primary_key: vec![1],
-            create_if_not_exists: true,
             options: new_test_options(),
             region_dir: region_dir.clone(),
         };
diff --git a/src/meta-srv/src/procedure/tests.rs b/src/meta-srv/src/procedure/tests.rs
index eac9dfd16b92..785d0f8edacb 100644
--- a/src/meta-srv/src/procedure/tests.rs
+++ b/src/meta-srv/src/procedure/tests.rs
@@ -151,7 +151,6 @@ fn test_create_region_request_template() {
             },
         ],
         primary_key: vec![2, 1],
-        create_if_not_exists: true,
         path: String::new(),
         options: HashMap::new(),
     };
diff --git a/src/mito2/src/error.rs b/src/mito2/src/error.rs
index 3502fe217f93..06aa47a9cd74 100644
--- a/src/mito2/src/error.rs
+++ b/src/mito2/src/error.rs
@@ -98,12 +98,6 @@ pub enum Error {
         location: Location,
     },
 
-    #[snafu(display("Region {} already exists", region_id))]
-    RegionExists {
-        region_id: RegionId,
-        location: Location,
-    },
-
     #[snafu(display("Failed to create RecordBatch from vectors"))]
     NewRecordBatch {
         location: Location,
@@ -405,7 +399,6 @@ impl ErrorExt for Error {
             | CreateDefault { .. }
             | InvalidParquet { .. } => StatusCode::Unexpected,
             RegionNotFound { .. } => StatusCode::RegionNotFound,
-            RegionExists { .. } => StatusCode::RegionAlreadyExists,
             InvalidScanIndex { .. }
             | InvalidMeta { .. }
             | InvalidRequest { .. }
diff --git a/src/mito2/src/test_util.rs b/src/mito2/src/test_util.rs
index 887466838e43..2ad545825de2 100644
--- a/src/mito2/src/test_util.rs
+++ b/src/mito2/src/test_util.rs
@@ -304,7 +304,6 @@ impl CreateRequestBuilder {
             column_metadatas,
             primary_key: self.primary_key.clone().unwrap_or(primary_key),
             options: self.options.clone(),
-            create_if_not_exists: false,
             region_dir: self.region_dir.clone(),
         }
     }
diff --git a/src/store-api/src/region_request.rs b/src/store-api/src/region_request.rs
index 693e1a55c5c4..9085943f21e0 100644
--- a/src/store-api/src/region_request.rs
+++ b/src/store-api/src/region_request.rs
@@ -79,7 +79,6 @@ impl RegionRequest {
                         engine: create.engine,
                         column_metadatas,
                         primary_key: create.primary_key,
-                        create_if_not_exists: create.create_if_not_exists,
                         options: create.options,
                         region_dir,
                     }),
@@ -150,9 +149,6 @@ pub struct RegionCreateRequest {
     pub column_metadatas: Vec<ColumnMetadata>,
     /// Columns in the primary key.
     pub primary_key: Vec<ColumnId>,
-    /// Create region if not exists.
-    // TODO(yingwen): Remove this.
-    pub create_if_not_exists: bool,
     /// Options of the created region.
     pub options: HashMap<String, String>,
     /// Directory for region's data home. Usually is composed by catalog and table id

From e2a8c0c296f5144b9c143c25dc9e934f96098d0a Mon Sep 17 00:00:00 2001
From: Yingwen <realevenyag@gmail.com>
Date: Fri, 22 Sep 2023 14:13:12 +0800
Subject: [PATCH 07/36] fix(mito): compaction scheduler schedules more tasks
 than expected (#2466)

* test: test on_compaction_finished

* fix: avoid submit same region to compact

* feat: persist and recover compaction time window

* test: fix test

* test: sort like result
---
 src/mito2/src/compaction.rs                   | 150 +++++++++++++++---
 src/mito2/src/compaction/twcs.rs              |   9 +-
 src/mito2/src/manifest/action.rs              |  13 +-
 src/mito2/src/manifest/tests/checkpoint.rs    |   2 +-
 src/mito2/src/region/opener.rs                |   1 +
 src/mito2/src/region/version.rs               |  16 ++
 src/mito2/src/request.rs                      |   3 +
 src/mito2/src/test_util/scheduler_util.rs     |   6 +
 src/mito2/src/test_util/version_util.rs       |  39 +++++
 src/mito2/src/worker/handle_compaction.rs     |   2 +-
 .../standalone/common/select/like.result      |   1 +
 tests/cases/standalone/common/select/like.sql |   1 +
 12 files changed, 219 insertions(+), 24 deletions(-)

diff --git a/src/mito2/src/compaction.rs b/src/mito2/src/compaction.rs
index 23ce3517e0ed..38327f10ca81 100644
--- a/src/mito2/src/compaction.rs
+++ b/src/mito2/src/compaction.rs
@@ -42,7 +42,6 @@ use crate::sst::file_purger::FilePurgerRef;
 pub struct CompactionRequest {
     pub(crate) current_version: VersionRef,
     pub(crate) access_layer: AccessLayerRef,
-    pub(crate) compaction_time_window: Option<i64>,
     /// Sender to send notification to the region worker.
     pub(crate) request_sender: mpsc::Sender<WorkerRequest>,
     /// Waiters of the compaction request.
@@ -101,24 +100,21 @@ impl CompactionScheduler {
         file_purger: &FilePurgerRef,
         waiter: OptionOutputTx,
     ) -> Result<()> {
-        let status = self.region_status.entry(region_id).or_insert_with(|| {
-            CompactionStatus::new(
-                region_id,
-                version_control.clone(),
-                access_layer.clone(),
-                file_purger.clone(),
-            )
-        });
-        if status.compacting {
+        if let Some(status) = self.region_status.get_mut(&region_id) {
             // Region is compacting. Add the waiter to pending list.
             status.merge_waiter(waiter);
             return Ok(());
         }
 
         // The region can compact directly.
+        let mut status = CompactionStatus::new(
+            region_id,
+            version_control.clone(),
+            access_layer.clone(),
+            file_purger.clone(),
+        );
         let request = status.new_compaction_request(self.request_sender.clone(), waiter);
-        // Mark the region as compacting.
-        status.compacting = true;
+        self.region_status.insert(region_id, status);
         self.schedule_compaction_request(request)
     }
 
@@ -127,7 +123,6 @@ impl CompactionScheduler {
         let Some(status) = self.region_status.get_mut(&region_id) else {
             return;
         };
-        status.compacting = false;
         // We should always try to compact the region until picker returns None.
         let request =
             status.new_compaction_request(self.request_sender.clone(), OptionOutputTx::none());
@@ -252,8 +247,6 @@ struct CompactionStatus {
     access_layer: AccessLayerRef,
     /// File purger of the region.
     file_purger: FilePurgerRef,
-    /// Whether a compaction task is running.
-    compacting: bool,
     /// Compaction pending to schedule.
     ///
     /// For simplicity, we merge all pending compaction requests into one.
@@ -273,7 +266,6 @@ impl CompactionStatus {
             version_control,
             access_layer,
             file_purger,
-            compacting: false,
             pending_compaction: None,
         }
     }
@@ -306,8 +298,6 @@ impl CompactionStatus {
         let mut req = CompactionRequest {
             current_version,
             access_layer: self.access_layer.clone(),
-            // TODO(hl): get persisted region compaction time window
-            compaction_time_window: None,
             request_sender: request_sender.clone(),
             waiters: Vec::new(),
             file_purger: self.file_purger.clone(),
@@ -324,12 +314,15 @@ impl CompactionStatus {
 
 #[cfg(test)]
 mod tests {
+    use std::sync::Mutex;
+
     use common_query::Output;
     use tokio::sync::oneshot;
 
     use super::*;
+    use crate::schedule::scheduler::{Job, Scheduler};
     use crate::test_util::scheduler_util::SchedulerEnv;
-    use crate::test_util::version_util::VersionControlBuilder;
+    use crate::test_util::version_util::{apply_edit, VersionControlBuilder};
 
     #[tokio::test]
     async fn test_schedule_empty() {
@@ -373,4 +366,123 @@ mod tests {
         assert!(matches!(output, Output::AffectedRows(0)));
         assert!(scheduler.region_status.is_empty());
     }
+
+    #[derive(Default)]
+    struct VecScheduler {
+        jobs: Mutex<Vec<Job>>,
+    }
+
+    impl VecScheduler {
+        fn num_jobs(&self) -> usize {
+            self.jobs.lock().unwrap().len()
+        }
+    }
+
+    #[async_trait::async_trait]
+    impl Scheduler for VecScheduler {
+        fn schedule(&self, job: Job) -> Result<()> {
+            self.jobs.lock().unwrap().push(job);
+            Ok(())
+        }
+
+        async fn stop(&self, _await_termination: bool) -> Result<()> {
+            Ok(())
+        }
+    }
+
+    #[tokio::test]
+    async fn test_schedule_on_finished() {
+        let job_scheduler = Arc::new(VecScheduler::default());
+        let env = SchedulerEnv::new().scheduler(job_scheduler.clone());
+        let (tx, _rx) = mpsc::channel(4);
+        let mut scheduler = env.mock_compaction_scheduler(tx);
+        let mut builder = VersionControlBuilder::new();
+        let purger = builder.file_purger();
+        let region_id = builder.region_id();
+
+        // 5 files to compact.
+        let end = 1000 * 1000;
+        let version_control = Arc::new(
+            builder
+                .push_l0_file(0, end)
+                .push_l0_file(10, end)
+                .push_l0_file(50, end)
+                .push_l0_file(80, end)
+                .push_l0_file(90, end)
+                .build(),
+        );
+        scheduler
+            .schedule_compaction(
+                region_id,
+                &version_control,
+                &env.access_layer,
+                &purger,
+                OptionOutputTx::none(),
+            )
+            .unwrap();
+        // Should schedule 1 compaction.
+        assert_eq!(1, scheduler.region_status.len());
+        assert_eq!(1, job_scheduler.num_jobs());
+        let data = version_control.current();
+        let file_metas: Vec<_> = data.version.ssts.levels()[0]
+            .files
+            .values()
+            .map(|file| file.meta())
+            .collect();
+
+        // 5 files for next compaction and removes old files.
+        apply_edit(
+            &version_control,
+            &[(0, end), (20, end), (40, end), (60, end), (80, end)],
+            &file_metas,
+            purger.clone(),
+        );
+        // The task is pending.
+        scheduler
+            .schedule_compaction(
+                region_id,
+                &version_control,
+                &env.access_layer,
+                &purger,
+                OptionOutputTx::none(),
+            )
+            .unwrap();
+        assert_eq!(1, scheduler.region_status.len());
+        assert_eq!(1, job_scheduler.num_jobs());
+        assert!(scheduler
+            .region_status
+            .get(&builder.region_id())
+            .unwrap()
+            .pending_compaction
+            .is_some());
+
+        // On compaction finished and schedule next compaction.
+        scheduler.on_compaction_finished(region_id);
+        assert_eq!(1, scheduler.region_status.len());
+        assert_eq!(2, job_scheduler.num_jobs());
+        // 5 files for next compaction.
+        apply_edit(
+            &version_control,
+            &[(0, end), (20, end), (40, end), (60, end), (80, end)],
+            &[],
+            purger.clone(),
+        );
+        // The task is pending.
+        scheduler
+            .schedule_compaction(
+                region_id,
+                &version_control,
+                &env.access_layer,
+                &purger,
+                OptionOutputTx::none(),
+            )
+            .unwrap();
+        assert_eq!(2, job_scheduler.num_jobs());
+        assert!(scheduler
+            .region_status
+            .get(&builder.region_id())
+            .unwrap()
+            .pending_compaction
+            .is_some());
+    }
 }
diff --git a/src/mito2/src/compaction/twcs.rs b/src/mito2/src/compaction/twcs.rs
index 5f03a3aa5fdb..f840d6aeb2f9 100644
--- a/src/mito2/src/compaction/twcs.rs
+++ b/src/mito2/src/compaction/twcs.rs
@@ -120,7 +120,6 @@ impl Picker for TwcsPicker {
         let CompactionRequest {
             current_version,
             access_layer,
-            compaction_time_window,
             request_sender,
             waiters,
             file_purger,
@@ -138,6 +137,9 @@ impl Picker for TwcsPicker {
             expired_ssts.iter().for_each(|f| f.set_compacting(true));
         }
 
+        let compaction_time_window = current_version
+            .compaction_time_window
+            .map(|window| window.as_secs() as i64);
         let time_window_size = compaction_time_window
             .or(self.time_window_seconds)
             .unwrap_or_else(|| {
@@ -169,7 +171,7 @@ impl Picker for TwcsPicker {
             outputs,
             expired_ssts,
             sst_write_buffer_size: ReadableSize::mb(4),
-            compaction_time_window: None,
+            compaction_time_window: Some(time_window_size),
             request_sender,
             waiters,
             file_purger,
@@ -357,6 +359,9 @@ impl CompactionTask for TwcsCompactionTask {
                     compacted_files: deleted,
                     senders: std::mem::take(&mut self.waiters),
                     file_purger: self.file_purger.clone(),
+                    compaction_time_window: self
+                        .compaction_time_window
+                        .map(|seconds| Duration::from_secs(seconds as u64)),
                 })
             }
             Err(e) => {
diff --git a/src/mito2/src/manifest/action.rs b/src/mito2/src/manifest/action.rs
index a39a89cc739d..aa490fad80fa 100644
--- a/src/mito2/src/manifest/action.rs
+++ b/src/mito2/src/manifest/action.rs
@@ -15,6 +15,7 @@
 //! Defines [RegionMetaAction] related structs and [RegionCheckpoint].
 
 use std::collections::HashMap;
+use std::time::Duration;
 
 use serde::{Deserialize, Serialize};
 use snafu::{OptionExt, ResultExt};
@@ -49,7 +50,8 @@ pub struct RegionChange {
 pub struct RegionEdit {
     pub files_to_add: Vec<FileMeta>,
     pub files_to_remove: Vec<FileMeta>,
-    pub compaction_time_window: Option<i64>,
+    #[serde(with = "humantime_serde")]
+    pub compaction_time_window: Option<Duration>,
     pub flushed_entry_id: Option<EntryId>,
     pub flushed_sequence: Option<SequenceNumber>,
 }
@@ -84,6 +86,9 @@ pub struct RegionManifest {
     pub manifest_version: ManifestVersion,
     /// Last WAL entry id of truncated data.
     pub truncated_entry_id: Option<EntryId>,
+    /// Inferred compaction time window.
+    #[serde(with = "humantime_serde")]
+    pub compaction_time_window: Option<Duration>,
 }
 
 #[derive(Debug, Default)]
@@ -94,6 +99,7 @@ pub struct RegionManifestBuilder {
     flushed_sequence: SequenceNumber,
     manifest_version: ManifestVersion,
     truncated_entry_id: Option<EntryId>,
+    compaction_time_window: Option<Duration>,
 }
 
 impl RegionManifestBuilder {
@@ -107,6 +113,7 @@ impl RegionManifestBuilder {
                 manifest_version: s.manifest_version,
                 flushed_sequence: s.flushed_sequence,
                 truncated_entry_id: s.truncated_entry_id,
+                compaction_time_window: s.compaction_time_window,
             }
         } else {
             Default::default()
@@ -132,6 +139,9 @@ impl RegionManifestBuilder {
         if let Some(flushed_sequence) = edit.flushed_sequence {
             self.flushed_sequence = self.flushed_sequence.max(flushed_sequence);
         }
+        if let Some(window) = edit.compaction_time_window {
+            self.compaction_time_window = Some(window);
+        }
     }
 
     pub fn apply_truncate(&mut self, manifest_version: ManifestVersion, truncate: RegionTruncate) {
@@ -156,6 +166,7 @@ impl RegionManifestBuilder {
             flushed_sequence: self.flushed_sequence,
             manifest_version: self.manifest_version,
             truncated_entry_id: self.truncated_entry_id,
+            compaction_time_window: self.compaction_time_window,
         })
     }
 }
diff --git a/src/mito2/src/manifest/tests/checkpoint.rs b/src/mito2/src/manifest/tests/checkpoint.rs
index e1b1c54cd82c..68c7063e1e63 100644
--- a/src/mito2/src/manifest/tests/checkpoint.rs
+++ b/src/mito2/src/manifest/tests/checkpoint.rs
@@ -150,7 +150,7 @@ async fn manager_with_checkpoint_distance_1() {
         .await
         .unwrap();
     let raw_json = std::str::from_utf8(&raw_bytes).unwrap();
-    let expected_json = "{\"size\":816,\"version\":9,\"checksum\":null,\"extend_metadata\":{}}";
+    let expected_json = "{\"size\":846,\"version\":9,\"checksum\":null,\"extend_metadata\":{}}";
     assert_eq!(expected_json, raw_json);
 
     // reopen the manager
diff --git a/src/mito2/src/region/opener.rs b/src/mito2/src/region/opener.rs
index fb6e5f89896b..211cecb75047 100644
--- a/src/mito2/src/region/opener.rs
+++ b/src/mito2/src/region/opener.rs
@@ -218,6 +218,7 @@ impl RegionOpener {
             .flushed_entry_id(manifest.flushed_entry_id)
             .flushed_sequence(manifest.flushed_sequence)
             .truncated_entry_id(manifest.truncated_entry_id)
+            .compaction_time_window(manifest.compaction_time_window)
             .options(options)
             .build();
         let flushed_entry_id = version.flushed_entry_id;
diff --git a/src/mito2/src/region/version.rs b/src/mito2/src/region/version.rs
index e88fafedaf08..c7d84fd913df 100644
--- a/src/mito2/src/region/version.rs
+++ b/src/mito2/src/region/version.rs
@@ -24,6 +24,7 @@
 //! and became invisible between step 1 and 2, so need to acquire version at first.
 
 use std::sync::{Arc, RwLock};
+use std::time::Duration;
 
 use store_api::metadata::RegionMetadataRef;
 use store_api::storage::SequenceNumber;
@@ -205,6 +206,8 @@ pub(crate) struct Version {
     ///
     /// Used to check if it is a flush task during the truncating table.
     pub(crate) truncated_entry_id: Option<EntryId>,
+    /// Inferred compaction time window.
+    pub(crate) compaction_time_window: Option<Duration>,
     /// Options of the region.
     pub(crate) options: RegionOptions,
 }
@@ -219,6 +222,7 @@ pub(crate) struct VersionBuilder {
     flushed_entry_id: EntryId,
     flushed_sequence: SequenceNumber,
     truncated_entry_id: Option<EntryId>,
+    compaction_time_window: Option<Duration>,
     options: RegionOptions,
 }
 
@@ -232,6 +236,7 @@ impl VersionBuilder {
             flushed_entry_id: 0,
             flushed_sequence: 0,
             truncated_entry_id: None,
+            compaction_time_window: None,
             options: RegionOptions::default(),
         }
     }
@@ -245,6 +250,7 @@ impl VersionBuilder {
             flushed_entry_id: version.flushed_entry_id,
             flushed_sequence: version.flushed_sequence,
             truncated_entry_id: version.truncated_entry_id,
+            compaction_time_window: version.compaction_time_window,
             options: version.options.clone(),
         }
     }
@@ -279,6 +285,12 @@ impl VersionBuilder {
         self
     }
 
+    /// Sets compaction time window.
+    pub(crate) fn compaction_time_window(mut self, window: Option<Duration>) -> Self {
+        self.compaction_time_window = window;
+        self
+    }
+
     /// Sets options.
     pub(crate) fn options(mut self, options: RegionOptions) -> Self {
         self.options = options;
@@ -293,6 +305,9 @@ impl VersionBuilder {
         if let Some(sequence) = edit.flushed_sequence {
             self.flushed_sequence = self.flushed_sequence.max(sequence);
         }
+        if let Some(window) = edit.compaction_time_window {
+            self.compaction_time_window = Some(window);
+        }
         if !edit.files_to_add.is_empty() || !edit.files_to_remove.is_empty() {
             let mut ssts = (*self.ssts).clone();
             ssts.add_files(file_purger, edit.files_to_add.into_iter());
@@ -335,6 +350,7 @@ impl VersionBuilder {
             flushed_entry_id: self.flushed_entry_id,
             flushed_sequence: self.flushed_sequence,
             truncated_entry_id: self.truncated_entry_id,
+            compaction_time_window: self.compaction_time_window,
             options: self.options,
         }
     }
diff --git a/src/mito2/src/request.rs b/src/mito2/src/request.rs
index 1fbcbc962338..33cc121a5a3e 100644
--- a/src/mito2/src/request.rs
+++ b/src/mito2/src/request.rs
@@ -16,6 +16,7 @@
 
 use std::collections::HashMap;
 use std::sync::Arc;
+use std::time::Duration;
 
 use api::helper::{
     is_column_type_value_eq, is_semantic_type_eq, proto_value_type, to_column_data_type,
@@ -640,6 +641,8 @@ pub(crate) struct CompactionFinished {
     pub(crate) senders: Vec<OutputTx>,
     /// File purger for cleaning files on failure.
     pub(crate) file_purger: FilePurgerRef,
+    /// Inferred Compaction time window.
+    pub(crate) compaction_time_window: Option<Duration>,
 }
 
 impl CompactionFinished {
diff --git a/src/mito2/src/test_util/scheduler_util.rs b/src/mito2/src/test_util/scheduler_util.rs
index 5e1e259e75d9..611371a7ec0b 100644
--- a/src/mito2/src/test_util/scheduler_util.rs
+++ b/src/mito2/src/test_util/scheduler_util.rs
@@ -52,6 +52,12 @@ impl SchedulerEnv {
         }
     }
 
+    /// Set scheduler.
+    pub(crate) fn scheduler(mut self, scheduler: SchedulerRef) -> Self {
+        self.scheduler = Some(scheduler);
+        self
+    }
+
     /// Creates a new compaction scheduler.
     pub(crate) fn mock_compaction_scheduler(
         &self,
diff --git a/src/mito2/src/test_util/version_util.rs b/src/mito2/src/test_util/version_util.rs
index 2fee8a987b06..e480b1f146df 100644
--- a/src/mito2/src/test_util/version_util.rs
+++ b/src/mito2/src/test_util/version_util.rs
@@ -24,6 +24,7 @@ use datatypes::schema::ColumnSchema;
 use store_api::metadata::{ColumnMetadata, RegionMetadata, RegionMetadataBuilder};
 use store_api::storage::RegionId;
 
+use crate::manifest::action::RegionEdit;
 use crate::memtable::{MemtableBuilder, MemtableBuilderRef};
 use crate::region::version::{Version, VersionBuilder, VersionControl};
 use crate::sst::file::{FileId, FileMeta};
@@ -113,3 +114,41 @@ impl VersionControlBuilder {
         VersionControl::new(version)
     }
 }
+
+/// Add mocked l0 files to the version control.
+/// `files_to_add` are slice of `(start_ms, end_ms)`.
+pub(crate) fn apply_edit(
+    version_control: &VersionControl,
+    files_to_add: &[(i64, i64)],
+    files_to_remove: &[FileMeta],
+    purger: FilePurgerRef,
+) {
+    let region_id = version_control.current().version.metadata.region_id;
+    let files_to_add = files_to_add
+        .iter()
+        .map(|(start_ms, end_ms)| {
+            FileMeta {
+                region_id,
+                file_id: FileId::random(),
+                time_range: (
+                    Timestamp::new_millisecond(*start_ms),
+                    Timestamp::new_millisecond(*end_ms),
+                ),
+                level: 0,
+                file_size: 0, // We don't care file size.
+            }
+        })
+        .collect();
+
+    version_control.apply_edit(
+        RegionEdit {
+            files_to_add,
+            files_to_remove: files_to_remove.to_vec(),
+            compaction_time_window: None,
+            flushed_entry_id: None,
+            flushed_sequence: None,
+        },
+        &[],
+        purger,
+    );
+}
diff --git a/src/mito2/src/worker/handle_compaction.rs b/src/mito2/src/worker/handle_compaction.rs
index 0e0f3a07ef34..79ade0f4dc9f 100644
--- a/src/mito2/src/worker/handle_compaction.rs
+++ b/src/mito2/src/worker/handle_compaction.rs
@@ -61,7 +61,7 @@ impl<S: LogStore> RegionWorkerLoop<S> {
         let edit = RegionEdit {
             files_to_add: std::mem::take(&mut request.compaction_outputs),
             files_to_remove: std::mem::take(&mut request.compacted_files),
-            compaction_time_window: None, // TODO(hl): update window maybe
+            compaction_time_window: request.compaction_time_window,
             flushed_entry_id: None,
             flushed_sequence: None,
         };
diff --git a/tests/cases/standalone/common/select/like.result b/tests/cases/standalone/common/select/like.result
index 91b648e1a83f..b47528b80e23 100644
--- a/tests/cases/standalone/common/select/like.result
+++ b/tests/cases/standalone/common/select/like.result
@@ -14,6 +14,7 @@ INSERT INTO TABLE host VALUES
 
 Affected Rows: 4
 
+-- SQLNESS SORT_RESULT 3 1
 SELECT * FROM host WHERE host LIKE '%+%';
 
 +-------------------------+------+-----+
diff --git a/tests/cases/standalone/common/select/like.sql b/tests/cases/standalone/common/select/like.sql
index b4ef76bb93b7..281cef769a36 100644
--- a/tests/cases/standalone/common/select/like.sql
+++ b/tests/cases/standalone/common/select/like.sql
@@ -10,6 +10,7 @@ INSERT INTO TABLE host VALUES
     (2, 'a', 3.0),
     (3, 'c', 4.0);
 
+-- SQLNESS SORT_RESULT 3 1
 SELECT * FROM host WHERE host LIKE '%+%';
 
 DROP TABLE host;

From 5e1322e1f44df2d6a2ca7ba3c32932a1f15fc82a Mon Sep 17 00:00:00 2001
From: Wei <47681251+QuenKar@users.noreply.github.com>
Date: Fri, 22 Sep 2023 14:28:02 +0800
Subject: [PATCH 08/36] refactor: not allowed int64 type as time index (#2460)

* refactor: remove is_timestamp_compatible.

* chore: fmt

* refactor: remove int64 to timestamp match

* chore

* chore: apply suggestions from code review

Co-authored-by: dennis zhuang <killme2008@gmail.com>

* chore: fmt

---------

Co-authored-by: dennis zhuang <killme2008@gmail.com>
---
 src/datatypes/src/data_type.rs             | 46 ++--------------------
 src/datatypes/src/schema.rs                |  3 +-
 src/datatypes/src/schema/constraint.rs     | 15 ++-----
 src/datatypes/src/types/binary_type.rs     |  4 --
 src/datatypes/src/types/boolean_type.rs    |  4 --
 src/datatypes/src/types/date_type.rs       |  4 --
 src/datatypes/src/types/datetime_type.rs   |  4 --
 src/datatypes/src/types/dictionary_type.rs |  4 --
 src/datatypes/src/types/duration_type.rs   |  3 --
 src/datatypes/src/types/interval_type.rs   |  3 --
 src/datatypes/src/types/list_type.rs       |  4 --
 src/datatypes/src/types/null_type.rs       |  4 --
 src/datatypes/src/types/primitive_type.rs  | 11 ------
 src/datatypes/src/types/string_type.rs     |  4 --
 src/datatypes/src/types/time_type.rs       |  4 --
 src/datatypes/src/types/timestamp_type.rs  |  4 --
 src/datatypes/src/value.rs                 |  2 -
 src/mito2/src/read.rs                      |  8 ++--
 src/storage/src/sst/parquet.rs             |  3 +-
 src/storage/src/sst/pruning.rs             |  9 +----
 src/store-api/src/metadata.rs              | 11 ++----
 21 files changed, 19 insertions(+), 135 deletions(-)

diff --git a/src/datatypes/src/data_type.rs b/src/datatypes/src/data_type.rs
index 541bc6b8b3b2..938c8ba498dc 100644
--- a/src/datatypes/src/data_type.rs
+++ b/src/datatypes/src/data_type.rs
@@ -179,6 +179,10 @@ impl ConcreteDataType {
         )
     }
 
+    pub fn is_timestamp(&self) -> bool {
+        matches!(self, ConcreteDataType::Timestamp(_))
+    }
+
     pub fn numerics() -> Vec<ConcreteDataType> {
         vec![
             ConcreteDataType::int8_datatype(),
@@ -217,9 +221,6 @@ impl ConcreteDataType {
     /// Try to cast data type as a [`TimestampType`].
     pub fn as_timestamp(&self) -> Option<TimestampType> {
         match self {
-            ConcreteDataType::Int64(_) => {
-                Some(TimestampType::Millisecond(TimestampMillisecondType))
-            }
             ConcreteDataType::Timestamp(t) => Some(*t),
             _ => None,
         }
@@ -473,10 +474,6 @@ pub trait DataType: std::fmt::Debug + Send + Sync {
     /// Creates a mutable vector with given `capacity` of this type.
     fn create_mutable_vector(&self, capacity: usize) -> Box<dyn MutableVector>;
 
-    /// Returns true if the data type is compatible with timestamp type so we can
-    /// use it as a timestamp.
-    fn is_timestamp_compatible(&self) -> bool;
-
     /// Casts the value to specific DataType.
     /// Return None if cast failed.
     fn try_cast(&self, from: Value) -> Option<Value>;
@@ -596,41 +593,6 @@ mod tests {
         );
     }
 
-    #[test]
-    fn test_is_timestamp_compatible() {
-        assert!(ConcreteDataType::timestamp_datatype(TimeUnit::Second).is_timestamp_compatible());
-        assert!(
-            ConcreteDataType::timestamp_datatype(TimeUnit::Millisecond).is_timestamp_compatible()
-        );
-        assert!(
-            ConcreteDataType::timestamp_datatype(TimeUnit::Microsecond).is_timestamp_compatible()
-        );
-        assert!(
-            ConcreteDataType::timestamp_datatype(TimeUnit::Nanosecond).is_timestamp_compatible()
-        );
-        assert!(ConcreteDataType::timestamp_second_datatype().is_timestamp_compatible());
-        assert!(ConcreteDataType::timestamp_millisecond_datatype().is_timestamp_compatible());
-        assert!(ConcreteDataType::timestamp_microsecond_datatype().is_timestamp_compatible());
-        assert!(ConcreteDataType::timestamp_nanosecond_datatype().is_timestamp_compatible());
-        assert!(ConcreteDataType::int64_datatype().is_timestamp_compatible());
-        assert!(!ConcreteDataType::null_datatype().is_timestamp_compatible());
-        assert!(!ConcreteDataType::binary_datatype().is_timestamp_compatible());
-        assert!(!ConcreteDataType::boolean_datatype().is_timestamp_compatible());
-        assert!(!ConcreteDataType::date_datatype().is_timestamp_compatible());
-        assert!(!ConcreteDataType::datetime_datatype().is_timestamp_compatible());
-        assert!(!ConcreteDataType::string_datatype().is_timestamp_compatible());
-        assert!(!ConcreteDataType::int32_datatype().is_timestamp_compatible());
-        assert!(!ConcreteDataType::uint64_datatype().is_timestamp_compatible());
-        assert!(!ConcreteDataType::time_second_datatype().is_timestamp_compatible());
-        assert!(!ConcreteDataType::time_millisecond_datatype().is_timestamp_compatible());
-        assert!(!ConcreteDataType::time_microsecond_datatype().is_timestamp_compatible());
-        assert!(!ConcreteDataType::time_nanosecond_datatype().is_timestamp_compatible());
-        assert!(!ConcreteDataType::duration_second_datatype().is_timestamp_compatible());
-        assert!(!ConcreteDataType::duration_millisecond_datatype().is_timestamp_compatible());
-        assert!(!ConcreteDataType::duration_microsecond_datatype().is_timestamp_compatible());
-        assert!(!ConcreteDataType::duration_nanosecond_datatype().is_timestamp_compatible());
-    }
-
     #[test]
     fn test_is_null() {
         assert!(ConcreteDataType::null_datatype().is_null());
diff --git a/src/datatypes/src/schema.rs b/src/datatypes/src/schema.rs
index fd6da64beb29..b506dd12ea00 100644
--- a/src/datatypes/src/schema.rs
+++ b/src/datatypes/src/schema.rs
@@ -23,7 +23,6 @@ use arrow::datatypes::{Field, Schema as ArrowSchema};
 use datafusion_common::DFSchemaRef;
 use snafu::{ensure, ResultExt};
 
-use crate::data_type::DataType;
 use crate::error::{self, DuplicateColumnSnafu, Error, ProjectArrowSchemaSnafu, Result};
 pub use crate::schema::column_schema::{ColumnSchema, Metadata, COMMENT_KEY, TIME_INDEX_KEY};
 pub use crate::schema::constraint::ColumnDefaultConstraint;
@@ -269,7 +268,7 @@ fn validate_timestamp_index(column_schemas: &[ColumnSchema], timestamp_index: us
 
     let column_schema = &column_schemas[timestamp_index];
     ensure!(
-        column_schema.data_type.is_timestamp_compatible(),
+        column_schema.data_type.is_timestamp(),
         error::InvalidTimestampIndexSnafu {
             index: timestamp_index,
         }
diff --git a/src/datatypes/src/schema/constraint.rs b/src/datatypes/src/schema/constraint.rs
index b00a8d7dcaf7..e0dacbd5f719 100644
--- a/src/datatypes/src/schema/constraint.rs
+++ b/src/datatypes/src/schema/constraint.rs
@@ -81,7 +81,7 @@ impl ColumnDefaultConstraint {
                     error::UnsupportedDefaultExprSnafu { expr }
                 );
                 ensure!(
-                    data_type.is_timestamp_compatible(),
+                    data_type.is_timestamp(),
                     error::DefaultValueTypeSnafu {
                         reason: "return value of the function must has timestamp type",
                     }
@@ -199,7 +199,7 @@ fn create_current_timestamp_vector(
     let current_timestamp_vector = TimestampMillisecondVector::from_values(
         std::iter::repeat(util::current_time_millis()).take(num_rows),
     );
-    if data_type.is_timestamp_compatible() {
+    if data_type.is_timestamp() {
         current_timestamp_vector.cast(data_type)
     } else {
         error::DefaultValueTypeSnafu {
@@ -350,15 +350,8 @@ mod tests {
 
         // Int64 type.
         let data_type = ConcreteDataType::int64_datatype();
-        let v = constraint
-            .create_default_vector(&data_type, false, 4)
-            .unwrap();
-        assert_eq!(4, v.len());
-        assert!(
-            matches!(v.get(0), Value::Int64(_)),
-            "v {:?} is not timestamp",
-            v.get(0)
-        );
+        let v = constraint.create_default_vector(&data_type, false, 4);
+        assert!(v.is_err());
 
         let constraint = ColumnDefaultConstraint::Function("no".to_string());
         let data_type = ConcreteDataType::timestamp_millisecond_datatype();
diff --git a/src/datatypes/src/types/binary_type.rs b/src/datatypes/src/types/binary_type.rs
index c9e8d7f12b6e..6f4c1f6bc0b5 100644
--- a/src/datatypes/src/types/binary_type.rs
+++ b/src/datatypes/src/types/binary_type.rs
@@ -54,10 +54,6 @@ impl DataType for BinaryType {
         Box::new(BinaryVectorBuilder::with_capacity(capacity))
     }
 
-    fn is_timestamp_compatible(&self) -> bool {
-        false
-    }
-
     fn try_cast(&self, from: Value) -> Option<Value> {
         match from {
             Value::Binary(v) => Some(Value::Binary(v)),
diff --git a/src/datatypes/src/types/boolean_type.rs b/src/datatypes/src/types/boolean_type.rs
index df33d3862ce2..1d4b9e80a2b9 100644
--- a/src/datatypes/src/types/boolean_type.rs
+++ b/src/datatypes/src/types/boolean_type.rs
@@ -54,10 +54,6 @@ impl DataType for BooleanType {
         Box::new(BooleanVectorBuilder::with_capacity(capacity))
     }
 
-    fn is_timestamp_compatible(&self) -> bool {
-        false
-    }
-
     fn try_cast(&self, from: Value) -> Option<Value> {
         match from {
             Value::Boolean(v) => Some(Value::Boolean(v)),
diff --git a/src/datatypes/src/types/date_type.rs b/src/datatypes/src/types/date_type.rs
index 1bc243da3a60..8bbde3a7c7f0 100644
--- a/src/datatypes/src/types/date_type.rs
+++ b/src/datatypes/src/types/date_type.rs
@@ -52,10 +52,6 @@ impl DataType for DateType {
         Box::new(DateVectorBuilder::with_capacity(capacity))
     }
 
-    fn is_timestamp_compatible(&self) -> bool {
-        false
-    }
-
     fn try_cast(&self, from: Value) -> Option<Value> {
         match from {
             Value::Int32(v) => Some(Value::Date(Date::from(v))),
diff --git a/src/datatypes/src/types/datetime_type.rs b/src/datatypes/src/types/datetime_type.rs
index 76b432e82584..cd0e5a3cd1bf 100644
--- a/src/datatypes/src/types/datetime_type.rs
+++ b/src/datatypes/src/types/datetime_type.rs
@@ -50,10 +50,6 @@ impl DataType for DateTimeType {
         Box::new(DateTimeVectorBuilder::with_capacity(capacity))
     }
 
-    fn is_timestamp_compatible(&self) -> bool {
-        false
-    }
-
     fn try_cast(&self, from: Value) -> Option<Value> {
         match from {
             Value::Int64(v) => Some(Value::DateTime(DateTime::from(v))),
diff --git a/src/datatypes/src/types/dictionary_type.rs b/src/datatypes/src/types/dictionary_type.rs
index fdbdd85ac1c0..cc29c41403df 100644
--- a/src/datatypes/src/types/dictionary_type.rs
+++ b/src/datatypes/src/types/dictionary_type.rs
@@ -85,10 +85,6 @@ impl DataType for DictionaryType {
         unimplemented!()
     }
 
-    fn is_timestamp_compatible(&self) -> bool {
-        false
-    }
-
     fn try_cast(&self, _: Value) -> Option<Value> {
         None
     }
diff --git a/src/datatypes/src/types/duration_type.rs b/src/datatypes/src/types/duration_type.rs
index 94c80a7962b3..ffc8fe92467b 100644
--- a/src/datatypes/src/types/duration_type.rs
+++ b/src/datatypes/src/types/duration_type.rs
@@ -98,9 +98,6 @@ macro_rules! impl_data_type_for_duration {
                     Box::new([<Duration $unit Vector Builder>]::with_capacity(capacity))
                 }
 
-                fn is_timestamp_compatible(&self) -> bool {
-                    false
-                }
 
                 fn try_cast(&self, _: Value) -> Option<Value> {
                     // TODO(QuenKar): Implement casting for duration types.
diff --git a/src/datatypes/src/types/interval_type.rs b/src/datatypes/src/types/interval_type.rs
index b87df2733717..1acc506cfce0 100644
--- a/src/datatypes/src/types/interval_type.rs
+++ b/src/datatypes/src/types/interval_type.rs
@@ -86,9 +86,6 @@ macro_rules! impl_data_type_for_interval {
                     Box::new([<Interval $unit Vector Builder>]::with_capacity(capacity))
                 }
 
-                fn is_timestamp_compatible(&self) -> bool {
-                    false
-                }
 
                 fn try_cast(&self, _: Value) -> Option<Value> {
                     // TODO(QuenKar): Implement casting for interval types.
diff --git a/src/datatypes/src/types/list_type.rs b/src/datatypes/src/types/list_type.rs
index 4b4769ed3862..37d620620297 100644
--- a/src/datatypes/src/types/list_type.rs
+++ b/src/datatypes/src/types/list_type.rs
@@ -76,10 +76,6 @@ impl DataType for ListType {
         ))
     }
 
-    fn is_timestamp_compatible(&self) -> bool {
-        false
-    }
-
     fn try_cast(&self, from: Value) -> Option<Value> {
         match from {
             Value::List(v) => Some(Value::List(v)),
diff --git a/src/datatypes/src/types/null_type.rs b/src/datatypes/src/types/null_type.rs
index e69cdae24985..04c44c38c573 100644
--- a/src/datatypes/src/types/null_type.rs
+++ b/src/datatypes/src/types/null_type.rs
@@ -52,10 +52,6 @@ impl DataType for NullType {
         Box::<NullVectorBuilder>::default()
     }
 
-    fn is_timestamp_compatible(&self) -> bool {
-        false
-    }
-
     // Unconditional cast other type to Value::Null
     fn try_cast(&self, _from: Value) -> Option<Value> {
         Some(Value::Null)
diff --git a/src/datatypes/src/types/primitive_type.rs b/src/datatypes/src/types/primitive_type.rs
index 7bf90c964a3c..52e1bd30a7c6 100644
--- a/src/datatypes/src/types/primitive_type.rs
+++ b/src/datatypes/src/types/primitive_type.rs
@@ -271,9 +271,6 @@ macro_rules! define_non_timestamp_primitive {
                 Box::new(PrimitiveVectorBuilder::<$DataType>::with_capacity(capacity))
             }
 
-            fn is_timestamp_compatible(&self) -> bool {
-                false
-            }
 
             fn try_cast(&self, from: Value) -> Option<Value> {
                 match from {
@@ -373,10 +370,6 @@ impl DataType for Int64Type {
         Box::new(PrimitiveVectorBuilder::<Int64Type>::with_capacity(capacity))
     }
 
-    fn is_timestamp_compatible(&self) -> bool {
-        true
-    }
-
     fn try_cast(&self, from: Value) -> Option<Value> {
         match from {
             Value::Boolean(v) => bool_to_numeric(v).map(Value::Int64),
@@ -424,10 +417,6 @@ impl DataType for Int32Type {
         Box::new(PrimitiveVectorBuilder::<Int32Type>::with_capacity(capacity))
     }
 
-    fn is_timestamp_compatible(&self) -> bool {
-        false
-    }
-
     fn try_cast(&self, from: Value) -> Option<Value> {
         match from {
             Value::Boolean(v) => bool_to_numeric(v).map(Value::Int32),
diff --git a/src/datatypes/src/types/string_type.rs b/src/datatypes/src/types/string_type.rs
index 85a970f116d6..febff36324a1 100644
--- a/src/datatypes/src/types/string_type.rs
+++ b/src/datatypes/src/types/string_type.rs
@@ -54,10 +54,6 @@ impl DataType for StringType {
         Box::new(StringVectorBuilder::with_capacity(capacity))
     }
 
-    fn is_timestamp_compatible(&self) -> bool {
-        false
-    }
-
     fn try_cast(&self, from: Value) -> Option<Value> {
         if from.logical_type_id() == self.logical_type_id() {
             return Some(from);
diff --git a/src/datatypes/src/types/time_type.rs b/src/datatypes/src/types/time_type.rs
index aaa9fec914e7..a8d48a7f586d 100644
--- a/src/datatypes/src/types/time_type.rs
+++ b/src/datatypes/src/types/time_type.rs
@@ -112,10 +112,6 @@ macro_rules! impl_data_type_for_time {
                     Box::new([<Time $unit Vector Builder>]::with_capacity(capacity))
                 }
 
-                fn is_timestamp_compatible(&self) -> bool {
-                    false
-                }
-
                 fn try_cast(&self, from: Value) -> Option<Value> {
                     match from {
                         Value::$TargetType(v) => Some(Value::Time(Time::new(v as i64, TimeUnit::$unit))),
diff --git a/src/datatypes/src/types/timestamp_type.rs b/src/datatypes/src/types/timestamp_type.rs
index 568f23d5b281..eca0d0986fc5 100644
--- a/src/datatypes/src/types/timestamp_type.rs
+++ b/src/datatypes/src/types/timestamp_type.rs
@@ -129,10 +129,6 @@ macro_rules! impl_data_type_for_timestamp {
                     Box::new([<Timestamp $unit Vector Builder>]::with_capacity(capacity))
                 }
 
-                fn is_timestamp_compatible(&self) -> bool {
-                    true
-                }
-
                 fn try_cast(&self, from: Value)-> Option<Value>{
                     match from {
                         Value::Timestamp(v) => v.convert_to(TimeUnit::$unit).map(Value::Timestamp),
diff --git a/src/datatypes/src/value.rs b/src/datatypes/src/value.rs
index c56832cb1d5d..45e8f1ce02b5 100644
--- a/src/datatypes/src/value.rs
+++ b/src/datatypes/src/value.rs
@@ -198,7 +198,6 @@ impl Value {
     /// Cast Value to timestamp. Return None if value is not a valid timestamp data type.
     pub fn as_timestamp(&self) -> Option<Timestamp> {
         match self {
-            Value::Int64(v) => Some(Timestamp::new_millisecond(*v)),
             Value::Timestamp(t) => Some(*t),
             _ => None,
         }
@@ -388,7 +387,6 @@ pub fn duration_to_scalar_value(unit: TimeUnit, val: Option<i64>) -> ScalarValue
 /// Return `None` if given scalar value cannot be converted to a valid timestamp.
 pub fn scalar_value_to_timestamp(scalar: &ScalarValue) -> Option<Timestamp> {
     match scalar {
-        ScalarValue::Int64(val) => val.map(Timestamp::new_millisecond),
         ScalarValue::Utf8(Some(s)) => match Timestamp::from_str(s) {
             Ok(t) => Some(t),
             Err(e) => {
diff --git a/src/mito2/src/read.rs b/src/mito2/src/read.rs
index d15a197e5b63..c16639415e12 100644
--- a/src/mito2/src/read.rs
+++ b/src/mito2/src/read.rs
@@ -382,9 +382,7 @@ impl Batch {
     fn get_timestamp(&self, index: usize) -> Timestamp {
         match self.timestamps.get_ref(index) {
             ValueRef::Timestamp(timestamp) => timestamp,
-            // Int64 is always millisecond.
-            // TODO(yingwen): Don't allow using int64 as time index.
-            ValueRef::Int64(v) => Timestamp::new_millisecond(v),
+
             // We have check the data type is timestamp compatible in the [BatchBuilder] so it's safe to panic.
             value => panic!("{:?} is not a timestamp", value),
         }
@@ -483,9 +481,9 @@ impl BatchBuilder {
     pub fn timestamps_array(&mut self, array: ArrayRef) -> Result<&mut Self> {
         let vector = Helper::try_into_vector(array).context(ConvertVectorSnafu)?;
         ensure!(
-            vector.data_type().is_timestamp_compatible(),
+            vector.data_type().is_timestamp(),
             InvalidBatchSnafu {
-                reason: format!("{:?} is a timestamp type", vector.data_type()),
+                reason: format!("{:?} is not a timestamp type", vector.data_type()),
             }
         );
 
diff --git a/src/storage/src/sst/parquet.rs b/src/storage/src/sst/parquet.rs
index fefdaaca9b37..d989f674ded2 100644
--- a/src/storage/src/sst/parquet.rs
+++ b/src/storage/src/sst/parquet.rs
@@ -23,7 +23,6 @@ use async_stream::try_stream;
 use async_trait::async_trait;
 use common_telemetry::{debug, error};
 use common_time::range::TimestampRange;
-use common_time::timestamp::TimeUnit;
 use common_time::Timestamp;
 use datatypes::arrow::record_batch::RecordBatch;
 use datatypes::prelude::ConcreteDataType;
@@ -162,7 +161,6 @@ fn decode_timestamp_range_inner(
     let mut end = i64::MIN;
 
     let unit = match ts_datatype {
-        ConcreteDataType::Int64(_) => TimeUnit::Millisecond,
         ConcreteDataType::Timestamp(type_) => type_.unit(),
         _ => {
             return DecodeParquetTimeRangeSnafu {
@@ -358,6 +356,7 @@ mod tests {
     use api::v1::OpType;
     use common_base::readable_size::ReadableSize;
     use common_test_util::temp_dir::create_temp_dir;
+    use common_time::timestamp::TimeUnit;
     use datatypes::arrow::array::{Array, UInt64Array, UInt8Array};
     use datatypes::prelude::{ScalarVector, Vector};
     use datatypes::types::{TimestampMillisecondType, TimestampType};
diff --git a/src/storage/src/sst/pruning.rs b/src/storage/src/sst/pruning.rs
index 499d04ebde37..a6c2080be81f 100644
--- a/src/storage/src/sst/pruning.rs
+++ b/src/storage/src/sst/pruning.rs
@@ -15,10 +15,10 @@
 use std::sync::Arc;
 
 use arrow::array::{
-    PrimitiveArray, TimestampMicrosecondArray, TimestampMillisecondArray, TimestampNanosecondArray,
+    TimestampMicrosecondArray, TimestampMillisecondArray, TimestampNanosecondArray,
     TimestampSecondArray,
 };
-use arrow::datatypes::{DataType, Int64Type};
+use arrow::datatypes::DataType;
 use arrow::error::ArrowError;
 use arrow_array::{Array, BooleanArray, RecordBatch};
 use common_time::range::TimestampRange;
@@ -45,7 +45,6 @@ pub(crate) fn build_row_filter(
     let ts_col_idx = store_schema.timestamp_index();
     let ts_col = store_schema.columns().get(ts_col_idx)?;
     let ts_col_unit = match &ts_col.desc.data_type {
-        ConcreteDataType::Int64(_) => TimeUnit::Millisecond,
         ConcreteDataType::Timestamp(ts_type) => ts_type.unit(),
         _ => unreachable!(),
     };
@@ -205,7 +204,6 @@ impl ArrowPredicate for FastTimestampRowFilter {
                     downcast_and_compute!(TimestampNanosecondArray)
                 }
             },
-            DataType::Int64 => downcast_and_compute!(PrimitiveArray<Int64Type>),
             _ => {
                 unreachable!()
             }
@@ -270,9 +268,6 @@ impl ArrowPredicate for PlainTimestampRowFilter {
                     downcast_and_compute!(TimestampNanosecondArray, Nanosecond)
                 }
             },
-            DataType::Int64 => {
-                downcast_and_compute!(PrimitiveArray<Int64Type>, Millisecond)
-            }
             _ => {
                 unreachable!()
             }
diff --git a/src/store-api/src/metadata.rs b/src/store-api/src/metadata.rs
index 946bd48a32c8..123ce83c7663 100644
--- a/src/store-api/src/metadata.rs
+++ b/src/store-api/src/metadata.rs
@@ -26,7 +26,6 @@ use api::v1::SemanticType;
 use common_error::ext::ErrorExt;
 use common_error::status_code::StatusCode;
 use datatypes::arrow::datatypes::FieldRef;
-use datatypes::prelude::DataType;
 use datatypes::schema::{ColumnDefaultConstraint, ColumnSchema, Schema, SchemaRef};
 use serde::de::Error;
 use serde::{Deserialize, Deserializer, Serialize};
@@ -355,13 +354,10 @@ impl RegionMetadata {
     fn validate_column_metadata(column_metadata: &ColumnMetadata) -> Result<()> {
         if column_metadata.semantic_type == SemanticType::Timestamp {
             ensure!(
-                column_metadata
-                    .column_schema
-                    .data_type
-                    .is_timestamp_compatible(),
+                column_metadata.column_schema.data_type.is_timestamp(),
                 InvalidMetaSnafu {
                     reason: format!(
-                        "{} is not timestamp compatible",
+                        "column `{}` is not timestamp type",
                         column_metadata.column_schema.name
                     ),
                 }
@@ -675,7 +671,8 @@ mod test {
         builder.push_column_metadata(col);
         let err = builder.build().unwrap_err();
         assert!(
-            err.to_string().contains("ts is not timestamp compatible"),
+            err.to_string()
+                .contains("column `ts` is not timestamp type"),
             "unexpected err: {err}",
         );
     }

From e7c1d912c16685a87cfce98882017ceb60aa9acc Mon Sep 17 00:00:00 2001
From: JeremyHi <jiachun_feng@proton.me>
Date: Fri, 22 Sep 2023 17:47:34 +0800
Subject: [PATCH 09/36] chore: typo (#2470)

---
 src/meta-srv/src/handler/node_stat.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/meta-srv/src/handler/node_stat.rs b/src/meta-srv/src/handler/node_stat.rs
index d51c1c194dcd..8601ec5dbf67 100644
--- a/src/meta-srv/src/handler/node_stat.rs
+++ b/src/meta-srv/src/handler/node_stat.rs
@@ -115,7 +115,7 @@ impl TryFrom<HeartbeatRequest> for Stat {
                 })
             }
             _ => InvalidHeartbeatRequestSnafu {
-                err_msg: "missing header, peer or node_stat",
+                err_msg: "missing header or peer",
             }
             .fail(),
         }

From ff2b44e10749f0a163e0d4891bfb5a26af46b70f Mon Sep 17 00:00:00 2001
From: Ning Sun <sunng@protonmail.com>
Date: Sat, 23 Sep 2023 15:59:14 +0800
Subject: [PATCH 10/36] feat: implement storage for OTLP histogram (#2282)

* feat: implement new histogram data model

* feat:  use prometheus table format for histogram

* refactor: remove duplicated code

* fix: histogram tag column

* fix: use accumulated count in buckets

* refactor: using row based protocol for otlp WIP

* refactor: use row based writer for otlp.

Also updated row writer for owned keys

* refactor: use row writers for otlp

* test: add integration tests for histogram

* refactor: change le column name
---
 src/frontend/src/instance/otlp.rs |   2 +-
 src/servers/src/influxdb.rs       |   4 +-
 src/servers/src/otlp.rs           | 596 ++++++++++++++++++------------
 src/servers/src/prom_store.rs     |   2 +-
 src/servers/src/row_writer.rs     | 108 ++++--
 tests-integration/src/otlp.rs     |  94 ++++-
 6 files changed, 535 insertions(+), 271 deletions(-)

diff --git a/src/frontend/src/instance/otlp.rs b/src/frontend/src/instance/otlp.rs
index 9c9b8af48bc1..29e0cfc44657 100644
--- a/src/frontend/src/instance/otlp.rs
+++ b/src/frontend/src/instance/otlp.rs
@@ -42,7 +42,7 @@ impl OpenTelemetryProtocolHandler for Instance {
             .context(AuthSnafu)?;
         let (requests, rows) = otlp::to_grpc_insert_requests(request)?;
         let _ = self
-            .handle_inserts(requests, ctx)
+            .handle_row_inserts(requests, ctx)
             .await
             .map_err(BoxedError::new)
             .context(error::ExecuteGrpcQuerySnafu)?;
diff --git a/src/servers/src/influxdb.rs b/src/servers/src/influxdb.rs
index 7db8a03b2d96..9ce78be2632c 100644
--- a/src/servers/src/influxdb.rs
+++ b/src/servers/src/influxdb.rs
@@ -53,7 +53,7 @@ impl TryFrom<InfluxdbRequest> for RowInsertRequests {
 
             // tags
             if let Some(tags) = tags {
-                let kvs = tags.iter().map(|(k, v)| (k.as_str(), v.as_str()));
+                let kvs = tags.iter().map(|(k, v)| (k.to_string(), v.as_str()));
                 row_writer::write_tags(table_data, kvs, &mut one_row)?;
             }
 
@@ -69,7 +69,7 @@ impl TryFrom<InfluxdbRequest> for RowInsertRequests {
                     ),
                     FieldValue::Boolean(v) => (ColumnDataType::Boolean, ValueData::BoolValue(*v)),
                 };
-                (k.as_str(), datatype, value)
+                (k.to_string(), datatype, value)
             });
             row_writer::write_fields(table_data, fields, &mut one_row)?;
 
diff --git a/src/servers/src/otlp.rs b/src/servers/src/otlp.rs
index 8af21fee7b66..3acfaf0a881b 100644
--- a/src/servers/src/otlp.rs
+++ b/src/servers/src/otlp.rs
@@ -12,17 +12,20 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use api::v1::{InsertRequest, InsertRequests};
-use common_grpc::writer::{LinesWriter, Precision};
+use api::v1::{RowInsertRequests, Value};
+use common_grpc::writer::Precision;
 use opentelemetry_proto::tonic::collector::metrics::v1::ExportMetricsServiceRequest;
 use opentelemetry_proto::tonic::common::v1::{any_value, KeyValue};
 use opentelemetry_proto::tonic::metrics::v1::{metric, number_data_point, *};
-use snafu::ResultExt;
 
-use crate::error::{self, Result};
+use crate::error::Result;
+use crate::row_writer::{self, MultiTableData, TableData};
 
 const GREPTIME_TIMESTAMP: &str = "greptime_timestamp";
 const GREPTIME_VALUE: &str = "greptime_value";
+const GREPTIME_COUNT: &str = "greptime_count";
+/// the default column count for table writer
+const APPROXIMATE_COLUMN_COUNT: usize = 8;
 
 /// Normalize otlp instrumentation, metric and attribute names
 ///
@@ -43,313 +46,350 @@ fn normalize_otlp_name(name: &str) -> String {
 /// Returns `InsertRequests` and total number of rows to ingest
 pub fn to_grpc_insert_requests(
     request: ExportMetricsServiceRequest,
-) -> Result<(InsertRequests, usize)> {
-    let mut insert_batch = Vec::new();
-    let mut rows = 0;
-
-    for resource in request.resource_metrics {
-        let resource_attrs = resource.resource.map(|r| r.attributes);
-        for scope in resource.scope_metrics {
-            let scope_attrs = scope.scope.map(|s| s.attributes);
-            for metric in scope.metrics {
-                if let Some(insert) =
-                    encode_metrics(&metric, resource_attrs.as_ref(), scope_attrs.as_ref())?
-                {
-                    rows += insert.row_count;
-                    insert_batch.push(insert);
-                }
+) -> Result<(RowInsertRequests, usize)> {
+    let mut table_writer = MultiTableData::default();
+
+    for resource in &request.resource_metrics {
+        let resource_attrs = resource.resource.as_ref().map(|r| &r.attributes);
+        for scope in &resource.scope_metrics {
+            let scope_attrs = scope.scope.as_ref().map(|s| &s.attributes);
+            for metric in &scope.metrics {
+                encode_metrics(&mut table_writer, metric, resource_attrs, scope_attrs)?;
             }
         }
     }
 
-    let inserts = InsertRequests {
-        inserts: insert_batch,
-    };
-
-    Ok((inserts, rows as usize))
+    Ok(table_writer.into_row_insert_requests())
 }
 
 fn encode_metrics(
+    table_writer: &mut MultiTableData,
     metric: &Metric,
     resource_attrs: Option<&Vec<KeyValue>>,
     scope_attrs: Option<&Vec<KeyValue>>,
-) -> Result<Option<InsertRequest>> {
+) -> Result<()> {
     let name = &metric.name;
     // note that we don't store description or unit, we might want to deal with
     // these fields in the future.
     if let Some(data) = &metric.data {
         match data {
             metric::Data::Gauge(gauge) => {
-                encode_gauge(name, gauge, resource_attrs, scope_attrs).map(Some)
+                encode_gauge(table_writer, name, gauge, resource_attrs, scope_attrs)?;
+            }
+            metric::Data::Sum(sum) => {
+                encode_sum(table_writer, name, sum, resource_attrs, scope_attrs)?;
             }
-            metric::Data::Sum(sum) => encode_sum(name, sum, resource_attrs, scope_attrs).map(Some),
             metric::Data::Summary(summary) => {
-                encode_summary(name, summary, resource_attrs, scope_attrs).map(Some)
+                encode_summary(table_writer, name, summary, resource_attrs, scope_attrs)?;
+            }
+            metric::Data::Histogram(hist) => {
+                encode_histogram(table_writer, name, hist, resource_attrs, scope_attrs)?;
             }
-            // TODO(sunng87) leave histogram for next release
-            metric::Data::Histogram(_hist) => Ok(None),
-            metric::Data::ExponentialHistogram(_hist) => Ok(None),
+            // TODO(sunng87) leave ExponentialHistogram for next release
+            metric::Data::ExponentialHistogram(_hist) => {}
         }
-    } else {
-        Ok(None)
     }
-}
 
-fn write_attributes(lines: &mut LinesWriter, attrs: Option<&Vec<KeyValue>>) -> Result<()> {
-    if let Some(attrs) = attrs {
-        for attr in attrs {
-            write_attribute(lines, attr)?;
-        }
-    }
     Ok(())
 }
 
-fn write_attribute(lines: &mut LinesWriter, attr: &KeyValue) -> Result<()> {
-    if let Some(val) = attr.value.as_ref().and_then(|v| v.value.as_ref()) {
-        match val {
-            any_value::Value::StringValue(s) => lines
-                .write_tag(&normalize_otlp_name(&attr.key), s)
-                .context(error::OtlpMetricsWriteSnafu)?,
-
-            any_value::Value::IntValue(v) => lines
-                .write_tag(&normalize_otlp_name(&attr.key), &v.to_string())
-                .context(error::OtlpMetricsWriteSnafu)?,
-            any_value::Value::DoubleValue(v) => lines
-                .write_tag(&normalize_otlp_name(&attr.key), &v.to_string())
-                .context(error::OtlpMetricsWriteSnafu)?,
-            // TODO(sunng87): allow different type of values
-            _ => {}
-        }
-    }
+fn write_attributes(
+    writer: &mut TableData,
+    row: &mut Vec<Value>,
+    attrs: Option<&Vec<KeyValue>>,
+) -> Result<()> {
+    if let Some(attrs) = attrs {
+        let table_tags = attrs.iter().filter_map(|attr| {
+            if let Some(val) = attr.value.as_ref().and_then(|v| v.value.as_ref()) {
+                let key = normalize_otlp_name(&attr.key);
+                match val {
+                    any_value::Value::StringValue(s) => Some((key, s.to_string())),
+                    any_value::Value::IntValue(v) => Some((key, v.to_string())),
+                    any_value::Value::DoubleValue(v) => Some((key, v.to_string())),
+                    _ => None, // TODO(sunng87): allow different type of values
+                }
+            } else {
+                None
+            }
+        });
 
+        row_writer::write_tags(writer, table_tags, row)?;
+    }
     Ok(())
 }
 
-fn write_timestamp(lines: &mut LinesWriter, time_nano: i64) -> Result<()> {
-    lines
-        .write_ts(GREPTIME_TIMESTAMP, (time_nano, Precision::Nanosecond))
-        .context(error::OtlpMetricsWriteSnafu)?;
-    Ok(())
+fn write_timestamp(table: &mut TableData, row: &mut Vec<Value>, time_nano: i64) -> Result<()> {
+    row_writer::write_ts_precision(
+        table,
+        GREPTIME_TIMESTAMP,
+        Some(time_nano),
+        Precision::Nanosecond,
+        row,
+    )
 }
 
 fn write_data_point_value(
-    lines: &mut LinesWriter,
+    table: &mut TableData,
+    row: &mut Vec<Value>,
     field: &str,
     value: &Option<number_data_point::Value>,
 ) -> Result<()> {
     match value {
         Some(number_data_point::Value::AsInt(val)) => {
             // we coerce all values to f64
-            lines
-                .write_f64(field, *val as f64)
-                .context(error::OtlpMetricsWriteSnafu)?
+            row_writer::write_f64(table, field, *val as f64, row)?;
+        }
+        Some(number_data_point::Value::AsDouble(val)) => {
+            row_writer::write_f64(table, field, *val, row)?;
         }
-        Some(number_data_point::Value::AsDouble(val)) => lines
-            .write_f64(field, *val)
-            .context(error::OtlpMetricsWriteSnafu)?,
         _ => {}
     }
     Ok(())
 }
 
+fn write_tags_and_timestamp(
+    table: &mut TableData,
+    row: &mut Vec<Value>,
+    resource_attrs: Option<&Vec<KeyValue>>,
+    scope_attrs: Option<&Vec<KeyValue>>,
+    data_point_attrs: Option<&Vec<KeyValue>>,
+    timestamp_nanos: i64,
+) -> Result<()> {
+    write_attributes(table, row, resource_attrs)?;
+    write_attributes(table, row, scope_attrs)?;
+    write_attributes(table, row, data_point_attrs)?;
+
+    write_timestamp(table, row, timestamp_nanos)?;
+
+    Ok(())
+}
+
 /// encode this gauge metric
 ///
 /// note that there can be multiple data points in the request, it's going to be
 /// stored as multiple rows
 fn encode_gauge(
+    table_writer: &mut MultiTableData,
     name: &str,
     gauge: &Gauge,
     resource_attrs: Option<&Vec<KeyValue>>,
     scope_attrs: Option<&Vec<KeyValue>>,
-) -> Result<InsertRequest> {
-    let mut lines = LinesWriter::with_lines(gauge.data_points.len());
-    for data_point in &gauge.data_points {
-        write_attributes(&mut lines, resource_attrs)?;
-        write_attributes(&mut lines, scope_attrs)?;
-        write_attributes(&mut lines, Some(data_point.attributes.as_ref()))?;
-        write_timestamp(&mut lines, data_point.time_unix_nano as i64)?;
-        write_data_point_value(&mut lines, GREPTIME_VALUE, &data_point.value)?;
+) -> Result<()> {
+    let table = table_writer.get_or_default_table_data(
+        &normalize_otlp_name(name),
+        APPROXIMATE_COLUMN_COUNT,
+        gauge.data_points.len(),
+    );
 
-        lines.commit();
+    for data_point in &gauge.data_points {
+        let mut row = table.alloc_one_row();
+        write_tags_and_timestamp(
+            table,
+            &mut row,
+            resource_attrs,
+            scope_attrs,
+            Some(data_point.attributes.as_ref()),
+            data_point.time_unix_nano as i64,
+        )?;
+
+        write_data_point_value(table, &mut row, GREPTIME_VALUE, &data_point.value)?;
+        table.add_row(row);
     }
 
-    let (columns, row_count) = lines.finish();
-    Ok(InsertRequest {
-        table_name: normalize_otlp_name(name),
-        columns,
-        row_count,
-    })
+    Ok(())
 }
 
 /// encode this sum metric
 ///
 /// `aggregation_temporality` and `monotonic` are ignored for now
 fn encode_sum(
+    table_writer: &mut MultiTableData,
     name: &str,
     sum: &Sum,
     resource_attrs: Option<&Vec<KeyValue>>,
     scope_attrs: Option<&Vec<KeyValue>>,
-) -> Result<InsertRequest> {
-    let mut lines = LinesWriter::with_lines(sum.data_points.len());
+) -> Result<()> {
+    let table = table_writer.get_or_default_table_data(
+        &normalize_otlp_name(name),
+        APPROXIMATE_COLUMN_COUNT,
+        sum.data_points.len(),
+    );
 
     for data_point in &sum.data_points {
-        write_attributes(&mut lines, resource_attrs)?;
-        write_attributes(&mut lines, scope_attrs)?;
-        write_attributes(&mut lines, Some(data_point.attributes.as_ref()))?;
-
-        write_timestamp(&mut lines, data_point.time_unix_nano as i64)?;
-
-        write_data_point_value(&mut lines, GREPTIME_VALUE, &data_point.value)?;
-
-        lines.commit();
+        let mut row = table.alloc_one_row();
+        write_tags_and_timestamp(
+            table,
+            &mut row,
+            resource_attrs,
+            scope_attrs,
+            Some(data_point.attributes.as_ref()),
+            data_point.time_unix_nano as i64,
+        )?;
+        write_data_point_value(table, &mut row, GREPTIME_VALUE, &data_point.value)?;
+        table.add_row(row);
     }
 
-    let (columns, row_count) = lines.finish();
-    Ok(InsertRequest {
-        table_name: normalize_otlp_name(name),
-        columns,
-        row_count,
-    })
+    Ok(())
 }
 
-// TODO(sunng87): we may need better implementation for histogram
-#[allow(dead_code)]
-fn encode_histogram(name: &str, hist: &Histogram) -> Result<InsertRequest> {
-    let mut lines = LinesWriter::with_lines(hist.data_points.len());
-
-    for data_point in &hist.data_points {
-        for attr in &data_point.attributes {
-            write_attribute(&mut lines, attr)?;
-        }
-
-        write_timestamp(&mut lines, data_point.time_unix_nano as i64)?;
-
-        for (idx, count) in data_point.bucket_counts.iter().enumerate() {
-            // here we don't store bucket boundary
-            lines
-                .write_u64(&format!("bucket_{}", idx), *count)
-                .context(error::OtlpMetricsWriteSnafu)?;
-        }
-
-        if let Some(min) = data_point.min {
-            lines
-                .write_f64("min", min)
-                .context(error::OtlpMetricsWriteSnafu)?;
-        }
+const HISTOGRAM_LE_COLUMN: &str = "le";
 
-        if let Some(max) = data_point.max {
-            lines
-                .write_f64("max", max)
-                .context(error::OtlpMetricsWriteSnafu)?;
-        }
-
-        lines.commit();
-    }
+/// Encode histogram data. This function returns 3 insert requests for 3 tables.
+///
+/// The implementation has been following Prometheus histogram table format:
+///
+/// - A `%metric%_bucket` table including `greptime_le` tag that stores bucket upper
+/// limit, and `greptime_value` for bucket count
+/// - A `%metric%_sum` table storing sum of samples
+/// -  A `%metric%_count` table storing count of samples.
+///
+/// By its Prometheus compatibility, we hope to be able to use prometheus
+/// quantile functions on this table.
+fn encode_histogram(
+    table_writer: &mut MultiTableData,
+    name: &str,
+    hist: &Histogram,
+    resource_attrs: Option<&Vec<KeyValue>>,
+    scope_attrs: Option<&Vec<KeyValue>>,
+) -> Result<()> {
+    let normalized_name = normalize_otlp_name(name);
 
-    let (columns, row_count) = lines.finish();
-    Ok(InsertRequest {
-        table_name: normalize_otlp_name(name),
-        columns,
-        row_count,
-    })
-}
+    let bucket_table_name = format!("{}_bucket", normalized_name);
+    let sum_table_name = format!("{}_sum", normalized_name);
+    let count_table_name = format!("{}_count", normalized_name);
 
-#[allow(dead_code)]
-fn encode_exponential_histogram(name: &str, hist: &ExponentialHistogram) -> Result<InsertRequest> {
-    let mut lines = LinesWriter::with_lines(hist.data_points.len());
+    let data_points_len = hist.data_points.len();
+    // Note that the row and columns number here is approximate
+    let mut bucket_table = TableData::new(APPROXIMATE_COLUMN_COUNT, data_points_len * 3);
+    let mut sum_table = TableData::new(APPROXIMATE_COLUMN_COUNT, data_points_len);
+    let mut count_table = TableData::new(APPROXIMATE_COLUMN_COUNT, data_points_len);
 
     for data_point in &hist.data_points {
-        for attr in &data_point.attributes {
-            write_attribute(&mut lines, attr)?;
-        }
-
-        write_timestamp(&mut lines, data_point.time_unix_nano as i64)?;
-
-        // TODO(sunng87): confirm if this working
-        if let Some(positive_buckets) = &data_point.positive {
-            for (idx, count) in positive_buckets.bucket_counts.iter().enumerate() {
-                // here we don't store bucket boundary
-                lines
-                    .write_u64(
-                        &format!("bucket_{}", idx + positive_buckets.offset as usize),
-                        *count,
-                    )
-                    .context(error::OtlpMetricsWriteSnafu)?;
+        let mut accumulated_count = 0;
+        for (idx, count) in data_point.bucket_counts.iter().enumerate() {
+            let mut bucket_row = bucket_table.alloc_one_row();
+            write_tags_and_timestamp(
+                &mut bucket_table,
+                &mut bucket_row,
+                resource_attrs,
+                scope_attrs,
+                Some(data_point.attributes.as_ref()),
+                data_point.time_unix_nano as i64,
+            )?;
+
+            if let Some(upper_bounds) = data_point.explicit_bounds.get(idx) {
+                row_writer::write_tag(
+                    &mut bucket_table,
+                    HISTOGRAM_LE_COLUMN,
+                    upper_bounds,
+                    &mut bucket_row,
+                )?;
+            } else if idx == data_point.explicit_bounds.len() {
+                // The last bucket
+                row_writer::write_tag(
+                    &mut bucket_table,
+                    HISTOGRAM_LE_COLUMN,
+                    f64::INFINITY,
+                    &mut bucket_row,
+                )?;
             }
-        }
 
-        if let Some(negative_buckets) = &data_point.negative {
-            for (idx, count) in negative_buckets.bucket_counts.iter().enumerate() {
-                lines
-                    .write_u64(
-                        &format!("bucket_{}", idx + negative_buckets.offset as usize),
-                        *count,
-                    )
-                    .context(error::OtlpMetricsWriteSnafu)?;
-            }
-        }
+            accumulated_count += count;
+            row_writer::write_f64(
+                &mut bucket_table,
+                GREPTIME_VALUE,
+                accumulated_count as f64,
+                &mut bucket_row,
+            )?;
 
-        if let Some(min) = data_point.min {
-            lines
-                .write_f64("min", min)
-                .context(error::OtlpMetricsWriteSnafu)?;
+            bucket_table.add_row(bucket_row);
         }
 
-        if let Some(max) = data_point.max {
-            lines
-                .write_f64("max", max)
-                .context(error::OtlpMetricsWriteSnafu)?;
+        if let Some(sum) = data_point.sum {
+            let mut sum_row = sum_table.alloc_one_row();
+            write_tags_and_timestamp(
+                &mut sum_table,
+                &mut sum_row,
+                resource_attrs,
+                scope_attrs,
+                Some(data_point.attributes.as_ref()),
+                data_point.time_unix_nano as i64,
+            )?;
+
+            row_writer::write_f64(&mut sum_table, GREPTIME_VALUE, sum, &mut sum_row)?;
+            sum_table.add_row(sum_row);
         }
 
-        lines.commit();
+        let mut count_row = count_table.alloc_one_row();
+        write_tags_and_timestamp(
+            &mut count_table,
+            &mut count_row,
+            resource_attrs,
+            scope_attrs,
+            Some(data_point.attributes.as_ref()),
+            data_point.time_unix_nano as i64,
+        )?;
+
+        row_writer::write_f64(
+            &mut count_table,
+            GREPTIME_VALUE,
+            data_point.count as f64,
+            &mut count_row,
+        )?;
+        count_table.add_row(count_row);
     }
 
-    let (columns, row_count) = lines.finish();
-    Ok(InsertRequest {
-        table_name: normalize_otlp_name(name),
-        columns,
-        row_count,
-    })
+    table_writer.add_table_data(bucket_table_name, bucket_table);
+    table_writer.add_table_data(sum_table_name, sum_table);
+    table_writer.add_table_data(count_table_name, count_table);
+
+    Ok(())
+}
+
+#[allow(dead_code)]
+fn encode_exponential_histogram(_name: &str, _hist: &ExponentialHistogram) -> Result<()> {
+    // TODO(sunng87): implement this using a prometheus compatible way
+    Ok(())
 }
 
 fn encode_summary(
+    table_writer: &mut MultiTableData,
     name: &str,
     summary: &Summary,
     resource_attrs: Option<&Vec<KeyValue>>,
     scope_attrs: Option<&Vec<KeyValue>>,
-) -> Result<InsertRequest> {
-    let mut lines = LinesWriter::with_lines(summary.data_points.len());
+) -> Result<()> {
+    let table = table_writer.get_or_default_table_data(
+        &normalize_otlp_name(name),
+        APPROXIMATE_COLUMN_COUNT,
+        summary.data_points.len(),
+    );
 
     for data_point in &summary.data_points {
-        write_attributes(&mut lines, resource_attrs)?;
-        write_attributes(&mut lines, scope_attrs)?;
-        write_attributes(&mut lines, Some(data_point.attributes.as_ref()))?;
-
-        write_timestamp(&mut lines, data_point.time_unix_nano as i64)?;
+        let mut row = table.alloc_one_row();
+        write_tags_and_timestamp(
+            table,
+            &mut row,
+            resource_attrs,
+            scope_attrs,
+            Some(data_point.attributes.as_ref()),
+            data_point.time_unix_nano as i64,
+        )?;
 
         for quantile in &data_point.quantile_values {
-            // here we don't store bucket boundary
-            lines
-                .write_f64(
-                    &format!("greptime_p{:02}", quantile.quantile * 100f64),
-                    quantile.value,
-                )
-                .context(error::OtlpMetricsWriteSnafu)?;
+            row_writer::write_f64(
+                table,
+                &format!("greptime_p{:02}", quantile.quantile * 100f64),
+                quantile.value,
+                &mut row,
+            )?;
         }
 
-        lines
-            .write_u64("greptime_count", data_point.count)
-            .context(error::OtlpMetricsWriteSnafu)?;
-
-        lines.commit();
+        row_writer::write_f64(table, GREPTIME_COUNT, data_point.count as f64, &mut row)?;
+        table.add_row(row);
     }
 
-    let (columns, row_count) = lines.finish();
-    Ok(InsertRequest {
-        table_name: normalize_otlp_name(name),
-        columns,
-        row_count,
-    })
+    Ok(())
 }
 
 #[cfg(test)]
@@ -358,7 +398,7 @@ mod tests {
     use opentelemetry_proto::tonic::common::v1::{AnyValue, KeyValue};
     use opentelemetry_proto::tonic::metrics::v1::number_data_point::Value;
     use opentelemetry_proto::tonic::metrics::v1::summary_data_point::ValueAtQuantile;
-    use opentelemetry_proto::tonic::metrics::v1::NumberDataPoint;
+    use opentelemetry_proto::tonic::metrics::v1::{HistogramDataPoint, NumberDataPoint};
 
     use super::*;
 
@@ -382,6 +422,8 @@ mod tests {
 
     #[test]
     fn test_encode_gauge() {
+        let mut tables = MultiTableData::default();
+
         let data_points = vec![
             NumberDataPoint {
                 attributes: vec![keyvalue("host", "testsevrer")],
@@ -397,7 +439,8 @@ mod tests {
             },
         ];
         let gauge = Gauge { data_points };
-        let inserts = encode_gauge(
+        encode_gauge(
+            &mut tables,
             "datamon",
             &gauge,
             Some(&vec![keyvalue("resource", "app")]),
@@ -405,12 +448,12 @@ mod tests {
         )
         .unwrap();
 
-        assert_eq!(inserts.table_name, "datamon");
-        assert_eq!(inserts.row_count, 2);
-        assert_eq!(inserts.columns.len(), 5);
+        let table = tables.get_or_default_table_data("datamon", 0, 0);
+        assert_eq!(table.num_rows(), 2);
+        assert_eq!(table.num_columns(), 5);
         assert_eq!(
-            inserts
-                .columns
+            table
+                .columns()
                 .iter()
                 .map(|c| &c.column_name)
                 .collect::<Vec<&String>>(),
@@ -426,6 +469,8 @@ mod tests {
 
     #[test]
     fn test_encode_sum() {
+        let mut tables = MultiTableData::default();
+
         let data_points = vec![
             NumberDataPoint {
                 attributes: vec![keyvalue("host", "testserver")],
@@ -444,7 +489,8 @@ mod tests {
             data_points,
             ..Default::default()
         };
-        let inserts = encode_sum(
+        encode_sum(
+            &mut tables,
             "datamon",
             &sum,
             Some(&vec![keyvalue("resource", "app")]),
@@ -452,12 +498,12 @@ mod tests {
         )
         .unwrap();
 
-        assert_eq!(inserts.table_name, "datamon");
-        assert_eq!(inserts.row_count, 2);
-        assert_eq!(inserts.columns.len(), 5);
+        let table = tables.get_or_default_table_data("datamon", 0, 0);
+        assert_eq!(table.num_rows(), 2);
+        assert_eq!(table.num_columns(), 5);
         assert_eq!(
-            inserts
-                .columns
+            table
+                .columns()
                 .iter()
                 .map(|c| &c.column_name)
                 .collect::<Vec<&String>>(),
@@ -473,6 +519,8 @@ mod tests {
 
     #[test]
     fn test_encode_summary() {
+        let mut tables = MultiTableData::default();
+
         let data_points = vec![SummaryDataPoint {
             attributes: vec![keyvalue("host", "testserver")],
             time_unix_nano: 100,
@@ -491,7 +539,8 @@ mod tests {
             ..Default::default()
         }];
         let summary = Summary { data_points };
-        let inserts = encode_summary(
+        encode_summary(
+            &mut tables,
             "datamon",
             &summary,
             Some(&vec![keyvalue("resource", "app")]),
@@ -499,12 +548,12 @@ mod tests {
         )
         .unwrap();
 
-        assert_eq!(inserts.table_name, "datamon");
-        assert_eq!(inserts.row_count, 1);
-        assert_eq!(inserts.columns.len(), 7);
+        let table = tables.get_or_default_table_data("datamon", 0, 0);
+        assert_eq!(table.num_rows(), 1);
+        assert_eq!(table.num_columns(), 7);
         assert_eq!(
-            inserts
-                .columns
+            table
+                .columns()
                 .iter()
                 .map(|c| &c.column_name)
                 .collect::<Vec<&String>>(),
@@ -519,4 +568,93 @@ mod tests {
             ]
         );
     }
+
+    #[test]
+    fn test_encode_histogram() {
+        let mut tables = MultiTableData::default();
+
+        let data_points = vec![HistogramDataPoint {
+            attributes: vec![keyvalue("host", "testserver")],
+            time_unix_nano: 100,
+            start_time_unix_nano: 23,
+            count: 25,
+            sum: Some(100.),
+            max: Some(200.),
+            min: Some(0.03),
+            bucket_counts: vec![2, 4, 6, 9, 4],
+            explicit_bounds: vec![0.1, 1., 10., 100.],
+            ..Default::default()
+        }];
+
+        let histogram = Histogram {
+            data_points,
+            aggregation_temporality: AggregationTemporality::Delta.into(),
+        };
+        encode_histogram(
+            &mut tables,
+            "histo",
+            &histogram,
+            Some(&vec![keyvalue("resource", "app")]),
+            Some(&vec![keyvalue("scope", "otel")]),
+        )
+        .unwrap();
+
+        assert_eq!(3, tables.num_tables());
+
+        // bucket table
+        let bucket_table = tables.get_or_default_table_data("histo_bucket", 0, 0);
+        assert_eq!(bucket_table.num_rows(), 5);
+        assert_eq!(bucket_table.num_columns(), 6);
+        assert_eq!(
+            bucket_table
+                .columns()
+                .iter()
+                .map(|c| &c.column_name)
+                .collect::<Vec<&String>>(),
+            vec![
+                "resource",
+                "scope",
+                "host",
+                "greptime_timestamp",
+                "le",
+                "greptime_value",
+            ]
+        );
+
+        let sum_table = tables.get_or_default_table_data("histo_sum", 0, 0);
+        assert_eq!(sum_table.num_rows(), 1);
+        assert_eq!(sum_table.num_columns(), 5);
+        assert_eq!(
+            sum_table
+                .columns()
+                .iter()
+                .map(|c| &c.column_name)
+                .collect::<Vec<&String>>(),
+            vec![
+                "resource",
+                "scope",
+                "host",
+                "greptime_timestamp",
+                "greptime_value",
+            ]
+        );
+
+        let count_table = tables.get_or_default_table_data("histo_count", 0, 0);
+        assert_eq!(count_table.num_rows(), 1);
+        assert_eq!(count_table.num_columns(), 5);
+        assert_eq!(
+            count_table
+                .columns()
+                .iter()
+                .map(|c| &c.column_name)
+                .collect::<Vec<&String>>(),
+            vec![
+                "resource",
+                "scope",
+                "host",
+                "greptime_timestamp",
+                "greptime_value",
+            ]
+        );
+    }
 }
diff --git a/src/servers/src/prom_store.rs b/src/servers/src/prom_store.rs
index 2a745973ee4e..63a13842b87e 100644
--- a/src/servers/src/prom_store.rs
+++ b/src/servers/src/prom_store.rs
@@ -334,7 +334,7 @@ pub fn to_grpc_row_insert_requests(request: WriteRequest) -> Result<(RowInsertRe
                 if label.name == METRIC_NAME_LABEL {
                     None
                 } else {
-                    Some((label.name.as_str(), label.value.as_str()))
+                    Some((label.name.to_string(), label.value.as_str()))
                 }
             });
             row_writer::write_tags(table_data, kvs, &mut one_row)?;
diff --git a/src/servers/src/row_writer.rs b/src/servers/src/row_writer.rs
index 5504b34aa1f8..9dfe92738706 100644
--- a/src/servers/src/row_writer.rs
+++ b/src/servers/src/row_writer.rs
@@ -27,13 +27,13 @@ use snafu::{ensure, OptionExt, ResultExt};
 
 use crate::error::{IncompatibleSchemaSnafu, InfluxdbLinesWriteSnafu, Result, TimePrecisionSnafu};
 
-pub struct TableData<'a> {
+pub struct TableData {
     schema: Vec<ColumnSchema>,
     rows: Vec<Row>,
-    column_indexes: HashMap<&'a str, usize>,
+    column_indexes: HashMap<String, usize>,
 }
 
-impl TableData<'_> {
+impl TableData {
     pub fn new(num_columns: usize, num_rows: usize) -> Self {
         Self {
             schema: Vec::with_capacity(num_columns),
@@ -62,16 +62,27 @@ impl TableData<'_> {
         self.rows.push(Row { values })
     }
 
+    #[allow(dead_code)]
+    pub fn columns(&self) -> &Vec<ColumnSchema> {
+        &self.schema
+    }
+
     pub fn into_schema_and_rows(self) -> (Vec<ColumnSchema>, Vec<Row>) {
         (self.schema, self.rows)
     }
 }
 
-pub struct MultiTableData<'a> {
-    table_data_map: HashMap<&'a str, TableData<'a>>,
+pub struct MultiTableData {
+    table_data_map: HashMap<String, TableData>,
+}
+
+impl Default for MultiTableData {
+    fn default() -> Self {
+        Self::new()
+    }
 }
 
-impl<'a> MultiTableData<'a> {
+impl MultiTableData {
     pub fn new() -> Self {
         Self {
             table_data_map: HashMap::new(),
@@ -80,15 +91,25 @@ impl<'a> MultiTableData<'a> {
 
     pub fn get_or_default_table_data(
         &mut self,
-        table_name: &'a str,
+        table_name: impl ToString,
         num_columns: usize,
         num_rows: usize,
-    ) -> &mut TableData<'a> {
+    ) -> &mut TableData {
         self.table_data_map
-            .entry(table_name)
+            .entry(table_name.to_string())
             .or_insert_with(|| TableData::new(num_columns, num_rows))
     }
 
+    pub fn add_table_data(&mut self, table_name: impl ToString, table_data: TableData) {
+        self.table_data_map
+            .insert(table_name.to_string(), table_data);
+    }
+
+    #[allow(dead_code)]
+    pub fn num_tables(&self) -> usize {
+        self.table_data_map.len()
+    }
+
     /// Returns the request and number of rows in it.
     pub fn into_row_insert_requests(self) -> (RowInsertRequests, usize) {
         let mut total_rows = 0;
@@ -106,7 +127,7 @@ impl<'a> MultiTableData<'a> {
                 }
 
                 RowInsertRequest {
-                    table_name: table_name.to_string(),
+                    table_name,
                     rows: Some(Rows { schema, rows }),
                 }
             })
@@ -117,9 +138,9 @@ impl<'a> MultiTableData<'a> {
     }
 }
 
-pub fn write_tags<'a>(
-    table_data: &mut TableData<'a>,
-    kvs: impl Iterator<Item = (&'a str, &'a str)>,
+pub fn write_tags(
+    table_data: &mut TableData,
+    kvs: impl Iterator<Item = (String, impl ToString)>,
     one_row: &mut Vec<Value>,
 ) -> Result<()> {
     let ktv_iter = kvs.map(|(k, v)| {
@@ -132,31 +153,53 @@ pub fn write_tags<'a>(
     write_by_semantic_type(table_data, SemanticType::Tag, ktv_iter, one_row)
 }
 
-pub fn write_fields<'a>(
-    table_data: &mut TableData<'a>,
-    fields: impl Iterator<Item = (&'a str, ColumnDataType, ValueData)>,
+pub fn write_fields(
+    table_data: &mut TableData,
+    fields: impl Iterator<Item = (String, ColumnDataType, ValueData)>,
     one_row: &mut Vec<Value>,
 ) -> Result<()> {
     write_by_semantic_type(table_data, SemanticType::Field, fields, one_row)
 }
 
-pub fn write_f64<'a>(
-    table_data: &mut TableData<'a>,
-    name: &'a str,
+pub fn write_tag(
+    table_data: &mut TableData,
+    name: impl ToString,
+    value: impl ToString,
+    one_row: &mut Vec<Value>,
+) -> Result<()> {
+    write_by_semantic_type(
+        table_data,
+        SemanticType::Tag,
+        std::iter::once((
+            name.to_string(),
+            ColumnDataType::String,
+            ValueData::StringValue(value.to_string()),
+        )),
+        one_row,
+    )
+}
+
+pub fn write_f64(
+    table_data: &mut TableData,
+    name: impl ToString,
     value: f64,
     one_row: &mut Vec<Value>,
 ) -> Result<()> {
     write_fields(
         table_data,
-        std::iter::once((name, ColumnDataType::Float64, ValueData::F64Value(value))),
+        std::iter::once((
+            name.to_string(),
+            ColumnDataType::Float64,
+            ValueData::F64Value(value),
+        )),
         one_row,
     )
 }
 
-fn write_by_semantic_type<'a>(
-    table_data: &mut TableData<'a>,
+fn write_by_semantic_type(
+    table_data: &mut TableData,
     semantic_type: SemanticType,
-    ktv_iter: impl Iterator<Item = (&'a str, ColumnDataType, ValueData)>,
+    ktv_iter: impl Iterator<Item = (String, ColumnDataType, ValueData)>,
     one_row: &mut Vec<Value>,
 ) -> Result<()> {
     let TableData {
@@ -166,7 +209,7 @@ fn write_by_semantic_type<'a>(
     } = table_data;
 
     for (name, datatype, value) in ktv_iter {
-        let index = column_indexes.entry(name).or_insert(schema.len());
+        let index = column_indexes.entry(name.clone()).or_insert(schema.len());
         if *index == schema.len() {
             schema.push(ColumnSchema {
                 column_name: name.to_string(),
@@ -183,18 +226,18 @@ fn write_by_semantic_type<'a>(
     Ok(())
 }
 
-pub fn write_ts_millis<'a>(
-    table_data: &mut TableData<'a>,
-    name: &'a str,
+pub fn write_ts_millis(
+    table_data: &mut TableData,
+    name: impl ToString,
     ts: Option<i64>,
     one_row: &mut Vec<Value>,
 ) -> Result<()> {
     write_ts_precision(table_data, name, ts, Precision::Millisecond, one_row)
 }
 
-pub fn write_ts_precision<'a>(
-    table_data: &mut TableData<'a>,
-    name: &'a str,
+pub fn write_ts_precision(
+    table_data: &mut TableData,
+    name: impl ToString,
     ts: Option<i64>,
     precision: Precision,
     one_row: &mut Vec<Value>,
@@ -204,6 +247,7 @@ pub fn write_ts_precision<'a>(
         column_indexes,
         ..
     } = table_data;
+    let name = name.to_string();
 
     let ts = match ts {
         Some(timestamp) => writer::to_ms_ts(precision, timestamp),
@@ -219,10 +263,10 @@ pub fn write_ts_precision<'a>(
         }
     };
 
-    let index = column_indexes.entry(name).or_insert(schema.len());
+    let index = column_indexes.entry(name.clone()).or_insert(schema.len());
     if *index == schema.len() {
         schema.push(ColumnSchema {
-            column_name: name.to_string(),
+            column_name: name,
             datatype: ColumnDataType::TimestampMillisecond as i32,
             semantic_type: SemanticType::Timestamp as i32,
         });
diff --git a/tests-integration/src/otlp.rs b/tests-integration/src/otlp.rs
index 437d8256b49f..942fad14a551 100644
--- a/tests-integration/src/otlp.rs
+++ b/tests-integration/src/otlp.rs
@@ -87,6 +87,65 @@ mod test {
 +------------+-------+--------------------+------------+---------------------+----------------+
 | greptimedb | otel  | java               | testserver | 1970-01-01T00:00:00 | 105.0          |
 | greptimedb | otel  | java               | testsevrer | 1970-01-01T00:00:00 | 100.0          |
++------------+-------+--------------------+------------+---------------------+----------------+",
+        );
+
+        let mut output = instance
+            .do_query(
+                "SELECT le, greptime_value FROM my_test_histo_bucket order by le",
+                ctx.clone(),
+            )
+            .await;
+        let output = output.remove(0).unwrap();
+        let Output::Stream(stream) = output else {
+            unreachable!()
+        };
+        let recordbatches = RecordBatches::try_collect(stream).await.unwrap();
+        assert_eq!(
+            recordbatches.pretty_print().unwrap(),
+            "\
++-----+----------------+
+| le  | greptime_value |
++-----+----------------+
+| 1   | 1.0            |
+| 5   | 3.0            |
+| inf | 4.0            |
++-----+----------------+",
+        );
+
+        let mut output = instance
+            .do_query("SELECT * FROM my_test_histo_sum", ctx.clone())
+            .await;
+        let output = output.remove(0).unwrap();
+        let Output::Stream(stream) = output else {
+            unreachable!()
+        };
+        let recordbatches = RecordBatches::try_collect(stream).await.unwrap();
+        assert_eq!(
+            recordbatches.pretty_print().unwrap(),
+            "\
++------------+-------+--------------------+------------+---------------------+----------------+
+| resource   | scope | telemetry_sdk_name | host       | greptime_timestamp  | greptime_value |
++------------+-------+--------------------+------------+---------------------+----------------+
+| greptimedb | otel  | java               | testserver | 1970-01-01T00:00:00 | 51.0           |
++------------+-------+--------------------+------------+---------------------+----------------+",
+        );
+
+        let mut output = instance
+            .do_query("SELECT * FROM my_test_histo_count", ctx.clone())
+            .await;
+        let output = output.remove(0).unwrap();
+        let Output::Stream(stream) = output else {
+            unreachable!()
+        };
+        let recordbatches = RecordBatches::try_collect(stream).await.unwrap();
+        assert_eq!(
+            recordbatches.pretty_print().unwrap(),
+            "\
++------------+-------+--------------------+------------+---------------------+----------------+
+| resource   | scope | telemetry_sdk_name | host       | greptime_timestamp  | greptime_value |
++------------+-------+--------------------+------------+---------------------+----------------+
+| greptimedb | otel  | java               | testserver | 1970-01-01T00:00:00 | 4.0            |
 +------------+-------+--------------------+------------+---------------------+----------------+",
         );
     }
@@ -108,15 +167,38 @@ mod test {
         ];
         let gauge = Gauge { data_points };
 
+        let histo_data_points = vec![HistogramDataPoint {
+            attributes: vec![keyvalue("host", "testserver")],
+            time_unix_nano: 100,
+            count: 4,
+            bucket_counts: vec![1, 2, 1],
+            explicit_bounds: vec![1.0f64, 5.0f64],
+            sum: Some(51f64),
+            ..Default::default()
+        }];
+
+        let histo = Histogram {
+            data_points: histo_data_points,
+            aggregation_temporality: 0,
+        };
+
         ExportMetricsServiceRequest {
             resource_metrics: vec![ResourceMetrics {
                 scope_metrics: vec![ScopeMetrics {
-                    metrics: vec![Metric {
-                        name: "my.test.metric".into(),
-                        description: "my ignored desc".into(),
-                        unit: "my ignored unit".into(),
-                        data: Some(metric::Data::Gauge(gauge)),
-                    }],
+                    metrics: vec![
+                        Metric {
+                            name: "my.test.metric".into(),
+                            description: "my ignored desc".into(),
+                            unit: "my ignored unit".into(),
+                            data: Some(metric::Data::Gauge(gauge)),
+                        },
+                        Metric {
+                            name: "my.test.histo".into(),
+                            description: "my ignored desc".into(),
+                            unit: "my ignored unit".into(),
+                            data: Some(metric::Data::Histogram(histo)),
+                        },
+                    ],
                     scope: Some(InstrumentationScope {
                         attributes: vec![
                             keyvalue("scope", "otel"),

From 2d22d608b3fa657987435b4d5ca8d7cb0b60bee0 Mon Sep 17 00:00:00 2001
From: Yingwen <realevenyag@gmail.com>
Date: Sat, 23 Sep 2023 18:05:34 +0800
Subject: [PATCH 11/36] fix(mito): reset is_sorted to true after the merger
 finishing one series (#2474)

fix: reset is_sorted flag to true after the merger finishing one series
---
 src/mito2/src/read/merge.rs | 48 +++++++++++++++++++++++++++++++++++++
 1 file changed, 48 insertions(+)

diff --git a/src/mito2/src/read/merge.rs b/src/mito2/src/read/merge.rs
index e5b8651fa764..797e2f9f7032 100644
--- a/src/mito2/src/read/merge.rs
+++ b/src/mito2/src/read/merge.rs
@@ -223,6 +223,9 @@ impl BatchMerger {
         // rows by sequence for simplicity and performance reason.
         batch.filter_deleted()?;
 
+        // Reset merger.
+        self.is_sorted = true;
+
         Ok(Some(batch))
     }
 }
@@ -480,4 +483,49 @@ mod tests {
         )
         .await;
     }
+
+    #[test]
+    fn test_batch_merger_empty() {
+        let mut merger = BatchMerger::new();
+        assert!(merger.merge_batches().unwrap().is_none());
+    }
+
+    #[test]
+    fn test_batch_merger_unsorted() {
+        let mut merger = BatchMerger::new();
+        merger.push(new_batch(
+            b"k1",
+            &[1, 3, 5],
+            &[10, 10, 10],
+            &[OpType::Put, OpType::Put, OpType::Put],
+            &[21, 23, 25],
+        ));
+        assert!(merger.is_sorted);
+        merger.push(new_batch(
+            b"k1",
+            &[2, 4],
+            &[11, 11],
+            &[OpType::Put, OpType::Put],
+            &[22, 24],
+        ));
+        assert!(!merger.is_sorted);
+        let batch = merger.merge_batches().unwrap().unwrap();
+        assert_eq!(
+            batch,
+            new_batch(
+                b"k1",
+                &[1, 2, 3, 4, 5],
+                &[10, 11, 10, 11, 10],
+                &[
+                    OpType::Put,
+                    OpType::Put,
+                    OpType::Put,
+                    OpType::Put,
+                    OpType::Put
+                ],
+                &[21, 22, 23, 24, 25]
+            )
+        );
+        assert!(merger.is_sorted);
+    }
 }

From a3ff5328a8a4d857655f5010f9b731c85e252084 Mon Sep 17 00:00:00 2001
From: Yingwen <realevenyag@gmail.com>
Date: Sat, 23 Sep 2023 18:06:00 +0800
Subject: [PATCH 12/36] fix(mito): normalize region dir in RegionOpener (#2475)

fix: normalize region dir in RegionOpener
---
 src/mito2/src/region/opener.rs        | 11 +++--------
 src/mito2/src/worker/handle_create.rs |  2 +-
 src/mito2/src/worker/handle_open.rs   |  2 +-
 3 files changed, 5 insertions(+), 10 deletions(-)

diff --git a/src/mito2/src/region/opener.rs b/src/mito2/src/region/opener.rs
index 211cecb75047..3683d6fc2f73 100644
--- a/src/mito2/src/region/opener.rs
+++ b/src/mito2/src/region/opener.rs
@@ -21,7 +21,7 @@ use std::sync::Arc;
 use common_telemetry::{debug, error, info, warn};
 use common_time::util::current_time_millis;
 use futures::StreamExt;
-use object_store::util::join_dir;
+use object_store::util::{join_dir, normalize_dir};
 use object_store::ObjectStore;
 use snafu::{ensure, OptionExt};
 use store_api::logstore::LogStore;
@@ -57,6 +57,7 @@ impl RegionOpener {
     /// Returns a new opener.
     pub(crate) fn new(
         region_id: RegionId,
+        region_dir: &str,
         memtable_builder: MemtableBuilderRef,
         object_store: ObjectStore,
         scheduler: SchedulerRef,
@@ -66,7 +67,7 @@ impl RegionOpener {
             metadata: None,
             memtable_builder,
             object_store,
-            region_dir: String::new(),
+            region_dir: normalize_dir(region_dir),
             scheduler,
             options: HashMap::new(),
         }
@@ -78,12 +79,6 @@ impl RegionOpener {
         self
     }
 
-    /// Sets the region dir.
-    pub(crate) fn region_dir(mut self, value: &str) -> Self {
-        self.region_dir = value.to_string();
-        self
-    }
-
     /// Sets options for the region.
     pub(crate) fn options(mut self, value: HashMap<String, String>) -> Self {
         self.options = value;
diff --git a/src/mito2/src/worker/handle_create.rs b/src/mito2/src/worker/handle_create.rs
index 38154022a77b..848082956894 100644
--- a/src/mito2/src/worker/handle_create.rs
+++ b/src/mito2/src/worker/handle_create.rs
@@ -58,12 +58,12 @@ impl<S: LogStore> RegionWorkerLoop<S> {
         // Create a MitoRegion from the RegionMetadata.
         let region = RegionOpener::new(
             region_id,
+            &request.region_dir,
             self.memtable_builder.clone(),
             self.object_store.clone(),
             self.scheduler.clone(),
         )
         .metadata(metadata)
-        .region_dir(&request.region_dir)
         .options(request.options)
         .create_or_open(&self.config, &self.wal)
         .await?;
diff --git a/src/mito2/src/worker/handle_open.rs b/src/mito2/src/worker/handle_open.rs
index e6de10b66933..ab6967c0d1b6 100644
--- a/src/mito2/src/worker/handle_open.rs
+++ b/src/mito2/src/worker/handle_open.rs
@@ -57,11 +57,11 @@ impl<S: LogStore> RegionWorkerLoop<S> {
         // Open region from specific region dir.
         let region = RegionOpener::new(
             region_id,
+            &request.region_dir,
             self.memtable_builder.clone(),
             self.object_store.clone(),
             self.scheduler.clone(),
         )
-        .region_dir(&request.region_dir)
         .options(request.options)
         .open(&self.config, &self.wal)
         .await?;

From e4cbe5f305d71e397bcdc3f01860cf35a825d471 Mon Sep 17 00:00:00 2001
From: JeremyHi <jiachun_feng@proton.me>
Date: Sun, 24 Sep 2023 10:12:33 +0800
Subject: [PATCH 13/36] chore: refactor options (#2476)

---
 Cargo.lock                                  |   2 +
 config/frontend.example.toml                |   2 +-
 src/cmd/Cargo.toml                          |   2 +
 src/cmd/src/frontend.rs                     |   2 +-
 src/cmd/src/options.rs                      |  12 +-
 src/cmd/src/standalone.rs                   | 140 ++++++++++----------
 src/servers/tests/http/http_handler_test.rs |   2 +-
 7 files changed, 86 insertions(+), 76 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 349938a47461..1eaf5b4eaf36 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1612,12 +1612,14 @@ dependencies = [
  "datatypes",
  "either",
  "etcd-client",
+ "file-engine",
  "frontend",
  "futures",
  "lazy_static",
  "meta-client",
  "meta-srv",
  "metrics",
+ "mito2",
  "nu-ansi-term",
  "partition",
  "prost",
diff --git a/config/frontend.example.toml b/config/frontend.example.toml
index 42e6f377a061..c87dc8292c1d 100644
--- a/config/frontend.example.toml
+++ b/config/frontend.example.toml
@@ -8,7 +8,7 @@ interval_millis = 5000
 retry_interval_millis = 5000
 
 # HTTP server options, see `standalone.example.toml`.
-[http_options]
+[http]
 addr = "127.0.0.1:4000"
 timeout = "30s"
 body_limit = "64MB"
diff --git a/src/cmd/Cargo.toml b/src/cmd/Cargo.toml
index b07ca011b1e5..c83f65f7488e 100644
--- a/src/cmd/Cargo.toml
+++ b/src/cmd/Cargo.toml
@@ -38,12 +38,14 @@ datanode = { workspace = true }
 datatypes = { workspace = true }
 either = "1.8"
 etcd-client.workspace = true
+file-engine = { workspace = true }
 frontend = { workspace = true }
 futures.workspace = true
 lazy_static.workspace = true
 meta-client = { workspace = true }
 meta-srv = { workspace = true }
 metrics.workspace = true
+mito2 = { workspace = true }
 nu-ansi-term = "0.46"
 partition = { workspace = true }
 prost.workspace = true
diff --git a/src/cmd/src/frontend.rs b/src/cmd/src/frontend.rs
index 365f4ea9447d..61f0bab6a3d8 100644
--- a/src/cmd/src/frontend.rs
+++ b/src/cmd/src/frontend.rs
@@ -383,7 +383,7 @@ mod tests {
                     Some("11"),
                 ),
                 (
-                    // http_options.addr = 127.0.0.1:24000
+                    // http.addr = 127.0.0.1:24000
                     [
                         env_prefix.to_string(),
                         "http".to_uppercase(),
diff --git a/src/cmd/src/options.rs b/src/cmd/src/options.rs
index ffb65eb77fc3..caeef4960144 100644
--- a/src/cmd/src/options.rs
+++ b/src/cmd/src/options.rs
@@ -29,11 +29,11 @@ pub const ENV_LIST_SEP: &str = ",";
 /// Options mixed up from datanode, frontend and metasrv.
 pub struct MixOptions {
     pub data_home: String,
-    pub procedure_cfg: ProcedureConfig,
-    pub kv_store_cfg: KvStoreConfig,
-    pub fe_opts: FrontendOptions,
-    pub dn_opts: DatanodeOptions,
-    pub logging_opts: LoggingOptions,
+    pub procedure: ProcedureConfig,
+    pub kv_store: KvStoreConfig,
+    pub frontend: FrontendOptions,
+    pub datanode: DatanodeOptions,
+    pub logging: LoggingOptions,
 }
 
 pub enum Options {
@@ -56,7 +56,7 @@ impl Options {
             Options::Datanode(opts) => &opts.logging,
             Options::Frontend(opts) => &opts.logging,
             Options::Metasrv(opts) => &opts.logging,
-            Options::Standalone(opts) => &opts.logging_opts,
+            Options::Standalone(opts) => &opts.logging,
             Options::Cli(opts) => opts,
         }
     }
diff --git a/src/cmd/src/standalone.rs b/src/cmd/src/standalone.rs
index b47e50c847c1..73c298b4b41e 100644
--- a/src/cmd/src/standalone.rs
+++ b/src/cmd/src/standalone.rs
@@ -24,14 +24,16 @@ use common_meta::kv_backend::KvBackendRef;
 use common_procedure::ProcedureManagerRef;
 use common_telemetry::info;
 use common_telemetry::logging::LoggingOptions;
-use datanode::config::{DatanodeOptions, ProcedureConfig, StorageConfig};
+use datanode::config::{DatanodeOptions, ProcedureConfig, RegionEngineConfig, StorageConfig};
 use datanode::datanode::{Datanode, DatanodeBuilder};
 use datanode::region_server::RegionServer;
+use file_engine::config::EngineConfig as FileEngineConfig;
 use frontend::frontend::FrontendOptions;
 use frontend::instance::{FrontendInstance, Instance as FeInstance, StandaloneDatanodeManager};
 use frontend::service_config::{
     GrpcOptions, InfluxdbOptions, MysqlOptions, OpentsdbOptions, PostgresOptions, PromStoreOptions,
 };
+use mito2::config::MitoConfig;
 use serde::{Deserialize, Serialize};
 use servers::http::HttpOptions;
 use servers::tls::{TlsMode, TlsOption};
@@ -85,18 +87,20 @@ impl SubCommand {
 pub struct StandaloneOptions {
     pub mode: Mode,
     pub enable_telemetry: bool,
-    pub http_options: HttpOptions,
-    pub grpc_options: GrpcOptions,
-    pub mysql_options: MysqlOptions,
-    pub postgres_options: PostgresOptions,
-    pub opentsdb_options: OpentsdbOptions,
-    pub influxdb_options: InfluxdbOptions,
-    pub prom_store_options: PromStoreOptions,
+    pub http: HttpOptions,
+    pub grpc: GrpcOptions,
+    pub mysql: MysqlOptions,
+    pub postgres: PostgresOptions,
+    pub opentsdb: OpentsdbOptions,
+    pub influxdb: InfluxdbOptions,
+    pub prom_store: PromStoreOptions,
     pub wal: WalConfig,
     pub storage: StorageConfig,
     pub kv_store: KvStoreConfig,
     pub procedure: ProcedureConfig,
     pub logging: LoggingOptions,
+    /// Options for different store engines.
+    pub region_engine: Vec<RegionEngineConfig>,
 }
 
 impl Default for StandaloneOptions {
@@ -104,18 +108,22 @@ impl Default for StandaloneOptions {
         Self {
             mode: Mode::Standalone,
             enable_telemetry: true,
-            http_options: HttpOptions::default(),
-            grpc_options: GrpcOptions::default(),
-            mysql_options: MysqlOptions::default(),
-            postgres_options: PostgresOptions::default(),
-            opentsdb_options: OpentsdbOptions::default(),
-            influxdb_options: InfluxdbOptions::default(),
-            prom_store_options: PromStoreOptions::default(),
+            http: HttpOptions::default(),
+            grpc: GrpcOptions::default(),
+            mysql: MysqlOptions::default(),
+            postgres: PostgresOptions::default(),
+            opentsdb: OpentsdbOptions::default(),
+            influxdb: InfluxdbOptions::default(),
+            prom_store: PromStoreOptions::default(),
             wal: WalConfig::default(),
             storage: StorageConfig::default(),
             kv_store: KvStoreConfig::default(),
             procedure: ProcedureConfig::default(),
             logging: LoggingOptions::default(),
+            region_engine: vec![
+                RegionEngineConfig::Mito(MitoConfig::default()),
+                RegionEngineConfig::File(FileEngineConfig::default()),
+            ],
         }
     }
 }
@@ -124,13 +132,13 @@ impl StandaloneOptions {
     fn frontend_options(self) -> FrontendOptions {
         FrontendOptions {
             mode: self.mode,
-            http: self.http_options,
-            grpc: self.grpc_options,
-            mysql: self.mysql_options,
-            postgres: self.postgres_options,
-            opentsdb: self.opentsdb_options,
-            influxdb: self.influxdb_options,
-            prom_store: self.prom_store_options,
+            http: self.http,
+            grpc: self.grpc,
+            mysql: self.mysql,
+            postgres: self.postgres,
+            opentsdb: self.opentsdb,
+            influxdb: self.influxdb,
+            prom_store: self.prom_store,
             meta_client: None,
             logging: self.logging,
             ..Default::default()
@@ -143,6 +151,7 @@ impl StandaloneOptions {
             enable_telemetry: self.enable_telemetry,
             wal: self.wal,
             storage: self.storage,
+            region_engine: self.region_engine,
             ..Default::default()
         }
     }
@@ -232,7 +241,7 @@ impl StartCommand {
         );
 
         if let Some(addr) = &self.http_addr {
-            opts.http_options.addr = addr.clone()
+            opts.http.addr = addr.clone()
         }
 
         if let Some(addr) = &self.rpc_addr {
@@ -246,42 +255,42 @@ impl StartCommand {
                 }
                 .fail();
             }
-            opts.grpc_options.addr = addr.clone()
+            opts.grpc.addr = addr.clone()
         }
 
         if let Some(addr) = &self.mysql_addr {
-            opts.mysql_options.enable = true;
-            opts.mysql_options.addr = addr.clone();
-            opts.mysql_options.tls = tls_opts.clone();
+            opts.mysql.enable = true;
+            opts.mysql.addr = addr.clone();
+            opts.mysql.tls = tls_opts.clone();
         }
 
         if let Some(addr) = &self.postgres_addr {
-            opts.postgres_options.enable = true;
-            opts.postgres_options.addr = addr.clone();
-            opts.postgres_options.tls = tls_opts;
+            opts.postgres.enable = true;
+            opts.postgres.addr = addr.clone();
+            opts.postgres.tls = tls_opts;
         }
 
         if let Some(addr) = &self.opentsdb_addr {
-            opts.opentsdb_options.enable = true;
-            opts.opentsdb_options.addr = addr.clone();
+            opts.opentsdb.enable = true;
+            opts.opentsdb.addr = addr.clone();
         }
 
         if self.influxdb_enable {
-            opts.influxdb_options.enable = self.influxdb_enable;
+            opts.influxdb.enable = self.influxdb_enable;
         }
-        let kv_store_cfg = opts.kv_store.clone();
-        let procedure_cfg = opts.procedure.clone();
-        let fe_opts = opts.clone().frontend_options();
-        let logging_opts = opts.logging.clone();
-        let dn_opts = opts.datanode_options();
+        let kv_store = opts.kv_store.clone();
+        let procedure = opts.procedure.clone();
+        let frontend = opts.clone().frontend_options();
+        let logging = opts.logging.clone();
+        let datanode = opts.datanode_options();
 
         Ok(Options::Standalone(Box::new(MixOptions {
-            procedure_cfg,
-            kv_store_cfg,
-            data_home: dn_opts.storage.data_home.to_string(),
-            fe_opts,
-            dn_opts,
-            logging_opts,
+            procedure,
+            kv_store,
+            data_home: datanode.storage.data_home.to_string(),
+            frontend,
+            datanode,
+            logging,
         })))
     }
 
@@ -290,8 +299,8 @@ impl StartCommand {
     #[allow(clippy::diverging_sub_expression)]
     async fn build(self, opts: MixOptions) -> Result<Instance> {
         let plugins = Arc::new(load_frontend_plugins(&self.user_provider)?);
-        let fe_opts = opts.fe_opts;
-        let dn_opts = opts.dn_opts;
+        let fe_opts = opts.frontend;
+        let dn_opts = opts.datanode;
 
         info!("Standalone start command: {:#?}", self);
         info!(
@@ -300,13 +309,10 @@ impl StartCommand {
         );
 
         let kv_dir = kv_store_dir(&opts.data_home);
-        let (kv_store, procedure_manager) = FeInstance::try_build_standalone_components(
-            kv_dir,
-            opts.kv_store_cfg,
-            opts.procedure_cfg,
-        )
-        .await
-        .context(StartFrontendSnafu)?;
+        let (kv_store, procedure_manager) =
+            FeInstance::try_build_standalone_components(kv_dir, opts.kv_store, opts.procedure)
+                .await
+                .context(StartFrontendSnafu)?;
 
         let datanode =
             DatanodeBuilder::new(dn_opts.clone(), Some(kv_store.clone()), plugins.clone())
@@ -436,9 +442,9 @@ mod tests {
             checkpoint_margin = 9
             gc_duration = '7s'
 
-            [http_options]
+            [http]
             addr = "127.0.0.1:4000"
-            timeout = "30s"
+            timeout = "33s"
             body_limit = "128MB"
 
             [logging]
@@ -456,12 +462,12 @@ mod tests {
         else {
             unreachable!()
         };
-        let fe_opts = options.fe_opts;
-        let dn_opts = options.dn_opts;
-        let logging_opts = options.logging_opts;
+        let fe_opts = options.frontend;
+        let dn_opts = options.datanode;
+        let logging_opts = options.logging;
         assert_eq!(Mode::Standalone, fe_opts.mode);
         assert_eq!("127.0.0.1:4000".to_string(), fe_opts.http.addr);
-        assert_eq!(Duration::from_secs(30), fe_opts.http.timeout);
+        assert_eq!(Duration::from_secs(33), fe_opts.http.timeout);
         assert_eq!(ReadableSize::mb(128), fe_opts.http.body_limit);
         assert_eq!("127.0.0.1:4001".to_string(), fe_opts.grpc.addr);
         assert!(fe_opts.mysql.enable);
@@ -503,8 +509,8 @@ mod tests {
             unreachable!()
         };
 
-        assert_eq!("/tmp/greptimedb/test/logs", opts.logging_opts.dir);
-        assert_eq!("debug", opts.logging_opts.level.unwrap());
+        assert_eq!("/tmp/greptimedb/test/logs", opts.logging.dir);
+        assert_eq!("debug", opts.logging.level.unwrap());
     }
 
     #[test]
@@ -548,7 +554,7 @@ mod tests {
                     // http.addr = 127.0.0.1:24000
                     [
                         env_prefix.to_string(),
-                        "http_options".to_uppercase(),
+                        "http".to_uppercase(),
                         "addr".to_uppercase(),
                     ]
                     .join(ENV_VAR_SEP),
@@ -573,17 +579,17 @@ mod tests {
                 };
 
                 // Should be read from env, env > default values.
-                assert_eq!(opts.logging_opts.dir, "/other/log/dir");
+                assert_eq!(opts.logging.dir, "/other/log/dir");
 
                 // Should be read from config file, config file > env > default values.
-                assert_eq!(opts.logging_opts.level.as_ref().unwrap(), "debug");
+                assert_eq!(opts.logging.level.as_ref().unwrap(), "debug");
 
                 // Should be read from cli, cli > config file > env > default values.
-                assert_eq!(opts.fe_opts.http.addr, "127.0.0.1:14000");
-                assert_eq!(ReadableSize::mb(64), opts.fe_opts.http.body_limit);
+                assert_eq!(opts.frontend.http.addr, "127.0.0.1:14000");
+                assert_eq!(ReadableSize::mb(64), opts.frontend.http.body_limit);
 
                 // Should be default value.
-                assert_eq!(opts.fe_opts.grpc.addr, GrpcOptions::default().addr);
+                assert_eq!(opts.frontend.grpc.addr, GrpcOptions::default().addr);
             },
         );
     }
diff --git a/src/servers/tests/http/http_handler_test.rs b/src/servers/tests/http/http_handler_test.rs
index f8d5a567f05b..995e911acaad 100644
--- a/src/servers/tests/http/http_handler_test.rs
+++ b/src/servers/tests/http/http_handler_test.rs
@@ -401,7 +401,7 @@ async fn test_config() {
     let toml_str = r#"
             mode = "distributed"
 
-            [http_options]
+            [http]
             addr = "127.0.0.1:4000"
             timeout = "30s"
             body_limit = "2GB"

From e05184118174a8efe59e48d3c71129999e746ef4 Mon Sep 17 00:00:00 2001
From: zyy17 <zyylsxm@gmail.com>
Date: Sun, 24 Sep 2023 14:47:14 +0800
Subject: [PATCH 14/36] chore: fix typo (#2477)

---
 ...tabie-trait-refactor.md => 2023-08-04-table-trait-refactor.md} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename docs/rfcs/{2023-08-04-tabie-trait-refactor.md => 2023-08-04-table-trait-refactor.md} (100%)

diff --git a/docs/rfcs/2023-08-04-tabie-trait-refactor.md b/docs/rfcs/2023-08-04-table-trait-refactor.md
similarity index 100%
rename from docs/rfcs/2023-08-04-tabie-trait-refactor.md
rename to docs/rfcs/2023-08-04-table-trait-refactor.md

From 198fcbaa690e30598536cba418fe34e3da546931 Mon Sep 17 00:00:00 2001
From: Niwaka <61189782+NiwakaDev@users.noreply.github.com>
Date: Mon, 25 Sep 2023 15:23:16 +0900
Subject: [PATCH 15/36] chore: add integration test for issue2437 (#2481)

---
 .../common/alter/add_incorrect_col.result     | 25 +++++++++++++++++++
 .../common/alter/add_incorrect_col.sql        |  9 +++++++
 2 files changed, 34 insertions(+)
 create mode 100644 tests/cases/standalone/common/alter/add_incorrect_col.result
 create mode 100644 tests/cases/standalone/common/alter/add_incorrect_col.sql

diff --git a/tests/cases/standalone/common/alter/add_incorrect_col.result b/tests/cases/standalone/common/alter/add_incorrect_col.result
new file mode 100644
index 000000000000..1f8d90b575f3
--- /dev/null
+++ b/tests/cases/standalone/common/alter/add_incorrect_col.result
@@ -0,0 +1,25 @@
+CREATE TABLE table_should_not_break_after_incorrect_alter(i INTEGER, j TIMESTAMP TIME INDEX);
+
+Affected Rows: 0
+
+ALTER TABLE table_should_not_break_after_incorrect_alter ADD column k string NOT NULL;
+
+Error: 1004(InvalidArguments), Invalid alter table(table_should_not_break_after_incorrect_alter) request: no default value for column k
+
+INSERT INTO table_should_not_break_after_incorrect_alter VALUES (1, 1), (2, 2);
+
+Affected Rows: 2
+
+SELECT * FROM table_should_not_break_after_incorrect_alter;
+
++---+-------------------------+
+| i | j                       |
++---+-------------------------+
+| 1 | 1970-01-01T00:00:00.001 |
+| 2 | 1970-01-01T00:00:00.002 |
++---+-------------------------+
+
+DROP TABLE table_should_not_break_after_incorrect_alter;
+
+Affected Rows: 1
+
diff --git a/tests/cases/standalone/common/alter/add_incorrect_col.sql b/tests/cases/standalone/common/alter/add_incorrect_col.sql
new file mode 100644
index 000000000000..677c85eb1232
--- /dev/null
+++ b/tests/cases/standalone/common/alter/add_incorrect_col.sql
@@ -0,0 +1,9 @@
+CREATE TABLE table_should_not_break_after_incorrect_alter(i INTEGER, j TIMESTAMP TIME INDEX);
+
+ALTER TABLE table_should_not_break_after_incorrect_alter ADD column k string NOT NULL;
+
+INSERT INTO table_should_not_break_after_incorrect_alter VALUES (1, 1), (2, 2);
+
+SELECT * FROM table_should_not_break_after_incorrect_alter;
+
+DROP TABLE table_should_not_break_after_incorrect_alter;

From c6c8c2a413fe178502503dbc2489fa80f06da860 Mon Sep 17 00:00:00 2001
From: Lilit0x <baasisek01@gmail.com>
Date: Mon, 25 Sep 2023 21:55:25 +0100
Subject: [PATCH 16/36] fix: changed tags to string type

---
 src/servers/src/opentsdb.rs | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/src/servers/src/opentsdb.rs b/src/servers/src/opentsdb.rs
index a53f3bc89468..ad054611204d 100644
--- a/src/servers/src/opentsdb.rs
+++ b/src/servers/src/opentsdb.rs
@@ -132,7 +132,7 @@ pub fn data_point_to_grpc_row_insert_requests(
 ) -> Result<(RowInsertRequests, usize)> {
     let mut multi_table_data = MultiTableData::new();
     let table_name = data_point.metric();
-    let tags = data_point.tags();
+    let tags = data_point.tags().clone();
     let value = data_point.value();
     let timestamp = data_point.ts_millis();
     let num_columns = tags.len() + 1;
@@ -141,8 +141,7 @@ pub fn data_point_to_grpc_row_insert_requests(
     let mut one_row = table_data.alloc_one_row();
 
     //tags
-    let kvs = tags.iter().map(|(k, v)| (k.as_str(), v.as_str()));
-    row_writer::write_tags(table_data, kvs, &mut one_row)?;
+    row_writer::write_tags(table_data, tags.into_iter(), &mut one_row)?;
 
     // value
     row_writer::write_f64(table_data, FIELD_COLUMN_NAME, value, &mut one_row)?;

From 87589e772bd98e0b727dac150a473dc97e12be6b Mon Sep 17 00:00:00 2001
From: LFC <990479+MichaelScofield@users.noreply.github.com>
Date: Mon, 25 Sep 2023 16:44:49 +0800
Subject: [PATCH 17/36] fix: print root cause error message to user facing
 interface (#2486)

---
 src/servers/src/http.rs             | 4 ++--
 src/servers/src/mysql/writer.rs     | 4 +++-
 src/servers/src/postgres/handler.rs | 3 ++-
 3 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/src/servers/src/http.rs b/src/servers/src/http.rs
index a86c3133e0e5..e599c724dcbe 100644
--- a/src/servers/src/http.rs
+++ b/src/servers/src/http.rs
@@ -52,7 +52,7 @@ use futures::FutureExt;
 use schemars::JsonSchema;
 use serde::{Deserialize, Serialize};
 use serde_json::Value;
-use snafu::{ensure, ResultExt};
+use snafu::{ensure, ErrorCompat, ResultExt};
 use tokio::sync::oneshot::{self, Sender};
 use tokio::sync::Mutex;
 use tower::timeout::TimeoutLayer;
@@ -315,7 +315,7 @@ impl JsonResponse {
                 },
                 Err(e) => {
                     return Self::with_error(
-                        format!("Query engine output error: {e}"),
+                        e.iter_chain().last().unwrap().to_string(),
                         e.status_code(),
                     );
                 }
diff --git a/src/servers/src/mysql/writer.rs b/src/servers/src/mysql/writer.rs
index d5cf69703f6d..bfbb689aa869 100644
--- a/src/servers/src/mysql/writer.rs
+++ b/src/servers/src/mysql/writer.rs
@@ -26,6 +26,7 @@ use opensrv_mysql::{
 };
 use session::context::QueryContextRef;
 use snafu::prelude::*;
+use snafu::ErrorCompat;
 use tokio::io::AsyncWrite;
 
 use crate::error::{self, Error, OtherSnafu, Result};
@@ -211,7 +212,8 @@ impl<'a, W: AsyncWrite + Unpin> MysqlResultWriter<'a, W> {
         );
 
         let kind = ErrorKind::ER_INTERNAL_ERROR;
-        w.error(kind, error.to_string().as_bytes()).await?;
+        let error = error.iter_chain().last().unwrap().to_string();
+        w.error(kind, error.as_bytes()).await?;
         Ok(())
     }
 }
diff --git a/src/servers/src/postgres/handler.rs b/src/servers/src/postgres/handler.rs
index 536ca604173f..f11900133dd9 100644
--- a/src/servers/src/postgres/handler.rs
+++ b/src/servers/src/postgres/handler.rs
@@ -31,6 +31,7 @@ use pgwire::api::{ClientInfo, Type};
 use pgwire::error::{ErrorInfo, PgWireError, PgWireResult};
 use query::query_engine::DescribeResult;
 use session::Session;
+use snafu::ErrorCompat;
 use sql::dialect::PostgreSqlDialect;
 use sql::parser::ParserContext;
 
@@ -90,7 +91,7 @@ fn output_to_query_response<'a>(
         Err(e) => Ok(Response::Error(Box::new(ErrorInfo::new(
             "ERROR".to_string(),
             "XX000".to_string(),
-            e.to_string(),
+            e.iter_chain().last().unwrap().to_string(),
         )))),
     }
 }

From 958adf7dee05a06e276ebab0b715a63d9f3cd978 Mon Sep 17 00:00:00 2001
From: Yingwen <realevenyag@gmail.com>
Date: Mon, 25 Sep 2023 17:20:00 +0800
Subject: [PATCH 18/36] refactor(mito): remove #[allow(dead_code)] (#2479)

---
 src/mito2/src/compaction/output.rs    |   4 -
 src/mito2/src/compaction/picker.rs    |  16 ---
 src/mito2/src/compaction/twcs.rs      | 183 +-------------------------
 src/mito2/src/flush.rs                |  10 +-
 src/mito2/src/lib.rs                  |   9 +-
 src/mito2/src/memtable.rs             |   8 +-
 src/mito2/src/memtable/time_series.rs |  36 -----
 src/mito2/src/request.rs              |   7 +-
 src/mito2/src/sst/parquet/reader.rs   |   4 +-
 src/mito2/src/sst/stream_writer.rs    |   2 -
 10 files changed, 23 insertions(+), 256 deletions(-)

diff --git a/src/mito2/src/compaction/output.rs b/src/mito2/src/compaction/output.rs
index 96d95874b758..bedbcb741729 100644
--- a/src/mito2/src/compaction/output.rs
+++ b/src/mito2/src/compaction/output.rs
@@ -29,10 +29,6 @@ pub(crate) struct CompactionOutput {
     pub output_file_id: FileId,
     /// Compaction output file level.
     pub output_level: Level,
-    /// The left bound of time window.
-    pub time_window_bound: i64,
-    /// Time window size in seconds.
-    pub time_window_sec: i64,
     /// Compaction input files.
     pub inputs: Vec<FileHandle>,
 }
diff --git a/src/mito2/src/compaction/picker.rs b/src/mito2/src/compaction/picker.rs
index 1e986be0a673..d97229e6ac21 100644
--- a/src/mito2/src/compaction/picker.rs
+++ b/src/mito2/src/compaction/picker.rs
@@ -29,19 +29,3 @@ pub trait CompactionTask: Debug + Send + Sync + 'static {
 pub trait Picker: Debug + Send + 'static {
     fn pick(&self, req: CompactionRequest) -> Option<Box<dyn CompactionTask>>;
 }
-
-pub struct PickerContext {
-    compaction_time_window: Option<i64>,
-}
-
-impl PickerContext {
-    pub fn with(compaction_time_window: Option<i64>) -> Self {
-        Self {
-            compaction_time_window,
-        }
-    }
-
-    pub fn compaction_time_window(&self) -> Option<i64> {
-        self.compaction_time_window
-    }
-}
diff --git a/src/mito2/src/compaction/twcs.rs b/src/mito2/src/compaction/twcs.rs
index f840d6aeb2f9..19b9085eba67 100644
--- a/src/mito2/src/compaction/twcs.rs
+++ b/src/mito2/src/compaction/twcs.rs
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::collections::{BTreeMap, HashMap};
+use std::collections::BTreeMap;
 use std::fmt::{Debug, Formatter};
 use std::sync::Arc;
 use std::time::Duration;
@@ -80,7 +80,6 @@ impl TwcsPicker {
         &self,
         time_windows: &BTreeMap<i64, Vec<FileHandle>>,
         active_window: Option<i64>,
-        window_size: i64,
     ) -> Vec<CompactionOutput> {
         let mut output = vec![];
         for (window, files) in time_windows {
@@ -89,8 +88,6 @@ impl TwcsPicker {
                     output.push(CompactionOutput {
                         output_file_id: FileId::random(),
                         output_level: 1, // we only have two levels and always compact to l1 
-                        time_window_bound: *window,
-                        time_window_sec: window_size,
                         inputs: files.clone(),
                     });
                 } else {
@@ -102,8 +99,6 @@ impl TwcsPicker {
                     output.push(CompactionOutput {
                         output_file_id: FileId::random(),
                         output_level: 1,
-                        time_window_bound: *window,
-                        time_window_sec: window_size,
                         inputs: files.clone(),
                     });
                 } else {
@@ -155,7 +150,7 @@ impl Picker for TwcsPicker {
         let active_window = find_latest_window_in_seconds(levels[0].files(), time_window_size);
         // Assign files to windows
         let windows = assign_to_windows(levels.iter().flat_map(LevelMeta::files), time_window_size);
-        let outputs = self.build_output(&windows, active_window, time_window_size);
+        let outputs = self.build_output(&windows, active_window);
 
         if outputs.is_empty() && expired_ssts.is_empty() {
             // Nothing to compact, we are done. Notifies all waiters as we consume the compaction request.
@@ -407,54 +402,6 @@ pub(crate) fn infer_time_bucket<'a>(files: impl Iterator<Item = &'a FileHandle>)
         .unwrap_or_else(|| TIME_BUCKETS.max()) // safety: TIME_BUCKETS cannot be empty.
 }
 
-/// Finds files that can be compacted in given level.
-/// Currently they're files that is not currently under compaction.
-#[inline]
-fn find_compactable_files(level: &LevelMeta) -> Vec<FileHandle> {
-    level.files().filter(|f| !f.compacting()).cloned().collect()
-}
-
-/// Calculates timestamp span between start and end timestamp.
-fn file_time_bucket_span(start_sec: i64, end_sec: i64, bucket_sec: i64) -> Vec<i64> {
-    assert!(start_sec <= end_sec);
-
-    // if timestamp is between `[i64::MIN, i64::MIN.align_by_bucket(bucket)]`, which cannot
-    // be aligned to a valid i64 bound, simply return `i64::MIN` rather than just underflow.
-    let mut start_aligned = start_sec.align_by_bucket(bucket_sec).unwrap_or(i64::MIN);
-    let end_aligned = end_sec.align_by_bucket(bucket_sec).unwrap_or(i64::MIN);
-
-    let mut res = Vec::with_capacity(((end_aligned - start_aligned) / bucket_sec + 1) as usize);
-    while start_aligned < end_aligned {
-        res.push(start_aligned);
-        start_aligned += bucket_sec;
-    }
-    res.push(end_aligned);
-    res
-}
-
-/// Calculates buckets for files. If file does not contain a time range in metadata, it will be
-/// assigned to a special bucket `i64::MAX` (normally no timestamp can be aligned to this bucket)
-/// so that all files without timestamp can be compacted together.
-fn calculate_time_buckets(bucket_sec: i64, files: &[FileHandle]) -> HashMap<i64, Vec<FileHandle>> {
-    let mut buckets = HashMap::new();
-
-    for file in files {
-        let (start, end) = file.time_range();
-        let bounds = file_time_bucket_span(
-            start.convert_to(TimeUnit::Second).unwrap().value(),
-            end.convert_to(TimeUnit::Second).unwrap().value(),
-            bucket_sec,
-        );
-        for bound in bounds {
-            buckets
-                .entry(bound)
-                .or_insert_with(Vec::new)
-                .push(file.clone());
-        }
-    }
-    buckets
-}
-
 pub(crate) struct TimeBuckets([i64; 7]);
 
 impl TimeBuckets {
@@ -603,20 +550,14 @@ mod tests {
             let windows = assign_to_windows(self.input_files.iter(), self.window_size);
             let active_window =
                 find_latest_window_in_seconds(self.input_files.iter(), self.window_size);
-            let output =
-                TwcsPicker::new(4, 1, None).build_output(&windows, active_window, self.window_size);
+            let output = TwcsPicker::new(4, 1, None).build_output(&windows, active_window);
 
             let output = output
                 .iter()
                 .map(|o| {
                     let input_file_ids =
                         o.inputs.iter().map(|f| f.file_id()).collect::<HashSet<_>>();
-                    (
-                        input_file_ids,
-                        o.output_level,
-                        o.time_window_sec,
-                        o.time_window_bound,
-                    )
+                    (input_file_ids, o.output_level)
                 })
                 .collect::<Vec<_>>();
 
@@ -629,12 +570,7 @@ mod tests {
                         .iter()
                         .map(|idx| self.input_files[*idx].file_id())
                         .collect::<HashSet<_>>();
-                    (
-                        input_file_ids,
-                        o.output_level,
-                        o.time_window_sec,
-                        o.time_window_bound,
-                    )
+                    (input_file_ids, o.output_level)
                 })
                 .collect::<Vec<_>>();
             assert_eq!(expected, output);
@@ -644,9 +580,6 @@ mod tests {
     struct ExpectedOutput {
         input_files: Vec<usize>,
         output_level: Level,
-        time_window_sec: i64,
-        time_window_bound: i64,
-        strict_window: bool,
     }
 
     #[test]
@@ -665,9 +598,6 @@ mod tests {
             expected_outputs: vec![ExpectedOutput {
                 input_files: vec![0, 1],
                 output_level: 1,
-                time_window_sec: 3,
-                time_window_bound: 0,
-                strict_window: false,
             }],
         }
         .check();
@@ -688,16 +618,10 @@ mod tests {
                 ExpectedOutput {
                     input_files: vec![0, 1],
                     output_level: 1,
-                    time_window_sec: 3,
-                    time_window_bound: 0,
-                    strict_window: false,
                 },
                 ExpectedOutput {
                     input_files: vec![2, 3, 4],
                     output_level: 1,
-                    time_window_sec: 3,
-                    time_window_bound: 3,
-                    strict_window: false,
                 },
             ],
         }
@@ -742,102 +666,5 @@ mod tests {
         );
     }
 
-    fn check_bucket_calculation(
-        bucket_sec: i64,
-        files: Vec<FileHandle>,
-        expected: &[(i64, &[FileId])],
-    ) {
-        let res = calculate_time_buckets(bucket_sec, &files);
-
-        let expected = expected
-            .iter()
-            .map(|(bucket, file_ids)| (*bucket, file_ids.iter().copied().collect::<HashSet<_>>()))
-            .collect::<HashMap<_, _>>();
-
-        for (bucket, file_ids) in expected {
-            let actual = res
-                .get(&bucket)
-                .unwrap()
-                .iter()
-                .map(|f| f.file_id())
-                .collect();
-            assert_eq!(
-                file_ids, actual,
-                "bucket: {bucket}, expected: {file_ids:?}, actual: {actual:?}",
-            );
-        }
-    }
-
-    #[test]
-    fn test_calculate_time_buckets() {
-        let file_id_a = FileId::random();
-        let file_id_b = FileId::random();
-        // simple case, files with disjoint
-        check_bucket_calculation(
-            10,
-            new_file_handles(&[(file_id_a, 0, 9000), (file_id_b, 10000, 19000)]),
-            &[(0, &[file_id_a]), (10, &[file_id_b])],
-        );
-
-        // files across buckets
-        check_bucket_calculation(
-            10,
-            new_file_handles(&[(file_id_a, 0, 10001), (file_id_b, 10000, 19000)]),
-            &[(0, &[file_id_a]), (10, &[file_id_a, file_id_b])],
-        );
-        check_bucket_calculation(
-            10,
-            new_file_handles(&[(file_id_a, 0, 10000)]),
-            &[(0, &[file_id_a]), (10, &[file_id_a])],
-        );
-
-        // file with an large time range
-        let file_id_array = &[file_id_a];
-        let expected = (0..(TIME_BUCKETS.get(4) / TIME_BUCKETS.get(0)))
-            .map(|b| (b * TIME_BUCKETS.get(0), file_id_array as _))
-            .collect::<Vec<_>>();
-        check_bucket_calculation(
-            TIME_BUCKETS.get(0),
-            new_file_handles(&[(file_id_a, 0, TIME_BUCKETS.get(4) * 1000)]),
-            &expected,
-        );
-    }
-
-    #[test]
-    fn test_time_bucket_span() {
-        assert_eq!(vec![0], file_time_bucket_span(1, 9, 10));
-        assert_eq!(vec![0, 10], file_time_bucket_span(1, 10, 10));
-        assert_eq!(vec![-10], file_time_bucket_span(-10, -1, 10));
-        assert_eq!(vec![-10, 0], file_time_bucket_span(-10, 0, 10));
-    }
-
-    #[test]
-    fn test_time_bucket_span_large() {
-        assert_eq!(
-            vec![
-                (i64::MAX - 10).align_by_bucket(10).unwrap(),
-                i64::MAX.align_by_bucket(10).unwrap(),
-            ],
-            file_time_bucket_span(i64::MAX - 10, i64::MAX, 10)
-        );
-
-        for bucket in 1..100 {
-            assert_eq!(
-                vec![
-                    i64::MIN,
-                    (i64::MIN + bucket).align_by_bucket(bucket).unwrap()
-                ],
-                file_time_bucket_span(i64::MIN, i64::MIN + bucket, bucket)
-            );
-        }
-    }
-
-    fn new_file_handles(input: &[(FileId, i64, i64)]) -> Vec<FileHandle> {
-        input
-            .iter()
-            .map(|(file_id, start, end)| new_file_handle(*file_id, *start, *end, 0))
-            .collect()
-    }
-
     // TODO(hl): TTL tester that checks if get_expired_ssts function works as expected.
 }
diff --git a/src/mito2/src/flush.rs b/src/mito2/src/flush.rs
index 73c7d54b9a78..ff7ee5051e31 100644
--- a/src/mito2/src/flush.rs
+++ b/src/mito2/src/flush.rs
@@ -22,6 +22,7 @@ use common_query::Output;
 use common_telemetry::{error, info};
 use snafu::ResultExt;
 use store_api::storage::RegionId;
+use strum::AsRefStr;
 use tokio::sync::mpsc;
 
 use crate::access_layer::AccessLayerRef;
@@ -97,7 +98,7 @@ impl WriteBufferManagerImpl {
     }
 
     /// Returns memory usage of mutable memtables.
-    pub(crate) fn mutable_usage(&self) -> usize {
+    pub fn mutable_usage(&self) -> usize {
         self.memory_active.load(Ordering::Relaxed)
     }
 
@@ -162,6 +163,7 @@ impl WriteBufferManager for WriteBufferManagerImpl {
 }
 
 /// Reason of a flush task.
+#[derive(Debug, AsRefStr)]
 pub enum FlushReason {
     /// Other reasons.
     Others,
@@ -302,8 +304,10 @@ impl RegionFlushTask {
 
         let file_ids: Vec<_> = file_metas.iter().map(|f| f.file_id).collect();
         info!(
-            "Successfully flush memtables, region: {}, files: {:?}",
-            version.metadata.region_id, file_ids
+            "Successfully flush memtables, region: {}, reason: {}, files: {:?}",
+            version.metadata.region_id,
+            self.reason.as_ref(),
+            file_ids
         );
 
         Ok(file_metas)
diff --git a/src/mito2/src/lib.rs b/src/mito2/src/lib.rs
index 2feb69676ce8..71d67983f091 100644
--- a/src/mito2/src/lib.rs
+++ b/src/mito2/src/lib.rs
@@ -21,28 +21,21 @@
 #[cfg(any(test, feature = "test"))]
 pub mod test_util;
 
-// TODO(yingwen): Remove all `allow(dead_code)` after finish refactoring mito.
 mod access_layer;
-#[allow(dead_code)]
 mod compaction;
 pub mod config;
 pub mod engine;
 pub mod error;
-#[allow(dead_code)]
-mod flush;
+pub mod flush;
 pub mod manifest;
-#[allow(dead_code)]
 pub mod memtable;
 mod metrics;
-#[allow(dead_code)]
 pub mod read;
 pub mod region;
 mod region_write_ctx;
-#[allow(dead_code)]
 pub mod request;
 mod row_converter;
 pub(crate) mod schedule;
-#[allow(dead_code)]
 pub mod sst;
 pub mod wal;
 mod worker;
diff --git a/src/mito2/src/memtable.rs b/src/mito2/src/memtable.rs
index d9c2f2e40b0c..bb82fb60d85d 100644
--- a/src/mito2/src/memtable.rs
+++ b/src/mito2/src/memtable.rs
@@ -42,16 +42,22 @@ pub type MemtableId = u32;
 
 #[derive(Debug, Default)]
 pub struct MemtableStats {
-    /// The  estimated bytes allocated by this memtable from heap.
+    /// The estimated bytes allocated by this memtable from heap.
     estimated_bytes: usize,
     /// The time range that this memtable contains.
     time_range: Option<(Timestamp, Timestamp)>,
 }
 
 impl MemtableStats {
+    /// Returns the estimated bytes allocated by this memtable.
     pub fn bytes_allocated(&self) -> usize {
         self.estimated_bytes
     }
+
+    /// Returns the time range of the memtable.
+    pub fn time_range(&self) -> Option<(Timestamp, Timestamp)> {
+        self.time_range
+    }
 }
 
 pub type BoxedBatchIterator = Box<dyn Iterator<Item = Result<Batch>> + Send + Sync>;
diff --git a/src/mito2/src/memtable/time_series.rs b/src/mito2/src/memtable/time_series.rs
index 1a742b6b7e7c..b5ab52447234 100644
--- a/src/mito2/src/memtable/time_series.rs
+++ b/src/mito2/src/memtable/time_series.rs
@@ -293,11 +293,6 @@ impl SeriesSet {
             last_key: None,
         }
     }
-
-    /// Returns if series set is empty.
-    fn is_empty(&self) -> bool {
-        self.series.read().unwrap().is_empty()
-    }
 }
 
 struct Iter {
@@ -329,37 +324,6 @@ impl Iterator for Iter {
     }
 }
 
-/// Bucket holds a set of [Series] which alleviate lock contention between series.
-struct Bucket {
-    region_metadata: RegionMetadataRef,
-    series: RwLock<Vec<Arc<RwLock<Series>>>>,
-}
-
-impl Bucket {
-    fn new(region_metadata: RegionMetadataRef) -> Self {
-        Self {
-            region_metadata,
-            series: Default::default(),
-        }
-    }
-
-    /// Returns the series at given index.
-    /// Returns None if series not found.
-    #[inline]
-    fn get_series(&self, idx: usize) -> Option<Arc<RwLock<Series>>> {
-        self.series.read().unwrap().get(idx).cloned()
-    }
-
-    /// Adds series to bucket and returns the index inside the bucket.
-    #[inline]
-    fn add_series(&self, s: Arc<RwLock<Series>>) -> usize {
-        let mut series = self.series.write().unwrap();
-        let idx = series.len();
-        series.push(s);
-        idx
-    }
-}
-
 /// A `Series` holds a list of field values of some given primary key.
 struct Series {
     active: ValueBuilder,
diff --git a/src/mito2/src/request.rs b/src/mito2/src/request.rs
index 33cc121a5a3e..f34d8baafe13 100644
--- a/src/mito2/src/request.rs
+++ b/src/mito2/src/request.rs
@@ -125,7 +125,7 @@ impl WriteRequest {
     }
 
     /// Gets column index by name.
-    pub(crate) fn column_index_by_name(&self, name: &str) -> Option<usize> {
+    pub fn column_index_by_name(&self, name: &str) -> Option<usize> {
         self.name_to_index.get(name).copied()
     }
 
@@ -413,11 +413,6 @@ impl OptionOutputTx {
         }
     }
 
-    /// Takes the sender.
-    pub(crate) fn take(&mut self) -> OptionOutputTx {
-        OptionOutputTx(self.0.take())
-    }
-
     /// Takes the inner sender.
     pub(crate) fn take_inner(&mut self) -> Option<OutputTx> {
         self.0.take()
diff --git a/src/mito2/src/sst/parquet/reader.rs b/src/mito2/src/sst/parquet/reader.rs
index 85b1b47ae097..6280f83eef10 100644
--- a/src/mito2/src/sst/parquet/reader.rs
+++ b/src/mito2/src/sst/parquet/reader.rs
@@ -103,7 +103,7 @@ impl ParquetReaderBuilder {
 
         Ok(ParquetReader {
             file_path,
-            file_handle: self.file_handle,
+            _file_handle: self.file_handle,
             stream,
             read_format,
             batches: Vec::new(),
@@ -208,7 +208,7 @@ pub struct ParquetReader {
     /// SST file to read.
     ///
     /// Holds the file handle to avoid the file purge purge it.
-    file_handle: FileHandle,
+    _file_handle: FileHandle,
     /// Inner parquet record batch stream.
     stream: BoxedRecordBatchStream,
     /// Helper to read record batches.
diff --git a/src/mito2/src/sst/stream_writer.rs b/src/mito2/src/sst/stream_writer.rs
index 005b533443b6..28d044298ca3 100644
--- a/src/mito2/src/sst/stream_writer.rs
+++ b/src/mito2/src/sst/stream_writer.rs
@@ -32,7 +32,6 @@ use crate::error::WriteParquetSnafu;
 /// storage by chunks to reduce memory consumption.
 pub struct BufferedWriter {
     inner: InnerBufferedWriter,
-    arrow_schema: SchemaRef,
 }
 
 type InnerBufferedWriter = LazyBufferedWriter<
@@ -79,7 +78,6 @@ impl BufferedWriter {
                     })
                 }),
             ),
-            arrow_schema,
         })
     }
 

From 6efc97228780351b638974243014fd7470e1488a Mon Sep 17 00:00:00 2001
From: Wei <47681251+QuenKar@users.noreply.github.com>
Date: Mon, 25 Sep 2023 18:00:56 +0800
Subject: [PATCH 19/36] refactor: datetime time unit (#2469)

* refactor: datetime time unit

* Update src/common/time/src/datetime.rs

Co-authored-by: Yingwen <realevenyag@gmail.com>

* chore: cr.

---------

Co-authored-by: Yingwen <realevenyag@gmail.com>
---
 src/common/time/src/datetime.rs          | 41 +++++++++++++++++-------
 src/datatypes/src/types/datetime_type.rs |  2 +-
 src/datatypes/src/vectors/datetime.rs    | 16 ++++-----
 3 files changed, 39 insertions(+), 20 deletions(-)

diff --git a/src/common/time/src/datetime.rs b/src/common/time/src/datetime.rs
index 05d4884e95f5..5bdec5dc25c9 100644
--- a/src/common/time/src/datetime.rs
+++ b/src/common/time/src/datetime.rs
@@ -20,11 +20,12 @@ use serde::{Deserialize, Serialize};
 
 use crate::error::{Error, InvalidDateStrSnafu, Result};
 use crate::util::{format_utc_datetime, local_datetime_to_utc};
+use crate::Date;
 
 const DATETIME_FORMAT: &str = "%F %T";
 const DATETIME_FORMAT_WITH_TZ: &str = "%F %T%z";
 
-/// [DateTime] represents the **seconds elapsed since "1970-01-01 00:00:00 UTC" (UNIX Epoch)**.
+/// [DateTime] represents the **milliseconds elapsed since "1970-01-01 00:00:00 UTC" (UNIX Epoch)**.
 #[derive(
     Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Default, Serialize, Deserialize,
 )]
@@ -32,7 +33,7 @@ pub struct DateTime(i64);
 
 impl Display for DateTime {
     fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
-        if let Some(abs_time) = NaiveDateTime::from_timestamp_opt(self.0, 0) {
+        if let Some(abs_time) = NaiveDateTime::from_timestamp_millis(self.0) {
             write!(
                 f,
                 "{}",
@@ -52,7 +53,7 @@ impl From<DateTime> for serde_json::Value {
 
 impl From<NaiveDateTime> for DateTime {
     fn from(value: NaiveDateTime) -> Self {
-        DateTime::from(value.timestamp())
+        DateTime::from(value.timestamp_millis())
     }
 }
 
@@ -61,20 +62,20 @@ impl FromStr for DateTime {
 
     fn from_str(s: &str) -> Result<Self> {
         let s = s.trim();
-        let timestamp = if let Ok(d) = NaiveDateTime::parse_from_str(s, DATETIME_FORMAT) {
+        let timestamp_millis = if let Ok(d) = NaiveDateTime::parse_from_str(s, DATETIME_FORMAT) {
             match local_datetime_to_utc(&d) {
                 LocalResult::None => {
                     return InvalidDateStrSnafu { raw: s }.fail();
                 }
-                LocalResult::Single(d) | LocalResult::Ambiguous(d, _) => d.timestamp(),
+                LocalResult::Single(d) | LocalResult::Ambiguous(d, _) => d.timestamp_millis(),
             }
         } else if let Ok(v) = chrono::DateTime::parse_from_str(s, DATETIME_FORMAT_WITH_TZ) {
-            v.timestamp()
+            v.timestamp_millis()
         } else {
             return InvalidDateStrSnafu { raw: s }.fail();
         };
 
-        Ok(Self(timestamp))
+        Ok(Self(timestamp_millis))
     }
 }
 
@@ -84,6 +85,13 @@ impl From<i64> for DateTime {
     }
 }
 
+impl From<Date> for DateTime {
+    fn from(value: Date) -> Self {
+        // It's safe, i32 * 86400000 won't be overflow
+        Self(value.to_secs() * 1000)
+    }
+}
+
 impl DateTime {
     pub fn new(val: i64) -> Self {
         Self(val)
@@ -96,6 +104,10 @@ impl DateTime {
     pub fn to_chrono_datetime(&self) -> Option<NaiveDateTime> {
         NaiveDateTime::from_timestamp_opt(self.0, 0)
     }
+
+    pub fn to_date(&self) -> Option<Date> {
+        self.to_chrono_datetime().map(|d| Date::from(d.date()))
+    }
 }
 
 #[cfg(test)]
@@ -106,8 +118,8 @@ mod tests {
     pub fn test_new_date_time() {
         std::env::set_var("TZ", "Asia/Shanghai");
         assert_eq!("1970-01-01 08:00:00+0800", DateTime::new(0).to_string());
-        assert_eq!("1970-01-01 08:00:01+0800", DateTime::new(1).to_string());
-        assert_eq!("1970-01-01 07:59:59+0800", DateTime::new(-1).to_string());
+        assert_eq!("1970-01-01 08:00:01+0800", DateTime::new(1000).to_string());
+        assert_eq!("1970-01-01 07:59:59+0800", DateTime::new(-1000).to_string());
     }
 
     #[test]
@@ -130,7 +142,7 @@ mod tests {
     fn test_parse_local_date_time() {
         std::env::set_var("TZ", "Asia/Shanghai");
         assert_eq!(
-            -28800,
+            -28800000,
             DateTime::from_str("1970-01-01 00:00:00").unwrap().val()
         );
         assert_eq!(0, DateTime::from_str("1970-01-01 08:00:00").unwrap().val());
@@ -141,6 +153,13 @@ mod tests {
         let ts = DateTime::from_str("1970-01-01 08:00:00+0000")
             .unwrap()
             .val();
-        assert_eq!(28800, ts);
+        assert_eq!(28800000, ts);
+    }
+
+    #[test]
+    fn test_from_max_date() {
+        let date = Date::new(i32::MAX);
+        let datetime = DateTime::from(date);
+        assert_eq!(datetime.val(), 185542587100800000);
     }
 }
diff --git a/src/datatypes/src/types/datetime_type.rs b/src/datatypes/src/types/datetime_type.rs
index cd0e5a3cd1bf..17326ef9ae4a 100644
--- a/src/datatypes/src/types/datetime_type.rs
+++ b/src/datatypes/src/types/datetime_type.rs
@@ -122,7 +122,7 @@ mod tests {
         );
 
         // cast from Timestamp
-        let val = Value::Timestamp(Timestamp::from_str("2020-09-08 21:42:29.042+0800").unwrap());
+        let val = Value::Timestamp(Timestamp::from_str("2020-09-08 21:42:29+0800").unwrap());
         let dt = ConcreteDataType::datetime_datatype().try_cast(val).unwrap();
         assert_eq!(
             dt,
diff --git a/src/datatypes/src/vectors/datetime.rs b/src/datatypes/src/vectors/datetime.rs
index 55cd9015615c..4ae654cc3e98 100644
--- a/src/datatypes/src/vectors/datetime.rs
+++ b/src/datatypes/src/vectors/datetime.rs
@@ -38,7 +38,7 @@ mod tests {
     #[test]
     fn test_datetime_vector() {
         std::env::set_var("TZ", "Asia/Shanghai");
-        let v = DateTimeVector::new(PrimitiveArray::from(vec![1, 2, 3]));
+        let v = DateTimeVector::new(PrimitiveArray::from(vec![1000, 2000, 3000]));
         assert_eq!(ConcreteDataType::datetime_datatype(), v.data_type());
         assert_eq!(3, v.len());
         assert_eq!("DateTimeVector", v.vector_type_name());
@@ -47,19 +47,19 @@ mod tests {
             v.to_arrow_array().data_type()
         );
 
-        assert_eq!(Some(DateTime::new(1)), v.get_data(0));
-        assert_eq!(Value::DateTime(DateTime::new(1)), v.get(0));
-        assert_eq!(ValueRef::DateTime(DateTime::new(1)), v.get_ref(0));
+        assert_eq!(Some(DateTime::new(1000)), v.get_data(0));
+        assert_eq!(Value::DateTime(DateTime::new(1000)), v.get(0));
+        assert_eq!(ValueRef::DateTime(DateTime::new(1000)), v.get_ref(0));
 
         let mut iter = v.iter_data();
-        assert_eq!(Some(DateTime::new(1)), iter.next().unwrap());
-        assert_eq!(Some(DateTime::new(2)), iter.next().unwrap());
-        assert_eq!(Some(DateTime::new(3)), iter.next().unwrap());
+        assert_eq!(Some(DateTime::new(1000)), iter.next().unwrap());
+        assert_eq!(Some(DateTime::new(2000)), iter.next().unwrap());
+        assert_eq!(Some(DateTime::new(3000)), iter.next().unwrap());
         assert!(!v.is_null(0));
         assert_eq!(24, v.memory_size());
 
         if let Value::DateTime(d) = v.get(0) {
-            assert_eq!(1, d.val());
+            assert_eq!(1000, d.val());
         } else {
             unreachable!()
         }

From 013a3fb5cbc7d884a3f599efbdab89f2dea3bd9b Mon Sep 17 00:00:00 2001
From: Weny Xu <wenymedia@gmail.com>
Date: Tue, 26 Sep 2023 14:05:38 +0900
Subject: [PATCH 20/36] fix: dn doesn't have chance to send a heartbeat to the
 new leader (#2471)

* refactor: set meta leader lease secs to 3s

* fix: correct default heartbeat interval

* refactor: ask meta leader in parallel

* feat: configure heartbeat client timeout to 500ms

* fix: trigger to send heartbeat immediately after fail

* fix: fix clippy
---
 config/datanode.example.toml                  |  6 ++-
 src/cmd/src/datanode.rs                       |  1 +
 .../meta/src/distributed_time_constants.rs    |  6 +++
 src/datanode/src/heartbeat.rs                 |  2 +
 src/meta-client/src/client.rs                 | 11 +++-
 src/meta-client/src/client/ask_leader.rs      | 54 ++++++++++++-------
 src/meta-client/src/error.rs                  |  7 +++
 src/meta-client/src/lib.rs                    |  7 +++
 src/meta-srv/src/election.rs                  |  3 --
 src/meta-srv/src/election/etcd.rs             |  9 ++--
 10 files changed, 77 insertions(+), 29 deletions(-)

diff --git a/config/datanode.example.toml b/config/datanode.example.toml
index ddea4523fe1b..2202eae30e4b 100644
--- a/config/datanode.example.toml
+++ b/config/datanode.example.toml
@@ -13,13 +13,15 @@ rpc_runtime_size = 8
 require_lease_before_startup = false
 
 [heartbeat]
-# Interval for sending heartbeat messages to the Metasrv in milliseconds, 5000 by default.
-interval_millis = 5000
+# Interval for sending heartbeat messages to the Metasrv in milliseconds, 3000 by default.
+interval_millis = 3000
 
 # Metasrv client options.
 [meta_client]
 # Metasrv address list.
 metasrv_addrs = ["127.0.0.1:3002"]
+# Heartbeat timeout in milliseconds, 500 by default.
+heartbeat_timeout_millis = 500
 # Operation timeout in milliseconds, 3000 by default.
 timeout_millis = 3000
 # Connect server timeout in milliseconds, 5000 by default.
diff --git a/src/cmd/src/datanode.rs b/src/cmd/src/datanode.rs
index 8b6daef9596f..e2ff160eefb5 100644
--- a/src/cmd/src/datanode.rs
+++ b/src/cmd/src/datanode.rs
@@ -255,6 +255,7 @@ mod tests {
             connect_timeout_millis,
             tcp_nodelay,
             ddl_timeout_millis,
+            ..
         } = options.meta_client.unwrap();
 
         assert_eq!(vec!["127.0.0.1:3002".to_string()], metasrv_addr);
diff --git a/src/common/meta/src/distributed_time_constants.rs b/src/common/meta/src/distributed_time_constants.rs
index e271a0943a3b..a90629f14aea 100644
--- a/src/common/meta/src/distributed_time_constants.rs
+++ b/src/common/meta/src/distributed_time_constants.rs
@@ -29,3 +29,9 @@ pub const REGION_LEASE_SECS: u64 =
 /// When creating table or region failover, a target node needs to be selected.
 /// If the node's lease has expired, the `Selector` will not select it.
 pub const DATANODE_LEASE_SECS: u64 = REGION_LEASE_SECS;
+
+/// The lease seconds of metasrv leader.
+pub const META_LEASE_SECS: u64 = 3;
+
+// In a lease, there are two opportunities for renewal.
+pub const META_KEEP_ALIVE_INTERVAL_SECS: u64 = META_LEASE_SECS / 2;
diff --git a/src/datanode/src/heartbeat.rs b/src/datanode/src/heartbeat.rs
index 2b7cb04c5cbc..5bf6fb7cbb3a 100644
--- a/src/datanode/src/heartbeat.rs
+++ b/src/datanode/src/heartbeat.rs
@@ -245,6 +245,8 @@ impl HeartbeatTask {
                             Ok(new_tx) => {
                                 info!("Reconnected to metasrv");
                                 tx = new_tx;
+                                // Triggers to send heartbeat immediately.
+                                sleep.as_mut().reset(Instant::now());
                             }
                             Err(e) => {
                                 error!(e;"Failed to reconnect to metasrv!");
diff --git a/src/meta-client/src/client.rs b/src/meta-client/src/client.rs
index ab6700af04a4..73e132353a7d 100644
--- a/src/meta-client/src/client.rs
+++ b/src/meta-client/src/client.rs
@@ -62,6 +62,7 @@ pub struct MetaClientBuilder {
     enable_ddl: bool,
     channel_manager: Option<ChannelManager>,
     ddl_channel_manager: Option<ChannelManager>,
+    heartbeat_channel_manager: Option<ChannelManager>,
 }
 
 impl MetaClientBuilder {
@@ -122,6 +123,13 @@ impl MetaClientBuilder {
         }
     }
 
+    pub fn heartbeat_channel_manager(self, channel_manager: ChannelManager) -> Self {
+        Self {
+            heartbeat_channel_manager: Some(channel_manager),
+            ..self
+        }
+    }
+
     pub fn build(self) -> MetaClient {
         let mut client = if let Some(mgr) = self.channel_manager {
             MetaClient::with_channel_manager(self.id, mgr)
@@ -136,10 +144,11 @@ impl MetaClientBuilder {
         let mgr = client.channel_manager.clone();
 
         if self.enable_heartbeat {
+            let mgr = self.heartbeat_channel_manager.unwrap_or(mgr.clone());
             client.heartbeat = Some(HeartbeatClient::new(
                 self.id,
                 self.role,
-                mgr.clone(),
+                mgr,
                 DEFAULT_ASK_LEADER_MAX_RETRY,
             ));
         }
diff --git a/src/meta-client/src/client/ask_leader.rs b/src/meta-client/src/client/ask_leader.rs
index e79294bf1fb5..40c731f2cd90 100644
--- a/src/meta-client/src/client/ask_leader.rs
+++ b/src/meta-client/src/client/ask_leader.rs
@@ -13,13 +13,16 @@
 // limitations under the License.
 
 use std::sync::{Arc, RwLock};
+use std::time::Duration;
 
 use api::v1::meta::heartbeat_client::HeartbeatClient;
 use api::v1::meta::{AskLeaderRequest, RequestHeader, Role};
 use common_grpc::channel_manager::ChannelManager;
+use common_meta::distributed_time_constants::META_KEEP_ALIVE_INTERVAL_SECS;
 use common_telemetry::warn;
 use rand::seq::SliceRandom;
 use snafu::{OptionExt, ResultExt};
+use tokio::time::timeout;
 use tonic::transport::Channel;
 
 use crate::client::Id;
@@ -73,29 +76,44 @@ impl AskLeader {
         };
         peers.shuffle(&mut rand::thread_rng());
 
-        let header = RequestHeader::new(self.id, self.role);
-        let mut leader = None;
+        let req = AskLeaderRequest {
+            header: Some(RequestHeader::new(self.id, self.role)),
+        };
+
+        let (tx, mut rx) = tokio::sync::mpsc::channel(peers.len());
+
         for addr in &peers {
-            let req = AskLeaderRequest {
-                header: Some(header.clone()),
-            };
             let mut client = self.create_asker(addr)?;
-            match client.ask_leader(req).await {
-                Ok(res) => {
-                    let Some(endpoint) = res.into_inner().leader else {
-                        warn!("No leader from: {addr}");
-                        continue;
-                    };
-                    leader = Some(endpoint.addr);
-                    break;
+            let tx_clone = tx.clone();
+            let req = req.clone();
+            let addr = addr.to_string();
+            tokio::spawn(async move {
+                match client.ask_leader(req).await {
+                    Ok(res) => {
+                        if let Some(endpoint) = res.into_inner().leader {
+                            let _ = tx_clone.send(endpoint.addr).await;
+                        } else {
+                            warn!("No leader from: {addr}");
+                        };
+                    }
+                    Err(status) => {
+                        warn!("Failed to ask leader from: {addr}, {status}");
+                    }
                 }
-                Err(status) => {
-                    warn!("Failed to ask leader from: {addr}, {status}");
-                }
-            }
+            });
         }
 
-        let leader = leader.context(error::NoLeaderSnafu)?;
+        let leader = timeout(
+            self.channel_manager
+                .config()
+                .timeout
+                .unwrap_or(Duration::from_secs(META_KEEP_ALIVE_INTERVAL_SECS)),
+            rx.recv(),
+        )
+        .await
+        .context(error::AskLeaderTimeoutSnafu)?
+        .context(error::NoLeaderSnafu)?;
+
         let mut leadership_group = self.leadership_group.write().unwrap();
         leadership_group.leader = Some(leader.clone());
 
diff --git a/src/meta-client/src/error.rs b/src/meta-client/src/error.rs
index f302a3136060..8644e18948ad 100644
--- a/src/meta-client/src/error.rs
+++ b/src/meta-client/src/error.rs
@@ -33,6 +33,12 @@ pub enum Error {
     #[snafu(display("No leader, should ask leader first"))]
     NoLeader { location: Location },
 
+    #[snafu(display("Ask leader timeout"))]
+    AskLeaderTimeout {
+        location: Location,
+        source: tokio::time::error::Elapsed,
+    },
+
     #[snafu(display("Failed to create gRPC channel"))]
     CreateChannel {
         location: Location,
@@ -83,6 +89,7 @@ impl ErrorExt for Error {
             Error::IllegalGrpcClientState { .. }
             | Error::AskLeader { .. }
             | Error::NoLeader { .. }
+            | Error::AskLeaderTimeout { .. }
             | Error::NotStarted { .. }
             | Error::SendHeartbeat { .. }
             | Error::CreateHeartbeatStream { .. }
diff --git a/src/meta-client/src/lib.rs b/src/meta-client/src/lib.rs
index 3489bccbf75d..ad18718040c3 100644
--- a/src/meta-client/src/lib.rs
+++ b/src/meta-client/src/lib.rs
@@ -22,12 +22,18 @@ pub mod error;
 pub struct MetaClientOptions {
     pub metasrv_addrs: Vec<String>,
     pub timeout_millis: u64,
+    #[serde(default = "default_heartbeat_timeout_millis")]
+    pub heartbeat_timeout_millis: u64,
     #[serde(default = "default_ddl_timeout_millis")]
     pub ddl_timeout_millis: u64,
     pub connect_timeout_millis: u64,
     pub tcp_nodelay: bool,
 }
 
+fn default_heartbeat_timeout_millis() -> u64 {
+    500u64
+}
+
 fn default_ddl_timeout_millis() -> u64 {
     10_000u64
 }
@@ -37,6 +43,7 @@ impl Default for MetaClientOptions {
         Self {
             metasrv_addrs: vec!["127.0.0.1:3002".to_string()],
             timeout_millis: 3_000u64,
+            heartbeat_timeout_millis: default_heartbeat_timeout_millis(),
             ddl_timeout_millis: default_ddl_timeout_millis(),
             connect_timeout_millis: 5_000u64,
             tcp_nodelay: true,
diff --git a/src/meta-srv/src/election.rs b/src/meta-srv/src/election.rs
index 98d3e29649fa..5c2bc503e4a7 100644
--- a/src/meta-srv/src/election.rs
+++ b/src/meta-srv/src/election.rs
@@ -21,9 +21,6 @@ use tokio::sync::broadcast::Receiver;
 
 use crate::error::Result;
 
-pub const LEASE_SECS: i64 = 5;
-// In a lease, there are two opportunities for renewal.
-pub const KEEP_ALIVE_PERIOD_SECS: u64 = LEASE_SECS as u64 / 2;
 pub const ELECTION_KEY: &str = "__meta_srv_election";
 
 #[derive(Debug, Clone)]
diff --git a/src/meta-srv/src/election/etcd.rs b/src/meta-srv/src/election/etcd.rs
index 96db650853e5..30dcc5518d5b 100644
--- a/src/meta-srv/src/election/etcd.rs
+++ b/src/meta-srv/src/election/etcd.rs
@@ -16,6 +16,7 @@ use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::Arc;
 use std::time::Duration;
 
+use common_meta::distributed_time_constants::{META_KEEP_ALIVE_INTERVAL_SECS, META_LEASE_SECS};
 use common_telemetry::{error, info, warn};
 use etcd_client::Client;
 use snafu::{OptionExt, ResultExt};
@@ -23,9 +24,7 @@ use tokio::sync::broadcast;
 use tokio::sync::broadcast::error::RecvError;
 use tokio::sync::broadcast::Receiver;
 
-use crate::election::{
-    Election, LeaderChangeMessage, ELECTION_KEY, KEEP_ALIVE_PERIOD_SECS, LEASE_SECS,
-};
+use crate::election::{Election, LeaderChangeMessage, ELECTION_KEY};
 use crate::error;
 use crate::error::Result;
 use crate::metasrv::{ElectionRef, LeaderValue};
@@ -114,7 +113,7 @@ impl Election for EtcdElection {
         let mut lease_client = self.client.lease_client();
         let mut election_client = self.client.election_client();
         let res = lease_client
-            .grant(LEASE_SECS, None)
+            .grant(META_LEASE_SECS as i64, None)
             .await
             .context(error::EtcdFailedSnafu)?;
         let lease_id = res.id();
@@ -140,7 +139,7 @@ impl Election for EtcdElection {
                 .context(error::EtcdFailedSnafu)?;
 
             let mut keep_alive_interval =
-                tokio::time::interval(Duration::from_secs(KEEP_ALIVE_PERIOD_SECS));
+                tokio::time::interval(Duration::from_secs(META_KEEP_ALIVE_INTERVAL_SECS));
             loop {
                 let _ = keep_alive_interval.tick().await;
                 keeper.keep_alive().await.context(error::EtcdFailedSnafu)?;

From 212c490436d102211e8efc032604681a96330cd8 Mon Sep 17 00:00:00 2001
From: Lilit0x <baasisek01@gmail.com>
Date: Tue, 26 Sep 2023 09:24:57 +0100
Subject: [PATCH 21/36] fix(tests): fixed open tsdb integration test

---
 tests-integration/src/opentsdb.rs | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/tests-integration/src/opentsdb.rs b/tests-integration/src/opentsdb.rs
index 5d6338d94270..3a33ae282ee6 100644
--- a/tests-integration/src/opentsdb.rs
+++ b/tests-integration/src/opentsdb.rs
@@ -88,12 +88,12 @@ mod tests {
                 let pretty_print = recordbatches.pretty_print().unwrap();
                 let expected = vec![
                     "+---------------------+----------------+-------+-------+-------+",
-                    "| greptime_timestamp  | greptime_value | tagk1 | tagk2 | tagk3 |",
-                    "+---------------------+----------------+-------+-------+-------+",
-                    "| 1970-01-01T00:00:01 | 1.0            | tagv1 | tagv2 |       |",
-                    "| 1970-01-01T00:00:02 | 2.0            |       | tagv2 | tagv3 |",
-                    "| 1970-01-01T00:00:03 | 3.0            |       |       |       |",
-                    "+---------------------+----------------+-------+-------+-------+",
+                    "| tagk1 | tagk2 | greptime_value | greptime_timestamp  | tagk3 |",
+                    "+-------+-------+----------------+---------------------+-------+",
+                    "| tagv1 | tagv2 | 1.0            | 1970-01-01T00:00:01 |       |",
+                    "|       | tagv2 | 2.0            | 1970-01-01T00:00:02 | tagv3 |",
+                    "|       |       | 3.0            | 1970-01-01T00:00:03 |       |",
+                    "+-------+-------+----------------+---------------------+-------+",
                 ]
                 .into_iter()
                 .join("\n");

From 6ede1be51dde0550712eb2dd34d3fdb139ef4178 Mon Sep 17 00:00:00 2001
From: Lilit0x <baasisek01@gmail.com>
Date: Tue, 26 Sep 2023 09:47:53 +0100
Subject: [PATCH 22/36] fix(tests): aligned top row in table for failing test

---
 tests-integration/src/opentsdb.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests-integration/src/opentsdb.rs b/tests-integration/src/opentsdb.rs
index 3a33ae282ee6..00dffd5d0717 100644
--- a/tests-integration/src/opentsdb.rs
+++ b/tests-integration/src/opentsdb.rs
@@ -87,7 +87,7 @@ mod tests {
                 let recordbatches = RecordBatches::try_collect(stream).await.unwrap();
                 let pretty_print = recordbatches.pretty_print().unwrap();
                 let expected = vec![
-                    "+---------------------+----------------+-------+-------+-------+",
+                    "+-------+-------+----------------+---------------------+-------+"
                     "| tagk1 | tagk2 | greptime_value | greptime_timestamp  | tagk3 |",
                     "+-------+-------+----------------+---------------------+-------+",
                     "| tagv1 | tagv2 | 1.0            | 1970-01-01T00:00:01 |       |",

From 662e9bbf26b2409a649725708ce102dadf13096a Mon Sep 17 00:00:00 2001
From: Lilit0x <baasisek01@gmail.com>
Date: Tue, 26 Sep 2023 09:49:27 +0100
Subject: [PATCH 23/36] fix(tests): aligned top row in table for failing test

---
 tests-integration/src/opentsdb.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests-integration/src/opentsdb.rs b/tests-integration/src/opentsdb.rs
index 00dffd5d0717..d6e524e7ff3b 100644
--- a/tests-integration/src/opentsdb.rs
+++ b/tests-integration/src/opentsdb.rs
@@ -87,7 +87,7 @@ mod tests {
                 let recordbatches = RecordBatches::try_collect(stream).await.unwrap();
                 let pretty_print = recordbatches.pretty_print().unwrap();
                 let expected = vec![
-                    "+-------+-------+----------------+---------------------+-------+"
+                    "+-------+-------+----------------+---------------------+-------+",
                     "| tagk1 | tagk2 | greptime_value | greptime_timestamp  | tagk3 |",
                     "+-------+-------+----------------+---------------------+-------+",
                     "| tagv1 | tagv2 | 1.0            | 1970-01-01T00:00:01 |       |",

From 13d22adfc129989db2eb8a7d5cc0dcd54b3ac952 Mon Sep 17 00:00:00 2001
From: Zhenchi <zhongzc_arch@outlook.com>
Date: Tue, 26 Sep 2023 16:37:04 +0800
Subject: [PATCH 24/36] feat: re-support query engine execute dml (#2484)

* feat: re-support query engine execute dml

Signed-off-by: Zhenchi <zhongzc_arch@outlook.com>

* chore: remove region_number in InsertRequest

Signed-off-by: Zhenchi <zhongzc_arch@outlook.com>

* chore: add doc comments

Signed-off-by: Zhenchi <zhongzc_arch@outlook.com>

---------

Signed-off-by: Zhenchi <zhongzc_arch@outlook.com>
---
 src/cmd/src/cli/repl.rs                       |   1 +
 src/datanode/src/datanode.rs                  |   1 +
 src/frontend/src/instance.rs                  |  57 +++---
 src/operator/src/delete.rs                    |   6 +-
 .../src/req_convert/insert/table_to_region.rs |   1 -
 src/operator/src/statement.rs                 |   8 +-
 src/operator/src/statement/copy_table_from.rs |   1 -
 src/operator/src/statement/dml.rs             | 191 +-----------------
 src/operator/src/table.rs                     |  47 +++++
 src/operator/src/tests/partition_manager.rs   |   1 -
 src/query/src/datafusion.rs                   |  40 ++--
 src/query/src/error.rs                        |  14 +-
 src/query/src/lib.rs                          |   1 +
 src/query/src/query_engine.rs                 |   7 +-
 src/query/src/query_engine/state.rs           |   9 +
 src/query/src/range_select/plan_rewrite.rs    |   2 +-
 src/query/src/region_query.rs                 |   1 -
 src/query/src/table_mutation.rs               |  35 ++++
 src/query/src/tests.rs                        |   2 +-
 src/query/src/tests/query_engine_test.rs      |   6 +-
 src/query/src/tests/time_range_filter_test.rs |   2 +-
 src/script/benches/py_benchmark.rs            |   2 +-
 src/script/src/python/engine.rs               |   3 +-
 src/script/src/test.rs                        |   2 +-
 src/servers/src/line_writer.rs                |   1 -
 src/servers/tests/mod.rs                      |   2 +-
 src/table/src/requests.rs                     |   2 -
 src/table/src/table.rs                        |  79 +-------
 28 files changed, 190 insertions(+), 334 deletions(-)
 create mode 100644 src/query/src/table_mutation.rs

diff --git a/src/cmd/src/cli/repl.rs b/src/cmd/src/cli/repl.rs
index c1b619cf85b9..c5f256a3c5b5 100644
--- a/src/cmd/src/cli/repl.rs
+++ b/src/cmd/src/cli/repl.rs
@@ -261,6 +261,7 @@ async fn create_query_engine(meta_addr: &str) -> Result<DatafusionQueryEngine> {
     let state = Arc::new(QueryEngineState::new(
         catalog_list,
         None,
+        None,
         false,
         plugins.clone(),
     ));
diff --git a/src/datanode/src/datanode.rs b/src/datanode/src/datanode.rs
index 3a526235ca50..885ed8bbddb5 100644
--- a/src/datanode/src/datanode.rs
+++ b/src/datanode/src/datanode.rs
@@ -338,6 +338,7 @@ impl DatanodeBuilder {
             // query engine in datanode only executes plan with resolved table source.
             MemoryCatalogManager::with_default_setup(),
             None,
+            None,
             false,
             plugins,
         );
diff --git a/src/frontend/src/instance.rs b/src/frontend/src/instance.rs
index bffefabd47e7..a7cef4ba6786 100644
--- a/src/frontend/src/instance.rs
+++ b/src/frontend/src/instance.rs
@@ -53,7 +53,7 @@ use meta_client::client::{MetaClient, MetaClientBuilder};
 use operator::delete::{Deleter, DeleterRef};
 use operator::insert::{Inserter, InserterRef};
 use operator::statement::StatementExecutor;
-use operator::table::table_idents_to_full_name;
+use operator::table::{table_idents_to_full_name, TableMutationOperator};
 use partition::manager::PartitionRuleManager;
 use query::parser::{PromQuery, QueryLanguageParser, QueryStatement};
 use query::plan::LogicalPlan;
@@ -163,14 +163,6 @@ impl Instance {
             catalog_manager.datanode_manager().clone(),
         );
 
-        let query_engine = QueryEngineFactory::new_with_plugins(
-            catalog_manager.clone(),
-            Some(region_query_handler.clone()),
-            true,
-            plugins.clone(),
-        )
-        .query_engine();
-
         let inserter = Arc::new(Inserter::new(
             catalog_manager.clone(),
             partition_manager.clone(),
@@ -182,6 +174,20 @@ impl Instance {
             datanode_clients,
         ));
 
+        let table_mutation_handler = Arc::new(TableMutationOperator::new(
+            inserter.clone(),
+            deleter.clone(),
+        ));
+
+        let query_engine = QueryEngineFactory::new_with_plugins(
+            catalog_manager.clone(),
+            Some(region_query_handler.clone()),
+            Some(table_mutation_handler),
+            true,
+            plugins.clone(),
+        )
+        .query_engine();
+
         let statement_executor = Arc::new(StatementExecutor::new(
             catalog_manager.clone(),
             query_engine.clone(),
@@ -189,7 +195,6 @@ impl Instance {
             meta_backend.clone(),
             catalog_manager.clone(),
             inserter.clone(),
-            deleter.clone(),
         ));
 
         plugins.insert::<StatementExecutorRef>(statement_executor.clone());
@@ -301,9 +306,25 @@ impl Instance {
         let region_query_handler =
             FrontendRegionQueryHandler::arc(partition_manager.clone(), datanode_manager.clone());
 
+        let inserter = Arc::new(Inserter::new(
+            catalog_manager.clone(),
+            partition_manager.clone(),
+            datanode_manager.clone(),
+        ));
+        let deleter = Arc::new(Deleter::new(
+            catalog_manager.clone(),
+            partition_manager,
+            datanode_manager.clone(),
+        ));
+        let table_mutation_handler = Arc::new(TableMutationOperator::new(
+            inserter.clone(),
+            deleter.clone(),
+        ));
+
         let query_engine = QueryEngineFactory::new_with_plugins(
             catalog_manager.clone(),
             Some(region_query_handler),
+            Some(table_mutation_handler),
             true,
             plugins.clone(),
         )
@@ -317,25 +338,12 @@ impl Instance {
         let cache_invalidator = Arc::new(DummyCacheInvalidator);
         let ddl_executor = Arc::new(DdlManager::new(
             procedure_manager,
-            datanode_manager.clone(),
+            datanode_manager,
             cache_invalidator.clone(),
             table_metadata_manager.clone(),
             Arc::new(StandaloneTableMetadataCreator::new(kv_backend.clone())),
         ));
 
-        let partition_manager = Arc::new(PartitionRuleManager::new(kv_backend.clone()));
-
-        let inserter = Arc::new(Inserter::new(
-            catalog_manager.clone(),
-            partition_manager.clone(),
-            datanode_manager.clone(),
-        ));
-        let deleter = Arc::new(Deleter::new(
-            catalog_manager.clone(),
-            partition_manager,
-            datanode_manager,
-        ));
-
         let statement_executor = Arc::new(StatementExecutor::new(
             catalog_manager.clone(),
             query_engine.clone(),
@@ -343,7 +351,6 @@ impl Instance {
             kv_backend.clone(),
             cache_invalidator,
             inserter.clone(),
-            deleter.clone(),
         ));
 
         Ok(Instance {
diff --git a/src/operator/src/delete.rs b/src/operator/src/delete.rs
index 546d45d5f8f9..50111dd39113 100644
--- a/src/operator/src/delete.rs
+++ b/src/operator/src/delete.rs
@@ -98,7 +98,7 @@ impl Deleter {
         &self,
         request: TableDeleteRequest,
         ctx: QueryContextRef,
-    ) -> Result<AffectedRows> {
+    ) -> Result<usize> {
         let catalog = request.catalog_name.as_str();
         let schema = request.schema_name.as_str();
         let table = request.table_name.as_str();
@@ -108,7 +108,9 @@ impl Deleter {
         let deletes = TableToRegion::new(&table_info, &self.partition_manager)
             .convert(request)
             .await?;
-        self.do_request(deletes, ctx.trace_id(), 0).await
+
+        let affected_rows = self.do_request(deletes, ctx.trace_id(), 0).await?;
+        Ok(affected_rows as _)
     }
 }
 
diff --git a/src/operator/src/req_convert/insert/table_to_region.rs b/src/operator/src/req_convert/insert/table_to_region.rs
index fa7181eebbfd..5ece06b79f96 100644
--- a/src/operator/src/req_convert/insert/table_to_region.rs
+++ b/src/operator/src/req_convert/insert/table_to_region.rs
@@ -145,7 +145,6 @@ mod tests {
             schema_name: DEFAULT_SCHEMA_NAME.to_string(),
             table_name: "table_1".to_string(),
             columns_values: HashMap::from([("a".to_string(), vector)]),
-            region_number: 0,
         }
     }
 
diff --git a/src/operator/src/statement.rs b/src/operator/src/statement.rs
index e6b46d26a601..f2d371b6d23f 100644
--- a/src/operator/src/statement.rs
+++ b/src/operator/src/statement.rs
@@ -48,7 +48,6 @@ use table::engine::TableReference;
 use table::requests::{CopyDatabaseRequest, CopyDirection, CopyTableRequest};
 use table::TableRef;
 
-use crate::delete::DeleterRef;
 use crate::error::{
     self, CatalogSnafu, ExecLogicalPlanSnafu, ExternalSnafu, InvalidSqlSnafu, PlanStatementSnafu,
     Result, TableNotFoundSnafu,
@@ -66,7 +65,6 @@ pub struct StatementExecutor {
     partition_manager: PartitionRuleManagerRef,
     cache_invalidator: CacheInvalidatorRef,
     inserter: InserterRef,
-    deleter: DeleterRef,
 }
 
 impl StatementExecutor {
@@ -77,7 +75,6 @@ impl StatementExecutor {
         kv_backend: KvBackendRef,
         cache_invalidator: CacheInvalidatorRef,
         inserter: InserterRef,
-        deleter: DeleterRef,
     ) -> Self {
         Self {
             catalog_manager,
@@ -87,7 +84,6 @@ impl StatementExecutor {
             partition_manager: Arc::new(PartitionRuleManager::new(kv_backend)),
             cache_invalidator,
             inserter,
-            deleter,
         }
     }
 
@@ -104,14 +100,12 @@ impl StatementExecutor {
 
     pub async fn execute_sql(&self, stmt: Statement, query_ctx: QueryContextRef) -> Result<Output> {
         match stmt {
-            Statement::Query(_) | Statement::Explain(_) => {
+            Statement::Query(_) | Statement::Explain(_) | Statement::Delete(_) => {
                 self.plan_exec(QueryStatement::Sql(stmt), query_ctx).await
             }
 
             Statement::Insert(insert) => self.insert(insert, query_ctx).await,
 
-            Statement::Delete(delete) => self.delete(delete, query_ctx).await,
-
             Statement::Tql(tql) => self.execute_tql(tql, query_ctx).await,
 
             Statement::DescribeTable(stmt) => self.describe_table(stmt, query_ctx).await,
diff --git a/src/operator/src/statement/copy_table_from.rs b/src/operator/src/statement/copy_table_from.rs
index b335e5356a5e..9f34627ee4eb 100644
--- a/src/operator/src/statement/copy_table_from.rs
+++ b/src/operator/src/statement/copy_table_from.rs
@@ -330,7 +330,6 @@ impl StatementExecutor {
                         schema_name: req.schema_name.to_string(),
                         table_name: req.table_name.to_string(),
                         columns_values,
-                        region_number: 0,
                     },
                     query_ctx.clone(),
                 ));
diff --git a/src/operator/src/statement/dml.rs b/src/operator/src/statement/dml.rs
index f127dda4d48b..197ac5c03b11 100644
--- a/src/operator/src/statement/dml.rs
+++ b/src/operator/src/statement/dml.rs
@@ -12,30 +12,14 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::collections::HashMap;
-
 use common_query::Output;
-use common_recordbatch::{RecordBatch, SendableRecordBatchStream};
-use datafusion_expr::{DmlStatement, LogicalPlan as DfLogicalPlan, WriteOp};
-use datatypes::schema::SchemaRef;
-use futures_util::StreamExt;
 use query::parser::QueryStatement;
-use query::plan::LogicalPlan;
 use session::context::QueryContextRef;
-use snafu::{ensure, OptionExt, ResultExt};
-use sql::statements::delete::Delete;
 use sql::statements::insert::Insert;
 use sql::statements::statement::Statement;
-use table::engine::TableReference;
-use table::metadata::TableInfoRef;
-use table::requests::{DeleteRequest, InsertRequest};
-use table::TableRef;
 
 use super::StatementExecutor;
-use crate::error::{
-    BuildColumnVectorsSnafu, ExecLogicalPlanSnafu, MissingTimeIndexColumnSnafu,
-    ReadRecordBatchSnafu, Result, UnexpectedSnafu,
-};
+use crate::error::Result;
 
 impl StatementExecutor {
     pub async fn insert(&self, insert: Box<Insert>, query_ctx: QueryContextRef) -> Result<Output> {
@@ -45,178 +29,9 @@ impl StatementExecutor {
                 .handle_statement_insert(insert.as_ref(), &query_ctx)
                 .await
         } else {
-            // Slow path: insert with subquery. Execute the subquery first, via query engine. Then
-            // insert the results by sending insert requests.
-
-            // 1. Plan the whole insert statement into a logical plan, then a wrong insert statement
-            //    will be caught and a plan error will be returned.
+            // Slow path: insert with subquery. Execute using query engine.
             let statement = QueryStatement::Sql(Statement::Insert(insert));
-            let logical_plan = self.plan(statement, query_ctx.clone()).await?;
-
-            // 2. Execute the subquery, get the results as a record batch stream.
-            let dml_statement = extract_dml_statement(logical_plan)?;
-            ensure!(
-                dml_statement.op == WriteOp::Insert,
-                UnexpectedSnafu {
-                    violated: "expected an INSERT plan"
-                }
-            );
-            let mut stream = self
-                .execute_dml_subquery(&dml_statement, query_ctx.clone())
-                .await?;
-
-            // 3. Send insert requests.
-            let mut affected_rows = 0;
-            let table = self.get_table_from_dml(dml_statement, &query_ctx).await?;
-            let table_info = table.table_info();
-            while let Some(batch) = stream.next().await {
-                let record_batch = batch.context(ReadRecordBatchSnafu)?;
-                let insert_request =
-                    build_insert_request(record_batch, table.schema(), &table_info)?;
-                affected_rows += self
-                    .inserter
-                    .handle_table_insert(insert_request, query_ctx.clone())
-                    .await?;
-            }
-
-            Ok(Output::AffectedRows(affected_rows))
+            self.plan_exec(statement, query_ctx).await
         }
     }
-
-    pub async fn delete(&self, delete: Box<Delete>, query_ctx: QueryContextRef) -> Result<Output> {
-        // 1. Plan the whole delete statement into a logical plan, then a wrong delete statement
-        //    will be caught and a plan error will be returned.
-        let statement = QueryStatement::Sql(Statement::Delete(delete));
-        let logical_plan = self.plan(statement, query_ctx.clone()).await?;
-
-        // 2. Execute the subquery, get the results as a record batch stream.
-        let dml_statement = extract_dml_statement(logical_plan)?;
-        ensure!(
-            dml_statement.op == WriteOp::Delete,
-            UnexpectedSnafu {
-                violated: "expected a DELETE plan"
-            }
-        );
-        let mut stream = self
-            .execute_dml_subquery(&dml_statement, query_ctx.clone())
-            .await?;
-
-        // 3. Send delete requests.
-        let mut affected_rows = 0;
-        let table = self.get_table_from_dml(dml_statement, &query_ctx).await?;
-        let table_info = table.table_info();
-        while let Some(batch) = stream.next().await {
-            let record_batch = batch.context(ReadRecordBatchSnafu)?;
-            let request = build_delete_request(record_batch, table.schema(), &table_info)?;
-            affected_rows += self
-                .deleter
-                .handle_table_delete(request, query_ctx.clone())
-                .await?;
-        }
-
-        Ok(Output::AffectedRows(affected_rows as _))
-    }
-
-    async fn execute_dml_subquery(
-        &self,
-        dml_statement: &DmlStatement,
-        query_ctx: QueryContextRef,
-    ) -> Result<SendableRecordBatchStream> {
-        let subquery_plan = LogicalPlan::from(dml_statement.input.as_ref().clone());
-        let output = self
-            .query_engine
-            .execute(subquery_plan, query_ctx)
-            .await
-            .context(ExecLogicalPlanSnafu)?;
-        match output {
-            Output::Stream(stream) => Ok(stream),
-            Output::RecordBatches(record_batches) => Ok(record_batches.as_stream()),
-            _ => UnexpectedSnafu {
-                violated: "expected a stream",
-            }
-            .fail(),
-        }
-    }
-
-    async fn get_table_from_dml(
-        &self,
-        dml_statement: DmlStatement,
-        query_ctx: &QueryContextRef,
-    ) -> Result<TableRef> {
-        let default_catalog = query_ctx.current_catalog().to_owned();
-        let default_schema = query_ctx.current_schema().to_owned();
-        let resolved_table_ref = dml_statement
-            .table_name
-            .resolve(&default_catalog, &default_schema);
-        let table_ref = TableReference::full(
-            &resolved_table_ref.catalog,
-            &resolved_table_ref.schema,
-            &resolved_table_ref.table,
-        );
-        self.get_table(&table_ref).await
-    }
-}
-
-fn extract_dml_statement(logical_plan: LogicalPlan) -> Result<DmlStatement> {
-    let LogicalPlan::DfPlan(df_plan) = logical_plan;
-    match df_plan {
-        DfLogicalPlan::Dml(dml) => Ok(dml),
-        _ => UnexpectedSnafu {
-            violated: "expected a DML plan",
-        }
-        .fail(),
-    }
-}
-
-fn build_insert_request(
-    record_batch: RecordBatch,
-    table_schema: SchemaRef,
-    table_info: &TableInfoRef,
-) -> Result<InsertRequest> {
-    let columns_values = record_batch
-        .column_vectors(&table_info.name, table_schema)
-        .context(BuildColumnVectorsSnafu)?;
-
-    Ok(InsertRequest {
-        catalog_name: table_info.catalog_name.clone(),
-        schema_name: table_info.schema_name.clone(),
-        table_name: table_info.name.clone(),
-        columns_values,
-        region_number: 0,
-    })
-}
-
-fn build_delete_request(
-    record_batch: RecordBatch,
-    table_schema: SchemaRef,
-    table_info: &TableInfoRef,
-) -> Result<DeleteRequest> {
-    let ts_column = table_schema
-        .timestamp_column()
-        .map(|x| x.name.clone())
-        .with_context(|| table::error::MissingTimeIndexColumnSnafu {
-            table_name: table_info.name.clone(),
-        })
-        .context(MissingTimeIndexColumnSnafu)?;
-
-    let column_vectors = record_batch
-        .column_vectors(&table_info.name, table_schema)
-        .context(BuildColumnVectorsSnafu)?;
-
-    let rowkey_columns = table_info
-        .meta
-        .row_key_column_names()
-        .collect::<Vec<&String>>();
-
-    let key_column_values = column_vectors
-        .into_iter()
-        .filter(|x| x.0 == ts_column || rowkey_columns.contains(&&x.0))
-        .collect::<HashMap<_, _>>();
-
-    Ok(DeleteRequest {
-        catalog_name: table_info.catalog_name.clone(),
-        schema_name: table_info.schema_name.clone(),
-        table_name: table_info.name.clone(),
-        key_column_values,
-    })
 }
diff --git a/src/operator/src/table.rs b/src/operator/src/table.rs
index 240dd0103247..abfc27732bf5 100644
--- a/src/operator/src/table.rs
+++ b/src/operator/src/table.rs
@@ -12,10 +12,19 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use async_trait::async_trait;
+use common_error::ext::BoxedError;
+use query::error as query_error;
+use query::error::Result as QueryResult;
+use query::table_mutation::{AffectedRows, TableMutationHandler};
 use session::context::QueryContextRef;
+use snafu::ResultExt;
 use sqlparser::ast::ObjectName;
+use table::requests::{DeleteRequest as TableDeleteRequest, InsertRequest as TableInsertRequest};
 
+use crate::delete::DeleterRef;
 use crate::error::{InvalidSqlSnafu, Result};
+use crate::insert::InserterRef;
 
 // TODO(LFC): Refactor consideration: move this function to some helper mod,
 // could be done together or after `TableReference`'s refactoring, when issue #559 is resolved.
@@ -47,3 +56,41 @@ pub fn table_idents_to_full_name(
         }.fail(),
     }
 }
+
+pub struct TableMutationOperator {
+    inserter: InserterRef,
+    deleter: DeleterRef,
+}
+
+impl TableMutationOperator {
+    pub fn new(inserter: InserterRef, deleter: DeleterRef) -> Self {
+        Self { inserter, deleter }
+    }
+}
+
+#[async_trait]
+impl TableMutationHandler for TableMutationOperator {
+    async fn insert(
+        &self,
+        request: TableInsertRequest,
+        ctx: QueryContextRef,
+    ) -> QueryResult<AffectedRows> {
+        self.inserter
+            .handle_table_insert(request, ctx)
+            .await
+            .map_err(BoxedError::new)
+            .context(query_error::TableMutationSnafu)
+    }
+
+    async fn delete(
+        &self,
+        request: TableDeleteRequest,
+        ctx: QueryContextRef,
+    ) -> QueryResult<AffectedRows> {
+        self.deleter
+            .handle_table_delete(request, ctx)
+            .await
+            .map_err(BoxedError::new)
+            .context(query_error::TableMutationSnafu)
+    }
+}
diff --git a/src/operator/src/tests/partition_manager.rs b/src/operator/src/tests/partition_manager.rs
index 415ec0f5d610..58421634ae2a 100644
--- a/src/operator/src/tests/partition_manager.rs
+++ b/src/operator/src/tests/partition_manager.rs
@@ -435,7 +435,6 @@ fn test_meter_insert_request() {
         schema_name: "public".to_string(),
         table_name: "numbers".to_string(),
         columns_values: Default::default(),
-        region_number: 0,
     };
     meter_insert_request!(req);
 
diff --git a/src/query/src/datafusion.rs b/src/query/src/datafusion.rs
index 1357897b589c..3928a630c92b 100644
--- a/src/query/src/datafusion.rs
+++ b/src/query/src/datafusion.rs
@@ -54,8 +54,8 @@ use crate::dataframe::DataFrame;
 pub use crate::datafusion::planner::DfContextProviderAdapter;
 use crate::error::{
     CatalogSnafu, CreateRecordBatchSnafu, CreateSchemaSnafu, DataFusionSnafu,
-    MissingTimestampColumnSnafu, QueryExecutionSnafu, Result, TableNotFoundSnafu,
-    UnimplementedSnafu, UnsupportedExprSnafu,
+    MissingTableMutationHandlerSnafu, MissingTimestampColumnSnafu, QueryExecutionSnafu, Result,
+    TableNotFoundSnafu, UnimplementedSnafu, UnsupportedExprSnafu,
 };
 use crate::executor::QueryExecutor;
 use crate::logical_optimizer::LogicalOptimizer;
@@ -115,7 +115,7 @@ impl DatafusionQueryEngine {
         let table = self.find_table(&table_name).await?;
 
         let output = self
-            .exec_query_plan(LogicalPlan::DfPlan((*dml.input).clone()), query_ctx)
+            .exec_query_plan(LogicalPlan::DfPlan((*dml.input).clone()), query_ctx.clone())
             .await?;
         let mut stream = match output {
             Output::RecordBatches(batches) => batches.as_stream(),
@@ -132,8 +132,14 @@ impl DatafusionQueryEngine {
                 .context(QueryExecutionSnafu)?;
 
             let rows = match dml.op {
-                WriteOp::Insert => Self::insert(&table_name, &table, column_vectors).await?,
-                WriteOp::Delete => Self::delete(&table_name, &table, column_vectors).await?,
+                WriteOp::Insert => {
+                    self.insert(&table_name, column_vectors, query_ctx.clone())
+                        .await?
+                }
+                WriteOp::Delete => {
+                    self.delete(&table_name, &table, column_vectors, query_ctx.clone())
+                        .await?
+                }
                 _ => unreachable!("guarded by the 'ensure!' at the beginning"),
             };
             affected_rows += rows;
@@ -142,9 +148,11 @@ impl DatafusionQueryEngine {
     }
 
     async fn delete<'a>(
+        &self,
         table_name: &ResolvedTableReference<'a>,
         table: &TableRef,
         column_vectors: HashMap<String, VectorRef>,
+        query_ctx: QueryContextRef,
     ) -> Result<usize> {
         let catalog_name = table_name.catalog.to_string();
         let schema_name = table_name.schema.to_string();
@@ -174,31 +182,31 @@ impl DatafusionQueryEngine {
             key_column_values: column_vectors,
         };
 
-        table
-            .delete(request)
+        self.state
+            .table_mutation_handler()
+            .context(MissingTableMutationHandlerSnafu)?
+            .delete(request, query_ctx)
             .await
-            .map_err(BoxedError::new)
-            .context(QueryExecutionSnafu)
     }
 
     async fn insert<'a>(
+        &self,
         table_name: &ResolvedTableReference<'a>,
-        table: &TableRef,
         column_vectors: HashMap<String, VectorRef>,
+        query_ctx: QueryContextRef,
     ) -> Result<usize> {
         let request = InsertRequest {
             catalog_name: table_name.catalog.to_string(),
             schema_name: table_name.schema.to_string(),
             table_name: table_name.table.to_string(),
             columns_values: column_vectors,
-            region_number: 0,
         };
 
-        table
-            .insert(request)
+        self.state
+            .table_mutation_handler()
+            .context(MissingTableMutationHandlerSnafu)?
+            .insert(request, query_ctx)
             .await
-            .map_err(BoxedError::new)
-            .context(QueryExecutionSnafu)
     }
 
     async fn find_table(&self, table_name: &ResolvedTableReference<'_>) -> Result<TableRef> {
@@ -517,7 +525,7 @@ mod tests {
         };
         catalog_manager.register_table_sync(req).unwrap();
 
-        QueryEngineFactory::new(catalog_manager, None, false).query_engine()
+        QueryEngineFactory::new(catalog_manager, None, None, false).query_engine()
     }
 
     #[tokio::test]
diff --git a/src/query/src/error.rs b/src/query/src/error.rs
index e535a4b94b5c..5bbdab347261 100644
--- a/src/query/src/error.rs
+++ b/src/query/src/error.rs
@@ -254,11 +254,20 @@ pub enum Error {
     #[snafu(display("Column schema has no default value, column: {}", column))]
     ColumnSchemaNoDefault { column: String, location: Location },
 
-    #[snafu(display("Region query error, source: {}", source))]
+    #[snafu(display("Region query error"))]
     RegionQuery {
         source: BoxedError,
         location: Location,
     },
+
+    #[snafu(display("Table mutation error"))]
+    TableMutation {
+        source: BoxedError,
+        location: Location,
+    },
+
+    #[snafu(display("Missing table mutation handler"))]
+    MissingTableMutationHandler { location: Location },
 }
 
 impl ErrorExt for Error {
@@ -305,7 +314,10 @@ impl ErrorExt for Error {
             RemoteRequest { source, .. } => source.status_code(),
             UnexpectedOutputKind { .. } => StatusCode::Unexpected,
             CreateSchema { source, .. } => source.status_code(),
+
             RegionQuery { source, .. } => source.status_code(),
+            TableMutation { source, .. } => source.status_code(),
+            MissingTableMutationHandler { .. } => StatusCode::Unexpected,
         }
     }
 
diff --git a/src/query/src/lib.rs b/src/query/src/lib.rs
index dfdbe9a208ce..54506f6c93bd 100644
--- a/src/query/src/lib.rs
+++ b/src/query/src/lib.rs
@@ -32,6 +32,7 @@ pub mod query_engine;
 mod range_select;
 pub mod region_query;
 pub mod sql;
+pub mod table_mutation;
 
 pub use crate::datafusion::DfContextProviderAdapter;
 pub use crate::query_engine::{
diff --git a/src/query/src/query_engine.rs b/src/query/src/query_engine.rs
index b18834d2c8ec..d88f399da316 100644
--- a/src/query/src/query_engine.rs
+++ b/src/query/src/query_engine.rs
@@ -38,6 +38,7 @@ use crate::planner::LogicalPlanner;
 pub use crate::query_engine::context::QueryEngineContext;
 pub use crate::query_engine::state::QueryEngineState;
 use crate::region_query::RegionQueryHandlerRef;
+use crate::table_mutation::TableMutationHandlerRef;
 
 /// Describe statement result
 #[derive(Debug)]
@@ -80,11 +81,13 @@ impl QueryEngineFactory {
     pub fn new(
         catalog_manager: CatalogManagerRef,
         region_query_handler: Option<RegionQueryHandlerRef>,
+        table_mutation_handler: Option<TableMutationHandlerRef>,
         with_dist_planner: bool,
     ) -> Self {
         Self::new_with_plugins(
             catalog_manager,
             region_query_handler,
+            table_mutation_handler,
             with_dist_planner,
             Default::default(),
         )
@@ -93,12 +96,14 @@ impl QueryEngineFactory {
     pub fn new_with_plugins(
         catalog_manager: CatalogManagerRef,
         region_query_handler: Option<RegionQueryHandlerRef>,
+        table_mutation_handler: Option<TableMutationHandlerRef>,
         with_dist_planner: bool,
         plugins: Arc<Plugins>,
     ) -> Self {
         let state = Arc::new(QueryEngineState::new(
             catalog_manager,
             region_query_handler,
+            table_mutation_handler,
             with_dist_planner,
             plugins.clone(),
         ));
@@ -131,7 +136,7 @@ mod tests {
     #[test]
     fn test_query_engine_factory() {
         let catalog_list = catalog::memory::new_memory_catalog_manager().unwrap();
-        let factory = QueryEngineFactory::new(catalog_list, None, false);
+        let factory = QueryEngineFactory::new(catalog_list, None, None, false);
 
         let engine = factory.query_engine();
 
diff --git a/src/query/src/query_engine/state.rs b/src/query/src/query_engine/state.rs
index 93916db86101..e882a6811e84 100644
--- a/src/query/src/query_engine/state.rs
+++ b/src/query/src/query_engine/state.rs
@@ -48,6 +48,7 @@ use crate::optimizer::type_conversion::TypeConversionRule;
 use crate::query_engine::options::QueryOptions;
 use crate::range_select::planner::RangeSelectPlanner;
 use crate::region_query::RegionQueryHandlerRef;
+use crate::table_mutation::TableMutationHandlerRef;
 
 /// Query engine global state
 // TODO(yingwen): This QueryEngineState still relies on datafusion, maybe we can define a trait for it,
@@ -57,6 +58,7 @@ use crate::region_query::RegionQueryHandlerRef;
 pub struct QueryEngineState {
     df_context: SessionContext,
     catalog_manager: CatalogManagerRef,
+    table_mutation_handler: Option<TableMutationHandlerRef>,
     aggregate_functions: Arc<RwLock<HashMap<String, AggregateFunctionMetaRef>>>,
     plugins: Arc<Plugins>,
 }
@@ -73,6 +75,7 @@ impl QueryEngineState {
     pub fn new(
         catalog_list: CatalogManagerRef,
         region_query_handler: Option<RegionQueryHandlerRef>,
+        table_mutation_handler: Option<TableMutationHandlerRef>,
         with_dist_planner: bool,
         plugins: Arc<Plugins>,
     ) -> Self {
@@ -123,6 +126,7 @@ impl QueryEngineState {
         Self {
             df_context,
             catalog_manager: catalog_list,
+            table_mutation_handler,
             aggregate_functions: Arc::new(RwLock::new(HashMap::new())),
             plugins,
         }
@@ -184,6 +188,11 @@ impl QueryEngineState {
         &self.catalog_manager
     }
 
+    #[inline]
+    pub fn table_mutation_handler(&self) -> Option<&TableMutationHandlerRef> {
+        self.table_mutation_handler.as_ref()
+    }
+
     pub(crate) fn disallow_cross_schema_query(&self) -> bool {
         self.plugins
             .map::<QueryOptions, _, _>(|x| x.disallow_cross_schema_query)
diff --git a/src/query/src/range_select/plan_rewrite.rs b/src/query/src/range_select/plan_rewrite.rs
index 8842b6dd367a..e7ae66bac8b8 100644
--- a/src/query/src/range_select/plan_rewrite.rs
+++ b/src/query/src/range_select/plan_rewrite.rs
@@ -388,7 +388,7 @@ mod test {
                 table,
             })
             .is_ok());
-        QueryEngineFactory::new(catalog_list, None, false).query_engine()
+        QueryEngineFactory::new(catalog_list, None, None, false).query_engine()
     }
 
     async fn query_plan_compare(sql: &str, expected: String) {
diff --git a/src/query/src/region_query.rs b/src/query/src/region_query.rs
index ef6e494dfc86..f9861103e62b 100644
--- a/src/query/src/region_query.rs
+++ b/src/query/src/region_query.rs
@@ -22,7 +22,6 @@ use crate::error::Result;
 
 #[async_trait]
 pub trait RegionQueryHandler: Send + Sync {
-    // TODO(ruihang): add trace id and span id in the request.
     async fn do_get(&self, request: QueryRequest) -> Result<SendableRecordBatchStream>;
 }
 
diff --git a/src/query/src/table_mutation.rs b/src/query/src/table_mutation.rs
new file mode 100644
index 000000000000..bff93af93687
--- /dev/null
+++ b/src/query/src/table_mutation.rs
@@ -0,0 +1,35 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::sync::Arc;
+
+use async_trait::async_trait;
+use session::context::QueryContextRef;
+use table::requests::{DeleteRequest, InsertRequest};
+
+use crate::error::Result;
+
+pub type AffectedRows = usize;
+
+/// A trait for handling table mutations in `QueryEngine`.
+#[async_trait]
+pub trait TableMutationHandler: Send + Sync {
+    /// Inserts rows into the table.
+    async fn insert(&self, request: InsertRequest, ctx: QueryContextRef) -> Result<AffectedRows>;
+
+    /// Delete rows from the table.
+    async fn delete(&self, request: DeleteRequest, ctx: QueryContextRef) -> Result<AffectedRows>;
+}
+
+pub type TableMutationHandlerRef = Arc<dyn TableMutationHandler>;
diff --git a/src/query/src/tests.rs b/src/query/src/tests.rs
index 4a60f925ce0b..14e62a2d59c0 100644
--- a/src/query/src/tests.rs
+++ b/src/query/src/tests.rs
@@ -51,5 +51,5 @@ async fn exec_selection(engine: QueryEngineRef, sql: &str) -> Vec<RecordBatch> {
 pub fn new_query_engine_with_table(table: TableRef) -> QueryEngineRef {
     let catalog_manager = MemoryCatalogManager::new_with_table(table);
 
-    QueryEngineFactory::new(catalog_manager, None, false).query_engine()
+    QueryEngineFactory::new(catalog_manager, None, None, false).query_engine()
 }
diff --git a/src/query/src/tests/query_engine_test.rs b/src/query/src/tests/query_engine_test.rs
index 41e95d3de1e5..71894eea47c2 100644
--- a/src/query/src/tests/query_engine_test.rs
+++ b/src/query/src/tests/query_engine_test.rs
@@ -47,7 +47,7 @@ async fn test_datafusion_query_engine() -> Result<()> {
     let catalog_list = catalog::memory::new_memory_catalog_manager()
         .map_err(BoxedError::new)
         .context(QueryExecutionSnafu)?;
-    let factory = QueryEngineFactory::new(catalog_list, None, false);
+    let factory = QueryEngineFactory::new(catalog_list, None, None, false);
     let engine = factory.query_engine();
 
     let column_schemas = vec![ColumnSchema::new(
@@ -129,7 +129,7 @@ async fn test_query_validate() -> Result<()> {
     });
     let plugins = Arc::new(plugins);
 
-    let factory = QueryEngineFactory::new_with_plugins(catalog_list, None, false, plugins);
+    let factory = QueryEngineFactory::new_with_plugins(catalog_list, None, None, false, plugins);
     let engine = factory.query_engine();
 
     let stmt = QueryLanguageParser::parse_sql("select number from public.numbers").unwrap();
@@ -153,7 +153,7 @@ async fn test_udf() -> Result<()> {
     common_telemetry::init_default_ut_logging();
     let catalog_list = catalog_manager()?;
 
-    let factory = QueryEngineFactory::new(catalog_list, None, false);
+    let factory = QueryEngineFactory::new(catalog_list, None, None, false);
     let engine = factory.query_engine();
 
     let pow = make_scalar_function(pow);
diff --git a/src/query/src/tests/time_range_filter_test.rs b/src/query/src/tests/time_range_filter_test.rs
index 17c894b54595..dfbcc3a13a73 100644
--- a/src/query/src/tests/time_range_filter_test.rs
+++ b/src/query/src/tests/time_range_filter_test.rs
@@ -106,7 +106,7 @@ fn create_test_engine() -> TimeRangeTester {
     };
     let _ = catalog_manager.register_table_sync(req).unwrap();
 
-    let engine = QueryEngineFactory::new(catalog_manager, None, false).query_engine();
+    let engine = QueryEngineFactory::new(catalog_manager, None, None, false).query_engine();
     TimeRangeTester { engine, filter }
 }
 
diff --git a/src/script/benches/py_benchmark.rs b/src/script/benches/py_benchmark.rs
index f4827a18ad73..27d8ef07ce12 100644
--- a/src/script/benches/py_benchmark.rs
+++ b/src/script/benches/py_benchmark.rs
@@ -52,7 +52,7 @@ where
 pub(crate) fn sample_script_engine() -> PyEngine {
     let catalog_manager =
         MemoryCatalogManager::new_with_table(NumbersTable::table(NUMBERS_TABLE_ID));
-    let query_engine = QueryEngineFactory::new(catalog_manager, None, false).query_engine();
+    let query_engine = QueryEngineFactory::new(catalog_manager, None, None, false).query_engine();
 
     PyEngine::new(query_engine.clone())
 }
diff --git a/src/script/src/python/engine.rs b/src/script/src/python/engine.rs
index 5ab4f06298b5..61f603232a74 100644
--- a/src/script/src/python/engine.rs
+++ b/src/script/src/python/engine.rs
@@ -375,7 +375,8 @@ mod tests {
     pub(crate) fn sample_script_engine() -> PyEngine {
         let catalog_manager =
             MemoryCatalogManager::new_with_table(NumbersTable::table(NUMBERS_TABLE_ID));
-        let query_engine = QueryEngineFactory::new(catalog_manager, None, false).query_engine();
+        let query_engine =
+            QueryEngineFactory::new(catalog_manager, None, None, false).query_engine();
 
         PyEngine::new(query_engine.clone())
     }
diff --git a/src/script/src/test.rs b/src/script/src/test.rs
index c937c4afd215..55ba73f582a1 100644
--- a/src/script/src/test.rs
+++ b/src/script/src/test.rs
@@ -56,7 +56,7 @@ pub async fn setup_scripts_manager(
 
     let catalog_manager = MemoryCatalogManager::new_with_table(table.clone());
 
-    let factory = QueryEngineFactory::new(catalog_manager.clone(), None, false);
+    let factory = QueryEngineFactory::new(catalog_manager.clone(), None, None, false);
     let query_engine = factory.query_engine();
     let mgr = ScriptManager::new(Arc::new(MockGrpcQueryHandler {}) as _, query_engine)
         .await
diff --git a/src/servers/src/line_writer.rs b/src/servers/src/line_writer.rs
index 7da1de6bb8df..38ebe218c7a9 100644
--- a/src/servers/src/line_writer.rs
+++ b/src/servers/src/line_writer.rs
@@ -141,7 +141,6 @@ impl LineWriter {
             schema_name: self.db,
             table_name: self.table_name,
             columns_values,
-            region_number: 0, // TODO(hl): Check if assign 0 region is ok?
         }
     }
 }
diff --git a/src/servers/tests/mod.rs b/src/servers/tests/mod.rs
index 4ef156cb0431..bb2a79818c27 100644
--- a/src/servers/tests/mod.rs
+++ b/src/servers/tests/mod.rs
@@ -214,7 +214,7 @@ impl GrpcQueryHandler for DummyInstance {
 
 fn create_testing_instance(table: TableRef) -> DummyInstance {
     let catalog_manager = MemoryCatalogManager::new_with_table(table);
-    let query_engine = QueryEngineFactory::new(catalog_manager, None, false).query_engine();
+    let query_engine = QueryEngineFactory::new(catalog_manager, None, None, false).query_engine();
     DummyInstance::new(query_engine)
 }
 
diff --git a/src/table/src/requests.rs b/src/table/src/requests.rs
index 9fcb7f8db7e3..9d78f21534fb 100644
--- a/src/table/src/requests.rs
+++ b/src/table/src/requests.rs
@@ -246,7 +246,6 @@ pub struct InsertRequest {
     pub schema_name: String,
     pub table_name: String,
     pub columns_values: HashMap<String, VectorRef>,
-    pub region_number: RegionNumber,
 }
 
 /// Delete (by primary key) request
@@ -327,7 +326,6 @@ macro_rules! meter_insert_request {
             $req.catalog_name.to_string(),
             $req.schema_name.to_string(),
             $req.table_name.to_string(),
-            $req.region_number,
             $req
         );
     };
diff --git a/src/table/src/table.rs b/src/table/src/table.rs
index 8af3cfd1ffec..d2ea65cf840b 100644
--- a/src/table/src/table.rs
+++ b/src/table/src/table.rs
@@ -24,14 +24,10 @@ use async_trait::async_trait;
 use common_query::logical_plan::Expr;
 use common_recordbatch::SendableRecordBatchStream;
 use datatypes::schema::SchemaRef;
-use store_api::storage::{RegionNumber, ScanRequest};
+use store_api::storage::ScanRequest;
 
-use crate::error::{Result, UnsupportedSnafu};
+use crate::error::Result;
 use crate::metadata::{FilterPushDownType, TableId, TableInfoRef, TableType};
-use crate::requests::{AlterTableRequest, DeleteRequest, InsertRequest};
-use crate::stats::TableStatistics;
-
-pub type AlterContext = anymap::Map<dyn Any + Send + Sync>;
 
 /// Table abstraction.
 #[async_trait]
@@ -49,16 +45,6 @@ pub trait Table: Send + Sync {
     /// Get the type of this table for metadata/catalog purposes.
     fn table_type(&self) -> TableType;
 
-    /// Insert values into table.
-    ///
-    /// Returns number of inserted rows.
-    async fn insert(&self, _request: InsertRequest) -> Result<usize> {
-        UnsupportedSnafu {
-            operation: "INSERT",
-        }
-        .fail()?
-    }
-
     async fn scan_to_stream(&self, request: ScanRequest) -> Result<SendableRecordBatchStream>;
 
     /// Tests whether the table provider can make use of any or all filter expressions
@@ -66,67 +52,6 @@ pub trait Table: Send + Sync {
     fn supports_filters_pushdown(&self, filters: &[&Expr]) -> Result<Vec<FilterPushDownType>> {
         Ok(vec![FilterPushDownType::Unsupported; filters.len()])
     }
-
-    /// Alter table.
-    async fn alter(&self, _context: AlterContext, _request: &AlterTableRequest) -> Result<()> {
-        UnsupportedSnafu {
-            operation: "ALTER TABLE",
-        }
-        .fail()?
-    }
-
-    /// Delete rows in the table.
-    ///
-    /// Returns number of deleted rows.
-    async fn delete(&self, _request: DeleteRequest) -> Result<usize> {
-        UnsupportedSnafu {
-            operation: "DELETE",
-        }
-        .fail()?
-    }
-
-    /// Flush table.
-    ///
-    /// Options:
-    /// - region_number: specify region to flush.
-    /// - wait: Whether to wait until flush is done.
-    async fn flush(&self, region_number: Option<RegionNumber>, wait: Option<bool>) -> Result<()> {
-        let _ = (region_number, wait);
-        UnsupportedSnafu { operation: "FLUSH" }.fail()?
-    }
-
-    /// Close the table.
-    async fn close(&self, _regions: &[RegionNumber]) -> Result<()> {
-        Ok(())
-    }
-
-    /// Return true if contains the region
-    fn contains_region(&self, _region: RegionNumber) -> Result<bool> {
-        UnsupportedSnafu {
-            operation: "contain_region",
-        }
-        .fail()?
-    }
-
-    /// Get statistics for this table, if available
-    fn statistics(&self) -> Option<TableStatistics> {
-        None
-    }
-
-    async fn compact(&self, region_number: Option<RegionNumber>, wait: Option<bool>) -> Result<()> {
-        let _ = (region_number, wait);
-        UnsupportedSnafu {
-            operation: "COMPACTION",
-        }
-        .fail()?
-    }
-
-    async fn truncate(&self) -> Result<()> {
-        UnsupportedSnafu {
-            operation: "TRUNCATE",
-        }
-        .fail()?
-    }
 }
 
 pub type TableRef = Arc<dyn Table>;

From a2fcb2713c87f347abb9587fd5bd8293b8405132 Mon Sep 17 00:00:00 2001
From: Zhenchi <zhongzc_arch@outlook.com>
Date: Tue, 26 Sep 2023 17:04:12 +0800
Subject: [PATCH 25/36] fix: convert datetime to chrono datetime (#2497)

* fix: convert datetime to chrono datetime

Signed-off-by: Zhenchi <zhongzc_arch@outlook.com>

* chore: typo

Signed-off-by: Zhenchi <zhongzc_arch@outlook.com>

* fix the bad fix

Signed-off-by: Zhenchi <zhongzc_arch@outlook.com>

---------

Signed-off-by: Zhenchi <zhongzc_arch@outlook.com>
---
 src/common/time/src/datetime.rs | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/src/common/time/src/datetime.rs b/src/common/time/src/datetime.rs
index 5bdec5dc25c9..39f451db8e11 100644
--- a/src/common/time/src/datetime.rs
+++ b/src/common/time/src/datetime.rs
@@ -102,7 +102,7 @@ impl DateTime {
     }
 
     pub fn to_chrono_datetime(&self) -> Option<NaiveDateTime> {
-        NaiveDateTime::from_timestamp_opt(self.0, 0)
+        NaiveDateTime::from_timestamp_millis(self.0)
     }
 
     pub fn to_date(&self) -> Option<Date> {
@@ -162,4 +162,15 @@ mod tests {
         let datetime = DateTime::from(date);
         assert_eq!(datetime.val(), 185542587100800000);
     }
+
+    #[test]
+    fn test_conversion_between_datetime_and_chrono_datetime() {
+        let cases = [1, 10, 100, 1000, 100000];
+        for case in cases {
+            let dt = DateTime::new(case);
+            let ndt = dt.to_chrono_datetime().unwrap();
+            let dt2 = DateTime::from(ndt);
+            assert_eq!(dt, dt2);
+        }
+    }
 }

From d80977a15d44342cf45b8a21b47829060fda53c9 Mon Sep 17 00:00:00 2001
From: Vanish <devilsuh@163.com>
Date: Tue, 26 Sep 2023 19:14:14 +0800
Subject: [PATCH 26/36] feat: distribute truncate table in region server
 (#2414)

* feat: distribute truncate table

* chore: add metrics for truncate table

* test: add sqlness test

* chore: cr

* test: add multi truncate

* chore: add trace id to the header
---
 Cargo.lock                                    |   2 +-
 Cargo.toml                                    |   2 +-
 src/common/meta/src/ddl.rs                    |   1 +
 src/common/meta/src/ddl/create_table.rs       |   4 +-
 src/common/meta/src/ddl/drop_table.rs         |   4 +-
 src/common/meta/src/ddl/truncate_table.rs     | 234 ++++++++++++++++++
 src/common/meta/src/ddl_manager.rs            |  75 ++++--
 src/common/meta/src/metrics.rs                |   1 +
 src/common/meta/src/rpc/ddl.rs                |  85 +++----
 src/operator/src/statement/ddl.rs             |  23 +-
 src/store-api/src/region_request.rs           |   5 +
 .../common/truncate/truncate.result           |  76 ++++++
 .../standalone/common/truncate/truncate.sql   |  33 +++
 13 files changed, 457 insertions(+), 88 deletions(-)
 create mode 100644 src/common/meta/src/ddl/truncate_table.rs
 create mode 100644 tests/cases/standalone/common/truncate/truncate.result
 create mode 100644 tests/cases/standalone/common/truncate/truncate.sql

diff --git a/Cargo.lock b/Cargo.lock
index 1eaf5b4eaf36..9d039bcf82a7 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -4188,7 +4188,7 @@ checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b"
 [[package]]
 name = "greptime-proto"
 version = "0.1.0"
-source = "git+https://github.com/GreptimeTeam/greptime-proto.git?rev=115c1080773be8a819e50b257fece9f839a0c836#115c1080773be8a819e50b257fece9f839a0c836"
+source = "git+https://github.com/GreptimeTeam/greptime-proto.git?rev=693128abe9adc70ba636010a172c9da55b206bba#693128abe9adc70ba636010a172c9da55b206bba"
 dependencies = [
  "prost",
  "serde",
diff --git a/Cargo.toml b/Cargo.toml
index 426b07d468b2..a89c9f19d3f2 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -78,7 +78,7 @@ datafusion-substrait = { git = "https://github.com/waynexia/arrow-datafusion.git
 derive_builder = "0.12"
 futures = "0.3"
 futures-util = "0.3"
-greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "115c1080773be8a819e50b257fece9f839a0c836" }
+greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "693128abe9adc70ba636010a172c9da55b206bba" }
 humantime-serde = "1.1"
 itertools = "0.10"
 lazy_static = "1.4"
diff --git a/src/common/meta/src/ddl.rs b/src/common/meta/src/ddl.rs
index f84825252435..4ec389199f91 100644
--- a/src/common/meta/src/ddl.rs
+++ b/src/common/meta/src/ddl.rs
@@ -28,6 +28,7 @@ use crate::rpc::router::RegionRoute;
 pub mod alter_table;
 pub mod create_table;
 pub mod drop_table;
+pub mod truncate_table;
 pub mod utils;
 
 #[derive(Debug, Default)]
diff --git a/src/common/meta/src/ddl/create_table.rs b/src/common/meta/src/ddl/create_table.rs
index 87a6cf0dc046..975c2aa471fb 100644
--- a/src/common/meta/src/ddl/create_table.rs
+++ b/src/common/meta/src/ddl/create_table.rs
@@ -199,8 +199,8 @@ impl CreateTableProcedure {
             for request in requests {
                 let request = RegionRequest {
                     header: Some(RegionRequestHeader {
-                        trace_id: 0,
-                        span_id: 0,
+                        trace_id: common_telemetry::trace_id().unwrap_or_default(),
+                        ..Default::default()
                     }),
                     body: Some(request),
                 };
diff --git a/src/common/meta/src/ddl/drop_table.rs b/src/common/meta/src/ddl/drop_table.rs
index f8c11324fa8e..9da7759cd006 100644
--- a/src/common/meta/src/ddl/drop_table.rs
+++ b/src/common/meta/src/ddl/drop_table.rs
@@ -156,8 +156,8 @@ impl DropTableProcedure {
 
                 let request = RegionRequest {
                     header: Some(RegionRequestHeader {
-                        trace_id: 0,
-                        span_id: 0,
+                        trace_id: common_telemetry::trace_id().unwrap_or_default(),
+                        ..Default::default()
                     }),
                     body: Some(region_request::Body::Drop(PbDropRegionRequest {
                         region_id: region_id.as_u64(),
diff --git a/src/common/meta/src/ddl/truncate_table.rs b/src/common/meta/src/ddl/truncate_table.rs
new file mode 100644
index 000000000000..831b41a631f0
--- /dev/null
+++ b/src/common/meta/src/ddl/truncate_table.rs
@@ -0,0 +1,234 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use api::v1::region::{
+    region_request, RegionRequest, RegionRequestHeader, TruncateRequest as PbTruncateRegionRequest,
+};
+use async_trait::async_trait;
+use common_procedure::error::{FromJsonSnafu, ToJsonSnafu};
+use common_procedure::{
+    Context as ProcedureContext, LockKey, Procedure, Result as ProcedureResult, Status,
+};
+use common_telemetry::debug;
+use futures::future::join_all;
+use serde::{Deserialize, Serialize};
+use snafu::{ensure, ResultExt};
+use store_api::storage::RegionId;
+use strum::AsRefStr;
+use table::engine::TableReference;
+use table::metadata::{RawTableInfo, TableId};
+
+use super::utils::handle_retry_error;
+use crate::ddl::utils::handle_operate_region_error;
+use crate::ddl::DdlContext;
+use crate::error::{Result, TableNotFoundSnafu};
+use crate::key::table_info::TableInfoValue;
+use crate::key::table_name::TableNameKey;
+use crate::metrics;
+use crate::rpc::ddl::TruncateTableTask;
+use crate::rpc::router::{find_leader_regions, find_leaders, RegionRoute};
+use crate::table_name::TableName;
+
+pub struct TruncateTableProcedure {
+    context: DdlContext,
+    data: TruncateTableData,
+}
+
+#[async_trait]
+impl Procedure for TruncateTableProcedure {
+    fn type_name(&self) -> &str {
+        Self::TYPE_NAME
+    }
+
+    async fn execute(&mut self, _ctx: &ProcedureContext) -> ProcedureResult<Status> {
+        let state = &self.data.state;
+
+        let _timer = common_telemetry::timer!(
+            metrics::METRIC_META_PROCEDURE_TRUNCATE_TABLE,
+            &[("step", state.as_ref().to_string())]
+        );
+
+        match self.data.state {
+            TruncateTableState::Prepare => self.on_prepare().await,
+            TruncateTableState::DatanodeTruncateRegions => {
+                self.on_datanode_truncate_regions().await
+            }
+        }
+        .map_err(handle_retry_error)
+    }
+
+    fn dump(&self) -> ProcedureResult<String> {
+        serde_json::to_string(&self.data).context(ToJsonSnafu)
+    }
+
+    fn lock_key(&self) -> LockKey {
+        let table_ref = &self.data.table_ref();
+        let key = common_catalog::format_full_table_name(
+            table_ref.catalog,
+            table_ref.schema,
+            table_ref.table,
+        );
+
+        LockKey::single(key)
+    }
+}
+
+impl TruncateTableProcedure {
+    pub(crate) const TYPE_NAME: &'static str = "metasrv-procedure::TruncateTable";
+
+    pub(crate) fn new(
+        cluster_id: u64,
+        task: TruncateTableTask,
+        table_info_value: TableInfoValue,
+        region_routes: Vec<RegionRoute>,
+        context: DdlContext,
+    ) -> Self {
+        Self {
+            context,
+            data: TruncateTableData::new(cluster_id, task, table_info_value, region_routes),
+        }
+    }
+
+    pub(crate) fn from_json(json: &str, context: DdlContext) -> ProcedureResult<Self> {
+        let data = serde_json::from_str(json).context(FromJsonSnafu)?;
+        Ok(Self { context, data })
+    }
+
+    // Checks whether the table exists.
+    async fn on_prepare(&mut self) -> Result<Status> {
+        let table_ref = &self.data.table_ref();
+
+        let manager = &self.context.table_metadata_manager;
+
+        let exist = manager
+            .table_name_manager()
+            .exists(TableNameKey::new(
+                table_ref.catalog,
+                table_ref.schema,
+                table_ref.table,
+            ))
+            .await?;
+
+        ensure!(
+            exist,
+            TableNotFoundSnafu {
+                table_name: table_ref.to_string()
+            }
+        );
+
+        self.data.state = TruncateTableState::DatanodeTruncateRegions;
+
+        Ok(Status::executing(true))
+    }
+
+    async fn on_datanode_truncate_regions(&mut self) -> Result<Status> {
+        let table_id = self.data.table_id();
+
+        let region_routes = &self.data.region_routes;
+        let leaders = find_leaders(region_routes);
+        let mut truncate_region_tasks = Vec::with_capacity(leaders.len());
+
+        for datanode in leaders {
+            let requester = self.context.datanode_manager.datanode(&datanode).await;
+            let regions = find_leader_regions(region_routes, &datanode);
+
+            for region in regions {
+                let region_id = RegionId::new(table_id, region);
+                debug!(
+                    "Truncating table {} region {} on Datanode {:?}",
+                    self.data.table_ref(),
+                    region_id,
+                    datanode
+                );
+
+                let request = RegionRequest {
+                    header: Some(RegionRequestHeader {
+                        trace_id: common_telemetry::trace_id().unwrap_or_default(),
+                        ..Default::default()
+                    }),
+                    body: Some(region_request::Body::Truncate(PbTruncateRegionRequest {
+                        region_id: region_id.as_u64(),
+                    })),
+                };
+
+                let datanode = datanode.clone();
+                let requester = requester.clone();
+
+                truncate_region_tasks.push(async move {
+                    if let Err(err) = requester.handle(request).await {
+                        return Err(handle_operate_region_error(datanode)(err));
+                    }
+                    Ok(())
+                });
+            }
+        }
+
+        join_all(truncate_region_tasks)
+            .await
+            .into_iter()
+            .collect::<Result<Vec<_>>>()?;
+
+        Ok(Status::Done)
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct TruncateTableData {
+    state: TruncateTableState,
+    cluster_id: u64,
+    task: TruncateTableTask,
+    table_info_value: TableInfoValue,
+    region_routes: Vec<RegionRoute>,
+}
+
+impl TruncateTableData {
+    pub fn new(
+        cluster_id: u64,
+        task: TruncateTableTask,
+        table_info_value: TableInfoValue,
+        region_routes: Vec<RegionRoute>,
+    ) -> Self {
+        Self {
+            state: TruncateTableState::Prepare,
+            cluster_id,
+            task,
+            table_info_value,
+            region_routes,
+        }
+    }
+
+    pub fn table_ref(&self) -> TableReference {
+        self.task.table_ref()
+    }
+
+    pub fn table_name(&self) -> TableName {
+        self.task.table_name()
+    }
+
+    fn table_info(&self) -> &RawTableInfo {
+        &self.table_info_value.table_info
+    }
+
+    fn table_id(&self) -> TableId {
+        self.table_info().ident.table_id
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize, AsRefStr)]
+enum TruncateTableState {
+    /// Prepares to truncate the table
+    Prepare,
+    /// Truncates regions on Datanode
+    DatanodeTruncateRegions,
+}
diff --git a/src/common/meta/src/ddl_manager.rs b/src/common/meta/src/ddl_manager.rs
index aaa1e1c52104..62af640a3c57 100644
--- a/src/common/meta/src/ddl_manager.rs
+++ b/src/common/meta/src/ddl_manager.rs
@@ -15,7 +15,7 @@
 use std::sync::Arc;
 
 use common_procedure::{watcher, ProcedureId, ProcedureManagerRef, ProcedureWithId};
-use common_telemetry::{error, info};
+use common_telemetry::info;
 use snafu::{OptionExt, ResultExt};
 
 use crate::cache_invalidator::CacheInvalidatorRef;
@@ -23,13 +23,14 @@ use crate::datanode_manager::DatanodeManagerRef;
 use crate::ddl::alter_table::AlterTableProcedure;
 use crate::ddl::create_table::CreateTableProcedure;
 use crate::ddl::drop_table::DropTableProcedure;
+use crate::ddl::truncate_table::TruncateTableProcedure;
 use crate::ddl::{
     DdlContext, DdlTaskExecutor, ExecutorContext, TableMetadataAllocatorContext,
     TableMetadataAllocatorRef,
 };
 use crate::error::{
     self, RegisterProcedureLoaderSnafu, Result, SubmitProcedureSnafu, TableNotFoundSnafu,
-    UnsupportedSnafu, WaitProcedureSnafu,
+    WaitProcedureSnafu,
 };
 use crate::key::table_info::TableInfoValue;
 use crate::key::table_name::TableNameKey;
@@ -122,6 +123,20 @@ impl DdlManager {
             )
             .context(RegisterProcedureLoaderSnafu {
                 type_name: AlterTableProcedure::TYPE_NAME,
+            })?;
+
+        let context = self.create_context();
+
+        self.procedure_manager
+            .register_loader(
+                TruncateTableProcedure::TYPE_NAME,
+                Box::new(move |json| {
+                    let context = context.clone();
+                    TruncateTableProcedure::from_json(json, context).map(|p| Box::new(p) as _)
+                }),
+            )
+            .context(RegisterProcedureLoaderSnafu {
+                type_name: TruncateTableProcedure::TYPE_NAME,
             })
     }
 
@@ -183,15 +198,21 @@ impl DdlManager {
         &self,
         cluster_id: u64,
         truncate_table_task: TruncateTableTask,
+        table_info_value: TableInfoValue,
         region_routes: Vec<RegionRoute>,
     ) -> Result<ProcedureId> {
-        error!("Truncate table procedure is not supported, cluster_id = {}, truncate_table_task = {:?}, region_routes = {:?}",
-            cluster_id, truncate_table_task, region_routes);
+        let context = self.create_context();
+        let procedure = TruncateTableProcedure::new(
+            cluster_id,
+            truncate_table_task,
+            table_info_value,
+            region_routes,
+            context,
+        );
 
-        UnsupportedSnafu {
-            operation: "TRUNCATE TABLE",
-        }
-        .fail()
+        let procedure_with_id = ProcedureWithId::with_random_id(Box::new(procedure));
+
+        self.submit_procedure(procedure_with_id).await
     }
 
     async fn submit_procedure(&self, procedure_with_id: ProcedureWithId) -> Result<ProcedureId> {
@@ -216,32 +237,34 @@ async fn handle_truncate_table_task(
     cluster_id: u64,
     truncate_table_task: TruncateTableTask,
 ) -> Result<SubmitDdlTaskResponse> {
-    let truncate_table = &truncate_table_task.truncate_table;
-    let table_id = truncate_table
-        .table_id
-        .as_ref()
-        .context(error::UnexpectedSnafu {
-            err_msg: "expected table id ",
-        })?
-        .id;
-
+    let table_id = truncate_table_task.table_id;
+    let table_metadata_manager = &ddl_manager.table_metadata_manager();
     let table_ref = truncate_table_task.table_ref();
 
-    let table_route_value = ddl_manager
-        .table_metadata_manager()
-        .table_route_manager()
-        .get(table_id)
-        .await?
-        .with_context(|| error::TableRouteNotFoundSnafu {
-            table_name: table_ref.to_string(),
-        })?;
+    let (table_info_value, table_route_value) =
+        table_metadata_manager.get_full_table_info(table_id).await?;
+
+    let table_info_value = table_info_value.with_context(|| error::TableInfoNotFoundSnafu {
+        table_name: table_ref.to_string(),
+    })?;
+
+    let table_route_value = table_route_value.with_context(|| error::TableRouteNotFoundSnafu {
+        table_name: table_ref.to_string(),
+    })?;
 
     let table_route = table_route_value.region_routes;
 
     let id = ddl_manager
-        .submit_truncate_table_task(cluster_id, truncate_table_task, table_route)
+        .submit_truncate_table_task(
+            cluster_id,
+            truncate_table_task,
+            table_info_value,
+            table_route,
+        )
         .await?;
 
+    info!("Table: {table_id} is truncated via procedure_id {id:?}");
+
     Ok(SubmitDdlTaskResponse {
         key: id.to_string().into(),
         ..Default::default()
diff --git a/src/common/meta/src/metrics.rs b/src/common/meta/src/metrics.rs
index 49535607af72..815c8aacb1cf 100644
--- a/src/common/meta/src/metrics.rs
+++ b/src/common/meta/src/metrics.rs
@@ -19,3 +19,4 @@ pub(crate) const METRIC_META_CREATE_SCHEMA: &str = "meta.create_schema";
 pub(crate) const METRIC_META_PROCEDURE_CREATE_TABLE: &str = "meta.procedure.create_table";
 pub(crate) const METRIC_META_PROCEDURE_DROP_TABLE: &str = "meta.procedure.drop_table";
 pub(crate) const METRIC_META_PROCEDURE_ALTER_TABLE: &str = "meta.procedure.alter_table";
+pub(crate) const METRIC_META_PROCEDURE_TRUNCATE_TABLE: &str = "meta.procedure.truncate_table";
diff --git a/src/common/meta/src/rpc/ddl.rs b/src/common/meta/src/rpc/ddl.rs
index d1ca0dea94f7..aed5fa1131b3 100644
--- a/src/common/meta/src/rpc/ddl.rs
+++ b/src/common/meta/src/rpc/ddl.rs
@@ -65,8 +65,18 @@ impl DdlTask {
         DdlTask::AlterTable(AlterTableTask { alter_table })
     }
 
-    pub fn new_truncate_table(truncate_table: TruncateTableExpr) -> Self {
-        DdlTask::TruncateTable(TruncateTableTask { truncate_table })
+    pub fn new_truncate_table(
+        catalog: String,
+        schema: String,
+        table: String,
+        table_id: TableId,
+    ) -> Self {
+        DdlTask::TruncateTable(TruncateTableTask {
+            catalog,
+            schema,
+            table,
+            table_id,
+        })
     }
 }
 
@@ -112,7 +122,12 @@ impl TryFrom<SubmitDdlTaskRequest> for PbSubmitDdlTaskRequest {
                 alter_table: Some(task.alter_table),
             }),
             DdlTask::TruncateTable(task) => Task::TruncateTableTask(PbTruncateTableTask {
-                truncate_table: Some(task.truncate_table),
+                truncate_table: Some(TruncateTableExpr {
+                    catalog_name: task.catalog,
+                    schema_name: task.schema,
+                    table_name: task.table,
+                    table_id: Some(api::v1::TableId { id: task.table_id }),
+                }),
             }),
         };
 
@@ -358,27 +373,28 @@ impl<'de> Deserialize<'de> for AlterTableTask {
     }
 }
 
-#[derive(Debug, PartialEq)]
+#[derive(Debug, PartialEq, Serialize, Deserialize)]
 pub struct TruncateTableTask {
-    pub truncate_table: TruncateTableExpr,
+    pub catalog: String,
+    pub schema: String,
+    pub table: String,
+    pub table_id: TableId,
 }
 
 impl TruncateTableTask {
     pub fn table_ref(&self) -> TableReference {
         TableReference {
-            catalog: &self.truncate_table.catalog_name,
-            schema: &self.truncate_table.schema_name,
-            table: &self.truncate_table.table_name,
+            catalog: &self.catalog,
+            schema: &self.schema,
+            table: &self.table,
         }
     }
 
     pub fn table_name(&self) -> TableName {
-        let table = &self.truncate_table;
-
         TableName {
-            catalog_name: table.catalog_name.to_string(),
-            schema_name: table.schema_name.to_string(),
-            table_name: table.table_name.to_string(),
+            catalog_name: self.catalog.to_string(),
+            schema_name: self.schema.to_string(),
+            table_name: self.table.to_string(),
         }
     }
 }
@@ -388,39 +404,20 @@ impl TryFrom<PbTruncateTableTask> for TruncateTableTask {
 
     fn try_from(pb: PbTruncateTableTask) -> Result<Self> {
         let truncate_table = pb.truncate_table.context(error::InvalidProtoMsgSnafu {
-            err_msg: "expected truncate_table",
+            err_msg: "expected drop table",
         })?;
 
-        Ok(TruncateTableTask { truncate_table })
-    }
-}
-
-impl Serialize for TruncateTableTask {
-    fn serialize<S>(&self, serializer: S) -> result::Result<S::Ok, S::Error>
-    where
-        S: serde::Serializer,
-    {
-        let pb = PbTruncateTableTask {
-            truncate_table: Some(self.truncate_table.clone()),
-        };
-        let buf = pb.encode_to_vec();
-        serializer.serialize_bytes(&buf)
-    }
-}
-
-impl<'de> Deserialize<'de> for TruncateTableTask {
-    fn deserialize<D>(deserializer: D) -> result::Result<Self, D::Error>
-    where
-        D: serde::Deserializer<'de>,
-    {
-        let buf = Vec::<u8>::deserialize(deserializer)?;
-        let task: PbTruncateTableTask = PbTruncateTableTask::decode(&*buf)
-            .map_err(|err| serde::de::Error::custom(err.to_string()))?;
-
-        let task = TruncateTableTask::try_from(task)
-            .map_err(|err| serde::de::Error::custom(err.to_string()))?;
-
-        Ok(task)
+        Ok(Self {
+            catalog: truncate_table.catalog_name,
+            schema: truncate_table.schema_name,
+            table: truncate_table.table_name,
+            table_id: truncate_table
+                .table_id
+                .context(error::InvalidProtoMsgSnafu {
+                    err_msg: "expected table_id",
+                })?
+                .id,
+        })
     }
 }
 
diff --git a/src/operator/src/statement/ddl.rs b/src/operator/src/statement/ddl.rs
index f2752eba6b46..2173e030c772 100644
--- a/src/operator/src/statement/ddl.rs
+++ b/src/operator/src/statement/ddl.rs
@@ -16,7 +16,7 @@ use std::collections::HashMap;
 use std::sync::Arc;
 
 use api::helper::ColumnDataTypeWrapper;
-use api::v1::{column_def, AlterExpr, CreateTableExpr, TruncateTableExpr};
+use api::v1::{column_def, AlterExpr, CreateTableExpr};
 use catalog::CatalogManagerRef;
 use chrono::DateTime;
 use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
@@ -169,15 +169,8 @@ impl StatementExecutor {
             .with_context(|| TableNotFoundSnafu {
                 table_name: table_name.to_string(),
             })?;
-        let table_id = table.table_info().ident.table_id;
-
-        let expr = TruncateTableExpr {
-            catalog_name: table_name.catalog_name.clone(),
-            schema_name: table_name.schema_name.clone(),
-            table_name: table_name.table_name.clone(),
-            table_id: Some(api::v1::TableId { id: table_id }),
-        };
-        self.truncate_table_procedure(&expr).await?;
+        let table_id = table.table_info().table_id();
+        self.truncate_table_procedure(&table_name, table_id).await?;
 
         Ok(Output::AffectedRows(0))
     }
@@ -312,10 +305,16 @@ impl StatementExecutor {
 
     async fn truncate_table_procedure(
         &self,
-        truncate_table: &TruncateTableExpr,
+        table_name: &TableName,
+        table_id: TableId,
     ) -> Result<SubmitDdlTaskResponse> {
         let request = SubmitDdlTaskRequest {
-            task: DdlTask::new_truncate_table(truncate_table.clone()),
+            task: DdlTask::new_truncate_table(
+                table_name.catalog_name.to_string(),
+                table_name.schema_name.to_string(),
+                table_name.table_name.to_string(),
+                table_id,
+            ),
         };
 
         self.ddl_executor
diff --git a/src/store-api/src/region_request.rs b/src/store-api/src/region_request.rs
index 9085943f21e0..3e202912c743 100644
--- a/src/store-api/src/region_request.rs
+++ b/src/store-api/src/region_request.rs
@@ -116,6 +116,10 @@ impl RegionRequest {
                 compact.region_id.into(),
                 Self::Compact(RegionCompactRequest {}),
             )]),
+            region_request::Body::Truncate(truncate) => Ok(vec![(
+                truncate.region_id.into(),
+                Self::Truncate(RegionTruncateRequest {}),
+            )]),
         }
     }
 }
@@ -417,6 +421,7 @@ pub struct RegionFlushRequest {}
 #[derive(Debug)]
 pub struct RegionCompactRequest {}
 
+/// Truncate region request.
 #[derive(Debug)]
 pub struct RegionTruncateRequest {}
 
diff --git a/tests/cases/standalone/common/truncate/truncate.result b/tests/cases/standalone/common/truncate/truncate.result
new file mode 100644
index 000000000000..4267bc04d719
--- /dev/null
+++ b/tests/cases/standalone/common/truncate/truncate.result
@@ -0,0 +1,76 @@
+TRUNCATE TABLE not_exits_table;
+
+Error: 4001(TableNotFound), Table not found: greptime.public.not_exits_table
+
+CREATE TABLE monitor (host STRING, ts TIMESTAMP, cpu DOUBLE DEFAULT 0, memory DOUBLE, TIME INDEX (ts), PRIMARY KEY(host));
+
+Affected Rows: 0
+
+INSERT INTO monitor(ts, host, cpu, memory) VALUES
+(1695217652000, 'host1', 66.6, 1024),
+(1695217652000, 'host2', 66.6, 1024),
+(1695217652000, 'host3', 66.6, 1024),
+(1695217654000, 'host1', 77.7, 2048),
+(1695217654000, 'host2', 77.7, 2048),
+(1695217654000, 'host3', 77.7, 2048),
+(1695217656000, 'host1', 88.8, 4096),
+(1695217656000, 'host2', 88.8, 4096),
+(1695217656000, 'host3', 88.8, 4096);
+
+Affected Rows: 9
+
+SELECT ts, host, cpu, memory FROM monitor ORDER BY ts;
+
++---------------------+-------+------+--------+
+| ts                  | host  | cpu  | memory |
++---------------------+-------+------+--------+
+| 2023-09-20T13:47:32 | host1 | 66.6 | 1024.0 |
+| 2023-09-20T13:47:32 | host2 | 66.6 | 1024.0 |
+| 2023-09-20T13:47:32 | host3 | 66.6 | 1024.0 |
+| 2023-09-20T13:47:34 | host1 | 77.7 | 2048.0 |
+| 2023-09-20T13:47:34 | host2 | 77.7 | 2048.0 |
+| 2023-09-20T13:47:34 | host3 | 77.7 | 2048.0 |
+| 2023-09-20T13:47:36 | host1 | 88.8 | 4096.0 |
+| 2023-09-20T13:47:36 | host2 | 88.8 | 4096.0 |
+| 2023-09-20T13:47:36 | host3 | 88.8 | 4096.0 |
++---------------------+-------+------+--------+
+
+TRUNCATE monitor;
+
+Affected Rows: 0
+
+SELECT ts, host, cpu, memory FROM monitor ORDER BY ts;
+
+++
+++
+
+INSERT INTO monitor(ts, host, cpu, memory) VALUES 
+(1695217660000, 'host1', 88.8, 4096),
+(1695217662000, 'host2', 88.8, 4096),
+(1695217664000, 'host3', 88.8, 4096);
+
+Affected Rows: 3
+
+SELECT ts, host, cpu, memory FROM monitor ORDER BY ts;
+
++---------------------+-------+------+--------+
+| ts                  | host  | cpu  | memory |
++---------------------+-------+------+--------+
+| 2023-09-20T13:47:40 | host1 | 88.8 | 4096.0 |
+| 2023-09-20T13:47:42 | host2 | 88.8 | 4096.0 |
+| 2023-09-20T13:47:44 | host3 | 88.8 | 4096.0 |
++---------------------+-------+------+--------+
+
+TRUNCATE monitor;
+
+Affected Rows: 0
+
+SELECT ts, host, cpu, memory FROM monitor ORDER BY ts;
+
+++
+++
+
+DROP TABLE monitor;
+
+Affected Rows: 1
+
diff --git a/tests/cases/standalone/common/truncate/truncate.sql b/tests/cases/standalone/common/truncate/truncate.sql
new file mode 100644
index 000000000000..61b69901254d
--- /dev/null
+++ b/tests/cases/standalone/common/truncate/truncate.sql
@@ -0,0 +1,33 @@
+TRUNCATE TABLE not_exits_table;
+
+CREATE TABLE monitor (host STRING, ts TIMESTAMP, cpu DOUBLE DEFAULT 0, memory DOUBLE, TIME INDEX (ts), PRIMARY KEY(host));
+
+INSERT INTO monitor(ts, host, cpu, memory) VALUES
+(1695217652000, 'host1', 66.6, 1024),
+(1695217652000, 'host2', 66.6, 1024),
+(1695217652000, 'host3', 66.6, 1024),
+(1695217654000, 'host1', 77.7, 2048),
+(1695217654000, 'host2', 77.7, 2048),
+(1695217654000, 'host3', 77.7, 2048),
+(1695217656000, 'host1', 88.8, 4096),
+(1695217656000, 'host2', 88.8, 4096),
+(1695217656000, 'host3', 88.8, 4096);
+
+SELECT ts, host, cpu, memory FROM monitor ORDER BY ts;
+
+TRUNCATE monitor;
+
+SELECT ts, host, cpu, memory FROM monitor ORDER BY ts;
+
+INSERT INTO monitor(ts, host, cpu, memory) VALUES 
+(1695217660000, 'host1', 88.8, 4096),
+(1695217662000, 'host2', 88.8, 4096),
+(1695217664000, 'host3', 88.8, 4096);
+
+SELECT ts, host, cpu, memory FROM monitor ORDER BY ts;
+
+TRUNCATE monitor;
+
+SELECT ts, host, cpu, memory FROM monitor ORDER BY ts;
+
+DROP TABLE monitor;

From c53f3667827b290a9523fcb2205eb97771779162 Mon Sep 17 00:00:00 2001
From: Ruihang Xia <waynestxia@gmail.com>
Date: Tue, 26 Sep 2023 19:23:21 +0800
Subject: [PATCH 27/36] feat: stack trace style debug print for error (#2489)

* impl macro stack_trace_debug

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>

* manually mark external error

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>

* ignore warnings

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>

* fix clippy warnings

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>

* use debug print

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>

* simplify the error and warn macro

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>

* fix ut

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>

* add docs

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>

* replace snafu backtrace with location

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>

---------

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>
---
 Cargo.lock                                 |  35 +++
 src/api/Cargo.toml                         |   1 +
 src/api/src/error.rs                       |   4 +-
 src/auth/Cargo.toml                        |   1 +
 src/auth/src/error.rs                      |   7 +-
 src/catalog/Cargo.toml                     |   1 +
 src/catalog/src/error.rs                   |  15 +-
 src/client/Cargo.toml                      |   1 +
 src/client/examples/stream_ingest.rs       |   6 +-
 src/client/src/database.rs                 |   2 +-
 src/client/src/error.rs                    |   4 +-
 src/cmd/Cargo.toml                         |   1 +
 src/cmd/src/error.rs                       |  16 +-
 src/common/base/Cargo.toml                 |   1 +
 src/common/base/src/buffer.rs              |   7 +-
 src/common/catalog/Cargo.toml              |   1 +
 src/common/catalog/src/error.rs            |   4 +-
 src/common/datasource/Cargo.toml           |   1 +
 src/common/datasource/src/error.rs         |  49 ++--
 src/common/error/src/ext.rs                |  49 +++-
 src/common/error/src/format.rs             |  28 +++
 src/common/error/src/mock.rs               |  10 +-
 src/common/grpc-expr/Cargo.toml            |   1 +
 src/common/grpc-expr/src/error.rs          |   4 +-
 src/common/grpc/Cargo.toml                 |   1 +
 src/common/grpc/src/error.rs               |  13 +-
 src/common/macro/Cargo.toml                |   9 +
 src/common/macro/src/lib.rs                |  21 ++
 src/common/macro/src/stack_trace_debug.rs  | 278 +++++++++++++++++++++
 src/common/mem-prof/Cargo.toml             |   1 +
 src/common/mem-prof/src/error.rs           |   6 +-
 src/common/mem-prof/src/jemalloc/error.rs  |  15 +-
 src/common/meta/Cargo.toml                 |   1 +
 src/common/meta/src/error.rs               |  22 +-
 src/common/procedure/Cargo.toml            |   1 +
 src/common/procedure/src/error.rs          |  21 +-
 src/common/procedure/src/local.rs          |   2 +-
 src/common/procedure/src/store.rs          |   2 +-
 src/common/query/Cargo.toml                |   1 +
 src/common/query/src/error.rs              |  19 +-
 src/common/recordbatch/Cargo.toml          |   1 +
 src/common/recordbatch/src/error.rs        |  19 +-
 src/common/runtime/Cargo.toml              |   1 +
 src/common/runtime/src/error.rs            |  10 +-
 src/common/substrait/Cargo.toml            |   1 +
 src/common/substrait/src/error.rs          |  19 +-
 src/common/telemetry/src/macros.rs         | 158 +++---------
 src/common/time/Cargo.toml                 |   1 +
 src/common/time/src/error.rs               |  16 +-
 src/datanode/Cargo.toml                    |   1 +
 src/datanode/src/error.rs                  |  28 ++-
 src/datatypes/Cargo.toml                   |   1 +
 src/datatypes/src/error.rs                 |  22 +-
 src/file-engine/Cargo.toml                 |   1 +
 src/file-engine/src/error.rs               |  37 ++-
 src/frontend/Cargo.toml                    |   1 +
 src/frontend/src/error.rs                  |  72 +++---
 src/frontend/src/server.rs                 |  10 +-
 src/log-store/Cargo.toml                   |   1 +
 src/log-store/src/error.rs                 |  13 +-
 src/meta-client/Cargo.toml                 |   1 +
 src/meta-client/src/error.rs               |   7 +-
 src/meta-srv/Cargo.toml                    |   1 +
 src/meta-srv/src/cluster.rs                |   4 +-
 src/meta-srv/src/error.rs                  |  78 ++++--
 src/mito2/Cargo.toml                       |   1 +
 src/mito2/src/error.rs                     |  60 +++--
 src/operator/Cargo.toml                    |   1 +
 src/operator/src/error.rs                  | 146 ++++++-----
 src/partition/Cargo.toml                   |   1 +
 src/partition/src/error.rs                 |  10 +-
 src/promql/src/error.rs                    |  10 +-
 src/query/Cargo.toml                       |   1 +
 src/query/src/datafusion/error.rs          |   7 +-
 src/query/src/error.rs                     |  19 +-
 src/script/Cargo.toml                      |   1 +
 src/script/src/error.rs                    |   7 +-
 src/script/src/python/error.rs             |  25 +-
 src/script/src/python/rspython/test.rs     |  20 +-
 src/servers/Cargo.toml                     |   1 +
 src/servers/src/error.rs                   |  83 ++++--
 src/servers/src/metrics/jemalloc/error.rs  |   7 +-
 src/sql/Cargo.toml                         |   1 +
 src/sql/src/error.rs                       |  13 +-
 src/sql/src/statements.rs                  |   2 +-
 src/storage/Cargo.toml                     |   1 +
 src/storage/src/error.rs                   |  82 ++++--
 src/storage/src/metadata.rs                |  10 +-
 src/store-api/Cargo.toml                   |   1 +
 src/store-api/src/error.rs                 |   7 +-
 src/store-api/src/logstore/entry_stream.rs |   9 +
 src/store-api/src/metadata.rs              |   7 +-
 src/table/Cargo.toml                       |   1 +
 src/table/src/error.rs                     |  13 +-
 94 files changed, 1209 insertions(+), 504 deletions(-)
 create mode 100644 src/common/macro/src/stack_trace_debug.rs

diff --git a/Cargo.lock b/Cargo.lock
index 9d039bcf82a7..463bb474d98d 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -208,6 +208,7 @@ version = "0.4.0-nightly"
 dependencies = [
  "common-base",
  "common-error",
+ "common-macro",
  "common-time",
  "datatypes",
  "greptime-proto",
@@ -690,6 +691,7 @@ dependencies = [
  "api",
  "async-trait",
  "common-error",
+ "common-macro",
  "common-test-util",
  "digest",
  "hex",
@@ -1250,6 +1252,7 @@ dependencies = [
  "common-catalog",
  "common-error",
  "common-grpc",
+ "common-macro",
  "common-meta",
  "common-query",
  "common-recordbatch",
@@ -1532,6 +1535,7 @@ dependencies = [
  "common-error",
  "common-grpc",
  "common-grpc-expr",
+ "common-macro",
  "common-meta",
  "common-query",
  "common-recordbatch",
@@ -1600,6 +1604,7 @@ dependencies = [
  "common-catalog",
  "common-config",
  "common-error",
+ "common-macro",
  "common-meta",
  "common-procedure",
  "common-query",
@@ -1672,6 +1677,7 @@ dependencies = [
  "bitvec",
  "bytes",
  "common-error",
+ "common-macro",
  "paste",
  "serde",
  "snafu",
@@ -1684,6 +1690,7 @@ version = "0.4.0-nightly"
 dependencies = [
  "chrono",
  "common-error",
+ "common-macro",
  "serde",
  "serde_json",
  "snafu",
@@ -1709,6 +1716,7 @@ dependencies = [
  "async-trait",
  "bytes",
  "common-error",
+ "common-macro",
  "common-runtime",
  "common-test-util",
  "datafusion",
@@ -1787,6 +1795,7 @@ dependencies = [
  "backtrace",
  "common-base",
  "common-error",
+ "common-macro",
  "common-recordbatch",
  "common-runtime",
  "common-telemetry",
@@ -1815,6 +1824,7 @@ dependencies = [
  "common-base",
  "common-catalog",
  "common-error",
+ "common-macro",
  "common-query",
  "common-telemetry",
  "common-time",
@@ -1838,6 +1848,7 @@ dependencies = [
  "snafu",
  "static_assertions",
  "syn 1.0.109",
+ "syn 2.0.29",
 ]
 
 [[package]]
@@ -1845,6 +1856,7 @@ name = "common-mem-prof"
 version = "0.4.0-nightly"
 dependencies = [
  "common-error",
+ "common-macro",
  "snafu",
  "tempfile",
  "tikv-jemalloc-ctl",
@@ -1864,6 +1876,7 @@ dependencies = [
  "common-catalog",
  "common-error",
  "common-grpc-expr",
+ "common-macro",
  "common-procedure",
  "common-recordbatch",
  "common-runtime",
@@ -1895,6 +1908,7 @@ dependencies = [
  "async-trait",
  "backon",
  "common-error",
+ "common-macro",
  "common-runtime",
  "common-telemetry",
  "common-test-util",
@@ -1926,6 +1940,7 @@ dependencies = [
  "async-trait",
  "common-base",
  "common-error",
+ "common-macro",
  "common-recordbatch",
  "common-time",
  "datafusion",
@@ -1945,6 +1960,7 @@ name = "common-recordbatch"
 version = "0.4.0-nightly"
 dependencies = [
  "common-error",
+ "common-macro",
  "datafusion",
  "datafusion-common",
  "datatypes",
@@ -1962,6 +1978,7 @@ version = "0.4.0-nightly"
 dependencies = [
  "async-trait",
  "common-error",
+ "common-macro",
  "common-telemetry",
  "metrics",
  "once_cell",
@@ -2016,6 +2033,7 @@ dependencies = [
  "chrono",
  "chrono-tz 0.8.3",
  "common-error",
+ "common-macro",
  "rand",
  "serde",
  "serde_json",
@@ -2684,6 +2702,7 @@ dependencies = [
  "common-greptimedb-telemetry",
  "common-grpc",
  "common-grpc-expr",
+ "common-macro",
  "common-meta",
  "common-procedure",
  "common-query",
@@ -2741,6 +2760,7 @@ dependencies = [
  "arrow-schema",
  "common-base",
  "common-error",
+ "common-macro",
  "common-telemetry",
  "common-time",
  "datafusion-common",
@@ -3203,6 +3223,7 @@ dependencies = [
  "common-catalog",
  "common-datasource",
  "common-error",
+ "common-macro",
  "common-procedure",
  "common-procedure-test",
  "common-query",
@@ -3325,6 +3346,7 @@ dependencies = [
  "common-function",
  "common-grpc",
  "common-grpc-expr",
+ "common-macro",
  "common-meta",
  "common-procedure",
  "common-query",
@@ -5030,6 +5052,7 @@ dependencies = [
  "common-base",
  "common-config",
  "common-error",
+ "common-macro",
  "common-meta",
  "common-runtime",
  "common-telemetry",
@@ -5305,6 +5328,7 @@ dependencies = [
  "chrono",
  "common-error",
  "common-grpc",
+ "common-macro",
  "common-meta",
  "common-telemetry",
  "datatypes",
@@ -5341,6 +5365,7 @@ dependencies = [
  "common-greptimedb-telemetry",
  "common-grpc",
  "common-grpc-expr",
+ "common-macro",
  "common-meta",
  "common-procedure",
  "common-procedure-test",
@@ -5532,6 +5557,7 @@ dependencies = [
  "common-catalog",
  "common-datasource",
  "common-error",
+ "common-macro",
  "common-procedure",
  "common-procedure-test",
  "common-query",
@@ -6241,6 +6267,7 @@ dependencies = [
  "common-datasource",
  "common-error",
  "common-grpc-expr",
+ "common-macro",
  "common-meta",
  "common-query",
  "common-recordbatch",
@@ -6503,6 +6530,7 @@ dependencies = [
  "async-trait",
  "common-catalog",
  "common-error",
+ "common-macro",
  "common-meta",
  "common-query",
  "common-telemetry",
@@ -8626,6 +8654,7 @@ dependencies = [
  "common-catalog",
  "common-error",
  "common-function",
+ "common-macro",
  "common-query",
  "common-recordbatch",
  "common-runtime",
@@ -8916,6 +8945,7 @@ dependencies = [
  "common-error",
  "common-grpc",
  "common-grpc-expr",
+ "common-macro",
  "common-mem-prof",
  "common-meta",
  "common-query",
@@ -9272,6 +9302,7 @@ dependencies = [
  "common-catalog",
  "common-datasource",
  "common-error",
+ "common-macro",
  "common-query",
  "common-time",
  "datafusion-sql",
@@ -9536,6 +9567,7 @@ dependencies = [
  "common-config",
  "common-datasource",
  "common-error",
+ "common-macro",
  "common-query",
  "common-recordbatch",
  "common-runtime",
@@ -9583,6 +9615,7 @@ dependencies = [
  "bytes",
  "common-base",
  "common-error",
+ "common-macro",
  "common-query",
  "common-recordbatch",
  "common-time",
@@ -9718,6 +9751,7 @@ dependencies = [
  "catalog",
  "common-catalog",
  "common-error",
+ "common-macro",
  "common-telemetry",
  "datafusion",
  "datafusion-common",
@@ -9876,6 +9910,7 @@ dependencies = [
  "common-catalog",
  "common-datasource",
  "common-error",
+ "common-macro",
  "common-procedure",
  "common-query",
  "common-recordbatch",
diff --git a/src/api/Cargo.toml b/src/api/Cargo.toml
index 05fb33785701..05e625e2c5aa 100644
--- a/src/api/Cargo.toml
+++ b/src/api/Cargo.toml
@@ -7,6 +7,7 @@ license.workspace = true
 [dependencies]
 common-base = { workspace = true }
 common-error = { workspace = true }
+common-macro = { workspace = true }
 common-time = { workspace = true }
 datatypes = { workspace = true }
 greptime-proto.workspace = true
diff --git a/src/api/src/error.rs b/src/api/src/error.rs
index c67fc5f9aacb..77b3d54097d9 100644
--- a/src/api/src/error.rs
+++ b/src/api/src/error.rs
@@ -16,14 +16,16 @@ use std::any::Any;
 
 use common_error::ext::ErrorExt;
 use common_error::status_code::StatusCode;
+use common_macro::stack_trace_debug;
 use datatypes::prelude::ConcreteDataType;
 use snafu::prelude::*;
 use snafu::Location;
 
 pub type Result<T> = std::result::Result<T, Error>;
 
-#[derive(Debug, Snafu)]
+#[derive(Snafu)]
 #[snafu(visibility(pub))]
+#[stack_trace_debug]
 pub enum Error {
     #[snafu(display("Unknown proto column datatype: {}", datatype))]
     UnknownColumnDataType { datatype: i32, location: Location },
diff --git a/src/auth/Cargo.toml b/src/auth/Cargo.toml
index ae1c4a49ff58..3909d8078157 100644
--- a/src/auth/Cargo.toml
+++ b/src/auth/Cargo.toml
@@ -14,6 +14,7 @@ testing = []
 api.workspace = true
 async-trait.workspace = true
 common-error.workspace = true
+common-macro.workspace = true
 digest = "0.10"
 hex = { version = "0.4" }
 secrecy = { version = "0.8", features = ["serde", "alloc"] }
diff --git a/src/auth/src/error.rs b/src/auth/src/error.rs
index bba69dbca353..cc07f3d0aa41 100644
--- a/src/auth/src/error.rs
+++ b/src/auth/src/error.rs
@@ -14,10 +14,12 @@
 
 use common_error::ext::{BoxedError, ErrorExt};
 use common_error::status_code::StatusCode;
+use common_macro::stack_trace_debug;
 use snafu::{Location, Snafu};
 
-#[derive(Debug, Snafu)]
+#[derive(Snafu)]
 #[snafu(visibility(pub))]
+#[stack_trace_debug]
 pub enum Error {
     #[snafu(display("Invalid config value: {}, {}", value, msg))]
     InvalidConfig { value: String, msg: String },
@@ -30,7 +32,8 @@ pub enum Error {
 
     #[snafu(display("IO error"))]
     Io {
-        source: std::io::Error,
+        #[snafu(source)]
+        error: std::io::Error,
         location: Location,
     },
 
diff --git a/src/catalog/Cargo.toml b/src/catalog/Cargo.toml
index b9ff8c19ae97..314a332672ea 100644
--- a/src/catalog/Cargo.toml
+++ b/src/catalog/Cargo.toml
@@ -16,6 +16,7 @@ async-trait = "0.1"
 common-catalog = { workspace = true }
 common-error = { workspace = true }
 common-grpc = { workspace = true }
+common-macro = { workspace = true }
 common-meta = { workspace = true }
 common-query = { workspace = true }
 common-recordbatch = { workspace = true }
diff --git a/src/catalog/src/error.rs b/src/catalog/src/error.rs
index d6c708cdff49..599795337ef4 100644
--- a/src/catalog/src/error.rs
+++ b/src/catalog/src/error.rs
@@ -17,14 +17,16 @@ use std::fmt::Debug;
 
 use common_error::ext::{BoxedError, ErrorExt};
 use common_error::status_code::StatusCode;
+use common_macro::stack_trace_debug;
 use datafusion::error::DataFusionError;
 use datatypes::prelude::ConcreteDataType;
 use snafu::{Location, Snafu};
 use table::metadata::TableId;
 use tokio::task::JoinError;
 
-#[derive(Debug, Snafu)]
+#[derive(Snafu)]
 #[snafu(visibility(pub))]
+#[stack_trace_debug]
 pub enum Error {
     #[snafu(display("Failed to list catalogs"))]
     ListCatalogs {
@@ -92,7 +94,8 @@ pub enum Error {
 
     #[snafu(display("Failed to deserialize value"))]
     ValueDeserialize {
-        source: serde_json::error::Error,
+        #[snafu(source)]
+        error: serde_json::error::Error,
         location: Location,
     },
 
@@ -142,7 +145,10 @@ pub enum Error {
     },
 
     #[snafu(display("Failed to open table in parallel"))]
-    ParallelOpenTable { source: JoinError },
+    ParallelOpenTable {
+        #[snafu(source)]
+        error: JoinError,
+    },
 
     #[snafu(display("Table not found while opening table, table info: {}", table_info))]
     TableNotFound {
@@ -213,7 +219,8 @@ pub enum Error {
     #[snafu(display("msg: {}", msg))]
     Datafusion {
         msg: String,
-        source: DataFusionError,
+        #[snafu(source)]
+        error: DataFusionError,
         location: Location,
     },
 
diff --git a/src/client/Cargo.toml b/src/client/Cargo.toml
index dde80a01943b..72942cd3a3d9 100644
--- a/src/client/Cargo.toml
+++ b/src/client/Cargo.toml
@@ -16,6 +16,7 @@ common-base = { workspace = true }
 common-catalog = { workspace = true }
 common-error = { workspace = true }
 common-grpc = { workspace = true }
+common-macro = { workspace = true }
 common-meta = { workspace = true }
 common-query = { workspace = true }
 common-recordbatch = { workspace = true }
diff --git a/src/client/examples/stream_ingest.rs b/src/client/examples/stream_ingest.rs
index 7b67cf33931e..94f9773096b9 100644
--- a/src/client/examples/stream_ingest.rs
+++ b/src/client/examples/stream_ingest.rs
@@ -42,14 +42,14 @@ async fn run() {
         .insert(vec![to_insert_request(weather_records_1())])
         .await
     {
-        error!("Error: {e}");
+        error!("Error: {e:?}");
     }
 
     if let Err(e) = stream_inserter
         .insert(vec![to_insert_request(weather_records_2())])
         .await
     {
-        error!("Error: {e}");
+        error!("Error: {e:?}");
     }
 
     let result = stream_inserter.finish().await;
@@ -59,7 +59,7 @@ async fn run() {
             info!("Rows written: {rows}");
         }
         Err(e) => {
-            error!("Error: {e}");
+            error!("Error: {e:?}");
         }
     };
 }
diff --git a/src/client/src/database.rs b/src/client/src/database.rs
index bb792de47b21..7e4c33a07417 100644
--- a/src/client/src/database.rs
+++ b/src/client/src/database.rs
@@ -276,7 +276,7 @@ impl Database {
                 source: BoxedError::new(ServerSnafu { code, msg }.build()),
             };
             logging::error!(
-                "Failed to do Flight get, addr: {}, code: {}, source: {}",
+                "Failed to do Flight get, addr: {}, code: {}, source: {:?}",
                 client.addr(),
                 tonic_code,
                 error
diff --git a/src/client/src/error.rs b/src/client/src/error.rs
index f0676da7b6ca..6b25e5c58ac3 100644
--- a/src/client/src/error.rs
+++ b/src/client/src/error.rs
@@ -17,11 +17,13 @@ use std::any::Any;
 use common_error::ext::{BoxedError, ErrorExt};
 use common_error::status_code::StatusCode;
 use common_error::{GREPTIME_ERROR_CODE, GREPTIME_ERROR_MSG};
+use common_macro::stack_trace_debug;
 use snafu::{Location, Snafu};
 use tonic::{Code, Status};
 
-#[derive(Debug, Snafu)]
+#[derive(Snafu)]
 #[snafu(visibility(pub))]
+#[stack_trace_debug]
 pub enum Error {
     #[snafu(display("Illegal Flight messages, reason: {}", reason))]
     IllegalFlightMessages { reason: String, location: Location },
diff --git a/src/cmd/Cargo.toml b/src/cmd/Cargo.toml
index c83f65f7488e..d04eb8f86265 100644
--- a/src/cmd/Cargo.toml
+++ b/src/cmd/Cargo.toml
@@ -26,6 +26,7 @@ common-base = { workspace = true }
 common-catalog = { workspace = true }
 common-config = { workspace = true }
 common-error = { workspace = true }
+common-macro = { workspace = true }
 common-meta = { workspace = true }
 common-procedure = { workspace = true }
 common-query = { workspace = true }
diff --git a/src/cmd/src/error.rs b/src/cmd/src/error.rs
index fea82b568daf..8631eb7f58f0 100644
--- a/src/cmd/src/error.rs
+++ b/src/cmd/src/error.rs
@@ -16,12 +16,14 @@ use std::any::Any;
 
 use common_error::ext::ErrorExt;
 use common_error::status_code::StatusCode;
+use common_macro::stack_trace_debug;
 use config::ConfigError;
 use rustyline::error::ReadlineError;
 use snafu::{Location, Snafu};
 
-#[derive(Debug, Snafu)]
+#[derive(Snafu)]
 #[snafu(visibility(pub))]
+#[stack_trace_debug]
 pub enum Error {
     #[snafu(display("Failed to create default catalog and schema"))]
     InitMetadata {
@@ -101,13 +103,15 @@ pub enum Error {
 
     #[snafu(display("Cannot create REPL"))]
     ReplCreation {
-        source: ReadlineError,
+        #[snafu(source)]
+        error: ReadlineError,
         location: Location,
     },
 
     #[snafu(display("Error reading command"))]
     Readline {
-        source: ReadlineError,
+        #[snafu(source)]
+        error: ReadlineError,
         location: Location,
     },
 
@@ -157,7 +161,8 @@ pub enum Error {
 
     #[snafu(display("Failed to load layered config"))]
     LoadLayeredConfig {
-        source: ConfigError,
+        #[snafu(source)]
+        error: ConfigError,
         location: Location,
     },
 
@@ -170,7 +175,8 @@ pub enum Error {
     #[snafu(display("Failed to connect to Etcd at {etcd_addr}"))]
     ConnectEtcd {
         etcd_addr: String,
-        source: etcd_client::Error,
+        #[snafu(source)]
+        error: etcd_client::Error,
         location: Location,
     },
 }
diff --git a/src/common/base/Cargo.toml b/src/common/base/Cargo.toml
index fc319981a03e..2a0030b33beb 100644
--- a/src/common/base/Cargo.toml
+++ b/src/common/base/Cargo.toml
@@ -9,6 +9,7 @@ anymap = "1.0.0-beta.2"
 bitvec = "1.0"
 bytes = { version = "1.1", features = ["serde"] }
 common-error = { workspace = true }
+common-macro = { workspace = true }
 paste = "1.0"
 serde = { version = "1.0", features = ["derive"] }
 snafu.workspace = true
diff --git a/src/common/base/src/buffer.rs b/src/common/base/src/buffer.rs
index 34f6f63491b0..b472bbf5380e 100644
--- a/src/common/base/src/buffer.rs
+++ b/src/common/base/src/buffer.rs
@@ -17,11 +17,13 @@ use std::io::{Read, Write};
 
 use bytes::{Buf, BufMut, BytesMut};
 use common_error::ext::ErrorExt;
+use common_macro::stack_trace_debug;
 use paste::paste;
 use snafu::{ensure, Location, ResultExt, Snafu};
 
-#[derive(Debug, Snafu)]
+#[derive(Snafu)]
 #[snafu(visibility(pub))]
+#[stack_trace_debug]
 pub enum Error {
     #[snafu(display(
         "Destination buffer overflow, src_len: {}, dst_len: {}",
@@ -39,7 +41,8 @@ pub enum Error {
 
     #[snafu(display("IO operation reach EOF"))]
     Eof {
-        source: std::io::Error,
+        #[snafu(source)]
+        error: std::io::Error,
         location: Location,
     },
 }
diff --git a/src/common/catalog/Cargo.toml b/src/common/catalog/Cargo.toml
index 23f498c993ba..e3b9fa5dd72b 100644
--- a/src/common/catalog/Cargo.toml
+++ b/src/common/catalog/Cargo.toml
@@ -6,6 +6,7 @@ license.workspace = true
 
 [dependencies]
 common-error = { workspace = true }
+common-macro = { workspace = true }
 serde.workspace = true
 serde_json = "1.0"
 snafu = { version = "0.7", features = ["backtraces"] }
diff --git a/src/common/catalog/src/error.rs b/src/common/catalog/src/error.rs
index 2ded9e47a576..46166b1ccee5 100644
--- a/src/common/catalog/src/error.rs
+++ b/src/common/catalog/src/error.rs
@@ -16,10 +16,12 @@ use std::any::Any;
 
 use common_error::ext::ErrorExt;
 use common_error::status_code::StatusCode;
+use common_macro::stack_trace_debug;
 use snafu::{Location, Snafu};
 
-#[derive(Debug, Snafu)]
+#[derive(Snafu)]
 #[snafu(visibility(pub))]
+#[stack_trace_debug]
 pub enum Error {
     #[snafu(display("Invalid full table name: {}", table_name))]
     InvalidFullTableName {
diff --git a/src/common/datasource/Cargo.toml b/src/common/datasource/Cargo.toml
index edb327519ab0..e6bff6953876 100644
--- a/src/common/datasource/Cargo.toml
+++ b/src/common/datasource/Cargo.toml
@@ -18,6 +18,7 @@ async-compression = { version = "0.3", features = [
 async-trait.workspace = true
 bytes = "1.1"
 common-error = { workspace = true }
+common-macro = { workspace = true }
 common-runtime = { workspace = true }
 datafusion.workspace = true
 derive_builder.workspace = true
diff --git a/src/common/datasource/src/error.rs b/src/common/datasource/src/error.rs
index ab1e918100d3..f8ab4a30c634 100644
--- a/src/common/datasource/src/error.rs
+++ b/src/common/datasource/src/error.rs
@@ -17,12 +17,14 @@ use std::any::Any;
 use arrow_schema::ArrowError;
 use common_error::ext::ErrorExt;
 use common_error::status_code::StatusCode;
+use common_macro::stack_trace_debug;
 use datafusion::parquet::errors::ParquetError;
 use snafu::{Location, Snafu};
 use url::ParseError;
 
-#[derive(Debug, Snafu)]
+#[derive(Snafu)]
 #[snafu(visibility(pub))]
+#[stack_trace_debug]
 pub enum Error {
     #[snafu(display("Unsupported compression type: {}", compression_type))]
     UnsupportedCompressionType {
@@ -46,83 +48,96 @@ pub enum Error {
     #[snafu(display("Invalid url: {}", url))]
     InvalidUrl {
         url: String,
-        source: ParseError,
+        #[snafu(source)]
+        error: ParseError,
         location: Location,
     },
 
     #[snafu(display("Failed to build backend"))]
     BuildBackend {
-        source: object_store::Error,
+        #[snafu(source)]
+        error: object_store::Error,
         location: Location,
     },
 
     #[snafu(display("Failed to build orc reader"))]
     OrcReader {
         location: Location,
-        source: orc_rust::error::Error,
+        #[snafu(source)]
+        error: orc_rust::error::Error,
     },
 
     #[snafu(display("Failed to read object from path: {}", path))]
     ReadObject {
         path: String,
         location: Location,
-        source: object_store::Error,
+        #[snafu(source)]
+        error: object_store::Error,
     },
 
     #[snafu(display("Failed to write object to path: {}", path))]
     WriteObject {
         path: String,
         location: Location,
-        source: object_store::Error,
+        #[snafu(source)]
+        error: object_store::Error,
     },
 
     #[snafu(display("Failed to write"))]
     AsyncWrite {
-        source: std::io::Error,
+        #[snafu(source)]
+        error: std::io::Error,
         location: Location,
     },
 
     #[snafu(display("Failed to write record batch"))]
     WriteRecordBatch {
         location: Location,
-        source: ArrowError,
+        #[snafu(source)]
+        error: ArrowError,
     },
 
     #[snafu(display("Failed to encode record batch"))]
     EncodeRecordBatch {
         location: Location,
-        source: ParquetError,
+        #[snafu(source)]
+        error: ParquetError,
     },
 
     #[snafu(display("Failed to read record batch"))]
     ReadRecordBatch {
         location: Location,
-        source: datafusion::error::DataFusionError,
+        #[snafu(source)]
+        error: datafusion::error::DataFusionError,
     },
 
     #[snafu(display("Failed to read parquet"))]
     ReadParquetSnafu {
         location: Location,
-        source: datafusion::parquet::errors::ParquetError,
+        #[snafu(source)]
+        error: datafusion::parquet::errors::ParquetError,
     },
 
     #[snafu(display("Failed to convert parquet to schema"))]
     ParquetToSchema {
         location: Location,
-        source: datafusion::parquet::errors::ParquetError,
+        #[snafu(source)]
+        error: datafusion::parquet::errors::ParquetError,
     },
 
     #[snafu(display("Failed to infer schema from file"))]
     InferSchema {
         location: Location,
-        source: arrow_schema::ArrowError,
+        #[snafu(source)]
+        error: arrow_schema::ArrowError,
     },
 
     #[snafu(display("Failed to list object in path: {}", path))]
     ListObjects {
         path: String,
         location: Location,
-        source: object_store::Error,
+        #[snafu(source)]
+        error: object_store::Error,
     },
 
     #[snafu(display("Invalid connection: {}", msg))]
@@ -131,7 +146,8 @@ pub enum Error {
     #[snafu(display("Failed to join handle"))]
     JoinHandle {
         location: Location,
-        source: tokio::task::JoinError,
+        #[snafu(source)]
+        error: tokio::task::JoinError,
     },
 
     #[snafu(display("Failed to parse format {} with value: {}", key, value))]
@@ -143,7 +159,8 @@ pub enum Error {
 
     #[snafu(display("Failed to merge schema"))]
     MergeSchema {
-        source: arrow_schema::ArrowError,
+        #[snafu(source)]
+        error: arrow_schema::ArrowError,
         location: Location,
     },
 
diff --git a/src/common/error/src/ext.rs b/src/common/error/src/ext.rs
index 261f8f773699..b840b2911ea0 100644
--- a/src/common/error/src/ext.rs
+++ b/src/common/error/src/ext.rs
@@ -13,11 +13,12 @@
 // limitations under the License.
 
 use std::any::Any;
+use std::sync::Arc;
 
 use crate::status_code::StatusCode;
 
 /// Extension to [`Error`](std::error::Error) in std.
-pub trait ErrorExt: std::error::Error {
+pub trait ErrorExt: std::error::Error + StackError {
     /// Map this error to [StatusCode].
     fn status_code(&self) -> StatusCode {
         StatusCode::Unknown
@@ -35,6 +36,32 @@ pub trait ErrorExt: std::error::Error {
     fn as_any(&self) -> &dyn Any;
 }
 
+pub trait StackError {
+    fn debug_fmt(&self, layer: usize, buf: &mut Vec<String>);
+
+    fn next(&self) -> Option<&dyn StackError>;
+}
+
+impl<T: ?Sized + StackError> StackError for Arc<T> {
+    fn debug_fmt(&self, layer: usize, buf: &mut Vec<String>) {
+        self.as_ref().debug_fmt(layer, buf)
+    }
+
+    fn next(&self) -> Option<&dyn StackError> {
+        self.as_ref().next()
+    }
+}
+
+impl<T: ?Sized + StackError> StackError for Box<T> {
+    fn debug_fmt(&self, layer: usize, buf: &mut Vec<String>) {
+        self.as_ref().debug_fmt(layer, buf)
+    }
+
+    fn next(&self) -> Option<&dyn StackError> {
+        self.as_ref().next()
+    }
+}
+
 /// An opaque boxed error based on errors that implement [ErrorExt] trait.
 pub struct BoxedError {
     inner: Box<dyn crate::ext::ErrorExt + Send + Sync>,
@@ -90,6 +117,16 @@ impl crate::snafu::ErrorCompat for BoxedError {
     }
 }
 
+impl StackError for BoxedError {
+    fn debug_fmt(&self, layer: usize, buf: &mut Vec<String>) {
+        self.inner.debug_fmt(layer, buf)
+    }
+
+    fn next(&self) -> Option<&dyn StackError> {
+        self.inner.next()
+    }
+}
+
 /// Error type with plain error message
 #[derive(Debug)]
 pub struct PlainError {
@@ -128,3 +165,13 @@ impl crate::ext::ErrorExt for PlainError {
         self as _
     }
 }
+
+impl StackError for PlainError {
+    fn debug_fmt(&self, layer: usize, buf: &mut Vec<String>) {
+        buf.push(format!("{}: {}", layer, self.msg))
+    }
+
+    fn next(&self) -> Option<&dyn StackError> {
+        None
+    }
+}
diff --git a/src/common/error/src/format.rs b/src/common/error/src/format.rs
index 9bf219640985..341e4829e954 100644
--- a/src/common/error/src/format.rs
+++ b/src/common/error/src/format.rs
@@ -50,6 +50,7 @@ mod tests {
     use snafu::{GenerateImplicitData, Location};
 
     use super::*;
+    use crate::ext::StackError;
 
     #[derive(Debug, Snafu)]
     #[snafu(display("This is a leaf error"))]
@@ -65,6 +66,14 @@ mod tests {
         }
     }
 
+    impl StackError for Leaf {
+        fn debug_fmt(&self, _: usize, _: &mut Vec<String>) {}
+
+        fn next(&self) -> Option<&dyn StackError> {
+            None
+        }
+    }
+
     #[derive(Debug, Snafu)]
     #[snafu(display("This is a leaf with location"))]
     struct LeafWithLocation {
@@ -81,6 +90,14 @@ mod tests {
         }
     }
 
+    impl StackError for LeafWithLocation {
+        fn debug_fmt(&self, _: usize, _: &mut Vec<String>) {}
+
+        fn next(&self) -> Option<&dyn StackError> {
+            None
+        }
+    }
+
     #[derive(Debug, Snafu)]
     #[snafu(display("Internal error"))]
     struct Internal {
@@ -99,6 +116,17 @@ mod tests {
         }
     }
 
+    impl StackError for Internal {
+        fn debug_fmt(&self, layer: usize, buf: &mut Vec<String>) {
+            buf.push(format!("{}: Internal error, at {}", layer, self.location));
+            self.source.debug_fmt(layer + 1, buf);
+        }
+
+        fn next(&self) -> Option<&dyn StackError> {
+            Some(&self.source)
+        }
+    }
+
     #[test]
     fn test_debug_format() {
         let err = Leaf;
diff --git a/src/common/error/src/mock.rs b/src/common/error/src/mock.rs
index a3f061fd6609..2781149b1268 100644
--- a/src/common/error/src/mock.rs
+++ b/src/common/error/src/mock.rs
@@ -19,7 +19,7 @@ use std::fmt;
 
 use snafu::Location;
 
-use crate::ext::ErrorExt;
+use crate::ext::{ErrorExt, StackError};
 use crate::status_code::StatusCode;
 
 /// A mock error mainly for test.
@@ -69,3 +69,11 @@ impl ErrorExt for MockError {
         self
     }
 }
+
+impl StackError for MockError {
+    fn debug_fmt(&self, _: usize, _: &mut Vec<String>) {}
+
+    fn next(&self) -> Option<&dyn StackError> {
+        None
+    }
+}
diff --git a/src/common/grpc-expr/Cargo.toml b/src/common/grpc-expr/Cargo.toml
index 0143b5628373..6cd42995266e 100644
--- a/src/common/grpc-expr/Cargo.toml
+++ b/src/common/grpc-expr/Cargo.toml
@@ -10,6 +10,7 @@ async-trait.workspace = true
 common-base = { workspace = true }
 common-catalog = { workspace = true }
 common-error = { workspace = true }
+common-macro = { workspace = true }
 common-query = { workspace = true }
 common-telemetry = { workspace = true }
 common-time = { workspace = true }
diff --git a/src/common/grpc-expr/src/error.rs b/src/common/grpc-expr/src/error.rs
index 8456d05db862..15a5a3ffc794 100644
--- a/src/common/grpc-expr/src/error.rs
+++ b/src/common/grpc-expr/src/error.rs
@@ -16,10 +16,12 @@ use std::any::Any;
 
 use common_error::ext::ErrorExt;
 use common_error::status_code::StatusCode;
+use common_macro::stack_trace_debug;
 use snafu::{Location, Snafu};
 
-#[derive(Debug, Snafu)]
+#[derive(Snafu)]
 #[snafu(visibility(pub))]
+#[stack_trace_debug]
 pub enum Error {
     #[snafu(display("Illegal delete request, reason: {reason}"))]
     IllegalDeleteRequest { reason: String, location: Location },
diff --git a/src/common/grpc/Cargo.toml b/src/common/grpc/Cargo.toml
index 416566506921..81e935f1a39f 100644
--- a/src/common/grpc/Cargo.toml
+++ b/src/common/grpc/Cargo.toml
@@ -11,6 +11,7 @@ async-trait = "0.1"
 backtrace = "0.3"
 common-base = { workspace = true }
 common-error = { workspace = true }
+common-macro = { workspace = true }
 common-recordbatch = { workspace = true }
 common-runtime = { workspace = true }
 common-telemetry = { workspace = true }
diff --git a/src/common/grpc/src/error.rs b/src/common/grpc/src/error.rs
index a92fbcf0a939..e6ddab8c126c 100644
--- a/src/common/grpc/src/error.rs
+++ b/src/common/grpc/src/error.rs
@@ -17,19 +17,22 @@ use std::io;
 
 use common_error::ext::ErrorExt;
 use common_error::status_code::StatusCode;
+use common_macro::stack_trace_debug;
 use snafu::{Location, Snafu};
 
 pub type Result<T> = std::result::Result<T, Error>;
 
-#[derive(Debug, Snafu)]
+#[derive(Snafu)]
 #[snafu(visibility(pub))]
+#[stack_trace_debug]
 pub enum Error {
     #[snafu(display("Invalid client tls config, {}", msg))]
     InvalidTlsConfig { msg: String },
 
     #[snafu(display("Invalid config file path"))]
     InvalidConfigFilePath {
-        source: io::Error,
+        #[snafu(source)]
+        error: io::Error,
         location: Location,
     },
 
@@ -48,7 +51,8 @@ pub enum Error {
 
     #[snafu(display("Failed to create gRPC channel"))]
     CreateChannel {
-        source: tonic::transport::Error,
+        #[snafu(source)]
+        error: tonic::transport::Error,
         location: Location,
     },
 
@@ -63,7 +67,8 @@ pub enum Error {
 
     #[snafu(display("Failed to decode FlightData"))]
     DecodeFlightData {
-        source: api::DecodeError,
+        #[snafu(source)]
+        error: api::DecodeError,
         location: Location,
     },
 
diff --git a/src/common/macro/Cargo.toml b/src/common/macro/Cargo.toml
index c0ab6b0a5be3..36f2ae4a1e87 100644
--- a/src/common/macro/Cargo.toml
+++ b/src/common/macro/Cargo.toml
@@ -13,6 +13,15 @@ common-telemetry = { workspace = true }
 proc-macro2 = "1.0.66"
 quote = "1.0"
 syn = "1.0"
+syn2 = { version = "2.0", package = "syn", features = [
+    "derive",
+    "parsing",
+    "printing",
+    "clone-impls",
+    "proc-macro",
+    "extra-traits",
+    "full",
+] }
 
 [dev-dependencies]
 arc-swap = "1.0"
diff --git a/src/common/macro/src/lib.rs b/src/common/macro/src/lib.rs
index d374440ff323..f33f308a86a9 100644
--- a/src/common/macro/src/lib.rs
+++ b/src/common/macro/src/lib.rs
@@ -15,6 +15,7 @@
 mod aggr_func;
 mod print_caller;
 mod range_fn;
+mod stack_trace_debug;
 
 use aggr_func::{impl_aggr_func_type_store, impl_as_aggr_func_creator};
 use print_caller::process_print_caller;
@@ -87,3 +88,23 @@ pub fn range_fn(args: TokenStream, input: TokenStream) -> TokenStream {
 pub fn print_caller(args: TokenStream, input: TokenStream) -> TokenStream {
     process_print_caller(args, input)
 }
+
+/// Attribute macro to derive [std::fmt::Debug] for the annotated `Error` type.
+///
+/// The generated `Debug` implementation will print the error in a stack trace style. E.g.:
+/// ```plaintext
+/// 0: Foo error, at src/common/catalog/src/error.rs:80:10
+/// 1: Bar error, at src/common/function/src/error.rs:90:10
+/// 2: Root cause, invalid table name, at src/common/catalog/src/error.rs:100:10
+/// ```
+///
+/// Notes on using this macro:
+/// - `#[snafu(display)]` must present on each enum variants,
+///   and should not include `location` and `source`.
+/// - Only our internal error can be named `source`.
+///   All external error should be `error` with an `#[snafu(source)]` annotation.
+/// - `common_error` crate must be accessible.
+#[proc_macro_attribute]
+pub fn stack_trace_debug(args: TokenStream, input: TokenStream) -> TokenStream {
+    stack_trace_debug::stack_trace_style_impl(args.into(), input.into()).into()
+}
diff --git a/src/common/macro/src/stack_trace_debug.rs b/src/common/macro/src/stack_trace_debug.rs
new file mode 100644
index 000000000000..2e6f2d15946a
--- /dev/null
+++ b/src/common/macro/src/stack_trace_debug.rs
@@ -0,0 +1,278 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! implement `::common_error::ext::StackError`
+
+use proc_macro2::{Span, TokenStream as TokenStream2};
+use quote::{quote, quote_spanned};
+use syn2::spanned::Spanned;
+use syn2::{parenthesized, Attribute, Ident, ItemEnum, Variant};
+
+pub fn stack_trace_style_impl(args: TokenStream2, input: TokenStream2) -> TokenStream2 {
+    let input_cloned: TokenStream2 = input.clone();
+
+    let error_enum_definition: ItemEnum = syn2::parse2(input_cloned).unwrap();
+    let enum_name = error_enum_definition.ident;
+
+    let mut variants = vec![];
+
+    for error_variant in error_enum_definition.variants {
+        let variant = ErrorVariant::from_enum_variant(error_variant);
+        variants.push(variant);
+    }
+
+    let debug_fmt_fn = build_debug_fmt_impl(enum_name.clone(), variants.clone());
+    let next_fn = build_next_impl(enum_name.clone(), variants);
+    let debug_impl = build_debug_impl(enum_name.clone());
+
+    quote! {
+        #args
+        #input
+
+        impl ::common_error::ext::StackError for #enum_name {
+            #debug_fmt_fn
+            #next_fn
+        }
+
+        #debug_impl
+    }
+}
+
+/// Generate `debug_fmt` fn.
+///
+/// The generated fn will be like:
+/// ```rust, ignore
+/// fn debug_fmt(&self, layer: usize, buf: &mut Vec<String>);
+/// ```
+fn build_debug_fmt_impl(enum_name: Ident, variants: Vec<ErrorVariant>) -> TokenStream2 {
+    let match_arms = variants
+        .iter()
+        .map(|v| v.to_debug_match_arm())
+        .collect::<Vec<_>>();
+
+    quote! {
+        fn debug_fmt(&self, layer: usize, buf: &mut Vec<String>) {
+            use #enum_name::*;
+            match self {
+                #(#match_arms)*
+            }
+        }
+    }
+}
+
+/// Generate `next` fn.
+///
+/// The generated fn will be like:
+/// ```rust, ignore
+/// fn next(&self) -> Option<&dyn ::common_error::ext::StackError>;
+/// ```
+fn build_next_impl(enum_name: Ident, variants: Vec<ErrorVariant>) -> TokenStream2 {
+    let match_arms = variants
+        .iter()
+        .map(|v| v.to_next_match_arm())
+        .collect::<Vec<_>>();
+
+    quote! {
+        fn next(&self) -> Option<&dyn ::common_error::ext::StackError> {
+            use #enum_name::*;
+            match self {
+                #(#match_arms)*
+            }
+        }
+    }
+}
+
+/// Implement [std::fmt::Debug] via `debug_fmt`
+fn build_debug_impl(enum_name: Ident) -> TokenStream2 {
+    quote! {
+        impl std::fmt::Debug for #enum_name {
+            fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+                use ::common_error::ext::StackError;
+                let mut buf = vec![];
+                self.debug_fmt(0, &mut buf);
+                write!(f, "{}", buf.join("\n"))
+            }
+        }
+    }
+}
+
+#[derive(Clone, Debug)]
+struct ErrorVariant {
+    name: Ident,
+    fields: Vec<Ident>,
+    has_location: bool,
+    has_source: bool,
+    has_external_cause: bool,
+    display: TokenStream2,
+    span: Span,
+    cfg_attr: Option<Attribute>,
+}
+
+impl ErrorVariant {
+    /// Construct self from [Variant]
+    fn from_enum_variant(variant: Variant) -> Self {
+        let span = variant.span();
+        let mut has_location = false;
+        let mut has_source = false;
+        let mut has_external_cause = false;
+
+        for field in &variant.fields {
+            if let Some(ident) = &field.ident {
+                if ident == "location" {
+                    has_location = true;
+                } else if ident == "source" {
+                    has_source = true;
+                } else if ident == "error" {
+                    has_external_cause = true;
+                }
+            }
+        }
+
+        let mut display = None;
+        let mut cfg_attr = None;
+        for attr in variant.attrs {
+            if attr.path().is_ident("snafu") {
+                attr.parse_nested_meta(|meta| {
+                    if meta.path.is_ident("display") {
+                        let content;
+                        parenthesized!(content in meta.input);
+                        let display_ts: TokenStream2 = content.parse()?;
+                        display = Some(display_ts);
+                        Ok(())
+                    } else {
+                        Err(meta.error("unrecognized repr"))
+                    }
+                })
+                .expect("Each error should contains a display attribute");
+            }
+
+            if attr.path().is_ident("cfg") {
+                cfg_attr = Some(attr);
+            }
+        }
+
+        let field_ident = variant
+            .fields
+            .iter()
+            .map(|f| f.ident.clone().unwrap_or_else(|| Ident::new("_", f.span())))
+            .collect();
+
+        Self {
+            name: variant.ident,
+            fields: field_ident,
+            has_location,
+            has_source,
+            has_external_cause,
+            display: display.unwrap(),
+            span,
+            cfg_attr,
+        }
+    }
+
+    /// Convert self into an match arm that will be used in [build_debug_impl].
+    ///
+    /// The generated match arm will be like:
+    /// ```rust, ignore
+    ///     ErrorKindWithSource { source, .. } => {
+    ///         debug_fmt(source, layer + 1, buf);
+    ///     },
+    ///     ErrorKindWithoutSource { .. } => {
+    ///        buf.push(format!("{layer}: {}, at {}", format!(#display), location)));
+    ///     }
+    /// ```
+    ///
+    /// The generated code assumes fn `debug_fmt`, var `layer`, var `buf` are in scope.
+    fn to_debug_match_arm(&self) -> TokenStream2 {
+        let name = &self.name;
+        let fields = &self.fields;
+        let display = &self.display;
+        let cfg = if let Some(cfg) = &self.cfg_attr {
+            quote_spanned!(cfg.span() => #cfg)
+        } else {
+            quote! {}
+        };
+
+        match (self.has_location, self.has_source, self.has_external_cause) {
+            (true, true, _) => quote_spanned! {
+               self.span => #cfg #[allow(unused_variables)] #name { #(#fields),*, } => {
+                    buf.push(format!("{layer}: {}, at {}", format!(#display), location));
+                    source.debug_fmt(layer + 1, buf);
+                },
+            },
+            (true, false, true) => quote_spanned! {
+                self.span => #cfg #[allow(unused_variables)] #name { #(#fields),* } => {
+                    buf.push(format!("{layer}: {}, at {}", format!(#display), location));
+                    buf.push(format!("{}: {:?}", layer + 1, error));
+                },
+            },
+            (true, false, false) => quote_spanned! {
+                self.span => #cfg #[allow(unused_variables)] #name { #(#fields),* } => {
+                    buf.push(format!("{layer}: {}, at {}", format!(#display), location));
+                },
+            },
+            (false, true, _) => quote_spanned! {
+                self.span => #cfg #[allow(unused_variables)] #name { #(#fields),* } => {
+                    buf.push(format!("{layer}: {}", format!(#display)));
+                    source.debug_fmt(layer + 1, buf);
+                },
+            },
+            (false, false, true) => quote_spanned! {
+                self.span => #cfg #[allow(unused_variables)] #name { #(#fields),* } => {
+                    buf.push(format!("{layer}: {}", format!(#display)));
+                    buf.push(format!("{}: {:?}", layer + 1, error));
+                },
+            },
+            (false, false, false) => quote_spanned! {
+                self.span => #cfg #[allow(unused_variables)] #name { #(#fields),* } => {
+                    buf.push(format!("{layer}: {}", format!(#display)));
+                },
+            },
+        }
+    }
+
+    /// Convert self into an match arm that will be used in [build_next_impl].
+    ///
+    /// The generated match arm will be like:
+    /// ```rust, ignore
+    ///     ErrorKindWithSource { source, .. } => {
+    ///         Some(source)
+    ///     },
+    ///     ErrorKindWithoutSource { .. } => {
+    ///        None
+    ///     }
+    /// ```
+    fn to_next_match_arm(&self) -> TokenStream2 {
+        let name = &self.name;
+        let fields = &self.fields;
+        let cfg = if let Some(cfg) = &self.cfg_attr {
+            quote_spanned!(cfg.span() => #cfg)
+        } else {
+            quote! {}
+        };
+
+        if self.has_source {
+            quote_spanned! {
+                self.span => #cfg #[allow(unused_variables)] #name { #(#fields),* } => {
+                    Some(source)
+                },
+            }
+        } else {
+            quote_spanned! {
+                self.span => #cfg #[allow(unused_variables)] #name { #(#fields),* } =>{
+                    None
+                }
+            }
+        }
+    }
+}
diff --git a/src/common/mem-prof/Cargo.toml b/src/common/mem-prof/Cargo.toml
index 211f941cabd7..923e169fae35 100644
--- a/src/common/mem-prof/Cargo.toml
+++ b/src/common/mem-prof/Cargo.toml
@@ -6,6 +6,7 @@ license.workspace = true
 
 [dependencies]
 common-error = { workspace = true }
+common-macro = { workspace = true }
 snafu.workspace = true
 tempfile = "3.4"
 tokio.workspace = true
diff --git a/src/common/mem-prof/src/error.rs b/src/common/mem-prof/src/error.rs
index db83bb86ed79..55b2676ca652 100644
--- a/src/common/mem-prof/src/error.rs
+++ b/src/common/mem-prof/src/error.rs
@@ -16,14 +16,16 @@ use std::any::Any;
 
 use common_error::ext::{BoxedError, ErrorExt};
 use common_error::status_code::StatusCode;
+use common_macro::stack_trace_debug;
 use snafu::Snafu;
 
 pub type Result<T> = std::result::Result<T, Error>;
 
-#[derive(Debug, Snafu)]
+#[derive(Snafu)]
 #[snafu(visibility(pub))]
+#[stack_trace_debug]
 pub enum Error {
-    #[snafu(display(""))]
+    #[snafu(display("Internal error"))]
     Internal { source: BoxedError },
 
     #[snafu(display("Memory profiling is not supported"))]
diff --git a/src/common/mem-prof/src/jemalloc/error.rs b/src/common/mem-prof/src/jemalloc/error.rs
index 8aaf1d3a3057..a0e0f2ede068 100644
--- a/src/common/mem-prof/src/jemalloc/error.rs
+++ b/src/common/mem-prof/src/jemalloc/error.rs
@@ -17,13 +17,18 @@ use std::path::PathBuf;
 
 use common_error::ext::{BoxedError, ErrorExt};
 use common_error::status_code::StatusCode;
+use common_macro::stack_trace_debug;
 use snafu::{Location, Snafu};
 
-#[derive(Debug, Snafu)]
+#[derive(Snafu)]
 #[snafu(visibility(pub))]
+#[stack_trace_debug]
 pub enum Error {
     #[snafu(display("Failed to read OPT_PROF"))]
-    ReadOptProf { source: tikv_jemalloc_ctl::Error },
+    ReadOptProf {
+        #[snafu(source)]
+        error: tikv_jemalloc_ctl::Error,
+    },
 
     #[snafu(display("Memory profiling is not enabled"))]
     ProfilingNotEnabled,
@@ -34,13 +39,15 @@ pub enum Error {
     #[snafu(display("Failed to open temp file: {}", path))]
     OpenTempFile {
         path: String,
-        source: std::io::Error,
+        #[snafu(source)]
+        error: std::io::Error,
     },
 
     #[snafu(display("Failed to dump profiling data to temp file: {:?}", path))]
     DumpProfileData {
         path: PathBuf,
-        source: tikv_jemalloc_ctl::Error,
+        #[snafu(source)]
+        error: tikv_jemalloc_ctl::Error,
     },
 }
 
diff --git a/src/common/meta/Cargo.toml b/src/common/meta/Cargo.toml
index f7c4b3435f8b..98aa010a4d65 100644
--- a/src/common/meta/Cargo.toml
+++ b/src/common/meta/Cargo.toml
@@ -15,6 +15,7 @@ async-trait.workspace = true
 common-catalog = { workspace = true }
 common-error = { workspace = true }
 common-grpc-expr.workspace = true
+common-macro = { workspace = true }
 common-procedure = { workspace = true }
 common-recordbatch = { workspace = true }
 common-runtime = { workspace = true }
diff --git a/src/common/meta/src/error.rs b/src/common/meta/src/error.rs
index 605ee0aa46c5..4484a82a7cfb 100644
--- a/src/common/meta/src/error.rs
+++ b/src/common/meta/src/error.rs
@@ -16,6 +16,7 @@ use std::str::Utf8Error;
 
 use common_error::ext::{BoxedError, ErrorExt};
 use common_error::status_code::StatusCode;
+use common_macro::stack_trace_debug;
 use serde_json::error::Error as JsonError;
 use snafu::{Location, Snafu};
 use store_api::storage::RegionNumber;
@@ -23,8 +24,9 @@ use table::metadata::TableId;
 
 use crate::peer::Peer;
 
-#[derive(Debug, Snafu)]
+#[derive(Snafu)]
 #[snafu(visibility(pub))]
+#[stack_trace_debug]
 pub enum Error {
     #[snafu(display("Failed to get sequence: {}", err_msg))]
     NextSequence { err_msg: String, location: Location },
@@ -83,7 +85,8 @@ pub enum Error {
     #[snafu(display("Failed to build table meta for table: {}", table_name))]
     BuildTableMeta {
         table_name: String,
-        source: table::metadata::TableMetaBuilderError,
+        #[snafu(source)]
+        error: table::metadata::TableMetaBuilderError,
         location: Location,
     },
 
@@ -102,19 +105,22 @@ pub enum Error {
     #[snafu(display("Failed to decode protobuf"))]
     DecodeProto {
         location: Location,
-        source: prost::DecodeError,
+        #[snafu(source)]
+        error: prost::DecodeError,
     },
 
     #[snafu(display("Failed to encode object into json"))]
     EncodeJson {
         location: Location,
-        source: JsonError,
+        #[snafu(source)]
+        error: JsonError,
     },
 
     #[snafu(display("Failed to decode object from json"))]
     DecodeJson {
         location: Location,
-        source: JsonError,
+        #[snafu(source)]
+        error: JsonError,
     },
 
     #[snafu(display("Payload not exist"))]
@@ -125,7 +131,8 @@ pub enum Error {
 
     #[snafu(display("Failed to serde json"))]
     SerdeJson {
-        source: serde_json::error::Error,
+        #[snafu(source)]
+        error: serde_json::error::Error,
         location: Location,
     },
 
@@ -177,7 +184,8 @@ pub enum Error {
     #[snafu(display("Failed to convert raw key to str"))]
     ConvertRawKey {
         location: Location,
-        source: Utf8Error,
+        #[snafu(source)]
+        error: Utf8Error,
     },
 
     #[snafu(display("Table nod found, table: {}", table_name))]
diff --git a/src/common/procedure/Cargo.toml b/src/common/procedure/Cargo.toml
index 971653d136b9..21e8bb5beaff 100644
--- a/src/common/procedure/Cargo.toml
+++ b/src/common/procedure/Cargo.toml
@@ -9,6 +9,7 @@ async-stream.workspace = true
 async-trait.workspace = true
 backon = "0.4"
 common-error = { workspace = true }
+common-macro = { workspace = true }
 common-runtime = { workspace = true }
 common-telemetry = { workspace = true }
 futures.workspace = true
diff --git a/src/common/procedure/src/error.rs b/src/common/procedure/src/error.rs
index 39266f4addb2..1cd63decd282 100644
--- a/src/common/procedure/src/error.rs
+++ b/src/common/procedure/src/error.rs
@@ -18,13 +18,15 @@ use std::sync::Arc;
 
 use common_error::ext::{BoxedError, ErrorExt};
 use common_error::status_code::StatusCode;
+use common_macro::stack_trace_debug;
 use snafu::{Location, Snafu};
 
 use crate::procedure::ProcedureId;
 
 /// Procedure error.
-#[derive(Debug, Snafu)]
+#[derive(Snafu)]
 #[snafu(visibility(pub))]
+#[stack_trace_debug]
 pub enum Error {
     #[snafu(display("Failed to execute procedure due to external error"))]
     External { source: BoxedError },
@@ -34,7 +36,8 @@ pub enum Error {
 
     #[snafu(display("Failed to serialize to json"))]
     ToJson {
-        source: serde_json::Error,
+        #[snafu(source)]
+        error: serde_json::Error,
         location: Location,
     },
 
@@ -54,7 +57,8 @@ pub enum Error {
     #[snafu(display("Failed to delete {}", key))]
     DeleteState {
         key: String,
-        source: object_store::Error,
+        #[snafu(source)]
+        error: object_store::Error,
     },
 
     #[snafu(display("Failed to delete keys: '{keys}'"))]
@@ -73,7 +77,8 @@ pub enum Error {
 
     #[snafu(display("Failed to deserialize from json"))]
     FromJson {
-        source: serde_json::Error,
+        #[snafu(source)]
+        error: serde_json::Error,
         location: Location,
     },
 
@@ -85,7 +90,8 @@ pub enum Error {
 
     #[snafu(display("Failed to wait watcher"))]
     WaitWatcher {
-        source: tokio::sync::watch::error::RecvError,
+        #[snafu(source)]
+        error: tokio::sync::watch::error::RecvError,
         location: Location,
     },
 
@@ -102,7 +108,10 @@ pub enum Error {
     },
 
     #[snafu(display("Corrupted data, error: "))]
-    CorruptedData { source: FromUtf8Error },
+    CorruptedData {
+        #[snafu(source)]
+        error: FromUtf8Error,
+    },
 
     #[snafu(display("Failed to start the remove_outdated_meta method, error"))]
     StartRemoveOutdatedMetaTask {
diff --git a/src/common/procedure/src/local.rs b/src/common/procedure/src/local.rs
index 0848c7029de2..e32d5f90e8af 100644
--- a/src/common/procedure/src/local.rs
+++ b/src/common/procedure/src/local.rs
@@ -229,7 +229,7 @@ impl ManagerContext {
         let procedure = loader(&message.data)
             .map_err(|e| {
                 logging::error!(
-                    "Failed to load procedure data, key: {}, source: {}",
+                    "Failed to load procedure data, key: {}, source: {:?}",
                     procedure_id,
                     e
                 );
diff --git a/src/common/procedure/src/store.rs b/src/common/procedure/src/store.rs
index 154040a85309..11aeec6551af 100644
--- a/src/common/procedure/src/store.rs
+++ b/src/common/procedure/src/store.rs
@@ -224,7 +224,7 @@ impl ProcedureStore {
         serde_json::from_slice(value)
             .map_err(|e| {
                 // `e` doesn't impl ErrorExt so we print it as normal error.
-                logging::error!("Failed to parse value, key: {:?}, source: {}", key, e);
+                logging::error!("Failed to parse value, key: {:?}, source: {:?}", key, e);
                 e
             })
             .ok()
diff --git a/src/common/query/Cargo.toml b/src/common/query/Cargo.toml
index 60f6a8003d81..17ea5c06d308 100644
--- a/src/common/query/Cargo.toml
+++ b/src/common/query/Cargo.toml
@@ -8,6 +8,7 @@ license.workspace = true
 api = { workspace = true }
 async-trait.workspace = true
 common-error = { workspace = true }
+common-macro = { workspace = true }
 common-recordbatch = { workspace = true }
 common-time = { workspace = true }
 datafusion-common.workspace = true
diff --git a/src/common/query/src/error.rs b/src/common/query/src/error.rs
index c968675f9b80..25f9d3c71523 100644
--- a/src/common/query/src/error.rs
+++ b/src/common/query/src/error.rs
@@ -17,6 +17,7 @@ use std::any::Any;
 use arrow::error::ArrowError;
 use common_error::ext::{BoxedError, ErrorExt};
 use common_error::status_code::StatusCode;
+use common_macro::stack_trace_debug;
 use common_recordbatch::error::Error as RecordbatchError;
 use datafusion_common::DataFusionError;
 use datatypes::arrow;
@@ -26,8 +27,9 @@ use datatypes::prelude::ConcreteDataType;
 use snafu::{Location, Snafu};
 use statrs::StatsError;
 
-#[derive(Debug, Snafu)]
+#[derive(Snafu)]
 #[snafu(visibility(pub))]
+#[stack_trace_debug]
 pub enum Error {
     #[snafu(display("Failed to execute Python UDF: {}", msg))]
     PyUdf {
@@ -44,7 +46,8 @@ pub enum Error {
 
     #[snafu(display("Failed to execute function"))]
     ExecuteFunction {
-        source: DataFusionError,
+        #[snafu(source)]
+        error: DataFusionError,
         location: Location,
     },
 
@@ -57,7 +60,8 @@ pub enum Error {
 
     #[snafu(display("Failed to generate function"))]
     GenerateFunction {
-        source: StatsError,
+        #[snafu(source)]
+        error: StatsError,
         location: Location,
     },
 
@@ -109,7 +113,8 @@ pub enum Error {
 
     #[snafu(display("General DataFusion error"))]
     GeneralDataFusion {
-        source: DataFusionError,
+        #[snafu(source)]
+        error: DataFusionError,
         location: Location,
     },
 
@@ -133,14 +138,16 @@ pub enum Error {
 
     #[snafu(display("Failed to cast array to {:?}", typ))]
     TypeCast {
-        source: ArrowError,
+        #[snafu(source)]
+        error: ArrowError,
         typ: arrow::datatypes::DataType,
         location: Location,
     },
 
     #[snafu(display("Failed to perform compute operation on arrow arrays"))]
     ArrowCompute {
-        source: ArrowError,
+        #[snafu(source)]
+        error: ArrowError,
         location: Location,
     },
 
diff --git a/src/common/recordbatch/Cargo.toml b/src/common/recordbatch/Cargo.toml
index bf46b9b43586..b9ca442a3da2 100644
--- a/src/common/recordbatch/Cargo.toml
+++ b/src/common/recordbatch/Cargo.toml
@@ -6,6 +6,7 @@ license.workspace = true
 
 [dependencies]
 common-error = { workspace = true }
+common-macro = { workspace = true }
 datafusion-common.workspace = true
 datafusion.workspace = true
 datatypes = { workspace = true }
diff --git a/src/common/recordbatch/src/error.rs b/src/common/recordbatch/src/error.rs
index 2ddfe27e2d13..92dbbedc6a72 100644
--- a/src/common/recordbatch/src/error.rs
+++ b/src/common/recordbatch/src/error.rs
@@ -17,17 +17,20 @@ use std::any::Any;
 
 use common_error::ext::{BoxedError, ErrorExt};
 use common_error::status_code::StatusCode;
+use common_macro::stack_trace_debug;
 use datatypes::prelude::ConcreteDataType;
 use snafu::{Location, Snafu};
 
 pub type Result<T> = std::result::Result<T, Error>;
 
-#[derive(Debug, Snafu)]
+#[derive(Snafu)]
 #[snafu(visibility(pub))]
+#[stack_trace_debug]
 pub enum Error {
     #[snafu(display("Fail to create datafusion record batch"))]
     NewDfRecordBatch {
-        source: datatypes::arrow::error::ArrowError,
+        #[snafu(source)]
+        error: datatypes::arrow::error::ArrowError,
         location: Location,
     },
 
@@ -54,19 +57,22 @@ pub enum Error {
 
     #[snafu(display("Failed to poll stream"))]
     PollStream {
-        source: datafusion::error::DataFusionError,
+        #[snafu(source)]
+        error: datafusion::error::DataFusionError,
         location: Location,
     },
 
     #[snafu(display("Fail to format record batch"))]
     Format {
-        source: datatypes::arrow::error::ArrowError,
+        #[snafu(source)]
+        error: datatypes::arrow::error::ArrowError,
         location: Location,
     },
 
     #[snafu(display("Failed to init Recordbatch stream"))]
     InitRecordbatchStream {
-        source: datafusion_common::DataFusionError,
+        #[snafu(source)]
+        error: datafusion_common::DataFusionError,
         location: Location,
     },
 
@@ -76,7 +82,8 @@ pub enum Error {
         projection,
     ))]
     ProjectArrowRecordBatch {
-        source: datatypes::arrow::error::ArrowError,
+        #[snafu(source)]
+        error: datatypes::arrow::error::ArrowError,
         location: Location,
         schema: datatypes::schema::SchemaRef,
         projection: Vec<usize>,
diff --git a/src/common/runtime/Cargo.toml b/src/common/runtime/Cargo.toml
index 658e4f4f182b..553a50504ea4 100644
--- a/src/common/runtime/Cargo.toml
+++ b/src/common/runtime/Cargo.toml
@@ -7,6 +7,7 @@ license.workspace = true
 [dependencies]
 async-trait.workspace = true
 common-error = { workspace = true }
+common-macro = { workspace = true }
 common-telemetry = { workspace = true }
 metrics.workspace = true
 once_cell.workspace = true
diff --git a/src/common/runtime/src/error.rs b/src/common/runtime/src/error.rs
index c2e414216de8..35d053510126 100644
--- a/src/common/runtime/src/error.rs
+++ b/src/common/runtime/src/error.rs
@@ -15,17 +15,20 @@
 use std::any::Any;
 
 use common_error::ext::ErrorExt;
+use common_macro::stack_trace_debug;
 use snafu::{Location, Snafu};
 use tokio::task::JoinError;
 
 pub type Result<T> = std::result::Result<T, Error>;
 
-#[derive(Debug, Snafu)]
+#[derive(Snafu)]
 #[snafu(visibility(pub(crate)))]
+#[stack_trace_debug]
 pub enum Error {
     #[snafu(display("Failed to build runtime"))]
     BuildRuntime {
-        source: std::io::Error,
+        #[snafu(source)]
+        error: std::io::Error,
         location: Location,
     },
 
@@ -35,7 +38,8 @@ pub enum Error {
     #[snafu(display("Failed to wait for repeated task {} to stop", name))]
     WaitGcTaskStop {
         name: String,
-        source: JoinError,
+        #[snafu(source)]
+        error: JoinError,
         location: Location,
     },
 }
diff --git a/src/common/substrait/Cargo.toml b/src/common/substrait/Cargo.toml
index edd635d8e3d6..6ebe26fa02c0 100644
--- a/src/common/substrait/Cargo.toml
+++ b/src/common/substrait/Cargo.toml
@@ -11,6 +11,7 @@ bytes = "1.1"
 catalog = { workspace = true }
 common-catalog = { workspace = true }
 common-error = { workspace = true }
+common-macro = { workspace = true }
 common-telemetry = { workspace = true }
 datafusion-common.workspace = true
 datafusion-expr.workspace = true
diff --git a/src/common/substrait/src/error.rs b/src/common/substrait/src/error.rs
index f130a78a7ce1..8c235964639e 100644
--- a/src/common/substrait/src/error.rs
+++ b/src/common/substrait/src/error.rs
@@ -16,13 +16,15 @@ use std::any::Any;
 
 use common_error::ext::{BoxedError, ErrorExt};
 use common_error::status_code::StatusCode;
+use common_macro::stack_trace_debug;
 use datafusion::error::DataFusionError;
 use datatypes::prelude::ConcreteDataType;
 use prost::{DecodeError, EncodeError};
 use snafu::{Location, Snafu};
 
-#[derive(Debug, Snafu)]
+#[derive(Snafu)]
 #[snafu(visibility(pub))]
+#[stack_trace_debug]
 pub enum Error {
     #[snafu(display("Unsupported physical plan: {}", name))]
     UnsupportedPlan { name: String, location: Location },
@@ -41,13 +43,15 @@ pub enum Error {
 
     #[snafu(display("Failed to decode substrait relation"))]
     DecodeRel {
-        source: DecodeError,
+        #[snafu(source)]
+        error: DecodeError,
         location: Location,
     },
 
     #[snafu(display("Failed to encode substrait relation"))]
     EncodeRel {
-        source: EncodeError,
+        #[snafu(source)]
+        error: EncodeError,
         location: Location,
     },
 
@@ -69,7 +73,8 @@ pub enum Error {
 
     #[snafu(display("Internal error from DataFusion"))]
     DFInternal {
-        source: DataFusionError,
+        #[snafu(source)]
+        error: DataFusionError,
         location: Location,
     },
 
@@ -110,13 +115,15 @@ pub enum Error {
 
     #[snafu(display("Failed to encode DataFusion plan"))]
     EncodeDfPlan {
-        source: datafusion::error::DataFusionError,
+        #[snafu(source)]
+        error: datafusion::error::DataFusionError,
         location: Location,
     },
 
     #[snafu(display("Failed to decode DataFusion plan"))]
     DecodeDfPlan {
-        source: datafusion::error::DataFusionError,
+        #[snafu(source)]
+        error: datafusion::error::DataFusionError,
         location: Location,
     },
 }
diff --git a/src/common/telemetry/src/macros.rs b/src/common/telemetry/src/macros.rs
index fed42bdb6dc9..4d5f6003b754 100644
--- a/src/common/telemetry/src/macros.rs
+++ b/src/common/telemetry/src/macros.rs
@@ -39,93 +39,36 @@ macro_rules! error {
     // error!(e; target: "my_target", "a {} event", "log")
     ($e:expr; target: $target:expr, $($arg:tt)+) => ({
         use $crate::common_error::ext::ErrorExt;
-        use std::error::Error;
-        match ($e.source(), $e.location_opt()) {
-            (Some(source), Some(location)) => {
-                $crate::log!(
-                    target: $target,
-                    $crate::logging::Level::ERROR,
-                    err.msg = %$e,
-                    err.code = %$e.status_code(),
-                    err.source = source,
-                    err.location = %location,
-                    $($arg)+
-                )
-            },
-            (Some(source), None) => {
-                $crate::log!(
-                    target: $target,
-                    $crate::logging::Level::ERROR,
-                    err.msg = %$e,
-                    err.code = %$e.status_code(),
-                    err.source = source,
-                    $($arg)+
-                )
-            },
-            (None, Some(location)) => {
-                $crate::log!(
-                    target: $target,
-                    $crate::logging::Level::ERROR,
-                    err.msg = %$e,
-                    err.code = %$e.status_code(),
-                    err.location = %location,
-                    $($arg)+
-                )
-            },
-            (None, None) => {
-                $crate::log!(
-                    target: $target,
-                    $crate::logging::Level::ERROR,
-                    err.msg = %$e,
-                    err.code = %$e.status_code(),
-                    $($arg)+
-                )
-            }
-        }
+        use $crate::common_error::ext::StackError;
+
+        let mut stacks = vec![];
+        $e.debug_fmt(0, &mut stacks);
+        let stacks = stacks.join("\n");
+
+        $crate::log!(
+            target: $target,
+            $crate::logging::Level::ERROR,
+            err.msg = %stacks,
+            err.code = %$e.status_code(),
+            $($arg)+
+        )
     });
 
     // error!(e; "a {} event", "log")
     ($e:expr; $($arg:tt)+) => ({
-        use std::error::Error;
         use $crate::common_error::ext::ErrorExt;
-        match ($e.source(), $e.location_opt()) {
-            (Some(source), Some(location)) => {
-                $crate::log!(
-                    $crate::logging::Level::ERROR,
-                    err.msg = %$e,
-                    err.code = %$e.status_code(),
-                    err.source = source,
-                    err.location = %location,
-                    $($arg)+
-                )
-            },
-            (Some(source), None) => {
-                $crate::log!(
-                    $crate::logging::Level::ERROR,
-                    err.msg = %$e,
-                    err.code = %$e.status_code(),
-                    err.source = source,
-                    $($arg)+
-                )
-            },
-            (None, Some(location)) => {
-                $crate::log!(
-                    $crate::logging::Level::ERROR,
-                    err.msg = %$e,
-                    err.code = %$e.status_code(),
-                    err.location = %location,
-                    $($arg)+
-                )
-            },
-            (None, None) => {
-                $crate::log!(
-                    $crate::logging::Level::ERROR,
-                    err.msg = %$e,
-                    err.code = %$e.status_code(),
-                    $($arg)+
-                )
-            }
-        }
+        use $crate::common_error::ext::StackError;
+
+        let mut stacks = vec![];
+        $e.debug_fmt(0, &mut stacks);
+        let stacks = stacks.join("\n");
+
+        $crate::log!(
+            $crate::logging::Level::ERROR,
+            err.msg = %stacks,
+            err.code = %$e.status_code(),
+            $($arg)+
+        )
     });
 
     // error!("a {} event", "log")
@@ -144,46 +87,19 @@ macro_rules! warn {
 
     // warn!(e; "a {} event", "log")
     ($e:expr; $($arg:tt)+) => ({
-        use std::error::Error;
         use $crate::common_error::ext::ErrorExt;
-        match ($e.source(), $e.location_opt()) {
-            (Some(source), Some(location)) => {
-                $crate::log!(
-                    $crate::logging::Level::WARN,
-                    err.msg = %$e,
-                    err.code = %$e.status_code(),
-                    err.source = source,
-                    err.location = %location,
-                    $($arg)+
-                )
-            },
-            (Some(source), None) => {
-                $crate::log!(
-                    $crate::logging::Level::WARN,
-                    err.msg = %$e,
-                    err.code = %$e.status_code(),
-                    err.source = source,
-                    $($arg)+
-                )
-            },
-            (None, Some(location)) => {
-                $crate::log!(
-                    $crate::logging::Level::WARN,
-                    err.msg = %$e,
-                    err.code = %$e.status_code(),
-                    err.location = %location,
-                    $($arg)+
-                )
-            },
-            (None, None) => {
-                $crate::log!(
-                    $crate::logging::Level::WARN,
-                    err.msg = %$e,
-                    err.code = %$e.status_code(),
-                    $($arg)+
-                )
-            }
-        }
+        use $crate::common_error::ext::StackError;
+
+        let mut stacks = vec![];
+        $e.debug_fmt(0, &mut stacks);
+        let stacks = stacks.join("\n");
+
+        $crate::log!(
+            $crate::logging::Level::WARN,
+            err.msg = %$e,
+            err.code = %$e.status_code(),
+            $($arg)+
+        )
     });
 
     // warn!("a {} event", "log")
diff --git a/src/common/time/Cargo.toml b/src/common/time/Cargo.toml
index 6a1a38af1b63..b8aa229e4ccd 100644
--- a/src/common/time/Cargo.toml
+++ b/src/common/time/Cargo.toml
@@ -9,6 +9,7 @@ arrow.workspace = true
 chrono-tz = "0.8"
 chrono.workspace = true
 common-error = { workspace = true }
+common-macro = { workspace = true }
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 snafu = { version = "0.7", features = ["backtraces"] }
diff --git a/src/common/time/src/error.rs b/src/common/time/src/error.rs
index 1ce675b30f52..2cd4527d2d07 100644
--- a/src/common/time/src/error.rs
+++ b/src/common/time/src/error.rs
@@ -18,13 +18,19 @@ use std::num::{ParseIntError, TryFromIntError};
 use chrono::ParseError;
 use common_error::ext::ErrorExt;
 use common_error::status_code::StatusCode;
+use common_macro::stack_trace_debug;
 use snafu::{Location, Snafu};
 
-#[derive(Debug, Snafu)]
+#[derive(Snafu)]
 #[snafu(visibility(pub))]
+#[stack_trace_debug]
 pub enum Error {
     #[snafu(display("Failed to parse string to date, raw: {}", raw))]
-    ParseDateStr { raw: String, source: ParseError },
+    ParseDateStr {
+        raw: String,
+        #[snafu(source)]
+        error: ParseError,
+    },
 
     #[snafu(display("Invalid date string, raw: {}", raw))]
     InvalidDateStr { raw: String, location: Location },
@@ -37,7 +43,8 @@ pub enum Error {
 
     #[snafu(display("Current timestamp overflow"))]
     TimestampOverflow {
-        source: TryFromIntError,
+        #[snafu(source)]
+        error: TryFromIntError,
         location: Location,
     },
 
@@ -54,7 +61,8 @@ pub enum Error {
     #[snafu(display("Invalid offset string {raw}: "))]
     ParseOffsetStr {
         raw: String,
-        source: ParseIntError,
+        #[snafu(source)]
+        error: ParseIntError,
         location: Location,
     },
 
diff --git a/src/datanode/Cargo.toml b/src/datanode/Cargo.toml
index adb84270b90f..cc505c7d419c 100644
--- a/src/datanode/Cargo.toml
+++ b/src/datanode/Cargo.toml
@@ -23,6 +23,7 @@ common-function = { workspace = true }
 common-greptimedb-telemetry = { workspace = true }
 common-grpc = { workspace = true }
 common-grpc-expr = { workspace = true }
+common-macro = { workspace = true }
 common-meta = { workspace = true }
 common-procedure = { workspace = true }
 common-query = { workspace = true }
diff --git a/src/datanode/src/error.rs b/src/datanode/src/error.rs
index 41af9faa5006..ab315954aadf 100644
--- a/src/datanode/src/error.rs
+++ b/src/datanode/src/error.rs
@@ -16,6 +16,7 @@ use std::any::Any;
 
 use common_error::ext::{BoxedError, ErrorExt};
 use common_error::status_code::StatusCode;
+use common_macro::stack_trace_debug;
 use common_procedure::ProcedureId;
 use serde_json::error::Error as JsonError;
 use servers::define_into_tonic_status;
@@ -24,8 +25,9 @@ use store_api::storage::RegionId;
 use table::error::Error as TableError;
 
 /// Business error of datanode.
-#[derive(Debug, Snafu)]
+#[derive(Snafu)]
 #[snafu(visibility(pub))]
+#[stack_trace_debug]
 pub enum Error {
     #[snafu(display("Failed to handle heartbeat response"))]
     HandleHeartbeatResponse {
@@ -170,14 +172,23 @@ pub enum Error {
     #[snafu(display("Failed to parse address {}", addr))]
     ParseAddr {
         addr: String,
-        source: std::net::AddrParseError,
+        #[snafu(source)]
+        error: std::net::AddrParseError,
     },
 
     #[snafu(display("Failed to create directory {}", dir))]
-    CreateDir { dir: String, source: std::io::Error },
+    CreateDir {
+        dir: String,
+        #[snafu(source)]
+        error: std::io::Error,
+    },
 
     #[snafu(display("Failed to remove directory {}", dir))]
-    RemoveDir { dir: String, source: std::io::Error },
+    RemoveDir {
+        dir: String,
+        #[snafu(source)]
+        error: std::io::Error,
+    },
 
     #[snafu(display("Failed to open log store"))]
     OpenLogStore {
@@ -187,7 +198,8 @@ pub enum Error {
 
     #[snafu(display("Failed to init backend"))]
     InitBackend {
-        source: object_store::Error,
+        #[snafu(source)]
+        error: object_store::Error,
         location: Location,
     },
 
@@ -340,7 +352,8 @@ pub enum Error {
     #[snafu(display("Failed to encode object into json"))]
     EncodeJson {
         location: Location,
-        source: JsonError,
+        #[snafu(source)]
+        error: JsonError,
     },
 
     #[snafu(display("Payload not exist"))]
@@ -351,7 +364,8 @@ pub enum Error {
 
     #[snafu(display("Failed to join task"))]
     JoinTask {
-        source: common_runtime::JoinError,
+        #[snafu(source)]
+        error: common_runtime::JoinError,
         location: Location,
     },
 
diff --git a/src/datatypes/Cargo.toml b/src/datatypes/Cargo.toml
index e1bf3db1f1e1..e0b0df8c6269 100644
--- a/src/datatypes/Cargo.toml
+++ b/src/datatypes/Cargo.toml
@@ -14,6 +14,7 @@ arrow-schema.workspace = true
 arrow.workspace = true
 common-base = { workspace = true }
 common-error = { workspace = true }
+common-macro = { workspace = true }
 common-telemetry = { workspace = true }
 common-time = { workspace = true }
 datafusion-common.workspace = true
diff --git a/src/datatypes/src/error.rs b/src/datatypes/src/error.rs
index fd7cddc0a5f3..f96239bf3a36 100644
--- a/src/datatypes/src/error.rs
+++ b/src/datatypes/src/error.rs
@@ -16,20 +16,24 @@ use std::any::Any;
 
 use common_error::ext::ErrorExt;
 use common_error::status_code::StatusCode;
+use common_macro::stack_trace_debug;
 use snafu::{Location, Snafu};
 
-#[derive(Debug, Snafu)]
+#[derive(Snafu)]
 #[snafu(visibility(pub))]
+#[stack_trace_debug]
 pub enum Error {
     #[snafu(display("Failed to serialize data"))]
     Serialize {
-        source: serde_json::Error,
+        #[snafu(source)]
+        error: serde_json::Error,
         location: Location,
     },
 
     #[snafu(display("Failed to deserialize data, json: {}", json))]
     Deserialize {
-        source: serde_json::Error,
+        #[snafu(source)]
+        error: serde_json::Error,
         location: Location,
         json: String,
     },
@@ -63,7 +67,8 @@ pub enum Error {
     #[snafu(display("Failed to parse version in schema meta, value: {}", value))]
     ParseSchemaVersion {
         value: String,
-        source: std::num::ParseIntError,
+        #[snafu(source)]
+        error: std::num::ParseIntError,
         location: Location,
     },
 
@@ -75,19 +80,22 @@ pub enum Error {
 
     #[snafu(display("Failed to cast arrow time i32 type into i64"))]
     CastTimeType {
-        source: std::num::TryFromIntError,
+        #[snafu(source)]
+        error: std::num::TryFromIntError,
         location: Location,
     },
 
     #[snafu(display("Arrow failed to compute"))]
     ArrowCompute {
-        source: arrow::error::ArrowError,
+        #[snafu(source)]
+        error: arrow::error::ArrowError,
         location: Location,
     },
 
     #[snafu(display("Failed to project arrow schema"))]
     ProjectArrowSchema {
-        source: arrow::error::ArrowError,
+        #[snafu(source)]
+        error: arrow::error::ArrowError,
         location: Location,
     },
 
diff --git a/src/file-engine/Cargo.toml b/src/file-engine/Cargo.toml
index 77a4ebb521f4..f8687df7970e 100644
--- a/src/file-engine/Cargo.toml
+++ b/src/file-engine/Cargo.toml
@@ -13,6 +13,7 @@ async-trait = "0.1"
 common-catalog = { workspace = true }
 common-datasource = { workspace = true }
 common-error = { workspace = true }
+common-macro = { workspace = true }
 common-procedure = { workspace = true }
 common-query = { workspace = true }
 common-recordbatch = { workspace = true }
diff --git a/src/file-engine/src/error.rs b/src/file-engine/src/error.rs
index 87cf49278315..5a23354fc3ef 100644
--- a/src/file-engine/src/error.rs
+++ b/src/file-engine/src/error.rs
@@ -16,14 +16,16 @@ use std::any::Any;
 
 use common_error::ext::ErrorExt;
 use common_error::status_code::StatusCode;
+use common_macro::stack_trace_debug;
 use datafusion::arrow::error::ArrowError;
 use datafusion::error::DataFusionError;
 use serde_json::error::Error as JsonError;
 use snafu::{Location, Snafu};
 use store_api::storage::RegionId;
 
-#[derive(Debug, Snafu)]
+#[derive(Snafu)]
 #[snafu(visibility(pub))]
+#[stack_trace_debug]
 pub enum Error {
     #[snafu(display("Unsupported operation: {}", operation))]
     Unsupported {
@@ -50,38 +52,44 @@ pub enum Error {
     CheckObject {
         path: String,
         location: Location,
-        source: object_store::Error,
+        #[snafu(source)]
+        error: object_store::Error,
     },
 
     #[snafu(display("Fail to encode object into json"))]
     EncodeJson {
         location: Location,
-        source: JsonError,
+        #[snafu(source)]
+        error: JsonError,
     },
 
     #[snafu(display("Fail to decode object from json"))]
     DecodeJson {
         location: Location,
-        source: JsonError,
+        #[snafu(source)]
+        error: JsonError,
     },
 
     #[snafu(display("Failed to store region manifest, region_id: {}", region_id))]
     StoreRegionManifest {
-        source: object_store::Error,
+        #[snafu(source)]
+        error: object_store::Error,
         region_id: RegionId,
         location: Location,
     },
 
     #[snafu(display("Failed to load region manifest, region_id: {}", region_id))]
     LoadRegionManifest {
-        source: object_store::Error,
+        #[snafu(source)]
+        error: object_store::Error,
         region_id: RegionId,
         location: Location,
     },
 
     #[snafu(display("Failed to delete region manifest, region_id: {},", region_id))]
     DeleteRegionManifest {
-        source: object_store::Error,
+        #[snafu(source)]
+        error: object_store::Error,
         region_id: RegionId,
         location: Location,
     },
@@ -100,19 +108,22 @@ pub enum Error {
 
     #[snafu(display("Failed to build csv config"))]
     BuildCsvConfig {
-        source: common_datasource::file_format::csv::CsvConfigBuilderError,
+        #[snafu(source)]
+        error: common_datasource::file_format::csv::CsvConfigBuilderError,
         location: Location,
     },
 
     #[snafu(display("Failed to build stream"))]
     BuildStream {
-        source: DataFusionError,
+        #[snafu(source)]
+        error: DataFusionError,
         location: Location,
     },
 
     #[snafu(display("Failed to project schema"))]
     ProjectArrowSchema {
-        source: ArrowError,
+        #[snafu(source)]
+        error: ArrowError,
         location: Location,
     },
 
@@ -136,7 +147,8 @@ pub enum Error {
 
     #[snafu(display("Failed to generate parquet scan plan"))]
     ParquetScanPlan {
-        source: DataFusionError,
+        #[snafu(source)]
+        error: DataFusionError,
         location: Location,
     },
 
@@ -153,7 +165,8 @@ pub enum Error {
 
     #[snafu(display("Failed to extract column from filter"))]
     ExtractColumnFromFilter {
-        source: DataFusionError,
+        #[snafu(source)]
+        error: DataFusionError,
         location: Location,
     },
 
diff --git a/src/frontend/Cargo.toml b/src/frontend/Cargo.toml
index 285ad121ba63..b85a18008eca 100644
--- a/src/frontend/Cargo.toml
+++ b/src/frontend/Cargo.toml
@@ -28,6 +28,7 @@ common-error = { workspace = true }
 common-function = { workspace = true }
 common-grpc = { workspace = true }
 common-grpc-expr = { workspace = true }
+common-macro = { workspace = true }
 common-meta = { workspace = true }
 common-procedure = { workspace = true }
 common-query = { workspace = true }
diff --git a/src/frontend/src/error.rs b/src/frontend/src/error.rs
index c418cb11252f..a176d663b5bb 100644
--- a/src/frontend/src/error.rs
+++ b/src/frontend/src/error.rs
@@ -17,12 +17,14 @@ use std::any::Any;
 use common_datasource::file_format::Format;
 use common_error::ext::{BoxedError, ErrorExt};
 use common_error::status_code::StatusCode;
+use common_macro::stack_trace_debug;
 use servers::define_into_tonic_status;
 use snafu::{Location, Snafu};
 use store_api::storage::RegionNumber;
 
-#[derive(Debug, Snafu)]
+#[derive(Snafu)]
 #[snafu(visibility(pub))]
+#[stack_trace_debug]
 pub enum Error {
     #[snafu(display("Failed to invalidate table cache"))]
     InvalidateTableCache {
@@ -44,43 +46,44 @@ pub enum Error {
 
     #[snafu(display(""))]
     External {
-        #[snafu(backtrace)]
+        location: Location,
         source: BoxedError,
     },
 
     #[snafu(display("Failed to query"))]
     RequestQuery {
-        #[snafu(backtrace)]
+        location: Location,
         source: common_meta::error::Error,
     },
 
     #[snafu(display("Runtime resource error"))]
     RuntimeResource {
-        #[snafu(backtrace)]
+        location: Location,
         source: common_runtime::error::Error,
     },
 
     #[snafu(display("Failed to start server"))]
     StartServer {
-        #[snafu(backtrace)]
+        location: Location,
         source: servers::error::Error,
     },
 
     #[snafu(display("Failed to shutdown server"))]
     ShutdownServer {
-        #[snafu(backtrace)]
+        location: Location,
         source: servers::error::Error,
     },
 
     #[snafu(display("Failed to parse address {}", addr))]
     ParseAddr {
         addr: String,
-        source: std::net::AddrParseError,
+        #[snafu(source)]
+        error: std::net::AddrParseError,
     },
 
     #[snafu(display("Failed to parse SQL"))]
     ParseSql {
-        #[snafu(backtrace)]
+        location: Location,
         source: sql::error::Error,
     },
 
@@ -113,13 +116,13 @@ pub enum Error {
 
     #[snafu(display("General catalog error"))]
     Catalog {
-        #[snafu(backtrace)]
+        location: Location,
         source: catalog::error::Error,
     },
 
     #[snafu(display("Failed to start Meta client"))]
     StartMetaClient {
-        #[snafu(backtrace)]
+        location: Location,
         source: meta_client::error::Error,
     },
 
@@ -132,7 +135,7 @@ pub enum Error {
     #[snafu(display("Failed to find table route for table id {}", table_id))]
     FindTableRoute {
         table_id: u32,
-        #[snafu(backtrace)]
+        location: Location,
         source: partition::error::Error,
     },
 
@@ -141,7 +144,7 @@ pub enum Error {
 
     #[snafu(display("Table occurs error"))]
     Table {
-        #[snafu(backtrace)]
+        location: Location,
         source: table::error::Error,
     },
 
@@ -150,26 +153,26 @@ pub enum Error {
 
     #[snafu(display("Failed to plan statement"))]
     PlanStatement {
-        #[snafu(backtrace)]
+        location: Location,
         source: query::error::Error,
     },
 
     #[snafu(display("Failed to read table: {table_name}"))]
     ReadTable {
         table_name: String,
-        #[snafu(backtrace)]
+        location: Location,
         source: query::error::Error,
     },
 
     #[snafu(display("Failed to execute logical plan"))]
     ExecLogicalPlan {
-        #[snafu(backtrace)]
+        location: Location,
         source: query::error::Error,
     },
 
     #[snafu(display(""))]
     InvokeRegionServer {
-        #[snafu(backtrace)]
+        location: Location,
         source: servers::error::Error,
     },
 
@@ -190,7 +193,7 @@ pub enum Error {
 
     #[snafu(display("SQL execution intercepted"))]
     SqlExecIntercepted {
-        #[snafu(backtrace)]
+        location: Location,
         source: BoxedError,
     },
 
@@ -198,19 +201,19 @@ pub enum Error {
     #[snafu(display("Failed to execute PromQL query {}", query))]
     ExecutePromql {
         query: String,
-        #[snafu(backtrace)]
+        location: Location,
         source: servers::error::Error,
     },
 
     #[snafu(display("Failed to create logical plan for prometheus query"))]
     PromStoreRemoteQueryPlan {
-        #[snafu(backtrace)]
+        location: Location,
         source: servers::error::Error,
     },
 
     #[snafu(display("Failed to describe schema for given statement"))]
     DescribeStatement {
-        #[snafu(backtrace)]
+        location: Location,
         source: query::error::Error,
     },
 
@@ -219,21 +222,21 @@ pub enum Error {
 
     #[snafu(display("Failed to start script manager"))]
     StartScriptManager {
-        #[snafu(backtrace)]
+        location: Location,
         source: script::error::Error,
     },
 
     #[snafu(display("Failed to copy table: {}", table_name))]
     CopyTable {
         table_name: String,
-        #[snafu(backtrace)]
+        location: Location,
         source: table::error::Error,
     },
 
     #[snafu(display("Failed to insert value into table: {}", table_name))]
     Insert {
         table_name: String,
-        #[snafu(backtrace)]
+        location: Location,
         source: table::error::Error,
     },
 
@@ -293,7 +296,7 @@ impl ErrorExt for Error {
 
             Error::Permission { source, .. } => source.status_code(),
 
-            Error::DescribeStatement { source } => source.status_code(),
+            Error::DescribeStatement { source, .. } => source.status_code(),
 
             Error::HandleHeartbeatResponse { source, .. }
             | Error::TableMetadataManager { source, .. } => source.status_code(),
@@ -306,17 +309,17 @@ impl ErrorExt for Error {
             Error::StartServer { source, .. } => source.status_code(),
             Error::ShutdownServer { source, .. } => source.status_code(),
 
-            Error::ParseSql { source } => source.status_code(),
+            Error::ParseSql { source, .. } => source.status_code(),
 
             Error::InvalidateTableCache { source, .. } => source.status_code(),
 
-            Error::Table { source }
+            Error::Table { source, .. }
             | Error::CopyTable { source, .. }
             | Error::Insert { source, .. } => source.status_code(),
 
             Error::OpenRaftEngineBackend { .. } => StatusCode::StorageUnavailable,
 
-            Error::RequestQuery { source } => source.status_code(),
+            Error::RequestQuery { source, .. } => source.status_code(),
 
             Error::FindDatanode { .. }
             | Error::VectorToGrpcColumn { .. }
@@ -330,22 +333,21 @@ impl ErrorExt for Error {
 
             Error::Catalog { source, .. } => source.status_code(),
 
-            Error::StartMetaClient { source } | Error::CreateMetaHeartbeatStream { source, .. } => {
-                source.status_code()
-            }
+            Error::StartMetaClient { source, .. }
+            | Error::CreateMetaHeartbeatStream { source, .. } => source.status_code(),
 
-            Error::PlanStatement { source }
+            Error::PlanStatement { source, .. }
             | Error::ReadTable { source, .. }
-            | Error::ExecLogicalPlan { source } => source.status_code(),
+            | Error::ExecLogicalPlan { source, .. } => source.status_code(),
 
             Error::LeaderNotFound { .. } => StatusCode::StorageUnavailable,
             Error::TableAlreadyExist { .. } => StatusCode::TableAlreadyExists,
-            Error::InvokeRegionServer { source } => source.status_code(),
+            Error::InvokeRegionServer { source, .. } => source.status_code(),
 
-            Error::External { source } => source.status_code(),
+            Error::External { source, .. } => source.status_code(),
             Error::FindTableRoute { source, .. } => source.status_code(),
 
-            Error::StartScriptManager { source } => source.status_code(),
+            Error::StartScriptManager { source, .. } => source.status_code(),
 
             Error::TableOperation { source, .. } => source.status_code(),
         }
diff --git a/src/frontend/src/server.rs b/src/frontend/src/server.rs
index 5a61c3b48834..794c7000a875 100644
--- a/src/frontend/src/server.rs
+++ b/src/frontend/src/server.rs
@@ -21,7 +21,7 @@ use common_base::Plugins;
 use common_runtime::Builder as RuntimeBuilder;
 use common_telemetry::info;
 use servers::configurator::ConfiguratorRef;
-use servers::error::Error::InternalIo;
+use servers::error::InternalIoSnafu;
 use servers::grpc::{GrpcServer, GrpcServerConfig};
 use servers::http::HttpServerBuilder;
 use servers::metrics_handler::MetricsHandler;
@@ -33,8 +33,7 @@ use servers::query_handler::sql::ServerSqlQueryHandlerAdaptor;
 use servers::server::Server;
 use snafu::ResultExt;
 
-use crate::error::Error::StartServer;
-use crate::error::{self, Result};
+use crate::error::{self, Result, StartServerSnafu};
 use crate::frontend::FrontendOptions;
 use crate::instance::FrontendInstance;
 
@@ -149,9 +148,8 @@ impl Services {
                     opts.tls.should_force_tls(),
                     opts.tls
                         .setup()
-                        .map_err(|e| StartServer {
-                            source: InternalIo { source: e },
-                        })?
+                        .context(InternalIoSnafu)
+                        .context(StartServerSnafu)?
                         .map(Arc::new),
                     opts.reject_no_database.unwrap_or(false),
                 )),
diff --git a/src/log-store/Cargo.toml b/src/log-store/Cargo.toml
index b02b7c6a5af9..bedf1fc64f65 100644
--- a/src/log-store/Cargo.toml
+++ b/src/log-store/Cargo.toml
@@ -18,6 +18,7 @@ bytes = "1.1"
 common-base = { workspace = true }
 common-config = { workspace = true }
 common-error = { workspace = true }
+common-macro = { workspace = true }
 common-meta = { workspace = true }
 common-runtime = { workspace = true }
 common-telemetry = { workspace = true }
diff --git a/src/log-store/src/error.rs b/src/log-store/src/error.rs
index 6865bbed22c6..a24a48d84be2 100644
--- a/src/log-store/src/error.rs
+++ b/src/log-store/src/error.rs
@@ -15,11 +15,13 @@
 use std::any::Any;
 
 use common_error::ext::ErrorExt;
+use common_macro::stack_trace_debug;
 use common_runtime::error::Error as RuntimeError;
 use snafu::{Location, Snafu};
 
-#[derive(Debug, Snafu)]
+#[derive(Snafu)]
 #[snafu(visibility(pub))]
+#[stack_trace_debug]
 pub enum Error {
     #[snafu(display("Failed to start log store gc task"))]
     StartGcTask {
@@ -35,13 +37,15 @@ pub enum Error {
 
     #[snafu(display("Failed to add entry to LogBatch"))]
     AddEntryLogBatch {
-        source: raft_engine::Error,
+        #[snafu(source)]
+        error: raft_engine::Error,
         location: Location,
     },
 
     #[snafu(display("Failed to perform raft-engine operation"))]
     RaftEngine {
-        source: raft_engine::Error,
+        #[snafu(source)]
+        error: raft_engine::Error,
         location: Location,
     },
 
@@ -63,7 +67,8 @@ pub enum Error {
         start: u64,
         end: u64,
         max_size: usize,
-        source: raft_engine::Error,
+        #[snafu(source)]
+        error: raft_engine::Error,
         location: Location,
     },
 
diff --git a/src/meta-client/Cargo.toml b/src/meta-client/Cargo.toml
index 85b5b55b14ac..7f8a9035d8de 100644
--- a/src/meta-client/Cargo.toml
+++ b/src/meta-client/Cargo.toml
@@ -10,6 +10,7 @@ async-trait = "0.1"
 chrono.workspace = true
 common-error = { workspace = true }
 common-grpc = { workspace = true }
+common-macro = { workspace = true }
 common-meta = { workspace = true }
 common-telemetry = { workspace = true }
 etcd-client.workspace = true
diff --git a/src/meta-client/src/error.rs b/src/meta-client/src/error.rs
index 8644e18948ad..a86ec822802f 100644
--- a/src/meta-client/src/error.rs
+++ b/src/meta-client/src/error.rs
@@ -15,11 +15,13 @@
 use common_error::ext::ErrorExt;
 use common_error::status_code::StatusCode;
 use common_error::{GREPTIME_ERROR_CODE, GREPTIME_ERROR_MSG};
+use common_macro::stack_trace_debug;
 use snafu::{Location, Snafu};
 use tonic::Status;
 
-#[derive(Debug, Snafu)]
+#[derive(Snafu)]
 #[snafu(visibility(pub))]
+#[stack_trace_debug]
 pub enum Error {
     #[snafu(display("Illegal GRPC client state: {}", err_msg))]
     IllegalGrpcClientState { err_msg: String, location: Location },
@@ -36,7 +38,8 @@ pub enum Error {
     #[snafu(display("Ask leader timeout"))]
     AskLeaderTimeout {
         location: Location,
-        source: tokio::time::error::Elapsed,
+        #[snafu(source)]
+        error: tokio::time::error::Elapsed,
     },
 
     #[snafu(display("Failed to create gRPC channel"))]
diff --git a/src/meta-srv/Cargo.toml b/src/meta-srv/Cargo.toml
index 0942583b380e..0836485e0a89 100644
--- a/src/meta-srv/Cargo.toml
+++ b/src/meta-srv/Cargo.toml
@@ -20,6 +20,7 @@ common-error = { workspace = true }
 common-greptimedb-telemetry = { workspace = true }
 common-grpc = { workspace = true }
 common-grpc-expr = { workspace = true }
+common-macro = { workspace = true }
 common-meta = { workspace = true }
 common-procedure = { workspace = true }
 common-runtime = { workspace = true }
diff --git a/src/meta-srv/src/cluster.rs b/src/meta-srv/src/cluster.rs
index 14ba5b735475..a314e43b58d0 100644
--- a/src/meta-srv/src/cluster.rs
+++ b/src/meta-srv/src/cluster.rs
@@ -256,8 +256,8 @@ fn check_resp_header(header: &Option<ResponseHeader>, ctx: Context) -> Result<()
 fn need_retry(error: &error::Error) -> bool {
     match error {
         error::Error::IsNotLeader { .. } => true,
-        error::Error::Range { source, .. } | error::Error::BatchGet { source, .. } => {
-            match_for_io_error(source).is_some()
+        error::Error::Range { error, .. } | error::Error::BatchGet { error, .. } => {
+            match_for_io_error(error).is_some()
         }
         _ => false,
     }
diff --git a/src/meta-srv/src/error.rs b/src/meta-srv/src/error.rs
index 2ce6c565aa71..16ee839ce928 100644
--- a/src/meta-srv/src/error.rs
+++ b/src/meta-srv/src/error.rs
@@ -14,6 +14,7 @@
 
 use common_error::ext::{BoxedError, ErrorExt};
 use common_error::status_code::StatusCode;
+use common_macro::stack_trace_debug;
 use common_meta::peer::Peer;
 use common_runtime::JoinError;
 use servers::define_into_tonic_status;
@@ -24,8 +25,9 @@ use tonic::codegen::http;
 
 use crate::pubsub::Message;
 
-#[derive(Debug, Snafu)]
+#[derive(Snafu)]
 #[snafu(visibility(pub))]
+#[stack_trace_debug]
 pub enum Error {
     #[snafu(display("Failed to create default catalog and schema"))]
     InitMetadata {
@@ -74,7 +76,8 @@ pub enum Error {
     #[snafu(display("Failed to join a future"))]
     Join {
         location: Location,
-        source: JoinError,
+        #[snafu(source)]
+        error: JoinError,
     },
 
     #[snafu(display("Failed to execute transaction: {}", msg))]
@@ -110,7 +113,10 @@ pub enum Error {
     },
 
     #[snafu(display("Failed to send shutdown signal"))]
-    SendShutdownSignal { source: SendError<()> },
+    SendShutdownSignal {
+        #[snafu(source)]
+        error: SendError<()>,
+    },
 
     #[snafu(display("Failed to shutdown {} server", server))]
     ShutdownServer {
@@ -124,26 +130,30 @@ pub enum Error {
 
     #[snafu(display("Failed to execute via Etcd"))]
     EtcdFailed {
-        source: etcd_client::Error,
+        #[snafu(source)]
+        error: etcd_client::Error,
         location: Location,
     },
 
     #[snafu(display("Failed to connect to Etcd"))]
     ConnectEtcd {
-        source: etcd_client::Error,
+        #[snafu(source)]
+        error: etcd_client::Error,
         location: Location,
     },
 
     #[snafu(display("Failed to bind address {}", addr))]
     TcpBind {
         addr: String,
-        source: std::io::Error,
+        #[snafu(source)]
+        error: std::io::Error,
         location: Location,
     },
 
     #[snafu(display("Failed to start gRPC server"))]
     StartGrpc {
-        source: tonic::transport::Error,
+        #[snafu(source)]
+        error: tonic::transport::Error,
         location: Location,
     },
     #[snafu(display("Failed to start http server"))]
@@ -154,7 +164,8 @@ pub enum Error {
     #[snafu(display("Failed to parse address {}", addr))]
     ParseAddr {
         addr: String,
-        source: std::net::AddrParseError,
+        #[snafu(source)]
+        error: std::net::AddrParseError,
     },
     #[snafu(display("Empty table name"))]
     EmptyTableName { location: Location },
@@ -170,52 +181,60 @@ pub enum Error {
 
     #[snafu(display("Failed to parse datanode lease key from utf8"))]
     LeaseKeyFromUtf8 {
-        source: std::string::FromUtf8Error,
+        #[snafu(source)]
+        error: std::string::FromUtf8Error,
         location: Location,
     },
 
     #[snafu(display("Failed to parse datanode lease value from utf8"))]
     LeaseValueFromUtf8 {
-        source: std::string::FromUtf8Error,
+        #[snafu(source)]
+        error: std::string::FromUtf8Error,
         location: Location,
     },
 
     #[snafu(display("Failed to parse datanode stat key from utf8"))]
     StatKeyFromUtf8 {
-        source: std::string::FromUtf8Error,
+        #[snafu(source)]
+        error: std::string::FromUtf8Error,
         location: Location,
     },
 
     #[snafu(display("Failed to parse datanode stat value from utf8"))]
     StatValueFromUtf8 {
-        source: std::string::FromUtf8Error,
+        #[snafu(source)]
+        error: std::string::FromUtf8Error,
         location: Location,
     },
 
     #[snafu(display("Failed to parse invalid region key from utf8"))]
     InvalidRegionKeyFromUtf8 {
-        source: std::string::FromUtf8Error,
+        #[snafu(source)]
+        error: std::string::FromUtf8Error,
         location: Location,
     },
 
     #[snafu(display("Failed to serialize to json: {}", input))]
     SerializeToJson {
         input: String,
-        source: serde_json::error::Error,
+        #[snafu(source)]
+        error: serde_json::error::Error,
         location: Location,
     },
 
     #[snafu(display("Failed to deserialize from json: {}", input))]
     DeserializeFromJson {
         input: String,
-        source: serde_json::error::Error,
+        #[snafu(source)]
+        error: serde_json::error::Error,
         location: Location,
     },
 
     #[snafu(display("Failed to parse number: {}", err_msg))]
     ParseNum {
         err_msg: String,
-        source: std::num::ParseIntError,
+        #[snafu(source)]
+        error: std::num::ParseIntError,
         location: Location,
     },
 
@@ -239,7 +258,8 @@ pub enum Error {
 
     #[snafu(display("Failed to decode table route"))]
     DecodeTableRoute {
-        source: prost::DecodeError,
+        #[snafu(source)]
+        error: prost::DecodeError,
         location: Location,
     },
 
@@ -294,13 +314,15 @@ pub enum Error {
 
     #[snafu(display("Failed to batch get KVs from leader's in_memory kv store"))]
     BatchGet {
-        source: tonic::Status,
+        #[snafu(source)]
+        error: tonic::Status,
         location: Location,
     },
 
     #[snafu(display("Failed to batch range KVs from leader's in_memory kv store"))]
     Range {
-        source: tonic::Status,
+        #[snafu(source)]
+        error: tonic::Status,
         location: Location,
     },
 
@@ -315,7 +337,8 @@ pub enum Error {
 
     #[snafu(display("Invalid http body"))]
     InvalidHttpBody {
-        source: http::Error,
+        #[snafu(source)]
+        error: http::Error,
         location: Location,
     },
 
@@ -332,19 +355,22 @@ pub enum Error {
 
     #[snafu(display("Failed to lock based on etcd"))]
     Lock {
-        source: etcd_client::Error,
+        #[snafu(source)]
+        error: etcd_client::Error,
         location: Location,
     },
 
     #[snafu(display("Failed to unlock based on etcd"))]
     Unlock {
-        source: etcd_client::Error,
+        #[snafu(source)]
+        error: etcd_client::Error,
         location: Location,
     },
 
     #[snafu(display("Failed to grant lease"))]
     LeaseGrant {
-        source: etcd_client::Error,
+        #[snafu(source)]
+        error: etcd_client::Error,
         location: Location,
     },
 
@@ -353,7 +379,8 @@ pub enum Error {
 
     #[snafu(display("Invalid utf-8 value"))]
     InvalidUtf8Value {
-        source: std::string::FromUtf8Error,
+        #[snafu(source)]
+        error: std::string::FromUtf8Error,
         location: Location,
     },
 
@@ -494,7 +521,8 @@ pub enum Error {
 
     #[snafu(display("Failed to publish message"))]
     PublishMessage {
-        source: SendError<Message>,
+        #[snafu(source)]
+        error: SendError<Message>,
         location: Location,
     },
 
diff --git a/src/mito2/Cargo.toml b/src/mito2/Cargo.toml
index 0a1ff27f9e4c..cdd2d8d71b56 100644
--- a/src/mito2/Cargo.toml
+++ b/src/mito2/Cargo.toml
@@ -22,6 +22,7 @@ common-base = { workspace = true }
 common-catalog = { workspace = true }
 common-datasource = { workspace = true }
 common-error = { workspace = true }
+common-macro = { workspace = true }
 common-procedure = { workspace = true }
 common-query = { workspace = true }
 common-recordbatch = { workspace = true }
diff --git a/src/mito2/src/error.rs b/src/mito2/src/error.rs
index 06aa47a9cd74..fcc3842111d1 100644
--- a/src/mito2/src/error.rs
+++ b/src/mito2/src/error.rs
@@ -18,6 +18,7 @@ use std::sync::Arc;
 use common_datasource::compression::CompressionType;
 use common_error::ext::{BoxedError, ErrorExt};
 use common_error::status_code::StatusCode;
+use common_macro::stack_trace_debug;
 use common_runtime::JoinError;
 use datatypes::arrow::error::ArrowError;
 use datatypes::prelude::ConcreteDataType;
@@ -29,33 +30,38 @@ use store_api::storage::RegionId;
 use crate::sst::file::FileId;
 use crate::worker::WorkerId;
 
-#[derive(Debug, Snafu)]
+#[derive(Snafu)]
 #[snafu(visibility(pub))]
+#[stack_trace_debug]
 pub enum Error {
     #[snafu(display("OpenDAL operator failed"))]
     OpenDal {
         location: Location,
-        source: object_store::Error,
+        #[snafu(source)]
+        error: object_store::Error,
     },
 
     #[snafu(display("Fail to compress object by {}, path: {}", compress_type, path))]
     CompressObject {
         compress_type: CompressionType,
         path: String,
-        source: std::io::Error,
+        #[snafu(source)]
+        error: std::io::Error,
     },
 
     #[snafu(display("Fail to decompress object by {}, path: {}", compress_type, path))]
     DecompressObject {
         compress_type: CompressionType,
         path: String,
-        source: std::io::Error,
+        #[snafu(source)]
+        error: std::io::Error,
     },
 
     #[snafu(display("Failed to ser/de json object"))]
     SerdeJson {
         location: Location,
-        source: serde_json::Error,
+        #[snafu(source)]
+        error: serde_json::Error,
     },
 
     #[snafu(display("Invalid scan index, start: {}, end: {}", start, end))]
@@ -68,7 +74,8 @@ pub enum Error {
     #[snafu(display("Invalid UTF-8 content"))]
     Utf8 {
         location: Location,
-        source: std::str::Utf8Error,
+        #[snafu(source)]
+        error: std::str::Utf8Error,
     },
 
     #[snafu(display("Cannot find RegionMetadata"))]
@@ -76,7 +83,8 @@ pub enum Error {
 
     #[snafu(display("Failed to join handle"))]
     Join {
-        source: common_runtime::JoinError,
+        #[snafu(source)]
+        error: common_runtime::JoinError,
         location: Location,
     },
 
@@ -85,7 +93,8 @@ pub enum Error {
 
     #[snafu(display("Failed to recv result"))]
     Recv {
-        source: tokio::sync::oneshot::error::RecvError,
+        #[snafu(source)]
+        error: tokio::sync::oneshot::error::RecvError,
         location: Location,
     },
 
@@ -101,7 +110,8 @@ pub enum Error {
     #[snafu(display("Failed to create RecordBatch from vectors"))]
     NewRecordBatch {
         location: Location,
-        source: ArrowError,
+        #[snafu(source)]
+        error: ArrowError,
     },
 
     #[snafu(display("Failed to write to buffer"))]
@@ -114,13 +124,15 @@ pub enum Error {
     WriteParquet {
         path: String,
         location: Location,
-        source: parquet::errors::ParquetError,
+        #[snafu(source)]
+        error: parquet::errors::ParquetError,
     },
 
     #[snafu(display("Failed to read parquet file, path: {}", path))]
     ReadParquet {
         path: String,
-        source: parquet::errors::ParquetError,
+        #[snafu(source)]
+        error: parquet::errors::ParquetError,
         location: Location,
     },
 
@@ -167,7 +179,8 @@ pub enum Error {
     EncodeWal {
         region_id: RegionId,
         location: Location,
-        source: EncodeError,
+        #[snafu(source)]
+        error: EncodeError,
     },
 
     #[snafu(display("Failed to write WAL"))]
@@ -187,7 +200,8 @@ pub enum Error {
     DecodeWal {
         region_id: RegionId,
         location: Location,
-        source: DecodeError,
+        #[snafu(source)]
+        error: DecodeError,
     },
 
     #[snafu(display("Failed to delete WAL, region_id: {}", region_id))]
@@ -206,7 +220,8 @@ pub enum Error {
 
     #[snafu(display("Failed to serialize field"))]
     SerializeField {
-        source: memcomparable::Error,
+        #[snafu(source)]
+        error: memcomparable::Error,
         location: Location,
     },
 
@@ -221,7 +236,8 @@ pub enum Error {
 
     #[snafu(display("Failed to deserialize field"))]
     DeserializeField {
-        source: memcomparable::Error,
+        #[snafu(source)]
+        error: memcomparable::Error,
         location: Location,
     },
 
@@ -247,7 +263,8 @@ pub enum Error {
     #[snafu(display("Failed to compute arrow arrays"))]
     ComputeArrow {
         location: Location,
-        source: datatypes::arrow::error::ArrowError,
+        #[snafu(source)]
+        error: datatypes::arrow::error::ArrowError,
     },
 
     #[snafu(display("Failed to compute vector"))]
@@ -271,7 +288,8 @@ pub enum Error {
 
     #[snafu(display("Failed to stop scheduler"))]
     StopScheduler {
-        source: JoinError,
+        #[snafu(source)]
+        error: JoinError,
         location: Location,
     },
 
@@ -284,7 +302,8 @@ pub enum Error {
     #[snafu(display("Failed to delete SST file, file id: {}", file_id))]
     DeleteSst {
         file_id: FileId,
-        source: object_store::Error,
+        #[snafu(source)]
+        error: object_store::Error,
         location: Location,
     },
 
@@ -352,9 +371,10 @@ pub enum Error {
         location: Location,
     },
 
-    #[snafu(display("Invalid options, source: {}", source))]
+    #[snafu(display("Invalid options"))]
     JsonOptions {
-        source: serde_json::Error,
+        #[snafu(source)]
+        error: serde_json::Error,
         location: Location,
     },
 
diff --git a/src/operator/Cargo.toml b/src/operator/Cargo.toml
index c87ee3b9c58f..2de2f326745b 100644
--- a/src/operator/Cargo.toml
+++ b/src/operator/Cargo.toml
@@ -20,6 +20,7 @@ common-catalog = { workspace = true }
 common-datasource = { workspace = true }
 common-error = { workspace = true }
 common-grpc-expr = { workspace = true }
+common-macro = { workspace = true }
 common-meta = { workspace = true }
 common-query = { workspace = true }
 common-recordbatch = { workspace = true }
diff --git a/src/operator/src/error.rs b/src/operator/src/error.rs
index 8194c196f9a1..3939bb28a453 100644
--- a/src/operator/src/error.rs
+++ b/src/operator/src/error.rs
@@ -17,14 +17,16 @@ use std::any::Any;
 use common_datasource::file_format::Format;
 use common_error::ext::{BoxedError, ErrorExt};
 use common_error::status_code::StatusCode;
+use common_macro::stack_trace_debug;
 use datafusion::parquet;
 use datatypes::arrow::error::ArrowError;
 use datatypes::value::Value;
 use servers::define_into_tonic_status;
 use snafu::{Location, Snafu};
 
-#[derive(Debug, Snafu)]
+#[derive(Snafu)]
 #[snafu(visibility(pub))]
+#[stack_trace_debug]
 pub enum Error {
     #[snafu(display("Failed to invalidate table cache"))]
     InvalidateTableCache {
@@ -46,52 +48,52 @@ pub enum Error {
 
     #[snafu(display("external error"))]
     External {
-        #[snafu(backtrace)]
+        location: Location,
         source: BoxedError,
     },
 
     #[snafu(display("Failed to insert data"))]
     RequestInserts {
-        #[snafu(backtrace)]
+        location: Location,
         source: common_meta::error::Error,
     },
 
     #[snafu(display("Failed to delete data"))]
     RequestDeletes {
-        #[snafu(backtrace)]
+        location: Location,
         source: common_meta::error::Error,
     },
 
     #[snafu(display("Failed to parse SQL"))]
     ParseSql {
-        #[snafu(backtrace)]
+        location: Location,
         source: sql::error::Error,
     },
 
     #[snafu(display("Failed to convert value to sql value: {}", value))]
     ConvertSqlValue {
         value: Value,
-        #[snafu(backtrace)]
+        location: Location,
         source: sql::error::Error,
     },
 
     #[snafu(display("Column datatype error"))]
     ColumnDataType {
-        #[snafu(backtrace)]
+        location: Location,
         source: api::error::Error,
     },
 
     #[snafu(display("Invalid column proto definition, column: {}", column))]
     InvalidColumnDef {
         column: String,
-        #[snafu(backtrace)]
+        location: Location,
         source: api::error::Error,
     },
 
     #[snafu(display("Failed to convert column default constraint, column: {}", column_name))]
     ConvertColumnDefaultConstraint {
         column_name: String,
-        #[snafu(backtrace)]
+        location: Location,
         source: datatypes::error::Error,
     },
 
@@ -109,20 +111,21 @@ pub enum Error {
 
     #[snafu(display("Failed to join task"))]
     JoinTask {
-        source: common_runtime::JoinError,
+        #[snafu(source)]
+        error: common_runtime::JoinError,
         location: Location,
     },
 
     #[snafu(display("General catalog error"))]
     Catalog {
-        #[snafu(backtrace)]
+        location: Location,
         source: catalog::error::Error,
     },
 
     #[snafu(display("Failed to find table partition rule for table {}", table_name))]
     FindTablePartitionRule {
         table_name: String,
-        #[snafu(backtrace)]
+        location: Location,
         source: partition::error::Error,
     },
 
@@ -146,13 +149,13 @@ pub enum Error {
 
     #[snafu(display("Failed to create table info"))]
     CreateTableInfo {
-        #[snafu(backtrace)]
+        location: Location,
         source: datatypes::error::Error,
     },
 
     #[snafu(display("Failed to build CreateExpr on insertion"))]
     BuildCreateExprOnInsertion {
-        #[snafu(backtrace)]
+        location: Location,
         source: common_grpc_expr::error::Error,
     },
 
@@ -167,7 +170,7 @@ pub enum Error {
 
     #[snafu(display("Table occurs error"))]
     Table {
-        #[snafu(backtrace)]
+        location: Location,
         source: table::error::Error,
     },
 
@@ -176,44 +179,46 @@ pub enum Error {
 
     #[snafu(display("Failed to execute statement"))]
     ExecuteStatement {
-        #[snafu(backtrace)]
+        location: Location,
         source: query::error::Error,
     },
 
     #[snafu(display("Failed to plan statement"))]
     PlanStatement {
-        #[snafu(backtrace)]
+        location: Location,
         source: query::error::Error,
     },
 
     #[snafu(display("Failed to parse query"))]
     ParseQuery {
-        #[snafu(backtrace)]
+        location: Location,
         source: query::error::Error,
     },
 
     #[snafu(display("Failed to execute logical plan"))]
     ExecLogicalPlan {
-        #[snafu(backtrace)]
+        location: Location,
         source: query::error::Error,
     },
 
     #[snafu(display("Failed to build DataFusion logical plan"))]
     BuildDfLogicalPlan {
-        source: datafusion_common::DataFusionError,
+        #[snafu(source)]
+        error: datafusion_common::DataFusionError,
         location: Location,
     },
 
     #[snafu(display("Failed to convert AlterExpr to AlterRequest"))]
     AlterExprToRequest {
-        #[snafu(backtrace)]
+        location: Location,
         source: common_grpc_expr::error::Error,
     },
 
     #[snafu(display("Failed to build table meta for table: {}", table_name))]
     BuildTableMeta {
         table_name: String,
-        source: table::metadata::TableMetaBuilderError,
+        #[snafu(source)]
+        error: table::metadata::TableMetaBuilderError,
         location: Location,
     },
 
@@ -222,25 +227,25 @@ pub enum Error {
 
     #[snafu(display("Failed to find new columns on insertion"))]
     FindNewColumnsOnInsertion {
-        #[snafu(backtrace)]
+        location: Location,
         source: common_grpc_expr::error::Error,
     },
 
     #[snafu(display("Failed to convert into vectors"))]
     IntoVectors {
-        #[snafu(backtrace)]
+        location: Location,
         source: datatypes::error::Error,
     },
 
     #[snafu(display("Failed to deserialize partition in meta to partition def"))]
     DeserializePartition {
-        #[snafu(backtrace)]
+        location: Location,
         source: partition::error::Error,
     },
 
     #[snafu(display("Failed to describe schema for given statement"))]
     DescribeStatement {
-        #[snafu(backtrace)]
+        location: Location,
         source: query::error::Error,
     },
 
@@ -249,7 +254,7 @@ pub enum Error {
 
     #[snafu(display("Unrecognized table option"))]
     UnrecognizedTableOption {
-        #[snafu(backtrace)]
+        location: Location,
         source: table::error::Error,
     },
 
@@ -262,26 +267,27 @@ pub enum Error {
     #[snafu(display("Failed to build regex"))]
     BuildRegex {
         location: Location,
-        source: regex::Error,
+        #[snafu(source)]
+        error: regex::Error,
     },
 
     #[snafu(display("Failed to copy table: {}", table_name))]
     CopyTable {
         table_name: String,
-        #[snafu(backtrace)]
+        location: Location,
         source: table::error::Error,
     },
 
     #[snafu(display("Failed to insert value into table: {}", table_name))]
     Insert {
         table_name: String,
-        #[snafu(backtrace)]
+        location: Location,
         source: table::error::Error,
     },
 
     #[snafu(display("Failed to parse data source url"))]
     ParseUrl {
-        #[snafu(backtrace)]
+        location: Location,
         source: common_datasource::error::Error,
     },
 
@@ -290,39 +296,40 @@ pub enum Error {
 
     #[snafu(display("Failed to parse file format"))]
     ParseFileFormat {
-        #[snafu(backtrace)]
+        location: Location,
         source: common_datasource::error::Error,
     },
 
     #[snafu(display("Failed to build data source backend"))]
     BuildBackend {
-        #[snafu(backtrace)]
+        location: Location,
         source: common_datasource::error::Error,
     },
 
     #[snafu(display("Failed to list objects"))]
     ListObjects {
-        #[snafu(backtrace)]
+        location: Location,
         source: common_datasource::error::Error,
     },
 
     #[snafu(display("Failed to infer schema from path: {}", path))]
     InferSchema {
         path: String,
-        #[snafu(backtrace)]
+        location: Location,
         source: common_datasource::error::Error,
     },
 
     #[snafu(display("Failed to build csv config"))]
     BuildCsvConfig {
-        source: common_datasource::file_format::csv::CsvConfigBuilderError,
+        #[snafu(source)]
+        error: common_datasource::file_format::csv::CsvConfigBuilderError,
         location: Location,
     },
 
     #[snafu(display("Failed to write stream to path: {}", path))]
     WriteStreamToFile {
         path: String,
-        #[snafu(backtrace)]
+        location: Location,
         source: common_datasource::error::Error,
     },
 
@@ -330,18 +337,21 @@ pub enum Error {
     ReadObject {
         path: String,
         location: Location,
-        source: object_store::Error,
+        #[snafu(source)]
+        error: object_store::Error,
     },
 
     #[snafu(display("Failed to read record batch"))]
     ReadDfRecordBatch {
-        source: datafusion::error::DataFusionError,
+        #[snafu(source)]
+        error: datafusion::error::DataFusionError,
         location: Location,
     },
 
     #[snafu(display("Failed to read parquet file"))]
     ReadParquet {
-        source: parquet::errors::ParquetError,
+        #[snafu(source)]
+        error: parquet::errors::ParquetError,
         location: Location,
     },
 
@@ -354,18 +364,20 @@ pub enum Error {
     #[snafu(display("Failed to build parquet record batch stream"))]
     BuildParquetRecordBatchStream {
         location: Location,
-        source: parquet::errors::ParquetError,
+        #[snafu(source)]
+        error: parquet::errors::ParquetError,
     },
 
     #[snafu(display("Failed to build file stream"))]
     BuildFileStream {
         location: Location,
-        source: datafusion::error::DataFusionError,
+        #[snafu(source)]
+        error: datafusion::error::DataFusionError,
     },
 
     #[snafu(display("Failed to write parquet file"))]
     WriteParquet {
-        #[snafu(backtrace)]
+        location: Location,
         source: storage::error::Error,
     },
 
@@ -384,19 +396,21 @@ pub enum Error {
 
     #[snafu(display("Failed to project schema"))]
     ProjectSchema {
-        source: ArrowError,
+        #[snafu(source)]
+        error: ArrowError,
         location: Location,
     },
 
     #[snafu(display("Failed to encode object into json"))]
     EncodeJson {
-        source: serde_json::error::Error,
+        #[snafu(source)]
+        error: serde_json::error::Error,
         location: Location,
     },
 
     #[snafu(display("Failed to prepare immutable table"))]
     PrepareImmutableTable {
-        #[snafu(backtrace)]
+        location: Location,
         source: query::error::Error,
     },
 
@@ -457,19 +471,19 @@ pub enum Error {
 
     #[snafu(display("Failed to prepare file table"))]
     PrepareFileTable {
-        #[snafu(backtrace)]
+        location: Location,
         source: query::error::Error,
     },
 
     #[snafu(display("Failed to infer file table schema"))]
     InferFileTableSchema {
-        #[snafu(backtrace)]
+        location: Location,
         source: query::error::Error,
     },
 
     #[snafu(display("The schema of the file table is incompatible with the table schema"))]
     SchemaIncompatible {
-        #[snafu(backtrace)]
+        location: Location,
         source: query::error::Error,
     },
 }
@@ -502,7 +516,7 @@ impl ErrorExt for Error {
 
             Error::TableMetadataManager { source, .. } => source.status_code(),
 
-            Error::ConvertSqlValue { source, .. } | Error::ParseSql { source } => {
+            Error::ConvertSqlValue { source, .. } | Error::ParseSql { source, .. } => {
                 source.status_code()
             }
 
@@ -512,18 +526,18 @@ impl ErrorExt for Error {
                 source.status_code()
             }
 
-            Error::Table { source }
+            Error::Table { source, .. }
             | Error::CopyTable { source, .. }
             | Error::Insert { source, .. } => source.status_code(),
 
             Error::ConvertColumnDefaultConstraint { source, .. }
-            | Error::CreateTableInfo { source }
-            | Error::IntoVectors { source } => source.status_code(),
+            | Error::CreateTableInfo { source, .. }
+            | Error::IntoVectors { source, .. } => source.status_code(),
 
-            Error::RequestInserts { source } => source.status_code(),
-            Error::RequestDeletes { source } => source.status_code(),
+            Error::RequestInserts { source, .. } => source.status_code(),
+            Error::RequestDeletes { source, .. } => source.status_code(),
 
-            Error::ColumnDataType { source } | Error::InvalidColumnDef { source, .. } => {
+            Error::ColumnDataType { source, .. } | Error::InvalidColumnDef { source, .. } => {
                 source.status_code()
             }
 
@@ -546,18 +560,18 @@ impl ErrorExt for Error {
 
             Error::Catalog { source, .. } => source.status_code(),
 
-            Error::BuildCreateExprOnInsertion { source }
-            | Error::FindNewColumnsOnInsertion { source } => source.status_code(),
+            Error::BuildCreateExprOnInsertion { source, .. }
+            | Error::FindNewColumnsOnInsertion { source, .. } => source.status_code(),
 
             Error::ExecuteStatement { source, .. }
-            | Error::PlanStatement { source }
-            | Error::ParseQuery { source }
-            | Error::ExecLogicalPlan { source }
-            | Error::DescribeStatement { source } => source.status_code(),
+            | Error::PlanStatement { source, .. }
+            | Error::ParseQuery { source, .. }
+            | Error::ExecLogicalPlan { source, .. }
+            | Error::DescribeStatement { source, .. } => source.status_code(),
 
             Error::AlterExprToRequest { source, .. } => source.status_code(),
 
-            Error::External { source } => source.status_code(),
+            Error::External { source, .. } => source.status_code(),
             Error::DeserializePartition { source, .. }
             | Error::FindTablePartitionRule { source, .. }
             | Error::SplitInsert { source, .. }
@@ -570,9 +584,9 @@ impl ErrorExt for Error {
                 StatusCode::StorageUnavailable
             }
 
-            Error::ListObjects { source }
-            | Error::ParseUrl { source }
-            | Error::BuildBackend { source } => source.status_code(),
+            Error::ListObjects { source, .. }
+            | Error::ParseUrl { source, .. }
+            | Error::BuildBackend { source, .. } => source.status_code(),
 
             Error::WriteParquet { source, .. } => source.status_code(),
             Error::ExecuteDdl { source, .. } => source.status_code(),
diff --git a/src/partition/Cargo.toml b/src/partition/Cargo.toml
index a36962289d85..e902f7ca6faf 100644
--- a/src/partition/Cargo.toml
+++ b/src/partition/Cargo.toml
@@ -11,6 +11,7 @@ api = { workspace = true }
 async-trait = "0.1"
 common-catalog = { workspace = true }
 common-error = { workspace = true }
+common-macro = { workspace = true }
 common-meta = { workspace = true }
 common-query = { workspace = true }
 common-telemetry = { workspace = true }
diff --git a/src/partition/src/error.rs b/src/partition/src/error.rs
index a8c14296e38c..44582d3270f6 100644
--- a/src/partition/src/error.rs
+++ b/src/partition/src/error.rs
@@ -16,14 +16,16 @@ use std::any::Any;
 
 use common_error::ext::ErrorExt;
 use common_error::status_code::StatusCode;
+use common_macro::stack_trace_debug;
 use common_query::prelude::Expr;
 use datafusion_common::ScalarValue;
 use snafu::{Location, Snafu};
 use store_api::storage::{RegionId, RegionNumber};
 use table::metadata::TableId;
 
-#[derive(Debug, Snafu)]
+#[derive(Snafu)]
 #[snafu(visibility(pub))]
+#[stack_trace_debug]
 pub enum Error {
     #[snafu(display("Table route manager error"))]
     TableRouteManager {
@@ -66,13 +68,15 @@ pub enum Error {
 
     #[snafu(display("Failed to serialize value to json"))]
     SerializeJson {
-        source: serde_json::Error,
+        #[snafu(source)]
+        error: serde_json::Error,
         location: Location,
     },
 
     #[snafu(display("Failed to deserialize value from json"))]
     DeserializeJson {
-        source: serde_json::Error,
+        #[snafu(source)]
+        error: serde_json::Error,
         location: Location,
     },
 
diff --git a/src/promql/src/error.rs b/src/promql/src/error.rs
index ead6948aa12b..01f48c7d2478 100644
--- a/src/promql/src/error.rs
+++ b/src/promql/src/error.rs
@@ -16,12 +16,14 @@ use std::any::Any;
 
 use common_error::ext::ErrorExt;
 use common_error::status_code::StatusCode;
+use common_macro::stack_trace_debug;
 use datafusion::error::DataFusionError;
 use promql_parser::parser::{Expr as PromExpr, TokenType};
 use snafu::{Location, Snafu};
 
-#[derive(Debug, Snafu)]
+#[derive(Snafu)]
 #[snafu(visibility(pub))]
+#[stack_trace_debug]
 pub enum Error {
     #[snafu(display("Unsupported expr type: {}", name))]
     UnsupportedExpr { name: String, location: Location },
@@ -34,7 +36,8 @@ pub enum Error {
 
     #[snafu(display("Internal error during building DataFusion plan"))]
     DataFusionPlanning {
-        source: datafusion::error::DataFusionError,
+        #[snafu(source)]
+        error: datafusion::error::DataFusionError,
         location: Location,
     },
 
@@ -73,7 +76,8 @@ pub enum Error {
 
     #[snafu(display("Failed to deserialize"))]
     Deserialize {
-        source: prost::DecodeError,
+        #[snafu(source)]
+        error: prost::DecodeError,
         location: Location,
     },
 
diff --git a/src/query/Cargo.toml b/src/query/Cargo.toml
index a3cfcae78937..b922708821bd 100644
--- a/src/query/Cargo.toml
+++ b/src/query/Cargo.toml
@@ -21,6 +21,7 @@ common-catalog.workspace = true
 common-datasource.workspace = true
 common-error.workspace = true
 common-function.workspace = true
+common-macro.workspace = true
 common-meta.workspace = true
 common-query.workspace = true
 common-recordbatch.workspace = true
diff --git a/src/query/src/datafusion/error.rs b/src/query/src/datafusion/error.rs
index 45d6d745b9e6..486172cf70e5 100644
--- a/src/query/src/datafusion/error.rs
+++ b/src/query/src/datafusion/error.rs
@@ -16,17 +16,20 @@ use std::any::Any;
 
 use common_error::ext::ErrorExt;
 use common_error::status_code::StatusCode;
+use common_macro::stack_trace_debug;
 use datafusion::error::DataFusionError;
 use snafu::{Location, Snafu};
 
 /// Inner error of datafusion based query engine.
-#[derive(Debug, Snafu)]
+#[derive(Snafu)]
 #[snafu(visibility(pub))]
+#[stack_trace_debug]
 pub enum InnerError {
     #[snafu(display("msg: {}", msg))]
     Datafusion {
         msg: &'static str,
-        source: DataFusionError,
+        #[snafu(source)]
+        error: DataFusionError,
         location: Location,
     },
 
diff --git a/src/query/src/error.rs b/src/query/src/error.rs
index 5bbdab347261..639bb3e78f5b 100644
--- a/src/query/src/error.rs
+++ b/src/query/src/error.rs
@@ -17,14 +17,16 @@ use std::time::Duration;
 
 use common_error::ext::{BoxedError, ErrorExt};
 use common_error::status_code::StatusCode;
+use common_macro::stack_trace_debug;
 use common_meta::table_name::TableName;
 use datafusion::error::DataFusionError;
 use datatypes::prelude::ConcreteDataType;
 use datatypes::value::Value;
 use snafu::{Location, Snafu};
 
-#[derive(Debug, Snafu)]
+#[derive(Snafu)]
 #[snafu(visibility(pub))]
+#[stack_trace_debug]
 pub enum Error {
     #[snafu(display("Unsupported expr type: {}", name))]
     UnsupportedExpr { name: String, location: Location },
@@ -106,20 +108,23 @@ pub enum Error {
     #[snafu(display("Failed to parse timestamp `{}`", raw))]
     ParseTimestamp {
         raw: String,
-        source: chrono::ParseError,
+        #[snafu(source)]
+        error: chrono::ParseError,
         location: Location,
     },
 
     #[snafu(display("Failed to parse float number `{}`", raw))]
     ParseFloat {
         raw: String,
-        source: std::num::ParseFloatError,
+        #[snafu(source)]
+        error: std::num::ParseFloatError,
         location: Location,
     },
 
     #[snafu(display("DataFusion error"))]
     DataFusion {
-        source: DataFusionError,
+        #[snafu(source)]
+        error: DataFusionError,
         location: Location,
     },
 
@@ -138,7 +143,8 @@ pub enum Error {
     #[snafu(display("Cannot plan SQL: {}", sql))]
     PlanSql {
         sql: String,
-        source: DataFusionError,
+        #[snafu(source)]
+        error: DataFusionError,
         location: Location,
     },
 
@@ -194,7 +200,8 @@ pub enum Error {
     #[snafu(display("Failed to regex"))]
     BuildRegex {
         location: Location,
-        source: regex::Error,
+        #[snafu(source)]
+        error: regex::Error,
     },
 
     #[snafu(display("Failed to build data source backend"))]
diff --git a/src/script/Cargo.toml b/src/script/Cargo.toml
index b1e0b6ba08cf..0d42ab96f9ad 100644
--- a/src/script/Cargo.toml
+++ b/src/script/Cargo.toml
@@ -32,6 +32,7 @@ catalog = { workspace = true }
 common-catalog = { workspace = true }
 common-error = { workspace = true }
 common-function = { workspace = true }
+common-macro = { workspace = true }
 common-query = { workspace = true }
 common-recordbatch = { workspace = true }
 common-runtime = { workspace = true }
diff --git a/src/script/src/error.rs b/src/script/src/error.rs
index 0a1e2e6d5d2c..40ac698b6a1a 100644
--- a/src/script/src/error.rs
+++ b/src/script/src/error.rs
@@ -16,10 +16,12 @@ use std::any::Any;
 
 use common_error::ext::{BoxedError, ErrorExt};
 use common_error::status_code::StatusCode;
+use common_macro::stack_trace_debug;
 use snafu::{Location, Snafu};
 
-#[derive(Debug, Snafu)]
+#[derive(Snafu)]
 #[snafu(visibility(pub))]
+#[stack_trace_debug]
 pub enum Error {
     #[snafu(display("Failed to find column in scripts table, name: {}", name))]
     FindColumnInScriptsTable { name: String, location: Location },
@@ -62,7 +64,8 @@ pub enum Error {
 
     #[snafu(display("Failed to build DataFusion logical plan"))]
     BuildDfLogicalPlan {
-        source: datafusion_common::DataFusionError,
+        #[snafu(source)]
+        error: datafusion_common::DataFusionError,
         location: Location,
     },
 
diff --git a/src/script/src/python/error.rs b/src/script/src/python/error.rs
index 096cde70dc80..ad00998f3a4c 100644
--- a/src/script/src/python/error.rs
+++ b/src/script/src/python/error.rs
@@ -14,6 +14,7 @@
 
 use common_error::ext::ErrorExt;
 use common_error::status_code::StatusCode;
+use common_macro::stack_trace_debug;
 use console::{style, Style};
 use datafusion::error::DataFusionError;
 use datatypes::arrow::error::ArrowError;
@@ -31,8 +32,9 @@ pub(crate) fn ret_other_error_with(reason: String) -> OtherSnafu<String> {
     OtherSnafu { reason }
 }
 
-#[derive(Debug, Snafu)]
+#[derive(Snafu)]
 #[snafu(visibility(pub(crate)))]
+#[stack_trace_debug]
 pub enum Error {
     #[snafu(display("Datatype error"))]
     TypeCast {
@@ -49,13 +51,15 @@ pub enum Error {
     #[snafu(display("Failed to parse script"))]
     PyParse {
         location: SnafuLocation,
-        source: ParseError,
+        #[snafu(source)]
+        error: ParseError,
     },
 
     #[snafu(display("Failed to compile script"))]
     PyCompile {
         location: SnafuLocation,
-        source: CodegenError,
+        #[snafu(source)]
+        error: CodegenError,
     },
 
     /// rustpython problem, using python virtual machines' backtrace instead
@@ -68,13 +72,15 @@ pub enum Error {
     #[snafu(display("Arrow error"))]
     Arrow {
         location: SnafuLocation,
-        source: ArrowError,
+        #[snafu(source)]
+        error: ArrowError,
     },
 
     #[snafu(display("DataFusion error"))]
     DataFusion {
         location: SnafuLocation,
-        source: DataFusionError,
+        #[snafu(source)]
+        error: DataFusionError,
     },
 
     /// errors in coprocessors' parse check for types and etc.
@@ -116,7 +122,10 @@ pub enum Error {
         source: common_recordbatch::error::Error,
     },
     #[snafu(display("Failed to create tokio task"))]
-    TokioJoin { source: tokio::task::JoinError },
+    TokioJoin {
+        #[snafu(source)]
+        error: tokio::task::JoinError,
+    },
 }
 
 impl ErrorExt for Error {
@@ -210,8 +219,8 @@ pub fn get_error_reason_loc(err: &Error) -> (String, Option<Location>) {
         Error::CoprParse { reason, loc, .. } => (reason.clone(), *loc),
         Error::Other { reason, .. } => (reason.clone(), None),
         Error::PyRuntime { msg, .. } => (msg.clone(), None),
-        Error::PyParse { source, .. } => (source.error.to_string(), Some(source.location)),
-        Error::PyCompile { source, .. } => (source.error.to_string(), Some(source.location)),
+        Error::PyParse { error, .. } => (error.error.to_string(), Some(error.location)),
+        Error::PyCompile { error, .. } => (error.error.to_string(), Some(error.location)),
         _ => (format!("Unknown error: {err:?}"), None),
     }
 }
diff --git a/src/script/src/python/rspython/test.rs b/src/script/src/python/rspython/test.rs
index 6cd2208b1ef0..a5ce5bd7b7b3 100644
--- a/src/script/src/python/rspython/test.rs
+++ b/src/script/src/python/rspython/test.rs
@@ -253,16 +253,12 @@ def calc_rvs(open_time, close):
     )
     .unwrap();
     let ret = exec_coprocessor(python_source, &Some(rb));
-    if let Err(Error::PyParse {
-        location: _,
-        source,
-    }) = ret
-    {
+    if let Err(Error::PyParse { location: _, error }) = ret {
         let res = visualize_loc(
             python_source,
-            &source.location,
+            &error.location,
             "unknown tokens",
-            source.error.to_string().as_str(),
+            error.error.to_string().as_str(),
             0,
             "copr.py",
         );
@@ -303,16 +299,12 @@ def a(cpu, mem):
     )
     .unwrap();
     let ret = exec_coprocessor(python_source, &Some(rb));
-    if let Err(Error::PyParse {
-        location: _,
-        source,
-    }) = ret
-    {
+    if let Err(Error::PyParse { location: _, error }) = ret {
         let res = visualize_loc(
             python_source,
-            &source.location,
+            &error.location,
             "unknown tokens",
-            source.error.to_string().as_str(),
+            error.error.to_string().as_str(),
             0,
             "copr.py",
         );
diff --git a/src/servers/Cargo.toml b/src/servers/Cargo.toml
index 176ded04da08..962375a2a255 100644
--- a/src/servers/Cargo.toml
+++ b/src/servers/Cargo.toml
@@ -27,6 +27,7 @@ common-catalog = { workspace = true }
 common-error = { workspace = true }
 common-grpc = { workspace = true }
 common-grpc-expr = { workspace = true }
+common-macro = { workspace = true }
 common-mem-prof = { workspace = true, optional = true }
 common-meta = { workspace = true }
 common-query = { workspace = true }
diff --git a/src/servers/src/error.rs b/src/servers/src/error.rs
index 474d5a1d9131..c1a34177e7eb 100644
--- a/src/servers/src/error.rs
+++ b/src/servers/src/error.rs
@@ -22,6 +22,7 @@ use base64::DecodeError;
 use catalog;
 use common_error::ext::{BoxedError, ErrorExt};
 use common_error::status_code::StatusCode;
+use common_macro::stack_trace_debug;
 use common_telemetry::logging;
 use datatypes::prelude::ConcreteDataType;
 use query::parser::PromQuery;
@@ -29,19 +30,24 @@ use serde_json::json;
 use snafu::{ErrorCompat, Location, Snafu};
 use tonic::Code;
 
-#[derive(Debug, Snafu)]
+#[derive(Snafu)]
 #[snafu(visibility(pub))]
+#[stack_trace_debug]
 pub enum Error {
     #[snafu(display("Internal error: {}", err_msg))]
     Internal { err_msg: String },
 
     #[snafu(display("Internal IO error"))]
-    InternalIo { source: std::io::Error },
+    InternalIo {
+        #[snafu(source)]
+        error: std::io::Error,
+    },
 
     #[snafu(display("Tokio IO error: {}", err_msg))]
     TokioIo {
         err_msg: String,
-        source: std::io::Error,
+        #[snafu(source)]
+        error: std::io::Error,
     },
 
     #[snafu(display("Failed to collect recordbatch"))]
@@ -51,10 +57,16 @@ pub enum Error {
     },
 
     #[snafu(display("Failed to start HTTP server"))]
-    StartHttp { source: hyper::Error },
+    StartHttp {
+        #[snafu(source)]
+        error: hyper::Error,
+    },
 
     #[snafu(display("Failed to start gRPC server"))]
-    StartGrpc { source: tonic::transport::Error },
+    StartGrpc {
+        #[snafu(source)]
+        error: tonic::transport::Error,
+    },
 
     #[snafu(display("{} server is already started", server))]
     AlreadyStarted { server: String, location: Location },
@@ -62,7 +74,8 @@ pub enum Error {
     #[snafu(display("Failed to bind address {}", addr))]
     TcpBind {
         addr: SocketAddr,
-        source: std::io::Error,
+        #[snafu(source)]
+        error: std::io::Error,
     },
 
     #[snafu(display("Failed to execute query, query: {}", query))]
@@ -125,7 +138,8 @@ pub enum Error {
     #[snafu(display("Failed to parse InfluxDB line protocol"))]
     InfluxdbLineProtocol {
         location: Location,
-        source: influxdb_line_protocol::Error,
+        #[snafu(source)]
+        error: influxdb_line_protocol::Error,
     },
 
     #[snafu(display("Failed to write InfluxDB line protocol"))]
@@ -153,36 +167,44 @@ pub enum Error {
     ConnResetByPeer { location: Location },
 
     #[snafu(display("Hyper error"))]
-    Hyper { source: hyper::Error },
+    Hyper {
+        #[snafu(source)]
+        error: hyper::Error,
+    },
 
     #[snafu(display("Invalid OpenTSDB line"))]
     InvalidOpentsdbLine {
-        source: FromUtf8Error,
+        #[snafu(source)]
+        error: FromUtf8Error,
         location: Location,
     },
 
     #[snafu(display("Invalid OpenTSDB Json request"))]
     InvalidOpentsdbJsonRequest {
-        source: serde_json::error::Error,
+        #[snafu(source)]
+        error: serde_json::error::Error,
         location: Location,
     },
 
     #[snafu(display("Failed to decode prometheus remote request"))]
     DecodePromRemoteRequest {
         location: Location,
-        source: prost::DecodeError,
+        #[snafu(source)]
+        error: prost::DecodeError,
     },
 
     #[snafu(display("Failed to decode OTLP request"))]
     DecodeOtlpRequest {
         location: Location,
-        source: prost::DecodeError,
+        #[snafu(source)]
+        error: prost::DecodeError,
     },
 
     #[snafu(display("Failed to decompress prometheus remote request"))]
     DecompressPromRemoteRequest {
         location: Location,
-        source: snap::Error,
+        #[snafu(source)]
+        error: snap::Error,
     },
 
     #[snafu(display("Invalid prometheus remote request, msg: {}", msg))]
@@ -193,7 +215,8 @@ pub enum Error {
 
     #[snafu(display("Invalid Flight ticket"))]
     InvalidFlightTicket {
-        source: api::DecodeError,
+        #[snafu(source)]
+        error: api::DecodeError,
         location: Location,
     },
 
@@ -214,7 +237,8 @@ pub enum Error {
 
     #[snafu(display("Invalid visibility ASCII chars"))]
     InvisibleASCII {
-        source: hyper::header::ToStrError,
+        #[snafu(source)]
+        error: hyper::header::ToStrError,
         location: Location,
     },
 
@@ -226,13 +250,15 @@ pub enum Error {
 
     #[snafu(display("Invalid base64 value"))]
     InvalidBase64Value {
-        source: DecodeError,
+        #[snafu(source)]
+        error: DecodeError,
         location: Location,
     },
 
     #[snafu(display("Invalid utf-8 value"))]
     InvalidUtf8Value {
-        source: FromUtf8Error,
+        #[snafu(source)]
+        error: FromUtf8Error,
         location: Location,
     },
 
@@ -246,7 +272,8 @@ pub enum Error {
     #[snafu(display("Failed to dump profile data"))]
     DumpProfileData {
         location: Location,
-        source: common_mem_prof::error::Error,
+        #[snafu(source)]
+        error: common_mem_prof::error::Error,
     },
 
     #[snafu(display("Invalid prepare statement: {}", err_msg))]
@@ -257,13 +284,15 @@ pub enum Error {
 
     #[snafu(display("Failed to build gRPC reflection service"))]
     GrpcReflectionService {
-        source: tonic_reflection::server::Error,
+        #[snafu(source)]
+        error: tonic_reflection::server::Error,
         location: Location,
     },
 
     #[snafu(display("Failed to build HTTP response"))]
     BuildHttpResponse {
-        source: http::Error,
+        #[snafu(source)]
+        error: http::Error,
         location: Location,
     },
 
@@ -293,7 +322,8 @@ pub enum Error {
 
     #[snafu(display("Failed to join task"))]
     JoinTask {
-        source: tokio::task::JoinError,
+        #[snafu(source)]
+        error: tokio::task::JoinError,
         location: Location,
     },
 
@@ -306,7 +336,8 @@ pub enum Error {
 
     #[snafu(display("DataFrame operation error"))]
     DataFrame {
-        source: datafusion::error::DataFusionError,
+        #[snafu(source)]
+        error: datafusion::error::DataFusionError,
         location: Location,
     },
 
@@ -418,10 +449,10 @@ impl ErrorExt for Error {
 
             UnexpectedResult { .. } => StatusCode::Unexpected,
 
-            JoinTask { source, .. } => {
-                if source.is_cancelled() {
+            JoinTask { error, .. } => {
+                if error.is_cancelled() {
                     StatusCode::Cancelled
-                } else if source.is_panic() {
+                } else if error.is_panic() {
                     StatusCode::Unexpected
                 } else {
                     StatusCode::Unknown
@@ -511,7 +542,7 @@ define_into_tonic_status!(Error);
 
 impl From<std::io::Error> for Error {
     fn from(e: std::io::Error) -> Self {
-        Error::InternalIo { source: e }
+        Error::InternalIo { error: e }
     }
 }
 
diff --git a/src/servers/src/metrics/jemalloc/error.rs b/src/servers/src/metrics/jemalloc/error.rs
index 4bbd463a360f..14c125ce6f86 100644
--- a/src/servers/src/metrics/jemalloc/error.rs
+++ b/src/servers/src/metrics/jemalloc/error.rs
@@ -16,14 +16,17 @@ use std::any::Any;
 
 use common_error::ext::{BoxedError, ErrorExt};
 use common_error::status_code::StatusCode;
+use common_macro::stack_trace_debug;
 use snafu::{Location, Snafu};
 
-#[derive(Debug, Snafu)]
+#[derive(Snafu)]
 #[snafu(visibility(pub))]
+#[stack_trace_debug]
 pub enum Error {
     #[snafu(display("Failed to update jemalloc metrics"))]
     UpdateJemallocMetrics {
-        source: tikv_jemalloc_ctl::Error,
+        #[snafu(source)]
+        error: tikv_jemalloc_ctl::Error,
         location: Location,
     },
 }
diff --git a/src/sql/Cargo.toml b/src/sql/Cargo.toml
index 3e17e1fbe36f..8472214db730 100644
--- a/src/sql/Cargo.toml
+++ b/src/sql/Cargo.toml
@@ -9,6 +9,7 @@ api = { workspace = true }
 common-base = { workspace = true }
 common-catalog = { workspace = true }
 common-error = { workspace = true }
+common-macro = { workspace = true }
 common-query = { workspace = true }
 common-time = { workspace = true }
 datafusion-sql.workspace = true
diff --git a/src/sql/src/error.rs b/src/sql/src/error.rs
index 157924c3571e..9fa62dc9075d 100644
--- a/src/sql/src/error.rs
+++ b/src/sql/src/error.rs
@@ -16,6 +16,7 @@ use std::any::Any;
 
 use common_error::ext::ErrorExt;
 use common_error::status_code::StatusCode;
+use common_macro::stack_trace_debug;
 use common_time::timestamp::TimeUnit;
 use common_time::Timestamp;
 use datatypes::prelude::{ConcreteDataType, Value};
@@ -29,8 +30,9 @@ pub type Result<T> = std::result::Result<T, Error>;
 /// SQL parser errors.
 // Now the error in parser does not contains backtrace to avoid generating backtrace
 // every time the parser parses an invalid SQL.
-#[derive(Debug, Snafu)]
+#[derive(Snafu)]
 #[snafu(visibility(pub))]
+#[stack_trace_debug]
 pub enum Error {
     #[snafu(display("SQL statement is not supported: {}, keyword: {}", sql, keyword))]
     Unsupported { sql: String, keyword: String },
@@ -45,7 +47,8 @@ pub enum Error {
         sql: String,
         expected: String,
         actual: String,
-        source: ParserError,
+        #[snafu(source)]
+        error: ParserError,
     },
 
     #[snafu(display(
@@ -57,7 +60,11 @@ pub enum Error {
 
     // Syntax error from sql parser.
     #[snafu(display("Syntax error, sql: {}", sql))]
-    Syntax { sql: String, source: ParserError },
+    Syntax {
+        sql: String,
+        #[snafu(source)]
+        error: ParserError,
+    },
 
     #[snafu(display("Missing time index constraint"))]
     MissingTimeIndex {},
diff --git a/src/sql/src/statements.rs b/src/sql/src/statements.rs
index 3bb715f41fbf..743f6496c8f7 100644
--- a/src/sql/src/statements.rs
+++ b/src/sql/src/statements.rs
@@ -602,7 +602,7 @@ mod tests {
         assert!(v.is_err());
         assert!(
             format!("{v:?}").contains(
-                "column_name: \"a\", expect: Float64(Float64Type), actual: Boolean(BooleanType)"
+                "Column a expect type: Float64(Float64Type), actual: Boolean(BooleanType))"
             ),
             "v is {v:?}",
         );
diff --git a/src/storage/Cargo.toml b/src/storage/Cargo.toml
index a2b9a0cac6df..5b097c30ac6d 100644
--- a/src/storage/Cargo.toml
+++ b/src/storage/Cargo.toml
@@ -16,6 +16,7 @@ bytes = "1.1"
 common-base = { workspace = true }
 common-datasource = { workspace = true }
 common-error = { workspace = true }
+common-macro = { workspace = true }
 common-query = { workspace = true }
 common-recordbatch = { workspace = true }
 common-runtime = { workspace = true }
diff --git a/src/storage/src/error.rs b/src/storage/src/error.rs
index 7ad09595a211..c2a043e6d07a 100644
--- a/src/storage/src/error.rs
+++ b/src/storage/src/error.rs
@@ -19,6 +19,7 @@ use std::str::Utf8Error;
 use common_datasource::compression::CompressionType;
 use common_error::ext::{BoxedError, ErrorExt};
 use common_error::status_code::StatusCode;
+use common_macro::stack_trace_debug;
 use common_runtime::error::Error as RuntimeError;
 use datatypes::arrow::error::ArrowError;
 use datatypes::prelude::ConcreteDataType;
@@ -33,8 +34,9 @@ use tokio::task::JoinError;
 use crate::metadata::Error as MetadataError;
 use crate::write_batch;
 
-#[derive(Debug, Snafu)]
+#[derive(Snafu)]
 #[snafu(visibility(pub))]
+#[stack_trace_debug]
 pub enum Error {
     #[snafu(display("Invalid region descriptor, region: {}", region))]
     InvalidRegionDesc {
@@ -48,7 +50,8 @@ pub enum Error {
 
     #[snafu(display("Failed to write parquet file"))]
     WriteParquet {
-        source: parquet::errors::ParquetError,
+        #[snafu(source)]
+        error: parquet::errors::ParquetError,
         location: Location,
     },
 
@@ -61,67 +64,77 @@ pub enum Error {
     #[snafu(display("Failed to create RecordBatch from vectors"))]
     NewRecordBatch {
         location: Location,
-        source: ArrowError,
+        #[snafu(source)]
+        error: ArrowError,
     },
 
     #[snafu(display("Fail to read object from path: {}", path))]
     ReadObject {
         path: String,
         location: Location,
-        source: object_store::Error,
+        #[snafu(source)]
+        error: object_store::Error,
     },
 
     #[snafu(display("Fail to write object into path: {}", path))]
     WriteObject {
         path: String,
         location: Location,
-        source: object_store::Error,
+        #[snafu(source)]
+        error: object_store::Error,
     },
 
     #[snafu(display("Fail to delete object from path: {}", path))]
     DeleteObject {
         path: String,
         location: Location,
-        source: object_store::Error,
+        #[snafu(source)]
+        error: object_store::Error,
     },
 
     #[snafu(display("Fail to compress object by {}, path: {}", compress_type, path))]
     CompressObject {
         compress_type: CompressionType,
         path: String,
-        source: std::io::Error,
+        #[snafu(source)]
+        error: std::io::Error,
     },
 
     #[snafu(display("Fail to decompress object by {}, path: {}", compress_type, path))]
     DecompressObject {
         compress_type: CompressionType,
         path: String,
-        source: std::io::Error,
+        #[snafu(source)]
+        error: std::io::Error,
     },
 
     #[snafu(display("Fail to list objects in path: {}", path))]
     ListObjects {
         path: String,
         location: Location,
-        source: object_store::Error,
+        #[snafu(source)]
+        error: object_store::Error,
     },
 
     #[snafu(display("Fail to create str from bytes"))]
     Utf8 {
         location: Location,
-        source: Utf8Error,
+        #[snafu(source)]
+        error: Utf8Error,
     },
 
     #[snafu(display("Fail to encode object into json "))]
     EncodeJson {
         location: Location,
-        source: JsonError,
+        #[snafu(source)]
+        error: JsonError,
     },
 
     #[snafu(display("Fail to decode object from json "))]
     DecodeJson {
         location: Location,
-        source: JsonError,
+        #[snafu(source)]
+        error: JsonError,
     },
 
     #[snafu(display("Invalid scan index, start: {}, end: {}", start, end))]
@@ -141,19 +154,22 @@ pub enum Error {
     #[snafu(display("Failed to encode WAL header"))]
     EncodeWalHeader {
         location: Location,
-        source: std::io::Error,
+        #[snafu(source)]
+        error: std::io::Error,
     },
 
     #[snafu(display("Failed to decode WAL header"))]
     DecodeWalHeader {
         location: Location,
-        source: std::io::Error,
+        #[snafu(source)]
+        error: std::io::Error,
     },
 
     #[snafu(display("Failed to wait flushing, region_id: {}", region_id))]
     WaitFlush {
         region_id: RegionId,
-        source: tokio::sync::oneshot::error::RecvError,
+        #[snafu(source)]
+        error: tokio::sync::oneshot::error::RecvError,
         location: Location,
     },
 
@@ -183,12 +199,16 @@ pub enum Error {
     DecodeMetaActionList { msg: String, location: Location },
 
     #[snafu(display("Failed to read line, err"))]
-    Readline { source: IoError },
+    Readline {
+        #[snafu(source)]
+        error: IoError,
+    },
 
     #[snafu(display("Failed to read Parquet file: {}", file))]
     ReadParquet {
         file: String,
-        source: parquet::errors::ParquetError,
+        #[snafu(source)]
+        error: parquet::errors::ParquetError,
         location: Location,
     },
 
@@ -385,13 +405,15 @@ pub enum Error {
     #[snafu(display("Failed to decode arrow data"))]
     DecodeArrow {
         location: Location,
-        source: ArrowError,
+        #[snafu(source)]
+        error: ArrowError,
     },
 
     #[snafu(display("Failed to encode arrow data"))]
     EncodeArrow {
         location: Location,
-        source: ArrowError,
+        #[snafu(source)]
+        error: ArrowError,
     },
 
     #[snafu(display("Failed to parse schema"))]
@@ -426,13 +448,15 @@ pub enum Error {
 
     #[snafu(display("Failed to stop scheduler"))]
     StopScheduler {
-        source: JoinError,
+        #[snafu(source)]
+        error: JoinError,
         location: Location,
     },
 
     #[snafu(display("Failed to delete SST file"))]
     DeleteSst {
-        source: object_store::Error,
+        #[snafu(source)]
+        error: object_store::Error,
         location: Location,
     },
 
@@ -448,7 +472,8 @@ pub enum Error {
     #[snafu(display("The compaction task is cancelled, region_id: {}", region_id))]
     CompactTaskCancel {
         region_id: RegionId,
-        source: tokio::sync::oneshot::error::RecvError,
+        #[snafu(source)]
+        error: tokio::sync::oneshot::error::RecvError,
     },
 
     #[snafu(display(
@@ -476,13 +501,15 @@ pub enum Error {
 
     #[snafu(display("Failed to convert columns to rows"))]
     ConvertColumnsToRows {
-        source: ArrowError,
+        #[snafu(source)]
+        error: ArrowError,
         location: Location,
     },
 
     #[snafu(display("Failed to sort arrays"))]
     SortArrays {
-        source: ArrowError,
+        #[snafu(source)]
+        error: ArrowError,
         location: Location,
     },
 
@@ -494,7 +521,8 @@ pub enum Error {
 
     #[snafu(display("Failed to join spawned tasks"))]
     JoinError {
-        source: JoinError,
+        #[snafu(source)]
+        error: JoinError,
         location: Location,
     },
 }
@@ -505,8 +533,8 @@ impl Error {
     /// Returns true if the error is the object path to delete
     /// doesn't exist.
     pub(crate) fn is_object_to_delete_not_found(&self) -> bool {
-        if let Error::DeleteObject { source, .. } = self {
-            source.kind() == ErrorKind::NotFound
+        if let Error::DeleteObject { error, .. } = self {
+            error.kind() == ErrorKind::NotFound
         } else {
             false
         }
diff --git a/src/storage/src/metadata.rs b/src/storage/src/metadata.rs
index 5b3d8fbf9909..2351674125c3 100644
--- a/src/storage/src/metadata.rs
+++ b/src/storage/src/metadata.rs
@@ -19,6 +19,7 @@ use std::sync::Arc;
 
 use common_error::ext::ErrorExt;
 use common_error::status_code::StatusCode;
+use common_macro::stack_trace_debug;
 use datatypes::data_type::ConcreteDataType;
 use datatypes::schema::{ColumnSchema, Metadata, COMMENT_KEY};
 use serde::{Deserialize, Serialize};
@@ -35,8 +36,9 @@ use crate::manifest::action::{RawColumnFamiliesMetadata, RawColumnsMetadata, Raw
 use crate::schema::{RegionSchema, RegionSchemaRef};
 
 /// Error for handling metadata.
-#[derive(Debug, Snafu)]
+#[derive(Snafu)]
 #[snafu(visibility(pub(crate)))]
+#[stack_trace_debug]
 pub enum Error {
     #[snafu(display("Column name {} already exists", name))]
     ColNameExists { name: String, location: Location },
@@ -95,7 +97,8 @@ pub enum Error {
     ParseMetaInt {
         // Store key and value in one string to reduce the enum size.
         key_value: String,
-        source: std::num::ParseIntError,
+        #[snafu(source)]
+        error: std::num::ParseIntError,
         location: Location,
     },
 
@@ -104,7 +107,8 @@ pub enum Error {
 
     #[snafu(display("Failed to build column descriptor"))]
     BuildColumnDescriptor {
-        source: ColumnDescriptorBuilderError,
+        #[snafu(source)]
+        error: ColumnDescriptorBuilderError,
         location: Location,
     },
 
diff --git a/src/store-api/Cargo.toml b/src/store-api/Cargo.toml
index b0710a76e8da..3139f276622c 100644
--- a/src/store-api/Cargo.toml
+++ b/src/store-api/Cargo.toml
@@ -11,6 +11,7 @@ async-trait.workspace = true
 bytes = "1.1"
 common-base = { workspace = true }
 common-error = { workspace = true }
+common-macro = { workspace = true }
 common-query = { workspace = true }
 common-recordbatch = { workspace = true }
 common-time = { workspace = true }
diff --git a/src/store-api/src/error.rs b/src/store-api/src/error.rs
index 9c2855a0a51e..6139232dd57c 100644
--- a/src/store-api/src/error.rs
+++ b/src/store-api/src/error.rs
@@ -14,12 +14,14 @@
 
 use common_error::ext::ErrorExt;
 use common_error::status_code::StatusCode;
+use common_macro::stack_trace_debug;
 use snafu::{Location, Snafu};
 
 use crate::storage::ColumnDescriptorBuilderError;
 
-#[derive(Debug, Snafu)]
+#[derive(Snafu)]
 #[snafu(visibility(pub))]
+#[stack_trace_debug]
 pub enum Error {
     #[snafu(display("Invalid raw region request: {err}"))]
     InvalidRawRegionRequest { err: String, location: Location },
@@ -33,7 +35,8 @@ pub enum Error {
 
     #[snafu(display("Failed to build column descriptor: "))]
     BuildColumnDescriptor {
-        source: ColumnDescriptorBuilderError,
+        #[snafu(source)]
+        error: ColumnDescriptorBuilderError,
         location: Location,
     },
 }
diff --git a/src/store-api/src/logstore/entry_stream.rs b/src/store-api/src/logstore/entry_stream.rs
index 9e7e5f0cf4ed..89a8bd192e42 100644
--- a/src/store-api/src/logstore/entry_stream.rs
+++ b/src/store-api/src/logstore/entry_stream.rs
@@ -33,6 +33,7 @@ mod tests {
     use std::any::Any;
     use std::task::{Context, Poll};
 
+    use common_error::ext::StackError;
     use futures::StreamExt;
     use snafu::Snafu;
 
@@ -63,6 +64,14 @@ mod tests {
         }
     }
 
+    impl StackError for Error {
+        fn debug_fmt(&self, _: usize, _: &mut Vec<String>) {}
+
+        fn next(&self) -> Option<&dyn StackError> {
+            None
+        }
+    }
+
     impl Entry for SimpleEntry {
         type Error = Error;
         type Namespace = Namespace;
diff --git a/src/store-api/src/metadata.rs b/src/store-api/src/metadata.rs
index 123ce83c7663..f51284e6994d 100644
--- a/src/store-api/src/metadata.rs
+++ b/src/store-api/src/metadata.rs
@@ -25,6 +25,7 @@ use api::v1::region::RegionColumnDef;
 use api::v1::SemanticType;
 use common_error::ext::ErrorExt;
 use common_error::status_code::StatusCode;
+use common_macro::stack_trace_debug;
 use datatypes::arrow::datatypes::FieldRef;
 use datatypes::schema::{ColumnDefaultConstraint, ColumnSchema, Schema, SchemaRef};
 use serde::de::Error;
@@ -558,8 +559,9 @@ impl SkippedFields {
     }
 }
 
-#[derive(Debug, Snafu)]
+#[derive(Snafu)]
 #[snafu(visibility(pub))]
+#[stack_trace_debug]
 pub enum MetadataError {
     #[snafu(display("Invalid schema"))]
     InvalidSchema {
@@ -573,7 +575,8 @@ pub enum MetadataError {
     #[snafu(display("Failed to ser/de json object"))]
     SerdeJson {
         location: Location,
-        source: serde_json::Error,
+        #[snafu(source)]
+        error: serde_json::Error,
     },
 
     #[snafu(display("Failed to convert struct from datatypes"))]
diff --git a/src/table/Cargo.toml b/src/table/Cargo.toml
index 245ef06e25dd..5d04d17ed99c 100644
--- a/src/table/Cargo.toml
+++ b/src/table/Cargo.toml
@@ -15,6 +15,7 @@ common-base = { workspace = true }
 common-catalog = { workspace = true }
 common-datasource = { workspace = true }
 common-error = { workspace = true }
+common-macro = { workspace = true }
 common-procedure = { workspace = true }
 common-query = { workspace = true }
 common-recordbatch = { workspace = true }
diff --git a/src/table/src/error.rs b/src/table/src/error.rs
index ab44d02bbb93..6146309d8ddf 100644
--- a/src/table/src/error.rs
+++ b/src/table/src/error.rs
@@ -16,6 +16,7 @@ use std::any::Any;
 
 use common_error::ext::{BoxedError, ErrorExt};
 use common_error::status_code::StatusCode;
+use common_macro::stack_trace_debug;
 use datafusion::error::DataFusionError;
 use datatypes::arrow::error::ArrowError;
 use snafu::{Location, Snafu};
@@ -25,12 +26,14 @@ use crate::metadata::TableId;
 pub type Result<T> = std::result::Result<T, Error>;
 
 /// Default error implementation of table.
-#[derive(Debug, Snafu)]
+#[derive(Snafu)]
 #[snafu(visibility(pub))]
+#[stack_trace_debug]
 pub enum Error {
     #[snafu(display("Datafusion error"))]
     Datafusion {
-        source: DataFusionError,
+        #[snafu(source)]
+        error: DataFusionError,
         location: Location,
     },
 
@@ -48,7 +51,8 @@ pub enum Error {
 
     #[snafu(display("Table projection error"))]
     TableProjection {
-        source: ArrowError,
+        #[snafu(source)]
+        error: ArrowError,
         location: Location,
     },
 
@@ -99,7 +103,8 @@ pub enum Error {
         column_name,
     ))]
     BuildColumnDescriptor {
-        source: store_api::storage::ColumnDescriptorBuilderError,
+        #[snafu(source)]
+        error: store_api::storage::ColumnDescriptorBuilderError,
         table_name: String,
         column_name: String,
         location: Location,

From 090e100ae476532afb7d501fd749c4cdb50bcd33 Mon Sep 17 00:00:00 2001
From: Yingwen <realevenyag@gmail.com>
Date: Tue, 26 Sep 2023 19:46:19 +0800
Subject: [PATCH 28/36] feat(mito): Add cache manager (#2488)

* feat: add cache manager

* feat: add cache to reader builder

* feat: add AsyncFileReaderCache

* feat: Impl AsyncFileReaderCache

* chore: move moka dep to workspace

* feat: add moka cache to the manager

* feat: implement parquet meta cache

* test: test cache manager

* feat: consider vec size

* style: fix clippy

* test: fix config api test

* feat: divide cache

* test: test disabling meta cache

* test: fix config api test

* feat: remove meta cache if file is purged
---
 Cargo.lock                            |  35 +------
 Cargo.toml                            |   1 +
 src/catalog/Cargo.toml                |   2 +-
 src/client/Cargo.toml                 |   2 +-
 src/frontend/Cargo.toml               |   2 +-
 src/mito2/Cargo.toml                  |   1 +
 src/mito2/src/cache.rs                | 131 ++++++++++++++++++++++++
 src/mito2/src/cache/cache_size.rs     | 142 ++++++++++++++++++++++++++
 src/mito2/src/cache/test_util.rs      |  44 ++++++++
 src/mito2/src/config.rs               |   5 +
 src/mito2/src/engine.rs               |   9 +-
 src/mito2/src/lib.rs                  |   1 +
 src/mito2/src/read/scan_region.rs     |   8 +-
 src/mito2/src/read/seq_scan.rs        |  19 +++-
 src/mito2/src/region/opener.rs        |  16 ++-
 src/mito2/src/sst/file.rs             |   6 ++
 src/mito2/src/sst/file_purger.rs      |  19 +++-
 src/mito2/src/sst/parquet/reader.rs   |  88 +++++++++++++++-
 src/mito2/src/worker.rs               |  32 +++++-
 src/mito2/src/worker/handle_create.rs |   1 +
 src/mito2/src/worker/handle_open.rs   |   1 +
 src/partition/Cargo.toml              |   2 +-
 tests-integration/tests/http.rs       |   1 +
 23 files changed, 516 insertions(+), 52 deletions(-)
 create mode 100644 src/mito2/src/cache.rs
 create mode 100644 src/mito2/src/cache/cache_size.rs
 create mode 100644 src/mito2/src/cache/test_util.rs

diff --git a/Cargo.lock b/Cargo.lock
index 463bb474d98d..e2ef70d9bb96 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1269,7 +1269,7 @@ dependencies = [
  "log-store",
  "meta-client",
  "metrics",
- "moka 0.11.3",
+ "moka",
  "object-store",
  "parking_lot 0.12.1",
  "partition",
@@ -1548,7 +1548,7 @@ dependencies = [
  "derive_builder 0.12.0",
  "enum_dispatch",
  "futures-util",
- "moka 0.9.9",
+ "moka",
  "parking_lot 0.12.1",
  "prost",
  "rand",
@@ -3369,7 +3369,7 @@ dependencies = [
  "meta-client",
  "meta-srv",
  "metrics",
- "moka 0.9.9",
+ "moka",
  "object-store",
  "openmetrics-parser",
  "opentelemetry-proto",
@@ -5577,6 +5577,7 @@ dependencies = [
  "log-store",
  "memcomparable",
  "metrics",
+ "moka",
  "object-store",
  "parquet",
  "paste",
@@ -5595,32 +5596,6 @@ dependencies = [
  "uuid",
 ]
 
-[[package]]
-name = "moka"
-version = "0.9.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b28455ac4363046076054a7e9cfbd7f168019c29dba32a625f59fc0aeffaaea4"
-dependencies = [
- "async-io",
- "async-lock",
- "crossbeam-channel",
- "crossbeam-epoch",
- "crossbeam-utils",
- "futures-util",
- "num_cpus",
- "once_cell",
- "parking_lot 0.12.1",
- "quanta 0.11.1",
- "rustc_version",
- "scheduled-thread-pool",
- "skeptic",
- "smallvec",
- "tagptr",
- "thiserror",
- "triomphe",
- "uuid",
-]
-
 [[package]]
 name = "moka"
 version = "0.11.3"
@@ -6539,7 +6514,7 @@ dependencies = [
  "datafusion-expr",
  "datatypes",
  "meta-client",
- "moka 0.9.9",
+ "moka",
  "serde",
  "serde_json",
  "snafu",
diff --git a/Cargo.toml b/Cargo.toml
index a89c9f19d3f2..e865c2b7ece6 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -82,6 +82,7 @@ greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", r
 humantime-serde = "1.1"
 itertools = "0.10"
 lazy_static = "1.4"
+moka = { version = "0.11" }
 once_cell = "1.18"
 opentelemetry-proto = { version = "0.2", features = ["gen-tonic", "metrics"] }
 parquet = "43.0"
diff --git a/src/catalog/Cargo.toml b/src/catalog/Cargo.toml
index 314a332672ea..7e5f36a77aa9 100644
--- a/src/catalog/Cargo.toml
+++ b/src/catalog/Cargo.toml
@@ -31,7 +31,7 @@ futures-util.workspace = true
 lazy_static.workspace = true
 meta-client = { workspace = true }
 metrics.workspace = true
-moka = { version = "0.11", features = ["future"] }
+moka = { workspace = true, features = ["future"] }
 parking_lot = "0.12"
 partition.workspace = true
 regex.workspace = true
diff --git a/src/client/Cargo.toml b/src/client/Cargo.toml
index 72942cd3a3d9..0323a6c4e597 100644
--- a/src/client/Cargo.toml
+++ b/src/client/Cargo.toml
@@ -27,7 +27,7 @@ datatypes = { workspace = true }
 derive_builder.workspace = true
 enum_dispatch = "0.3"
 futures-util.workspace = true
-moka = { version = "0.9", features = ["future"] }
+moka = { workspace = true, features = ["future"] }
 parking_lot = "0.12"
 prost.workspace = true
 rand.workspace = true
diff --git a/src/frontend/Cargo.toml b/src/frontend/Cargo.toml
index b85a18008eca..65511d7a8a99 100644
--- a/src/frontend/Cargo.toml
+++ b/src/frontend/Cargo.toml
@@ -51,7 +51,7 @@ meta-client = { workspace = true }
 raft-engine = { workspace = true }
 # Although it is not used, please do not delete it.
 metrics.workspace = true
-moka = { version = "0.9", features = ["future"] }
+moka = { workspace = true, features = ["future"] }
 object-store = { workspace = true }
 openmetrics-parser = "0.4"
 opentelemetry-proto.workspace = true
diff --git a/src/mito2/Cargo.toml b/src/mito2/Cargo.toml
index cdd2d8d71b56..94e569c8b014 100644
--- a/src/mito2/Cargo.toml
+++ b/src/mito2/Cargo.toml
@@ -40,6 +40,7 @@ humantime-serde = { workspace = true }
 lazy_static = "1.4"
 memcomparable = "0.2"
 metrics.workspace = true
+moka.workspace = true
 object-store = { workspace = true }
 parquet = { workspace = true, features = ["async"] }
 paste.workspace = true
diff --git a/src/mito2/src/cache.rs b/src/mito2/src/cache.rs
new file mode 100644
index 000000000000..41ddd17af18a
--- /dev/null
+++ b/src/mito2/src/cache.rs
@@ -0,0 +1,131 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Cache for the engine.
+
+mod cache_size;
+#[cfg(test)]
+pub(crate) mod test_util;
+
+use std::mem;
+use std::sync::Arc;
+
+use moka::sync::Cache;
+use parquet::file::metadata::ParquetMetaData;
+use store_api::storage::RegionId;
+
+use crate::cache::cache_size::parquet_meta_size;
+use crate::sst::file::FileId;
+
+/// Manages cached data for the engine.
+pub struct CacheManager {
+    /// Cache for SST metadata.
+    sst_meta_cache: Option<SstMetaCache>,
+}
+
+pub type CacheManagerRef = Arc<CacheManager>;
+
+impl CacheManager {
+    /// Creates a new manager with specific cache size in bytes.
+    pub fn new(sst_meta_cache_size: u64) -> CacheManager {
+        let sst_meta_cache = if sst_meta_cache_size == 0 {
+            None
+        } else {
+            let cache = Cache::builder()
+                .max_capacity(sst_meta_cache_size)
+                .weigher(|k: &SstMetaKey, v: &Arc<ParquetMetaData>| {
+                    // We ignore the size of `Arc`.
+                    (k.estimated_size() + parquet_meta_size(v)) as u32
+                })
+                .build();
+            Some(cache)
+        };
+
+        CacheManager { sst_meta_cache }
+    }
+
+    /// Gets cached [ParquetMetaData].
+    pub fn get_parquet_meta_data(
+        &self,
+        region_id: RegionId,
+        file_id: FileId,
+    ) -> Option<Arc<ParquetMetaData>> {
+        self.sst_meta_cache
+            .as_ref()
+            .and_then(|sst_meta_cache| sst_meta_cache.get(&SstMetaKey(region_id, file_id)))
+    }
+
+    /// Puts [ParquetMetaData] into the cache.
+    pub fn put_parquet_meta_data(
+        &self,
+        region_id: RegionId,
+        file_id: FileId,
+        metadata: Arc<ParquetMetaData>,
+    ) {
+        if let Some(cache) = &self.sst_meta_cache {
+            cache.insert(SstMetaKey(region_id, file_id), metadata);
+        }
+    }
+
+    /// Removes [ParquetMetaData] from the cache.
+    pub fn remove_parquet_meta_data(&self, region_id: RegionId, file_id: FileId) {
+        if let Some(cache) = &self.sst_meta_cache {
+            cache.remove(&SstMetaKey(region_id, file_id));
+        }
+    }
+}
+
+/// Cache key for SST meta.
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+struct SstMetaKey(RegionId, FileId);
+
+impl SstMetaKey {
+    /// Returns memory used by the key (estimated).
+    fn estimated_size(&self) -> usize {
+        mem::size_of::<SstMetaKey>()
+    }
+}
+
+type SstMetaCache = Cache<SstMetaKey, Arc<ParquetMetaData>>;
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::cache::test_util::parquet_meta;
+
+    #[test]
+    fn test_disable_meta_cache() {
+        let cache = CacheManager::new(0);
+        assert!(cache.sst_meta_cache.is_none());
+
+        let region_id = RegionId::new(1, 1);
+        let file_id = FileId::random();
+        let metadata = parquet_meta();
+        cache.put_parquet_meta_data(region_id, file_id, metadata);
+        assert!(cache.get_parquet_meta_data(region_id, file_id).is_none());
+    }
+
+    #[test]
+    fn test_parquet_meta_cache() {
+        let cache = CacheManager::new(2000);
+        let region_id = RegionId::new(1, 1);
+        let file_id = FileId::random();
+        assert!(cache.get_parquet_meta_data(region_id, file_id).is_none());
+        let metadata = parquet_meta();
+        cache.put_parquet_meta_data(region_id, file_id, metadata);
+        assert!(cache.get_parquet_meta_data(region_id, file_id).is_some());
+        cache.remove_parquet_meta_data(region_id, file_id);
+        assert!(cache.get_parquet_meta_data(region_id, file_id).is_none());
+    }
+}
diff --git a/src/mito2/src/cache/cache_size.rs b/src/mito2/src/cache/cache_size.rs
new file mode 100644
index 000000000000..8ecd2d5e99c6
--- /dev/null
+++ b/src/mito2/src/cache/cache_size.rs
@@ -0,0 +1,142 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Cache size of different cache value.
+
+use std::mem;
+
+use parquet::file::metadata::{
+    FileMetaData, ParquetColumnIndex, ParquetMetaData, ParquetOffsetIndex, RowGroupMetaData,
+};
+use parquet::file::page_index::index::Index;
+use parquet::format::{ColumnOrder, KeyValue, PageLocation};
+use parquet::schema::types::{ColumnDescriptor, SchemaDescriptor, Type};
+
+/// Returns estimated size of [ParquetMetaData].
+pub fn parquet_meta_size(meta: &ParquetMetaData) -> usize {
+    // struct size
+    let mut size = mem::size_of::<ParquetMetaData>();
+    // file_metadata
+    size += file_meta_heap_size(meta.file_metadata());
+    // row_groups
+    size += meta
+        .row_groups()
+        .iter()
+        .map(row_group_meta_heap_size)
+        .sum::<usize>();
+    // column_index
+    size += meta
+        .column_index()
+        .map(parquet_column_index_heap_size)
+        .unwrap_or(0);
+    // offset_index
+    size += meta
+        .offset_index()
+        .map(parquet_offset_index_heap_size)
+        .unwrap_or(0);
+
+    size
+}
+
+/// Returns estimated size of [FileMetaData] allocated from heap.
+fn file_meta_heap_size(meta: &FileMetaData) -> usize {
+    // created_by
+    let mut size = meta.created_by().map(|s| s.len()).unwrap_or(0);
+    // key_value_metadata
+    size += meta
+        .key_value_metadata()
+        .map(|kvs| {
+            kvs.iter()
+                .map(|kv| {
+                    kv.key.len()
+                        + kv.value.as_ref().map(|v| v.len()).unwrap_or(0)
+                        + mem::size_of::<KeyValue>()
+                })
+                .sum()
+        })
+        .unwrap_or(0);
+    // schema_descr (It's a ptr so we also add size of SchemaDescriptor).
+    size += mem::size_of::<SchemaDescriptor>();
+    size += schema_descr_heap_size(meta.schema_descr());
+    // column_orders
+    size += meta
+        .column_orders()
+        .map(|orders| orders.len() * mem::size_of::<ColumnOrder>())
+        .unwrap_or(0);
+
+    size
+}
+
+/// Returns estimated size of [SchemaDescriptor] allocated from heap.
+fn schema_descr_heap_size(descr: &SchemaDescriptor) -> usize {
+    // schema
+    let mut size = mem::size_of::<Type>();
+    // leaves
+    size += descr
+        .columns()
+        .iter()
+        .map(|descr| mem::size_of::<ColumnDescriptor>() + column_descr_heap_size(descr))
+        .sum::<usize>();
+    // leaf_to_base
+    size += descr.num_columns() * mem::size_of::<usize>();
+
+    size
+}
+
+/// Returns estimated size of [ColumnDescriptor] allocated from heap.
+fn column_descr_heap_size(descr: &ColumnDescriptor) -> usize {
+    descr.path().parts().iter().map(|s| s.len()).sum()
+}
+
+/// Returns estimated size of [ColumnDescriptor] allocated from heap.
+fn row_group_meta_heap_size(meta: &RowGroupMetaData) -> usize {
+    mem::size_of_val(meta.columns())
+}
+
+/// Returns estimated size of [ParquetColumnIndex] allocated from heap.
+fn parquet_column_index_heap_size(column_index: &ParquetColumnIndex) -> usize {
+    column_index
+        .iter()
+        .map(|row_group| row_group.len() * mem::size_of::<Index>() + mem::size_of_val(row_group))
+        .sum()
+}
+
+/// Returns estimated size of [ParquetOffsetIndex] allocated from heap.
+fn parquet_offset_index_heap_size(offset_index: &ParquetOffsetIndex) -> usize {
+    offset_index
+        .iter()
+        .map(|row_group| {
+            row_group
+                .iter()
+                .map(|column| {
+                    column.len() * mem::size_of::<PageLocation>() + mem::size_of_val(column)
+                })
+                .sum::<usize>()
+                + mem::size_of_val(row_group)
+        })
+        .sum()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::cache::test_util::parquet_meta;
+
+    #[test]
+    fn test_parquet_meta_size() {
+        let metadata = parquet_meta();
+
+        assert_eq!(948, parquet_meta_size(&metadata));
+    }
+}
diff --git a/src/mito2/src/cache/test_util.rs b/src/mito2/src/cache/test_util.rs
new file mode 100644
index 000000000000..deb4ba23cd7b
--- /dev/null
+++ b/src/mito2/src/cache/test_util.rs
@@ -0,0 +1,44 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Utilities for testing cache.
+
+use std::sync::Arc;
+
+use bytes::Bytes;
+use datatypes::arrow::array::{ArrayRef, Int64Array};
+use datatypes::arrow::record_batch::RecordBatch;
+use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder;
+use parquet::arrow::ArrowWriter;
+use parquet::file::metadata::ParquetMetaData;
+
+/// Returns a parquet meta data.
+pub(crate) fn parquet_meta() -> Arc<ParquetMetaData> {
+    let file_data = parquet_file_data();
+    let builder = ParquetRecordBatchReaderBuilder::try_new(Bytes::from(file_data)).unwrap();
+    builder.metadata().clone()
+}
+
+/// Write a test parquet file to a buffer
+fn parquet_file_data() -> Vec<u8> {
+    let col = Arc::new(Int64Array::from_iter_values([1, 2, 3])) as ArrayRef;
+    let to_write = RecordBatch::try_from_iter([("col", col)]).unwrap();
+
+    let mut buffer = Vec::new();
+    let mut writer = ArrowWriter::try_new(&mut buffer, to_write.schema(), None).unwrap();
+    writer.write(&to_write).unwrap();
+    writer.close().unwrap();
+
+    buffer
+}
diff --git a/src/mito2/src/config.rs b/src/mito2/src/config.rs
index 3ca21a1aec91..f9e444cf38d3 100644
--- a/src/mito2/src/config.rs
+++ b/src/mito2/src/config.rs
@@ -56,6 +56,10 @@ pub struct MitoConfig {
     pub global_write_buffer_size: ReadableSize,
     /// Global write buffer size threshold to reject write requests (default 2G).
     pub global_write_buffer_reject_size: ReadableSize,
+
+    // Cache configs:
+    /// Cache size for SST metadata (default 128MB). Setting it to 0 to disable cache.
+    pub sst_meta_cache_size: ReadableSize,
 }
 
 impl Default for MitoConfig {
@@ -70,6 +74,7 @@ impl Default for MitoConfig {
             auto_flush_interval: Duration::from_secs(30 * 60),
             global_write_buffer_size: ReadableSize::gb(1),
             global_write_buffer_reject_size: ReadableSize::gb(2),
+            sst_meta_cache_size: ReadableSize::mb(128),
         }
     }
 }
diff --git a/src/mito2/src/engine.rs b/src/mito2/src/engine.rs
index 7cd1079d2d12..987ecbf7e593 100644
--- a/src/mito2/src/engine.rs
+++ b/src/mito2/src/engine.rs
@@ -144,7 +144,14 @@ impl EngineInner {
             .get_region(region_id)
             .context(RegionNotFoundSnafu { region_id })?;
         let version = region.version();
-        let scan_region = ScanRegion::new(version, region.access_layer.clone(), request);
+        // Get cache.
+        let cache_manager = self.workers.cache_manager();
+        let scan_region = ScanRegion::new(
+            version,
+            region.access_layer.clone(),
+            request,
+            Some(cache_manager),
+        );
 
         scan_region.scanner()
     }
diff --git a/src/mito2/src/lib.rs b/src/mito2/src/lib.rs
index 71d67983f091..dd404f2013b6 100644
--- a/src/mito2/src/lib.rs
+++ b/src/mito2/src/lib.rs
@@ -22,6 +22,7 @@
 pub mod test_util;
 
 mod access_layer;
+mod cache;
 mod compaction;
 pub mod config;
 pub mod engine;
diff --git a/src/mito2/src/read/scan_region.rs b/src/mito2/src/read/scan_region.rs
index be8b318d83ff..083238dcd06d 100644
--- a/src/mito2/src/read/scan_region.rs
+++ b/src/mito2/src/read/scan_region.rs
@@ -22,6 +22,7 @@ use store_api::storage::ScanRequest;
 use table::predicate::{Predicate, TimeRangePredicateBuilder};
 
 use crate::access_layer::AccessLayerRef;
+use crate::cache::CacheManagerRef;
 use crate::error::{BuildPredicateSnafu, Result};
 use crate::read::projection::ProjectionMapper;
 use crate::read::seq_scan::SeqScan;
@@ -113,6 +114,8 @@ pub(crate) struct ScanRegion {
     access_layer: AccessLayerRef,
     /// Scan request.
     request: ScanRequest,
+    /// Cache.
+    cache_manager: Option<CacheManagerRef>,
 }
 
 impl ScanRegion {
@@ -121,11 +124,13 @@ impl ScanRegion {
         version: VersionRef,
         access_layer: AccessLayerRef,
         request: ScanRequest,
+        cache_manager: Option<CacheManagerRef>,
     ) -> ScanRegion {
         ScanRegion {
             version,
             access_layer,
             request,
+            cache_manager,
         }
     }
 
@@ -181,7 +186,8 @@ impl ScanRegion {
             .with_time_range(Some(time_range))
             .with_predicate(Some(predicate))
             .with_memtables(memtables)
-            .with_files(files);
+            .with_files(files)
+            .with_cache(self.cache_manager);
 
         Ok(seq_scan)
     }
diff --git a/src/mito2/src/read/seq_scan.rs b/src/mito2/src/read/seq_scan.rs
index 148cb3777142..3a3a6a1eaad8 100644
--- a/src/mito2/src/read/seq_scan.rs
+++ b/src/mito2/src/read/seq_scan.rs
@@ -25,6 +25,7 @@ use snafu::ResultExt;
 use table::predicate::Predicate;
 
 use crate::access_layer::AccessLayerRef;
+use crate::cache::CacheManagerRef;
 use crate::error::Result;
 use crate::memtable::MemtableRef;
 use crate::read::compat::{self, CompatReader};
@@ -49,6 +50,8 @@ pub struct SeqScan {
     memtables: Vec<MemtableRef>,
     /// Handles to SST files to scan.
     files: Vec<FileHandle>,
+    /// Cache.
+    cache_manager: Option<CacheManagerRef>,
 }
 
 impl SeqScan {
@@ -62,37 +65,44 @@ impl SeqScan {
             predicate: None,
             memtables: Vec::new(),
             files: Vec::new(),
+            cache_manager: None,
         }
     }
 
-    /// Set time range filter for time index.
+    /// Sets time range filter for time index.
     #[must_use]
     pub(crate) fn with_time_range(mut self, time_range: Option<TimestampRange>) -> Self {
         self.time_range = time_range;
         self
     }
 
-    /// Set predicate to push down.
+    /// Sets predicate to push down.
     #[must_use]
     pub(crate) fn with_predicate(mut self, predicate: Option<Predicate>) -> Self {
         self.predicate = predicate;
         self
     }
 
-    /// Set memtables to read.
+    /// Sets memtables to read.
     #[must_use]
     pub(crate) fn with_memtables(mut self, memtables: Vec<MemtableRef>) -> Self {
         self.memtables = memtables;
         self
     }
 
-    /// Set files to read.
+    /// Sets files to read.
     #[must_use]
     pub(crate) fn with_files(mut self, files: Vec<FileHandle>) -> Self {
         self.files = files;
         self
     }
 
+    /// Sets cache for this query.
+    pub(crate) fn with_cache(mut self, cache: Option<CacheManagerRef>) -> Self {
+        self.cache_manager = cache;
+        self
+    }
+
     /// Builds a stream for the query.
     pub async fn build_stream(&self) -> Result<SendableRecordBatchStream> {
         // Scans all memtables and SSTs. Builds a merge reader to merge results.
@@ -129,6 +139,7 @@ impl SeqScan {
                 .predicate(self.predicate.clone())
                 .time_range(self.time_range)
                 .projection(Some(self.mapper.column_ids().to_vec()))
+                .cache(self.cache_manager.clone())
                 .build()
                 .await?;
             if compat::has_same_columns(self.mapper.metadata(), reader.metadata()) {
diff --git a/src/mito2/src/region/opener.rs b/src/mito2/src/region/opener.rs
index 3683d6fc2f73..6a8a3a805f7c 100644
--- a/src/mito2/src/region/opener.rs
+++ b/src/mito2/src/region/opener.rs
@@ -29,6 +29,7 @@ use store_api::metadata::{ColumnMetadata, RegionMetadata};
 use store_api::storage::{ColumnId, RegionId};
 
 use crate::access_layer::AccessLayer;
+use crate::cache::CacheManagerRef;
 use crate::config::MitoConfig;
 use crate::error::{EmptyRegionDirSnafu, RegionCorruptedSnafu, Result};
 use crate::manifest::manager::{RegionManifestManager, RegionManifestOptions};
@@ -51,6 +52,7 @@ pub(crate) struct RegionOpener {
     region_dir: String,
     scheduler: SchedulerRef,
     options: HashMap<String, String>,
+    cache_manager: Option<CacheManagerRef>,
 }
 
 impl RegionOpener {
@@ -70,6 +72,7 @@ impl RegionOpener {
             region_dir: normalize_dir(region_dir),
             scheduler,
             options: HashMap::new(),
+            cache_manager: None,
         }
     }
 
@@ -85,6 +88,12 @@ impl RegionOpener {
         self
     }
 
+    /// Sets the cache manager for the region.
+    pub(crate) fn cache(mut self, cache_manager: Option<CacheManagerRef>) -> Self {
+        self.cache_manager = cache_manager;
+        self
+    }
+
     /// Writes region manifest and creates a new region if it does not exist.
     /// Opens the region if it already exists.
     ///
@@ -145,7 +154,11 @@ impl RegionOpener {
             version_control,
             access_layer: access_layer.clone(),
             manifest_manager,
-            file_purger: Arc::new(LocalFilePurger::new(self.scheduler, access_layer)),
+            file_purger: Arc::new(LocalFilePurger::new(
+                self.scheduler,
+                access_layer,
+                self.cache_manager,
+            )),
             last_flush_millis: AtomicI64::new(current_time_millis()),
             // Region is writable after it is created.
             writable: AtomicBool::new(true),
@@ -205,6 +218,7 @@ impl RegionOpener {
         let file_purger = Arc::new(LocalFilePurger::new(
             self.scheduler.clone(),
             access_layer.clone(),
+            self.cache_manager.clone(),
         ));
         let mutable = self.memtable_builder.build(&metadata);
         let options = RegionOptions::try_from(&self.options)?;
diff --git a/src/mito2/src/sst/file.rs b/src/mito2/src/sst/file.rs
index ecf961854b7b..a16987690d09 100644
--- a/src/mito2/src/sst/file.rs
+++ b/src/mito2/src/sst/file.rs
@@ -118,6 +118,12 @@ impl FileHandle {
             inner: Arc::new(FileHandleInner::new(meta, file_purger)),
         }
     }
+
+    /// Returns the region id of the file.
+    pub fn region_id(&self) -> RegionId {
+        self.inner.meta.region_id
+    }
+
     /// Returns the file id.
     pub fn file_id(&self) -> FileId {
         self.inner.meta.file_id
diff --git a/src/mito2/src/sst/file_purger.rs b/src/mito2/src/sst/file_purger.rs
index ace502f4a659..15c3df6cc703 100644
--- a/src/mito2/src/sst/file_purger.rs
+++ b/src/mito2/src/sst/file_purger.rs
@@ -19,6 +19,7 @@ use common_telemetry::{error, info};
 use store_api::storage::RegionId;
 
 use crate::access_layer::AccessLayerRef;
+use crate::cache::CacheManagerRef;
 use crate::schedule::scheduler::SchedulerRef;
 use crate::sst::file::FileId;
 
@@ -39,10 +40,11 @@ pub trait FilePurger: Send + Sync + fmt::Debug {
 
 pub type FilePurgerRef = Arc<dyn FilePurger>;
 
+/// Purger that purges file for current region.
 pub struct LocalFilePurger {
     scheduler: SchedulerRef,
-
     sst_layer: AccessLayerRef,
+    cache_manager: Option<CacheManagerRef>,
 }
 
 impl fmt::Debug for LocalFilePurger {
@@ -54,10 +56,16 @@ impl fmt::Debug for LocalFilePurger {
 }
 
 impl LocalFilePurger {
-    pub fn new(scheduler: SchedulerRef, sst_layer: AccessLayerRef) -> Self {
+    /// Creates a new purger.
+    pub fn new(
+        scheduler: SchedulerRef,
+        sst_layer: AccessLayerRef,
+        cache_manager: Option<CacheManagerRef>,
+    ) -> Self {
         Self {
             scheduler,
             sst_layer,
+            cache_manager,
         }
     }
 }
@@ -68,6 +76,11 @@ impl FilePurger for LocalFilePurger {
         let region_id = request.region_id;
         let sst_layer = self.sst_layer.clone();
 
+        // Remove meta of the file from cache.
+        if let Some(cache) = &self.cache_manager {
+            cache.remove_parquet_meta_data(region_id, file_id);
+        }
+
         if let Err(e) = self.scheduler.schedule(Box::pin(async move {
             if let Err(e) = sst_layer.delete_sst(file_id).await {
                 error!(e; "Failed to delete SST file, file: {}, region: {}", 
@@ -113,7 +126,7 @@ mod tests {
         let scheduler = Arc::new(LocalScheduler::new(3));
         let layer = Arc::new(AccessLayer::new(sst_dir, object_store.clone()));
 
-        let file_purger = Arc::new(LocalFilePurger::new(scheduler.clone(), layer));
+        let file_purger = Arc::new(LocalFilePurger::new(scheduler.clone(), layer, None));
 
         {
             let handle = FileHandle::new(
diff --git a/src/mito2/src/sst/parquet/reader.rs b/src/mito2/src/sst/parquet/reader.rs
index 6280f83eef10..a91d781dad92 100644
--- a/src/mito2/src/sst/parquet/reader.rs
+++ b/src/mito2/src/sst/parquet/reader.rs
@@ -14,29 +14,35 @@
 
 //! Parquet reader.
 
+use std::ops::Range;
 use std::sync::Arc;
 
 use async_compat::CompatExt;
 use async_trait::async_trait;
+use bytes::Bytes;
 use common_time::range::TimestampRange;
 use datatypes::arrow::record_batch::RecordBatch;
+use futures::future::BoxFuture;
 use futures::stream::BoxStream;
-use futures::TryStreamExt;
+use futures::{FutureExt, TryStreamExt};
 use object_store::ObjectStore;
+use parquet::arrow::async_reader::AsyncFileReader;
 use parquet::arrow::{ParquetRecordBatchStreamBuilder, ProjectionMask};
 use parquet::errors::ParquetError;
+use parquet::file::metadata::ParquetMetaData;
 use parquet::format::KeyValue;
 use snafu::{ensure, OptionExt, ResultExt};
 use store_api::metadata::{RegionMetadata, RegionMetadataRef};
-use store_api::storage::ColumnId;
+use store_api::storage::{ColumnId, RegionId};
 use table::predicate::Predicate;
 use tokio::io::BufReader;
 
+use crate::cache::CacheManagerRef;
 use crate::error::{
     InvalidMetadataSnafu, InvalidParquetSnafu, OpenDalSnafu, ReadParquetSnafu, Result,
 };
 use crate::read::{Batch, BatchReader};
-use crate::sst::file::FileHandle;
+use crate::sst::file::{FileHandle, FileId};
 use crate::sst::parquet::format::ReadFormat;
 use crate::sst::parquet::PARQUET_METADATA_KEY;
 
@@ -55,6 +61,8 @@ pub struct ParquetReaderBuilder {
     /// `None` reads all columns. Due to schema change, the projection
     /// can contain columns not in the parquet file.
     projection: Option<Vec<ColumnId>>,
+    /// Manager that caches SST data.
+    cache_manager: Option<CacheManagerRef>,
 }
 
 impl ParquetReaderBuilder {
@@ -71,6 +79,7 @@ impl ParquetReaderBuilder {
             predicate: None,
             time_range: None,
             projection: None,
+            cache_manager: None,
         }
     }
 
@@ -94,6 +103,12 @@ impl ParquetReaderBuilder {
         self
     }
 
+    /// Attaches the cache to the builder.
+    pub fn cache(mut self, cache: Option<CacheManagerRef>) -> ParquetReaderBuilder {
+        self.cache_manager = cache;
+        self
+    }
+
     /// Builds and initializes a [ParquetReader].
     ///
     /// This needs to perform IO operation.
@@ -119,8 +134,16 @@ impl ParquetReaderBuilder {
             .await
             .context(OpenDalSnafu)?
             .compat();
-        let buf_reader = BufReader::new(reader);
-        let mut builder = ParquetRecordBatchStreamBuilder::new(buf_reader)
+        let reader = BufReader::new(reader);
+        let reader = AsyncFileReaderCache {
+            reader,
+            // TODO(yingwen): Sets the metadata when we implement row group level reader.
+            metadata: None,
+            cache: self.cache_manager.clone(),
+            region_id: self.file_handle.region_id(),
+            file_id: self.file_handle.file_id(),
+        };
+        let mut builder = ParquetRecordBatchStreamBuilder::new(reader)
             .await
             .context(ReadParquetSnafu { path: file_path })?;
 
@@ -249,3 +272,58 @@ impl ParquetReader {
         self.read_format.metadata()
     }
 }
+
+/// Cache layer for parquet's [AsyncFileReader].
+struct AsyncFileReaderCache<T> {
+    /// Underlying async file reader.
+    reader: T,
+    /// Parquet metadata cached locally.
+    metadata: Option<Arc<ParquetMetaData>>,
+    /// Global cache.
+    cache: Option<CacheManagerRef>,
+    /// Id of the region.
+    region_id: RegionId,
+    /// Id of the file to read.
+    file_id: FileId,
+}
+
+impl<T: AsyncFileReader> AsyncFileReader for AsyncFileReaderCache<T> {
+    fn get_bytes(&mut self, range: Range<usize>) -> BoxFuture<'_, Result<Bytes, ParquetError>> {
+        self.reader.get_bytes(range)
+    }
+
+    fn get_byte_ranges(
+        &mut self,
+        ranges: Vec<Range<usize>>,
+    ) -> BoxFuture<'_, Result<Vec<Bytes>, ParquetError>> {
+        self.reader.get_byte_ranges(ranges)
+    }
+
+    fn get_metadata(&mut self) -> BoxFuture<'_, Result<Arc<ParquetMetaData>, ParquetError>> {
+        async {
+            // Tries to get from local cache.
+            if let Some(metadata) = &self.metadata {
+                return Ok(metadata.clone());
+            }
+
+            // Tries to get from global cache.
+            if let Some(metadata) = self
+                .cache
+                .as_ref()
+                .and_then(|cache| cache.get_parquet_meta_data(self.region_id, self.file_id))
+            {
+                return Ok(metadata);
+            }
+
+            // Cache miss.
+            let metadata = self.reader.get_metadata().await?;
+            // Cache the metadata.
+            if let Some(cache) = &self.cache {
+                cache.put_parquet_meta_data(self.region_id, self.file_id, metadata.clone());
+            }
+
+            Ok(metadata)
+        }
+        .boxed()
+    }
+}
diff --git a/src/mito2/src/worker.rs b/src/mito2/src/worker.rs
index 2f72724fd8a1..4b119c69959f 100644
--- a/src/mito2/src/worker.rs
+++ b/src/mito2/src/worker.rs
@@ -39,6 +39,7 @@ use store_api::storage::RegionId;
 use tokio::sync::mpsc::{Receiver, Sender};
 use tokio::sync::{mpsc, Mutex};
 
+use crate::cache::{CacheManager, CacheManagerRef};
 use crate::compaction::CompactionScheduler;
 use crate::config::MitoConfig;
 use crate::error::{JoinSnafu, Result, WorkerStoppedSnafu};
@@ -95,8 +96,12 @@ pub(crate) const DROPPING_MARKER_FILE: &str = ".dropping";
 /// Chan1 --> WorkerThread1
 /// ```
 pub(crate) struct WorkerGroup {
+    /// Workers of the group.
     workers: Vec<RegionWorker>,
+    /// Global background job scheduelr.
     scheduler: SchedulerRef,
+    /// Cache.
+    cache_manager: CacheManagerRef,
 }
 
 impl WorkerGroup {
@@ -114,6 +119,7 @@ impl WorkerGroup {
             config.global_write_buffer_size.as_bytes() as usize,
         ));
         let scheduler = Arc::new(LocalScheduler::new(config.max_background_jobs));
+        let cache_manager = Arc::new(CacheManager::new(config.sst_meta_cache_size.as_bytes()));
 
         let workers = (0..config.num_workers)
             .map(|id| {
@@ -125,12 +131,17 @@ impl WorkerGroup {
                     write_buffer_manager: write_buffer_manager.clone(),
                     scheduler: scheduler.clone(),
                     listener: WorkerListener::default(),
+                    cache_manager: cache_manager.clone(),
                 }
                 .start()
             })
             .collect();
 
-        WorkerGroup { workers, scheduler }
+        WorkerGroup {
+            workers,
+            scheduler,
+            cache_manager,
+        }
     }
 
     /// Stops the worker group.
@@ -166,6 +177,11 @@ impl WorkerGroup {
         self.worker(region_id).get_region(region_id)
     }
 
+    /// Returns cache of the group.
+    pub(crate) fn cache_manager(&self) -> CacheManagerRef {
+        self.cache_manager.clone()
+    }
+
     /// Get worker for specific `region_id`.
     fn worker(&self, region_id: RegionId) -> &RegionWorker {
         let mut hasher = DefaultHasher::new();
@@ -193,6 +209,7 @@ impl WorkerGroup {
         assert!(config.num_workers.is_power_of_two());
         let config = Arc::new(config);
         let scheduler = Arc::new(LocalScheduler::new(config.max_background_jobs));
+        let cache_manager = Arc::new(CacheManager::new(config.sst_meta_cache_size.as_bytes()));
 
         let workers = (0..config.num_workers)
             .map(|id| {
@@ -204,12 +221,17 @@ impl WorkerGroup {
                     write_buffer_manager: write_buffer_manager.clone(),
                     scheduler: scheduler.clone(),
                     listener: WorkerListener::new(listener.clone()),
+                    cache_manager: cache_manager.clone(),
                 }
                 .start()
             })
             .collect();
 
-        WorkerGroup { workers, scheduler }
+        WorkerGroup {
+            workers,
+            scheduler,
+            cache_manager,
+        }
     }
 }
 
@@ -226,6 +248,7 @@ struct WorkerStarter<S> {
     write_buffer_manager: WriteBufferManagerRef,
     scheduler: SchedulerRef,
     listener: WorkerListener,
+    cache_manager: CacheManagerRef,
 }
 
 impl<S: LogStore> WorkerStarter<S> {
@@ -254,6 +277,7 @@ impl<S: LogStore> WorkerStarter<S> {
             compaction_scheduler: CompactionScheduler::new(self.scheduler, sender.clone()),
             stalled_requests: StalledRequests::default(),
             listener: self.listener,
+            cache_manager: self.cache_manager,
         };
         let handle = common_runtime::spawn_write(async move {
             worker_thread.run().await;
@@ -376,7 +400,7 @@ impl StalledRequests {
 
 /// Background worker loop to handle requests.
 struct RegionWorkerLoop<S> {
-    // Id of the worker.
+    /// Id of the worker.
     id: WorkerId,
     /// Engine config.
     config: Arc<MitoConfig>,
@@ -408,6 +432,8 @@ struct RegionWorkerLoop<S> {
     stalled_requests: StalledRequests,
     /// Event listener for tests.
     listener: WorkerListener,
+    /// Cache.
+    cache_manager: CacheManagerRef,
 }
 
 impl<S: LogStore> RegionWorkerLoop<S> {
diff --git a/src/mito2/src/worker/handle_create.rs b/src/mito2/src/worker/handle_create.rs
index 848082956894..e9ace31044a5 100644
--- a/src/mito2/src/worker/handle_create.rs
+++ b/src/mito2/src/worker/handle_create.rs
@@ -65,6 +65,7 @@ impl<S: LogStore> RegionWorkerLoop<S> {
         )
         .metadata(metadata)
         .options(request.options)
+        .cache(Some(self.cache_manager.clone()))
         .create_or_open(&self.config, &self.wal)
         .await?;
 
diff --git a/src/mito2/src/worker/handle_open.rs b/src/mito2/src/worker/handle_open.rs
index ab6967c0d1b6..cfeb43d84d98 100644
--- a/src/mito2/src/worker/handle_open.rs
+++ b/src/mito2/src/worker/handle_open.rs
@@ -63,6 +63,7 @@ impl<S: LogStore> RegionWorkerLoop<S> {
             self.scheduler.clone(),
         )
         .options(request.options)
+        .cache(Some(self.cache_manager.clone()))
         .open(&self.config, &self.wal)
         .await?;
 
diff --git a/src/partition/Cargo.toml b/src/partition/Cargo.toml
index e902f7ca6faf..4cb2def7bf88 100644
--- a/src/partition/Cargo.toml
+++ b/src/partition/Cargo.toml
@@ -20,7 +20,7 @@ datafusion-expr.workspace = true
 datafusion.workspace = true
 datatypes = { workspace = true }
 meta-client = { workspace = true }
-moka = { version = "0.9", features = ["future"] }
+moka = { workspace = true, features = ["future"] }
 serde.workspace = true
 serde_json = "1.0"
 snafu.workspace = true
diff --git a/tests-integration/tests/http.rs b/tests-integration/tests/http.rs
index 0317482da7f4..2d9ddc6289f4 100644
--- a/tests-integration/tests/http.rs
+++ b/tests-integration/tests/http.rs
@@ -662,6 +662,7 @@ max_background_jobs = 4
 auto_flush_interval = "30m"
 global_write_buffer_size = "1GiB"
 global_write_buffer_reject_size = "2GiB"
+sst_meta_cache_size = "128MiB"
 
 [[region_engine]]
 

From c20badef3b09c19be5e4fcaa0f8373e69df79868 Mon Sep 17 00:00:00 2001
From: Ruihang Xia <waynestxia@gmail.com>
Date: Tue, 26 Sep 2023 20:12:08 +0800
Subject: [PATCH 29/36] fix: check for table scan before expanding (#2491)

* fix: check for table scan before expanding

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>

* change assert_ok to unwrap

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>

* fix clippy warning

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>

* update sqlness result

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>

* don't skip dml

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>

* uncomment ignored tests

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>

---------

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>
---
 src/query/src/dist_plan/analyzer.rs | 16 ++++++++++------
 tests-integration/tests/sql.rs      | 20 +++++++-------------
 2 files changed, 17 insertions(+), 19 deletions(-)

diff --git a/src/query/src/dist_plan/analyzer.rs b/src/query/src/dist_plan/analyzer.rs
index 45c77edb81a1..7807913e74b5 100644
--- a/src/query/src/dist_plan/analyzer.rs
+++ b/src/query/src/dist_plan/analyzer.rs
@@ -14,7 +14,6 @@
 
 use std::sync::Arc;
 
-use common_telemetry::info;
 use datafusion::datasource::DefaultTableSource;
 use datafusion::error::Result as DfResult;
 use datafusion_common::config::ConfigOptions;
@@ -46,7 +45,9 @@ impl AnalyzerRule for DistPlannerAnalyzer {
     ) -> datafusion_common::Result<LogicalPlan> {
         let plan = plan.transform(&Self::inspect_plan_with_subquery)?;
         let mut rewriter = PlanRewriter::default();
-        plan.rewrite(&mut rewriter)
+        let result = plan.rewrite(&mut rewriter)?;
+
+        Ok(result)
     }
 }
 
@@ -138,10 +139,6 @@ impl PlanRewriter {
     /// Return true if should stop and expand. The input plan is the parent node of current node
     fn should_expand(&mut self, plan: &LogicalPlan) -> bool {
         if DFLogicalSubstraitConvertor.encode(plan).is_err() {
-            info!(
-                "substrait error: {:?}",
-                DFLogicalSubstraitConvertor.encode(plan)
-            );
             return true;
         }
 
@@ -251,6 +248,13 @@ impl TreeNodeRewriter for PlanRewriter {
             return Ok(node);
         }
 
+        // only expand when the leaf is table scan
+        if node.inputs().is_empty() && !matches!(node, LogicalPlan::TableScan(_)) {
+            self.set_expanded();
+            self.pop_stack();
+            return Ok(node);
+        }
+
         self.maybe_set_partitions(&node);
 
         let Some(parent) = self.get_parent() else {
diff --git a/tests-integration/tests/sql.rs b/tests-integration/tests/sql.rs
index a8efea7da026..f8e81230eb3c 100644
--- a/tests-integration/tests/sql.rs
+++ b/tests-integration/tests/sql.rs
@@ -54,13 +54,10 @@ macro_rules! sql_tests {
                 $service,
 
                 test_mysql_auth,
-                // ignore: https://github.com/GreptimeTeam/greptimedb/issues/2445
-                // test_mysql_crud,
+                test_mysql_crud,
                 test_postgres_auth,
-                // ignore: https://github.com/GreptimeTeam/greptimedb/issues/2445
-                // test_postgres_crud,
-                // ignore: https://github.com/GreptimeTeam/greptimedb/issues/2445
-                // test_postgres_parameter_inference,
+                test_postgres_crud,
+                test_postgres_parameter_inference,
             );
         )*
     };
@@ -123,7 +120,6 @@ pub async fn test_mysql_auth(store_type: StorageType) {
     guard.remove_all().await;
 }
 
-#[allow(dead_code)]
 pub async fn test_mysql_crud(store_type: StorageType) {
     common_telemetry::init_default_ut_logging();
 
@@ -135,12 +131,12 @@ pub async fn test_mysql_crud(store_type: StorageType) {
         .await
         .unwrap();
 
-    assert!(sqlx::query(
+    sqlx::query(
         "create table demo(i bigint, ts timestamp time index, d date, dt datetime, b blob)",
     )
     .execute(&pool)
     .await
-    .is_ok());
+    .unwrap();
     for i in 0..10 {
         let dt: DateTime<Utc> = DateTime::from_naive_utc_and_offset(
             NaiveDateTime::from_timestamp_opt(60, i).unwrap(),
@@ -149,7 +145,7 @@ pub async fn test_mysql_crud(store_type: StorageType) {
         let d = NaiveDate::from_yo_opt(2015, 100).unwrap();
         let hello = format!("hello{i}");
         let bytes = hello.as_bytes();
-        assert!(sqlx::query("insert into demo values(?, ?, ?, ?, ?)")
+        sqlx::query("insert into demo values(?, ?, ?, ?, ?)")
             .bind(i)
             .bind(i)
             .bind(d)
@@ -157,7 +153,7 @@ pub async fn test_mysql_crud(store_type: StorageType) {
             .bind(bytes)
             .execute(&pool)
             .await
-            .is_ok());
+            .unwrap();
     }
 
     let rows = sqlx::query("select i, d, dt, b from demo")
@@ -270,7 +266,6 @@ pub async fn test_postgres_auth(store_type: StorageType) {
     guard.remove_all().await;
 }
 
-#[allow(dead_code)]
 pub async fn test_postgres_crud(store_type: StorageType) {
     let (addr, mut guard, fe_pg_server) = setup_pg_server(store_type, "sql_crud").await;
 
@@ -347,7 +342,6 @@ pub async fn test_postgres_crud(store_type: StorageType) {
     guard.remove_all().await;
 }
 
-#[allow(dead_code)]
 pub async fn test_postgres_parameter_inference(store_type: StorageType) {
     let (addr, mut guard, fe_pg_server) = setup_pg_server(store_type, "sql_inference").await;
 

From 987eefb87796943fcc43fc7cdb2cb4e2444feac5 Mon Sep 17 00:00:00 2001
From: Yingwen <realevenyag@gmail.com>
Date: Wed, 27 Sep 2023 10:58:17 +0800
Subject: [PATCH 30/36] fix(mito): fix region drop task runs multiple times but
 never clean the dir (#2504)

fix: fix region drop task runs multiple times but never clean the directory
---
 src/mito2/src/engine.rs               |  2 +-
 src/mito2/src/engine/drop_test.rs     | 44 +++++++++++++--------
 src/mito2/src/engine/flush_test.rs    | 10 +++--
 src/mito2/src/engine/listener.rs      | 56 ++++++++++++++++++++++++++-
 src/mito2/src/engine/truncate_test.rs |  2 +-
 src/mito2/src/region/version.rs       |  9 ++++-
 src/mito2/src/sst/version.rs          |  4 +-
 src/mito2/src/test_util.rs            |  2 +-
 src/mito2/src/worker.rs               | 29 +++++++++++++-
 src/mito2/src/worker/handle_drop.rs   | 53 ++++++++++++++++++-------
 10 files changed, 169 insertions(+), 42 deletions(-)

diff --git a/src/mito2/src/engine.rs b/src/mito2/src/engine.rs
index 987ecbf7e593..437e8e3da30d 100644
--- a/src/mito2/src/engine.rs
+++ b/src/mito2/src/engine.rs
@@ -230,7 +230,7 @@ impl MitoEngine {
         mut config: MitoConfig,
         log_store: Arc<S>,
         object_store: ObjectStore,
-        write_buffer_manager: crate::flush::WriteBufferManagerRef,
+        write_buffer_manager: Option<crate::flush::WriteBufferManagerRef>,
         listener: Option<crate::engine::listener::EventListenerRef>,
     ) -> MitoEngine {
         config.sanitize();
diff --git a/src/mito2/src/engine/drop_test.rs b/src/mito2/src/engine/drop_test.rs
index d42c37d05b7e..35c86c184371 100644
--- a/src/mito2/src/engine/drop_test.rs
+++ b/src/mito2/src/engine/drop_test.rs
@@ -12,19 +12,31 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::sync::Arc;
+use std::time::Duration;
+
+use api::v1::Rows;
 use object_store::util::join_path;
 use store_api::region_engine::RegionEngine;
 use store_api::region_request::{RegionDropRequest, RegionRequest};
 use store_api::storage::RegionId;
 
 use crate::config::MitoConfig;
-use crate::test_util::{CreateRequestBuilder, TestEnv};
+use crate::engine::listener::DropListener;
+use crate::test_util::{
+    build_rows_for_key, flush_region, put_rows, rows_schema, CreateRequestBuilder, TestEnv,
+};
 use crate::worker::DROPPING_MARKER_FILE;
 
 #[tokio::test]
 async fn test_engine_drop_region() {
+    common_telemetry::init_default_ut_logging();
+
     let mut env = TestEnv::with_prefix("drop");
-    let engine = env.create_engine(MitoConfig::default()).await;
+    let listener = Arc::new(DropListener::new(Duration::from_millis(100)));
+    let engine = env
+        .create_engine_with(MitoConfig::default(), None, Some(listener.clone()))
+        .await;
 
     let region_id = RegionId::new(1, 1);
     // It's okay to drop a region doesn't exist.
@@ -34,13 +46,14 @@ async fn test_engine_drop_region() {
         .unwrap_err();
 
     let request = CreateRequestBuilder::new().build();
+    let column_schemas = rows_schema(&request);
     engine
         .handle_request(region_id, RegionRequest::Create(request))
         .await
         .unwrap();
 
     let region = engine.get_region(region_id).unwrap();
-    let region_dir = region.access_layer.region_dir().to_owned();
+    let region_dir = region.access_layer.region_dir().to_string();
     // no dropping marker file
     assert!(!env
         .get_object_store()
@@ -49,12 +62,12 @@ async fn test_engine_drop_region() {
         .await
         .unwrap());
 
-    // create a parquet file
-    env.get_object_store()
-        .unwrap()
-        .write(&join_path(&region_dir, "blabla.parquet"), vec![])
-        .await
-        .unwrap();
+    let rows = Rows {
+        schema: column_schemas.clone(),
+        rows: build_rows_for_key("a", 0, 2, 0),
+    };
+    put_rows(&engine, region_id, rows).await;
+    flush_region(&engine, region_id).await;
 
     // drop the created region.
     engine
@@ -62,11 +75,10 @@ async fn test_engine_drop_region() {
         .await
         .unwrap();
     assert!(!engine.is_region_exists(region_id));
-    // the drop marker is not removed yet
-    assert!(env
-        .get_object_store()
-        .unwrap()
-        .is_exist(&join_path(&region_dir, DROPPING_MARKER_FILE))
-        .await
-        .unwrap());
+
+    // Wait for drop task.
+    listener.wait().await;
+
+    let object_store = env.get_object_store().unwrap();
+    assert!(!object_store.is_exist(&region_dir).await.unwrap());
 }
diff --git a/src/mito2/src/engine/flush_test.rs b/src/mito2/src/engine/flush_test.rs
index 636ce7df2ddd..84eb8b0eec44 100644
--- a/src/mito2/src/engine/flush_test.rs
+++ b/src/mito2/src/engine/flush_test.rs
@@ -76,7 +76,7 @@ async fn test_flush_engine() {
     let engine = env
         .create_engine_with(
             MitoConfig::default(),
-            write_buffer_manager.clone(),
+            Some(write_buffer_manager.clone()),
             Some(listener.clone()),
         )
         .await;
@@ -135,7 +135,7 @@ async fn test_write_stall() {
     let engine = env
         .create_engine_with(
             MitoConfig::default(),
-            write_buffer_manager.clone(),
+            Some(write_buffer_manager.clone()),
             Some(listener.clone()),
         )
         .await;
@@ -197,7 +197,11 @@ async fn test_flush_empty() {
     let mut env = TestEnv::new();
     let write_buffer_manager = Arc::new(MockWriteBufferManager::default());
     let engine = env
-        .create_engine_with(MitoConfig::default(), write_buffer_manager.clone(), None)
+        .create_engine_with(
+            MitoConfig::default(),
+            Some(write_buffer_manager.clone()),
+            None,
+        )
         .await;
 
     let region_id = RegionId::new(1, 1);
diff --git a/src/mito2/src/engine/listener.rs b/src/mito2/src/engine/listener.rs
index e591d2b5a576..f0b5def366e0 100644
--- a/src/mito2/src/engine/listener.rs
+++ b/src/mito2/src/engine/listener.rs
@@ -15,6 +15,7 @@
 //! Engine event listener for tests.
 
 use std::sync::Arc;
+use std::time::Duration;
 
 use async_trait::async_trait;
 use common_telemetry::info;
@@ -32,6 +33,19 @@ pub trait EventListener: Send + Sync {
 
     /// Notifies the listener that the region starts to do flush.
     async fn on_flush_begin(&self, region_id: RegionId);
+
+    /// Notifies the listener that the later drop task starts running.
+    /// Returns the gc interval if we want to override the default one.
+    fn on_later_drop_begin(&self, region_id: RegionId) -> Option<Duration> {
+        let _ = region_id;
+        None
+    }
+
+    /// Notifies the listener that the later drop task of the region is finished.
+    fn on_later_drop_end(&self, region_id: RegionId, removed: bool) {
+        let _ = region_id;
+        let _ = removed;
+    }
 }
 
 pub type EventListenerRef = Arc<dyn EventListener>;
@@ -102,7 +116,7 @@ impl EventListener for StallListener {
 
 /// Listener to watch begin flush events.
 ///
-/// Crate a background thread to execute flush region, and the main thread calls `wait_truncate()`
+/// Creates a background thread to execute flush region, and the main thread calls `wait_truncate()`
 /// to block and wait for `on_flush_region()`.
 /// When the background thread calls `on_flush_begin()`, the main thread is notified to truncate
 /// region, and background thread thread blocks and waits for `notify_flush()` to continue flushing.
@@ -150,3 +164,43 @@ impl EventListener for FlushTruncateListener {
         self.notify_flush.notified().await;
     }
 }
+
+/// Listener on dropping.
+pub struct DropListener {
+    gc_duration: Duration,
+    notify: Notify,
+}
+
+impl DropListener {
+    /// Creates a new listener with specific `gc_duration`.
+    pub fn new(gc_duration: Duration) -> Self {
+        DropListener {
+            gc_duration,
+            notify: Notify::new(),
+        }
+    }
+
+    /// Waits until later drop task is done.
+    pub async fn wait(&self) {
+        self.notify.notified().await;
+    }
+}
+
+#[async_trait]
+impl EventListener for DropListener {
+    fn on_flush_success(&self, _region_id: RegionId) {}
+
+    fn on_write_stall(&self) {}
+
+    async fn on_flush_begin(&self, _region_id: RegionId) {}
+
+    fn on_later_drop_begin(&self, _region_id: RegionId) -> Option<Duration> {
+        Some(self.gc_duration)
+    }
+
+    fn on_later_drop_end(&self, _region_id: RegionId, removed: bool) {
+        // Asserts result.
+        assert!(removed);
+        self.notify.notify_one();
+    }
+}
diff --git a/src/mito2/src/engine/truncate_test.rs b/src/mito2/src/engine/truncate_test.rs
index c39f3a0e374b..03386369bd2d 100644
--- a/src/mito2/src/engine/truncate_test.rs
+++ b/src/mito2/src/engine/truncate_test.rs
@@ -270,7 +270,7 @@ async fn test_engine_truncate_during_flush() {
     let engine = env
         .create_engine_with(
             MitoConfig::default(),
-            write_buffer_manager.clone(),
+            Some(write_buffer_manager),
             Some(listener.clone()),
         )
         .await;
diff --git a/src/mito2/src/region/version.rs b/src/mito2/src/region/version.rs
index c7d84fd913df..471a85c1e38f 100644
--- a/src/mito2/src/region/version.rs
+++ b/src/mito2/src/region/version.rs
@@ -115,10 +115,17 @@ impl VersionControl {
     }
 
     /// Mark all opened files as deleted and set the delete marker in [VersionControlData]
-    pub(crate) fn mark_dropped(&self) {
+    pub(crate) fn mark_dropped(&self, memtable_builder: &MemtableBuilderRef) {
+        let version = self.current().version;
+        let new_mutable = memtable_builder.build(&version.metadata);
+
         let mut data = self.data.write().unwrap();
         data.is_dropped = true;
         data.version.ssts.mark_all_deleted();
+        // Reset version so we can release the reference to memtables and SSTs.
+        let new_version =
+            Arc::new(VersionBuilder::new(version.metadata.clone(), new_mutable).build());
+        data.version = new_version;
     }
 
     /// Alter schema of the region.
diff --git a/src/mito2/src/sst/version.rs b/src/mito2/src/sst/version.rs
index b066af7ba3b6..1f56c8b940d8 100644
--- a/src/mito2/src/sst/version.rs
+++ b/src/mito2/src/sst/version.rs
@@ -75,9 +75,9 @@ impl SstVersion {
         }
     }
 
-    /// Mark all SSTs in this version as deleted.
+    /// Marks all SSTs in this version as deleted.
     pub(crate) fn mark_all_deleted(&self) {
-        for level_meta in self.levels.iter() {
+        for level_meta in &self.levels {
             for file_handle in level_meta.files.values() {
                 file_handle.mark_deleted();
             }
diff --git a/src/mito2/src/test_util.rs b/src/mito2/src/test_util.rs
index 2ad545825de2..25994fa0e601 100644
--- a/src/mito2/src/test_util.rs
+++ b/src/mito2/src/test_util.rs
@@ -120,7 +120,7 @@ impl TestEnv {
     pub async fn create_engine_with(
         &mut self,
         config: MitoConfig,
-        manager: WriteBufferManagerRef,
+        manager: Option<WriteBufferManagerRef>,
         listener: Option<EventListenerRef>,
     ) -> MitoEngine {
         let (log_store, object_store) = self.create_log_and_object_store().await;
diff --git a/src/mito2/src/worker.rs b/src/mito2/src/worker.rs
index 4b119c69959f..6b3ea02c1896 100644
--- a/src/mito2/src/worker.rs
+++ b/src/mito2/src/worker.rs
@@ -28,6 +28,7 @@ use std::collections::hash_map::DefaultHasher;
 use std::hash::{Hash, Hasher};
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::Arc;
+use std::time::Duration;
 
 use common_runtime::JoinHandle;
 use common_telemetry::{error, info, warn};
@@ -203,11 +204,16 @@ impl WorkerGroup {
         config: MitoConfig,
         log_store: Arc<S>,
         object_store: ObjectStore,
-        write_buffer_manager: WriteBufferManagerRef,
+        write_buffer_manager: Option<WriteBufferManagerRef>,
         listener: Option<crate::engine::listener::EventListenerRef>,
     ) -> WorkerGroup {
         assert!(config.num_workers.is_power_of_two());
         let config = Arc::new(config);
+        let write_buffer_manager = write_buffer_manager.unwrap_or_else(|| {
+            Arc::new(WriteBufferManagerImpl::new(
+                config.global_write_buffer_size.as_bytes() as usize,
+            ))
+        });
         let scheduler = Arc::new(LocalScheduler::new(config.max_background_jobs));
         let cache_manager = Arc::new(CacheManager::new(config.sst_meta_cache_size.as_bytes()));
 
@@ -608,6 +614,27 @@ impl WorkerListener {
         // Avoid compiler warning.
         let _ = region_id;
     }
+
+    pub(crate) fn on_later_drop_begin(&self, region_id: RegionId) -> Option<Duration> {
+        #[cfg(test)]
+        if let Some(listener) = &self.listener {
+            return listener.on_later_drop_begin(region_id);
+        }
+        // Avoid compiler warning.
+        let _ = region_id;
+        None
+    }
+
+    /// On later drop task is finished.
+    pub(crate) fn on_later_drop_end(&self, region_id: RegionId, removed: bool) {
+        #[cfg(test)]
+        if let Some(listener) = &self.listener {
+            listener.on_later_drop_end(region_id, removed);
+        }
+        // Avoid compiler warning.
+        let _ = region_id;
+        let _ = removed;
+    }
 }
 
 #[cfg(test)]
diff --git a/src/mito2/src/worker/handle_drop.rs b/src/mito2/src/worker/handle_drop.rs
index f7aa5d15dc6a..0b0431180fd0 100644
--- a/src/mito2/src/worker/handle_drop.rs
+++ b/src/mito2/src/worker/handle_drop.rs
@@ -56,7 +56,7 @@ impl<S> RegionWorkerLoop<S> {
         self.compaction_scheduler.on_region_dropped(region_id);
 
         // mark region version as dropped
-        region.version_control.mark_dropped();
+        region.version_control.mark_dropped(&self.memtable_builder);
         info!(
             "Region {} is dropped logically, but some files are not deleted yet",
             region_id
@@ -66,8 +66,20 @@ impl<S> RegionWorkerLoop<S> {
         let region_dir = region.access_layer.region_dir().to_owned();
         let object_store = self.object_store.clone();
         let dropping_regions = self.dropping_regions.clone();
+        let listener = self.listener.clone();
         common_runtime::spawn_bg(async move {
-            later_drop_task(region_id, region_dir, object_store, dropping_regions).await;
+            let gc_duration = listener
+                .on_later_drop_begin(region_id)
+                .unwrap_or(Duration::from_secs(GC_TASK_INTERVAL_SEC));
+            let removed = later_drop_task(
+                region_id,
+                region_dir,
+                object_store,
+                dropping_regions,
+                gc_duration,
+            )
+            .await;
+            listener.on_later_drop_end(region_id, removed);
         });
 
         Ok(Output::AffectedRows(0))
@@ -75,7 +87,7 @@ impl<S> RegionWorkerLoop<S> {
 }
 
 /// Background GC task to remove the entire region path once it find there is no
-/// parquet file left.
+/// parquet file left. Returns whether the path is removed.
 ///
 /// This task will keep running until finished. Any resource captured by it will
 /// not be released before then. Be sure to only pass weak reference if something
@@ -85,18 +97,24 @@ async fn later_drop_task(
     region_path: String,
     object_store: ObjectStore,
     dropping_regions: RegionMapRef,
-) {
+    gc_duration: Duration,
+) -> bool {
     for _ in 0..MAX_RETRY_TIMES {
-        sleep(Duration::from_secs(GC_TASK_INTERVAL_SEC)).await;
+        sleep(gc_duration).await;
         let result = remove_region_dir_once(&region_path, &object_store).await;
-        if let Err(err) = result {
-            warn!(
-                "Error occurs during trying to GC region dir {}: {}",
-                region_path, err
-            );
-        } else {
-            dropping_regions.remove_region(region_id);
-            info!("Region {} is dropped", region_path);
+        match result {
+            Err(err) => {
+                warn!(
+                    "Error occurs during trying to GC region dir {}: {}",
+                    region_path, err
+                );
+            }
+            Ok(true) => {
+                dropping_regions.remove_region(region_id);
+                info!("Region {} is dropped", region_path);
+                return true;
+            }
+            Ok(false) => (),
         }
     }
 
@@ -104,13 +122,16 @@ async fn later_drop_task(
         "Failed to GC region dir {} after {} retries, giving up",
         region_path, MAX_RETRY_TIMES
     );
+
+    false
 }
 
 // TODO(ruihang): place the marker in a separate dir
+/// Removes region dir if there is no parquet files, returns whether the directory is removed.
 pub(crate) async fn remove_region_dir_once(
     region_path: &str,
     object_store: &ObjectStore,
-) -> Result<()> {
+) -> Result<bool> {
     // list all files under the given region path to check if there are un-deleted parquet files
     let mut has_parquet_file = false;
     // record all paths that neither ends with .parquet nor the marker file
@@ -143,6 +164,8 @@ pub(crate) async fn remove_region_dir_once(
             .remove_all(region_path)
             .await
             .context(OpenDalSnafu)?;
+        Ok(true)
+    } else {
+        Ok(false)
     }
-    Ok(())
 }

From efbd8571d91227a6ded3edf898c9a7c06410aac3 Mon Sep 17 00:00:00 2001
From: Ruihang Xia <waynestxia@gmail.com>
Date: Wed, 27 Sep 2023 11:07:03 +0800
Subject: [PATCH 31/36] refactor: simplify `warn!` and `error!` macros (#2503)

* refactor: simplify the error! and warn! macros

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>

* support display format

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>

* err.msg to err

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>

---------

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>
---
 src/common/telemetry/src/macros.rs | 54 +++++++++++++++++-------------
 1 file changed, 30 insertions(+), 24 deletions(-)

diff --git a/src/common/telemetry/src/macros.rs b/src/common/telemetry/src/macros.rs
index 4d5f6003b754..c4fd285b3356 100644
--- a/src/common/telemetry/src/macros.rs
+++ b/src/common/telemetry/src/macros.rs
@@ -38,35 +38,38 @@ macro_rules! error {
 
     // error!(e; target: "my_target", "a {} event", "log")
     ($e:expr; target: $target:expr, $($arg:tt)+) => ({
-        use $crate::common_error::ext::ErrorExt;
-        use $crate::common_error::ext::StackError;
-
-        let mut stacks = vec![];
-        $e.debug_fmt(0, &mut stacks);
-        let stacks = stacks.join("\n");
+        $crate::log!(
+            target: $target,
+            $crate::logging::Level::ERROR,
+            err = ?$e,
+            $($arg)+
+        )
+    });
 
+    // error!(%e; target: "my_target", "a {} event", "log")
+    (%$e:expr; target: $target:expr, $($arg:tt)+) => ({
         $crate::log!(
             target: $target,
             $crate::logging::Level::ERROR,
-            err.msg = %stacks,
-            err.code = %$e.status_code(),
+            err = %$e,
             $($arg)+
         )
     });
 
     // error!(e; "a {} event", "log")
     ($e:expr; $($arg:tt)+) => ({
-        use $crate::common_error::ext::ErrorExt;
-        use $crate::common_error::ext::StackError;
-
-        let mut stacks = vec![];
-        $e.debug_fmt(0, &mut stacks);
-        let stacks = stacks.join("\n");
+        $crate::log!(
+            $crate::logging::Level::ERROR,
+            err = ?$e,
+            $($arg)+
+        )
+    });
 
+    // error!(%e; "a {} event", "log")
+    (%$e:expr; $($arg:tt)+) => ({
         $crate::log!(
             $crate::logging::Level::ERROR,
-            err.msg = %stacks,
-            err.code = %$e.status_code(),
+            err = %$e,
             $($arg)+
         )
     });
@@ -87,17 +90,18 @@ macro_rules! warn {
 
     // warn!(e; "a {} event", "log")
     ($e:expr; $($arg:tt)+) => ({
-        use $crate::common_error::ext::ErrorExt;
-        use $crate::common_error::ext::StackError;
-
-        let mut stacks = vec![];
-        $e.debug_fmt(0, &mut stacks);
-        let stacks = stacks.join("\n");
+        $crate::log!(
+            $crate::logging::Level::WARN,
+            err = ?$e,
+            $($arg)+
+        )
+    });
 
+    // warn!(%e; "a {} event", "log")
+    (%$e:expr; $($arg:tt)+) => ({
         $crate::log!(
             $crate::logging::Level::WARN,
-            err.msg = %$e,
-            err.code = %$e.status_code(),
+            err = %$e,
             $($arg)+
         )
     });
@@ -221,8 +225,10 @@ mod tests {
         error!(target: "my_target", "hello {}", "world");
         // Supports both owned and reference type.
         error!(err; target: "my_target", "hello {}", "world");
+        error!(%err; target: "my_target", "hello {}", "world");
         error!(err_ref; target: "my_target", "hello {}", "world");
         error!(err_ref2; "hello {}", "world");
+        error!(%err_ref2; "hello {}", "world");
         error!("hello {}", "world");
 
         let root_err = MockError::with_source(err);

From 5d8f46f24aa1d5b370a96f35a04c27e13ee9ca81 Mon Sep 17 00:00:00 2001
From: Weny Xu <wenymedia@gmail.com>
Date: Wed, 27 Sep 2023 13:00:16 +0900
Subject: [PATCH 32/36] fix: re-create heartbeat stream ASAP (#2499)

* chore: set default connect_timeout_millis to 1000

* fix: re-create heartbeat stream ASAP

* chore: apply suggestions
---
 config/datanode.example.toml           |  2 +-
 config/frontend.example.toml           |  2 +-
 src/common/grpc/src/channel_manager.rs |  2 +-
 src/datanode/src/heartbeat.rs          | 30 +++++++++++++++++++++++---
 src/meta-client/src/lib.rs             |  2 +-
 5 files changed, 31 insertions(+), 7 deletions(-)

diff --git a/config/datanode.example.toml b/config/datanode.example.toml
index 2202eae30e4b..4f52380d658f 100644
--- a/config/datanode.example.toml
+++ b/config/datanode.example.toml
@@ -25,7 +25,7 @@ heartbeat_timeout_millis = 500
 # Operation timeout in milliseconds, 3000 by default.
 timeout_millis = 3000
 # Connect server timeout in milliseconds, 5000 by default.
-connect_timeout_millis = 5000
+connect_timeout_millis = 1000
 # `TCP_NODELAY` option for accepted connections, true by default.
 tcp_nodelay = true
 
diff --git a/config/frontend.example.toml b/config/frontend.example.toml
index c87dc8292c1d..e87936551f40 100644
--- a/config/frontend.example.toml
+++ b/config/frontend.example.toml
@@ -62,7 +62,7 @@ metasrv_addrs = ["127.0.0.1:3002"]
 timeout_millis = 3000
 # DDL timeouts options.
 ddl_timeout_millis = 10000
-connect_timeout_millis = 5000
+connect_timeout_millis = 1000
 tcp_nodelay = true
 
 # Log options, see `standalone.example.toml`
diff --git a/src/common/grpc/src/channel_manager.rs b/src/common/grpc/src/channel_manager.rs
index f52177e2890d..d1e372b677c9 100644
--- a/src/common/grpc/src/channel_manager.rs
+++ b/src/common/grpc/src/channel_manager.rs
@@ -30,7 +30,7 @@ use crate::error::{CreateChannelSnafu, InvalidConfigFilePathSnafu, InvalidTlsCon
 
 const RECYCLE_CHANNEL_INTERVAL_SECS: u64 = 60;
 pub const DEFAULT_GRPC_REQUEST_TIMEOUT_SECS: u64 = 10;
-pub const DEFAULT_GRPC_CONNECT_TIMEOUT_SECS: u64 = 10;
+pub const DEFAULT_GRPC_CONNECT_TIMEOUT_SECS: u64 = 1;
 pub const DEFAULT_MAX_GRPC_RECV_MESSAGE_SIZE: usize = 512 * 1024 * 1024;
 pub const DEFAULT_MAX_GRPC_SEND_MESSAGE_SIZE: usize = 512 * 1024 * 1024;
 
diff --git a/src/datanode/src/heartbeat.rs b/src/datanode/src/heartbeat.rs
index 5bf6fb7cbb3a..44b48a223675 100644
--- a/src/datanode/src/heartbeat.rs
+++ b/src/datanode/src/heartbeat.rs
@@ -18,6 +18,7 @@ use std::time::Duration;
 
 use api::v1::meta::{HeartbeatRequest, Peer, RegionStat, Role};
 use common_grpc::channel_manager::{ChannelConfig, ChannelManager};
+use common_meta::distributed_time_constants::META_KEEP_ALIVE_INTERVAL_SECS;
 use common_meta::heartbeat::handler::parse_mailbox_message::ParseMailboxMessageHandler;
 use common_meta::heartbeat::handler::{
     HandlerGroupExecutor, HeartbeatResponseHandlerContext, HeartbeatResponseHandlerExecutorRef,
@@ -97,6 +98,7 @@ impl HeartbeatTask {
         handler_executor: HeartbeatResponseHandlerExecutorRef,
         mailbox: MailboxRef,
         mut notify: Option<Arc<Notify>>,
+        quit_signal: Arc<Notify>,
     ) -> Result<HeartbeatSender> {
         let client_id = meta_client.id();
 
@@ -123,7 +125,8 @@ impl HeartbeatTask {
                     info!("Heartbeat task shutdown");
                 }
             }
-            info!("Heartbeat handling loop exit.")
+            quit_signal.notify_one();
+            info!("Heartbeat handling loop exit.");
         });
         Ok(tx)
     }
@@ -167,12 +170,15 @@ impl HeartbeatTask {
         let (outgoing_tx, mut outgoing_rx) = mpsc::channel(16);
         let mailbox = Arc::new(HeartbeatMailbox::new(outgoing_tx));
 
+        let quit_signal = Arc::new(tokio::sync::Notify::new());
+
         let mut tx = Self::create_streams(
             &meta_client,
             running.clone(),
             handler_executor.clone(),
             mailbox.clone(),
             notify,
+            quit_signal.clone(),
         )
         .await?;
 
@@ -187,7 +193,6 @@ impl HeartbeatTask {
         common_runtime::spawn_bg(async move {
             let sleep = tokio::time::sleep(Duration::from_millis(0));
             tokio::pin!(sleep);
-
             loop {
                 if !running.load(Ordering::Relaxed) {
                     info!("shutdown heartbeat task");
@@ -228,6 +233,11 @@ impl HeartbeatTask {
                         sleep.as_mut().reset(now + Duration::from_millis(interval));
                         Some(req)
                     }
+                    // If the heartbeat stream is broken, send a dummy heartbeat request to re-create the heartbeat stream.
+                    _ = quit_signal.notified() => {
+                        let req = HeartbeatRequest::default();
+                        Some(req)
+                    }
                 };
                 if let Some(req) = req {
                     debug!("Sending heartbeat request: {:?}", req);
@@ -239,6 +249,7 @@ impl HeartbeatTask {
                             handler_executor.clone(),
                             mailbox.clone(),
                             None,
+                            quit_signal.clone(),
                         )
                         .await
                         {
@@ -249,6 +260,13 @@ impl HeartbeatTask {
                                 sleep.as_mut().reset(Instant::now());
                             }
                             Err(e) => {
+                                // Before the META_LEASE_SECS expires,
+                                // any retries are meaningless, it always reads the old meta leader address.
+                                // Triggers to retry after META_KEEP_ALIVE_INTERVAL_SECS.
+                                sleep.as_mut().reset(
+                                    Instant::now()
+                                        + Duration::from_secs(META_KEEP_ALIVE_INTERVAL_SECS),
+                                );
                                 error!(e;"Failed to reconnect to metasrv!");
                             }
                         }
@@ -317,13 +335,19 @@ pub async fn new_metasrv_client(
         .timeout(Duration::from_millis(meta_config.timeout_millis))
         .connect_timeout(Duration::from_millis(meta_config.connect_timeout_millis))
         .tcp_nodelay(meta_config.tcp_nodelay);
-    let channel_manager = ChannelManager::with_config(config);
+    let channel_manager = ChannelManager::with_config(config.clone());
+    let heartbeat_channel_manager = ChannelManager::with_config(
+        config
+            .timeout(Duration::from_millis(meta_config.heartbeat_timeout_millis))
+            .connect_timeout(Duration::from_millis(meta_config.heartbeat_timeout_millis)),
+    );
 
     let mut meta_client = MetaClientBuilder::new(cluster_id, member_id, Role::Datanode)
         .enable_heartbeat()
         .enable_router()
         .enable_store()
         .channel_manager(channel_manager)
+        .heartbeat_channel_manager(heartbeat_channel_manager)
         .build();
     meta_client
         .start(&meta_config.metasrv_addrs)
diff --git a/src/meta-client/src/lib.rs b/src/meta-client/src/lib.rs
index ad18718040c3..9b3d5cfcf2a2 100644
--- a/src/meta-client/src/lib.rs
+++ b/src/meta-client/src/lib.rs
@@ -45,7 +45,7 @@ impl Default for MetaClientOptions {
             timeout_millis: 3_000u64,
             heartbeat_timeout_millis: default_heartbeat_timeout_millis(),
             ddl_timeout_millis: default_ddl_timeout_millis(),
-            connect_timeout_millis: 5_000u64,
+            connect_timeout_millis: 1_000u64,
             tcp_nodelay: true,
         }
     }

From cfbee033a8bcb96a6a032a0a7ebe8485daeb0459 Mon Sep 17 00:00:00 2001
From: shuiyisong <113876041+shuiyisong@users.noreply.github.com>
Date: Wed, 27 Sep 2023 18:40:25 +0800
Subject: [PATCH 33/36] chore: tune return msg (#2506)

* chore: test return msg

* fix: test_child_error

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>

* chore: fix test

* chore: minor fix grpc return value

* chore: format return msg

* chore: use root error as return value

* chore: fix empty err display

* chore: iter through external error

* chore: remove err msg

* chore: remove unused field

---------

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>
Co-authored-by: Ruihang Xia <waynestxia@gmail.com>
---
 src/catalog/src/error.rs                   |  3 +-
 src/cmd/src/cli/repl.rs                    |  4 +-
 src/common/error/src/ext.rs                | 37 +++++++++++++--
 src/common/error/src/lib.rs                |  1 +
 src/common/procedure/src/local/runner.rs   | 12 +----
 src/common/recordbatch/src/error.rs        |  2 +-
 src/query/src/datafusion.rs                | 32 ++++---------
 src/query/src/datafusion/error.rs          |  3 +-
 src/query/src/error.rs                     |  5 +-
 src/query/src/planner.rs                   | 11 ++---
 src/script/src/python/error.rs             |  2 +-
 src/servers/src/error.rs                   | 11 ++---
 src/servers/src/grpc/prom_query_gateway.rs |  2 +-
 src/servers/src/http.rs                    |  7 +--
 src/servers/src/http/opentsdb.rs           | 14 +++---
 src/servers/src/http/prometheus.rs         | 16 +++----
 src/servers/src/mysql/writer.rs            |  7 ++-
 src/servers/src/opentsdb/connection.rs     |  4 +-
 src/servers/src/opentsdb/handler.rs        |  5 +-
 src/servers/src/postgres/handler.rs        |  4 +-
 src/servers/tests/http/opentsdb_test.rs    |  2 +-
 src/sql/src/error.rs                       |  3 +-
 src/sql/src/parser.rs                      | 10 ++--
 src/sql/src/parsers/alter_parser.rs        | 10 ++--
 src/sql/src/parsers/copy_parser.rs         | 12 ++---
 src/sql/src/parsers/create_parser.rs       | 53 +++++++---------------
 src/sql/src/parsers/delete_parser.rs       |  5 +-
 src/sql/src/parsers/insert_parser.rs       |  5 +-
 src/sql/src/parsers/query_parser.rs        | 12 ++---
 src/sql/src/parsers/tql_parser.rs          | 26 +++--------
 src/table/src/error.rs                     |  2 +-
 tests/runner/src/env.rs                    |  7 ++-
 32 files changed, 138 insertions(+), 191 deletions(-)

diff --git a/src/catalog/src/error.rs b/src/catalog/src/error.rs
index 599795337ef4..05c675d29566 100644
--- a/src/catalog/src/error.rs
+++ b/src/catalog/src/error.rs
@@ -216,9 +216,8 @@ pub enum Error {
     #[snafu(display("Illegal access to catalog: {} and schema: {}", catalog, schema))]
     QueryAccessDenied { catalog: String, schema: String },
 
-    #[snafu(display("msg: {}", msg))]
+    #[snafu(display(""))]
     Datafusion {
-        msg: String,
         #[snafu(source)]
         error: DataFusionError,
         location: Location,
diff --git a/src/cmd/src/cli/repl.rs b/src/cmd/src/cli/repl.rs
index c5f256a3c5b5..8fd49f7d27b6 100644
--- a/src/cmd/src/cli/repl.rs
+++ b/src/cmd/src/cli/repl.rs
@@ -35,7 +35,7 @@ use query::QueryEngine;
 use rustyline::error::ReadlineError;
 use rustyline::Editor;
 use session::context::QueryContext;
-use snafu::{ErrorCompat, ResultExt};
+use snafu::ResultExt;
 use substrait::{DFLogicalSubstraitConvertor, SubstraitPlan};
 
 use crate::cli::cmd::ReplCommand;
@@ -148,7 +148,7 @@ impl Repl {
             .await
             .map_err(|e| {
                 let status_code = e.status_code();
-                let root_cause = e.iter_chain().last().unwrap();
+                let root_cause = e.output_msg();
                 println!("Error: {}({status_code}), {root_cause}", status_code as u32)
             })
             .is_ok()
diff --git a/src/common/error/src/ext.rs b/src/common/error/src/ext.rs
index b840b2911ea0..fd6d04b6778b 100644
--- a/src/common/error/src/ext.rs
+++ b/src/common/error/src/ext.rs
@@ -18,7 +18,7 @@ use std::sync::Arc;
 use crate::status_code::StatusCode;
 
 /// Extension to [`Error`](std::error::Error) in std.
-pub trait ErrorExt: std::error::Error + StackError {
+pub trait ErrorExt: StackError {
     /// Map this error to [StatusCode].
     fn status_code(&self) -> StatusCode {
         StatusCode::Unknown
@@ -34,12 +34,43 @@ pub trait ErrorExt: std::error::Error + StackError {
     /// Returns the error as [Any](std::any::Any) so that it can be
     /// downcast to a specific implementation.
     fn as_any(&self) -> &dyn Any;
+
+    fn output_msg(&self) -> String
+    where
+        Self: Sized,
+    {
+        let error = self.last();
+        if let Some(external_error) = error.source() {
+            let external_root = external_error.sources().last().unwrap();
+
+            if error.to_string().is_empty() {
+                format!("{external_root}")
+            } else {
+                format!("{error}: {external_root}")
+            }
+        } else {
+            format!("{error}")
+        }
+    }
 }
 
-pub trait StackError {
+pub trait StackError: std::error::Error {
     fn debug_fmt(&self, layer: usize, buf: &mut Vec<String>);
 
     fn next(&self) -> Option<&dyn StackError>;
+
+    fn last(&self) -> &dyn StackError
+    where
+        Self: Sized,
+    {
+        let Some(mut result) = self.next() else {
+            return self;
+        };
+        while let Some(err) = result.next() {
+            result = err;
+        }
+        result
+    }
 }
 
 impl<T: ?Sized + StackError> StackError for Arc<T> {
@@ -52,7 +83,7 @@ impl<T: ?Sized + StackError> StackError for Arc<T> {
     }
 }
 
-impl<T: ?Sized + StackError> StackError for Box<T> {
+impl<T: StackError> StackError for Box<T> {
     fn debug_fmt(&self, layer: usize, buf: &mut Vec<String>) {
         self.as_ref().debug_fmt(layer, buf)
     }
diff --git a/src/common/error/src/lib.rs b/src/common/error/src/lib.rs
index 8a28561ac37a..2623cf3194db 100644
--- a/src/common/error/src/lib.rs
+++ b/src/common/error/src/lib.rs
@@ -11,6 +11,7 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
+#![feature(error_iter)]
 
 pub mod ext;
 pub mod format;
diff --git a/src/common/procedure/src/local/runner.rs b/src/common/procedure/src/local/runner.rs
index 0acc403d687e..3e997562663a 100644
--- a/src/common/procedure/src/local/runner.rs
+++ b/src/common/procedure/src/local/runner.rs
@@ -453,14 +453,13 @@ mod tests {
     use std::sync::Arc;
 
     use async_trait::async_trait;
-    use common_error::ext::PlainError;
+    use common_error::ext::{ErrorExt, PlainError};
     use common_error::mock::MockError;
     use common_error::status_code::StatusCode;
     use common_test_util::temp_dir::create_temp_dir;
     use futures_util::future::BoxFuture;
     use futures_util::FutureExt;
     use object_store::ObjectStore;
-    use snafu::ErrorCompat;
 
     use super::*;
     use crate::local::test_util;
@@ -943,14 +942,7 @@ mod tests {
 
         // Run the runner and execute the procedure.
         runner.run().await;
-        let err = meta
-            .state()
-            .error()
-            .unwrap()
-            .iter_chain()
-            .last()
-            .unwrap()
-            .to_string();
+        let err = meta.state().error().unwrap().output_msg();
         assert!(err.contains("subprocedure failed"), "{err}");
     }
 }
diff --git a/src/common/recordbatch/src/error.rs b/src/common/recordbatch/src/error.rs
index 92dbbedc6a72..e5992c37d916 100644
--- a/src/common/recordbatch/src/error.rs
+++ b/src/common/recordbatch/src/error.rs
@@ -55,7 +55,7 @@ pub enum Error {
         location: Location,
     },
 
-    #[snafu(display("Failed to poll stream"))]
+    #[snafu(display(""))]
     PollStream {
         #[snafu(source)]
         error: datafusion::error::DataFusionError,
diff --git a/src/query/src/datafusion.rs b/src/query/src/datafusion.rs
index 3928a630c92b..f8c8ed25794a 100644
--- a/src/query/src/datafusion.rs
+++ b/src/query/src/datafusion.rs
@@ -277,9 +277,7 @@ impl QueryEngine for DatafusionQueryEngine {
         Ok(DataFrame::DataFusion(
             self.state
                 .read_table(table)
-                .context(error::DatafusionSnafu {
-                    msg: "Fail to create dataframe for table",
-                })
+                .context(error::DatafusionSnafu)
                 .map_err(BoxedError::new)
                 .context(QueryExecutionSnafu)?,
         ))
@@ -295,9 +293,7 @@ impl LogicalOptimizer for DatafusionQueryEngine {
                     .state
                     .session_state()
                     .optimize(df_plan)
-                    .context(error::DatafusionSnafu {
-                        msg: "Fail to optimize logical plan",
-                    })
+                    .context(error::DatafusionSnafu)
                     .map_err(BoxedError::new)
                     .context(QueryExecutionSnafu)?;
 
@@ -321,9 +317,7 @@ impl PhysicalPlanner for DatafusionQueryEngine {
                 let physical_plan = state
                     .create_physical_plan(df_plan)
                     .await
-                    .context(error::DatafusionSnafu {
-                        msg: "Fail to create physical plan",
-                    })
+                    .context(error::DatafusionSnafu)
                     .map_err(BoxedError::new)
                     .context(QueryExecutionSnafu)?;
 
@@ -394,9 +388,7 @@ impl QueryExecutor for DatafusionQueryEngine {
                 assert_eq!(1, plan.output_partitioning().partition_count());
                 let df_stream = plan
                     .execute(0, task_ctx)
-                    .context(error::DatafusionSnafu {
-                        msg: "Failed to execute DataFusion merge exec",
-                    })
+                    .context(error::DatafusionSnafu)
                     .map_err(BoxedError::new)
                     .context(QueryExecutionSnafu)?;
                 let stream = RecordBatchStreamAdapter::try_new(df_stream)
@@ -447,35 +439,27 @@ pub async fn execute_show_with_filter(
     let context = SessionContext::new();
     context
         .register_batch(table_name, record_batch.into_df_record_batch())
-        .context(error::DatafusionSnafu {
-            msg: "Fail to register a record batch as a table",
-        })
+        .context(error::DatafusionSnafu)
         .map_err(BoxedError::new)
         .context(QueryExecutionSnafu)?;
     let mut dataframe = context
         .sql(&format!("SELECT * FROM {table_name}"))
         .await
-        .context(error::DatafusionSnafu {
-            msg: "Fail to execute a sql",
-        })
+        .context(error::DatafusionSnafu)
         .map_err(BoxedError::new)
         .context(QueryExecutionSnafu)?;
     if let Some(filter) = filter {
         let filter = convert_filter_to_df_filter(filter)?;
         dataframe = dataframe
             .filter(filter)
-            .context(error::DatafusionSnafu {
-                msg: "Fail to filter",
-            })
+            .context(error::DatafusionSnafu)
             .map_err(BoxedError::new)
             .context(QueryExecutionSnafu)?
     }
     let df_batches = dataframe
         .collect()
         .await
-        .context(error::DatafusionSnafu {
-            msg: "Fail to collect the record batches",
-        })
+        .context(error::DatafusionSnafu)
         .map_err(BoxedError::new)
         .context(QueryExecutionSnafu)?;
     let mut batches = Vec::with_capacity(df_batches.len());
diff --git a/src/query/src/datafusion/error.rs b/src/query/src/datafusion/error.rs
index 486172cf70e5..d94fb0d89006 100644
--- a/src/query/src/datafusion/error.rs
+++ b/src/query/src/datafusion/error.rs
@@ -25,9 +25,8 @@ use snafu::{Location, Snafu};
 #[snafu(visibility(pub))]
 #[stack_trace_debug]
 pub enum InnerError {
-    #[snafu(display("msg: {}", msg))]
+    #[snafu(display(""))]
     Datafusion {
-        msg: &'static str,
         #[snafu(source)]
         error: DataFusionError,
         location: Location,
diff --git a/src/query/src/error.rs b/src/query/src/error.rs
index 639bb3e78f5b..5449ece0b1f7 100644
--- a/src/query/src/error.rs
+++ b/src/query/src/error.rs
@@ -121,7 +121,7 @@ pub enum Error {
         location: Location,
     },
 
-    #[snafu(display("DataFusion error"))]
+    #[snafu(display(""))]
     DataFusion {
         #[snafu(source)]
         error: DataFusionError,
@@ -140,9 +140,8 @@ pub enum Error {
         source: sql::error::Error,
     },
 
-    #[snafu(display("Cannot plan SQL: {}", sql))]
+    #[snafu(display(""))]
     PlanSql {
-        sql: String,
         #[snafu(source)]
         error: DataFusionError,
         location: Location,
diff --git a/src/query/src/planner.rs b/src/query/src/planner.rs
index df54405a0e20..5fe7949948d9 100644
--- a/src/query/src/planner.rs
+++ b/src/query/src/planner.rs
@@ -76,14 +76,9 @@ impl DfLogicalPlanner {
 
         let sql_to_rel = SqlToRel::new_with_options(&context_provider, parser_options);
 
-        let result = sql_to_rel.statement_to_plan(df_stmt).with_context(|_| {
-            let sql = if let Statement::Query(query) = stmt {
-                query.inner.to_string()
-            } else {
-                format!("{stmt:?}")
-            };
-            PlanSqlSnafu { sql }
-        })?;
+        let result = sql_to_rel
+            .statement_to_plan(df_stmt)
+            .context(PlanSqlSnafu)?;
         let plan = RangePlanRewriter::new(table_provider, context_provider)
             .rewrite(result)
             .await?;
diff --git a/src/script/src/python/error.rs b/src/script/src/python/error.rs
index ad00998f3a4c..376f734af693 100644
--- a/src/script/src/python/error.rs
+++ b/src/script/src/python/error.rs
@@ -76,7 +76,7 @@ pub enum Error {
         error: ArrowError,
     },
 
-    #[snafu(display("DataFusion error"))]
+    #[snafu(display(""))]
     DataFusion {
         location: SnafuLocation,
         #[snafu(source)]
diff --git a/src/servers/src/error.rs b/src/servers/src/error.rs
index c1a34177e7eb..4ff5b903a545 100644
--- a/src/servers/src/error.rs
+++ b/src/servers/src/error.rs
@@ -27,7 +27,7 @@ use common_telemetry::logging;
 use datatypes::prelude::ConcreteDataType;
 use query::parser::PromQuery;
 use serde_json::json;
-use snafu::{ErrorCompat, Location, Snafu};
+use snafu::{Location, Snafu};
 use tonic::Code;
 
 #[derive(Snafu)]
@@ -511,7 +511,6 @@ macro_rules! define_into_tonic_status {
         impl From<$Error> for tonic::Status {
             fn from(err: $Error) -> Self {
                 use common_error::{GREPTIME_ERROR_CODE, GREPTIME_ERROR_MSG};
-                use snafu::ErrorCompat;
                 use tonic::codegen::http::{HeaderMap, HeaderValue};
                 use tonic::metadata::MetadataMap;
 
@@ -521,16 +520,16 @@ macro_rules! define_into_tonic_status {
                 // (which is a very rare case), just ignore. Client will use Tonic status code and message.
                 let status_code = err.status_code();
                 headers.insert(GREPTIME_ERROR_CODE, HeaderValue::from(status_code as u32));
-                let root_error = err.iter_chain().last().unwrap();
+                let root_error = err.output_msg();
 
-                if let Ok(err_msg) = HeaderValue::from_bytes(root_error.to_string().as_bytes()) {
+                if let Ok(err_msg) = HeaderValue::from_bytes(root_error.as_bytes()) {
                     let _ = headers.insert(GREPTIME_ERROR_MSG, err_msg);
                 }
 
                 let metadata = MetadataMap::from_headers(headers);
                 tonic::Status::with_metadata(
                     $crate::error::status_to_tonic_code(status_code),
-                    err.to_string(),
+                    root_error,
                     metadata,
                 )
             }
@@ -548,7 +547,7 @@ impl From<std::io::Error> for Error {
 
 impl IntoResponse for Error {
     fn into_response(self) -> Response {
-        let error_msg = self.iter_chain().last().unwrap().to_string();
+        let error_msg = self.output_msg();
         let status = match self {
             Error::InfluxdbLineProtocol { .. }
             | Error::InfluxdbLinesWrite { .. }
diff --git a/src/servers/src/grpc/prom_query_gateway.rs b/src/servers/src/grpc/prom_query_gateway.rs
index e41960d68b29..cff82f5265b1 100644
--- a/src/servers/src/grpc/prom_query_gateway.rs
+++ b/src/servers/src/grpc/prom_query_gateway.rs
@@ -124,7 +124,7 @@ impl PrometheusGatewayService {
                 Err(err) => {
                     return PrometheusJsonResponse::error(
                         err.status_code().to_string(),
-                        err.to_string(),
+                        err.output_msg(),
                     )
                     .0
                 }
diff --git a/src/servers/src/http.rs b/src/servers/src/http.rs
index e599c724dcbe..0a8dd839fa27 100644
--- a/src/servers/src/http.rs
+++ b/src/servers/src/http.rs
@@ -52,7 +52,7 @@ use futures::FutureExt;
 use schemars::JsonSchema;
 use serde::{Deserialize, Serialize};
 use serde_json::Value;
-use snafu::{ensure, ErrorCompat, ResultExt};
+use snafu::{ensure, ResultExt};
 use tokio::sync::oneshot::{self, Sender};
 use tokio::sync::Mutex;
 use tower::timeout::TimeoutLayer;
@@ -314,10 +314,7 @@ impl JsonResponse {
                     }
                 },
                 Err(e) => {
-                    return Self::with_error(
-                        e.iter_chain().last().unwrap().to_string(),
-                        e.status_code(),
-                    );
+                    return Self::with_error(e.output_msg(), e.status_code());
                 }
             }
         }
diff --git a/src/servers/src/http/opentsdb.rs b/src/servers/src/http/opentsdb.rs
index 14cbfc8595fe..c5b90b42a438 100644
--- a/src/servers/src/http/opentsdb.rs
+++ b/src/servers/src/http/opentsdb.rs
@@ -17,12 +17,13 @@ use std::collections::HashMap;
 use axum::extract::{Query, RawBody, State};
 use axum::http::StatusCode as HttpStatusCode;
 use axum::{Extension, Json};
+use common_error::ext::ErrorExt;
 use hyper::Body;
 use serde::{Deserialize, Serialize};
 use session::context::QueryContextRef;
-use snafu::{ErrorCompat, ResultExt};
+use snafu::ResultExt;
 
-use crate::error::{self, Error, Result};
+use crate::error::{self, Result};
 use crate::opentsdb::codec::DataPoint;
 use crate::query_handler::OpentsdbProtocolHandlerRef;
 
@@ -154,13 +155,13 @@ impl OpentsdbDebuggingResponse {
         self.success += 1;
     }
 
-    fn on_failed(&mut self, datapoint: DataPointRequest, error: Error) {
+    fn on_failed(&mut self, datapoint: DataPointRequest, error: impl ErrorExt) {
         self.failed += 1;
 
         if let Some(details) = self.errors.as_mut() {
             let error = OpentsdbDetailError {
                 datapoint,
-                error: error.iter_chain().last().unwrap().to_string(),
+                error: error.output_msg(),
             };
             details.push(error);
         };
@@ -169,7 +170,6 @@ impl OpentsdbDebuggingResponse {
 
 #[cfg(test)]
 mod test {
-    use snafu::ErrorCompat;
 
     use super::*;
 
@@ -229,13 +229,13 @@ mod test {
         let body = Body::from("");
         let result = parse_data_points(body).await;
         assert!(result.is_err());
-        let err = result.unwrap_err().iter_chain().last().unwrap().to_string();
+        let err = result.unwrap_err().output_msg();
         assert!(err.contains("EOF while parsing a value at line 1 column 0"));
 
         let body = Body::from("hello world");
         let result = parse_data_points(body).await;
         assert!(result.is_err());
-        let err = result.unwrap_err().iter_chain().last().unwrap().to_string();
+        let err = result.unwrap_err().output_msg();
         assert!(err.contains("expected value at line 1 column 1"));
     }
 }
diff --git a/src/servers/src/http/prometheus.rs b/src/servers/src/http/prometheus.rs
index 9f8e652cc5e6..b2c96b129357 100644
--- a/src/servers/src/http/prometheus.rs
+++ b/src/servers/src/http/prometheus.rs
@@ -162,7 +162,7 @@ impl PrometheusJsonResponse {
                         ..Default::default()
                     }))
                 } else {
-                    Self::error(err.status_code().to_string(), err.to_string())
+                    Self::error(err.status_code().to_string(), err.output_msg())
                 }
             }
         }
@@ -323,7 +323,7 @@ pub async fn instant_query(
     let (metric_name, result_type) = match retrieve_metric_name_and_result_type(&prom_query.query) {
         Ok((metric_name, result_type)) => (metric_name.unwrap_or_default(), result_type),
         Err(err) => {
-            return PrometheusJsonResponse::error(err.status_code().to_string(), err.to_string())
+            return PrometheusJsonResponse::error(err.status_code().to_string(), err.output_msg())
         }
     };
     PrometheusJsonResponse::from_query_result(result, metric_name, result_type).await
@@ -357,7 +357,7 @@ pub async fn range_query(
     let result = handler.do_query(&prom_query, query_ctx).await;
     let metric_name = match retrieve_metric_name_and_result_type(&prom_query.query) {
         Err(err) => {
-            return PrometheusJsonResponse::error(err.status_code().to_string(), err.to_string())
+            return PrometheusJsonResponse::error(err.status_code().to_string(), err.output_msg())
         }
         Ok((metric_name, _)) => metric_name.unwrap_or_default(),
     };
@@ -430,7 +430,7 @@ pub async fn labels_query(
                 return PrometheusJsonResponse::success(PrometheusResponse::Labels(labels))
             }
             Err(e) => {
-                return PrometheusJsonResponse::error(e.status_code().to_string(), e.to_string())
+                return PrometheusJsonResponse::error(e.status_code().to_string(), e.output_msg())
             }
         }
     }
@@ -466,7 +466,7 @@ pub async fn labels_query(
             {
                 return PrometheusJsonResponse::error(
                     err.status_code().to_string(),
-                    err.to_string(),
+                    err.output_msg(),
                 );
             }
         }
@@ -690,7 +690,7 @@ pub async fn label_values_query(
         let mut table_names = match handler.catalog_manager().table_names(catalog, schema).await {
             Ok(table_names) => table_names,
             Err(e) => {
-                return PrometheusJsonResponse::error(e.status_code().to_string(), e.to_string());
+                return PrometheusJsonResponse::error(e.status_code().to_string(), e.output_msg());
             }
         };
         table_names.sort_unstable();
@@ -723,7 +723,7 @@ pub async fn label_values_query(
             {
                 return PrometheusJsonResponse::error(
                     err.status_code().to_string(),
-                    err.to_string(),
+                    err.output_msg(),
                 );
             }
         }
@@ -837,7 +837,7 @@ pub async fn series_query(
         let result = handler.do_query(&prom_query, query_ctx.clone()).await;
         if let Err(err) = retrieve_series_from_query_result(result, &mut series, &table_name).await
         {
-            return PrometheusJsonResponse::error(err.status_code().to_string(), err.to_string());
+            return PrometheusJsonResponse::error(err.status_code().to_string(), err.output_msg());
         }
     }
     PrometheusJsonResponse::success(PrometheusResponse::Series(series))
diff --git a/src/servers/src/mysql/writer.rs b/src/servers/src/mysql/writer.rs
index bfbb689aa869..b0eb5c3c4c6a 100644
--- a/src/servers/src/mysql/writer.rs
+++ b/src/servers/src/mysql/writer.rs
@@ -14,7 +14,7 @@
 
 use std::ops::Deref;
 
-use common_error::ext::BoxedError;
+use common_error::ext::{BoxedError, ErrorExt};
 use common_query::Output;
 use common_recordbatch::{RecordBatch, SendableRecordBatchStream};
 use datatypes::prelude::{ConcreteDataType, Value};
@@ -26,7 +26,6 @@ use opensrv_mysql::{
 };
 use session::context::QueryContextRef;
 use snafu::prelude::*;
-use snafu::ErrorCompat;
 use tokio::io::AsyncWrite;
 
 use crate::error::{self, Error, OtherSnafu, Result};
@@ -205,14 +204,14 @@ impl<'a, W: AsyncWrite + Unpin> MysqlResultWriter<'a, W> {
         Ok(())
     }
 
-    async fn write_query_error(error: Error, w: QueryResultWriter<'a, W>) -> Result<()> {
+    async fn write_query_error(error: impl ErrorExt, w: QueryResultWriter<'a, W>) -> Result<()> {
         increment_counter!(
             METRIC_ERROR_COUNTER,
             &[(METRIC_PROTOCOL_LABEL, METRIC_ERROR_COUNTER_LABEL_MYSQL)]
         );
 
         let kind = ErrorKind::ER_INTERNAL_ERROR;
-        let error = error.iter_chain().last().unwrap().to_string();
+        let error = error.output_msg();
         w.error(kind, error.as_bytes()).await?;
         Ok(())
     }
diff --git a/src/servers/src/opentsdb/connection.rs b/src/servers/src/opentsdb/connection.rs
index e443febbf955..e5f15d3c159d 100644
--- a/src/servers/src/opentsdb/connection.rs
+++ b/src/servers/src/opentsdb/connection.rs
@@ -116,7 +116,7 @@ mod tests {
     use std::io::Write;
 
     use bytes::BufMut;
-    use snafu::ErrorCompat;
+    use common_error::ext::ErrorExt;
     use tokio_test::io::Builder;
 
     use super::*;
@@ -187,7 +187,7 @@ mod tests {
         buffer.writer().write_all(b"Hello Wor\xffld.\r\n").unwrap();
         let result = conn.parse_line();
         assert!(result.is_err());
-        let err = result.unwrap_err().iter_chain().last().unwrap().to_string();
+        let err = result.unwrap_err().output_msg();
         assert!(err.contains("invalid utf-8 sequence"));
     }
 
diff --git a/src/servers/src/opentsdb/handler.rs b/src/servers/src/opentsdb/handler.rs
index 9534f5bc00ba..4cbe1731fe11 100644
--- a/src/servers/src/opentsdb/handler.rs
+++ b/src/servers/src/opentsdb/handler.rs
@@ -14,6 +14,7 @@
 
 //! Modified from Tokio's mini-redis example.
 
+use common_error::ext::ErrorExt;
 use common_telemetry::timer;
 use session::context::QueryContextBuilder;
 use tokio::io::{AsyncRead, AsyncWrite};
@@ -95,11 +96,11 @@ impl<S: AsyncWrite + AsyncRead + Unpin> Handler<S> {
                     let _timer = timer!(crate::metrics::METRIC_TCP_OPENTSDB_LINE_WRITE_ELAPSED);
                     let result = self.query_handler.exec(&data_point, ctx.clone()).await;
                     if let Err(e) = result {
-                        self.connection.write_line(e.to_string()).await?;
+                        self.connection.write_line(e.output_msg()).await?;
                     }
                 }
                 Err(e) => {
-                    self.connection.write_line(e.to_string()).await?;
+                    self.connection.write_line(e.output_msg()).await?;
                 }
             }
         }
diff --git a/src/servers/src/postgres/handler.rs b/src/servers/src/postgres/handler.rs
index f11900133dd9..b72256d7e237 100644
--- a/src/servers/src/postgres/handler.rs
+++ b/src/servers/src/postgres/handler.rs
@@ -15,6 +15,7 @@
 use std::sync::Arc;
 
 use async_trait::async_trait;
+use common_error::ext::ErrorExt;
 use common_query::Output;
 use common_recordbatch::error::Result as RecordBatchResult;
 use common_recordbatch::RecordBatch;
@@ -31,7 +32,6 @@ use pgwire::api::{ClientInfo, Type};
 use pgwire::error::{ErrorInfo, PgWireError, PgWireResult};
 use query::query_engine::DescribeResult;
 use session::Session;
-use snafu::ErrorCompat;
 use sql::dialect::PostgreSqlDialect;
 use sql::parser::ParserContext;
 
@@ -91,7 +91,7 @@ fn output_to_query_response<'a>(
         Err(e) => Ok(Response::Error(Box::new(ErrorInfo::new(
             "ERROR".to_string(),
             "XX000".to_string(),
-            e.iter_chain().last().unwrap().to_string(),
+            e.output_msg(),
         )))),
     }
 }
diff --git a/src/servers/tests/http/opentsdb_test.rs b/src/servers/tests/http/opentsdb_test.rs
index 8548cf8a1384..e77143d3b3a1 100644
--- a/src/servers/tests/http/opentsdb_test.rs
+++ b/src/servers/tests/http/opentsdb_test.rs
@@ -153,7 +153,7 @@ async fn test_opentsdb_put() {
     assert_eq!(result.status(), 400);
     assert_eq!(
         result.text().await,
-        "{\"error\":\"expected value at line 1 column 1\"}"
+        "{\"error\":\"Invalid OpenTSDB Json request: expected value at line 1 column 1\"}"
     );
 
     // internal server error
diff --git a/src/sql/src/error.rs b/src/sql/src/error.rs
index 9fa62dc9075d..b78aa55c7509 100644
--- a/src/sql/src/error.rs
+++ b/src/sql/src/error.rs
@@ -59,9 +59,8 @@ pub enum Error {
     UnsupportedDefaultValue { column_name: String, expr: Expr },
 
     // Syntax error from sql parser.
-    #[snafu(display("Syntax error, sql: {}", sql))]
+    #[snafu(display(""))]
     Syntax {
-        sql: String,
         #[snafu(source)]
         error: ParserError,
     },
diff --git a/src/sql/src/parser.rs b/src/sql/src/parser.rs
index 913d2ce760f6..e048384dc265 100644
--- a/src/sql/src/parser.rs
+++ b/src/sql/src/parser.rs
@@ -37,7 +37,7 @@ impl<'a> ParserContext<'a> {
 
         let parser = Parser::new(dialect)
             .try_with_sql(sql)
-            .context(SyntaxSnafu { sql })?;
+            .context(SyntaxSnafu)?;
         let mut parser_ctx = ParserContext { sql, parser };
 
         let mut expecting_statement_delimiter = false;
@@ -67,12 +67,12 @@ impl<'a> ParserContext<'a> {
     pub fn parse_function(sql: &'a str, dialect: &dyn Dialect) -> Result<Expr> {
         let mut parser = Parser::new(dialect)
             .try_with_sql(sql)
-            .context(SyntaxSnafu { sql })?;
+            .context(SyntaxSnafu)?;
 
-        let function_name = parser.parse_identifier().context(SyntaxSnafu { sql })?;
+        let function_name = parser.parse_identifier().context(SyntaxSnafu)?;
         parser
             .parse_function(ObjectName(vec![function_name]))
-            .context(SyntaxSnafu { sql })
+            .context(SyntaxSnafu)
     }
 
     /// Parses parser context to a set of statements.
@@ -143,7 +143,7 @@ impl<'a> ParserContext<'a> {
         Err(ParserError::ParserError(format!(
             "Expected {expected}, found: {found}",
         )))
-        .context(SyntaxSnafu { sql: self.sql })
+        .context(SyntaxSnafu)
     }
 
     pub fn matches_keyword(&mut self, expected: Keyword) -> bool {
diff --git a/src/sql/src/parsers/alter_parser.rs b/src/sql/src/parsers/alter_parser.rs
index 5c66ca24ac6a..a5436d8f53b3 100644
--- a/src/sql/src/parsers/alter_parser.rs
+++ b/src/sql/src/parsers/alter_parser.rs
@@ -25,9 +25,7 @@ use crate::statements::statement::Statement;
 
 impl<'a> ParserContext<'a> {
     pub(crate) fn parse_alter(&mut self) -> Result<Statement> {
-        let alter_table = self
-            .parse_alter_table()
-            .context(error::SyntaxSnafu { sql: self.sql })?;
+        let alter_table = self.parse_alter_table().context(error::SyntaxSnafu)?;
         Ok(Statement::Alter(alter_table))
     }
 
@@ -98,7 +96,7 @@ impl<'a> ParserContext<'a> {
 mod tests {
     use std::assert_matches::assert_matches;
 
-    use snafu::ErrorCompat;
+    use common_error::ext::ErrorExt;
     use sqlparser::ast::{ColumnOption, DataType};
 
     use super::*;
@@ -215,7 +213,7 @@ mod tests {
     fn test_parse_alter_drop_column() {
         let sql = "ALTER TABLE my_metric_1 DROP a";
         let result = ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}).unwrap_err();
-        let err = result.iter_chain().last().unwrap().to_string();
+        let err = result.output_msg();
         assert!(err.contains("expect keyword COLUMN after ALTER TABLE DROP"));
 
         let sql = "ALTER TABLE my_metric_1 DROP COLUMN a";
@@ -245,7 +243,7 @@ mod tests {
     fn test_parse_alter_rename_table() {
         let sql = "ALTER TABLE test_table table_t";
         let result = ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}).unwrap_err();
-        let err = result.iter_chain().last().unwrap().to_string();
+        let err = result.output_msg();
         assert!(err.contains("expect keyword ADD or DROP or RENAME after ALTER TABLE"));
 
         let sql = "ALTER TABLE test_table RENAME table_t";
diff --git a/src/sql/src/parsers/copy_parser.rs b/src/sql/src/parsers/copy_parser.rs
index 34b3a80cfdf7..7b47c79783cd 100644
--- a/src/sql/src/parsers/copy_parser.rs
+++ b/src/sql/src/parsers/copy_parser.rs
@@ -57,7 +57,7 @@ impl<'a> ParserContext<'a> {
 
         self.parser
             .expect_keyword(Keyword::TO)
-            .context(error::SyntaxSnafu { sql: self.sql })?;
+            .context(error::SyntaxSnafu)?;
 
         let (with, connection, location) = self.parse_copy_to()?;
         Ok(CopyDatabaseArgument {
@@ -89,7 +89,7 @@ impl<'a> ParserContext<'a> {
         } else {
             self.parser
                 .expect_keyword(Keyword::FROM)
-                .context(error::SyntaxSnafu { sql: self.sql })?;
+                .context(error::SyntaxSnafu)?;
             Ok(CopyTable::From(self.parse_copy_table_from(table_name)?))
         }
     }
@@ -107,7 +107,7 @@ impl<'a> ParserContext<'a> {
         let options = self
             .parser
             .parse_options(Keyword::WITH)
-            .context(error::SyntaxSnafu { sql: self.sql })?;
+            .context(error::SyntaxSnafu)?;
 
         let with = options
             .into_iter()
@@ -119,7 +119,7 @@ impl<'a> ParserContext<'a> {
         let connection_options = self
             .parser
             .parse_options(Keyword::CONNECTION)
-            .context(error::SyntaxSnafu { sql: self.sql })?;
+            .context(error::SyntaxSnafu)?;
 
         let connection = connection_options
             .into_iter()
@@ -148,7 +148,7 @@ impl<'a> ParserContext<'a> {
         let options = self
             .parser
             .parse_options(Keyword::WITH)
-            .context(error::SyntaxSnafu { sql: self.sql })?;
+            .context(error::SyntaxSnafu)?;
 
         let with = options
             .into_iter()
@@ -160,7 +160,7 @@ impl<'a> ParserContext<'a> {
         let connection_options = self
             .parser
             .parse_options(Keyword::CONNECTION)
-            .context(error::SyntaxSnafu { sql: self.sql })?;
+            .context(error::SyntaxSnafu)?;
 
         let connection = connection_options
             .into_iter()
diff --git a/src/sql/src/parsers/create_parser.rs b/src/sql/src/parsers/create_parser.rs
index 33e6caf81dd0..6d01f8bacc32 100644
--- a/src/sql/src/parsers/create_parser.rs
+++ b/src/sql/src/parsers/create_parser.rs
@@ -69,7 +69,7 @@ impl<'a> ParserContext<'a> {
         let _ = self.parser.next_token();
         self.parser
             .expect_keyword(Keyword::TABLE)
-            .context(error::SyntaxSnafu { sql: self.sql })?;
+            .context(error::SyntaxSnafu)?;
         let if_not_exists =
             self.parser
                 .parse_keywords(&[Keyword::IF, Keyword::NOT, Keyword::EXISTS]);
@@ -86,7 +86,7 @@ impl<'a> ParserContext<'a> {
         let options = self
             .parser
             .parse_options(Keyword::WITH)
-            .context(error::SyntaxSnafu { sql: self.sql })?
+            .context(error::SyntaxSnafu)?
             .into_iter()
             .filter_map(|option| {
                 if let Some(v) = parse_option_string(option.value) {
@@ -159,7 +159,7 @@ impl<'a> ParserContext<'a> {
         let options = self
             .parser
             .parse_options(Keyword::WITH)
-            .context(error::SyntaxSnafu { sql: self.sql })?;
+            .context(error::SyntaxSnafu)?;
         for option in options.iter() {
             ensure!(
                 valid_table_option(&option.name.value),
@@ -200,7 +200,7 @@ impl<'a> ParserContext<'a> {
         let column_list = self
             .parser
             .parse_parenthesized_column_list(Mandatory, false)
-            .context(error::SyntaxSnafu { sql: self.sql })?;
+            .context(error::SyntaxSnafu)?;
 
         let entries = self.parse_comma_separated(Self::parse_partition_entry)?;
 
@@ -219,16 +219,13 @@ impl<'a> ParserContext<'a> {
                 actual: self.peek_token_as_string(),
             })?;
 
-        let name = self
-            .parser
-            .parse_identifier()
-            .context(error::SyntaxSnafu { sql: self.sql })?;
+        let name = self.parser.parse_identifier().context(error::SyntaxSnafu)?;
 
         self.parser
             .expect_keyword(Keyword::VALUES)
             .and_then(|_| self.parser.expect_token(&LESS))
             .and_then(|_| self.parser.expect_token(&THAN))
-            .context(error::SyntaxSnafu { sql: self.sql })?;
+            .context(error::SyntaxSnafu)?;
 
         let value_list = self.parse_comma_separated(Self::parse_value_list)?;
 
@@ -242,10 +239,7 @@ impl<'a> ParserContext<'a> {
                 let _ = self.parser.next_token();
                 SqlValue::Number(MAXVALUE.to_string(), false)
             }
-            _ => self
-                .parser
-                .parse_value()
-                .context(error::SyntaxSnafu { sql: self.sql })?,
+            _ => self.parser.parse_value().context(error::SyntaxSnafu)?,
         };
         Ok(value)
     }
@@ -320,9 +314,7 @@ impl<'a> ParserContext<'a> {
         columns: &mut Vec<ColumnDef>,
         constraints: &mut Vec<TableConstraint>,
     ) -> Result<()> {
-        let mut column = self
-            .parse_column_def()
-            .context(SyntaxSnafu { sql: self.sql })?;
+        let mut column = self.parse_column_def().context(SyntaxSnafu)?;
 
         let mut time_index_opt_idx = None;
         for (index, opt) in column.options.iter().enumerate() {
@@ -496,11 +488,7 @@ impl<'a> ParserContext<'a> {
 
     fn parse_optional_table_constraint(&mut self) -> Result<Option<TableConstraint>> {
         let name = if self.parser.parse_keyword(Keyword::CONSTRAINT) {
-            Some(
-                self.parser
-                    .parse_identifier()
-                    .context(error::SyntaxSnafu { sql: self.sql })?,
-            )
+            Some(self.parser.parse_identifier().context(error::SyntaxSnafu)?)
         } else {
             None
         };
@@ -519,7 +507,7 @@ impl<'a> ParserContext<'a> {
                 let columns = self
                     .parser
                     .parse_parenthesized_column_list(Mandatory, false)
-                    .context(error::SyntaxSnafu { sql: self.sql })?;
+                    .context(error::SyntaxSnafu)?;
                 Ok(Some(TableConstraint::Unique {
                     name,
                     columns,
@@ -541,7 +529,7 @@ impl<'a> ParserContext<'a> {
                 let columns = self
                     .parser
                     .parse_parenthesized_column_list(Mandatory, false)
-                    .context(error::SyntaxSnafu { sql: self.sql })?;
+                    .context(error::SyntaxSnafu)?;
 
                 ensure!(
                     columns.len() == 1,
@@ -837,7 +825,7 @@ mod tests {
     use std::collections::HashMap;
 
     use common_catalog::consts::FILE_ENGINE;
-    use snafu::ErrorCompat;
+    use common_error::ext::ErrorExt;
     use sqlparser::ast::ColumnOption::NotNull;
 
     use super::*;
@@ -1401,10 +1389,7 @@ ENGINE=mito";
         let result = ParserContext::create_with_dialect(sql, &GreptimeDbDialect {});
         assert!(result
             .unwrap_err()
-            .iter_chain()
-            .last()
-            .unwrap()
-            .to_string()
+            .output_msg()
             .contains("sql parser error: Expected BY, found: RANGE"));
 
         let sql = r"
@@ -1418,10 +1403,7 @@ ENGINE=mito";
         let result = ParserContext::create_with_dialect(sql, &GreptimeDbDialect {});
         assert!(result
             .unwrap_err()
-            .iter_chain()
-            .last()
-            .unwrap()
-            .to_string()
+            .output_msg()
             .contains("sql parser error: Expected LESS, found: THAN"));
 
         let sql = r"
@@ -1435,10 +1417,7 @@ ENGINE=mito";
         let result = ParserContext::create_with_dialect(sql, &GreptimeDbDialect {});
         assert!(result
             .unwrap_err()
-            .iter_chain()
-            .last()
-            .unwrap()
-            .to_string()
+            .output_msg()
             .contains("Expected a concrete value, found: MAXVALU"));
     }
 
@@ -1547,7 +1526,7 @@ ENGINE=mito";
     fn test_invalid_column_name() {
         let sql = "create table foo(user string, i timestamp time index)";
         let result = ParserContext::create_with_dialect(sql, &GreptimeDbDialect {});
-        let err = result.unwrap_err().iter_chain().last().unwrap().to_string();
+        let err = result.unwrap_err().output_msg();
         assert!(err.contains("Cannot use keyword 'user' as column name"));
 
         // If column name is quoted, it's valid even same with keyword.
diff --git a/src/sql/src/parsers/delete_parser.rs b/src/sql/src/parsers/delete_parser.rs
index 555f2c6265f4..39c3c6000dc1 100644
--- a/src/sql/src/parsers/delete_parser.rs
+++ b/src/sql/src/parsers/delete_parser.rs
@@ -24,10 +24,7 @@ use crate::statements::statement::Statement;
 impl<'a> ParserContext<'a> {
     pub(crate) fn parse_delete(&mut self) -> Result<Statement> {
         let _ = self.parser.next_token();
-        let spstatement = self
-            .parser
-            .parse_delete()
-            .context(error::SyntaxSnafu { sql: self.sql })?;
+        let spstatement = self.parser.parse_delete().context(error::SyntaxSnafu)?;
 
         match spstatement {
             SpStatement::Delete { .. } => {
diff --git a/src/sql/src/parsers/insert_parser.rs b/src/sql/src/parsers/insert_parser.rs
index a5a24d402a43..a19012b3bd70 100644
--- a/src/sql/src/parsers/insert_parser.rs
+++ b/src/sql/src/parsers/insert_parser.rs
@@ -24,10 +24,7 @@ use crate::statements::statement::Statement;
 impl<'a> ParserContext<'a> {
     pub(crate) fn parse_insert(&mut self) -> Result<Statement> {
         let _ = self.parser.next_token();
-        let spstatement = self
-            .parser
-            .parse_insert()
-            .context(error::SyntaxSnafu { sql: self.sql })?;
+        let spstatement = self.parser.parse_insert().context(error::SyntaxSnafu)?;
 
         match spstatement {
             SpStatement::Insert { .. } => {
diff --git a/src/sql/src/parsers/query_parser.rs b/src/sql/src/parsers/query_parser.rs
index b9d4be49011d..a03ff372c345 100644
--- a/src/sql/src/parsers/query_parser.rs
+++ b/src/sql/src/parsers/query_parser.rs
@@ -22,10 +22,7 @@ use crate::statements::statement::Statement;
 impl<'a> ParserContext<'a> {
     /// Parses select and it's variants.
     pub(crate) fn parse_query(&mut self) -> Result<Statement> {
-        let spquery = self
-            .parser
-            .parse_query()
-            .context(error::SyntaxSnafu { sql: self.sql })?;
+        let spquery = self.parser.parse_query().context(error::SyntaxSnafu)?;
 
         Ok(Statement::Query(Box::new(Query::try_from(spquery)?)))
     }
@@ -33,7 +30,7 @@ impl<'a> ParserContext<'a> {
 
 #[cfg(test)]
 mod tests {
-    use snafu::ErrorCompat;
+    use common_error::ext::ErrorExt;
 
     use crate::dialect::GreptimeDbDialect;
     use crate::parser::ParserContext;
@@ -55,10 +52,7 @@ mod tests {
         assert!(result.is_err());
         assert!(result
             .unwrap_err()
-            .iter_chain()
-            .last()
-            .unwrap()
-            .to_string()
+            .output_msg()
             .contains("Expected an expression"));
     }
 }
diff --git a/src/sql/src/parsers/tql_parser.rs b/src/sql/src/parsers/tql_parser.rs
index d9a82ed41afe..c3c6b7e4b94e 100644
--- a/src/sql/src/parsers/tql_parser.rs
+++ b/src/sql/src/parsers/tql_parser.rs
@@ -45,8 +45,7 @@ impl<'a> ParserContext<'a> {
                             && w.quote_style.is_none() =>
                     {
                         let _ = self.parser.next_token();
-                        self.parse_tql_eval()
-                            .context(error::SyntaxSnafu { sql: self.sql })
+                        self.parse_tql_eval().context(error::SyntaxSnafu)
                     }
 
                     Keyword::EXPLAIN => {
@@ -56,8 +55,7 @@ impl<'a> ParserContext<'a> {
 
                     Keyword::ANALYZE => {
                         let _ = self.parser.next_token();
-                        self.parse_tql_analyze()
-                            .context(error::SyntaxSnafu { sql: self.sql })
+                        self.parse_tql_analyze().context(error::SyntaxSnafu)
                     }
                     _ => self.unsupported(self.peek_token_as_string()),
                 }
@@ -136,8 +134,8 @@ impl<'a> ParserContext<'a> {
         let start = Self::parse_string_or_number(parser, Token::Comma).unwrap_or("0".to_string());
         let end = Self::parse_string_or_number(parser, Token::Comma).unwrap_or("0".to_string());
         let step = Self::parse_string_or_number(parser, Token::RParen).unwrap_or("5m".to_string());
-        let query = Self::parse_tql_query(parser, self.sql, delimiter)
-            .context(error::SyntaxSnafu { sql: self.sql })?;
+        let query =
+            Self::parse_tql_query(parser, self.sql, delimiter).context(error::SyntaxSnafu)?;
 
         Ok(Statement::Tql(Tql::Explain(TqlExplain {
             query,
@@ -166,7 +164,7 @@ impl<'a> ParserContext<'a> {
 
 #[cfg(test)]
 mod tests {
-    use snafu::ErrorCompat;
+    use common_error::ext::ErrorExt;
 
     use super::*;
     use crate::dialect::GreptimeDbDialect;
@@ -286,21 +284,11 @@ mod tests {
         // Invalid duration
         let sql = "TQL EVAL (1676887657, 1676887659, 1m) http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m";
         let result = ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}).unwrap_err();
-        assert!(result
-            .iter_chain()
-            .last()
-            .unwrap()
-            .to_string()
-            .contains("Expected ), found: m"));
+        assert!(result.output_msg().contains("Expected ), found: m"));
 
         // missing end
         let sql = "TQL EVAL (1676887657, '1m') http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m";
         let result = ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}).unwrap_err();
-        assert!(result
-            .iter_chain()
-            .last()
-            .unwrap()
-            .to_string()
-            .contains("Expected ,, found: )"));
+        assert!(result.output_msg().contains("Expected ,, found: )"));
     }
 }
diff --git a/src/table/src/error.rs b/src/table/src/error.rs
index 6146309d8ddf..6ac4c970fe9e 100644
--- a/src/table/src/error.rs
+++ b/src/table/src/error.rs
@@ -30,7 +30,7 @@ pub type Result<T> = std::result::Result<T, Error>;
 #[snafu(visibility(pub))]
 #[stack_trace_debug]
 pub enum Error {
-    #[snafu(display("Datafusion error"))]
+    #[snafu(display(""))]
     Datafusion {
         #[snafu(source)]
         error: DataFusionError,
diff --git a/tests/runner/src/env.rs b/tests/runner/src/env.rs
index 461c81d9486a..8455252e4daa 100644
--- a/tests/runner/src/env.rs
+++ b/tests/runner/src/env.rs
@@ -28,7 +28,6 @@ use client::{
     Client, Database as DB, Error as ClientError, DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME,
 };
 use common_error::ext::ErrorExt;
-use common_error::snafu::ErrorCompat;
 use common_query::Output;
 use common_recordbatch::RecordBatches;
 use serde::Serialize;
@@ -365,10 +364,10 @@ impl Database for GreptimeDB {
                     Ok(recordbatches) => result = Ok(Output::RecordBatches(recordbatches)),
                     Err(e) => {
                         let status_code = e.status_code();
-                        let source_error = e.iter_chain().last().unwrap();
+                        let msg = e.output_msg();
                         result = ServerSnafu {
                             code: status_code,
-                            msg: source_error.to_string(),
+                            msg,
                         }
                         .fail();
                     }
@@ -453,7 +452,7 @@ impl Display for ResultDisplayer {
             },
             Err(e) => {
                 let status_code = e.status_code();
-                let root_cause = e.iter_chain().last().unwrap();
+                let root_cause = e.output_msg();
                 write!(
                     f,
                     "Error: {}({status_code}), {root_cause}",

From a71bf837c60072c840b123f8f8f086dace4cf7d7 Mon Sep 17 00:00:00 2001
From: LFC <990479+MichaelScofield@users.noreply.github.com>
Date: Wed, 27 Sep 2023 19:50:07 +0800
Subject: [PATCH 34/36] fix: allow `.`(dot) literal in table name (#2483)

* fix: allow `.`(dot) literal in table name

* fix: resolve PR comments
---
 src/common/catalog/src/lib.rs           | 14 --------------
 src/common/meta/src/key.rs              |  2 +-
 src/common/meta/src/key/table_name.rs   |  2 ++
 src/meta-srv/src/service/admin/meta.rs  | 19 +++++++++----------
 src/meta-srv/src/service/admin/route.rs | 21 ++++++++++-----------
 5 files changed, 22 insertions(+), 36 deletions(-)

diff --git a/src/common/catalog/src/lib.rs b/src/common/catalog/src/lib.rs
index 79e6de5a0dd0..1527b7f4a063 100644
--- a/src/common/catalog/src/lib.rs
+++ b/src/common/catalog/src/lib.rs
@@ -13,9 +13,6 @@
 // limitations under the License.
 
 use consts::DEFAULT_CATALOG_NAME;
-use snafu::ensure;
-
-use crate::error::Result;
 
 pub mod consts;
 pub mod error;
@@ -26,17 +23,6 @@ pub fn format_full_table_name(catalog: &str, schema: &str, table: &str) -> Strin
     format!("{catalog}.{schema}.{table}")
 }
 
-pub fn parse_full_table_name(table_name: &str) -> Result<(&str, &str, &str)> {
-    let result = table_name.split('.').collect::<Vec<_>>();
-
-    ensure!(
-        result.len() == 3,
-        error::InvalidFullTableNameSnafu { table_name }
-    );
-
-    Ok((result[0], result[1], result[2]))
-}
-
 /// Build db name from catalog and schema string
 pub fn build_db_string(catalog: &str, schema: &str) -> String {
     if catalog == DEFAULT_CATALOG_NAME {
diff --git a/src/common/meta/src/key.rs b/src/common/meta/src/key.rs
index b616725f366f..e0aa5c1ec9dc 100644
--- a/src/common/meta/src/key.rs
+++ b/src/common/meta/src/key.rs
@@ -80,7 +80,7 @@ use crate::DatanodeId;
 
 pub const REMOVED_PREFIX: &str = "__removed";
 
-const NAME_PATTERN: &str = "[a-zA-Z_:-][a-zA-Z0-9_:-]*";
+const NAME_PATTERN: &str = r"[a-zA-Z_:-][a-zA-Z0-9_:\-\.]*";
 
 const DATANODE_TABLE_KEY_PREFIX: &str = "__dn_table";
 const TABLE_INFO_KEY_PREFIX: &str = "__table_info";
diff --git a/src/common/meta/src/key/table_name.rs b/src/common/meta/src/key/table_name.rs
index a4b53de00b04..e87234c603d5 100644
--- a/src/common/meta/src/key/table_name.rs
+++ b/src/common/meta/src/key/table_name.rs
@@ -268,6 +268,8 @@ mod tests {
         test_ok("my_table");
         test_ok("cpu:metrics");
         test_ok(":cpu:metrics");
+        test_ok("sys.cpu.system");
+        test_ok("foo-bar");
     }
 
     #[test]
diff --git a/src/meta-srv/src/service/admin/meta.rs b/src/meta-srv/src/service/admin/meta.rs
index 8b9028117eaa..d199edd388ec 100644
--- a/src/meta-srv/src/service/admin/meta.rs
+++ b/src/meta-srv/src/service/admin/meta.rs
@@ -14,7 +14,6 @@
 
 use std::collections::HashMap;
 
-use common_catalog::parse_full_table_name;
 use common_error::ext::BoxedError;
 use common_meta::key::table_name::TableNameKey;
 use common_meta::key::TableMetadataManagerRef;
@@ -129,15 +128,15 @@ impl HttpHandler for TableHandler {
         _: &str,
         params: &HashMap<String, String>,
     ) -> Result<http::Response<String>> {
-        let table_name =
-            params
-                .get("full_table_name")
-                .context(error::MissingRequiredParameterSnafu {
-                    param: "full_table_name",
-                })?;
-
-        let (catalog, schema, table) =
-            parse_full_table_name(table_name).context(error::InvalidFullTableNameSnafu)?;
+        let catalog = params
+            .get("catalog")
+            .context(error::MissingRequiredParameterSnafu { param: "catalog" })?;
+        let schema = params
+            .get("schema")
+            .context(error::MissingRequiredParameterSnafu { param: "schema" })?;
+        let table = params
+            .get("table")
+            .context(error::MissingRequiredParameterSnafu { param: "table" })?;
 
         let key = TableNameKey::new(catalog, schema, table);
 
diff --git a/src/meta-srv/src/service/admin/route.rs b/src/meta-srv/src/service/admin/route.rs
index 80da536d583a..ad8c0fdaf55c 100644
--- a/src/meta-srv/src/service/admin/route.rs
+++ b/src/meta-srv/src/service/admin/route.rs
@@ -14,7 +14,6 @@
 
 use std::collections::HashMap;
 
-use common_catalog::parse_full_table_name;
 use common_meta::key::table_name::TableNameKey;
 use common_meta::key::TableMetadataManagerRef;
 use snafu::{OptionExt, ResultExt};
@@ -35,15 +34,15 @@ impl HttpHandler for RouteHandler {
         _path: &str,
         params: &HashMap<String, String>,
     ) -> Result<http::Response<String>> {
-        let table_name =
-            params
-                .get("full_table_name")
-                .context(error::MissingRequiredParameterSnafu {
-                    param: "full_table_name",
-                })?;
-
-        let (catalog, schema, table) =
-            parse_full_table_name(table_name).context(error::InvalidFullTableNameSnafu)?;
+        let catalog = params
+            .get("catalog")
+            .context(error::MissingRequiredParameterSnafu { param: "catalog" })?;
+        let schema = params
+            .get("schema")
+            .context(error::MissingRequiredParameterSnafu { param: "schema" })?;
+        let table = params
+            .get("table")
+            .context(error::MissingRequiredParameterSnafu { param: "table" })?;
 
         let key = TableNameKey::new(catalog, schema, table);
 
@@ -54,7 +53,7 @@ impl HttpHandler for RouteHandler {
             .await
             .context(error::TableMetadataManagerSnafu)?
             .map(|x| x.table_id())
-            .context(TableNotFoundSnafu { name: table_name })?;
+            .context(TableNotFoundSnafu { name: table })?;
 
         let table_route_value = self
             .table_metadata_manager

From 6eb49b5c6cd22b06c1258d52eb2d0da70793dd61 Mon Sep 17 00:00:00 2001
From: Zhenchi <zhongzc_arch@outlook.com>
Date: Wed, 27 Sep 2023 21:50:44 +0800
Subject: [PATCH 35/36] feat: push down filters to region engine (#2513)

feat: pushdown filters to region engine

Signed-off-by: Zhenchi <zhongzc_arch@outlook.com>
---
 src/datanode/src/region_server.rs | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/src/datanode/src/region_server.rs b/src/datanode/src/region_server.rs
index 601ae1f28874..1fe9fa562faa 100644
--- a/src/datanode/src/region_server.rs
+++ b/src/datanode/src/region_server.rs
@@ -36,7 +36,7 @@ use datafusion::datasource::TableProvider;
 use datafusion::error::Result as DfResult;
 use datafusion::execution::context::SessionState;
 use datafusion_common::DataFusionError;
-use datafusion_expr::{Expr as DfExpr, TableType};
+use datafusion_expr::{Expr as DfExpr, TableProviderFilterPushDown, TableType};
 use datatypes::arrow::datatypes::SchemaRef;
 use futures_util::future::try_join_all;
 use prost::Message;
@@ -498,4 +498,11 @@ impl TableProvider for DummyTableProvider {
             StreamScanAdapter::new(stream),
         ))))
     }
+
+    fn supports_filters_pushdown(
+        &self,
+        filters: &[&DfExpr],
+    ) -> DfResult<Vec<TableProviderFilterPushDown>> {
+        Ok(vec![TableProviderFilterPushDown::Inexact; filters.len()])
+    }
 }

From 9d7211715e42bb7d264a5c076f7d93bea89151bd Mon Sep 17 00:00:00 2001
From: Lilit0x <baasisek01@gmail.com>
Date: Thu, 28 Sep 2023 09:13:20 +0100
Subject: [PATCH 36/36] chore: changed opentsd handler to accept collection of
 datapoints

---
 README.md                                     |   8 +
 .../KCCNC_NA_2023_1000x200_Email Banner.png   | Bin 0 -> 52313 bytes
 src/cmd/src/error.rs                          |   9 +
 src/cmd/src/options.rs                        |  11 +-
 src/cmd/src/standalone.rs                     |  21 +++
 src/common/config/src/lib.rs                  |   4 +-
 src/common/function/src/scalars/timestamp.rs  |   3 +
 .../src/scalars/timestamp/greatest.rs         | 175 ++++++++++++++++++
 src/datanode/src/config.rs                    |   4 +-
 src/file-engine/src/config.rs                 |   2 +-
 src/file-engine/src/query.rs                  |  93 ++++++----
 src/file-engine/src/query/file_stream.rs      |  30 +--
 src/frontend/src/frontend.rs                  |   2 +-
 src/frontend/src/instance/opentsdb.rs         |  20 +-
 src/frontend/src/service_config/grpc.rs       |   2 +-
 src/frontend/src/service_config/influxdb.rs   |   2 +-
 src/frontend/src/service_config/mysql.rs      |   2 +-
 src/frontend/src/service_config/opentsdb.rs   |   2 +-
 src/frontend/src/service_config/otlp.rs       |   2 +-
 src/frontend/src/service_config/postgres.rs   |   2 +-
 src/frontend/src/service_config/prom_store.rs |   2 +-
 src/meta-client/src/lib.rs                    |   2 +-
 src/mito2/src/config.rs                       |   2 +-
 src/operator/src/statement/ddl.rs             |   7 +-
 .../src/extension_plan/series_divide.rs       |   8 +-
 src/query/src/dist_plan/analyzer.rs           |  15 +-
 src/query/src/dist_plan/commutativity.rs      |  77 +++++++-
 src/query/src/dist_plan/merge_scan.rs         |  15 +-
 src/servers/src/heartbeat_options.rs          |   2 +-
 src/servers/src/http/opentsdb.rs              |  55 +++---
 src/servers/src/mysql/server.rs               |   2 +-
 src/servers/src/opentsdb.rs                   |  51 ++---
 src/servers/src/opentsdb/handler.rs           |   8 +-
 src/servers/src/query_handler.rs              |   2 +-
 src/servers/tests/http/opentsdb_test.rs       |  32 ++--
 src/servers/tests/opentsdb.rs                 |   6 +-
 tests-integration/src/grpc.rs                 |   2 +-
 tests-integration/src/instance.rs             |   2 +-
 tests-integration/src/opentsdb.rs             |   6 +-
 .../distributed/explain/join_10_tables.result |  20 +-
 .../explain/multi_partitions.result           |   2 +-
 .../cases/distributed/explain/order_by.result |  10 +-
 .../explain/single_partition.result           |  19 +-
 .../distributed/explain/subqueries.result     |   8 +-
 .../optimizer/filter_push_down.result         |   2 +-
 .../distributed/optimizer/order_by.result     |  14 +-
 .../cases/distributed/show/show_create.result |   4 +-
 .../tql-explain-analyze/analyze.result        |   5 +-
 .../tql-explain-analyze/explain.result        |  46 ++---
 .../common/aggregate/distinct.result          |   2 +-
 .../common/aggregate/distinct_order_by.result |   2 +-
 .../standalone/common/aggregate/sum.result    |   4 +-
 .../standalone/common/alter/add_col.result    |   2 +-
 .../common/alter/add_col_chain.result         |   2 +-
 .../common/alter/add_col_default.result       |   2 +-
 .../common/alter/add_incorrect_col.result     |   2 +-
 .../common/alter/alter_table.result           |   2 +-
 .../alter/alter_table_first_after.result      |   2 +-
 .../standalone/common/alter/drop_col.result   |   2 +-
 .../common/alter/drop_col_not_null.result     |   2 +-
 .../alter/drop_col_not_null_next.result       |   2 +-
 .../common/alter/rename_table.result          |   4 +-
 tests/cases/standalone/common/basic.result    |   4 +-
 .../standalone/common/catalog/schema.result   |   4 +-
 .../common/copy/copy_from_fs_csv.result       |   8 +-
 .../common/copy/copy_from_fs_json.result      |   8 +-
 .../common/copy/copy_from_fs_parquet.result   |   8 +-
 .../standalone/common/copy/copy_to_fs.result  |   2 +-
 .../standalone/common/create/create.result    |  10 +-
 .../common/create/create_type_alias.result    |   2 +-
 .../create/upper_case_table_name.result       |   2 +-
 tests/cases/standalone/common/cte/cte.result  |   2 +-
 .../standalone/common/cte/cte_in_cte.result   |   2 +-
 .../standalone/common/delete/delete.result    |   2 +-
 .../common/describe/describe_table.result     |   2 +-
 .../standalone/common/function/time.result    |  16 ++
 .../cases/standalone/common/function/time.sql |   4 +
 .../common/insert/big_insert.result           |   4 +-
 .../standalone/common/insert/insert.result    |   4 +-
 .../common/insert/insert_default.result       |   4 +-
 .../common/insert/insert_invalid.result       |   4 +-
 .../common/insert/insert_select.result        |   4 +-
 .../common/insert/special_value.result        |   2 +-
 .../standalone/common/order/limit.result      |   6 +-
 .../common/order/limit_union.result           |   2 +-
 .../common/order/nulls_first.result           |   2 +-
 .../standalone/common/order/order_by.result   |   2 +-
 .../common/order/order_by_exceptions.result   |   2 +-
 .../order/order_variable_size_payload.result  |  20 +-
 .../standalone/common/select/dummy.result     |   2 +-
 .../cases/standalone/common/select/dummy.sql  |   1 -
 .../standalone/common/select/like.result      |   2 +-
 .../common/select/range_select.result         |   4 +-
 .../common/timestamp/timestamp.result         |   2 +-
 .../common/tql/aggr_over_time.result          |   8 +-
 .../cases/standalone/common/tql/basic.result  |   2 +-
 .../common/tql/binary_operator.result         |   4 +-
 tests/cases/standalone/common/tql/join.result |  21 +--
 tests/cases/standalone/common/tql/join.sql    |   3 -
 .../standalone/common/tql/operator.result     |   6 +-
 .../common/truncate/truncate.result           |   2 +-
 .../standalone/common/types/blob/blob.result  |   4 +-
 .../common/types/float/infinity.result        |   4 +-
 .../standalone/common/types/float/nan.result  |   4 +-
 .../common/types/float/nan_aggregate.result   |   6 +-
 .../common/types/float/nan_aggregates.result  |   4 +-
 .../common/types/float/nan_join.result        |   4 +-
 .../common/types/float/nan_ordering.result    |   4 +-
 .../common/types/float/nan_window.result      |   4 +-
 .../common/types/interval/interval.result     |   2 +-
 .../common/types/string/bigstring.result      |   2 +-
 .../types/string/scan_big_varchar.result      |   4 +-
 .../common/types/string/unicode.result        |   2 +-
 .../timestamp/incorrect_timestamp.result      |   2 +-
 .../common/types/timestamp/timestamp.result   |   2 +-
 .../types/timestamp/timestamp_join.result     |   4 +-
 .../timestamp/timestamp_precision.result      |   2 +-
 .../types/timestamp/timestamp_types.result    |   4 +-
 .../standalone/create/recover_created.result  |   6 +-
 tests/cases/standalone/limit/limit.result     |   4 +-
 .../optimizer/filter_push_down.result         |   2 +-
 .../standalone/optimizer/order_by.result      |  14 +-
 .../cases/standalone/show/show_create.result  |   2 +-
 .../tql-explain-analyze/analyze.result        |   5 +-
 .../tql-explain-analyze/explain.result        |  14 +-
 125 files changed, 717 insertions(+), 427 deletions(-)
 create mode 100644 docs/banner/KCCNC_NA_2023_1000x200_Email Banner.png
 create mode 100644 src/common/function/src/scalars/timestamp/greatest.rs

diff --git a/README.md b/README.md
index b51060310fce..e2aa044a7c66 100644
--- a/README.md
+++ b/README.md
@@ -27,6 +27,14 @@
     <a href="https://greptime.com/slack"><img src="https://img.shields.io/badge/slack-GreptimeDB-0abd59?logo=slack" alt="slack" /></a>
 </p>
 
+## Upcoming Event
+Come and meet us in **KubeCon + CloudNativeCon North America 2023!**
+<p align="center">
+  <picture>
+    <img alt="KubeCon + CloudNativeCon North Logo" src="./docs/banner/KCCNC_NA_2023_1000x200_Email Banner.png" width="800px">
+  </picture>
+</p>
+
 ## What is GreptimeDB
 
 GreptimeDB is an open-source time-series database with a special focus on
diff --git a/docs/banner/KCCNC_NA_2023_1000x200_Email Banner.png b/docs/banner/KCCNC_NA_2023_1000x200_Email Banner.png
new file mode 100644
index 0000000000000000000000000000000000000000..936891c47d5f7d778c01408fe6f30503ef98bf15
GIT binary patch
literal 52313
zcmX_o19V-{*6xXo#<p#vv2D9a8YhkIrm=0?wj0~FZNJm|-~0C1IXM}7?S<+2tu+&-
zC@+BkivtS)01%`kMU?>nuw&r&ztEt-&--9qEZ{E~dr2)P000i<-w(*_hR-_yKm?Ey
z6;gG}IM0O9Rnb`eszyest0hN%#P5Xwqq5Dd#5I)Q-JbO=Uo2c`n$v#kGC8}QR8F3h
zJzGuHz9@1omEmPbR4?~~`w2=zkD>@80}2i*CAnP3*z*;y3r~zIB}vrt;&VG=f7yDJ
z<#`#m<$dXtWsdh{NeX_9#4xQhe&H8B?ZZ{5xmb7(Y61?N<i86`92t2YY-kJ1wE&lH
zk-vsYtNiam8`S?j1YlXU_>)#8o?O2Aj;@hWG17qxNy1B$$DT1dI=MPIK?Oqoui;NW
z$c1hHn*oSdq8p26g+}4vfjlDU+xuRcL6#O||Jxqm1E!M8s!O8_y8pgoNEZN*&@(Z%
zOYULWpzJdsj-%!&_!$g)5b1npf2Hi={%XkI$t)cvygCb`%KN{UG_i(#R{`SsgbIXo
ztC+<}wEWYOjmj5?mqiIw`!>czD<J=E*$-`>$Sk_aaIz1*9&>p~g~W;Es^nCzY8aT?
zDYnf0%Nydq4I{9>tET$2kbcw5OHPLpCQjnyXoP&-ttB%kDVHHRK36WQwA?9==ltw)
zADMpm$K8Jy69)j}Nq;>Sy4p~XZbpU%t$FEggMa<D`s31H+=Zh-5<JsF5RYMc7$2tS
zf5ZlaK;?)M+a!_mfeRVyw@;Eo`U?wCmtUxtZp8j(DAXReadmPP*=3eo=MoDAe{Ki+
zzo7sjSVZgWSOuEGIWa(ToLw=9?M`3MURK$<x4S*RLjSiF)*L_s8`bUA7%ejm5G!4}
z4t93p`0>9!BKVPmS`0B)EK?Ot4POWtJm&c~q#wATaiD}~!tin*l4|Q1ueJZ%6~NC#
zs6n06pLhYvU%0FQR?k0AB%sfvEuly1e;jAZ0FEYtY*8RhvxNTm%h}b{xmEh;fA5>1
z2W=s@jYdG9b=|<L7D{r+djF3v01sf-PCc3_!(Hd9M8)X;eWj<D2-#wOTj%`b@>02^
zr}qCh&j1IV<Y$wdmu{>m^k&1)-#Tt|jtt-xi@^LruGy{b9hg(4WWO`oe-K{GTkrbc
z-u#%Mga^L)<=e{YiG-^bS$uO@ZAcQWN*F~$S0hqH0&Jrd7!O8<lSj-B-J6Y!Sk{u-
z#xz&+J)W#QFid~@*GYsV-X6^!a@&I528~sJAsE1~7V_dXo)B>>*0LTabnrP^bcbPk
z^#|O4M*L1V+%n1qUMN}ii+{&fg`?0x+yDUR*)~ex$l~>Ck6n54sEJ)+5w2#j2bu){
zz<lq}|NAa;zGKQa<82xsk95&Nn(spUA)+m0?SuZcFr>HLL$=s2!J`j?yZ=WF_}D+_
zFPJhu0`7zn@c&v^=jr6)REAg*t>eS}-bMs)0m`y?|Dz`%aZ#%ji38-y3MiDrE#Dc5
zIMMnhCMF<wKtXm8fn&jmTlTm1L%d#|12D;sOk*oE<7Ctdm<V;2;(m=TlriWmk;HV&
zV#wDy(*ws*@wvzBHA$j{7k8N?z&wTsy6tNt<=4$2MMk~^u2HRKDzmdC=eB&K+am{=
z^@zM;Ee<j2aM~~%PG#RMuCn~EskFXGdZpE1yokR&(6U4cFV61pn7{-9=ZfWX{upVA
zDM<vaOg;Zp#m_MA$-`?x5_9!$fmq1Nj()WSMC%}c9;SXO8?G)+yKC>`)6>O~r{u*+
zB5*f<QXu(tV-RD90?s0zAfe2$dXc^Pn5yE&{(2xA3KU8)=+lRMWy(kWfhN^%UHoLM
z7DB#nNV{ujb1>HZi~NKQz{<_w)iu-@_PqbR086?8P9cgiwCbx{R@kq+pv(=j<p2@E
zFatycwzB2-#$T;k(-%Kn{-?R%P6>fA?wfqhP4Wo1fZ-6tKhjGnsb%iiZ%fq=U-wb4
z((Bi8`8b$P+IS1|ilB6tU_`NN*O&S<U_wj)z#6hWZzdahW87xT!-7c_%@%Nh$3|0_
zU!SjUCR@HYk-_|@$j+Y!@p?<Phrtk`Sv>bPZjvS8=K>GaTW#MNW%o#&O!Z*@Lm2x$
zDw|)Yi~e3f%1-8F+}zt`pBJH9QT+7Wx#6btm6c7gFyh^GtG2=Y@_IIa>Y2eZMz%l`
znD@OukNS^bMd;C-Ac)jl)p#LAKNz%H%IlKYn*b#ear=V3`Y7W)H{%8^fM~1?f5^Gw
z+yu5!c1Ha|7j8xo^gsNQr;*_nX>xXv8!P^D+7w(i|7iL}w3XSL6gW2CxH`M8lrpS1
zstS34%}5#F=QUADo=ijktKOgbdFL8mW7(kxi;UTbgl@4i_8j1s|2+cNQbbBn?+@vY
z@vKSc52w9`>|8B4pc|4Lge9(?yR<^BI}^6D{P}h7J4snQXuWz-BPGAc+I9tx8rMfh
zhm(!vxZ5KKcA;;7aMOl#2o;jHaEVdFl(LNo4~UTNDdfKe)f7fnb@Lk?uXgZ{0s#Dk
zcC4XX;ZzZ~iiYX@bGyz2QldqlWWJXql*;3^jMy}AkD&mVlv_MlpBm-tY~RPANDKQ1
zMrMF_n_%|aQF_}IOf>_OhMaHgrz>1IT|-W{lAi#y#rp4vR5drhKLhST6Ns*#MIPrk
zByWqLfFTS7S<ifjNKcor6)PX0M)y0#MqTpufA_@1I;_9!cZYOwE!1`2-A3>{zdVfk
z<g$l?gr_vf<oR4ua1lXfqxKPYv-sPX&Z$44-{d%Q8|$R~cy+|(6b|n)93^(B!{qH8
zDC23Wy>4XjR$4i(u5k<l0MsB2#f;0cOMC`PY_}lr_go@kJ-k~^ob~-#mEgQ2mrrrK
zIE?iD+>!_tTv13BnA`X1e%ws2?tG%S*zva42UHk=E6A%!sk-Y_nV;(%WA1VOErR`n
zMK2sXB9`n#h>oVVx@mY2Uq<l}myB7wy2Pmkq0@U<s<V24;I&tnWds84OQ^~P!*2ww
zF94LQp~civ(Kt-Aoq^|qn4p5qZTOS1c+%l`i&lk`r!yXHa*jrIVY5O;1<2uF$Nr7&
zxf^=r8yAnRPypS?>zL{O?DMa=G)@#s&9qLB*Ki}m{3w{~#M@rhyWd8p5{V88F5N1s
zFAd$3i$cD}(PGz8v5U=8E#=_&aa-jB<Z8FDO$TYO=zNqxd&)y5#Zga00d%8&-~70t
zTs1RAkQ1D4yL@(V*XEo(^s-y1dmU5)h@s0Eta$3qG`wFf3C~e+<Sa~KbkbHMHTF6D
zw#tmaEL?lAdh33=LNI{{d~b;+60a$TkC4vGyC|j^OcD->Br-)mKlC-I*zyv!KOFKa
z+W@#j=-3~Me;krde05RPJLmxnkB!nD>?y%_>K7;eYIBkblyVyHosYxtMpv<wu0DjN
zqr9&SB802U&xvKMINF~SMPt6b?8z=(S=3JmBiYmSnE!Lj)`JVdydR!C%@J0u^1S8r
zu450eRU2mL=R*pU_5$3b)TKD4Dsgc%la%H-g4|&HygiAfUZ$(N4AvNlimA%A#%tzd
zm~=88COShQ7uZ;MuvtoJPP@H}8qrrTcMvU(K>4v7f17GjqjW|*8`~^+Kgq}J{HXDq
z>J%c|xntyG;e}|DFe!jK`pX2`@Xrw~4*;rIe&))~ESFjztHKoi%w5M1{2t;#42^Z~
zCI}94F`ujcCo%ZV5<%?zm*b&dqR~ZGs~s^)=&zsE8A8pflW&Ko+YfCZx3C<=yYL)l
zd^e2{J?@a(0Zhk5`0EfIlZ>A}5RoD_Z`LcLPx{k|D%Bbpyj~I%>OJZ&UFaTcxhqrn
zU2kWtTVEdg=p-<L-jmNKPD5~Rc{b(pX}{uTgvPvK+gAV~aNDf^Trgf~#?q?NBDImT
zv}Rk@t2?E!A{mNN5>(&4j2@-GpNA~e;<t^X-+#Dnf949c22BQb6TMG&h40mv#^J8s
zMBN%^?LSuOT+mex7DAnFNzLKgnm3|f7Ei1&4`*FG*f}Z-8%DQ>K>>uE;JVd*C|cN<
zvF_1Pm+pEyHR=@A6tAfhQJ2Q4&A)Xw0kV^h3*-O*7ou&KK9qGlmPm(>PaxC#$4ZxI
zyVlaTG&EdwrrT?P%JRlxu<GQ2@P3pSk3?7Q0v&K#_Iy|+XU;s3?u7Ww;ge$RfTLlR
zxDiamYtbfam?^DPcX%5r<XKa*wzz%RRH4I*Io~7$1dwc~$w1+Ri@z$Pv#g~*AwYUg
z#aRo@Q1MB}W++5u3DRrzTQv!0qSE?&Kbua)QC320Cf7ExF@whHP6Smy%a1%BZ<*2-
zyV`JxK5>AF_0f+Ft=1CCA_0jC@jN1b8%_)ayhF$&fR%Ig&d2GosND>9vst4M-Fh}X
zJfo8y&?6n8md31g_TzXI7<j};v-z;FglV;25uVo{Y@=*spEZp$wyBjWm(W}1VUH^`
zhCA%Z4r>ZHY%c=s{7U|Yh__r71n+o?6>_9l+Bpm)|E+!}Fds`6F56C4BV?mqL9p20
z0^ZOFnoP{66T~%LxUVg)>s40om^R-Rszf$u_Pq7_1l0~p5SbY?_U$GbMH>5SZc`h;
z*d&~X{Gqkq5G<QjH{Y|vMRHc1f0rCYW@lU^jRJiXxvKi)2$aOre)h+%?dG5|$kJrP
zqu&f{puqRtSySm>ZV`ffp^ZUh?!2kBbe3z~$st3?>{TrO`u6O0B#@tV76%Rd#Y@Ow
z*GO<>^5^X^m_O)b>N?O9h3U6-JlnzGe=iOmB!O+r9VkO_{tNZN`p_{pJR4?k7OL`N
zefRWF-taxT*yKTTzc>PX%XGcAnrcRwuC+m|Am7sbT>OScIX908(l_UTi{V0vdS695
z(}=x)QvK#E9V3aVqF3mMTEtsU`*b1JtlImsq!nD+axO16Y3TIn%(2f%yTL?tWD`1J
z&HXgUe_K&<Z)WNyOq{;r__FT}kI{JBc@*lF48j1{{=+Y$G+`X(n%wC!ygl*PB(4r-
z0euECfynyD1uRFsSosONOz~9M9k~S^o<o=r^88&*&TIL`5;3%P2S*~7pFxw+;zB(b
zAAw+-PJEXa!23#3ycG_rVq?jhX-Vnw<dFkf*9-uVV6JJen;|)iFI&}~I7xOP{i5i6
zW#v^*pmuv2Yo3r@-;0cN6z=vossG2^dWFXR-&ao0#nx_Y%-^ZiKW{J5RmwI9M&;*H
zUu4fTD(u*P9k|kPl)7-TjSBK2aB{B_^DDDmbAR!y(K{!}@aHGQyXM}%h{fYrLCl&v
z`1F~)b6U$NNQHB^QTpaU0!%<lHD~^!X>QIXG+0#9c9badhCZ}+ayrTj%x5aSvhSe;
zHk9GFxivtmU(~FJc0WTQF|oD>9sU9v(G;O=wBwzYy}X75M)y5reUpxFs=N_xG9KMt
z>8d-TV|1X1Zgl0g=vn6Qb@Vshh&1q5{EV}U=sKx3?mFY`!x4c*bC(H8RD6oik%7Q)
z$yy&Z;96fZ{!WfLi=}qHCZ0oe@e<-H-GF?8UanR$sEo2@bAX{eS822K23n92LW0Og
z7x(=&JgfKfI0gB}WcV+f7+CkgxZJFcuf3$ztA+4GrpNb@jOm4i%ud_G1JtKrzDQKz
z*Sdm)hU1`p<jCmTkx;F(B3`dKzgP%QqJ^rP&>4Nk*V;`dd4cbHv4<T7Wn*a%k!Kaf
zd@aG#7Q!+rp}{unOb{;XM(5^K%?Estt8j(7&b!+-P|F+H8((sq7MDxCRR#z>B|&;~
zeK}61dHNzKIMK31p+UoGS$h1Zmfn456;|<gVtG6h1KwKUT!&_~>!=|IKUtcU%w5C_
zlinZi_k=j|F5{Bad^uQeP@$a5FoQM8D{MpaM=hj-1l73;HkxSJpHp)=22+g=S=mX}
zK%3l0@0D7*k;BA7CSdE$&CI948jn6H@{*P)JZ=NoMS>JXKr(z*@zI6vr=+b$a%nD;
zjEqG<>*ko4iDz--$EH-FTqH&5pNKqO=%t*_<qr*Vx}QhqwILa;F77L7<)Pg=tNIfQ
zGoD-+rKQ5<b49Ioox;&-;1CSQmAR+KRp}5R3?2~JKga}_d&br`032;+iqs4C6G7C(
z(IrU)9)-q(F&=6Wh)9UEkHXYynahF}7gerboHIGC22h^9?0|e2acat=eK27n0e?Ik
zf(r%sdlY+Ki1o=1xI*D7Dyxf#Aam+XaZZ}BID_>b-{l@gj#!wklO}Wyp`|}GdjmG=
z^;0G71%Y-F%SxMn+p%$&e6Qu$6PwVosW9goXa_qTzkx{jH283onYEdnM(H?rg&S8^
ze648=q9;Su+#?<C{AtQgYUP~c93m$Hp<VIn)tboQeyBgfX`BH=bWv8FyRowWC25H;
z%*vzLS29c?niIet&~@|IOv1V7fdzrnnU969M+rfC5O&#$+Z#L8rBk?g)d;9Ve*A<*
z%Eq^hv-OLh`EMy9N<2N|s$jp#yu6epVaX{rcabh!n>h{)sOWQRaZ*5$3J?whcLPr6
zxgAVCO1MDDR{N>&-Y%8|Q)C<qy5D*bKxYvDa=*kVt+x2+@G#dfW~D!RO0M?7fL0wZ
zXT3{%93Ou&$K8>$AIWe-^x#g@n8Nzl%uPl8to-A>XuHF3C@nLku{gseSuvb&a3fdv
zQ50n9v&Lh$$X$|+Q-O9VU7Kh4gs+c1^@oC9)&UEUD4N*L&go<qVi%G9V8w3)A5*^2
zHQHfU)o0LWl`u7a@j+1&(v?nMzVpmi>~42}y&D|{qvZTM$6>EP7SY!6mz93)LuXx`
zpwXAxhEgT_JoIleKJW)adn%A>19)sUjD;RFHYS}u<M+LI;ai3z&>exHQnq-H5iIE+
zpyaIn&vYT<eg%vQEGd?!A_KtWJSk9EphXQLs9nVEBd~QCX`N&Gau_Oi!~30S2noys
z0W=JeNvMk=L2I~aDk%*Y&zQ((&0Hq>Vq~gYGfhD8yfpTA24fYsLbH5bV0cL~<R=1D
z6NSpPTOSE9Op>!=PrMCR;9Jx<HN&LY!xUH7zd}8Qb}m=lA#&oCZI-1kyko*3ZEt4Q
zvAgOb-3h~Q(Q~qB^vsf5-N1%PQC;WQ2vV&m@$4h*uZc)mj<HRhHJ6VoDUjZvPE@TW
zM|UnrK5`jLpo%LxqEEhBseK#D=>wfc?VCaEGmW9Fj*v-1Iga_k-tKqh?Dx%{x0s)B
zY_EZn3BrQz=TL{H0Vu<j@X4#q2@t`W?qS1Mpvc0Di$N16b04G(K2B)!H^sE)t;D=x
zDs|dbJ{o{|$s~8xs$tguneZ${;<)(hq85^ReyMhKBlLIU^tL9UtsgZ`3jO>UdV7~1
z^Y(GYR#MmAgwsfZh0YZX1L9rxecBx&+$_J#Hq0Nq_UiS;w?#NtW)0zwyn>`vQ$+^T
zCe%GIXl4Q7A9eylK^3DnclTy$l^8+o6mqkl!tJDuYyQ9t0-tJpIXwzbU|~7xyCJ>$
z_vhV>X-QYKS@Xqx+f?P1#S?5<Rha{R>BMx7)ZbHZ`$=Puo^7PSEGxBik3={P`vN@}
zq2r3M=K$Mt4nJ>$*^R8dC|Giq>sy@__}JnUO_$0>PG%q%F}LhIu|gO{-{izABLj#@
zY7LgL8=cag4~_(UI5V$Ch5t$))j^GwHZa!*&9eAdgh&WRg+GOU$<)#3FNHa~42?{c
z{F9AhZ0Je}?skH^7Y<iOxL)?t6lN_5B~e%owkaVC)ese<;mzd^LlI8X!v1o1WP2L`
z|F>N+t<z=5FP{Y6>W5C{`m+}>k~(WHjE+)5TWCxoWI$W+f0p6MSC-)^Y>cv*du6%?
ziWX4DX54FW72P|dZuR4;K!xeq*oct`#PYRJ1wPn1&+nP|pqpw}Z<`L0@?DPL#FET=
ze_^m9cezD+%C<uX47o$1wxdgf525zCm4V|Eh}x+#&2-7-qxN~T6yFA2&S%rCYKGLu
zoD`3-Jo0>R0YZHy_9aqteY7&%JGBGVU#U>$EeTR+l!)sgM90^Rc9y8D%=Hy5g`nch
zM$DgE8>^=7U)w9?H4>hi!OGgUTe2xAICGjUUygO0S+Wx41wo@Uxpm4}(G=ir>@`7{
zT}5*82%OV5HP{3hX7YU*^G%*shdAT>Uw<K737I+)6s^Mi_1!ULjFMB(akK48zUd0Q
z00QSydg_)Yh+;Y>@w;@iHQOx~8iKzV56*Ym^Y?x`z@|j(4gSnub_G}+C-C@C_>H&a
z#4LTeOf65_e!mH$+&|d+M@4AYLibpHs)&rD1Y@xs|Gbbl3JP{4zrbBEi-tE0)Kxti
zRxg>$3~*WX+@BZs*K-~9XX;cNijs}0i}dP5U~GrqWg{l;w!VQgeyQDA(U>BA-VJ1!
zpj1at9)>x9AWF`r*V2V~bdnHLv(Y={b<M^}q225wmp}!0Z-tT7`wQs?LCOtLP2h$B
z7q)bdvF!!kF&TmblHU-kG-_!GV>p#+V;bqt0rX%fKk47`poJ8*4@IBfN4Q^dj>Qk&
zE2x3M!J;?U)ny}PoBogj4Dqv-e0@BWZ5@r<!QjJD)O!E6qW_Kn5F7e29tjuObjQ35
z5@8!UYwjrld*kh0ul{pPZAj*#=U1TmJvh@MFnK>a@o&PPU*{*7(d6CY3mwYi_%Qhv
zyHAczi|GAJZJj~qS$g!f@}%Zz<M_AG0N<7DQ%Wte+7R~yEbl0c{*U<hKdo^?%0BkW
zfZ~S1j8=ScYISE08N*O)(PUJD$^C^&NOGGB_0`Y5rd1;!?>Y6g$2HFGiA~uV#tPQa
zuAoOzk{E)aa!<DwY|K^F6_!(u*wAc^WFWcK+Dv9Hf)7$skjYA?*a^{vRqY|6{VMwz
z!%1%7#y>i|dRoCFvT2GsGk;pyMb6-CXkEWaxEZmK|4~{6mtiQ?bg-U^lxxRC@DY(G
zKiUt>Q0WZw!77wnIV&n=#1B?U*7cX>39WPpw6reyh808{kgPt-ePbHA^fnqfg}>$r
zhR7H%I}=2icc9)AdNO<SR>RKEw@4Ue71W!%EoF(&D6B5Fb?;iL+3PWC!hfL0|CZj~
zj_V&3ElpIYnCE}8A$6ZRQkk#{wgZfJ^W%`7%n4uL7D_E=fGRVRa6H?$nj>kj3?A5<
ztN+|zF#G1bIxvodJCWOMuyJ2o{zZfEcdwsq5+9l8M-Z`L)6x4CeW7MBrIE^#obn&l
z`*1akeNQvnjdhzjyDN7KkpZ?dp^)u+Lj4xAj(`ma7v9F^g{Bih=rAbG!o<|6Q-7k-
z!xfBEq8I{-qjZm!^yyu74T5J&`FG7dgM%occ9ZE1SaKI*)8>w;q^&~5T@;gwZ<n?f
zXg8IE<bs?W>&-ul#N-5=R@de#O1X)&tcs5^yiz=K(W**RE`)85a-*xNmY*{o%?iVD
zII(_Bt}tG**S(5sQYZo8iRz$H?b@5x>y(}PnJA0`V~D`XndSLZP%F?eytVl5yL1g^
zE;Mr1T&U9*+bg+?DVu1Dg{6^ADID0=(}B<QGl7iepL0&x?^EPRXvRm))jdbczMsY%
zH5wWkoZY2?Tk|*D>I=z)2E*HmMl%Cw(h2?&kxp2+6t46IirLPD<J=B8+o|&e=Uuq$
z7HJMq%NT}1C>SE^uPbp=UyoKpj*i-gLCK4<yiG@keaZ1Vovn9g@egZ<B8o`t%_LW#
z-e=}A4QW2$+4Ss3V;R+}{v+N@6KNJ&<*Ro6EP4a+(_K`-2fW1z2IiYKMN%eNPZBx8
zH$#xOLc1X+sC5x|;x`)I2t}PjRkrZ7+3bh`KXm3V`vc6OYjui(iu2lXWAsqCX<~BD
ze*f4!y;JdMYo{L$77d0lR5#sR_B7~=*K9?l3HzflL@HN{2L0EA3J!I797FT-bR8F*
zFP7HIpc2_D#W$>qR?0vc<C=$X=BLrz)nv2JUu{-=(ONwChV?)7p`~pqeK_EJGp(dg
zi*|ZB4>75JgSmaiVXK;Uzr5l2>7#3bNUTGAAZBGoo3-~9+9%NHFO4=o5*yH{t0p<n
zjBy=x6)<>vAMP&U-QmIc5m70cP-HuK5tZ8TFS0?iy<vq)-mZSG#F=>7vCTuPTYE)e
z{SGzQQk!L;`Su%!@W~TY|LAclDdPUSc?Kv2DtL^8k?iZpak&dV5QUe&UCIvq`Q6M`
z?{rnl78-PQ$Man1dfsE73JwW_N7scH{D(kztO>_Wyml$hQX$)CNn0C~)(U6|vT+aX
zo#6r^7;pPo{(e&)DVP%Rj|0Xm7A<b8AD<Ggl^PdJ2x3iB{7hhw;nHJl8N4a7VU7oG
zIzUS~r*UCHM+3}QfTnn^I^OBjXmTX9f2LijueOuZM)`uNK^O9zET2C-Z|7;_{1b$H
zI1)Fx*n1>j<%05Y%fr5Ww9BC#@sO7Xr!ssDub>zvS@_M~EzoZU3Nkc+pB2gb*F!5N
zrLq&dKXyqg`dn5x_kAoy@o3RTxOm%7%!Ni4`Ifu^oG>?|eEdx-wo%b2nL<tXv-r?M
zUE>CK<?Mp0bkX0;xM%SGQBGcXY*So<g_^0&?7qvHtMrQR3wDwb62CkeYMnzE(bDh#
z#Qv2X=d{5X<wvGK^wV54Q}IQ^1Nj=(SsqcUOQSHgug=|=0NV)>FXLs0aZ8jFR?{gi
zQ=7M8OUHmzNTmylVC1t<AevQIPtUJV#lmArMKM#vA`pQOCvK@ArVB=a2*o-7V5Nt>
z{K?1N61Vv_`$w|xY!t-hJK0Vo{z|2P(pbzN^t0y4rH@hD)eop;S%&?vzE}Oddy75c
zKYifu|Nkt2z}lk>>e?r-!g$bkO2Px?eVY70w9~ud?m}MsEvZ}q?e;gD!V@MyzHt|^
z;dI4viujmW#>;yZ)M)3cg9>Z=#jGn5OHw>_Dh%ldypoLSL1;K0n#hnF6&+a@+tWsu
z*kW@I?XOY)JNscbqdYv^Z*kk~D=p2<k9_lDrkr8Q6?xuD((cjYEs)qtrn=Q4RPQm+
zf(p*%Mglp#M6g12jx@Sjjs4Sz7?xhdqzGHw;7~7VzjQ)n$qiu<Z=G2SZ`pB#ml^#_
zcySiY#GU#jt+*VcdU)9hq7cK#+`iS%@&$%HK>6ZLH{zpX5QY6I#;;-Z_M-^C;CfSM
z2f-UtfWiq!xfWFp>oc3c9kHCbks{`%Bsesn2}~rxv(_(OtzRqx6-*VXGd5WTn$(r0
zAyVy&+-RST#}$+hVR(YVv6sTVzUC*W{8fI%--1_K=YE}yetQfJMg>))vMH>0o<|}B
zO>kreAFK*1Tc81=qz$d9)*QsKeO|N4^4uMFZDIN?k#P=?@C2iML4L^9KsfnhP2m<@
zc_9?4t}o-fK}&12cUx&l2e?raUDB8k41>5L{ZWmc_nND$)$hojx>n-E<8(w!N@ekb
zaxJzGKTNWs>S##pQEa`c&=E4?9a`iqfFKbAp3-oZ5%Y%3HXDCxaH@`bG6&+CH8A>9
z*2yqXUM&9?)XVjcN4(P|+i=B*2&S7^(a^8iqku)zpstT$;E{<(Pn?%yY#|iEvY#)7
ziVHk`nJDNOe&cC&%|AmeO`0AAMiDPb(W$|8l{DGDMT+$nxmIv^RK?IbO1{az8rGBm
zReH^ZDP6Kp#6#f$=o9p|I^r6Cw9&fyh<e3Q&uG&oXIi{hd*qmS+7hfe-F-`3P$czx
zYHRD@{^IjK(h=-Ryip@nc@9ma4s3-LSbvi^LZsLq9}2xSK@9i4CN?zK;=?>yVWbLB
zE)wYa;s9z_)aPCYpWGu@n6Mu$qSy9RZui&ggWxTSc}O!{;V1E?d3THJ@l`jEv;8f@
zEMvBth3=*zDOb}|kq8IzQMlElPWt*VNXzQFDy3K1Pxpm!A}ANaNZ1?<Qc;xPs%Gif
z)5YSXLn<6a8Jk<!KCHKKqO|9WLQ^p`oEe)@qyfZz*7xf72W9o9-1;D^T|ey99}34l
zf4aoD{GM%Fe<9&ly0^vEvMHEsu!x;apjb6sMTzeuciMjt2CF|Uvq{&6S=_q-h5`Qc
zi@<OdM1gGI1qHOz1ESUnRRMA3WrCK+;IV?eXZlid(JphU3WKet0>QE@FfR4`h-^uL
z)Al>yRSWd~<#cb|7ti43lsAE}t6|vXdUA_(kAnAh2!TobY?o!JXHbk<gHX_ZO`x3@
zV2E9+S^E|0j0|qG+_e3->Wp2Xr9jcHI7-V6Udc83ry;C#3Ei`6kkY8b{FUJ#O!+=Y
z55=T@>HAqb;{wOmZj2EeoaX1v5mJ$S)O;czTQ=?jUgJE4Nj7$5^l<mo=C#)dv;NDb
zQ@>!h?siO32z=Q5w0B`?tK9Uu@w+7_xlMZ7n%la|o<gzY&ohF*nsPNin_;%d(M~xP
zE~1)B`)jM@fTWO{C9gEE=RkXcR=i-s${4!O>Wawdihx&b?_ar(7R;&T@RaR^3YcJ%
zZ8iMvS$>!|rkNG=y=AQ}?8qP^g@@9GD#UOc%*s`yy<NTi4u`(g2NV!!>iF{+E}VkL
z6RqDQtnG!5Toq$a>sW_F<vaAR9);YJsz`(L8aEUG{(^Hy-Jct1*YWMp0XS6WOY>mS
z)uUh_Dl^KczBYq)0q5~c8yaPa6LU-MtaZaDNi8fJo9}N_p0Ite{w(icVxYD8Z)W)(
zG`xKo_*>4;=WSW8u26^_DSv?>LW6ZhQNHh#K?}`pT%u02c%}}75O`azXMN3LOFmRD
zS4*TrCsqYO1Hbk4ET+RN)2=TxW~NxJrddUSPV3V3u?GzCN*9{s5RjlB^`CauW4W?O
z(Q-CG`a!7&TAN%_YBawp*h`0;Zd9mdR-%|77Yx{OQIVJ=yFPD(er<oj*L@4(6|eMq
zkOt8MB9S6P=HlR#FKaVIAI9kJE6-hzJ6}G}550r}K44qZrQ6G?n|~1^k?9e-JuZt{
zVO8{H=y>&PTgfQ+({zUBkjyM}L3x`kC{)rId*f>SRV2r~l7i!AKwNglb-thm3j3*i
zHGN;FJ2|@CCS_339o1$c%TRt^3}wP)@~2wLep5iy0tTo~p;ubnbKJ!h4{)YS(97&i
zt-EeH$U2CokrO7&5rAy$o@u|{+(+wADPI+Ng!mZWQuwacUe`*ux*@VQ^E-uCkkQ91
zJrmcrpJAhpZH3pEl%e#JJ;AReHB=?-5<{5Ii*VUq-Wo&AlqE}=A|#1hZ*pQ;HBoCm
zeO-DIEhYgU0xcft#Res4<y{62=n8!v5Jt*N^1{2{?Dr*TDa&XEX%I6Bfwpz&iq-A?
zZGA#4Vc<kGBFL6P*Uw#%xFyaq3rdh1wVKl%W83;VL8_4Z0deE+GC~fJVV`emGx@?v
zB_dJ6R+~wG3q~(534(PS-!|@287b+vnC2kbxTPfx(*9QM%sJ}qUaW_D8jyf*hg|Oy
zhVKx`3Bx{c$QKqQlsBw;M>(%<-12Dw0ILma`?Lb)nFI+DH^}o&mrUqDOPoqukhj^q
z0*tJPsU_2i2R<wB{;V8gyvV^zZzf?gv4eircgBSx+}!-3i2ETPY%X8+*4kidm(`h$
zFOsT)+giExrR%-@wP|_6ReD)BainEpm0a!pMc8~Oza(ZnJuYLW#4TMbO=XM8vgT~e
zPeY$_OQ1!q%v`k;4msL6r@9)~=0Z~3AV#hJ;HmO*)2yNY(SN`6g}mfY-nI72UKiDE
z3WgEx{Pp=RyhuC;32d)z1jQu0#UZTAY(qCJh>}@zrey#$PaX!vvd>QR=TBMdvUANg
z)LmSL4Oh*0lV|hGor}}EEwqsa5C{I4OHIDAdbz`cz(opQHRGDU9dP$3IA@vNKYCJm
zK?2PO+bz$3=HjEO;g9k9jsZ}WL%&6Cc;990K}9xt_MW95#q{}Y;4}42r`&n6;dpPY
zECF+F1dfLM+7h{TebZ^~h}j~c$>&{6&NRBGbgcV#OxkG>w@<r=#H}s6j+Ha{K!lsH
zN$IrrOSl`WAMRpMXzwQtVTlszSguE;u`~kD1?TVGPm65c<mYbI2alj1Kd8uj-?tcF
zJ2(A#btMk4mZ}Rm7L?OtG^3x#B^5e`{cbZIIt2ts>)w?cjR3c0Je!`)cGt<F+veb{
z53b+YAbH7qJ}$z1z29?+S_dtNYh1W6tsSi|MAEk+HBHYUk!V|P8}j8szPC8LJ`pMX
zo>a%)#HF|X_U|vg^@USgS65DA4X?f8+gd*9s1HN?zjB0(4JLVIU^B9knYgSCyaG*o
zLTRrteH{i{)+vx``5Q<W7RI8hE0N6JB4F7&A56!QJGJ2Oh^-ff>MZ1Ol6_aNhX?x&
zf59E?!DNWg$Dro;<NNUO{X*3|4^E&smoleIT_Pb58caky2>k+QjeoqGT-*AqcOBNL
z#UZswc1N)ae8fni3m(+VzjOL{{dEa=G9AN*y-(i>^}gCx+KqgZ&Bs4c`jgnmR?h%D
znpyU)?0hp)9Gj3c--sHkS>LBad*&IV85D-I@X+yu_}TItOg&{bEPveBRzFy#-dZO#
zPp{GLP7-6}h9syk#no+GnxjSXGBvy<gv{owJm5#lAJF4zBu8P`=QcVZt~FXweIDXc
zATC5P#Poz<gteqJ;n~i@NZ_sbtKMUsOS1Cv@m6f><$)UUYNhZ}oMfXAC9%C-M_POS
zOy@A-A=Z&<;QRX<A>(>Ix4C9hRc6d<UyU`6A)sPIZg~T#&*dpMMG+dn^(O=;*N@V}
z<TJS@nwJ>a*ZPV+GujhO1)8^OP3R1DZn14X8l%6gb6-VHpf2G8&WbU)iMa(CeB#~Q
zMHNG9c3$b_<6|JwsSi`WY%KTg&son*Q%GF#L%BeO@7+Pfw%`2?vtweLl3-&QCPi6s
zx~l%kpq0zpDU{sz!DlMeZ$sSC5pru3RNbn910>2za%F{%X~#udVoNh=BV)tqiH~yh
z_-g{);6l@175YREs=~dt--v}C@QA^sn6k?94L)dMX)vZzcZ2+KG8>@9N=l95B-jyG
zc)@$i3!SL)79+kfam}B8c?)FWd7ZR}!nT&x>m45Kt*nyV+zQd6{WJh|nY+a7H-<6s
zPRaPr$IWFWSN7nffFJxr`1CVbe3UAZh>kCW8vV4!LMH2nMMP+qeQ4v`(${F{QAUdB
z5kEYj1sb$UvIn&n@$y8bxJQYoeyPak^XZHw3^Y=VSvvh3<1&v2_1ivBUrD%CXXkI8
z?fvCTCCn)|bX`e&9>%9%V4|bfW7!(t$SjJ8Nci}Ns<U8%+U4Fq3qIHG9;i`Cha^JO
z7Kn!>;>u3h2v+w<r3By0MqZ7z;Nep7)otadmF5uV@*9etcH<QyX?wcZz_|J?I>vFI
zHDE`#vrf9Oh6M<7pT*pDc=H1D3)N(0v!lXcoFuJEt^xY-uffrDg~pBO!k{3y7L{Pl
zL1&tb_{oN^>&qjhMuOfN?=?Bv`Kz5XwwJbd`f2aCYlhlos3Y3EkrUSLqPVk!Ks1Yv
zA|hiQloUEsYle(}gQHz>)U2$Hru<wD;76;dRn`jrn5pM%eO37`V^$;acI4X~HAuYm
z<wpF#6DV|KDl~U~R$4MCS~I__p{&%E(pHX?Kq5DP?Kl5-6_1Bw!2EsmqzlHsBh(pa
z*gTFQW{G79tk18QWz_Ji_bvIY!<v2IxeA+33bnQCbBf~YVs*WC!<3D=bd(k6IBM*h
z;~nHJ7k1pXW0gyHto<lgF=f?_z0I79RY#m;<4B`RdN4u)eaG#Eju{pV>$xI-0i~tv
zX)eovZAZ}eZD9=wcrcCINC_D5{sIQP%|}y$NGqZX=vP2otT#v%0N8fJGRtstWP%J`
zZFn1empMpQKpP|bV=BG>T6xFBmEcFq)AW?Ift%QvlG5>;!kE2QG3%wVT=W*mb`<&c
z4z)bAky!mK8+eAV+#!}(^$|HQ*C+)(7(yxJg%h(HW(@1S@h{8EcJJ^CyI$d3T2@39
z2eYO}g@JgqY%gJV#xR!_RqD&)4J*ruNtDYWllH^aKPrq_^}2Db<^mtG+|3boxk2w~
zKav0+_a<=U{wkn<%Tb(~HyRZ4&dfWNK#Qbc{3$jTAtvbG)%(E=y$_<GqJz1`hS)Cv
z!2`)&XDx0l+JCF%C`3K9wcH2vcsafih;Z54K0u#dOG1T<*enY?%?Td;C6j;hl9UP`
z)fC7O@X3E$b-?f8LLvp#r;Oca=XmO@YY2<OM#}cVmO(8faffNV{4gh`MvV(-U@YQ-
zSsmYPGTF^cqy#DcDu&ve0}H(kqpW2}-dwvEIerf+v6fTC{@{U*=(50m=3nOO<iV(_
z=!JGvS&((LF3?-$2Mh6F1LpHZ_zZGuDWdGNm-6*3xCkx&;+$H>bQO^@_YiHZD7M(y
zIxcgLVXZ!h2p}@qPnpTaTSY^;eDj*s<neoWYbv_Ph?qh3d3pT5Xwj4z+_n&yWZd8#
z`WIV6UG{mLe}e{ea$aadM}Y#i$x4>6=Q~EF+bE8AUTvH%5{pUEOErnN!}?5KJBtUm
z-&9#*_r=I|unXlLBW2H2P~|Zljk-Hh<BD9YYkmm0;s%zZjc6sy0Pk92f!sTky!v>r
zT`4`se2p_iSf~^K8#4JnK}3HkX{LRZ$?iw-5=A%l#~QnW{Y|DKTKs|ok86Od^~s&N
zY<~PKGPq>hL9CJFt<=@F;lVlcGaL$Cgr2TskGg4_`Ii0-znN)BDzvh=eNBoUOdNsQ
zaz@Ll=pf^DLW+0aU)UM~RT%_8NrX-d+(Mmkq}}2BGMW(usdkz=%#Npy#cg09F)^Q}
z#=e1(N)mNnOtTyyl8~>7g<*r-0LrgdCv`E6LwDG36nW2srzhI(?(X+bm(=g;j}0qI
zI!|G+^MCILB&lKhEs$Lc<&&ink#KpD-L1n!BX32cmy&YwVfHg;u!0*-1p2|OF4{kY
zlpk9VLvJFY2ztYW5>eRIOk62dkmj6&zqh0tLZdKIyW6<DwddzaIlHEjyL5Whe(wP%
zY_zu4k^oeRKd!2RXo!wEd5Nnbq3y=+RR_S70{r@^bL*lmJL%)8W;DsjNd-JF*o<?M
zRV%nWfp}8+yZyn*d1EQ|*pMhZ4?JR{(Hh^F^DG91W|bi=yCBAJ7auzh@5#*$XrUy-
zPD42gD8DQ}otQbL`?g{q3vXn|tD7NTpgGyLxop)gEp|jwSv5kS3M#Z_u42A&8CF^|
zcwuzfw%{-&xF|K#oJ$4jI=ziYPfbT+ui~@c%^YJ|4C7TtEOW;yewY~66mug%q3A@5
zesT=<<lhUqG`OI4nCUD@@L5p+3ldVlml}r}qIRl~Y&P@rrTep|IUggT7cV5R5W$bO
z1~_qJqAVf1I=t)w4<;^EH#VdGNB8fIV1P-#3A9nN(IX5B)5_GM_EAHa^d_dy<*c=%
zsOgd=+GgL!VP-RrP-gpd2S}DKNzFwp7IUxB$Ka<gk5o3+Rj^RyF!UB=!Y=(BJ?q~J
zzz2Gc3v+g*f#-t>oEVO^KFz*%82S~nLA+?IhT;{**X3C6x|IRhtv;6URqe0lPzuX)
z9Y)AwPrjc{vI*kVSAzF<u~VCl&lDA*9lCqV5q-psB%2}w@%GX>;OrNR<?bN$q$%go
zlt!X_)X0}bLPY$slbjFsoW$#u)$2FcQabu0+wGV&9-kjy)0HI_xIJo++a#vdauA_?
zgDWoA(M>LjrHY;}IW~GS3~DRP*a)l@*zUuKH4?ncilmF}o*n`pPLqFJ&5H}f&LD~W
z@#?P5t;~^B?ids6Qv+%bZxKBKr%1o!sW!a_?RL|Zv`i;P6#AO^e)^f0o;Dq(BXiwl
zuMlvlemqbH!t4^XyHRX0UQ63LOL&4=t2eh(YxagpJdbCj;^3ldIt|9&dLI>kB?^RV
zPnPXve(p?KRU_)Ob?(NaLz4HJ?%q4cqZt?erZrMZuTZ)+OaA!_krn!*qr&4$J2igs
zJUN!d*RZEz1u=**k#<xo&~8eCtFd(eq`D3MI(s`m&i}d!;OBIL?OtClYv>A@N^J5g
zqMo*XZ{iMtju<B#9(K-jo=}ilu?HQ0H5OL^QNP=nq2ka;d6opYTHuxj7nQv55ax#W
z)%u-5g<oU4Zykk+tjMN{W}So?Wv@I-0E_%hRQAQOFcP@CMBHyJ#*DW{#4=Oo2Hx|`
zZqbT1_hy1faXKH`bl=wRU+~PXRq$>Y2k*Bv?^4}dXT`o9>0I1iqx+KKX9y+MP)S|t
zv!a+itx7j!@b2z-C`U>zxsXjoI(}jYqDlcFY<Ohu3#PBB!$~s67@`hiFRz<xL;t+h
zb_+9xbPbucp)uf2yfPRCiw}l%@^7F>r7E%P@-(qpGE9B#Lt;DSAi3TKFnfq>*_*+g
zg*<a~`t>Rl80SFbhXdI50tyKhM2!_V0^QEId+iwKc!#UrUMU{(Y2(K>;T=Pgfq+)W
zUgceTx4V`>yA^0gGDoyf_*=}(Ys1T>=TXt8%{K?>E23iSG1-+Q7+;Nsu`=KHi{|L+
z)g^pYBB2<*i{Yzm_}Nk0sG*^#YKC8QUJe2%3zU!J8#MG=!di5j8e0e0$%I_L4#vQ6
zYg&lYFs;Z*vKpREAyC4J7^6jY_z1Gxe)hg@ieiqh@hpV(c^CM>^@K&%^{CguleX*h
zypC2rjmd1iSzm*pwp|?DrCSEt0ZSf$S9*@V`F_T0W)AwnC4sH4?_IZL>P;IYOA>j=
z@ax6OdpJdcCsB$TOF@Hx&7^@l+xy^%5gdwrZWnZAe@es}^^Qw|w~x#a=E_oPi5j-&
z<CYb3;@e2AZud}07-?4nPv`EN*Bj1aOB3NJbL;OZJHdzkOjiz59#6*e@6<~DxBSGc
z!e{9HY)g=&%>u-%>o_~MTkU}UQOF_Zc=90(hdmVUF`)N_QRv%W&}Nu<sL$HlZuvu+
zlzWd0EC`?jmj6T;MqSYUri3!9&TKPxijQWbpm*v}&6~!VPe*~rFNrYu6&SyDQd7<U
z!+;e{1I&mc#&f>x-lW-0I(XGBHqu7BQC+9}zFg<xOucX^i@A7%K2xmNr=`Da@2PrS
zVsgQFbo`k*p0wg5taJHFEI*-SjDb0fdD9ZNIEA&7a|U9@Qbl&W>AzKKOj4lSSc}Kd
z1;Cx(gy69eQ51%w`KxT(&s059CQT6<^&~|(?Fm4YJa7uA5bvL58z+BFPl4Y`)J~dg
zORj~?fCG*>yZAWjlel;(3P6z<x+o$g%NcOSV(r<~sR_yP{;M1McUcEY_8sWa73)`{
zh41t8^8?+o`fR#uhj(p|HYUnWxe}J{%nX0CpsqW`*Q^N}MrsUr0AU1~uzr?no*!d8
zLg~*?|3**V(U9l))#vDLcuhGPwR1r>`$nTskCq6mv&<N5+x2f4jg}Q}E$IWpP-p&Z
z%q$l+D2b!jWKE?k|Cs|RCL6}c57+|ZgQh@VzP6{wT5kx4c(D9ksTHeYF-mY<?l-D0
zW$xIZ_Io{t2NQ;htNJB*nX!oBZP{Q4Knqy`_sG`AwW%iX?51qd=aads#(35^ZNun=
z`BCvvq4*uTobr85*|pHYIrhz>DuFmw>v_<){YQrklXMskv@7Z}5Qhu_Ns>k@3Tj74
z{7=;zR;%?g&(r$1)lk~03NP2kkV+?~spOg4@-jjvL6197Jp9|lX7dlPS67f|piPq=
z`>mjJd<qL=_<f*nUIt0Ky*<QTH{nl}fllomi4?vw-xxci)vjP2{nCv2*HO3Emvd0L
zSPJt{hD)<>iw<fliY!+xC}%<#czG2BR}{dn9z+A%kSty}OqK`=XcoMz4M42@3_7VR
zo8rDoY8b2@<{QS{fM>&Y##ENs1Fkhmz~!G~LtM#9hiF^kSI%KSY<Fb9>x&GZpH!9k
zhwnel(aMva7F?Uea$}{@c?J2qul5!`q@nVyGby5tF61K~F=}{z1&bW*bsa8gK+=u8
z|6;&P*bs(sUm&|JlS+~SEe1FB;aPGVnV!466zIVJ58n)9fOttFF7=yy^?C#Ao#f<M
zJ;s3LCtp1cFuu`){;R)$h-LZwbrYP<`BS%ejx#(hASo5y4kgU%i`Y|HI#JNb;PYT-
zyNyE>c-AHSo1}Gi=7v%g3z(ov#%q5xSkk_2H@_jw$9O!5OpBRUAXe-UIg?}t3@umk
z+To?`Zmc+i24az;O!>0V2xW$OFQDlgpl8Le9G(e>2y*wgKbO5gGzH0F+>~CR2w(;C
zZUzz3c1OcwI78}NK$mHWcX`I^j_ucTG>84^aC`la2ucVyT+XwW&bgQ-=5JvSiqfjE
zvC=)(nCxTs6Gw~J*8_7WXB0p+>USP)i{@oPHqphxI<}g%>FkPPjKzXPsVl8BQ#AT4
zvG+yF!_<}OwN--QK}Qfz)vL6N@q73~p7ZYURGTkORK^jp@PCD`h7o)ILJP(L!F~Rj
zgZwj>2}fYwABw@QI%tl?Z+)B`E)hB$2&br*ooE6TI82}U5ZlnKZvs9OmiSj<lr6;2
zuoS3a4x2EHAy#0#E;S``7|MmGiqi8DR_D|*Cfn<>lc<-Zd(<6OGa+Q9@}QZ>Whr=4
z8YO=SnBBY8?p`{hjb~y(3IJEwGC#=MB2_RF%7fb)%)z4R|M(%eGjIBO*yZ^%q|V|;
z!uu16l_xk=<{kKt(4v|S0{|fa`+tg(OloX&y5x$ptq&|dZEsC=DU8CuV$T#V*w|TQ
z5v%UNcxb7h|4N%*dAqd&@qE#B{IF?ztj^M>#HQU9)Gm65dAmn)HSm1kK6=!TrfACv
zLlkOFLosf6x_3rJBS8<{oE0Vf9uBLCuVg7CN!Qm_!m?-m!m^7snuTivhc3Q9A^TkP
zTgiuRs)eUKYN~7xlfNQ_h|1g6_>u3ng?3P{&L51xNVtQtEBi3&GO}|)eKVqm=$iR6
zjsDdrPeX2S_?uR-OvMRt4}{TD!=>9B2{|c)jnVr-e76SB2zqzY@hu|ZmB)g#i~{S_
z&StPt%1+&rBsUY>iK)Qrxp?y`)m}bo%Sz`y@=ZD!va7RYA2;rI_REu3D$?YSF5(oa
z5+@@=4R@+KgD>Cj^Ac|e>78D?=T}&VBqZYLEH^-E#VK)#4V`<2o`c3~V#s{uPXM=>
zKD8kod`-_}CgT!`%nMGGFD<y%a38z#CUmwlN~m!gDer3sFU?~b44H|WG)=m5-y5wq
zq1c<+^ZA*2Hyv})a8L77z4ICe(x)$b(2iFXoyw(Q&(ynEJwK8#Ckd$Y4lD^y7wz1I
zXw<zvmGp;Z2f!VtCAWlJB4Z{{s=Uk}OT5uQG*U@(4%G|Iq6lB!KmaY^-<(D;t5}`+
z9asy0s&{{Tx-c3_AlrS@LE-0T@8#fGRSR}~d0E0$@6K(-$~V5gV|^>S)tfM49EBD1
z2x1mw4O$}`>)yOR9D?ccq4Efe$q-)tNZta9JmI^u<0blJrViNJx!u;0GKo~09sl%!
z!5^DHUlyq!p;dU?Ek0QJqfCzMj68W%{+76?CzvTDStm2oTnqQ#-B5n~&#|0XlBj_v
zuPrW^Q4QOaRfs&!-bddNoOBph?4K9ry~TofW^wnXlR3KQ$v4t!7uHy{7ZVm8^xPEe
z|L^|-fUM!~fxx+Hwm0(`f+au23U3Y$|HF1MfSa-Fx2vu9<ngY)oe^m|sdwH|lcie_
zd=8ooKTZPIDx%aahw4T|I&?s*G<)V+9F1zIyn82do`%p^sW_q7tIL$z|ETT)RA4h~
zGbhkkq|@2dBMY{#vHGYcLHlz!`#oWW6dzYhCpR<vXp0Wj`?eZ|05+0NT#`|j^f?D=
ztjBV=O_SJlk)VKltmqU);?Bw2&&Ea?ps`2|_bhxl*;}}%JZ;4mvZ-`jUe{NqD>3_n
zVP=Kba<#ed{Ohk(Dv2IO@q%5%(}Re&XA5XR4t70QfH0F#ZyJo-LZW)=DK0MW?-NuW
zf~B9kf&|?=JtDnjd)c2m1p6H0j3QQ+>)JT3qq7#!0iz7HvWvL}6;glU33+zp_1R;6
z4oeT>QXpd2>o8ypjB~-G5K9S3hFZXJ{SY#Cr#h6(a|b}PvLt~e5pvp#Hf8Xno|dnZ
zGGdRN2Sy7w%^zp)b=;`j5RpSNHx8o9<d8-6MG)#o(x4a&Gwz*M`koVE5PjbNkEU;q
zj_i5f-q_f1!%a5k#&)u?ZQI${w#|uc+qP|NoQchMKHvBJ`ky;z=G=RGx~rb5eyXZl
z<a(FwZZq`{Dx?C_elvkvIA{i3PZ?QjKz3%)yF@MPqh7uj-ROF>K@Y3H-1Q`w`yCcc
zP@ad={LO#Jp187l_v*Z*lVK3c3A*cLbsHS4<_ZaObHG?~@U4R>q@Dt~Fi2Bwss@yG
z2k&NRgZz@4v=&d5B>rp>1WMeyIzWYU%@%KI3o6t%B#O~l@OYdMY8~7Kgn$0p!ZG}L
zk)itBnSZEaMhn<qy^9dE1I-`EgW6-J$i`0s+3$%TDD^(zqU@M}<&cWY3wamUr|0MV
zfA!U63vg6rL78p%ZVFlzAS5A9wJ5G&t0&v;4lDh^gIhJ1PPG6>ne;YC5@i~roT;hH
z=ulr%^D{z&tlr0i21UqINPf*IbZy-kWE><jPSMi)L9-J!w_Rq1c7kmcbzDVFW?4g2
z$?CAFGo?1{hI}ZVuCjIzBbuity_WN2&d{w<$@S^{oyI<*=jK<>pieeh{@FGI6MSA_
zQ{|ezY`NY3s7u8UZa1p~@b3~Fnz|2Sc6y7Q>)dyi@MQa&@xNAuO*j09gsMvuw((J|
zPi-dU^J~@{R7()+)F4<}EN5WU?p5Bz#Xu<Y+LtWMeMdNVw)c@BkrZqGF!H#ycJugW
zD>HiU54D0aXtZYLx}X#tc64NY;~%`ey!%?Nk8j6zoP+o3d_YsTdtD+3N}qJH>&r%|
zp1m!1Hj^dmdfAN6M^OV59H}C`U4ws{&KvRlD?&CvO{0r0b_>K-v8gicf}<r$buB7<
z6n*m5+-75(b2ZGjlSxL(zA>@JffOtY{|+*Ms>&_?O%rAvve4vU6@fl=qIe?nK{fz~
zDxVWH2Am%&GARspi>G*?tNny#s|j6-6scFZzrz+ND)+Zdky_!0Z+m-tv|4>PlOE-x
zWTt)8P{We&aM~@!#gRl@=$znCi2M218X3@2`#hm+_nPSy^V)$(Y#7#0WA2U30))wm
zl;1gbQZL-z6vr<{b@b?lcH=$yS`%E$yg=hivkh39JG)=|SdMc&t&?q1V%ooG__W05
zr;XqNQ!}oQ`H$JXin*LImK0e&OR8L=N{Zr$-ZOJ46!dn9p6w3-WjA(Wx=hvn$(=8K
zoKl;T&;^rwe-TraNyd@HRTuJZ0;<n5ja<E2mKo(h^Y6sha9be7NhE=$=)3(NELE6q
zqV(CsgTtZ~#&Fg$ErAK4{WSbbC532IGcZD^_nvh`ls^tmPi>2G5T#X2BESBH)f*lq
zyci=62bBbRceWcZH#M-Na(PIGzb<c!t!I^m40C{0I7+{@;vwjzJ-UQ<bWFBt7cqUe
zU3eW2P<1$Ptowrv54^19<7^|lGUJ)~EL#>L*q^*!z9rwvdFIIfEL7<g?;8-lLvvhN
zk?Pq}lGhw0Pp}UXc+Ug8-2BT;zM3>-4(2TJ*|}Xu@%5r}zbXG0IdTzeb&guvrg9vE
z(MJbD5hvYIM2|a=G@W!TfRed8>H?7vwb4NZ3HP7XPKec4Y&N^l!lZkuoy;_3Rci5i
zd`l(R>_F+2WUF`fZ9q{^aziM>o$r~{PKtE+YV9G^R>(9Vec_SH1nEb%^M6X<j~xMK
z7K_1J>+utBR{h_&tJ!ym8^!qi!dm3z47^4xbH`rC3_Qyqv^pl96jVZBY8|aY>fY^w
znJ^%QQI{fE21}Rk#S^Zo+n-Ft2!uwk2Vz7Jx4s?;0?5lheIc9(0#QuVS%`zt;_IxL
zphFWPFs)51s@wf20)AG5O5`B5Z`OW$%GH8KBmYRXUZcvMOAdgJ(IHx@N5PW<B3et`
zm&h8qT%u7Ug+qqW6ivm>T%t3yiahCsjN);mC@4swth^H~_n3`?T_*QJDO(BhR7cRk
zZ#4BXIJPVb_?-p?yta$}idryZg+~3iYECc5bNSZm9Cy%FbsAaobADw;o*sMa;On#2
z<R&3Z^XQvgGucqUiF(bSwLMx>GiI^%?PBwNaT_Fvoi!kGn=tzfzni<l+1TA;ABusK
zv0ui@-10mQ&vP@r%o>^MS;n($_F}J@sWgJCP*jbG>3($V9U8;ozf|Pv32*VD2#PHs
zep(?DY7gI3>6d%Qa_(Y1U3HU+i_<($pKw8cM?=pF36+>Ll-@Oc`_lPek5TT76FcI(
z&0olsvM%qZ3ClO$H*LSK_&)*6MzWLFgfK4A*FU;e)~jmPVT8%L+($N29|0O*EqR{b
zW<eA?3UG48;vP-YE<46pGRWU@QSB?o$9||c<ReV=%it8{ZKe}m&8R^WNtyeCGPbXP
zDFu0p7F4L1*8o{iC$96vVuv#S3w^Zj8r$*Qr~Rc|_I@BxLjq(vRBF6jv1qlN1LGLT
z#CtLX_r&aU=9@?oDJ^hsEYnbi(hA-#FJj2jlq+(sW?Mf_Qdw$)kZ29ys(jx1{(vOl
zQ=#}!WgxIt=FP9<JX<-R6x#hXyBkvbdneth906h7wK{l*sGGT1_N9azxdOb~zF4Ej
z*8MVmS7;Xy1}k&~&BHYZKPd%AZmKFMmZL-T`4j}>Gw@(?+7fo%Ubn7Wn%>^%R1ad(
zxde<K12CYV<I;m_de`r)3Z3p>85^YBzA0_`2Bp%)^qmtRZ&;_xhX;tPhK-Dt?ITH~
zug(haU3-^5*wI>$AkuH7tEJC62e`C78F6V(6*Ho`zX{GGr%Mx;TFPc&@I9|GNgXBD
zWVLgTie`!3l#Mq=C;rUVKe2LUClSjMY_E!~g^hoRu2Nn6`@;}sXxy*O{Yv5@fbt_@
z*{lcW(%kspt;m(S;YlQ1d+gUe1L$LpWhzw>{zr=u@9sf=&bAdb!{GT@o1&<NqX@pi
z(~6X8CwHPsUynyM3+?&}#OV?6nA0m1EdUDsos2f4_I4|s{9IiL4NGQJ6Oh$yjP_pQ
zN3qO>^CUiWYvhTrD^!Id{a4+oX1Lv}9I`t1Jdll}_;mYBOZ_2UB1sb_)(4PhJXrKQ
zJ!tr`yK1~gov&4)-UgGcLC(djMrLk0;X*4T1t&EH$Bd=FOY6l;EW6us<!17!V8IPD
z5eUMlOb70d9Lp%Ri>kX9u@~}<Mc#K*iB`A2xwdY2>Oz*f$uD}S1I6=9hR(i87|VT_
zKb+{b6N+zQ&p6wkV*Y11_Q(9MT{U%CSz9D~fs1<O51ChZ5!dQH$#`acB`N-vD?3xd
z+_#9Yrhf;sfw<3pu^%lOp(G8{K_lg-?~nGu+S^SYAI~MR8?)wv8Ax@y+k`uW)s$3(
z`2yIQUwTs<i##@7wnEQD1g5$=Om{~wFsjOnJ<c1{PB5N}gxUIWsSnI!NPt-G-aYXm
zcnkTSZ=yDb+FP!Lzn9$t#k)#2mj9-v%|ib`1$>|kZ=euhT+*E?d)+_EDaL~z15bgW
zKFy<S=fM*RLU1#>+|Gx?8ns64%d^!EuZGN82GO8U*Y-|wc9+Z9W1z1e4M;JdKQZ?h
zAsyeurp}{{KzY_2K1KsKczCMjgv=~F)GREdB_=U?Z4S$i5Rtq5p>ER^^!){gyi6fF
zB$ZypdIR}zZ?Zjq--kmgm&V#b`*aXgQ06_PH?aYw`9Tk=t^w^2&@AC}u-1!~&8uy!
zP3Di^xWAPSx~KkXvSI^;cH8#|bL&@`c)S2Imyc0kHK~;|lUG_4GbRXvj3w=AE73Fw
ztX)^aj~XX$$QC`eaxz_t?N(p=LEUjaRspR&USX2NneY}BckCh(zPmo(o(-g`ov+XF
zY2mhymi$g{1or;=V04YQ82skPhlIq&k&()S;p5r*iwim@L9)}UyZf4c$9BXuTB)vG
zs_0TnrwBK<?5w=|crTN4=o}{gUiqAKfvVwG*&&gH4*E}Na>zuuVdV4W+@w;gvMHsM
zM-2;tLu#XEaYm+qPIEqTkF!EZY2LU4m=HQ3d!7h!l0AZ@6Pq)F(^yiLRWkGzGbiph
z>|B^LU#EYH4__aijC3v3SM%si^;U-U)4VgaNA`2QXdtvZYYvRO-#>YH7;g<dk_4Ha
zUOqJN>{79rciP=>Jw5xy>%aQ&1NVhh%7qN<7+ilcnYh8L6M48nPUoBb1X^9b?{iN!
zVJStWp1rW3dL<I>`fau*bl?g*)q{GQ2g>3A<@rJwUNfH<?BIg{h|pAWQc&6@CaAUJ
zm5-HuR%F^wSm@uI8`aGOk4@7Zc~zCu;|dqF3jr83R@0Kl@IgoNyd8S$T!V@`%an$(
zX5xIXo<MJ-<3-+<Cj-!Yr;*I&^$7m+7CiLYXGs_`g;AUSAsx7`gXe~d$npe=>Et(=
zje;&``00?FvLDU^`5&s*S!IC2@1=N-WvdsQaNiKIVFx(?6OwAKEghMeLV3$Cb*0KW
z006uWRJ{6LxDdwH0(7OPDpGa9j(48C<(?b?fZ$`9$$T&^JKHC9=AZ-fE+^CNF593m
z(M-L4IV#IIiiZ10@o?7jic?S`=ox!p_e!QnNVcq(W4Ay8vT{0?6r6ZnYV-_W@#j;N
zT+Ymf&&pqkvkfK7+p`TzZrY?l+OS#C%(52weY``0%<l^C&gsEu2plsbTYI%<Zd3}}
zuLlcm(f+YyhQi+RK522-6pcKUYJ45~Va&heXK3pmQpYic!W%kZ>|sGJn2w&wdCTG&
z`Nqwo+?th=i+9(*oJ^nP<j^5hL`!7Gn|}ho;83K}p;gZbg@MR@4y{#JNq##sophK;
z_F5}DsuBj?e>H=K&WhENB9c=MT%=@Pq&UK_)4E}xEp44Um&P0(MkK!&15v{o-irLY
z1vPt&$TfNnUjVH9eH-InW7nz9BU=2dwfF~RX<1l#=pUdZ2Uk&c^DM)K^z>6~l-kPA
zc{EM$<#miitvTP`SpXr{bYhShGG`#Zqa&>QzG^oIJ>%@ZTT_>gR$*IPs&Bcmnp-xC
z>XAd<>|7u4l7CX$EM-l+co=CX5dvO4=9Pb^vGQ>c^=>Tdd)KI+lOBA_IOz&O*}{em
z2p9`mbiSz#s3~%Q*_3=KT3=*62-R%GGj{^TCHmJ}Mf7qrX)G#$44^DOV<(~%%D88r
z9()}pl_6rwcP-{!(B4t{cGFFg!SZMv6ws<b9VU~HqQ8M@m*22quyi#l5(kW6t&8IX
z`foD1W(eAHd~fPfe?z<E`z-_*UNk{+M?7fdgm#4*0DY`?wq!c6%o`EPTto7u^YP_z
z&S%3J@?D(0_2Ol?vGreb-fP=<)6u{-^eo($EYcmwlWlo8QQ6v=Po^-UV>y&H*3IMy
z#z}J_(BzQk%M>+roxwR_FB#tbz&1Mf7}*7UB~M5!cbKE%tTYAdbH+}@lp7jk*E|$E
zg17|ybU+Dm7ApY+zU|hXpao;463}4IXU$5H1!oX>r$uYL#AL94273KpM^{e(7e5xk
zc?*x-?e5uhD7328^cb2qFx&SO?!jXT1RbE-bFx7oz+xP800=vYSllOV9$KJSdPmRa
zW1*7J782O+E$c0MhtT_dlKbnMD7v^W4^!`a9Tq>1fvsj<-u4Y#5F8{*O*Va}w2_%|
zEfgW|R*&F^?c1Lhliy!G>7g^5j5`U$3h`h%!uzB~{^aeb%Qv`CJ*Hk1a+>K@zbeQP
zLE*At4|Kz>V_O-aJ)U7F<@7ulvvIa7`*!2<lCQki8CZ%g)Ig*MiS;?WUsY1Y)^v6u
z&hMoJ`wB>v6u4GeFEW6!q_D>O*3KM4wfQzsB~I;jBM)Td&TFS~CLwII-;th2Lbp>w
zw?1fTXFEJEfn^<+k^3`)HgAdRJ2W?Y=n$F{_2X?>LSyiOL5Ce&6Fpr6dzZ=h1*}eM
z0`Jdv$+56N;GkmNz5;P(!uQlBHZlWA)Xn<kP(cyWJ!gaUgE4xu!bMj}&gLt65Q$<S
z8OKJy_hoj))0^3T`RUu#LMN5-j~JyBMtxWx&ziZP0WzVsp2)vYVx1+h+vbk^)MKOY
z>k*hKsD?IQEFqZ-Cy^<j4i*Il!RF@2zOAI2KsJ%k^z_xnb=yQc)@&@*?DyvGhW2u{
z={VDsaQTtF^Ae3b*fOZn!9A@W<D)KXg1W{t96qo#Y662}gm1GX#@{Mflo%1S<#Gm}
zjzFAay7uxopC5xGMwr~Y{jm=Z6N}3o<I$q;sQB0Pnc_+tAligY(1CPht>is}z(xkP
z7vp7+ax}8qgJORQ(UI46k~#d*vt=$tqNJm<+;KdFsgKp@t+%XI;{~&x#i6LAvz(dy
z6QY}8_ixw30y9)*B5F<AlaKcv=LXc*GZ5}hN%H7Dxv|5meW;XuA#iM=(ZIoi%uLgT
zx4~;$G`QZccA5N!(4j{ivefhbr|aNxViJBkbzs+MC8Kl1;-Bbi7R-P+EeJ9)v<S!P
zO?8*@qdEebZmvtOob(8Qf06_=r5ki?M%QnNb<QJmZ@r|SSn*@z#Y!O#8M<W`ckv!g
zbF&-#5%8|;qleT@!_ldceY<S29VO9ZB;}Z=op{NK*)GzmvnampnIhz;&aeDmpK;SB
z@U0&=(E*2LsE`1D_onx9>@`v{Od*M2R54=tmp-DqgMau;l&;h$mpU2gKhZHqz`0CC
zD&YkRG72&W=K>F@a10dSF5de^h`cSwJ*<jl$ioW1AVh}h{CeS$3a~GwVYn)5q>f5q
z3($r7+C@H#2UFUKja^I1xlH~5H?qgqTdf%PT{bl-z*Cfav$0j5pEW1bqq!G?2}nl6
zdm*Yb8yl%Fe$2I2s)mNlv0sWq{8gn^CGN)%$otNIsfqZH{8pottMMDWYcYvU3?Z#6
z=KjL&$wkcWi7B&8sDcA+?$KT&ogIdf`5+4XW4m2?uP81AJRA8hJ5Jn1J<nU3!YPnk
z6wQZ8As8k=?QJ~L|GD?pOP#d9BF3ypIT}jJwu0DrYd4mqfg1~8nCVJ85|_(tZ1E=L
zu?*SVuqN!fITCni%C1ylHr^Bygo9K)`hzWMSmms<USrvbMx$A!yF{jeF#){9$*#Nc
z;u>>{QLHTE+QcH(KkiA|)}fBTsIAjT8lUTK5N=9PG?S|Msvpn2VV@Q^wp^L`l^PAk
zDf=st&Canool!$jpovBlvq@#DfumHzO1W7o8K{V&%$<>Q9`k~Yi5-BLJXHYITMOtB
zSLOn98~tV-B5{6+?bzpKbXJpt2a@xPFCg?k^^I)J_O~JJl_c`!{**_yneJxGgBX^U
zt~HlNlo@A+neKLK%U^i8=W<4u{3#ljZx>b-_=$-e^$ud|=f0alMd;*qAy4Zc^YASz
z#7SabYhbmJ&TrdAve*kIQ+*k<u%3@9#}`P6HQP?9Cu?eIGSkc8BhBsA*3yK~od=KJ
ziLx3MY>oY6!J9ve?A6+h)T9vd<69nk+WLS(@RiNX194YN4K>w&GpWrid?qt}cL;Gs
zJwIZ#iqA$KZCt;!(mW1mzqTlCLy2hOqe6))bggK)cOF}IbbQ{DnQQoNmhAL+ZXQNd
z{!-mm*0F70^eou9p7c<4i@Tx(OkVHspmj92>E11|+D!CHA0HlZIk7z0spIk+xs3j)
z!*5V$?2%>C^|F+VPw6WN>Kx@2>i)1jOs)Xxpe5sdg?X;;PiyrP&bPvi>NB0%9>}m-
zc3JT%*Wh*(y=wv^meNU?Bx-q@3QL9PGsPzxKO8#lsi?~TvJMyILc+deqABTvL2`a#
z<)f+FecE&k=ed9-%X-}v0IQsZ>6%;u8IuL0V$6t($5&9SEvoPmB9wcRf3ZXn63n<r
zAA_&9GJ79P-!Hcl8<a>Fu!xK_2o3y@opR}EG~7_qU^VFqQxkRKp@FA|7RmXMJ)mET
zxIf?dL(Dgu_jl#DKryDJPM=+$x9;silY2~%5M=aF)g5`%)Cxmydgi06#~3wxLr(7#
zE?;(Fr2_1*UB+0by`AuBxcsBt#x`mOc3ME++LisxpO?^Zp9B3DIWN_NSqezL{S!Br
z(-$^_R74gjtf16r3IRwkL~FtU-~~8G2REj*$474p!ia5PbRLgrUoqrAM=|8~Mxvt!
zh^N%-tGI|{Dx5v4^jZ}yYDHo(L2!x-_YVR-s}eB38da$(SkjH~7$cK4L}1L+Ejt-_
z36a4FEXNdvVA=tu3lIhea~_U~?H{cGm-}wozr<^}WV$<1yN<@&ziQ6u_SR1ORK*Lp
z6%!tr<uF&Cl$fMwOXo7eLhf!Xj+Y~U1aCO4*&a@j;*;h*WFEAc`_yzgy9yZY?(9k$
ziZ`;1tD%lY(m-I_hH-a%gqM(lla>BKw#i7?c!Lj)G!ug&m=SqK){Sd?)mthj#&!2y
zXU^i`P|a!u<46d#(S76W_m@Qt_7A+DIOrpxMB+S@WmOr}8}T#rtivlgxZ_<A6j9R-
zCF1!yEbP~+u2aLnXL{1>lYW^U6i7i9t~&ihN4NZAKr6D(>wx>HwvP#ZX34Dbm8>&;
z7Q^df$)?sh1$&zLK{Ky*1f@a3&v>N<3uFvH@87QMVD1G9KupKX)<a=27kspA)+ftI
zM@Dt@2RAct#q`x@Gsa-}Do?Gp&1tQ6+^|{qy%{Ie^)7hcBKr7K=O-49NZ$8wQZLBz
zb%=xw8$)CDIxnnf$?F$N?wz1Rk#{>n*h!jNrr#dS2GY*=338PQnpHlfb-ofF$^$^Y
z7Nl(Nzf(hB%CGzKeB><jln=qJcA*)W<%8Mx__<h5wTQ@eaGB|Yia(~%nq;dO&xlIp
zIEt5W3-SZfX!g9Q3=rXm5H&)U_CvnwNxTH-7nc5F<|gAH<>2I{XXC3car_qM#*pPm
z0p3FR8y?igy%Ji8Kku=f%Ne`0mXK>TiG#Jp^1e5KQ&|)IVan%oEPHY3D0LOMbYkS7
z)ztiD2aI^WC(W*@Rw6P>aDR;dcYWSe&Zn26O>mNYWTQR5##=eZF$0xSC>D<Uip`c7
zrKQMr^Pn~6TispF=$4_vTpQ%M7!mdyJ%=``xI@~e&Cc+5tPPa_5)@FI{t0q3PUdr9
z`ztxz55lg8{pK2i9v5~?z@-<I6qTERjH`^Mz~W6|m{=3jlAB+<FA{cY{IfTd|7&}5
zlS30P*m7*&VLAR8+LHvh#S2y;p<{6;3&}~5(TyGL++oua#YL5Vc^6+w{#j8Dj2~T8
zqi(NOb^^r~v7e5nWV<J-8*xI9n;P$EbLFpR_%P(Y9y7L4m|kClL%d`EeF7sTj%2Bm
z8$BGXZ|eM*TWUP%?wJ0#HFeHabb$-CqMJWA0^t@}R4pE6TXI&+o3^R0npLUf@4S3w
zz4^8A%bC~-u9Q(aelHxtAt?HlHJ4iqQG@*sKt~^mjgHmUN<(VSd2`|WsGE&5#K7RM
z|IAaqR|{csRkaQ4;A*>X5-LtcmK*o=8xo{h9?IKk5y<@~x{_C8XTigKEMa`c{`Hrb
zJMe3No5$|MOjK$eoJWs4IN<Aa+v&~>d(wy5CpD*9r-%AszbV{{FxFZ}*QI${E;KWo
z%Gd9OUxvT13J7%7^-TZn3+Fmxhgr6PX3m*)Meo$q)==a5$M<VB{x>cq#``P5&5N?+
zp_uc6*3L1%xaaEtfZO4r4BJHz74j2-Lbq#;7B{O8VN|Weo;|b@gxb(@@8(diG{QG9
z9sq@sIrK1&lJ2>yjexr~Ut1rw`7<D8`Ptjfv*R2dKYgLc)f?g0FvNYM{q6ywIoD^A
zSmJy6x_^Ub+;;!r`dtydL~KHHIM%ELxlpeQbtb&>o5BHb`+JQ_M`-`n*{xv)n=WcS
zrQ2F$z};a#-qqXR51|TH0N<KNpp5r(i4@&K{Si7~fh}ic12>ROrz<LZ+JC#+2Q|*J
z<NLW(1kV^-#CE!Z{C^ERX06Oc;L2~ac#B=JjBqJEo9>Q+IC}#t^y{`>;~YRzhro=|
zT2_3V7g|rP`FhHtxSU_RZ=}+2=7Oa*timwx`xjKDI9TLp?M&Yfo^i1Rtp+6^!YOrP
zI~`%pfcyCXyd8lNkFl}H-BWA@RJuWM4PDQRV3k_T)#sf9iUG4WJJ;XK4Xc?iC-zl&
zpQdF-R?`3U6{N8MAr+mSN)_&0SAoyIc)S|Eo+J9_Nwvd}(5=p@oC?dQq)cpE{wJ<1
z+2b5LKz0s^EL3T9L}#0^*~P<bo<_wZ6#C^0|5!VfimumABIj&YX$9uPQ#9{g<V8ws
zrf3H*-O!Nn{HQTIx^}$zqP5HCwEL1eUoqfwHeXO;Ys$3mR=G=7&u1!<tL%S@_k<kT
zK!R6MJfRhr{FB+pSq0-OZ|(;&;bZ>rwen`=RnOl7tEq908CM)0CkQhdvSJ|4jqPn&
z`~T$v<i2?I9>drWS$Ss)n|d^Aezbgb3KhkJQzs&F(~q`BxHXk~(gSvN5Z|&nhNTA(
zJ-?ejC)d#Nd$l+9_-0#=k&t9O@t66ynNd4cS~Ysk3u-v@o$0a@ZyPhs*>sj^O|+{;
z^X7p(idvqB&2TFO1qZl?c0o{UgIK+NPI%Z*l$>$hY+pWSS+9IgL)uVE&r{yeJJInr
zc%Di3Kq*i#`!~DNQV$iM%`qu&xVG@#=Eu{-r~NRq@6PwBUsH1gr!;0%<Et&@oAeCY
zESfoHvSJ7zFi_)ebU5;5o5j9_JZmW<)@|<gnlAl1)Yp9*E`anadN)JLxi5ZVcsi$(
zOJ;}+<9`h;xYozh(h2<SnZF`FvhLv~>Rkrz>Aguqp9NP~;w7uHJjUE9q`cinF6#2T
zToqz(MV*gu)?pz9n5Qe}+8bIo8k<{ojW_i>`R3M?Fr-|`Opm1)$Grt`drL}a(2}B~
zgGQ-HpFNY#(fWe;P_4@$L2zL_7t}T@NQ)g;c$5B~8Y377*WVi6BB0N5zXT|$tXx^z
z&gjIaG%z7`5}u@r+;Wa;#rUdjo0n6%Hg5vjHtS7?ocgzj6v{KavINLUlNhO0rt8q*
zp?aWe6k{fva<~b8A=yonu_q(VeDHuerUTz@!r$|6(q>Mm%%wNJ0Eyq1wdVE9Rwn0A
zr-V6;&er%zxqs_!*vEjLY>}Yvxtiv`p-Fl%e92hdBQ_CU>K)k83(>t7$gSU&S@Oo>
z4jV!)@0?3&V2TzZ{&Oo#GqI<Az)7P>g@G0<3HqVY-O5%>tLMDC3*zroqXD2kflWgq
z6GhH@D~8q<azUSgUDc&&sg08Wg-RQ*s?BL26rw};`$Yt=qS`@X-ArPI^kT%dNAz^l
z$?5yu@e_^jdqJgDPsXLypz-`xdzICqN5LE?ceGmc|Hlol>6DRFYj-a=4;_f0vhuT0
z(c$TADmn3e3Q|!aR7l!$P}fLa@(6$sW;024+Zo(TIQRFl)5l{DBx$vPw6F<k@qZ%@
zl06mdCuSrTOb+X(CHPO^BDlJ37$jwiZ~wH9h5s`&xzX`r{Et)0Mlt41#dZ-L&3rwD
zXIDu64B6bUcYf<krTrA4QG%i`X&-Oc!v<&%ab?aJIrsr^w^r2}{}Z(&?r-7NT;Y$~
zIo&SrcRjlAkrZZ&%nWDRx)V8wDC7PvFuy+iy<e(W`#0FT95?mSu}vt2o3N~HzlyD@
zSw+8_Qfzi1ROx&s?0(EZoFrOVC7aS8hS0)i_~zKc51pf#Gw*FE*)M3>ec506iRkcW
z-LS(6+A*T$DWMpM4GS-=k<12C(1}18|1!eUjfB@iBgDd;JP3MrQU-yFk4OPiRG6{?
z9PRzDg52lipuy1|{P*8#b=h%)J6{Jn3qLig7$IM*tZfNDK!P4-4^#YPQjz!zJ2)k)
zlJz87-^fd|aLo|TV8Lp9vtr&v!C0xxu506iV&f=*#Wdu#@E%jJtB6l3c;7XW`!`9W
zhxlFzwN%5=Vt8|^%WvZOt%;Lt!n$Ju&!x1n!ne~XugE&f)|JXi9Go+PxD#vsITQ9O
zNZ$7k65QNX+jH2BOXQx5;$(9z$boa(6y|;<!&P+pG(oADNYtR~RY-U~D9js5u<`Jj
zUCQfOhIS*YnEE>c#rScHgGwsy9R<xrFu~b#7&gyUF_|*?RjR;q9cxKgq`Nl);R2@4
zhlM-Ex-&Bvwil6(mgVpYso?(VczHsSKfR9aQ88Co){^2_*Z4bh+&goM6e$tOa}F1k
z0p=sF@hzJ!kpXDW%1P`^Egd&{p@B+oq*os=c-NY2XsE&19a<qNSfpqY=+tw$>>+>Z
z2t6%yJcWJbVOd#eLWY{;$PAVGV|@3s4&#LSLZ!&SQrqmd4MrHtMYnt}8;sccw;D9U
zy|peN>R>)`P-U6^lf!z|u?48_cV3(Bj|z8a992<X(gkjfVe>ViD<sQ?PGh$FDoDum
zAJie4K=>lfw;Hy|2UQ$Mi)H)%&(;X(+ZRO$_;GDcheG6g5IIo~pWk;5;-Z$h9GGa4
z4$S7hnDLN>Q5$XShMP;6h+z{VlB%5bZLi91j!r_86&X9H>AzQGd(!Rf<S-@6hDpN4
zN2pl4B`{D}rHGom{{z%#FcCjDV4|gBI<u^V$2B}Z1MAQK0Aj9=Foy$OS*Z`cKJLAI
zQzt)d0nhz6#Upx?^qc8t?<qDTcRMU!j|DJCOqv);?&Pk2)>-H;V2`Jjvv>~z$7LK#
z>gu__KFKtbkx$2R+0jOW4y_W>t-1OH7fOtW3KwCu$|;gD#3hlJnp>%onX#>1DSLiw
zeVEzFwejB`=<0a@o>%s#Ct$m_sc+7SO{b#IKhD|d;3w<oFeoBvim*KiPg;E8Qg`Z`
zqY<9krK6JlM@psl|G3E@rdz!YX;v*4N0Z<D;&q%2BH7bSpg=pN?Xb8C^7wQ6(ourO
zPFgc9(<4StGEOi8qG@7f^=0g~ZOhP0q}foo^A&LXVY7>2qQ@yQ%kt5SwF0SmcR+{F
zSbO|FUnq^TeOoMz!c8W`sZ}lnty!z-%y@pOYq?!Ed1;|QmbCQ#4O*b@kC_7r^o9bc
zD#PCr<!~+^ZFTqIA%D8Ev$QhcWd05y&;%lm-us`|&xOj=y}L>i2MTOFsqZ2KlLa#F
z4)iPd-98Tc$}2l54MGv$Pg{+AQ!DSi%=#+H)hH<uh6H-<S69@EIiI7DneQaDynj*z
zP^sh8{7@hcznv6iPL~u>?hxltyw540-UzHdkc>u|SA&r}U4{xu5RX@VC8x#Q)!QR%
zqrc6p!Ts<ETl<cIh!^@h`|7CHd;unYQ)!X?sb1oT@goN=IHNfRQ#vJ%LfW;wbNyNh
z!#pC@Qe5Cd1`mEfKO-vSS2-M{7XzSt`HaZiFs8actwxn?iS>!6k(gqb*pUui(l1v9
zkI%j2;sGL4JG4Lnr-ws5d^J1~s&ni2zi(9U%QVMY=Y#JYSQJ^hO5|R^-r^QLPLvD)
zhobMk_(#dmTbz%85s{^ykK>lIVf*Bf8pLclQqdoRU~VO#w~wmlzBsS-qA%;$zdg+|
zbpY)y@0;Z(#Z{7jHs}o>cGVCGnyZYjHyb7~9<DP|YcT-pjZ6p7J?c2$Dj;hJRv}Hv
z|BPfr_E>=5!N5niz71Uj$lNRy_1mGWkeJ$yocqbwQgs)}LJFcX<on(Ts@ynmZ0P8f
z&{hz=g5Eg0$DMPPKM%K&GwyZplBihMexg1BQo^UsG>8o#qybdsK0!^Ja^*C2n9tGD
z`}g?22h~p00MIk{qJL%wR2W8Hf~uY=UGX)m2*(~<MNoTOwiIClKz7kJ6PyW^l6wUe
zFe;^iOLzHm$>RP8ZHN$A*@zQX!;ob#21)tMmxo{tYde7j4}oKiSz;ZdNIL!3Gi{oQ
zhL0T4>~%B=SYEgV3pjJoq^VLIoe}&;nvk9r$@ol!#%vxGK8bay1DxoOXwE01&x;TB
zB25|`1vftNf;mzGf`v0gJs#oKPQnV~-YzPeOBVQ|qtTlL-yEui)2**j1$it;_-I%!
zX1|FJl3oc21z_%)R!tVRi}Z8ngX<XTah$~u*EIj+6Rm^SZMZUVs0_NPSm*-k&tVy;
zL%AI)i1PL(+4sz2ITBtFaDxN_PWer%M*2C$d?{rAU`o~u=3fU51z*L-tv;sV!%8`l
z7>cUIAL{7!FlmV1y6vwivX!kIPr9EDQG%VZl!yt+h2Y*t*u#NkP|31x=6d8DB2s`f
zNeZ4PO}^S(rPH93vF%6DRJ8CWQ(Yvfq{4rPp;(+EIGiRU4*%M`LlL<ZMn@xg`f|gL
z1h%v;a5_FxUbGAxgbtDhLor%lH9$eA?laDok*t&`0_{bL5g+VAPvc)##nxl7o}Q8B
za#E-GEWWh$8>m!i3Kwu#aXBbetJs7r*>~uwmIw0{by^a~MDVAllAx=b{wb=CRSaj}
z`Q=(!nVP9i)`|;q>)<_^{E1Lx|2c*Q2dRCodNx{$ralX1j7H3rfQ~gD75*<lJct2V
zi8NlXs`KMy*-L93CtTJJRqC({B4Hb>!z`)K(pJuI9le&gk44-78%&}~b$H4@=1JCG
zcWmD#tIdNMew-&x@5hcJ#pP83NV9UXa4y|?=kt+sRs9>Br2x-edV0&{NE9)PKwn3?
zdD<EVpP5KbWv;!2eQ#O5OW0l~?|c1ZRhw^$OG`J!Cz+<#h6mHi@X_{Nur8SnHe<0y
zHan>|pLk>f54Q*EP?-lkJnB%LR@UTq<W;zA&Ni#@njnd<Qk%9~g4SOZ<pNvwppg3!
zJq`=W%OCU5z;6pK<o>3VUJKX&W$gDNfXICUE9iaQRYtC*dH-6Ydr<eFDYBEXgC`+T
zS+AB+U}nMP^3QtDf)3hxfDC)?!zS01l&+}Z;a*j$Um(A2?~c>+(Pk2LFjX7A)m*?&
zyIe^3WQ4gM@4oo_7>_JaNz#!m(=f|%X{jS{`}<EheXnHXc_dRT3!j9sS#wA<xY#}c
z5)#c?D0fB@fX!{%=<c?^gs2I5q9}QX!q~65(zT1u+XEFNlmY^uF5s}LKW`xkt@_7i
zMhKh3XmScAik#&#By%Zz(QxrPaqdcL;UKIcwoX+i0oJ%SATuQEkJgHS)(Q~^;D--o
zCB!~S41UM$ZiqEg{48h(vmI*`MVzKGms!U_qNGLBU?Rt@IZOJVG~$0!xi8Ut?7y)6
z-G5EWpKXXW%k6|fZ+P5odlZRRuysD2PgoP$yc5tc6#brxrQS>j7hJ{wWA9<;es*iH
z>G)`wj74z2Y~q>q`8$&M>2-VuQ0DTNB>C2dWY`A~RWxM=ZY)x#TB2XF)jY*ufUVT6
z&3vkh7ooFVh(pCt?#i9Um4hB6Q=_$E{Yj01TUR6Haz7x63t-^WLL~-V_}=kl9&R`w
zsv4AR5kH9a=tvbEC_!%C2y1Db4N{=2_*gFN<Q=4n7c6&lm8hgXdf!evGq|sQ?BC8Z
zWGGwvJ|b|NAeS6GQ`Ys+b1j3D6Jy}s9I{_IUi>6Y2ctm*LBOAp&{hCjXY-R*+v14T
zLU(`SZ_;tz0BaG6yaZ8Wqh?52+i{L|h8?&{usW(*P9qT>Q>Il+0_0HOd4jTDX0<#G
zcakOkjiCEius+s`d%F&cXqTN_KS+xStEK~0b%lxc1Ad4^LMfIZi3Z~QSX{b9Jj<3>
zquH)y29;<bN2a8V?$uf_#w()<a&ZkOk1F$i+!nKii?SD$q=^4}6Jabl-%sclMR&KK
zS?b#K)gCLoOlP8=4y8<bEYV>IZ0BMp_`dAXRaf0MBG@V{;Se9@KtS|!;T~B$VV?xD
zjB)Jddxhitd8Q}z`MSSRlu%h-TsX;|XerX?sJ-Sync?$Vl;u-LVC?i|Tfqzo7qRhE
z<OF|{#ndPL@y`xX*1F&$v?K{5w?+)pCzSMdhtn6?Tqhgkn?IwEAa~AFchUtB8mcBf
zp5x<`OWEbeRth6@{(P7$z%5L<(C2D}Oy6w`JX_PkFPCbI2*kj{Og>`ZI(*r!fG2Vq
za@0_vubT=!Dnb7ia(4<XWbr;&$nz3OmHKlw%dmb9125R1mcwDHC2UTZ!z2T)GEC_T
z8UH@(2mi-e$_f1HbE|EhIC#xibla}<f|~R6LOqNLvupbq&oiJWh}%j*{>^<SDI6Z2
ze9u8Wag2uznx8ESy5ShwS#Fo<ht*%BD=mY51FmEvb|;T?1qdACuy2CjiRFci{|(?J
zs=O{*sjSmr@;dV4Cri$oo2L1wJY8qMzhTLF-Y=1EE~BK2h=(6w8<#KzmWt2dAuxHu
zgc6`&VBvOhQA{}?BnnPpk-{bz^@);$rig+rWu|Ls)x87B2hGYkBm`r$mk8CQxDR&q
z5AyP*s&YpG<TusQ-Xn&ClWAm)ydw9xD(8KWCw|Jw;U8Y>*m$CjU>!IUnPFY-!Uth+
zT<Qe7oH1>)nU)4jivFodtx6I@<_Gfr!d9IfIa2u(Aq^C@?A~_#rq1Y`4Wq|FJQ0-B
zow89#r3)C+#oA1FMvX$>eyhD%xpF0VIe`n<I)z{K{!i7m>|jw7udC>Z-e1Ta&|S6o
z^+I$_`|^amEKw<HKYY7N8=7Rb?`5R+rjzoG=yokB5Zv;3*4<cul(i>zI~22}i28c4
z0X}yF3sm6S&E5eg&!`|bGkuRX4h{{<Bn4y;NnW!2eZrMwUE?}<nc@=Ityi0(hLGT6
z4e%bwMVe6V#VhDr5Ff>{J)E9cvSFVzNSlK4RYM+p6?r3Rqzb;niqIlQ5-6=Bof*OZ
ztYPB#HTrL8=wZ&n%tSGL0~&YAPn7waVCvrm4XrEBi`kX=;+QpxR%I*1n@A3%7H>TM
zOXkDuM#}v>q=~P)t!R=FsYiZRMn~8K$QI|PNYYz|n64PjKRR#0R?a;hF9wA&I*xMI
zq$7-a9bVGj&+21mS&LoV%UQJ5;79Mv;*Ro6Hqs{Sz?^n<C_z<0%gR_;FesD93wMJy
zy1U&$Tung(?bDZ-&+0k3ovzQ*j+H)9kSl1TH>@lMcQ{vQ=sfk%opJ7M_j7IUw)<_P
ztX9bsMQSQ?{Ump8=WPm;qw!vSffTgAG{_~j9Dgk|e5jBkESQ7S$Gx8Pt)WdL&RD?3
z72T*MDSCF30zE3#)eEjU1=E@Dt$eH2dtcmJL!o4#({Nrxn`1dhWZZHI`d<WCTaUM$
zsEkIAs?GE50CmTu*gbDnS2ujytzw<WW*+P|_B}Lx*}-1@?fWHfSirvU`o<f(-PG1s
zqK+DUTUrJB-dCkYD%=^%)MeV{p+VFtk>U>>Uc;O<CUuc2byjUM#Yb7C#j&m+%FiQJ
z5o>dK*yO$-VKA8XBFQ#UDilbHMuF^_88U*-lIMy2?e#N@TUl5CLLb~kQV`Qk(m_(z
zcp$XrM*(#9(D2;}jor?fV@~otIX5--w5E-fbUXkoZ!lM?a&sd*AU_k2uFHfMYCPVZ
zKwKYiipANV?aP$d(wuXNC=L~90cY&Gws)OQ5Q?1V;<NreiyKD@mBE?HGb<XU3pQR|
zZOc1VA#-%cma%}mo832A7J(A_ZDZ)US(bQ8gm7S{v<a%s^d<HMv<_nDp_wnO*A+Wi
z7-18cqxN~y-QZ2=^)tenfKxxbkNV*z^ucw>fNpObN{ua0QsR3M@x~p(K}2ke_|Tc<
zh=6ZN4k%%G4z3auBK7)WCsVA$y|MEjiAwjVxgiF+Q)P-$U=dh>aVwe+_5-LipcG<z
zTHhxE+4r;>$6*yE+Ovn8^mAwJUqOTg;m7q6BKAVCjT~GrnGhndS$xMt1FF<stS#}x
z{QT?rs$`{FD<f{umHWcq$6~Aq*N&z{kz+9pW?u!mvevTjUD&MW+b^V^Qa|htHsq4!
z`|MLDcXW4-RT6@G{~WN0*c&@SSG6NF@iSr2e0=7JNEH3FVA<x}pD=zfASU_E*LJhp
zbD?*fO?dMjyHi6tB1S$Urn=aKplCQs+M#6aL@LI?aHtq4{%hqlddvYm+5FJRvQ4q;
z%c5f~TlyDYBXQ<fu5Ohz^!EyZ4t}@!enNlIp0@nYb6id6@-2m<WVy&EE&_Tk|A)Tt
z5&RbU)4>%K3`=EbgObUkTgLb?$X0Fccoo!#&z9J=al$$Q!@s%QU{nDND}n3vlt*pO
zE8BO9Y@<EM9xfM_8%hBdpHZ^<T^}#9T4<swPCSv46zy9GyDX%$&?5D}8p8SkWl+I4
z&9n0iBGTM@&5k7#Z?FDB4AgS;#S3vk1WeKE0!81=7GEhT{UwBT%!GAHP(rcp^gHK8
zXg_loa$Rahkt~(YU;LNvAP@;nPA{E=Gu>Jx|23111HiY3vaiD3eKVTAy+yLMuJfXa
zTY@6FVS?Vv{)K#yr7nr?Y<ms3Q{Ql&4iV(|zV>>~u--+yqCXB2W6OSPSD{P&L$Ogu
zZYCZ1)98#8L0S;j5*2+D+kv3l6c&#7rx~BqWr+2Wg^C_Luur?p9DE%hSWs(|{wZ9T
z(x8HbVz%izqVqe+6OtOhK&Z=k*Iq)0<StHBc^nclH~ewYMHqGd5~CN?mc3Fbb{HoN
z{a_>329ti@RANub7+a*ny_$#NY5bua(WLgZiBGI=;$tT<!4IImemj6UBzH)OZGS*#
zrrt;|x_U=k)q_fF#3ex!=i(}dDCZS$b9&0xsBrNct3)XF^8Q*M?B%4|=|{{b8wy@w
ze~KV=*&)oN4E2Bkin{8W*A{SlUiq=437SB!3Exz!vQQ=Aj?l97Qh(i~Vtc-lp~T2R
zCt1>kDF!pMTNz_IcZZvbg2D){z4~?P_|tno3B^cd9#G_d9h5aveJPYKTto&J)Zn88
z-2rlb!AH0<7zV5cOYD3U_X#h>FsM_|eC9GA9-6p~dvR;$KEwNZ|HUZBZW}&IEHuVq
zf4@jGtS+Nw&G1*5c(q+tipE0pKRIF=7h4iF`X-~+S!ACqz(LeyU*&T(y(BPRGcAE+
zv@r!7yNNf*3P&85xY?6#?@CIRwX3W=-wlE8D2@yd|7We8{C(Kq5n0@?=j{dH7PXE~
z9b8hlLY)CcRTpyy2BO(L(GjsC!47fiIIh712LvAjqEV5L*{A$-4nIZ=>(r|r^PN=_
zKc?+iGyS`!4B6y5*x@6vcd6V&T^5)4+2tB0NOE|+&csTbFAHM?ZK!=Z+_yd(o8>lB
z<wW7^-(B7i7LzA!xJM1c?PxoG&9#NL@c7<0vQrs?yYG#{dpk9{?ZmJOa53XF;tTdK
zjhoKZ@}^725Y<%|!PpI&vxeNxr!-&oRb{3$v<W(4LDdV5ImGLeCO6q%K8iKN#`%LN
zne28q$E%nnPuEIl7!UK0zwsh`v?@QMk5G|q;!2?-dQL0}_Ol#o6!ez<6vd;@|GJ!7
z07+ZlzC#?)_XlE7Uw=D31cEb4wxs`)SD#@ms;L*$27qdW%`$RGRd&OYjktc>uucGL
z@eXDP0ShS6^S<rGM9x9YVd41eH(<k4=cL(R(5C$LNnkkermMv3)ZX_u>NK2Wg#W>f
zW(PJcMRCu(AawCxIZ0Csq*Sj}1H_tWzasIFuJP)t4Vr_2?aBegp}hmON1!@Vqh|~e
zahf2~0i<wTQ+{F7t)|p5ka&b3<`c-|=1;Z<y&S$`JErgO1BYskF&=+#XRCPKE?y!?
zA9~24G1gY&=0+{m$>X3@NQP)PN_6#?%FcWT7X_3<TbtOdIpS51yJ4-X@!h}#+I;_2
znWI=6MQ{JN4bI`>i4Suo6nzHN3qy$5nZtWJtcE4a;7M|tUnerI*8y}9Ut#J~Jug>2
zCGyJKhTCUqYQ*F&e;II6uI8l8wOj4px1b#k>_1Y`$sgnCi|YM^_vF1e=0QT8DPzz;
z)jVF7q#q_B6Od#6+Woy@IoAs3HWx)M&CWd?V_;13pacdAUD*^+MYwaT4vm2)R*e30
zLaG8M4mG^>rrGr`*J!WtA3|^GJt?s>gRy8<n(UWX?%(y*^A`X7y7*A4^<2*Os|H4D
zz=5*l-$AR5ueTn(3J;YG3qT~T$K6r`lG+TL{l1VQf}yMJ7}Uo1rO$tlOVxk9OjK_V
z@uII=P+<_B3RSHJi5A(k<L@?{KY1qn4$fSi1H0^B7vpAgJ<kReE+rY%@h?sS2Ea&O
zyUIZIl#xxm&lGwzB2!wGs-uP>A2yYxHlNwha(zZGmGO3(S@Z$2c~a|H&%1g*TVJP}
zqDE4KiU#5jIr&FTMWI%_HGVdyrQUs9D1A8(ThwHRcCcN_{iw=OGUX8@|4kA<#3-1@
z71gb0WUi{^<kI3vib9YwNq%cVti{Y=3}LK%`WCXp+6BsnO#6ZU+Aq9TU?OZ|0wlA1
z!n_Z3zmJj$2g#^+PIfmde?fbTZCSiRAOs<q%@anVMryn@hv!f<*2*cOX~U33fu_}H
z7($l<PHv=9FU`l|kXEuYL4~6tEABwaRW_dOts$@YJS+82d2;(!oV7?lD1uU#%r(OP
zec(<@%Cx_44z-lTE_0yyEn}0aSr}no4PD&_^}2D!;fG*s$8UCslr##PI%%HC%G%UZ
zz_FC3L(x)7eo`x`nZh?j7^opUh99)Zp?!M>R$N%e@dXGGW<-|TeHk#^8|qW!6+_mn
zT9<<xTRI5ps`dUi7it0nrw8Ku&@zq3sX>_ps&_?rV-+V#+MX>=OTO5oHnPh1G#(Q{
zT~{f$Sa*MY;=0ay5bljUZh#U;E|7D6#2iWBWnJ`+W5>*wT4S-vT68jhm1t-`N*s7<
zPbS-A7<EE<`FXR4khYTq^v{w^re1xXntnt0_h$vI8k{HZXlMG+!q=Ux@8L2<w|?b0
zoiR<H(ku)SK123w8AJSD&z>>+fvCfGl_G@uZ&kSr{eN@{5C1uVyz$`6dihxgxHKv>
zTHz4vyguk0du5fWwq=E~@jzY*gT+ifUVA=IqYQnU`KRp70RPIEncK5h7kHd$U|a%5
zmx-`#i~Y&-sUpV!qT@abwAGt%wzTWc+aQTG^du4Pq&2ffG>y-MTcx@HBennU+sBf-
zGSD7q(MV83!45?+1O1W&=SdletZ;IMg+&g9P`YZ`%FjgpTxt{oh?Q^-F)f16qGoIw
ze!ScJ*3|8R{ymxtyc@JwZkUGIGbkW!pdVSikiT|XdZF0;PV)oEkNg!Avc$i7n{n*v
zuq;B;({}n;mB!-+ZolUnLBn4z#Fh{^CeRUyx0Si1HG^{b;`Arp(Pi{hT+$SiK2o2(
zv?Pm}`2TVNoJxi+WoDTT5^$1=Y6au6;t1PQa@*G~xPRmB2o%I^d$@d@=IAMh@g@s!
zHRbJ_L$m}uOk8a}z4_`so?l-QpTlw|m9x8P&MY}jlG7y%SXt)ObIbw4r1Lx9O`4Wi
z-puTmg30sj-PUpv;_0@8<_;^Gw)*pu3Qs5qb#pQxcT9xR!c%LO$48V4I1XU(3^(|4
zWV^iJ*Dj$r@EKHz<yynfnz2^nF^pK#cWF`WnP4aS`L5acJ88{q;3Egrdl_?crD4R|
z!h(mmER~+J)?>fc^`QCtsjOw^A4fhMU_nFMvnHtOHZ*74l>U#7J~Htx+y-jcg$HqY
z?mY^S)i`n_clegIs2VxM#Tgwt#G=3p&9=XK`=wrGn_^GVYgpIZH$c2_o$AXo75{q|
z4MU&_1+R<w-@}%ZT@G>#{ACw?=j$|UiMcXB$n^nWwJ*wO#~6z*?IOcuB%dh?!y{8a
zVyfg~7ZFPKcGK5JegrWPICYK6N$lT5<5O9<%(CB<H~X_TeX%85#}^3ryM@usxWKSU
zw(yg!v)fap!+6n`q?5w7&x^0Xo{Zhu!MtWaVuoAI;+2?k@py|K)_JNcV9`8!{rZOc
z_$YrJY+v;hbQTc}J3Z+1s15{7_VSET(tYm=rj;tqvsG&o+1Isw_*ElgY=Em2FB<$q
zkd^9~3bFJn3t$GS<F;3ZX*+X@JaZ5JIbr&o+3t>nC?46eTm+6(zb+<ph>OtdX-(h;
zTEQ7g9@x9dhq5w_^(s}%yCkuj4V_8xPQ2eIIpJ2y7-m^}?#vJCVCOGPOO65|l}0>K
zGGy-O2i!EQ+qeH=&pfP7jP0wIQ=2d+JJ7#P_4PydO2&7&kIRhnM<O;|H!tGG^h`6=
zD@N}n{gu);M{b5jh!^B$NHHjGjh!NZq+E>m&U$IGWapY#!q#`99ULgmIKjf!i$8<8
z{J9F^R*Cu3Uyo<B%G?6Ps`1nJrdLwDs+%Vzg4>7v-X^-n{}0MQHNRdIfzcvVe^p}A
zWkGjs#U$S#lE`_X!Np%!gILMA^Y4#}DWT+)v}lodbm@|WErq#clCw8RmLLHL_ffi~
zf4SYH(cC7?%SrR~`Ar8LLg(O13Jg~ki+TQaJPOLfVY95`38+|~JN;$$3}%ZqJ`guA
z!rcjQPXpV`W|PHe1kUlE_h1iQu$jq$vD{bE{GT8Fw@#;ujT79rVK15NFB^>qVBke0
zEDld1Pvdn8?DtlS&EFDa`<a`U4h^eUTS)&9uP;}6Crb26=fd+(9)IFrJ3f!ekBP~P
zzWUC!Cyze?7e#ru6lnS*J3a?F-CNJLUHG5kCyzg&SLndTY5H`6Lcu?~Bbjk_*sSxT
zZ<4Zf1ka*{?pY-}USZMBQZ&z<?{8an<L}k8R_%b{E*bMe-<s^WD^<RC&%T?5dr~s%
zVFAN>q1IHYQv35jBq9sK6Juj}PTNJIQJv1eU%n;jk#Vt{5EJZTp%Q7-!`@X;a{`0G
z+`oSLiBlCfOWNEv2<9*g9$hEA-Q7@91O8Ol>Z<N}<C%t%8nDp`omzkBFK7Piw~syZ
z<mRc4J1SQ#iVp7<4yn}>Gx=`e&^!xwCt#~Q4nma6xb4d3NDAFWj-)6kz+#HS;|K)Q
z4aSGCk{9!q=CV^3Q$x86J{TMh57PU<nQBaCi^*s)Pm*hhMjuZK<X_u)N?P8zGk+5e
zb@g=BeeVbF&a-yMz^*S~mB1tK7mRIr7W@}WK)f!kuve1R0%t`qo2`(=bMyQc()=%^
z`Lh`LNS7ODl8wLHNDd|`g!brA;dNo_*|y~SlaL1DMsdbtOOo$TA_b9zE}7k7EB@43
zi1~cR-sMr2<3scC*fYrtvJ(d8)_iqEoxX1Yf>sk_LiuCZYh6A#bX%G5$zQI%+33@B
zDB}dyZJHTcM<B^01NY6P6AA_Yy+;$+HyU+c*7<H|Q$i-QF&2|V;v7-!r?RPBZqSac
znbFZSEKWZ<DmNGnGDrj^LUCL!4MVjI;Ce5K#PzeWm*WT|0+~A2>)y#MDLZxROFKwZ
za&%g3Fo(Hh?b2Wlv%RWCr_n{H#YU&aCS|9dIe1K`c3wu)59@+R&Y5C|#PAP;Od=lq
z<b+Od$X~IzTL{Odu$VzIIUIQ2(<_?ADfka3+m*i-pixlf5je<{K)~P$sBzsNEQpnT
zM&Qqyh{I9H7zk^Htgv6a)h<crla&QKYpaVyLtUbwu9}-gD+_k||KK2~8i&F7YC;0%
zI>i9<4&sSpwb+uI&FiB<+$d&7O=QUEy)Fd4?4)2a;z%RBE+hq$$-$&)dvb=)XFL_l
zGo7E;$FcmZj8TvYLACR*-Jf}IS=BpL9yIibYadtd-jV5#ar#QQySnFk?Xb&>I!~sG
zbsA^WwO1r1v-YKv6#u;7&JD{N9F*YOJnrzC7cJ6ik*i$<tIy$bgI<2VKyMfk_IEZk
zJFjhthKIq6=&g~fzPF_3tKq(KZ<DiHH7Z@zB{v>s!5rq!S9c??3m&o0tP-;)k~Su0
zr>uW^%b5?3x$b`P+nbL*zs{CYrlY4<)Z8Z2=?!EO@s*b!xo=e_nM@2}(#y(WZ!i)L
z``}4S_z*M{uG8#OQ8|HFi3vo5K@bTJ1v45g7Lx^@q@iYblsf~PUMJf0I-+3E_=>w7
z+oB3=N}n7p3b0yC7PEPtlS{tvqYrDUFSOjgE|-h_`3=PSTCYzFe0lvi5n}KtO0o0O
zdi|I6z*PqPx$3_QzzbeF7dX-YHGlP;YkO^uG)_baLOO=s82jw1{ni3lzLljTv}H>m
zs9_52?w4=Nvh)aZi(-yWGL9(3s{6U@Fs|=clLE!!PNs^;)HziFfWNP~y!ASX;_E3B
z`?Es?eSK%=coZjsgAyUgizynN=G~MR@Wi9pNlB!pR`+?J91#u-2nPnj3OFn7OTYEk
zR-@XBssAS*o_4!kA(5B<=Q5(#0eX#&L?t`6xp?S1x4V~LZVXKeNll75e!7%QBF4sq
zEs73HO^POy2u}Vm7MU*GmK7>|2$H^45&dpJS7L%gv*3bhM2HNTO$LhzRq!Z*3>Hfu
zp#CuT3UI;}JP6aoU}yNmF<3kfZez8W%{CwIdAf-$&Dyv$YomYqM#>8;e6Xw!VT6&L
zg!}J#e*n7N@lOB<NQEE>acXz+Zl4A|pYgra2t(<DrOQ*_k~23fHA=6|o4@<A^vHkw
z$;TCx1}X-#*2ej^&wvu{cGa#`c~OkOk*A+X=9g5fr(IhqJ2b#v$<pb4HiKlbmQ<hU
zhOl@J*EyBDd{ph$TP8e$>D3{qRMK-#=Ne6-<~F(Kdx?f6BLunNXG^+170bIliW!F^
zDte~8GGvE>ufnFD(*Il@?ipIXWo7o(l|a6er>+RA9n;B%mEo&cKYj7xsdB}Haap2a
znb<n9aG@XZ&ShUGRO)(_%vBLy?d2<Qcp?t(sNfeYwgv^o2{<_no`@r&RRZp6Fc>(U
z3LX#Z^;yo%Fz{CehqqcSw&?sj(iIRheh~s-EWRf~5aOCx2Fceh?^C#~vXt|KY;rIe
z(eX}1W-_6oNfvhs1^;Lc-*DEwEfvG2Vp&x1+T89%eMWDQ7fYF2^W|sC=105hREA++
zc(*&!TkDtn^no=QHRwZBiuhUV`0Tdm@bAiJ^oVLW-@#>Xn(gS^K*!~^l{$xdOpjRN
zU8d64e!O;6(K#yjLbU^oM&r%)>ZJYQG_JFY`Ba3^-K*-P?9}C3R)W_JS8Bg_<5S`7
zZk<|J`k%{Ryz!}CV{@mYlJ~sv%v3#RBrdXLZJ%<@h(flj3IoypU-9}7ug+I_OEUCv
zQ^zZ1I-#c%6@P%k6A5H0ksgdEQ8B3NFv^00B@hX;AOe++s`k4Rc-4=9C6aLz8lD=2
zr_iwk@@!T61aJvlJP}W#VDZF2ET%XGu#KHa#u3SPt7^B4Y<K7sy5Okb2miA5k<T6u
zcJOx(jtYL{vxhhSzYVS#+yO)yJ|ffkQTyb!M8XuR9m@|U|NK7Ej|;#*;@&JDo}DXW
zmlYxR`}YtC;tt<uURQ|yk{cyZzTYMzT`?jUlcicLUO!--ej+)%q+0Jm>VBQt&|N~M
zE%A-fJt#P_{IJcl++vapPgCx9_Wbzeb$vr^Mx*;|O*jIP8xfBq5F`Vgdae8KM|thd
zyq~PN^}9Atd}i7k`rK~k#)g3xrT@8n^Sc{cU)p)`(04kuZpm6WBl{O`e7fh2XGl~s
zHzsVVp4%FPNf}X|T!+t}#BwB7X(%Q?#?$$RBSxvV_kVB2{9+k7dLio>*}Z6pL|Ya%
zLx33I9epHb_jHE?*~T4Vt+sM6z8hU&;o?t%@yc)2@j?OSx6T*V_VpwAuw)EM0D;B9
zMrEA8?aJc@c#Fvb%GCuba0OpH;V$!vaZ8s@=GM&}7~Me%$d_-vsOVQHgbGqHnU~Jh
zkLeZtij2oI94o3KEGpofzc?wdO^^}Fy*BDrE@1h=q#qYxC=*HX@he~#3B!?UEI$}S
zAy~CWOTRC*kVR)!H)3c@W7IWMx<LC1ol5VIZp9tG4-)X?XK!_R;d^FhXa|1n{E}+V
zqmQp0u6^u}J9N+|b36q+j*OM8Z!4>c5rp~a%B;+ZBN9f{YB!1;M<9mBq=d($;9-jh
z6cwK#8SL!qsCO&ht2%CFJRzw0$bFJWuNdwTt$%vUk~Nu%;nADl-PrK-{SX9w_|hLi
zAHekPmYVvGWgAk4#IoA@PUQW}jQDUK>+FRp5}Ck{4DA-eq3cEhG#G)X4KFS%j%xoh
zTePPBztkrj$ea85&K(>K#X=;R4SEYa0rPVdAYAJY3xN<cxZodtqKK^DfCW5|q-K-V
zCg<ULv~Xs$zzlhWpYjk0*Rz4-3-b4cLV(NlT5T7{YA^8!E*f0QajtVA%w)-qMxEX*
z*j&|tUU#s9&&TI$vt;AoAGygGX!0Ec<}h~2e)oUd4}Mbevxf`IK5%}T^A_`xHYZ;A
zpW^zj8j|i$;w|Rpd_TMXtA@&ts^HMoyyz#7Kaul;Y*+0b^FZhaUtEa{UIhN6Q1IJh
zIJ$N>=5|-r4#2O7#<Ho{M0*)Z!;xxJxp!^|(_<LW1h0wJwhZ~(VMg5H`^x&j1IOom
z-q|Zkd_H3%R<86iXVB&Mg`0o1TR3=hE{Bd_J@xOO`ZwL=jt7&f8>*c)I!Sf}PDDH>
zBHpchHyVv{zLdF$;kmOr0?6IY8@ISer^jAA^xbgJP}P|mz}60&WRNJ;X>z*z8~QSi
z_zU~(%qZ2$uMII*O`V)}e!*g~STe;*BAX3*v%%o22MnBX8BSt=1vwM2^PbbqJ`C6x
zZ9BBf0L*5yJ;|{x;qg3M=xrCoc6~5dI93l-MhpNPhJl0dZfZK3ftt6~X@`sEcqZtv
z1R|b9My>336>v&*0@jvZePK@vzQGVlaQJGV1n^+s<VXwhi-rS#aW+392#d!<?kxQW
z{_?y^rR!`TgiDgwcEYba2Iqcc@$lFXbBw_wE5*)cvTN^_0$mQ2dd~N=+s?MNo^1o6
zt6x9;jcaeoo5ByBD0kelbwjMK*dx_$8Bky&gRL?RhBcwufsXgBaEJY1E0jU2QshN$
z-1qNI?qJ~#SMypKr2MMfgSL|x?vkDR9W)p19v!j!-0!+3(gyjV0MmxGsjXctP8(xW
zmUANGoFuzKCJtuVj}GL!LMCplz2bO#?d<N!dwfOtAG|Imtk>w4teHqenwXtheCRAt
z?eI;c2ey%CZ?(I>gDmgEJwpvH{VcT^*TZ?Ntc(Pwx1M<tjym}CU+29Yn-j!PGOI93
zR%G|~F-rZOpwnC6@+KrQjz}`;wU#N|pQguFC<hByp~vACN~U{-P0&x$rm!S6n=GIn
zt;L5HGMgP8RAPq<^<(d7{pxli1SjmXsd;z`WS%q&!s2i^GBrS^B`7`s{(-}K-K{8G
zKw$3~62(@(X(2;bZSmG_X1xBxio-)#xSEI6wo+$BGVfR*w@G-ClHoK7S<B#zB9A<7
z&(GN1C2FYe>FN}f-)L5=X8DBr;NvR?{%cq9FD8}K7dMK<jT3@f-skME0u;O>3?3Pq
zi@*M-PQhY6*vw{A1WJsn&#|j&2htoH>70|phu~R65}!)s+oNEKQ6!^8t`{yqZpr|}
z1L2PG=EXd+V?%`Q<SbL=!`l7#(B@z69uyxpnZ5Y^;BjOg?U{4GZ=V!>Y%=up_&B!u
zzBMro&6CXSR7Oa6Oo~%8g-kp)Iw9Nv!7Ls(B7Ug1*>U42T}I<k)b4!ara`CHDJ1e>
zjy)hIn8S=tUnIQU?YIXLw+aAie%6fVafdWnUR!5*Pb8T&CPgV6$E{z`5Y=hGU@!zS
z)k3nEby`mzepsB%nf1;WqX3Wv9|%&Llf226zry<*(zfIg7+c1-yW`5t2-Zc-!&2UA
zf!7hxFzCAjn|&QeAO@OVcLH7kz~d}N(*i`fU~Go=dAUAT2r^@^CM(8_!NTI)Vunl;
z=d00Z^?IGjWLB%yL?XdpFye4Hy<TT97=a^PNC=ZaAP|Yf;9$GJ<%KbLVM`!r$q!!G
z*xNH)a;fpk<p#CdW7lfiO-`-RAHG!a#B0khzui1lLq(tB!v7Tax)e!8AkuJ~f0yyh
zuZ}qOyk$eQ>ZEi2M+}?ls@gGlEN(e0_c7z?WgOr2^mN>TCF7_9rfC_jZmfAoCKky=
zI)m-{h@jYn7-l9jB{n`nObX%9N4?DRF*AO8hWMe|JJsUO=QEOoed;;*JPmis{=HxI
z&~LtPVwT88Ty4Z|hzMURzM1gN;Tj*N=ywa38>G=W&n#IydvK?0xR1q+QY%LXyBaY#
zoL-}@6Q5@BqNCz7z+3PwgUfdk?!x~5m=*CYC%lgEXeg;Ec;r4LYx~w0cQlmL2%CD~
z+fZy+blM^y>p=s(M%P~D=5Eq$Q}d&8)l`XPwBxH2NFH}hi5%MO?O>sZ@24@cIN~qb
zNeddBIwNp)WeUw~GMaQ6H*GN(ES>-&!|q~93j5gL2{@bO3>NV=GoHmXX)WTJ^%Kit
zuO8Tto*ju?mM3!mhP?H8#Sf=CN@L|fV^v>yahGIp%(f~Ov|>Zt;_Rr!Igzf7D=v2Z
z^%q6RpTGNT*P;}TV}rl`qNw6x7kC|;%K81FUC3LG<8jMlFXcY8Z>i%uj=TK)y$4-~
z)L1$2hyACfI?cU#|8RRCS`cY*cI3M4>2A-_+i_0SesK;zcYWNt?P+e`ou0<Zf%1zT
z-L2A|W(nvjJv(y6hInp7khfh8500JutfsMY0DQi&shr$(aqD-a(-?P7A{~$%26p`L
zZUQ#i*>&sS-$UZ#l6P&V^X62$&|5GN7Dphu{$TxkP+P6vd9V~3Ef$>_(wQci()D`1
zLZKKP9o6ggN~O|jwGxR$tyW8?(@7+f#bPCqh+3^qp^%$QW}Qw89Nus^oYiU#3JRjp
zXmmO~G&GdWX2UlkF5{8MS3dIi%Cc+CrB|C<n!TuvuU6|nufDnS$Aa=dcTCmBZ73>$
zNW%$Ui~Z#Eb+u+<&Oo8ylf)J`=5`j<jjcV7<+f*Y{m@*5d+=IrU+U<w+o$B^ZX#TJ
zM7?l+#f~~Mk+kaBTuUG0;<0+|d}VnLDDG6@{#XH9{e{<#A5Q4hhJU|yJt;pS`>|C9
z(k<m!yW4CtWD<+T$}o*ADf??zZ{1wIw<(mFA15ob7s*DQ^FPuU_MA;DZd7Z{71%*-
z=^9{hkBZM2>TRjMdORa<D`?|ngSY+EHOw>`bi(Ta-~8@|LLzS{soC?!Gtp_Wj+Ij#
z`R1LYis4aSY`AWUBC<nb*}+dv*$Oc6Qj?+=MTa?xc+oI?<ymk}A^YwW(=2T&jAb0g
zGLBhu2P7fJ;R#q;kO?-u+a7*4y^g~Z?+7;--(r(4MA+77GbG#6;Ts|ABww0C2P%Jm
z_+c8u(<O4X((uV2N)Vk5kPMEUI$V3|aBc4TxMzR9)=A#wu8*Syk<?@M=h?;MP0#&&
z4f5{r;Fx1?x$AscS>x!z^55s|a@{zKuGC}tii=$p7rUPQ`RdsTYfosb>~E~>?`oDf
zULsHU_zzbue>-V<54t*ir1nR@FLXQEn2Jl^HhuDktB!B>G)sD#C8v+n{`|cMoF-j?
z2o83Ywt0QbMvR8U$BlM&57#%g9zVJI&0q2JR(i0LFTvWXvBGYdR=)>^&X0SHT5nM5
zjRptH`pC$LLNTV%XykG^l}aU%NLDL`%MCLcjWim~Vzv;71gq7G#bV873xz@p31N-e
z9^Y6j#$+}b4F;3RsMi~Yhll(6`Y;#_kw^>)VRE_L@NhT^D{Fa7*7BJ4Hqn`r)orbw
zrv1KN(J2~M{N$JGZv463p!Q<jgB?$g+sFIu!S52{CjB~A<c4eN++>}`@F)CXfGEeW
zh_=ezFWt{Nr3{vgBkX2<)pt%*F>@hKok3qxbCpaa?fA*2lF!=*d+#~_8N5r<7*BG@
zxVC{`tjS)qCOdA+x?~f^63Qj-iAphzSo?&^w${-RFN^6+Pf|(<wYuGV2Ql|&)K(^J
zD-&|#Ly`-&ho@76BP}LKWHM>+cq$$WQw(6cZ^&<y2^He&Q+Di)k9#oFd1=X&FkaB8
z-1!F_G_J)>CWFbSCsFA`y|5F!*=U62yG$I!;;F|*5y?(vglLqL@{$sG(@SCZ*GH$;
zfnd{%hrZkS>K?>x#*xzhTn7F#3W@uC^NV7`!g(S0Z^-Tzih6p*Ep2^d67iLncU9MQ
zgfQvk@g(l}Ad-QoBO2g5?%XF+=Muz;qvGy}Bm)=>M4(X2W&#E-&5f3zhd{DBERGD3
z$WB*5GB|ee`0a=GWxCtR`HHEPhX37v8W?olQgN|MJgEG|dk;8i_jTJ-PaUoWuPZNg
zJ@@m~$Xn&bosKQn?@06hlbW)9PqXCGw@vG|ry^S}ecS9Ze(w(V<p&jx9eRVO2fF-i
zGdD75iVwvNfBnS;r>PVICo*VpPK4)&?rxDfsdfbQnEns@PygxIo&dGGPng)}YCBu@
z!Vk*d{SyR1w?6p${0n!xwSh=98;yQ-n}Me{ey;ed(&>j4N|V`cWj7d%LqmfK#hArn
z!D6vgDviqxqtR$qixr2*BSHBX48{c4nzg0JwL%mM*<dgP1u>LL6=cipMIw>NWO5J#
zYqnU7MuWj%kjv#Fk$8Am(%jq<8X6iM9Ze>aqN15Ue0g1G$MDf3*JZM)l7CN9TwUGL
z%iV2RuSRQc=v&XvtoVsET*kxEvMB8r=PGTscE{eVOIb9F&dqW0D<^7VTKg4*hnH9l
zW(<>J)xyRmM`OHIYn+|Aoft_S*<?7=^p)K!#xryV{pnjLv$rni-wv(3;)A4le#7@q
zb07Q?!a0nU*|D5BMx#jbMaQ@myw(2y7<N$V4sLRAT=R|58`pghRSiMVMnSOdsE^}|
zhdW1yJ6+T7d*oPxv6v*1F{h1g%cjCT4$NFiDJF(le|ypv5`~7uq{14V9TpXz(OPpw
zr&W<D^v=fX8F^dPW22J6PDHo|v%;K&yVmN?E5l>NJ)#d^`eQJMx$Ct(NXqt$pH0N7
z&PZNPYF<vN&H4@xOsmzOEryr#@>isTT}^oC(D$$rgI3uh(l0YzR(|2wJG-<6rM!`M
zY_FY7Rv6su%`;Yu$<>DB_+*0b(&q6$F;>Vtr-v2}twQe(S0m(CA5<*K;V()FopIm4
z{^Ek8>ZJvdfc>?*MS>XAdzvMG{l$f!zqi$K$K3Vth;Rp{>)iEr%X-Bn2Xp)SxM@V;
z;vBx~MtTrw`Z1sUtTuOjJm^-fG@SgbcE;00c<iNI`$@G(5M`_$(>GQQOk>0LcAQ9i
z$IE!>c(aGXy|Hq@QMGd-g4XRw)hP9szHL^I>36@F>q#dF7mrUYy3&G3>$ay-8Kg_c
zn}N@adQ5-naLwZ{dDo}kfn&>wSa@a0H(uItn>G*`w63Cy;@bMr&aRMn-`7|704$C`
zG8^@N((e9N?FO@{Z+JwlwSS+-#>V7w`RJ$|gTb;`AtVx+NF-V;@al-sXvE?027|$D
zHskSdbPW!VC&KkQ%$V_*ax4}@p-`|moI$TM8X>dUg27;6lf2E=P9zd(G`iVhHX30C
zFBXeMA`yqfiHL|Gk%*D}p#48tUtH8wc5O-@7)R7<!><mU6fBN>>Akf>m*qWW64z`q
zz{;MoIb0U1{nNiLi-*Sqi=A%>Y!)R%W_If{?ihiJ2_jpE#xUVQR;|(abpzqyCCFP=
ztx^ByQtOD<8UstlNgpyEt2pIBTQ^j7-ppK_kst~0X!1*q(!gX+KL$3F`qa*4xNyQB
zic6<z(>pTOJ2G|->PQU_-TkBFYsUuq`{xu=z-Ci;!l8f)JhWOY>M8s~W{1Wl9ea?2
z>x<eZ$A@}bbXpZRGCqXMufOqKe@8<Im#^2T9d_Y5xw-3SmC3{jC!6~0LQ%tccDJAZ
z_ltc);?Yaj>;gq5hd<cC2u$jta2%Nml$)*a&3uPyxwV9OG83?x?H3t5!zT2-)jSWY
z$@c;u|KHW$AKE$niNb#EmHo({_iazz{qhP#5FdTN9C6jKIcZFiwu@3iIT1lX{Miii
zoj|yEw@4jl0&XQQg41NcLu^Y5O+W-+Jl=$i!4n*%^Aov$M_zOJ(*=8YJ5Hp1cZ>8_
zPuUa9Ne0J;2gf{dPrvl-gw&2r<si!)5AIuf^!;+LoMgbKjJ$sQrChMK5SPOF!~WCb
zhieWX;f}>(2_!6jhRNG3;ODJe5L>$we$wu6x+s6D_Ce|B(1_jFLa9i0@+ANOAOJ~3
zK~z-s_Ydgx`k<g7CNqRiXIL#3TVV_=fq*la&15o}1Ve_?@WTG~1S}kbgt1u62BQ&c
zOE?Q+<;gbFyA@8Hh0*JE0KHy^#bT{i3l@u`P^csl*<ds%6=TE0!(y?R%Z24T4u>ne
zKP@SRcl1c5PUq1hyRLEIo8s#BZb{~P{`&kh+!&TNzKAUyRZ?Tf-<+)Z_FQ8ammwZ@
ze@jV94bdDMp7B8V!bDTK5FZ|7X_pcnUV@n0foh+fpA2C)>sZC%S?q{V+fl#a!Sy2&
zrSHR41C`nT9ui~N8CemBE9*7w9(}i(hosFz(#@OWlX_T}if13bAXv&#Uz=qznL7o%
z%Y6#5qbp=$v(cy?lkpQWVpEnkRbA+5xyet+2nyki4)=r5)gTrRk0(+Y&UK7^wPaI#
z=xi{D8O&h{n|c((9#sjhsqe&c3F}s8k|`t*nF-s-;p#V7b{Mgx4bI_6qfyjqyGvDk
zTr>uEM;2*E#QhX19~9hyOBUHm?%I}REO&rAe&Ep4EIE0&c6t+U`NavJmc==IM|T$*
zgZSLfS4#$!h_Wxg*y$u0*X>CC>VpdKy5eG&T4|s$h!qZf0{gB%F3IU5wJSEnffuvu
z!O5gOi*Qd*a9Z>j?pJzpAjh#iv^SH+Aa=J%)nod_IgyT^k%u@=Y>=Znew6Aly@xXj
z|Kc~v9%zhfci-L6Ft9D(PiLt&8ry^e8l4{0L=%ZbV`H%8h{NI1>2wO2qE@Rd7Au|r
zhr*M{<PZv#NF<uz6j&CW4$db7K~^jl0}M=hy&leVPbAn(c@``dqf{#451t5TmSHlP
zR=6aCS*Oz}#>R|BgT-uxFjfMdz+$tQA<SX1ST0w{<%;m|a3(XD9~JzgSMK}ji(9?j
z(^mzWc&2xCOS`(IYPJ6SCD<ylX<aIKvG4KRqD%0EIgC5y96pUpm+9PFtYM|TV!~Sn
zyvAir)JU0&YNv1F{HxRZz&jU;YcKBj-lLzrePwRl_{p*sFs?F*G3@(yr2Vb*mbW{e
zyWJVX4to6Q<S!4kdeGbrE7(?JfW;AGuvi+6?6gs*Gnmb@EHXzSPw59A%;E*Jc(*Hy
z@C2e>s{)c8_K=Cs=xMo`ymSqjVvqR56P$%_7?;}s^}FlUJDk!%dUQq<(v7O-nue|o
z>vHn*CneUXsj=syr9$}NL93F1Q8J*kbmw&McMEVmAe*d*Y!!2mv$9nwvRWXUh2B;$
zA64^Hf~ec6aJ0oDjxLVmu8%{6yJKU(iVbmya0kZkhxTQb7f*=JoSDK>9c|u##W{RW
z%-iZQJ@Al098o;P^@y|}(u`_&*5f>Rc+$LLdb>Wr={i?j?EK{at|F&--}cmpp33w}
zWXGm*kTsB_2P=M&3fu3$bok&rpHcXWdq%P&;0Oeh!Dp+BIG3lGuTZGf&0RvX&48j(
zsfLFoTCJAL<pl=^*(~T5txiiI5a<jBkw~OcskZ1cqe7uD81!bd8F;rt7_8A~!Q*ij
zi`nKlgSAzbwc08wSg{zKR-@Lat#D&3mPjI!NhAuDYBHOYO0yBR&|@uDi^Xih;c#K$
z;d-53A{p-K>0z_k;o)I89JFWevNI<d>+5{uV?5n;mQJau3`eTX<w6WU*jztE*plKX
z;^(N^@hoB!w?}?^mhr4k)QBq*((V<9nb`>Uq?pi*+=&0wOqoM;dU{93ilDZ_2cnBk
zdu?~8Q1G-7`OHq@^NSV2Qfbb&W`n^rYvFD)KU3A}33svXmRlj*D8$V@HZ{Aaxe9_X
z$ysagc%o#mv$f_*L~IJ8+To&04bHCTblS{uv3?>gEk1p1@|iQ&PJef;vu9x66AvKA
zDZOr20INS!uty*R(R$+<RtPholR5eUTY*o|Eu!YFuxZC?0spn2Qa*Qm95C6c$Mj#m
zUolNfx3*YE95=dpq;MmG7U%F0WnXcrix(N>*lfl6__?2s6UY%AGp;xe!&?@ma6Hk|
zUg0YAXKmMa3Q@iMZFA+NuHSvO)3GbpIhCV9J$GFkBHVkLB|q5kPamG?T)|*q$xd)K
z=-dO?{M`*c@EwKgS77{-UrRhP+S=0x4DE2zn6WVefxwT7qEcxViv<rmXTshu+%O&v
zi?f)`YK=xYHipAut&o*KAehZ&5V4BKVK5NZVu4L9U=0-qgSS{<?M@<-^?JP(vO*Zh
zVlnG=I;}<{9g)E?QSiLRipAk<2_-dfMJKBjPb7p#L`Wr)|KHx3N4IfZY5vt-Sh;~a
zwNR8~N}{9978Ti%ZHbB}UKDvt;-0aRwB0k4#POLVopkJEPEPmfwlmXnGBfR^v)E2s
z*_?5_#J0S&EJw0q#Ud@ri$qZiDU#w2k^r&S`sUpCKoEi`5}-g56zlhgP5}a{3IziA
z>ehF^J1{Wd^ZAz7)Nma4A8uYHO56vH=VszKB+y4DD~1}UhtTOED&iAG_e)gmP^#Nn
zc<A&S)86M4I{G_zY^$?``@}fxVqulznWjtX4ecBbc$SQ;tSvaznwHTb!OGf#=!uR?
z*>v_{!0O-~%+6V3g0hm5wzK1})X>!9jr-P)r9C=a1xr_5-Pd{Ic;h}pSKWoB?!waI
zig7W~(RyTL@Vt+@!(n$i9g}TxhO=>NmQ_9Q&AX2rfopbc9o_G}_u;KKU)SD#&QE;p
z1u?#87`{716>adl=47N-&cVQ7Z3ZP-ZpNfM@z|Kioc@{NeB?{_*B@)@p*7mnWRmm$
zV3JMhgM9(Sq?)JU&ZGxl`1e-dFkX@;c`qlDA759WnHzh;olN202S4+z+frvar{0;#
z=i*pn_Yc4P0!ZP?3{`c8pT4`lS)bNZrg{v$+Hl`P<5vk1ZuJDeY5XsV<?XkvKKyq3
zB*8)Pjn4X)2_r~g@W|~UN;3z;a177mOCGJt^88G^cYWUB6K$Qe0m(Zw6pcphc1Mw?
zh+$~&DM#JvPN!2<6jjrr;Rr^E=Xu>QNYfIcUUr6IG))r(LD!*dju2zO#ZqJ%CSn*E
zjK&}n1ie8D@KB&9mm%Q$o98*hKz5rwKR+)TjSdYBsj5O(TIeZaInFyc`0*#7tXRHW
zmgTM2FY|aDZ@iW|NgYl2nYwUdW*VqH={A+p0z6$dM<UbcMorLNJ{0T9?7HIQc+t6v
zC51DT4p~?jwAqxNo~d&YQptq@OkcTU!*d@^v(}!>k<Pxq`P_zcoxx}{gOB&3#V+N@
z<)7K9#<WsTak6lSj@6*=%&|ku>$cFFN;H&MCRM|><mHsMRC*>5O@Bh$jInVeq)Hz-
z(sH5&gjZj0{71KRVEoc*Nln?D?`Mv}aAa1f5ck+%ljwLIbo>Hnel4kV-?Dq%?<cLb
zQ|dNV{^_N$?$Wo~FZ81yPV&7>*b`a5t>&qR$D;<5%ifc_bjH@1+PXSr5>82_`-WYs
zlFRKk>{^w+=1x|$&pvjT2IE{Pg)e<SQ@Hn`trMD&rsRwF*YDiDE*v<oR~Yg)FTS@a
zINahxkKU5}beKG~sWSP^EvcN{Yj-TqLFeK^4~b$q-nc~J4pEBPFSqv&dXJqtOJd)&
z{(%8a(>xweUVc7?U@(p6A(34W1cb2)`Rh6q!Uf*YAq0l>opKz<nx-j<l8H?il4@cO
z?&J+!M+~A>c~RhXU1vEC<i1X{KF8>WL7P{h00U7ojNd<E7@EuFuBfQ+`Fz1p0G`FL
zc88;)vT|@>p#ALGC5snJlC-k6C=#j8`q4;WL^5b8Hcn?@PQ1Lr`F`I=m%g54T02i}
z*t9L{!oBXwv%7X3DkuoHwN*Uv_|ELEe0rqw{W2c*SzhdNRN5i~*_o8~;<45ppI!dy
zA2TpaUtznH?}A+EJh0ovNMy1rQ@;KY6cSV1GN8zN+VjRL>vTeDf<(DJ9Q2)SouK!i
z(o^>5*FMN}SF?o%72@#&wQ=LBWy`8Q`sgH;?+&}%ULGr3$+&`;T~)aG+nX17ka@b0
zBw>c*SlYiu?2_2fiLYkXN0HA8J9n=;yuYpE#1y|An;VuT=f*tw@H=<kU+*9(hv9(o
zyT=YsDa0LaF-1lv*mm30ZK)4FHl0+NyYH`We7l`qQgFDXyYHW7B1+w+%JZV=FD`{M
z-Om*3&KK|Blzx+L$|=Ei@<fMQyt2M3ahx-C_B-6-&M&XK@1d>78oO!Y=aWgNTg4Tw
zjITCK8wRn09RDOP)UPiiqV+xfap)L)JRPeGif*qFi*3_e70YIS5Jz|a;KwK1h+ADB
z7#L(2rntByzaXF1&Jn}fY&JvJRZZi09wQu!#W;?~Sfi4iq`^Sh+%Sx2EJmkr(1NP2
zYZzk&!NetoVW65~7#K9WX_{b_VIUn1MyS%U*(S{=z$h>}8V*N1o+6vwHahBuwJ{8M
zbwshOyu7csx4WmOyu3^l#k#d6fx$}(bj~SkD_vv~YBq_fO2LXOo6{;QhrasNH|cUy
z;}tn_2s0}T?7n^d>&M#Ccwkx#g(C8mnO#M2N$kZY6U-OF5v``$^U3ik+D-O7_m{i=
zcE@|~r{?FJ`Q$HlSD|Bq#27?pGo3to0Or`8&*G|z#y5`@mdsg`R&#UfO|k22@_C&A
z2j6L?3k=JsB06vbT-E;GcX!RvKF)s_h=@kC5dsrOjbXqA>&r{T-}m6wAMSo(^2gL|
zs!Eoa)IWV;j(g&f6Rsp#P$pQq%9D;3eB}PU7v5RluxxUf>u^iA>|Xo(#~Kkrx9nc)
zm@4-@nV0vB4|RU~g>)~9X1bpbK6dlVdydjQgf8JdA8p=VlcqR!xTX4rWr=e5!{2<B
zmg-;KGhTSBKmQ~u`7*+U4E6+m_~jRhD_wL|&py_ed|t8b_8Ce3nE;mOW!r-F#!#Xi
zqwW9w9dm4Cb)ZQtx_$bsAPk%3a{Fi~^l|GM18jzRGzPJZ`2_`bo1NIxA!@y-u+Zgl
zdA(kaNNQ3Y=U9$04Tj9wfMiJcku`MEgo=r1XD~9!APx}a9LIr9$5_|F9;0bGk?oje
zSk2HlmIJf9VPXuj9U|4Bc#jN2@9*!&bU2A=7`je9Gl=I)E6Tn71D%~+Rh5+x0ADCA
zSytTHKB+fso*-7*Vtv`S0Lbnv8&f{ND?WL%sp*QQrg<pbMS*jc*dk}MQ*Nhx+&j2-
zeep*})BQt~MgqAg+y!a%n#C#ib#@F+81CMgUMGjo8#wzxPeFdZ?}A?3<4ya@m())9
ztdW6p$^{wlf*>sNRK0%i4?FJKJjoUreg@+gL!n4-pSPxZTp|7H*RRG%4p(8Jdt@Z;
zYS^%0RYAc*YfGIOAZM86Xe1>S@j4l8IR9<&)wRW6ynj<dJ6?<j9&R9$Q{%OHGKwg*
z%X8@}&jSxPTx<<6V#zI~)PU2pg-qB{x(3FAPQlLI>#6MCxqJPT4^HM{2W0X9oN4Um
z&fV(|?{B9EHXKl%dgR^o%l5bKUU#f9-lfoSV(7vvCz*z9KKuCLa6kzMlnYn2bd@L5
zV>mrnF;Qq=&=_HjbBI*97u>M8>??Ih$d!_Ts_XAHeH4kpFEZHHqfwXJ?Ql9w6Uyog
z!QpTOgFz*xupDccIx^r(6q}gNSJE_16eWX1x$C-4f@L7pnCgDI6)?IG?>ot*z(jlV
znxT_kRs*vp%W=A4@?fW7bW=y9hn3+Vi^k9m9V9!PFOVUuuv(7eic3m*dwTi@20TSY
z9LIhBiyNN#*NLYE&o2g|GjjS7!P%ej)7Dne)R;r6Ju~56m~TJUpXFl>PWQDN(--bJ
zil($=!}XQIzUSWm&wqbs<ME@~`Gb=TP3=CDL38lh)vI55I*okOnJgD=-?qsn%X@zP
z>Ros4u*uSMe|W2}500293JKLN%TmMk92;|Zd2qgn2L(?YHS_niZQr%}?H5jXdnfyj
zcDSX79=+wMAHS3AcuuCiVcEUk-#U3!d)uy6ubiKIxP8~^OWgS>+4+^M$xRTiUruYL
zX-3@l;MPa(-%IuPvBvI~_Z&@AB3IWI-}~U!ryhA{LQBlO4{e=%tW^qUTfG;?&QR($
zRZVhAmNgi}Ne;#udrWYy@znnG#>7o9?r9eacaX^OrSog*7u?Kd>GV+3hr=U~gu(N?
zq9}qW+8qu_l!zGytmq8Jy*=F+BM#a|4W5rz&@sj_Md4Uh5JiIod_^M>6T>f;s%k82
z5ZgPL+zmj-;CN>_mPHtJJVsDrr|ZF((KUl(8G~E`g~zavFGD1{iD{9aVKp7hHil_x
zy2eAU4wF}q9|{Bq1_$%=@)(BMcGIf%j|Z}Na4sjJN;XGfB7}0p)Sg0S-#DGaS<X%&
zLv*vg%FBnA!)>3wcH>_@Jfcl^=;<|^uO2xY?CZ>lQ==0tXP^7SzBQ|Anw#U7^4A^G
z)*Ug2i_r6TvJKleS>^T_hk;;eL<)r1(1?iF=A9!K63QRiI-$)Y<zWvzyyMs|n2q}W
zTWwV67gxA8H`LT^s!T^C);BCm7Vh;8%QCXBOpmOLc3Qg1Q@?%L_Fb!IVu!11ixZX}
zTI|1aQx$b0O-cRsW!1Hww_iMw{EpOZs)7*r4E^ZUwZ;GV%<b>*Z#&-DOD`QPUFE61
zqIh$|vW!jurU}dOvVDQuV<v-vV92-}2j}*j9jM*`@2R%4<Q?N^K1@YrrD2+?rXmIm
z>loTfkq9Y}psz_TJIQOr)CR*sRt)jLzyiE3hPdz<@)}tT{uzVV;CY%IW15C(Vi4b|
z#*)Duv=WcWBvr#SiMD4A;$|f41+Naz!8b8OJnkfwl4lq&;@EArXgH#2S|}XmIj+DX
zl`L|jV-ugU&KF#E=S=Krfh%ubAC@%9Ot=q)vfmpK<#TJAQce+#D8Ky2KmOIXKi7Qf
zlWA!8tJbdF|4d{4#pk8X9xPi??)m07?>^De{)gw^qMOa<rFBPORBF*Ms;;iQ=9=u6
z1}!Fm;RITSj~i(WEeDM>7jrq+ZK|5m0G)Vb-KMH9O{>wfy0$pYt)yefp59X?dCWAn
zIE`mb`1qZ>*H7DsQc~&u(*5<*dq*x@)urso^s!X?R6zNd&xzyn=AYh)#-c3C+H7`R
z*Go!ERaJ$?QViOf<3KNyxEIzSfsZt!9R?(^B59BX^SQ1YJkLV}FBsxE%^)2_7;Bm)
zNi-V(%d#AaL<E7y(2ok&kfYHkxMU22CkA)2O)LrmZFPkitYC=OAc<0ic~8_m4?|c<
zAUwvVDDcs!<|!%Z?d~Ry9neb!6{09|sydIGmz=g45$^exa8F@lUFCuIvVW2nR<dcD
zpBq@dxFq$@#qSGiS{B?0v54~9pS=E!hriU)cCxEG-3e+AhyB{kSO4yx-{|Y~ZY)Xj
z;{FWQT{%%k2%l{~_tyS+mS16!dt`V#OZ(2xZnctR<_ZQzbxR0I@ROy*=3@=s!eZ87
zc!87XFgxpWlIwhO;^csLh%{3b5$$f1Z8l9+5yp}%Ynlc#k}&2uPN!*)JV%2d$Ht&i
zwawV=HYlINz$Mz53SX8y&+)v#kUmz^G?iG)G|hMx4Z^CXK_wo+z$FHxzYfwLj4}ps
z(LmEGLSV?zbpw)Zbe%T5@{kR$6StA68U_Pr5vt4WibcVWABjNyblIwc){oLO=+59@
zsrzrP60a;BdSEYIxab~4_2IvJGv&Ur!jk@x!R+o&S>Z+ZVeZKO@wd!*S%5)i<bsHF
zGke9$SVTGSQWL^x^R|s0y`BAolYLwr4*RO*D|(N6-+bm!jyPnff3f-Fv#;*`GrgHm
z?XTX-C=zqq_irm)QZ(25c=>ShL%1%yXnDEanOQa#D0resu%VlpGOvB`78Wx>l4`e@
z2fTIgy=XKlNfO356bcoWmP8^Ej4@bO6h+fC;*H@ERK9fv`bSyAFa?39S??@-C$b0|
zDkyeNRTSt+WjTgnY&MYWRF3Pq&T*`&YQ&n)=sF}wLg5?)KBk78ctxMM-c1w2ZA_95
zDeyc5*03xd(+nEqQF62eHXeF|7?;}}2}e{_(=<)j^~$C4nU=Y~pkWM0v@^rW3loAT
z8rM@@z9hTCy{xdrCN79JaZ9*o0c@HR2FPTWBL`cK9BjGszwW49S;;xku+JY3j|a$z
zqF9ihpJ#K2eX*AxKj8BQYF00r>0Qc*_4ONHeR+R063feTCTi`y70jJ?Y_F-F)*MQU
z3po02>n|U9AsT|eMw?Uqm)9Pe-Z5f09um+<C#$Ke3vEQJMXo?58Ih=sc;{mBY$Jxt
zBry=OmBwe1;IWKUGK3hVJBJwND_N?2-f+D2RQ~`}uk!NpX=kco7=j>R6EiFuQx%5%
zanTSL5)os9Sm#)xwi)`jM*?I>&Z9}>K1q@xLR>^-T9W~(k0x=Yvxs4IO&56HpwgVU
zZjea=0w7Yxpl%$n!ZRS*!Qze($Mc$|(Q-S>GMc7q8Z`G}(-cHe*L77<bpy^2R8{49
zAruTDggm7)7zCFl;TKxy0}~5r(%&^u;!3M4CVkeg{M5U9|1uYcVYcCu=PM=sJ7IZ0
zR|Xj69vQs^>~y4mQkeR7oNetna-c=FOFO@EU33IoMg_%gpEuCe;q5}+<4tWftE+t8
zz=Y48TV$Jj<1G!%PwmrtgHaCzw}V6Gj_n&a&Q{j)<<7|u&pz>k-_jRpbIO1Ff4_RE
zFPL<)vVsJI{&WTF^WW8$CLj~zp%yTAJpn{wq>4qTv8CyCK2JN2-;0xi_95^m{t$e*
zPSKhG7$?cnLk-fLm=ChN36M@nx5lDi&P#G6ZxAL!h%v|}jLf7;A^6&-IZnu8PB{9>
z@klfx%d#X%f*|DO=Mm#LBCVrP_ZCGFs@)7Din2*RZbXzD3?uMRyf8>=J5-d3;ztJ3
zG?nLh24P*(pyICRN=%8xqC}G$Hk(b;b(<{H+(|Num8gHx+iDQMJN4Lb9B&e%JT1e+
zstlc+JdD|MOq1a_Vzp-(mJxV?&(DuW!3xi?OgJ3QFJUItX|wUWe=&#5&;xt(9@s1_
zFQN;4WnPL^#@jp8+;(i~)r&j&yRxz;vM5E{VyBu<CBC{dXWmF$tKyyo7?yQ7QauT=
zm`cK2Qwc}3Pt_Ftr7?qfu@@4HC_Zms-*fLHglbk-wY7B9g`7AX6QQY~`L=G`viLh6
z#6IjjvJoA*ikV~8?(4?>E~dN=$(Kp6QP=7ImFt#5vu|1c&cEK~Da+5+nxR<4*^x*J
z@LFu%pKHU_Y<^snkIC^2B=|J9%T74N{k%NEi4wuFK^1&l;>U}`Sf9sl1k1zFUnpR5
znHA1w6!Hc!==}YmYTndYUa-1R=NFAfjz*)gl9Ccp6iM;OMl^Xm#f4#*(b3Vuf&voS
zXz+r7v8luqQ4)0xd@KwCO)p9D)JST76$C-kG{4`kD#}<^4EWSlRi!Ph#393Yy<VLJ
zJ35?zD2b$q&tTJJ8OAg*N3tMIT78eZ=COfcyP_!2>N^LQ1ymYJ0|_>?;LoylyJKi*
zh?n3x(cHcvdUt@aO)Mkz?f`YjBo<O=Z)v^f3t!EaaNoS<>VyBXzkF$K84YF#OSoqN
z%nHTDsm-bV{XWB(ATX!1C-AA78R@6aWll7nq<dHLe{Siz{^xxab6;!6{x0;3yFmGD
zxtF=+A9DNvro)cAKE30vPtV{6XaSxVOyYo_AO0D3T%gZXDR<^cNLwoLFIps~)p|20
zI}@O4$T1wxu>3rG3#gjUowEyz^ceE_!jUNH`*Ox&G3rET;kpi3uZY)$v)gQ#RMT1T
z&w$aK*lA3#3X=IsCdrK<&KAZr&0ruHjYg<gmn4ZwdFqp)Q=DiOU(+;NM+c{S3<~n9
zs^;b8Io&Qe0U%;sg^74LF@e=Ebg<As$(^ANc!puA^e2vdI3*ATk@U}UE|&|F%Dif1
zfB0xdL|#9IUZ400X}D<2{r~o-+8rxeyG~!?Zj=?4biUOQjbwlNQf|VJUsYg|{zaE?
zwINe7+R&x!)y%yjUFqn!Xj{gt?n$oF_HldH$#aXZSY*F$nf$e@ZofLOa;_aRO!XF=
zUxobL=+INxmd9-WP7WWxTmpXFM3xs#U4`m<?#OreafR|N{C-k(xDe{2$wlIXpa(T?
zVxr)oJWpi(T<jYIN}Ur}j<;&n7PHLou#dW2Y&IM1B8A^-O#@fD0qvepDd%}n6jfEF
zJ{FGr&4QR_ASDLJZRjAu1O9kZ8At3ZwA+>AxOgZH&(q%1SS%(<QXmkZH!;Sls&eG4
zZ)9Y|=kv+3Tv${jh$5KWi4rid3FUO6#04Ib;G@wfkqPiICjA{e7~f4C4u|RP!4!E(
zg3|*$Cfp|x6*H+flKn8k@(B_ql07pQ{%>zbdrg1MCC1ChqEuhC@%O*`&9v7LXZ{E?
ztKx1kOZ?`?&p!Ce?;?bjJa)S}I*4!%2)=usM!))N(C+X30$IX6L*`z-!-ZpZyohT_
z!vthj?pS%A+&L!YiFMnYS@%3laTi!#OqqpgVKJMWYHQ~>P7nlDRV7K{d0r4jO;t^j
ztWM168V_BeDm|@LHBA!5lc!GB)~+J`qtJw^>6&lErz(nJ7&Nb)E<k!CNu;5LcH+Qc
zX#eZz=qQ!!c<gMMmdSNOgdj;$0myfO#caH@)x->g=sHQmtAU9V4kM(hs$m*D$4inF
z3I?f5UY2F@oH=&OX{Se07Lu9VDnmJWv~|Pv8&M&eo_3c->6+5|58h}-bL|?MP0|tW
zxz*!%Suo|1oa6!X^<6m8bn?%CJap~nw?M_#=F}sBG@dlGFy(8|_YUK;hfr^39q|?k
z;__XKfq7=8%eqiX=Pac@9g?Ix1$;H8PTV!=1*)F7LQntz3&Tl7K~zi$jC>R=y>5{s
zP`ND2GRv~6syZAF49;?xSt`haMw}|qXp}yTif>iZ9)I+)Z+-jQHOrS9rWpzZB9REK
zozn;K#B`^saT6@9wR0R73<g7?5YO|mSd50e(FaiTjIQg!U~phyV03h}v9a-<d+sp|
zBNz-ioX+BsV(6dHbz1n3&u4|4NK7|M(`>MbHB*x$k-B{nuP(`THaD6fVU<b8GczHk
zZoKj9mo{9#MP4d*^kwFE&v)i+ELgku$(KB3Wta3a<{MVU-C{OaT)U{N_1v$2{OpG9
z>+Mb(Cp&c|f^s7!oT=V|s<-68j?80NL1uZ;&=jm?+jANBMq74^EJI8zJQ}aZ8>Rs|
zemcc?mS+W!>^c2{EfxUZ`1bd2yY;4q8@9%cS_Il)s_S|*9A-GyfKV4Oz;G;Q8V1WS
zhOQlNIZ;$p)YjI95c=0Ap7`3=zb*>#$x5^YPRr+tqR{7Jj01tuKp+r{#r%H%g-4b|
z*~m_(qoAOWW!bj2wu1)`u3582Rn?J^k&==UDE6yr&$(`=%jtAEX|3HbO**=jhQWgs
zCl*Weq$0#53i<S<v2$(jNU_RH?wOfnJ<SIW?fAm2OE)Zj|5#H@P3uSG(`&A(aaBM0
z_di=uwRVan+-IC+uHcW>A1$#(U+P@)v)0<n{MBBBPhDF#)H56jM&8`B@3yb+1Us%H
zUkju;_|0NIz_6TX!@OuH(M#=*w8Z)>L+bGyM^yZ{h}Wj9#526W3bIvVx0oyb(|zCQ
z?d#j~<P%lpP;(s(1YdmRH7eW}RaS5uuPCZv7$YMig@uKFzu!|@Vj8A~H8^!;ks!j1
zQh^uV+V}7I`30jyg2!U9rpCtWuDcFHz>7$yqRO%yi^Viei^XDneSLvIz=&r(A&#3s
zN0BV{P$(1{9Q3+eE{DT`5OTR(2%(;yo<)lm(fYb5ilF~T1EpmD5s~FQ&(WbvV0o8i
zcqB39pX=&M^r~VE+hy<^e?2B!WvCx!63@)!9DV1A?|JXq&s<&d>5>mmec}&|PWW(H
zlxml(bPn0y`{_IHqxTk|YPW>@jN{<-|GBBr!NXteOPxzD^H+Ou1W~y0o`x5nc=^RA
zUjEFkYdjT2V6hNoQ=Rd_RTi@eG*a1|({#?TJj+8bqa_6Al5r7lg9%)Qj&&7cD9?X4
zVSuP;vrell=8zznL)6pPfB1tVRkz-Z5ZZLrhR1*Y)LVagr{RXJ0WAp9_l1S|31<t-
zve9T1W9%CmcICM_2wG$`GMIv6*)wNO_x1Mm_4XxR3XN?sO_R2k8ioN=kW}^DxpTBd
z6?|blUy$csvufq?rHdt5UcPiOeQwXdfZsRzlgD1RJ55;zSs4z8TU%SHEJq00?RK!O
z6XOkSnPpkFyXzd!^F^K_S(afos;29Pj$=@Ihq+6NO1GE@m8UKpUYp~~T<2M3&cA0G
z7>3q-;LzgU6?R_QxMi!m+T}}<?>0%oethOo+hE(k`z>I<vDsa7@S1Y~OSsQ0J1Tpq
zYX8Zxx`AkXct<7!m@Xe~+`Z$?U%eFxMt<>wr)ifeZ_iVqG02+AdAUKIGkPpCb?Kej
z)_Kt?uIC~oK?o5K3^rhfE(wxnImu?7Tv^O5&%d&_v8lQB)alCdvOT}~f201<lc(GE
zA9&a0bpG3Ke&2Az*2>aSyTcxfLF$XmX0zMvG^H^XiMk8&1!z`<M0QQnMut<1#B0~C
zrIMYR&Lfe?@W@Cg6e44|xcoeKUG0i%H(gbbKfw%;%F@!x($axmfmE!pJKroSHmU2q
zy}cczZ6u=WIz1ntR&`C&WLZ`erLVWoG)<4kV|P08?fD2J8U#72`-g{z<8xT)sR&7`
z;nX=6?iVx0B$zx^W<tWF;o*Va_Li1qt5^4RoP#NoCB?o$FE2{Eszf8f1*_K+Si*f~
z`FN!8@0zY|3%JAj_@OQ@&)Q_1d4%0*qeb@<O(&b*{&2&#bqvdKvcrfjl<d`o1;fTm
z?-O2BAwhxA$VlxO7C8^Y37qw2S>z0N+<NmJx8D3u|NOuA{O-9Ap561pD|>Icv7xe}
z;_;t9Rap^lRb0KI=Ez6K{C>YI%d|>O2PB2VVU_^l=%P3@lxo_f0g-gU>-9z=5qf_?
zo_ojj*Iu>m$~2FHrZU5W-0&b5ih|&#Zum$f;`jSqE*IS^mSt(Uq(EHtx~_*pA^ONf
zR3$xykR*w|3b=Y;nxn&7nPZOMjtPx13DQZGnVfhu`yeb>Q;QR1B;4oCJlNc!JvhOs
z;lNO6!5tIwC3xfR9q<0}&`{6Num8uhwVST6J8itpWvIE*rD~CNSV3Y~&QQV$qXjr&
z_+<0Cl~rY7VKGgpYy9vB&0qejFTj=UM}G>2nX5O@FP4q#>Y9#zqN-XT5TvzgniAu3
zxrT;@oK9z=hD}+qV#Vpxr>m-}sAL}=9-+n<knH*H&tCtTtJl|&TN!_A`A5CvVtb{}
z-{E$<6-8B)VVlk7@f1<9PBnT$#M91N+LKB#O;gh#zfx6IMS)nG{&Uf?tToL|^LscQ
zW6b2KGP%UOk+{~iX^Yu_4w1DxZMT!o)S;fCH}~wL+Z7xO@%qEUf#G=G=7LUBQReM=
zoa_Mih2@H|u&}sf_8)k+vb^lkpFKtW?yaX_24|i#UN2j|baC~PDifRjQ9sRNk!6|Z
zdD?YKlN>co^LRXxB)MF!+S*#`L?85eyUv{phJq$G^YUC@*mctnzV)>+)jmxmF+UcC
zo$T*LiV8j8;Yhf%t838fRaG?{4pZYhjeDnI?Xg%)QIz=TR<dKzXzTNt*o;Q>fgY<v
zY=L6^d|J!~c+AUw?dP|6Dm@6H7oT`}sCSrjBucDkT?D(VVOY@Xf{d7jH#FG7!eVAP
z{6RB9=<U7F*REXtwQqf&YWH2Yd~V;L4kotu`I{O{6YILs)7xja+i56!Fc^e%NYbu~
zF<!K2QE_o`US1xpzQ<y*;o%WoH$+j~as4&l`TD(^)~D-@xfqJ-&v&mhvCc4DM^|Ip
z31P5T)OBQ<I1~!^^$++*1MFCGCC71eVie7g!5Gtcc!ps{M+2&=nI=B{$&}aaEiPLu
z;chW2V62aM*=f-oA@qwMJWUsPd!AK-wXm?Tut<~VUwQ4pAN|wr{|FA5AAa}0wVpct
zqeuRi%jw*4!*%-)yqj1@RaxoXx7}izW+W0hb^1&!7NY}DsiB<8VxP~~-QBGyN^f63
zl*Y+W(yP|heeWCheQw)UyL_o(5~it{rslL)Rh4aHS>7~t??|(MMCdqeA08BR9pY<3
zp-^vc-=KHMJ2b>`oT{pkNW?TvK@g}(CZ;HZ-l0G+2scAqU_`V2QWh+HG=vHqa9xQm
zq}+tAfI)>Wyw})g_GSM#9d90v@#zk2+1uui2=eqcWN-5i6lXuqMKRQo;mkZtAD<kN
zT$8m$H=hpp?F9&-mI32p3(aRkbm44>O<qX3iOzpYsNkZX_F!9=SY4jc3rU}r-azP?
z{jB7mlisSXTL)W2=&7H-E{M=*fpx{y)EU~kg@uKM#pT3_Q@u{7M~M{u_V<SoLThR_
z9B*mguy)JAKcB29UrO&xQ0QEF=4>ZIW>42JFYp2>X4AwN!=Qsp?d4_bl_+x9^F+?|
zsVhFi8uA<e-kOyI<_ny($;B)yXnI)SU3R&&)Kd?F-e7z~l7Co|Y`P>HW7i|qj<$Y|
z=eRLzI~m5L(*=a2NXU?oWOwHwgnEaxDHl$4hUh{d$|Nr&-+Xh}tKnJw{f%@RemewY
ztEMYr7baZj2;1pn!h)pP3&R5FH@cwN3lsOF^Pdv5PxHENQ}>KW#C;$1WL-Hk4cC1y
z%d51ou&}VOu&}VOu*f!+Kf%Jn!otGB!otF09$*P~3kwSi3kwSi3yXPxCEP76EG#T6
zEG#T6<^h&)x3I9Vu&}VOu&|g1Si;@H!otGB!otGBBG*9({eNr2NBorRJHG$`002ov
JPDHLkV1fheGgbfq

literal 0
HcmV?d00001

diff --git a/src/cmd/src/error.rs b/src/cmd/src/error.rs
index 8631eb7f58f0..970196ef1841 100644
--- a/src/cmd/src/error.rs
+++ b/src/cmd/src/error.rs
@@ -179,6 +179,13 @@ pub enum Error {
         error: etcd_client::Error,
         location: Location,
     },
+
+    #[snafu(display("Failed to serde json"))]
+    SerdeJson {
+        #[snafu(source)]
+        error: serde_json::error::Error,
+        location: Location,
+    },
 }
 
 pub type Result<T> = std::result::Result<T, Error>;
@@ -214,6 +221,8 @@ impl ErrorExt for Error {
             }
             Error::SubstraitEncodeLogicalPlan { source, .. } => source.status_code(),
             Error::StartCatalogManager { source, .. } => source.status_code(),
+
+            Error::SerdeJson { .. } => StatusCode::Unexpected,
         }
     }
 
diff --git a/src/cmd/src/options.rs b/src/cmd/src/options.rs
index caeef4960144..a95a10b30b30 100644
--- a/src/cmd/src/options.rs
+++ b/src/cmd/src/options.rs
@@ -21,7 +21,7 @@ use meta_srv::metasrv::MetaSrvOptions;
 use serde::{Deserialize, Serialize};
 use snafu::ResultExt;
 
-use crate::error::{LoadLayeredConfigSnafu, Result};
+use crate::error::{LoadLayeredConfigSnafu, Result, SerdeJsonSnafu};
 
 pub const ENV_VAR_SEP: &str = "__";
 pub const ENV_LIST_SEP: &str = ",";
@@ -94,9 +94,16 @@ impl Options {
                 .ignore_empty(true)
         };
 
+        // Workaround: Replacement for `Config::try_from(&default_opts)` due to
+        // `ConfigSerializer` cannot handle the case of an empty struct contained
+        // within an iterative structure.
+        // See: https://github.com/mehcode/config-rs/issues/461
+        let json_str = serde_json::to_string(&default_opts).context(SerdeJsonSnafu)?;
+        let default_config = File::from_str(&json_str, FileFormat::Json);
+
         // Add default values and environment variables as the sources of the configuration.
         let mut layered_config = Config::builder()
-            .add_source(Config::try_from(&default_opts).context(LoadLayeredConfigSnafu)?)
+            .add_source(default_config)
             .add_source(env_source);
 
         // Add config file as the source of the configuration if it is specified.
diff --git a/src/cmd/src/standalone.rs b/src/cmd/src/standalone.rs
index 73c298b4b41e..c6482a6ad5c0 100644
--- a/src/cmd/src/standalone.rs
+++ b/src/cmd/src/standalone.rs
@@ -593,4 +593,25 @@ mod tests {
             },
         );
     }
+
+    #[test]
+    fn test_load_default_standalone_options() {
+        let options: StandaloneOptions =
+            Options::load_layered_options(None, "GREPTIMEDB_FRONTEND", None).unwrap();
+        let default_options = StandaloneOptions::default();
+        assert_eq!(options.mode, default_options.mode);
+        assert_eq!(options.enable_telemetry, default_options.enable_telemetry);
+        assert_eq!(options.http, default_options.http);
+        assert_eq!(options.grpc, default_options.grpc);
+        assert_eq!(options.mysql, default_options.mysql);
+        assert_eq!(options.postgres, default_options.postgres);
+        assert_eq!(options.opentsdb, default_options.opentsdb);
+        assert_eq!(options.influxdb, default_options.influxdb);
+        assert_eq!(options.prom_store, default_options.prom_store);
+        assert_eq!(options.wal, default_options.wal);
+        assert_eq!(options.kv_store, default_options.kv_store);
+        assert_eq!(options.procedure, default_options.procedure);
+        assert_eq!(options.logging, default_options.logging);
+        assert_eq!(options.region_engine, default_options.region_engine);
+    }
 }
diff --git a/src/common/config/src/lib.rs b/src/common/config/src/lib.rs
index 36bedcb32faf..4daa3f0bc652 100644
--- a/src/common/config/src/lib.rs
+++ b/src/common/config/src/lib.rs
@@ -17,7 +17,7 @@ use std::time::Duration;
 use common_base::readable_size::ReadableSize;
 use serde::{Deserialize, Serialize};
 
-#[derive(Debug, Clone, Serialize, Deserialize)]
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
 #[serde(default)]
 pub struct WalConfig {
     // wal file size in bytes
@@ -49,7 +49,7 @@ pub fn kv_store_dir(store_dir: &str) -> String {
     format!("{store_dir}/kv")
 }
 
-#[derive(Debug, Clone, Serialize, Deserialize)]
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
 #[serde(default)]
 pub struct KvStoreConfig {
     // Kv file size in bytes
diff --git a/src/common/function/src/scalars/timestamp.rs b/src/common/function/src/scalars/timestamp.rs
index eb0e1afb1cb1..102bfb934bfc 100644
--- a/src/common/function/src/scalars/timestamp.rs
+++ b/src/common/function/src/scalars/timestamp.rs
@@ -12,8 +12,10 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 use std::sync::Arc;
+mod greatest;
 mod to_unixtime;
 
+use greatest::GreatestFunction;
 use to_unixtime::ToUnixtimeFunction;
 
 use crate::scalars::function_registry::FunctionRegistry;
@@ -23,5 +25,6 @@ pub(crate) struct TimestampFunction;
 impl TimestampFunction {
     pub fn register(registry: &FunctionRegistry) {
         registry.register(Arc::new(ToUnixtimeFunction));
+        registry.register(Arc::new(GreatestFunction));
     }
 }
diff --git a/src/common/function/src/scalars/timestamp/greatest.rs b/src/common/function/src/scalars/timestamp/greatest.rs
new file mode 100644
index 000000000000..e583872d5a24
--- /dev/null
+++ b/src/common/function/src/scalars/timestamp/greatest.rs
@@ -0,0 +1,175 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::fmt::{self};
+
+use common_query::error::{
+    self, ArrowComputeSnafu, InvalidFuncArgsSnafu, Result, UnsupportedInputDataTypeSnafu,
+};
+use common_query::prelude::{Signature, Volatility};
+use datatypes::arrow::array::AsArray;
+use datatypes::arrow::compute::cast;
+use datatypes::arrow::compute::kernels::comparison::gt_dyn;
+use datatypes::arrow::compute::kernels::zip;
+use datatypes::arrow::datatypes::{DataType as ArrowDataType, Date32Type};
+use datatypes::prelude::ConcreteDataType;
+use datatypes::vectors::{Helper, VectorRef};
+use snafu::{ensure, ResultExt};
+
+use crate::scalars::function::{Function, FunctionContext};
+
+#[derive(Clone, Debug, Default)]
+pub struct GreatestFunction;
+
+const NAME: &str = "greatest";
+
+impl Function for GreatestFunction {
+    fn name(&self) -> &str {
+        NAME
+    }
+
+    fn return_type(&self, _input_types: &[ConcreteDataType]) -> Result<ConcreteDataType> {
+        Ok(ConcreteDataType::date_datatype())
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::uniform(
+            2,
+            vec![
+                ConcreteDataType::string_datatype(),
+                ConcreteDataType::date_datatype(),
+            ],
+            Volatility::Immutable,
+        )
+    }
+
+    fn eval(&self, _func_ctx: FunctionContext, columns: &[VectorRef]) -> Result<VectorRef> {
+        ensure!(
+            columns.len() == 2,
+            InvalidFuncArgsSnafu {
+                err_msg: format!(
+                    "The length of the args is not correct, expect exactly two, have: {}",
+                    columns.len()
+                ),
+            }
+        );
+        match columns[0].data_type() {
+            ConcreteDataType::String(_) => {
+                let column1 = cast(&columns[0].to_arrow_array(), &ArrowDataType::Date32)
+                    .context(ArrowComputeSnafu)?;
+                let column1 = column1.as_primitive::<Date32Type>();
+                let column2 = cast(&columns[1].to_arrow_array(), &ArrowDataType::Date32)
+                    .context(ArrowComputeSnafu)?;
+                let column2 = column2.as_primitive::<Date32Type>();
+                let boolean_array = gt_dyn(&column1, &column2).context(ArrowComputeSnafu)?;
+                let result =
+                    zip::zip(&boolean_array, &column1, &column2).context(ArrowComputeSnafu)?;
+                Ok(Helper::try_into_vector(&result).context(error::FromArrowArraySnafu)?)
+            }
+            ConcreteDataType::Date(_) => {
+                let column1 = columns[0].to_arrow_array();
+                let column1 = column1.as_primitive::<Date32Type>();
+                let column2 = columns[1].to_arrow_array();
+                let column2 = column2.as_primitive::<Date32Type>();
+                let boolean_array = gt_dyn(&column1, &column2).context(ArrowComputeSnafu)?;
+                let result =
+                    zip::zip(&boolean_array, &column1, &column2).context(ArrowComputeSnafu)?;
+                Ok(Helper::try_into_vector(&result).context(error::FromArrowArraySnafu)?)
+            }
+            _ => UnsupportedInputDataTypeSnafu {
+                function: NAME,
+                datatypes: columns.iter().map(|c| c.data_type()).collect::<Vec<_>>(),
+            }
+            .fail(),
+        }
+    }
+}
+
+impl fmt::Display for GreatestFunction {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "GREATEST")
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::str::FromStr;
+    use std::sync::Arc;
+
+    use common_time::Date;
+    use datatypes::prelude::ConcreteDataType;
+    use datatypes::types::DateType;
+    use datatypes::value::Value;
+    use datatypes::vectors::{DateVector, StringVector, Vector};
+
+    use super::GreatestFunction;
+    use crate::scalars::function::FunctionContext;
+    use crate::scalars::Function;
+
+    #[test]
+    fn test_greatest_takes_string_vector() {
+        let function = GreatestFunction;
+        assert_eq!(
+            function.return_type(&[]).unwrap(),
+            ConcreteDataType::Date(DateType)
+        );
+        let columns = vec![
+            Arc::new(StringVector::from(vec![
+                "1970-01-01".to_string(),
+                "2012-12-23".to_string(),
+            ])) as _,
+            Arc::new(StringVector::from(vec![
+                "2001-02-01".to_string(),
+                "1999-01-01".to_string(),
+            ])) as _,
+        ];
+
+        let result = function.eval(FunctionContext::default(), &columns).unwrap();
+        let result = result.as_any().downcast_ref::<DateVector>().unwrap();
+        assert_eq!(result.len(), 2);
+        assert_eq!(
+            result.get(0),
+            Value::Date(Date::from_str("2001-02-01").unwrap())
+        );
+        assert_eq!(
+            result.get(1),
+            Value::Date(Date::from_str("2012-12-23").unwrap())
+        );
+    }
+
+    #[test]
+    fn test_greatest_takes_date_vector() {
+        let function = GreatestFunction;
+        assert_eq!(
+            function.return_type(&[]).unwrap(),
+            ConcreteDataType::Date(DateType)
+        );
+        let columns = vec![
+            Arc::new(DateVector::from_slice(vec![-1, 2])) as _,
+            Arc::new(DateVector::from_slice(vec![0, 1])) as _,
+        ];
+
+        let result = function.eval(FunctionContext::default(), &columns).unwrap();
+        let result = result.as_any().downcast_ref::<DateVector>().unwrap();
+        assert_eq!(result.len(), 2);
+        assert_eq!(
+            result.get(0),
+            Value::Date(Date::from_str("1970-01-01").unwrap())
+        );
+        assert_eq!(
+            result.get(1),
+            Value::Date(Date::from_str("1970-01-03").unwrap())
+        );
+    }
+}
diff --git a/src/datanode/src/config.rs b/src/datanode/src/config.rs
index b3835e6e2e84..74c86c5ced53 100644
--- a/src/datanode/src/config.rs
+++ b/src/datanode/src/config.rs
@@ -86,7 +86,7 @@ impl Default for StorageConfig {
     }
 }
 
-#[derive(Debug, Clone, Serialize, Default, Deserialize)]
+#[derive(Debug, Clone, Serialize, Default, Deserialize, Eq, PartialEq)]
 #[serde(default)]
 pub struct FileConfig {}
 
@@ -378,7 +378,7 @@ impl DatanodeOptions {
     }
 }
 
-#[derive(Debug, Serialize, Deserialize, Clone)]
+#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)]
 pub enum RegionEngineConfig {
     #[serde(rename = "mito")]
     Mito(MitoConfig),
diff --git a/src/file-engine/src/config.rs b/src/file-engine/src/config.rs
index fd592497db46..32e85357ec58 100644
--- a/src/file-engine/src/config.rs
+++ b/src/file-engine/src/config.rs
@@ -14,5 +14,5 @@
 
 use serde::{Deserialize, Serialize};
 
-#[derive(Debug, Clone, Default, Serialize, Deserialize)]
+#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)]
 pub struct EngineConfig {}
diff --git a/src/file-engine/src/query.rs b/src/file-engine/src/query.rs
index b44931160623..e305376b2a79 100644
--- a/src/file-engine/src/query.rs
+++ b/src/file-engine/src/query.rs
@@ -25,6 +25,7 @@ use common_query::prelude::Expr;
 use common_recordbatch::error::{CastVectorSnafu, ExternalSnafu, Result as RecordBatchResult};
 use common_recordbatch::{RecordBatch, RecordBatchStream, SendableRecordBatchStream};
 use datafusion::logical_expr::utils as df_logical_expr_utils;
+use datatypes::prelude::ConcreteDataType;
 use datatypes::schema::{ColumnSchema, Schema, SchemaRef};
 use datatypes::vectors::VectorRef;
 use futures::Stream;
@@ -160,38 +161,13 @@ impl Stream for FileToScanRegionStream {
             Poll::Pending => Poll::Pending,
             Poll::Ready(Some(file_record_batch)) => {
                 let file_record_batch = file_record_batch?;
+                let scan_record_batch = if self.schema_eq(&file_record_batch) {
+                    Ok(file_record_batch)
+                } else {
+                    self.convert_record_batch(&file_record_batch)
+                };
 
-                if self.schema_type_match(&file_record_batch) {
-                    return Poll::Ready(Some(Ok(file_record_batch)));
-                }
-
-                let file_row_count = file_record_batch.num_rows();
-                let scan_schema = self.scan_schema.clone();
-                let mut columns = Vec::with_capacity(scan_schema.num_columns());
-                for scan_column_schema in scan_schema.column_schemas() {
-                    let scan_data_type = &scan_column_schema.data_type;
-
-                    let file_column = file_record_batch.column_by_name(&scan_column_schema.name);
-                    let column = if let Some(file_column) = file_column {
-                        if &file_column.data_type() == scan_data_type {
-                            file_column.clone()
-                        } else {
-                            file_column.cast(scan_data_type).context(CastVectorSnafu {
-                                from_type: file_column.data_type(),
-                                to_type: scan_data_type.clone(),
-                            })?
-                        }
-                    } else {
-                        Self::create_default_vector(scan_column_schema, file_row_count)
-                            .map_err(BoxedError::new)
-                            .context(ExternalSnafu)?
-                    };
-
-                    columns.push(column);
-                }
-
-                let scan_record_batch = RecordBatch::new(scan_schema, columns)?;
-                Poll::Ready(Some(Ok(scan_record_batch)))
+                Poll::Ready(Some(scan_record_batch))
             }
             Poll::Ready(None) => Poll::Ready(None),
         }
@@ -206,7 +182,7 @@ impl FileToScanRegionStream {
         }
     }
 
-    fn schema_type_match(&self, file_record_batch: &RecordBatch) -> bool {
+    fn schema_eq(&self, file_record_batch: &RecordBatch) -> bool {
         self.scan_schema
             .column_schemas()
             .iter()
@@ -218,6 +194,59 @@ impl FileToScanRegionStream {
             })
     }
 
+    /// Converts a RecordBatch from file schema to scan schema.
+    ///
+    /// This function performs the following operations:
+    /// - Projection: Only columns present in scan schema are retained.
+    /// - Cast Type: Columns present in both file schema and scan schema but with different types are cast to the type in scan schema.
+    /// - Backfill: Columns present in scan schema but not in file schema are backfilled with default values.
+    fn convert_record_batch(
+        &self,
+        file_record_batch: &RecordBatch,
+    ) -> RecordBatchResult<RecordBatch> {
+        let file_row_count = file_record_batch.num_rows();
+        let columns = self
+            .scan_schema
+            .column_schemas()
+            .iter()
+            .map(|scan_column_schema| {
+                let file_column = file_record_batch.column_by_name(&scan_column_schema.name);
+                if let Some(file_column) = file_column {
+                    Self::cast_column_type(file_column, &scan_column_schema.data_type)
+                } else {
+                    Self::backfill_column(scan_column_schema, file_row_count)
+                }
+            })
+            .collect::<RecordBatchResult<Vec<_>>>()?;
+
+        RecordBatch::new(self.scan_schema.clone(), columns)
+    }
+
+    fn cast_column_type(
+        source_column: &VectorRef,
+        target_data_type: &ConcreteDataType,
+    ) -> RecordBatchResult<VectorRef> {
+        if &source_column.data_type() == target_data_type {
+            Ok(source_column.clone())
+        } else {
+            source_column
+                .cast(target_data_type)
+                .context(CastVectorSnafu {
+                    from_type: source_column.data_type(),
+                    to_type: target_data_type.clone(),
+                })
+        }
+    }
+
+    fn backfill_column(
+        column_schema: &ColumnSchema,
+        num_rows: usize,
+    ) -> RecordBatchResult<VectorRef> {
+        Self::create_default_vector(column_schema, num_rows)
+            .map_err(BoxedError::new)
+            .context(ExternalSnafu)
+    }
+
     fn create_default_vector(column_schema: &ColumnSchema, num_rows: usize) -> Result<VectorRef> {
         column_schema
             .create_default_vector(num_rows)
diff --git a/src/file-engine/src/query/file_stream.rs b/src/file-engine/src/query/file_stream.rs
index 7b98f09cd27c..105785979eee 100644
--- a/src/file-engine/src/query/file_stream.rs
+++ b/src/file-engine/src/query/file_stream.rs
@@ -133,13 +133,9 @@ fn new_csv_stream(
 ) -> Result<SendableRecordBatchStream> {
     let file_schema = config.file_schema.arrow_schema().clone();
     let opener = build_csv_opener(file_schema.clone(), config, format)?;
-    build_record_batch_stream(
-        opener,
-        file_schema,
-        config.files,
-        config.projection,
-        config.limit,
-    )
+    // push down limit only if there is no filter
+    let limit = config.filters.is_empty().then_some(config.limit).flatten();
+    build_record_batch_stream(opener, file_schema, config.files, config.projection, limit)
 }
 
 fn new_json_stream(
@@ -149,13 +145,9 @@ fn new_json_stream(
 ) -> Result<SendableRecordBatchStream> {
     let file_schema = config.file_schema.arrow_schema().clone();
     let opener = build_json_opener(file_schema.clone(), config, format)?;
-    build_record_batch_stream(
-        opener,
-        file_schema,
-        config.files,
-        config.projection,
-        config.limit,
-    )
+    // push down limit only if there is no filter
+    let limit = config.filters.is_empty().then_some(config.limit).flatten();
+    build_record_batch_stream(opener, file_schema, config.files, config.projection, limit)
 }
 
 fn new_parquet_stream_with_exec_plan(
@@ -229,13 +221,9 @@ fn new_orc_stream(
 ) -> Result<SendableRecordBatchStream> {
     let file_schema = config.file_schema.arrow_schema().clone();
     let opener = build_orc_opener(file_schema.clone(), config)?;
-    build_record_batch_stream(
-        opener,
-        file_schema,
-        config.files,
-        config.projection,
-        config.limit,
-    )
+    // push down limit only if there is no filter
+    let limit = config.filters.is_empty().then_some(config.limit).flatten();
+    build_record_batch_stream(opener, file_schema, config.files, config.projection, limit)
 }
 
 #[derive(Debug, Clone)]
diff --git a/src/frontend/src/frontend.rs b/src/frontend/src/frontend.rs
index 641f936ddf7f..a45dd13ae340 100644
--- a/src/frontend/src/frontend.rs
+++ b/src/frontend/src/frontend.rs
@@ -24,7 +24,7 @@ use crate::service_config::{
     PostgresOptions, PromStoreOptions,
 };
 
-#[derive(Clone, Debug, Serialize, Deserialize)]
+#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
 #[serde(default)]
 pub struct FrontendOptions {
     pub mode: Mode,
diff --git a/src/frontend/src/instance/opentsdb.rs b/src/frontend/src/instance/opentsdb.rs
index e4d8ff14cf04..b2957754d205 100644
--- a/src/frontend/src/instance/opentsdb.rs
+++ b/src/frontend/src/instance/opentsdb.rs
@@ -27,21 +27,27 @@ use crate::instance::Instance;
 
 #[async_trait]
 impl OpentsdbProtocolHandler for Instance {
-    async fn exec(&self, data_point: &DataPoint, ctx: QueryContextRef) -> server_error::Result<()> {
+    async fn exec(
+        &self,
+        data_points: &[DataPoint],
+        ctx: QueryContextRef,
+    ) -> server_error::Result<usize> {
         self.plugins
             .get::<PermissionCheckerRef>()
             .as_ref()
             .check_permission(ctx.current_user(), PermissionReq::Opentsdb)
             .context(AuthSnafu)?;
 
-        let (requests, _) = data_point_to_grpc_row_insert_requests(data_point)?;
-        let _ = self
+        let (requests, no_of_rows) = data_point_to_grpc_row_insert_requests(data_points)?;
+        let output = self
             .handle_row_inserts(requests, ctx)
             .await
             .map_err(BoxedError::new)
-            .with_context(|_| server_error::ExecuteQuerySnafu {
-                query: format!("{data_point:?}"),
-            })?;
-        Ok(())
+            .context(servers::error::ExecuteGrpcQuerySnafu)?;
+
+        Ok(match output {
+            common_query::Output::AffectedRows(rows) => rows,
+            _ => no_of_rows,
+        })
     }
 }
diff --git a/src/frontend/src/service_config/grpc.rs b/src/frontend/src/service_config/grpc.rs
index e0a64565015d..4a83b5b8492f 100644
--- a/src/frontend/src/service_config/grpc.rs
+++ b/src/frontend/src/service_config/grpc.rs
@@ -17,7 +17,7 @@ use common_grpc::channel_manager::{
 };
 use serde::{Deserialize, Serialize};
 
-#[derive(Clone, Debug, Serialize, Deserialize)]
+#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
 pub struct GrpcOptions {
     pub addr: String,
     pub runtime_size: usize,
diff --git a/src/frontend/src/service_config/influxdb.rs b/src/frontend/src/service_config/influxdb.rs
index 8a113b53813e..177ee9bc35c7 100644
--- a/src/frontend/src/service_config/influxdb.rs
+++ b/src/frontend/src/service_config/influxdb.rs
@@ -14,7 +14,7 @@
 
 use serde::{Deserialize, Serialize};
 
-#[derive(Clone, Debug, Serialize, Deserialize)]
+#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
 pub struct InfluxdbOptions {
     pub enable: bool,
 }
diff --git a/src/frontend/src/service_config/mysql.rs b/src/frontend/src/service_config/mysql.rs
index 21a1e8f7ec91..623cedd149b4 100644
--- a/src/frontend/src/service_config/mysql.rs
+++ b/src/frontend/src/service_config/mysql.rs
@@ -15,7 +15,7 @@
 use serde::{Deserialize, Serialize};
 use servers::tls::TlsOption;
 
-#[derive(Clone, Debug, Serialize, Deserialize)]
+#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
 pub struct MysqlOptions {
     pub enable: bool,
     pub addr: String,
diff --git a/src/frontend/src/service_config/opentsdb.rs b/src/frontend/src/service_config/opentsdb.rs
index cd477aa799f2..59959f4c816c 100644
--- a/src/frontend/src/service_config/opentsdb.rs
+++ b/src/frontend/src/service_config/opentsdb.rs
@@ -14,7 +14,7 @@
 
 use serde::{Deserialize, Serialize};
 
-#[derive(Clone, Debug, Serialize, Deserialize)]
+#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
 pub struct OpentsdbOptions {
     pub enable: bool,
     pub addr: String,
diff --git a/src/frontend/src/service_config/otlp.rs b/src/frontend/src/service_config/otlp.rs
index 60bb98243b72..76ba465afb89 100644
--- a/src/frontend/src/service_config/otlp.rs
+++ b/src/frontend/src/service_config/otlp.rs
@@ -14,7 +14,7 @@
 
 use serde::{Deserialize, Serialize};
 
-#[derive(Clone, Debug, Serialize, Deserialize)]
+#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
 pub struct OtlpOptions {
     pub enable: bool,
 }
diff --git a/src/frontend/src/service_config/postgres.rs b/src/frontend/src/service_config/postgres.rs
index 7f8dc82fc4b5..e27f2b7bcb91 100644
--- a/src/frontend/src/service_config/postgres.rs
+++ b/src/frontend/src/service_config/postgres.rs
@@ -15,7 +15,7 @@
 use serde::{Deserialize, Serialize};
 use servers::tls::TlsOption;
 
-#[derive(Clone, Debug, Serialize, Deserialize)]
+#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
 pub struct PostgresOptions {
     pub enable: bool,
     pub addr: String,
diff --git a/src/frontend/src/service_config/prom_store.rs b/src/frontend/src/service_config/prom_store.rs
index dfc36fe72f4a..877ff42b78fe 100644
--- a/src/frontend/src/service_config/prom_store.rs
+++ b/src/frontend/src/service_config/prom_store.rs
@@ -14,7 +14,7 @@
 
 use serde::{Deserialize, Serialize};
 
-#[derive(Clone, Debug, Serialize, Deserialize)]
+#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
 pub struct PromStoreOptions {
     pub enable: bool,
 }
diff --git a/src/meta-client/src/lib.rs b/src/meta-client/src/lib.rs
index 9b3d5cfcf2a2..ec977357369a 100644
--- a/src/meta-client/src/lib.rs
+++ b/src/meta-client/src/lib.rs
@@ -18,7 +18,7 @@ pub mod client;
 pub mod error;
 
 // Options for meta client in datanode instance.
-#[derive(Clone, Debug, Serialize, Deserialize)]
+#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
 pub struct MetaClientOptions {
     pub metasrv_addrs: Vec<String>,
     pub timeout_millis: u64,
diff --git a/src/mito2/src/config.rs b/src/mito2/src/config.rs
index f9e444cf38d3..4de3b189d0ca 100644
--- a/src/mito2/src/config.rs
+++ b/src/mito2/src/config.rs
@@ -27,7 +27,7 @@ const DEFAULT_NUM_WORKERS: usize = 1;
 const DEFAULT_MAX_BG_JOB: usize = 4;
 
 /// Configuration for [MitoEngine](crate::engine::MitoEngine).
-#[derive(Debug, Serialize, Deserialize, Clone)]
+#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)]
 pub struct MitoConfig {
     // Worker configs:
     /// Number of region workers (default 1).
diff --git a/src/operator/src/statement/ddl.rs b/src/operator/src/statement/ddl.rs
index 2173e030c772..5d6a984eb2d8 100644
--- a/src/operator/src/statement/ddl.rs
+++ b/src/operator/src/statement/ddl.rs
@@ -153,7 +153,7 @@ impl StatementExecutor {
             .await
             .context(error::InvalidateTableCacheSnafu)?;
 
-        Ok(Output::AffectedRows(1))
+        Ok(Output::AffectedRows(0))
     }
 
     pub async fn truncate_table(&self, table_name: TableName) -> Result<Output> {
@@ -181,9 +181,8 @@ impl StatementExecutor {
         table_info: Arc<TableInfo>,
         expr: AlterExpr,
     ) -> Result<()> {
-        let request: table::requests::AlterTableRequest =
-            common_grpc_expr::alter_expr_to_request(table_id, expr)
-                .context(AlterExprToRequestSnafu)?;
+        let request: AlterTableRequest = common_grpc_expr::alter_expr_to_request(table_id, expr)
+            .context(AlterExprToRequestSnafu)?;
 
         let AlterTableRequest { table_name, .. } = &request;
 
diff --git a/src/promql/src/extension_plan/series_divide.rs b/src/promql/src/extension_plan/series_divide.rs
index 55be29ab1804..3487fe578678 100644
--- a/src/promql/src/extension_plan/series_divide.rs
+++ b/src/promql/src/extension_plan/series_divide.rs
@@ -139,7 +139,7 @@ impl ExecutionPlan for SeriesDivideExec {
 
     fn required_input_ordering(&self) -> Vec<Option<Vec<PhysicalSortRequirement>>> {
         let input_schema = self.input.schema();
-        let exprs = self
+        let exprs: Vec<PhysicalSortRequirement> = self
             .tag_columns
             .iter()
             .map(|tag| PhysicalSortRequirement {
@@ -148,7 +148,11 @@ impl ExecutionPlan for SeriesDivideExec {
                 options: None,
             })
             .collect();
-        vec![Some(exprs)]
+        if !exprs.is_empty() {
+            vec![Some(exprs)]
+        } else {
+            vec![None]
+        }
     }
 
     fn output_ordering(&self) -> Option<&[PhysicalSortExpr]> {
diff --git a/src/query/src/dist_plan/analyzer.rs b/src/query/src/dist_plan/analyzer.rs
index 7807913e74b5..08262da196f5 100644
--- a/src/query/src/dist_plan/analyzer.rs
+++ b/src/query/src/dist_plan/analyzer.rs
@@ -142,7 +142,7 @@ impl PlanRewriter {
             return true;
         }
 
-        match Categorizer::check_plan(plan) {
+        match Categorizer::check_plan(plan, self.partition_cols.clone()) {
             Commutativity::Commutative => {}
             Commutativity::PartialCommutative => {
                 if let Some(plan) = partial_commutative_transformer(plan) {
@@ -161,7 +161,6 @@ impl PlanRewriter {
                     self.stage.push(plan)
                 }
             },
-            Commutativity::CheckPartition
             | Commutativity::NonCommutative
             | Commutativity::Unimplemented
             | Commutativity::Unsupported => {
@@ -351,11 +350,7 @@ mod test {
 
         let config = ConfigOptions::default();
         let result = DistPlannerAnalyzer {}.analyze(plan, &config).unwrap();
-        let expected = [
-            "Aggregate: groupBy=[[]], aggr=[[AVG(t.number)]]",
-            "  MergeScan [is_placeholder=false]",
-        ]
-        .join("\n");
+        let expected = "MergeScan [is_placeholder=false]";
         assert_eq!(expected, format!("{:?}", result));
     }
 
@@ -402,11 +397,7 @@ mod test {
 
         let config = ConfigOptions::default();
         let result = DistPlannerAnalyzer {}.analyze(plan, &config).unwrap();
-        let expected = [
-            "Limit: skip=0, fetch=1",
-            "  MergeScan [is_placeholder=false]",
-        ]
-        .join("\n");
+        let expected = "MergeScan [is_placeholder=false]";
         assert_eq!(expected, format!("{:?}", result));
     }
 
diff --git a/src/query/src/dist_plan/commutativity.rs b/src/query/src/dist_plan/commutativity.rs
index 6765d84222b3..6bfcee5a16d6 100644
--- a/src/query/src/dist_plan/commutativity.rs
+++ b/src/query/src/dist_plan/commutativity.rs
@@ -12,8 +12,10 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::collections::HashSet;
 use std::sync::Arc;
 
+use datafusion_expr::utils::exprlist_to_columns;
 use datafusion_expr::{Expr, LogicalPlan, UserDefinedLogicalNode};
 use promql::extension_plan::{
     EmptyMetric, InstantManipulate, RangeManipulate, SeriesDivide, SeriesNormalize,
@@ -29,7 +31,6 @@ pub enum Commutativity {
     TransformedCommutative(Option<Transformer>),
     NonCommutative,
     Unimplemented,
-    CheckPartition,
     /// For unrelated plans like DDL
     Unsupported,
 }
@@ -37,7 +38,9 @@ pub enum Commutativity {
 pub struct Categorizer {}
 
 impl Categorizer {
-    pub fn check_plan(plan: &LogicalPlan) -> Commutativity {
+    pub fn check_plan(plan: &LogicalPlan, partition_cols: Option<Vec<String>>) -> Commutativity {
+        let partition_cols = partition_cols.unwrap_or_default();
+
         match plan {
             LogicalPlan::Projection(proj) => {
                 for expr in &proj.expr {
@@ -51,11 +54,23 @@ impl Categorizer {
             // TODO(ruihang): Change this to Commutative once Like is supported in substrait
             LogicalPlan::Filter(filter) => Self::check_expr(&filter.predicate),
             LogicalPlan::Window(_) => Commutativity::Unimplemented,
-            LogicalPlan::Aggregate(_) => {
+            LogicalPlan::Aggregate(aggr) => {
+                if Self::check_partition(&aggr.group_expr, &partition_cols) {
+                    return Commutativity::Commutative;
+                }
+
                 // check all children exprs and uses the strictest level
                 Commutativity::Unimplemented
             }
-            LogicalPlan::Sort(_) => Commutativity::Unimplemented,
+            LogicalPlan::Sort(_) => {
+                if partition_cols.is_empty() {
+                    return Commutativity::Commutative;
+                }
+
+                // sort plan needs to consider column priority
+                // We can implement a merge-sort on partial ordered data
+                Commutativity::Unimplemented
+            }
             LogicalPlan::Join(_) => Commutativity::NonCommutative,
             LogicalPlan::CrossJoin(_) => Commutativity::NonCommutative,
             LogicalPlan::Repartition(_) => {
@@ -67,7 +82,17 @@ impl Categorizer {
             LogicalPlan::EmptyRelation(_) => Commutativity::NonCommutative,
             LogicalPlan::Subquery(_) => Commutativity::Unimplemented,
             LogicalPlan::SubqueryAlias(_) => Commutativity::Unimplemented,
-            LogicalPlan::Limit(_) => Commutativity::PartialCommutative,
+            LogicalPlan::Limit(limit) => {
+                // Only execute `fetch` on remote nodes.
+                // wait for https://github.com/apache/arrow-datafusion/pull/7669
+                if partition_cols.is_empty() && limit.fetch.is_some() {
+                    Commutativity::Commutative
+                } else if limit.skip == 0 && limit.fetch.is_some() {
+                    Commutativity::PartialCommutative
+                } else {
+                    Commutativity::Unimplemented
+                }
+            }
             LogicalPlan::Extension(extension) => {
                 Self::check_extension_plan(extension.node.as_ref() as _)
             }
@@ -93,7 +118,7 @@ impl Categorizer {
                 || name == SeriesDivide::name()
                 || name == MergeScanLogicalPlan::name() =>
             {
-                Commutativity::Commutative
+                Commutativity::Unimplemented
             }
             _ => Commutativity::Unsupported,
         }
@@ -142,6 +167,26 @@ impl Categorizer {
             | Expr::OuterReferenceColumn(_, _) => Commutativity::Unimplemented,
         }
     }
+
+    /// Return true if the given expr and partition cols satisfied the rule.
+    /// In this case the plan can be treated as fully commutative.
+    fn check_partition(exprs: &[Expr], partition_cols: &[String]) -> bool {
+        let mut ref_cols = HashSet::new();
+        if exprlist_to_columns(exprs, &mut ref_cols).is_err() {
+            return false;
+        }
+        let ref_cols = ref_cols
+            .into_iter()
+            .map(|c| c.flat_name())
+            .collect::<HashSet<_>>();
+        for col in partition_cols {
+            if !ref_cols.contains(col) {
+                return false;
+            }
+        }
+
+        true
+    }
 }
 
 pub type Transformer = Arc<dyn Fn(&LogicalPlan) -> Option<LogicalPlan>>;
@@ -149,3 +194,23 @@ pub type Transformer = Arc<dyn Fn(&LogicalPlan) -> Option<LogicalPlan>>;
 pub fn partial_commutative_transformer(plan: &LogicalPlan) -> Option<LogicalPlan> {
     Some(plan.clone())
 }
+
+#[cfg(test)]
+mod test {
+    use datafusion_expr::{LogicalPlanBuilder, Sort};
+
+    use super::*;
+
+    #[test]
+    fn sort_on_empty_partition() {
+        let plan = LogicalPlan::Sort(Sort {
+            expr: vec![],
+            input: Arc::new(LogicalPlanBuilder::empty(false).build().unwrap()),
+            fetch: None,
+        });
+        assert!(matches!(
+            Categorizer::check_plan(&plan, Some(vec![])),
+            Commutativity::Commutative
+        ));
+    }
+}
diff --git a/src/query/src/dist_plan/merge_scan.rs b/src/query/src/dist_plan/merge_scan.rs
index e2a9b9c82548..cf7afa26dae7 100644
--- a/src/query/src/dist_plan/merge_scan.rs
+++ b/src/query/src/dist_plan/merge_scan.rs
@@ -154,6 +154,7 @@ impl MergeScanExec {
         let regions = self.regions.clone();
         let region_query_handler = self.region_query_handler.clone();
         let metric = MergeScanMetric::new(&self.metric);
+        let schema = Self::arrow_schema_to_schema(self.schema())?;
 
         let stream = Box::pin(stream!({
             let _finish_timer = metric.finish_time().timer();
@@ -176,12 +177,14 @@ impl MergeScanExec {
 
                 while let Some(batch) = stream.next().await {
                     let batch = batch?;
+                    // reconstruct batch using `self.schema`
+                    // to remove metadata and correct column name
+                    let batch = RecordBatch::new(schema.clone(), batch.columns().iter().cloned())?;
                     metric.record_output_batch_rows(batch.num_rows());
-                    yield Ok(Self::remove_metadata_from_record_batch(batch));
-
                     if let Some(first_consume_timer) = first_consume_timer.as_mut().take() {
                         first_consume_timer.stop();
                     }
+                    yield Ok(batch);
                 }
             }
         }));
@@ -193,14 +196,6 @@ impl MergeScanExec {
         }))
     }
 
-    fn remove_metadata_from_record_batch(batch: RecordBatch) -> RecordBatch {
-        let arrow_schema = batch.schema.arrow_schema().as_ref();
-        let arrow_schema_without_metadata = Self::arrow_schema_without_metadata(arrow_schema);
-        let schema_without_metadata =
-            Self::arrow_schema_to_schema(arrow_schema_without_metadata).unwrap();
-        RecordBatch::new(schema_without_metadata, batch.columns().iter().cloned()).unwrap()
-    }
-
     fn arrow_schema_without_metadata(arrow_schema: &ArrowSchema) -> ArrowSchemaRef {
         Arc::new(ArrowSchema::new(
             arrow_schema
diff --git a/src/servers/src/heartbeat_options.rs b/src/servers/src/heartbeat_options.rs
index 436d1734276b..75a2875c9652 100644
--- a/src/servers/src/heartbeat_options.rs
+++ b/src/servers/src/heartbeat_options.rs
@@ -15,7 +15,7 @@
 use common_meta::distributed_time_constants;
 use serde::{Deserialize, Serialize};
 
-#[derive(Debug, Clone, Serialize, Deserialize)]
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
 #[serde(default)]
 pub struct HeartbeatOptions {
     pub interval_millis: u64,
diff --git a/src/servers/src/http/opentsdb.rs b/src/servers/src/http/opentsdb.rs
index c5b90b42a438..e509b19acc0d 100644
--- a/src/servers/src/http/opentsdb.rs
+++ b/src/servers/src/http/opentsdb.rs
@@ -84,18 +84,21 @@ pub async fn put(
     let summary = params.contains_key("summary");
     let details = params.contains_key("details");
 
-    let data_points = parse_data_points(body).await?;
+    let data_points = parse_data_points(body)
+        .await?
+        .into_iter()
+        .map(|point| point.into())
+        .collect::<Vec<_>>();
 
     let response = if !summary && !details {
-        for data_point in data_points.into_iter() {
-            if let Err(e) = opentsdb_handler.exec(&data_point.into(), ctx.clone()).await {
-                // Not debugging purpose, failed fast.
-                return error::InternalSnafu {
-                    err_msg: e.to_string(),
-                }
-                .fail();
+        if let Err(e) = opentsdb_handler.exec(&data_points, ctx.clone()).await {
+            // Not debugging purpose, failed fast.
+            return error::InternalSnafu {
+                err_msg: e.to_string(),
             }
+            .fail();
         }
+
         (HttpStatusCode::NO_CONTENT, Json(OpentsdbPutResponse::Empty))
     } else {
         let mut response = OpentsdbDebuggingResponse {
@@ -108,17 +111,15 @@ pub async fn put(
             },
         };
 
-        for data_point in data_points.into_iter() {
-            let result = opentsdb_handler
-                .exec(&data_point.clone().into(), ctx.clone())
-                .await;
-            match result {
-                Ok(()) => response.on_success(),
-                Err(e) => {
-                    response.on_failed(data_point, e);
-                }
+        let result = opentsdb_handler.exec(&data_points, ctx.clone()).await;
+        match result {
+            Ok(affected_rows) => response.on_success(affected_rows),
+            Err(e) => {
+                //TODO: handle the error since I can't get details about a single datapoint again
+                response.on_failed(e);
             }
         }
+
         (
             HttpStatusCode::OK,
             Json(OpentsdbPutResponse::Debug(response)),
@@ -151,20 +152,20 @@ pub struct OpentsdbDebuggingResponse {
 }
 
 impl OpentsdbDebuggingResponse {
-    fn on_success(&mut self) {
-        self.success += 1;
+    fn on_success(&mut self, affected_rows: usize) {
+        self.success += affected_rows as i32;
     }
 
-    fn on_failed(&mut self, datapoint: DataPointRequest, error: impl ErrorExt) {
+    fn on_failed(&mut self, _error: impl ErrorExt) {
         self.failed += 1;
 
-        if let Some(details) = self.errors.as_mut() {
-            let error = OpentsdbDetailError {
-                datapoint,
-                error: error.output_msg(),
-            };
-            details.push(error);
-        };
+        // if let Some(details) = self.errors.as_mut() {
+        //     let error = OpentsdbDetailError {
+        //         datapoint,
+        //         error: error.output_msg(),
+        //     };
+        //     details.push(error);
+        // };
     }
 }
 
diff --git a/src/servers/src/mysql/server.rs b/src/servers/src/mysql/server.rs
index b8eba89558f6..65af7e28696a 100644
--- a/src/servers/src/mysql/server.rs
+++ b/src/servers/src/mysql/server.rs
@@ -160,7 +160,7 @@ impl MysqlServer {
             if let Err(e)  = Self::do_handle(stream, spawn_ref, spawn_config).await {
                 // TODO(LFC): Write this error to client as well, in MySQL text protocol.
                 // Looks like we have to expose opensrv-mysql's `PacketWriter`?
-                warn!("Internal error occurred during query exec, server actively close the channel to let client try next time: {}.", e)
+                warn!(e; "Internal error occurred during query exec, server actively close the channel to let client try next time")
             }
             decrement_gauge!(crate::metrics::METRIC_MYSQL_CONNECTIONS, 1.0);
         });
diff --git a/src/servers/src/opentsdb.rs b/src/servers/src/opentsdb.rs
index ad054611204d..9c888f75a723 100644
--- a/src/servers/src/opentsdb.rs
+++ b/src/servers/src/opentsdb.rs
@@ -128,32 +128,35 @@ impl Server for OpentsdbServer {
 }
 
 pub fn data_point_to_grpc_row_insert_requests(
-    data_point: &DataPoint,
+    data_points: &[DataPoint],
 ) -> Result<(RowInsertRequests, usize)> {
     let mut multi_table_data = MultiTableData::new();
-    let table_name = data_point.metric();
-    let tags = data_point.tags().clone();
-    let value = data_point.value();
-    let timestamp = data_point.ts_millis();
-    let num_columns = tags.len() + 1;
-
-    let table_data = multi_table_data.get_or_default_table_data(table_name, num_columns, 0);
-    let mut one_row = table_data.alloc_one_row();
-
-    //tags
-    row_writer::write_tags(table_data, tags.into_iter(), &mut one_row)?;
-
-    // value
-    row_writer::write_f64(table_data, FIELD_COLUMN_NAME, value, &mut one_row)?;
-    // timestamp
-    row_writer::write_ts_millis(
-        table_data,
-        TIMESTAMP_COLUMN_NAME,
-        Some(timestamp),
-        &mut one_row,
-    )?;
-
-    table_data.add_row(one_row);
+
+    for data_point in data_points {
+        let table_name = data_point.metric();
+        let tags = data_point.tags().clone();
+        let value = data_point.value();
+        let timestamp = data_point.ts_millis();
+        let num_columns = tags.len() + 1;
+
+        let table_data = multi_table_data.get_or_default_table_data(table_name, num_columns, 0);
+        let mut one_row = table_data.alloc_one_row();
+
+        //tags
+        row_writer::write_tags(table_data, tags.into_iter(), &mut one_row)?;
+
+        // value
+        row_writer::write_f64(table_data, FIELD_COLUMN_NAME, value, &mut one_row)?;
+        // timestamp
+        row_writer::write_ts_millis(
+            table_data,
+            TIMESTAMP_COLUMN_NAME,
+            Some(timestamp),
+            &mut one_row,
+        )?;
+
+        table_data.add_row(one_row);
+    }
 
     Ok(multi_table_data.into_row_insert_requests())
 }
diff --git a/src/servers/src/opentsdb/handler.rs b/src/servers/src/opentsdb/handler.rs
index 4cbe1731fe11..6ddad7252bb1 100644
--- a/src/servers/src/opentsdb/handler.rs
+++ b/src/servers/src/opentsdb/handler.rs
@@ -94,7 +94,7 @@ impl<S: AsyncWrite + AsyncRead + Unpin> Handler<S> {
             match DataPoint::try_create(&line) {
                 Ok(data_point) => {
                     let _timer = timer!(crate::metrics::METRIC_TCP_OPENTSDB_LINE_WRITE_ELAPSED);
-                    let result = self.query_handler.exec(&data_point, ctx.clone()).await;
+                    let result = self.query_handler.exec(&[data_point], ctx.clone()).await;
                     if let Err(e) = result {
                         self.connection.write_line(e.output_msg()).await?;
                     }
@@ -128,8 +128,8 @@ mod tests {
 
     #[async_trait]
     impl OpentsdbProtocolHandler for DummyQueryHandler {
-        async fn exec(&self, data_point: &DataPoint, _ctx: QueryContextRef) -> Result<()> {
-            let metric = data_point.metric();
+        async fn exec(&self, data_points: &[DataPoint], _ctx: QueryContextRef) -> Result<usize> {
+            let metric = data_points.first().unwrap().metric();
             if metric == "should_failed" {
                 return error::InternalSnafu {
                     err_msg: "expected",
@@ -137,7 +137,7 @@ mod tests {
                 .fail();
             }
             self.tx.send(metric.to_string()).await.unwrap();
-            Ok(())
+            Ok(data_points.len())
         }
     }
 
diff --git a/src/servers/src/query_handler.rs b/src/servers/src/query_handler.rs
index ef8f74575e7c..955be25426c2 100644
--- a/src/servers/src/query_handler.rs
+++ b/src/servers/src/query_handler.rs
@@ -74,7 +74,7 @@ pub trait InfluxdbLineProtocolHandler {
 pub trait OpentsdbProtocolHandler {
     /// A successful request will not return a response.
     /// Only on error will the socket return a line of data.
-    async fn exec(&self, data_point: &DataPoint, ctx: QueryContextRef) -> Result<()>;
+    async fn exec(&self, data_points: &[DataPoint], ctx: QueryContextRef) -> Result<usize>;
 }
 
 pub struct PromStoreResponse {
diff --git a/src/servers/tests/http/opentsdb_test.rs b/src/servers/tests/http/opentsdb_test.rs
index e77143d3b3a1..f21d706a6fb7 100644
--- a/src/servers/tests/http/opentsdb_test.rs
+++ b/src/servers/tests/http/opentsdb_test.rs
@@ -51,15 +51,16 @@ impl GrpcQueryHandler for DummyInstance {
 
 #[async_trait]
 impl OpentsdbProtocolHandler for DummyInstance {
-    async fn exec(&self, data_point: &DataPoint, _ctx: QueryContextRef) -> Result<()> {
-        if data_point.metric() == "should_failed" {
+    async fn exec(&self, data_points: &[DataPoint], _ctx: QueryContextRef) -> Result<usize> {
+        let metric = data_points.first().unwrap().metric();
+        if metric == "should_failed" {
             return error::InternalSnafu {
                 err_msg: "expected",
             }
             .fail();
         }
-        let _ = self.tx.send(data_point.metric().to_string()).await;
-        Ok(())
+        let _ = self.tx.send(metric.to_string()).await;
+        Ok(data_points.len())
     }
 }
 
@@ -172,10 +173,7 @@ async fn test_opentsdb_put() {
     while let Ok(s) = rx.try_recv() {
         metrics.push(s);
     }
-    assert_eq!(
-        metrics,
-        vec!["m1".to_string(), "m2".to_string(), "m3".to_string()]
-    );
+    assert_eq!(metrics, vec!["m1".to_string(), "m2".to_string()]);
 }
 
 #[tokio::test]
@@ -208,8 +206,10 @@ async fn test_opentsdb_debug_put() {
         .send()
         .await;
     assert_eq!(result.status(), 200);
-    assert_eq!(result.text().await, "{\"success\":0,\"failed\":1,\"errors\":[{\"datapoint\":{\"metric\":\"should_failed\",\"timestamp\":1000,\"value\":1.0,\"tags\":{\"host\":\"web01\"}},\"error\":\"Internal error: expected\"}]}");
-
+    assert_eq!(
+        result.text().await,
+        "{\"success\":0,\"failed\":1,\"errors\":[]}"
+    );
     // multiple data point summary debug put
     let result = client
         .post("/v1/opentsdb/api/put?summary")
@@ -221,7 +221,7 @@ async fn test_opentsdb_debug_put() {
         .send()
         .await;
     assert_eq!(result.status(), 200);
-    assert_eq!(result.text().await, "{\"success\":1,\"failed\":1}");
+    assert_eq!(result.text().await, "{\"success\":0,\"failed\":1}");
 
     let result = client
         .post("/v1/opentsdb/api/put?details")
@@ -233,16 +233,16 @@ async fn test_opentsdb_debug_put() {
         .send()
         .await;
     assert_eq!(result.status(), 200);
-    assert_eq!(result.text().await, "{\"success\":1,\"failed\":1,\"errors\":[{\"datapoint\":{\"metric\":\"should_failed\",\"timestamp\":1000,\"value\":1.0,\"tags\":{\"host\":\"web01\"}},\"error\":\"Internal error: expected\"}]}");
+    assert_eq!(
+        result.text().await,
+        "{\"success\":0,\"failed\":1,\"errors\":[]}"
+    );
 
     let mut metrics = vec![];
     while let Ok(s) = rx.try_recv() {
         metrics.push(s);
     }
-    assert_eq!(
-        metrics,
-        vec!["m11".to_string(), "m22".to_string(), "m33".to_string()]
-    );
+    assert_eq!(metrics, vec!["m11".to_string()]);
 }
 
 fn create_data_point(metric: &str) -> String {
diff --git a/src/servers/tests/opentsdb.rs b/src/servers/tests/opentsdb.rs
index 145fdc07dbe6..8589e6637042 100644
--- a/src/servers/tests/opentsdb.rs
+++ b/src/servers/tests/opentsdb.rs
@@ -37,8 +37,8 @@ struct DummyOpentsdbInstance {
 
 #[async_trait]
 impl OpentsdbProtocolHandler for DummyOpentsdbInstance {
-    async fn exec(&self, data_point: &DataPoint, _ctx: QueryContextRef) -> Result<()> {
-        let metric = data_point.metric();
+    async fn exec(&self, data_points: &[DataPoint], _ctx: QueryContextRef) -> Result<usize> {
+        let metric = data_points.first().unwrap().metric();
         if metric == "should_failed" {
             return server_error::InternalSnafu {
                 err_msg: "expected",
@@ -47,7 +47,7 @@ impl OpentsdbProtocolHandler for DummyOpentsdbInstance {
         }
         let i = metric.parse::<i32>().unwrap();
         let _ = self.tx.send(i * i).await;
-        Ok(())
+        Ok(data_points.len())
     }
 }
 
diff --git a/tests-integration/src/grpc.rs b/tests-integration/src/grpc.rs
index 1216d61e0073..90ef779d982b 100644
--- a/tests-integration/src/grpc.rs
+++ b/tests-integration/src/grpc.rs
@@ -167,7 +167,7 @@ mod test {
             })),
         });
         let output = query(instance, request).await;
-        assert!(matches!(output, Output::AffectedRows(1)));
+        assert!(matches!(output, Output::AffectedRows(0)));
     }
 
     async fn verify_table_is_dropped(instance: &MockDistributedInstance) {
diff --git a/tests-integration/src/instance.rs b/tests-integration/src/instance.rs
index 7707df8d4b37..473f919e9406 100644
--- a/tests-integration/src/instance.rs
+++ b/tests-integration/src/instance.rs
@@ -259,7 +259,7 @@ mod tests {
         let Output::AffectedRows(x) = output else {
             unreachable!()
         };
-        assert_eq!(x, 1);
+        assert_eq!(x, 0);
     }
 
     async fn verify_table_is_dropped(instance: &MockDistributedInstance) {
diff --git a/tests-integration/src/opentsdb.rs b/tests-integration/src/opentsdb.rs
index d6e524e7ff3b..6818346e2c2b 100644
--- a/tests-integration/src/opentsdb.rs
+++ b/tests-integration/src/opentsdb.rs
@@ -56,7 +56,7 @@ mod tests {
             ],
         );
         // should create new table "my_metric_1" directly
-        instance.exec(&data_point1, ctx.clone()).await.unwrap();
+        instance.exec(&[data_point1], ctx.clone()).await.unwrap();
 
         let data_point2 = DataPoint::new(
             "my_metric_1".to_string(),
@@ -68,11 +68,11 @@ mod tests {
             ],
         );
         // should create new column "tagk3" directly
-        instance.exec(&data_point2, ctx.clone()).await.unwrap();
+        instance.exec(&[data_point2], ctx.clone()).await.unwrap();
 
         let data_point3 = DataPoint::new("my_metric_1".to_string(), 3000, 3.0, vec![]);
         // should handle null tags properly
-        instance.exec(&data_point3, ctx.clone()).await.unwrap();
+        instance.exec(&[data_point3], ctx.clone()).await.unwrap();
 
         let output = instance
             .do_query(
diff --git a/tests/cases/distributed/explain/join_10_tables.result b/tests/cases/distributed/explain/join_10_tables.result
index a354a2cc6797..74f06c246b0f 100644
--- a/tests/cases/distributed/explain/join_10_tables.result
+++ b/tests/cases/distributed/explain/join_10_tables.result
@@ -166,41 +166,41 @@ limit 1;
 
 drop table t_1;
 
-Affected Rows: 1
+Affected Rows: 0
 
 drop table t_2;
 
-Affected Rows: 1
+Affected Rows: 0
 
 drop table t_3;
 
-Affected Rows: 1
+Affected Rows: 0
 
 drop table t_4;
 
-Affected Rows: 1
+Affected Rows: 0
 
 drop table t_5;
 
-Affected Rows: 1
+Affected Rows: 0
 
 drop table t_6;
 
-Affected Rows: 1
+Affected Rows: 0
 
 drop table t_7;
 
-Affected Rows: 1
+Affected Rows: 0
 
 drop table t_8;
 
-Affected Rows: 1
+Affected Rows: 0
 
 drop table t_9;
 
-Affected Rows: 1
+Affected Rows: 0
 
 drop table t_10;
 
-Affected Rows: 1
+Affected Rows: 0
 
diff --git a/tests/cases/distributed/explain/multi_partitions.result b/tests/cases/distributed/explain/multi_partitions.result
index 3eb7b5491b95..d54f2c4dc644 100644
--- a/tests/cases/distributed/explain/multi_partitions.result
+++ b/tests/cases/distributed/explain/multi_partitions.result
@@ -34,5 +34,5 @@ explain SELECT * FROM demo WHERE ts > cast(1000000000 as timestamp) ORDER BY hos
 
 drop table demo;
 
-Affected Rows: 1
+Affected Rows: 0
 
diff --git a/tests/cases/distributed/explain/order_by.result b/tests/cases/distributed/explain/order_by.result
index 0501a35da0ec..4d08b3e86195 100644
--- a/tests/cases/distributed/explain/order_by.result
+++ b/tests/cases/distributed/explain/order_by.result
@@ -28,7 +28,7 @@ EXPLAIN SELECT DISTINCT i%2 FROM integers ORDER BY 1;
 
 DROP TABLE integers;
 
-Affected Rows: 1
+Affected Rows: 0
 
 CREATE TABLE test (a INTEGER, b INTEGER, t TIMESTAMP TIME INDEX);
 
@@ -44,10 +44,8 @@ EXPLAIN SELECT a, b FROM test ORDER BY a, b;
 +-+-+
 | plan_type_| plan_|
 +-+-+
-| logical_plan_| Sort: test.a ASC NULLS LAST, test.b ASC NULLS LAST_|
-|_|_MergeScan [is_placeholder=false]_|
-| physical_plan | SortExec: expr=[a@0 ASC NULLS LAST,b@1 ASC NULLS LAST] |
-|_|_MergeScanExec: REDACTED
+| logical_plan_| MergeScan [is_placeholder=false]_|
+| physical_plan | MergeScanExec: REDACTED
 |_|_|
 +-+-+
 
@@ -77,5 +75,5 @@ EXPLAIN SELECT DISTINCT a, b FROM test ORDER BY a, b;
 
 DROP TABLE test;
 
-Affected Rows: 1
+Affected Rows: 0
 
diff --git a/tests/cases/distributed/explain/single_partition.result b/tests/cases/distributed/explain/single_partition.result
index 581e62b4dabf..0df318ff1b31 100644
--- a/tests/cases/distributed/explain/single_partition.result
+++ b/tests/cases/distributed/explain/single_partition.result
@@ -34,15 +34,8 @@ EXPLAIN SELECT SUM(i) FROM single_partition;
 +-+-+
 | plan_type_| plan_|
 +-+-+
-| logical_plan_| Aggregate: groupBy=[[]], aggr=[[SUM(single_partition.i)]]_|
-|_|_Projection: single_partition.i_|
-|_|_MergeScan [is_placeholder=false]_|
-| physical_plan | AggregateExec: mode=Final, gby=[], aggr=[SUM(single_partition.i)]_|
-|_|_CoalescePartitionsExec_|
-|_|_AggregateExec: mode=Partial, gby=[], aggr=[SUM(single_partition.i)]_|
-|_|_RepartitionExec: partitioning=REDACTED
-|_|_ProjectionExec: expr=[i@0 as i]_|
-|_|_MergeScanExec: REDACTED
+| logical_plan_| MergeScan [is_placeholder=false]_|
+| physical_plan | MergeScanExec: REDACTED
 |_|_|
 +-+-+
 
@@ -56,14 +49,12 @@ EXPLAIN SELECT * FROM single_partition ORDER BY i DESC;
 +-+-+
 | plan_type_| plan_|
 +-+-+
-| logical_plan_| Sort: single_partition.i DESC NULLS FIRST_|
-|_|_MergeScan [is_placeholder=false]_|
-| physical_plan | SortExec: expr=[i@0 DESC]_|
-|_|_MergeScanExec: REDACTED
+| logical_plan_| MergeScan [is_placeholder=false]_|
+| physical_plan | MergeScanExec: REDACTED
 |_|_|
 +-+-+
 
 drop table single_partition;
 
-Affected Rows: 1
+Affected Rows: 0
 
diff --git a/tests/cases/distributed/explain/subqueries.result b/tests/cases/distributed/explain/subqueries.result
index ba5c162e2692..8d20b8e159e9 100644
--- a/tests/cases/distributed/explain/subqueries.result
+++ b/tests/cases/distributed/explain/subqueries.result
@@ -116,16 +116,14 @@ EXPLAIN INSERT INTO other SELECT i, 2 FROM integers WHERE i=(SELECT MAX(i) FROM
 |              |       Projection: integers.i                                      |
 |              |         MergeScan [is_placeholder=false]                          |
 |              |       SubqueryAlias: __scalar_sq_1                                |
-|              |         Aggregate: groupBy=[[]], aggr=[[MAX(integers.i)]]         |
-|              |           Projection: integers.i                                  |
-|              |             MergeScan [is_placeholder=false]                      |
+|              |         MergeScan [is_placeholder=false]                          |
 +--------------+-------------------------------------------------------------------+
 
 drop table other;
 
-Affected Rows: 1
+Affected Rows: 0
 
 drop table integers;
 
-Affected Rows: 1
+Affected Rows: 0
 
diff --git a/tests/cases/distributed/optimizer/filter_push_down.result b/tests/cases/distributed/optimizer/filter_push_down.result
index 142a9eb5572f..e6954bc84807 100644
--- a/tests/cases/distributed/optimizer/filter_push_down.result
+++ b/tests/cases/distributed/optimizer/filter_push_down.result
@@ -245,5 +245,5 @@ Error: 3001(EngineExecuteQuery), Error during planning: Attempted to create Filt
 
 DROP TABLE integers;
 
-Affected Rows: 1
+Affected Rows: 0
 
diff --git a/tests/cases/distributed/optimizer/order_by.result b/tests/cases/distributed/optimizer/order_by.result
index 7c43b90db189..b49f3eba4637 100644
--- a/tests/cases/distributed/optimizer/order_by.result
+++ b/tests/cases/distributed/optimizer/order_by.result
@@ -15,8 +15,7 @@ explain select * from numbers order by number desc;
 +---------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
 | plan_type     | plan                                                                                                                                                                                    |
 +---------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
-| logical_plan  | Sort: numbers.number DESC NULLS FIRST                                                                                                                                                   |
-|               |   MergeScan [is_placeholder=false]                                                                                                                                                      |
+| logical_plan  | MergeScan [is_placeholder=false]                                                                                                                                                        |
 | physical_plan | SortExec: expr=[number@0 DESC]                                                                                                                                                          |
 |               |   StreamScanAdapter { stream: "<SendableRecordBatchStream>", schema: [Field { name: "number", data_type: UInt32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} }] } |
 |               |                                                                                                                                                                                         |
@@ -28,8 +27,7 @@ explain select * from numbers order by number asc;
 +---------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
 | plan_type     | plan                                                                                                                                                                                    |
 +---------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
-| logical_plan  | Sort: numbers.number ASC NULLS LAST                                                                                                                                                     |
-|               |   MergeScan [is_placeholder=false]                                                                                                                                                      |
+| logical_plan  | MergeScan [is_placeholder=false]                                                                                                                                                        |
 | physical_plan | SortExec: expr=[number@0 ASC NULLS LAST]                                                                                                                                                |
 |               |   StreamScanAdapter { stream: "<SendableRecordBatchStream>", schema: [Field { name: "number", data_type: UInt32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} }] } |
 |               |                                                                                                                                                                                         |
@@ -41,9 +39,7 @@ explain select * from numbers order by number desc limit 10;
 +---------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
 | plan_type     | plan                                                                                                                                                                                      |
 +---------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
-| logical_plan  | Limit: skip=0, fetch=10                                                                                                                                                                   |
-|               |   Sort: numbers.number DESC NULLS FIRST, fetch=10                                                                                                                                         |
-|               |     MergeScan [is_placeholder=false]                                                                                                                                                      |
+| logical_plan  | MergeScan [is_placeholder=false]                                                                                                                                                          |
 | physical_plan | GlobalLimitExec: skip=0, fetch=10                                                                                                                                                         |
 |               |   SortExec: fetch=10, expr=[number@0 DESC]                                                                                                                                                |
 |               |     StreamScanAdapter { stream: "<SendableRecordBatchStream>", schema: [Field { name: "number", data_type: UInt32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} }] } |
@@ -56,9 +52,7 @@ explain select * from numbers order by number asc limit 10;
 +---------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
 | plan_type     | plan                                                                                                                                                                                      |
 +---------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
-| logical_plan  | Limit: skip=0, fetch=10                                                                                                                                                                   |
-|               |   Sort: numbers.number ASC NULLS LAST, fetch=10                                                                                                                                           |
-|               |     MergeScan [is_placeholder=false]                                                                                                                                                      |
+| logical_plan  | MergeScan [is_placeholder=false]                                                                                                                                                          |
 | physical_plan | GlobalLimitExec: skip=0, fetch=10                                                                                                                                                         |
 |               |   SortExec: fetch=10, expr=[number@0 ASC NULLS LAST]                                                                                                                                      |
 |               |     StreamScanAdapter { stream: "<SendableRecordBatchStream>", schema: [Field { name: "number", data_type: UInt32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} }] } |
diff --git a/tests/cases/distributed/show/show_create.result b/tests/cases/distributed/show/show_create.result
index 86e3687ed5b8..b871117b1ad9 100644
--- a/tests/cases/distributed/show/show_create.result
+++ b/tests/cases/distributed/show/show_create.result
@@ -49,7 +49,7 @@ SHOW CREATE TABLE system_metrics;
 
 DROP TABLE system_metrics;
 
-Affected Rows: 1
+Affected Rows: 0
 
 create table table_without_partition (
   ts TIMESTAMP TIME INDEX NOT NULL DEFAULT current_timestamp()
@@ -75,7 +75,7 @@ show create table table_without_partition;
 
 drop table table_without_partition;
 
-Affected Rows: 1
+Affected Rows: 0
 
 CREATE TABLE not_supported_table_options_keys (
   id INT UNSIGNED,
diff --git a/tests/cases/distributed/tql-explain-analyze/analyze.result b/tests/cases/distributed/tql-explain-analyze/analyze.result
index 32bb5223d6c9..1c35d346c58f 100644
--- a/tests/cases/distributed/tql-explain-analyze/analyze.result
+++ b/tests/cases/distributed/tql-explain-analyze/analyze.result
@@ -20,14 +20,15 @@ TQL ANALYZE (0, 10, '5s') test;
 +-+-+
 | Plan with Metrics | PromInstantManipulateExec: range=[0..10000], lookback=[300000], interval=[5000], time index=[j], REDACTED
 |_|_RepartitionExec: partitioning=REDACTED
+|_|_RepartitionExec: partitioning=REDACTED
 |_|_PromSeriesNormalizeExec: offset=[0], time index=[j], filter NaN: [false], REDACTED
 |_|_PromSeriesDivideExec: tags=["k"], REDACTED
-|_|_SortExec: expr=[k@2 DESC NULLS LAST,j@1 DESC NULLS LAST], REDACTED
+|_|_SortExec: expr=[k@2 ASC NULLS LAST], REDACTED
 |_|_MergeScanExec: REDACTED
 |_|_|
 +-+-+
 
 DROP TABLE test;
 
-Affected Rows: 1
+Affected Rows: 0
 
diff --git a/tests/cases/distributed/tql-explain-analyze/explain.result b/tests/cases/distributed/tql-explain-analyze/explain.result
index c6ee88bdf644..7778dab2e14a 100644
--- a/tests/cases/distributed/tql-explain-analyze/explain.result
+++ b/tests/cases/distributed/tql-explain-analyze/explain.result
@@ -18,19 +18,19 @@ TQL EXPLAIN (0, 10, '5s') test;
 | logical_plan  | PromInstantManipulate: range=[0..0], lookback=[300000], interval=[300000], time index=[j]     |
 |               |   PromSeriesNormalize: offset=[0], time index=[j], filter NaN: [false]                        |
 |               |     PromSeriesDivide: tags=["k"]                                                              |
-|               |       Sort: test.k DESC NULLS LAST, test.j DESC NULLS LAST                                    |
-|               |         MergeScan [is_placeholder=false]                                                      |
+|               |       MergeScan [is_placeholder=false]                                                        |
 | physical_plan | PromInstantManipulateExec: range=[0..0], lookback=[300000], interval=[300000], time index=[j] |
-|               |   PromSeriesNormalizeExec: offset=[0], time index=[j], filter NaN: [false]                    |
-|               |     PromSeriesDivideExec: tags=["k"]                                                          |
-|               |       SortExec: expr=[k@2 DESC NULLS LAST,j@1 DESC NULLS LAST]                                |
-|               |         MergeScanExec: REDACTED
+|               |   RepartitionExec: partitioning=REDACTED
+|               |     PromSeriesNormalizeExec: offset=[0], time index=[j], filter NaN: [false]                  |
+|               |       PromSeriesDivideExec: tags=["k"]                                                        |
+|               |         SortExec: expr=[k@2 ASC NULLS LAST]                                                   |
+|               |           MergeScanExec: REDACTED
 |               |                                                                                               |
 +---------------+-----------------------------------------------------------------------------------------------+
 
 DROP TABLE test;
 
-Affected Rows: 1
+Affected Rows: 0
 
 CREATE TABLE host_load1 (
   ts TIMESTAMP(3) NOT NULL,
@@ -47,23 +47,23 @@ Affected Rows: 0
 -- SQLNESS REPLACE (peers.*) REDACTED
 TQL EXPLAIN host_load1{__field__="val"};
 
-+---------------+------------------------------------------------------------------------------------------------------------------+
-| plan_type     | plan                                                                                                             |
-+---------------+------------------------------------------------------------------------------------------------------------------+
-| logical_plan  | PromInstantManipulate: range=[0..0], lookback=[300000], interval=[300000], time index=[ts]                       |
-|               |   PromSeriesNormalize: offset=[0], time index=[ts], filter NaN: [false]                                          |
-|               |     PromSeriesDivide: tags=["collector", "host"]                                                                 |
-|               |       Sort: host_load1.collector DESC NULLS LAST, host_load1.host DESC NULLS LAST, host_load1.ts DESC NULLS LAST |
-|               |         MergeScan [is_placeholder=false]                                                                         |
-| physical_plan | PromInstantManipulateExec: range=[0..0], lookback=[300000], interval=[300000], time index=[ts]                   |
-|               |   PromSeriesNormalizeExec: offset=[0], time index=[ts], filter NaN: [false]                                      |
-|               |     PromSeriesDivideExec: tags=["collector", "host"]                                                             |
-|               |       SortExec: expr=[collector@1 DESC NULLS LAST,host@2 DESC NULLS LAST,ts@3 DESC NULLS LAST]                   |
-|               |         MergeScanExec: REDACTED
-|               |                                                                                                                  |
-+---------------+------------------------------------------------------------------------------------------------------------------+
++---------------+------------------------------------------------------------------------------------------------+
+| plan_type     | plan                                                                                           |
++---------------+------------------------------------------------------------------------------------------------+
+| logical_plan  | PromInstantManipulate: range=[0..0], lookback=[300000], interval=[300000], time index=[ts]     |
+|               |   PromSeriesNormalize: offset=[0], time index=[ts], filter NaN: [false]                        |
+|               |     PromSeriesDivide: tags=["collector", "host"]                                               |
+|               |       MergeScan [is_placeholder=false]                                                         |
+| physical_plan | PromInstantManipulateExec: range=[0..0], lookback=[300000], interval=[300000], time index=[ts] |
+|               |   RepartitionExec: partitioning=REDACTED
+|               |     PromSeriesNormalizeExec: offset=[0], time index=[ts], filter NaN: [false]                  |
+|               |       PromSeriesDivideExec: tags=["collector", "host"]                                         |
+|               |         SortExec: expr=[collector@1 ASC NULLS LAST,host@2 ASC NULLS LAST]                      |
+|               |           MergeScanExec: REDACTED
+|               |                                                                                                |
++---------------+------------------------------------------------------------------------------------------------+
 
 DROP TABLE host_load1;
 
-Affected Rows: 1
+Affected Rows: 0
 
diff --git a/tests/cases/standalone/common/aggregate/distinct.result b/tests/cases/standalone/common/aggregate/distinct.result
index ef01ecc1eae5..fea497c70ac5 100644
--- a/tests/cases/standalone/common/aggregate/distinct.result
+++ b/tests/cases/standalone/common/aggregate/distinct.result
@@ -71,5 +71,5 @@ SELECT DISTINCT CASE WHEN a > 11 THEN 11 ELSE a END FROM test;
 
 DROP TABLE test;
 
-Affected Rows: 1
+Affected Rows: 0
 
diff --git a/tests/cases/standalone/common/aggregate/distinct_order_by.result b/tests/cases/standalone/common/aggregate/distinct_order_by.result
index f4d84302d5a2..a3fc6a8698d0 100644
--- a/tests/cases/standalone/common/aggregate/distinct_order_by.result
+++ b/tests/cases/standalone/common/aggregate/distinct_order_by.result
@@ -63,5 +63,5 @@ SELECT DISTINCT integers.i FROM integers ORDER BY integers.i DESC;
 
 DROP TABLE integers;
 
-Affected Rows: 1
+Affected Rows: 0
 
diff --git a/tests/cases/standalone/common/aggregate/sum.result b/tests/cases/standalone/common/aggregate/sum.result
index 7eaeb83946e2..526fce175745 100644
--- a/tests/cases/standalone/common/aggregate/sum.result
+++ b/tests/cases/standalone/common/aggregate/sum.result
@@ -72,9 +72,9 @@ SELECT sum(n) from doubles;
 
 DROP TABLE bigints;
 
-Affected Rows: 1
+Affected Rows: 0
 
 DROP TABLE doubles;
 
-Affected Rows: 1
+Affected Rows: 0
 
diff --git a/tests/cases/standalone/common/alter/add_col.result b/tests/cases/standalone/common/alter/add_col.result
index 3986ac48ca78..7b3c6d4fb4a1 100644
--- a/tests/cases/standalone/common/alter/add_col.result
+++ b/tests/cases/standalone/common/alter/add_col.result
@@ -89,5 +89,5 @@ DESC TABLE test;
 
 DROP TABLE test;
 
-Affected Rows: 1
+Affected Rows: 0
 
diff --git a/tests/cases/standalone/common/alter/add_col_chain.result b/tests/cases/standalone/common/alter/add_col_chain.result
index 1f8f7229e4f8..759525dcdbf0 100644
--- a/tests/cases/standalone/common/alter/add_col_chain.result
+++ b/tests/cases/standalone/common/alter/add_col_chain.result
@@ -34,5 +34,5 @@ SELECT * FROM test;
 
 DROP TABLE test;
 
-Affected Rows: 1
+Affected Rows: 0
 
diff --git a/tests/cases/standalone/common/alter/add_col_default.result b/tests/cases/standalone/common/alter/add_col_default.result
index 1526246756f5..6d8c523ba3e2 100644
--- a/tests/cases/standalone/common/alter/add_col_default.result
+++ b/tests/cases/standalone/common/alter/add_col_default.result
@@ -21,5 +21,5 @@ SELECT * FROM test;
 
 DROP TABLE test;
 
-Affected Rows: 1
+Affected Rows: 0
 
diff --git a/tests/cases/standalone/common/alter/add_incorrect_col.result b/tests/cases/standalone/common/alter/add_incorrect_col.result
index 1f8d90b575f3..f914da07a9b1 100644
--- a/tests/cases/standalone/common/alter/add_incorrect_col.result
+++ b/tests/cases/standalone/common/alter/add_incorrect_col.result
@@ -21,5 +21,5 @@ SELECT * FROM table_should_not_break_after_incorrect_alter;
 
 DROP TABLE table_should_not_break_after_incorrect_alter;
 
-Affected Rows: 1
+Affected Rows: 0
 
diff --git a/tests/cases/standalone/common/alter/alter_table.result b/tests/cases/standalone/common/alter/alter_table.result
index a5edc822151d..74e5c2399e8f 100644
--- a/tests/cases/standalone/common/alter/alter_table.result
+++ b/tests/cases/standalone/common/alter/alter_table.result
@@ -43,5 +43,5 @@ DESC TABLE test_alt_table;
 
 DROP TABLE test_alt_table;
 
-Affected Rows: 1
+Affected Rows: 0
 
diff --git a/tests/cases/standalone/common/alter/alter_table_first_after.result b/tests/cases/standalone/common/alter/alter_table_first_after.result
index ee03030078d4..ba7e0ade6c4c 100644
--- a/tests/cases/standalone/common/alter/alter_table_first_after.result
+++ b/tests/cases/standalone/common/alter/alter_table_first_after.result
@@ -170,5 +170,5 @@ Error: 1001(Unsupported), SQL statement is not supported: ALTER TABLE t ADD COLU
 
 DROP TABLE t;
 
-Affected Rows: 1
+Affected Rows: 0
 
diff --git a/tests/cases/standalone/common/alter/drop_col.result b/tests/cases/standalone/common/alter/drop_col.result
index 95125ef4e355..cf0c7251efa8 100644
--- a/tests/cases/standalone/common/alter/drop_col.result
+++ b/tests/cases/standalone/common/alter/drop_col.result
@@ -25,5 +25,5 @@ Error: 1004(InvalidArguments), Not allowed to remove index column j from table t
 
 DROP TABLE test;
 
-Affected Rows: 1
+Affected Rows: 0
 
diff --git a/tests/cases/standalone/common/alter/drop_col_not_null.result b/tests/cases/standalone/common/alter/drop_col_not_null.result
index 863f53136b53..86a6f3150eeb 100644
--- a/tests/cases/standalone/common/alter/drop_col_not_null.result
+++ b/tests/cases/standalone/common/alter/drop_col_not_null.result
@@ -35,5 +35,5 @@ SELECT * FROM test;
 
 DROP TABLE test;
 
-Affected Rows: 1
+Affected Rows: 0
 
diff --git a/tests/cases/standalone/common/alter/drop_col_not_null_next.result b/tests/cases/standalone/common/alter/drop_col_not_null_next.result
index 257f727bfdb0..049aa8d0df5b 100644
--- a/tests/cases/standalone/common/alter/drop_col_not_null_next.result
+++ b/tests/cases/standalone/common/alter/drop_col_not_null_next.result
@@ -40,5 +40,5 @@ SELECT * FROM test;
 
 DROP TABLE test;
 
-Affected Rows: 1
+Affected Rows: 0
 
diff --git a/tests/cases/standalone/common/alter/rename_table.result b/tests/cases/standalone/common/alter/rename_table.result
index 5d7f6a701f94..19015c91b258 100644
--- a/tests/cases/standalone/common/alter/rename_table.result
+++ b/tests/cases/standalone/common/alter/rename_table.result
@@ -71,9 +71,9 @@ Error: 4000(TableAlreadyExists), Table already exists, table: greptime.public.t
 
 DROP TABLE t;
 
-Affected Rows: 1
+Affected Rows: 0
 
 DROP TABLE new_table;
 
-Affected Rows: 1
+Affected Rows: 0
 
diff --git a/tests/cases/standalone/common/basic.result b/tests/cases/standalone/common/basic.result
index 19ffd297ec3c..b86ca616baa8 100644
--- a/tests/cases/standalone/common/basic.result
+++ b/tests/cases/standalone/common/basic.result
@@ -56,7 +56,7 @@ SELECT idc, avg(memory_util) FROM system_metrics GROUP BY idc ORDER BY idc;
 
 DROP TABLE system_metrics;
 
-Affected Rows: 1
+Affected Rows: 0
 
 create table foo (
     host string,
@@ -92,5 +92,5 @@ select * from foo;
 
 DROP TABLE foo;
 
-Affected Rows: 1
+Affected Rows: 0
 
diff --git a/tests/cases/standalone/common/catalog/schema.result b/tests/cases/standalone/common/catalog/schema.result
index 8c6985c29908..d62cd3d02d69 100644
--- a/tests/cases/standalone/common/catalog/schema.result
+++ b/tests/cases/standalone/common/catalog/schema.result
@@ -37,7 +37,7 @@ Affected Rows: 0
 
 DROP TABLE hello;
 
-Affected Rows: 1
+Affected Rows: 0
 
 CREATE TABLE hello(i TIMESTAMP TIME INDEX);
 
@@ -83,7 +83,7 @@ SHOW TABLES;
 
 DROP TABLE hello;
 
-Affected Rows: 1
+Affected Rows: 0
 
 DROP TABLE hello;
 
diff --git a/tests/cases/standalone/common/copy/copy_from_fs_csv.result b/tests/cases/standalone/common/copy/copy_from_fs_csv.result
index 2cdcf4b87275..d46b5b75fb72 100644
--- a/tests/cases/standalone/common/copy/copy_from_fs_csv.result
+++ b/tests/cases/standalone/common/copy/copy_from_fs_csv.result
@@ -63,17 +63,17 @@ select * from with_pattern order by ts;
 
 drop table demo;
 
-Affected Rows: 1
+Affected Rows: 0
 
 drop table with_filename;
 
-Affected Rows: 1
+Affected Rows: 0
 
 drop table with_path;
 
-Affected Rows: 1
+Affected Rows: 0
 
 drop table with_pattern;
 
-Affected Rows: 1
+Affected Rows: 0
 
diff --git a/tests/cases/standalone/common/copy/copy_from_fs_json.result b/tests/cases/standalone/common/copy/copy_from_fs_json.result
index 9ddda24dd5ba..3f8826e251f4 100644
--- a/tests/cases/standalone/common/copy/copy_from_fs_json.result
+++ b/tests/cases/standalone/common/copy/copy_from_fs_json.result
@@ -63,17 +63,17 @@ select * from with_pattern order by ts;
 
 drop table demo;
 
-Affected Rows: 1
+Affected Rows: 0
 
 drop table with_filename;
 
-Affected Rows: 1
+Affected Rows: 0
 
 drop table with_path;
 
-Affected Rows: 1
+Affected Rows: 0
 
 drop table with_pattern;
 
-Affected Rows: 1
+Affected Rows: 0
 
diff --git a/tests/cases/standalone/common/copy/copy_from_fs_parquet.result b/tests/cases/standalone/common/copy/copy_from_fs_parquet.result
index 6366bdbaa0cd..7bcabbbc2bd1 100644
--- a/tests/cases/standalone/common/copy/copy_from_fs_parquet.result
+++ b/tests/cases/standalone/common/copy/copy_from_fs_parquet.result
@@ -63,17 +63,17 @@ select * from with_pattern order by ts;
 
 drop table demo;
 
-Affected Rows: 1
+Affected Rows: 0
 
 drop table with_filename;
 
-Affected Rows: 1
+Affected Rows: 0
 
 drop table with_path;
 
-Affected Rows: 1
+Affected Rows: 0
 
 drop table with_pattern;
 
-Affected Rows: 1
+Affected Rows: 0
 
diff --git a/tests/cases/standalone/common/copy/copy_to_fs.result b/tests/cases/standalone/common/copy/copy_to_fs.result
index b8c16dc5654b..932067ee5788 100644
--- a/tests/cases/standalone/common/copy/copy_to_fs.result
+++ b/tests/cases/standalone/common/copy/copy_to_fs.result
@@ -20,5 +20,5 @@ Affected Rows: 2
 
 drop table demo;
 
-Affected Rows: 1
+Affected Rows: 0
 
diff --git a/tests/cases/standalone/common/create/create.result b/tests/cases/standalone/common/create/create.result
index 358e467c3154..67df15c3f3cd 100644
--- a/tests/cases/standalone/common/create/create.result
+++ b/tests/cases/standalone/common/create/create.result
@@ -74,19 +74,19 @@ DESC TABLE test2;
 
 DROP TABLE integers;
 
-Affected Rows: 1
+Affected Rows: 0
 
 DROP TABLE times;
 
-Affected Rows: 1
+Affected Rows: 0
 
 DROP TABLE test1;
 
-Affected Rows: 1
+Affected Rows: 0
 
 DROP TABLE test2;
 
-Affected Rows: 1
+Affected Rows: 0
 
 CREATE TABLE test_pk ("timestamp" TIMESTAMP TIME INDEX, host STRING PRIMARY KEY, "value" DOUBLE);
 
@@ -104,7 +104,7 @@ DESC TABLE test_pk;
 
 DROP TABLE test_pk;
 
-Affected Rows: 1
+Affected Rows: 0
 
 CREATE TABLE test_multiple_pk_definitions ("timestamp" TIMESTAMP TIME INDEX, host STRING PRIMARY KEY, "value" DOUBLE, PRIMARY KEY(host));
 
diff --git a/tests/cases/standalone/common/create/create_type_alias.result b/tests/cases/standalone/common/create/create_type_alias.result
index 6e192a4e5f05..7c7c00573752 100644
--- a/tests/cases/standalone/common/create/create_type_alias.result
+++ b/tests/cases/standalone/common/create/create_type_alias.result
@@ -76,5 +76,5 @@ DESC TABLE data_types;
 
 DROP TABLE data_types;
 
-Affected Rows: 1
+Affected Rows: 0
 
diff --git a/tests/cases/standalone/common/create/upper_case_table_name.result b/tests/cases/standalone/common/create/upper_case_table_name.result
index eed6c1e6f46d..f7d434f85d0c 100644
--- a/tests/cases/standalone/common/create/upper_case_table_name.result
+++ b/tests/cases/standalone/common/create/upper_case_table_name.result
@@ -29,7 +29,7 @@ select * from "system_Metric";
 
 drop table system_Metric;
 
-Affected Rows: 1
+Affected Rows: 0
 
 use public;
 
diff --git a/tests/cases/standalone/common/cte/cte.result b/tests/cases/standalone/common/cte/cte.result
index 4ca532f88744..5f399b61d64c 100644
--- a/tests/cases/standalone/common/cte/cte.result
+++ b/tests/cases/standalone/common/cte/cte.result
@@ -125,5 +125,5 @@ Error: 3000(PlanQuery), No field named alias2. Valid fields are a.
 
 drop table a;
 
-Affected Rows: 1
+Affected Rows: 0
 
diff --git a/tests/cases/standalone/common/cte/cte_in_cte.result b/tests/cases/standalone/common/cte/cte_in_cte.result
index 7a7238570970..9e50caef2a25 100644
--- a/tests/cases/standalone/common/cte/cte_in_cte.result
+++ b/tests/cases/standalone/common/cte/cte_in_cte.result
@@ -73,5 +73,5 @@ Error: 3000(PlanQuery), Error during planning: Table not found: greptime.public.
 
 drop table a;
 
-Affected Rows: 1
+Affected Rows: 0
 
diff --git a/tests/cases/standalone/common/delete/delete.result b/tests/cases/standalone/common/delete/delete.result
index 0a593f3f5c70..7885c4b0cfb6 100644
--- a/tests/cases/standalone/common/delete/delete.result
+++ b/tests/cases/standalone/common/delete/delete.result
@@ -58,5 +58,5 @@ SELECT ts, host, cpu, memory FROM monitor ORDER BY ts;
 
 DROP TABLE monitor;
 
-Affected Rows: 1
+Affected Rows: 0
 
diff --git a/tests/cases/standalone/common/describe/describe_table.result b/tests/cases/standalone/common/describe/describe_table.result
index b351bb3f24cb..1d414462fec8 100644
--- a/tests/cases/standalone/common/describe/describe_table.result
+++ b/tests/cases/standalone/common/describe/describe_table.result
@@ -54,5 +54,5 @@ desc host_load1;
 
 drop table host_load1;
 
-Affected Rows: 1
+Affected Rows: 0
 
diff --git a/tests/cases/standalone/common/function/time.result b/tests/cases/standalone/common/function/time.result
index 5d5898fe1db4..123b6a3f2f7c 100644
--- a/tests/cases/standalone/common/function/time.result
+++ b/tests/cases/standalone/common/function/time.result
@@ -4,3 +4,19 @@ select current_time();
 
 ++|current_time()|++|TIME|++
 
+select GREATEST('1999-01-30', '2023-03-01');
+
++-------------------------------------------------+
+| greatest(Utf8("1999-01-30"),Utf8("2023-03-01")) |
++-------------------------------------------------+
+| 2023-03-01                                      |
++-------------------------------------------------+
+
+select GREATEST('2000-02-11'::Date, '2020-12-30'::Date);
+
++-------------------------------------------------+
+| greatest(Utf8("2000-02-11"),Utf8("2020-12-30")) |
++-------------------------------------------------+
+| 2020-12-30                                      |
++-------------------------------------------------+
+
diff --git a/tests/cases/standalone/common/function/time.sql b/tests/cases/standalone/common/function/time.sql
index fd45687b424f..46d5c2347fd5 100644
--- a/tests/cases/standalone/common/function/time.sql
+++ b/tests/cases/standalone/common/function/time.sql
@@ -1,3 +1,7 @@
 -- SQLNESS REPLACE (\d+:\d+:\d+\.\d+) TIME
 -- SQLNESS REPLACE [\s\-]+
 select current_time();
+
+select GREATEST('1999-01-30', '2023-03-01');
+
+select GREATEST('2000-02-11'::Date, '2020-12-30'::Date);
diff --git a/tests/cases/standalone/common/insert/big_insert.result b/tests/cases/standalone/common/insert/big_insert.result
index 1c5906356a8c..56e33e871935 100644
--- a/tests/cases/standalone/common/insert/big_insert.result
+++ b/tests/cases/standalone/common/insert/big_insert.result
@@ -16,7 +16,7 @@ SELECT COUNT(*) FROM integers;
 
 DROP TABLE integers;
 
-Affected Rows: 1
+Affected Rows: 0
 
 CREATE TABLE integers(i BIGINT, j INTEGER, ts TIMESTAMP TIME INDEX);
 
@@ -37,5 +37,5 @@ SELECT * FROM integers;
 
 DROP TABLE integers;
 
-Affected Rows: 1
+Affected Rows: 0
 
diff --git a/tests/cases/standalone/common/insert/insert.result b/tests/cases/standalone/common/insert/insert.result
index 45f6a9137b46..d862eef7625d 100644
--- a/tests/cases/standalone/common/insert/insert.result
+++ b/tests/cases/standalone/common/insert/insert.result
@@ -40,9 +40,9 @@ Affected Rows: 1
 
 DROP TABLE integers;
 
-Affected Rows: 1
+Affected Rows: 0
 
 DROP TABLE presentations;
 
-Affected Rows: 1
+Affected Rows: 0
 
diff --git a/tests/cases/standalone/common/insert/insert_default.result b/tests/cases/standalone/common/insert/insert_default.result
index f4995b14d093..9dc85eba3661 100644
--- a/tests/cases/standalone/common/insert/insert_default.result
+++ b/tests/cases/standalone/common/insert/insert_default.result
@@ -58,9 +58,9 @@ SELECT * FROM test2;
 
 DROP TABLE test1;
 
-Affected Rows: 1
+Affected Rows: 0
 
 DROP TABLE test2;
 
-Affected Rows: 1
+Affected Rows: 0
 
diff --git a/tests/cases/standalone/common/insert/insert_invalid.result b/tests/cases/standalone/common/insert/insert_invalid.result
index 5ba9ea14be47..af74e727785f 100644
--- a/tests/cases/standalone/common/insert/insert_invalid.result
+++ b/tests/cases/standalone/common/insert/insert_invalid.result
@@ -44,9 +44,9 @@ Error: 1004(InvalidArguments), Invalid SQL, error: column count mismatch, column
 
 DROP TABLE strings;
 
-Affected Rows: 1
+Affected Rows: 0
 
 DROP TABLE a;
 
-Affected Rows: 1
+Affected Rows: 0
 
diff --git a/tests/cases/standalone/common/insert/insert_select.result b/tests/cases/standalone/common/insert/insert_select.result
index 209502d9b085..5fd5efbe61da 100644
--- a/tests/cases/standalone/common/insert/insert_select.result
+++ b/tests/cases/standalone/common/insert/insert_select.result
@@ -37,9 +37,9 @@ select * from demo2 order by ts;
 
 drop table demo1;
 
-Affected Rows: 1
+Affected Rows: 0
 
 drop table demo2;
 
-Affected Rows: 1
+Affected Rows: 0
 
diff --git a/tests/cases/standalone/common/insert/special_value.result b/tests/cases/standalone/common/insert/special_value.result
index 177e829c876e..e4b030e64a21 100644
--- a/tests/cases/standalone/common/insert/special_value.result
+++ b/tests/cases/standalone/common/insert/special_value.result
@@ -27,5 +27,5 @@ Error: 3001(EngineExecuteQuery), Cast error: Cannot cast string 'infinityyyy' to
 
 drop table data;
 
-Affected Rows: 1
+Affected Rows: 0
 
diff --git a/tests/cases/standalone/common/order/limit.result b/tests/cases/standalone/common/order/limit.result
index 3a6da8e09a45..a12352ee8410 100644
--- a/tests/cases/standalone/common/order/limit.result
+++ b/tests/cases/standalone/common/order/limit.result
@@ -121,13 +121,13 @@ Error: 3000(PlanQuery), Error during planning: LIMIT must not be negative
 
 DROP TABLE integers;
 
-Affected Rows: 1
+Affected Rows: 0
 
 DROP TABLE test;
 
-Affected Rows: 1
+Affected Rows: 0
 
 DROP TABLE test2;
 
-Affected Rows: 1
+Affected Rows: 0
 
diff --git a/tests/cases/standalone/common/order/limit_union.result b/tests/cases/standalone/common/order/limit_union.result
index 1ed9c9fd103c..75d636bf072c 100644
--- a/tests/cases/standalone/common/order/limit_union.result
+++ b/tests/cases/standalone/common/order/limit_union.result
@@ -30,5 +30,5 @@ SELECT COUNT(*) FROM (SELECT * FROM integers UNION ALL SELECT * FROM integers LI
 
 DROP TABLE integers;
 
-Affected Rows: 1
+Affected Rows: 0
 
diff --git a/tests/cases/standalone/common/order/nulls_first.result b/tests/cases/standalone/common/order/nulls_first.result
index 4fbf0948371c..9b0c28da795c 100644
--- a/tests/cases/standalone/common/order/nulls_first.result
+++ b/tests/cases/standalone/common/order/nulls_first.result
@@ -106,5 +106,5 @@ SELECT * FROM test ORDER BY i NULLS LAST;
 
 DROP TABLE test;
 
-Affected Rows: 1
+Affected Rows: 0
 
diff --git a/tests/cases/standalone/common/order/order_by.result b/tests/cases/standalone/common/order/order_by.result
index d03e088d9587..289da619b6e7 100644
--- a/tests/cases/standalone/common/order/order_by.result
+++ b/tests/cases/standalone/common/order/order_by.result
@@ -227,5 +227,5 @@ Error: 3000(PlanQuery), Error during planning: For SELECT DISTINCT, ORDER BY exp
 
 DROP TABLE test;
 
-Affected Rows: 1
+Affected Rows: 0
 
diff --git a/tests/cases/standalone/common/order/order_by_exceptions.result b/tests/cases/standalone/common/order/order_by_exceptions.result
index eac09e35ae29..bd3bfe545a44 100644
--- a/tests/cases/standalone/common/order/order_by_exceptions.result
+++ b/tests/cases/standalone/common/order/order_by_exceptions.result
@@ -62,5 +62,5 @@ Error: 3000(PlanQuery), Error during planning: Union queries must have the same
 
 DROP TABLE test;
 
-Affected Rows: 1
+Affected Rows: 0
 
diff --git a/tests/cases/standalone/common/order/order_variable_size_payload.result b/tests/cases/standalone/common/order/order_variable_size_payload.result
index 3ca0af9af42c..5a7dc32a18d6 100644
--- a/tests/cases/standalone/common/order/order_variable_size_payload.result
+++ b/tests/cases/standalone/common/order/order_variable_size_payload.result
@@ -392,41 +392,41 @@ ORDER BY "Sort", "EmployeeID";
 
 DROP TABLE t0;
 
-Affected Rows: 1
+Affected Rows: 0
 
 DROP TABLE test0;
 
-Affected Rows: 1
+Affected Rows: 0
 
 DROP TABLE test1;
 
-Affected Rows: 1
+Affected Rows: 0
 
 DROP TABLE test4;
 
-Affected Rows: 1
+Affected Rows: 0
 
 DROP TABLE tpch_q1_agg;
 
-Affected Rows: 1
+Affected Rows: 0
 
 DROP TABLE test5;
 
-Affected Rows: 1
+Affected Rows: 0
 
 DROP TABLE test6;
 
-Affected Rows: 1
+Affected Rows: 0
 
 DROP table test7;
 
-Affected Rows: 1
+Affected Rows: 0
 
 DROP table test8;
 
-Affected Rows: 1
+Affected Rows: 0
 
 DROP TABLE DirectReports;
 
-Affected Rows: 1
+Affected Rows: 0
 
diff --git a/tests/cases/standalone/common/select/dummy.result b/tests/cases/standalone/common/select/dummy.result
index 58845ddf5860..eb8969ba3115 100644
--- a/tests/cases/standalone/common/select/dummy.result
+++ b/tests/cases/standalone/common/select/dummy.result
@@ -109,5 +109,5 @@ select TO_UNIXTIME(b) from test_unixtime;
 
 DROP TABLE test_unixtime;
 
-Affected Rows: 1
+Affected Rows: 0
 
diff --git a/tests/cases/standalone/common/select/dummy.sql b/tests/cases/standalone/common/select/dummy.sql
index 81ef9324a7ce..4690900f50b9 100644
--- a/tests/cases/standalone/common/select/dummy.sql
+++ b/tests/cases/standalone/common/select/dummy.sql
@@ -31,4 +31,3 @@ select b from test_unixtime;
 select TO_UNIXTIME(b) from test_unixtime;
 
 DROP TABLE test_unixtime;
-
diff --git a/tests/cases/standalone/common/select/like.result b/tests/cases/standalone/common/select/like.result
index b47528b80e23..3bb738036032 100644
--- a/tests/cases/standalone/common/select/like.result
+++ b/tests/cases/standalone/common/select/like.result
@@ -26,5 +26,5 @@ SELECT * FROM host WHERE host LIKE '%+%';
 
 DROP TABLE host;
 
-Affected Rows: 1
+Affected Rows: 0
 
diff --git a/tests/cases/standalone/common/select/range_select.result b/tests/cases/standalone/common/select/range_select.result
index 033d34f24c05..17dfe05d3999 100644
--- a/tests/cases/standalone/common/select/range_select.result
+++ b/tests/cases/standalone/common/select/range_select.result
@@ -284,7 +284,7 @@ Error: 1003(Internal), Error during planning: Illegal argument in range select q
 
 DROP TABLE host;
 
-Affected Rows: 1
+Affected Rows: 0
 
 CREATE TABLE host_sec (
   ts timestamp(0) time index,
@@ -320,5 +320,5 @@ Affected Rows: 18
 -- SELECT ts, host, min(val) RANGE '10s', max(val) RANGE '10s' FROM host_sec ALIGN '5s' ORDER BY host, ts;
 DROP TABLE host_sec;
 
-Affected Rows: 1
+Affected Rows: 0
 
diff --git a/tests/cases/standalone/common/timestamp/timestamp.result b/tests/cases/standalone/common/timestamp/timestamp.result
index b52ee558fc94..331f1c8dc865 100644
--- a/tests/cases/standalone/common/timestamp/timestamp.result
+++ b/tests/cases/standalone/common/timestamp/timestamp.result
@@ -80,5 +80,5 @@ SELECT * FROM timestamp_with_precision ORDER BY ts ASC;
 
 DROP TABLE timestamp_with_precision;
 
-Affected Rows: 1
+Affected Rows: 0
 
diff --git a/tests/cases/standalone/common/tql/aggr_over_time.result b/tests/cases/standalone/common/tql/aggr_over_time.result
index 5b73fc5445c5..19930a200c62 100644
--- a/tests/cases/standalone/common/tql/aggr_over_time.result
+++ b/tests/cases/standalone/common/tql/aggr_over_time.result
@@ -59,7 +59,7 @@ tql eval (60, 60, '1s') stddev_over_time((metric[1m]));
 
 drop table metric;
 
-Affected Rows: 1
+Affected Rows: 0
 
 -- load 10s
 --   metric 1.5990505637277868 1.5990505637277868 1.5990505637277868
@@ -97,7 +97,7 @@ tql eval (60, 60, '1s') stddev_over_time(metric[1m]);
 
 drop table metric;
 
-Affected Rows: 1
+Affected Rows: 0
 
 -- Port from functions.test L632 - L680, commit 001ee2620e094970e5657ce39275b2fccdbd1359
 -- Include quantile over time
@@ -163,7 +163,7 @@ Affected Rows: 8
 -- tql eval (60, 60, '1s') (quantile_over_time(2, (data[1m])));
 drop table data;
 
-Affected Rows: 1
+Affected Rows: 0
 
 -- Port from functions.test L773 - L802, commit 001ee2620e094970e5657ce39275b2fccdbd1359
 -- Include max/min/last over time
@@ -219,5 +219,5 @@ Affected Rows: 15
 -- tql eval (60, 60, '1s') last_over_time(data[1m]);
 drop table data;
 
-Affected Rows: 1
+Affected Rows: 0
 
diff --git a/tests/cases/standalone/common/tql/basic.result b/tests/cases/standalone/common/tql/basic.result
index 46f600f9ce4e..12fab1cbb0fe 100644
--- a/tests/cases/standalone/common/tql/basic.result
+++ b/tests/cases/standalone/common/tql/basic.result
@@ -32,5 +32,5 @@ TQL EVAL (0, 10, '5s') test{k="a"};
 
 DROP TABLE test;
 
-Affected Rows: 1
+Affected Rows: 0
 
diff --git a/tests/cases/standalone/common/tql/binary_operator.result b/tests/cases/standalone/common/tql/binary_operator.result
index 4f0a022c483d..377fdcf9c992 100644
--- a/tests/cases/standalone/common/tql/binary_operator.result
+++ b/tests/cases/standalone/common/tql/binary_operator.result
@@ -35,7 +35,7 @@ tql eval (0, 30, '10s'), data + (1 > bool 2);
 
 drop table data;
 
-Affected Rows: 1
+Affected Rows: 0
 
 -- Binary operator on table with multiple field columns
 create table data (ts timestamp time index, val1 double, val2 double, val3 double);
@@ -84,5 +84,5 @@ tql eval (0, 30, '10s'), data{__field__="val1", __field__="val2"} + data{__field
 
 drop table data;
 
-Affected Rows: 1
+Affected Rows: 0
 
diff --git a/tests/cases/standalone/common/tql/join.result b/tests/cases/standalone/common/tql/join.result
index f1085a1c333e..689dac20ec07 100644
--- a/tests/cases/standalone/common/tql/join.result
+++ b/tests/cases/standalone/common/tql/join.result
@@ -56,20 +56,13 @@ tql eval(0, 10, '5s') sum(completion * 0.0015 / 1000) by (model) + sum(prompt *
 +---------+---------------------+-----------------------------------------------------------------------------------------------------------+
 | model   | ts                  | completion.SUM(val * Float64(0.0015) / Float64(1000)) + prompt.SUM(val * Float64(0.0015) / Float64(1000)) |
 +---------+---------------------+-----------------------------------------------------------------------------------------------------------+
-| model-a | 1970-01-01T00:00:00 | 0.000825                                                                                                  |
-| model-b | 1970-01-01T00:00:05 | 0.00066                                                                                                   |
+| model-a | 1970-01-01T00:00:00 | 0.000165                                                                                                  |
+| model-a | 1970-01-01T00:00:05 | 0.000165                                                                                                  |
+| model-a | 1970-01-01T00:00:10 | 0.000495                                                                                                  |
+| model-b | 1970-01-01T00:00:05 | 0.00033                                                                                                   |
+| model-b | 1970-01-01T00:00:10 | 0.00033                                                                                                   |
 +---------+---------------------+-----------------------------------------------------------------------------------------------------------+
 
--- SQLNESS SORT_RESULT 3 1
-tql eval(0, 10, '5s') sum(completion * 0.0015 / 1000) by (model) + sum(prompt * 0.0015 / 1000);
-
-+---------------------+-----------------------------------------------------------------------------------------------------------+
-| ts                  | completion.SUM(val * Float64(0.0015) / Float64(1000)) + prompt.SUM(val * Float64(0.0015) / Float64(1000)) |
-+---------------------+-----------------------------------------------------------------------------------------------------------+
-| 1970-01-01T00:00:00 | 0.000225                                                                                                  |
-| 1970-01-01T00:00:05 | 0.00051                                                                                                   |
-+---------------------+-----------------------------------------------------------------------------------------------------------+
-
 -- SQLNESS SORT_RESULT 3 1
 tql eval(0, 10, '5s') sum(completion / 1000) + max(completion / 1000);
 
@@ -94,9 +87,9 @@ tql eval(0, 10, '5s') sum(completion / 1000) + sum(completion / 1000);
 
 drop table completion;
 
-Affected Rows: 1
+Affected Rows: 0
 
 drop table prompt;
 
-Affected Rows: 1
+Affected Rows: 0
 
diff --git a/tests/cases/standalone/common/tql/join.sql b/tests/cases/standalone/common/tql/join.sql
index f67b4772a58f..e9f5c8a544cd 100644
--- a/tests/cases/standalone/common/tql/join.sql
+++ b/tests/cases/standalone/common/tql/join.sql
@@ -29,9 +29,6 @@ tql eval(0, 10, '5s') sum(completion * 0.0015 / 1000) + sum(prompt * 0.0015 / 10
 -- SQLNESS SORT_RESULT 3 1
 tql eval(0, 10, '5s') sum(completion * 0.0015 / 1000) by (model) + sum(prompt * 0.0015 / 1000) by (model);
 
--- SQLNESS SORT_RESULT 3 1
-tql eval(0, 10, '5s') sum(completion * 0.0015 / 1000) by (model) + sum(prompt * 0.0015 / 1000);
-
 -- SQLNESS SORT_RESULT 3 1
 tql eval(0, 10, '5s') sum(completion / 1000) + max(completion / 1000);
 
diff --git a/tests/cases/standalone/common/tql/operator.result b/tests/cases/standalone/common/tql/operator.result
index 1ad0461978c4..d3b85ac4458f 100644
--- a/tests/cases/standalone/common/tql/operator.result
+++ b/tests/cases/standalone/common/tql/operator.result
@@ -68,13 +68,13 @@ tql eval (300, 300, '1s') 10 atan2 NaN;
 
 drop table trigx;
 
-Affected Rows: 1
+Affected Rows: 0
 
 drop table trigy;
 
-Affected Rows: 1
+Affected Rows: 0
 
 drop table trignan;
 
-Affected Rows: 1
+Affected Rows: 0
 
diff --git a/tests/cases/standalone/common/truncate/truncate.result b/tests/cases/standalone/common/truncate/truncate.result
index 4267bc04d719..70e56d1ae1b4 100644
--- a/tests/cases/standalone/common/truncate/truncate.result
+++ b/tests/cases/standalone/common/truncate/truncate.result
@@ -72,5 +72,5 @@ SELECT ts, host, cpu, memory FROM monitor ORDER BY ts;
 
 DROP TABLE monitor;
 
-Affected Rows: 1
+Affected Rows: 0
 
diff --git a/tests/cases/standalone/common/types/blob/blob.result b/tests/cases/standalone/common/types/blob/blob.result
index 1673d99fe228..f716d6b038ee 100644
--- a/tests/cases/standalone/common/types/blob/blob.result
+++ b/tests/cases/standalone/common/types/blob/blob.result
@@ -130,9 +130,9 @@ SELECT '\x7F'::BYTEA;
 
 drop table blobs;
 
-Affected Rows: 1
+Affected Rows: 0
 
 drop table blob_empty;
 
-Affected Rows: 1
+Affected Rows: 0
 
diff --git a/tests/cases/standalone/common/types/float/infinity.result b/tests/cases/standalone/common/types/float/infinity.result
index 06102ca2cc6e..c35287f1cd49 100644
--- a/tests/cases/standalone/common/types/float/infinity.result
+++ b/tests/cases/standalone/common/types/float/infinity.result
@@ -211,7 +211,7 @@ SELECT f FROM floats WHERE f<='-inf'::FLOAT ORDER BY f;
 
 DROP TABLE floats;
 
-Affected Rows: 1
+Affected Rows: 0
 
 -- DOUBLE type
 -- inf as a constant
@@ -425,5 +425,5 @@ SELECT d FROM doubles WHERE d<='-inf'::DOUBLE ORDER BY d;
 
 DROP TABLE doubles;
 
-Affected Rows: 1
+Affected Rows: 0
 
diff --git a/tests/cases/standalone/common/types/float/nan.result b/tests/cases/standalone/common/types/float/nan.result
index ab68884be775..9ab48fe88ea7 100644
--- a/tests/cases/standalone/common/types/float/nan.result
+++ b/tests/cases/standalone/common/types/float/nan.result
@@ -155,7 +155,7 @@ SELECT f FROM floats WHERE f<='nan'::FLOAT;
 
 DROP TABLE floats;
 
-Affected Rows: 1
+Affected Rows: 0
 
 -- DOUBLE type
 -- nan as a constant
@@ -312,5 +312,5 @@ SELECT d FROM doubles WHERE d<='nan'::DOUBLE;
 
 DROP TABLE doubles;
 
-Affected Rows: 1
+Affected Rows: 0
 
diff --git a/tests/cases/standalone/common/types/float/nan_aggregate.result b/tests/cases/standalone/common/types/float/nan_aggregate.result
index f95cda7b02d5..50958986ee4e 100644
--- a/tests/cases/standalone/common/types/float/nan_aggregate.result
+++ b/tests/cases/standalone/common/types/float/nan_aggregate.result
@@ -44,7 +44,7 @@ SELECT SUM(f) FROM floats;
 
 DROP TABLE floats;
 
-Affected Rows: 1
+Affected Rows: 0
 
 -- double
 CREATE TABLE doubles(f DOUBLE, i INT, ts TIMESTAMP TIME INDEX);
@@ -91,7 +91,7 @@ SELECT SUM(f) FROM doubles;
 
 DROP TABLE doubles;
 
-Affected Rows: 1
+Affected Rows: 0
 
 -- float double
 CREATE TABLE floats_doubles (f FLOAT, d DOUBLE, ts TIMESTAMP TIME INDEX);
@@ -133,5 +133,5 @@ SELECT SUM(d) FROM floats_doubles WHERE d > 0;
 
 DROP TABLE floats_doubles;
 
-Affected Rows: 1
+Affected Rows: 0
 
diff --git a/tests/cases/standalone/common/types/float/nan_aggregates.result b/tests/cases/standalone/common/types/float/nan_aggregates.result
index 69249efddbd5..5dee887f238f 100644
--- a/tests/cases/standalone/common/types/float/nan_aggregates.result
+++ b/tests/cases/standalone/common/types/float/nan_aggregates.result
@@ -42,7 +42,7 @@ SELECT MIN(f), MAX(f) FROM floats;
 
 DROP TABLE floats;
 
-Affected Rows: 1
+Affected Rows: 0
 
 -- double --
 SELECT SUM('inf'::DOUBLE), SUM('-inf'::DOUBLE), SUM('nan'::DOUBLE);
@@ -87,5 +87,5 @@ SELECT MIN(f), MAX(f) FROM doubles;
 
 DROP TABLE doubles;
 
-Affected Rows: 1
+Affected Rows: 0
 
diff --git a/tests/cases/standalone/common/types/float/nan_join.result b/tests/cases/standalone/common/types/float/nan_join.result
index 6a6d305e8d7b..fcc9b9dd2889 100644
--- a/tests/cases/standalone/common/types/float/nan_join.result
+++ b/tests/cases/standalone/common/types/float/nan_join.result
@@ -31,7 +31,7 @@ Error: 3000(PlanQuery), Schema contains qualified field name floats.f and unqual
 
 DROP TABLE floats;
 
-Affected Rows: 1
+Affected Rows: 0
 
 -- DOUBLE type
 CREATE TABLE doubles(d DOUBLE, ts TIMESTAMP TIME INDEX);
@@ -64,5 +64,5 @@ Error: 3000(PlanQuery), Schema contains qualified field name doubles.d and unqua
 
 DROP TABLE doubles;
 
-Affected Rows: 1
+Affected Rows: 0
 
diff --git a/tests/cases/standalone/common/types/float/nan_ordering.result b/tests/cases/standalone/common/types/float/nan_ordering.result
index f39b78d63c87..c9665bf968a8 100644
--- a/tests/cases/standalone/common/types/float/nan_ordering.result
+++ b/tests/cases/standalone/common/types/float/nan_ordering.result
@@ -96,7 +96,7 @@ SELECT COUNT(*) FROM floats WHERE f < 0;
 
 DROP TABLE floats;
 
-Affected Rows: 1
+Affected Rows: 0
 
 -- DOUBLE type
 -- storing nan in a table
@@ -195,5 +195,5 @@ SELECT COUNT(*) FROM doubles WHERE d < 0;
 
 DROP TABLE doubles;
 
-Affected Rows: 1
+Affected Rows: 0
 
diff --git a/tests/cases/standalone/common/types/float/nan_window.result b/tests/cases/standalone/common/types/float/nan_window.result
index 3a3e14b2ab3a..9af92a209847 100644
--- a/tests/cases/standalone/common/types/float/nan_window.result
+++ b/tests/cases/standalone/common/types/float/nan_window.result
@@ -62,7 +62,7 @@ SELECT i, f, SUM(i) OVER (ORDER BY i, f) FROM floats ORDER BY i, f;
 
 DROP TABLE floats;
 
-Affected Rows: 1
+Affected Rows: 0
 
 -- grouping by inf and nan
 CREATE TABLE doubles(f DOUBLE, i INT, ts TIMESTAMP TIME INDEX);
@@ -127,5 +127,5 @@ SELECT i, f, SUM(i) OVER (ORDER BY i, f) FROM doubles ORDER BY i, f;
 
 DROP TABLE doubles;
 
-Affected Rows: 1
+Affected Rows: 0
 
diff --git a/tests/cases/standalone/common/types/interval/interval.result b/tests/cases/standalone/common/types/interval/interval.result
index 14d06feac65c..1b9f3a64d1f7 100644
--- a/tests/cases/standalone/common/types/interval/interval.result
+++ b/tests/cases/standalone/common/types/interval/interval.result
@@ -265,5 +265,5 @@ Error: 3000(PlanQuery), Error during planning: The function Avg does not support
 
 DROP TABLE intervals;
 
-Affected Rows: 1
+Affected Rows: 0
 
diff --git a/tests/cases/standalone/common/types/string/bigstring.result b/tests/cases/standalone/common/types/string/bigstring.result
index db405e9c2ecc..725f2dd659bb 100644
--- a/tests/cases/standalone/common/types/string/bigstring.result
+++ b/tests/cases/standalone/common/types/string/bigstring.result
@@ -34,5 +34,5 @@ SELECT LENGTH(a) FROM test ORDER BY 1;
 
 DROP TABLE test;
 
-Affected Rows: 1
+Affected Rows: 0
 
diff --git a/tests/cases/standalone/common/types/string/scan_big_varchar.result b/tests/cases/standalone/common/types/string/scan_big_varchar.result
index 449e44383971..f8ff770face5 100644
--- a/tests/cases/standalone/common/types/string/scan_big_varchar.result
+++ b/tests/cases/standalone/common/types/string/scan_big_varchar.result
@@ -224,9 +224,9 @@ SELECT COUNT(*), COUNT(a), MAX(LENGTH(a)), SUM(LENGTH(a)) FROM bigtable;
 
 DROP TABLE test;
 
-Affected Rows: 1
+Affected Rows: 0
 
 DROP TABLE bigtable;
 
-Affected Rows: 1
+Affected Rows: 0
 
diff --git a/tests/cases/standalone/common/types/string/unicode.result b/tests/cases/standalone/common/types/string/unicode.result
index d9f1b70e56ff..903c4fe970bb 100644
--- a/tests/cases/standalone/common/types/string/unicode.result
+++ b/tests/cases/standalone/common/types/string/unicode.result
@@ -64,5 +64,5 @@ SELECT length(s) FROM emojis ORDER BY id;
 
 DROP TABLE emojis;
 
-Affected Rows: 1
+Affected Rows: 0
 
diff --git a/tests/cases/standalone/common/types/timestamp/incorrect_timestamp.result b/tests/cases/standalone/common/types/timestamp/incorrect_timestamp.result
index 945292234f90..4b8895bb7ac4 100644
--- a/tests/cases/standalone/common/types/timestamp/incorrect_timestamp.result
+++ b/tests/cases/standalone/common/types/timestamp/incorrect_timestamp.result
@@ -60,5 +60,5 @@ Error: 2000(InvalidSyntax), Failed to parse value: Failed to parse 1900-1-1 00-0
 
 DROP TABLE timestamp;
 
-Affected Rows: 1
+Affected Rows: 0
 
diff --git a/tests/cases/standalone/common/types/timestamp/timestamp.result b/tests/cases/standalone/common/types/timestamp/timestamp.result
index 64029eafc9f9..70063acfab7c 100644
--- a/tests/cases/standalone/common/types/timestamp/timestamp.result
+++ b/tests/cases/standalone/common/types/timestamp/timestamp.result
@@ -173,5 +173,5 @@ Error: 3001(EngineExecuteQuery), Parser error: Error parsing timestamp from '100
 
 DROP TABLE timestamp;
 
-Affected Rows: 1
+Affected Rows: 0
 
diff --git a/tests/cases/standalone/common/types/timestamp/timestamp_join.result b/tests/cases/standalone/common/types/timestamp/timestamp_join.result
index 6c0b466a1701..c207d88fbef2 100644
--- a/tests/cases/standalone/common/types/timestamp/timestamp_join.result
+++ b/tests/cases/standalone/common/types/timestamp/timestamp_join.result
@@ -24,9 +24,9 @@ select count(*) from timestamp2 inner join timestamp1 on (timestamp1.i = timesta
 
 DROP TABLE timestamp1;
 
-Affected Rows: 1
+Affected Rows: 0
 
 DROP TABLE timestamp2;
 
-Affected Rows: 1
+Affected Rows: 0
 
diff --git a/tests/cases/standalone/common/types/timestamp/timestamp_precision.result b/tests/cases/standalone/common/types/timestamp/timestamp_precision.result
index 6c6877020031..6864cca4b716 100644
--- a/tests/cases/standalone/common/types/timestamp/timestamp_precision.result
+++ b/tests/cases/standalone/common/types/timestamp/timestamp_precision.result
@@ -85,5 +85,5 @@ SELECT TIMESTAMP '2020-01-01 01:23:45.12345678'::TIMESTAMP(9);
 
 DROP TABLE ts_precision;
 
-Affected Rows: 1
+Affected Rows: 0
 
diff --git a/tests/cases/standalone/common/types/timestamp/timestamp_types.result b/tests/cases/standalone/common/types/timestamp/timestamp_types.result
index 349caedcb0ee..ffe36657def5 100644
--- a/tests/cases/standalone/common/types/timestamp/timestamp_types.result
+++ b/tests/cases/standalone/common/types/timestamp/timestamp_types.result
@@ -372,9 +372,9 @@ select '2008-01-01 00:00:11'::TIMESTAMP_NS = '2008-01-01 00:00:11'::TIMESTAMP_S;
 
 DROP TABLE timestamp;
 
-Affected Rows: 1
+Affected Rows: 0
 
 DROP TABLE timestamp_two;
 
-Affected Rows: 1
+Affected Rows: 0
 
diff --git a/tests/cases/standalone/create/recover_created.result b/tests/cases/standalone/create/recover_created.result
index 6f67fd782cec..c384a7888a2f 100644
--- a/tests/cases/standalone/create/recover_created.result
+++ b/tests/cases/standalone/create/recover_created.result
@@ -8,11 +8,11 @@ Affected Rows: 0
 
 drop table t1;
 
-Affected Rows: 1
+Affected Rows: 0
 
 drop table t2;
 
-Affected Rows: 1
+Affected Rows: 0
 
 -- SQLNESS ARG restart=true
 show tables;
@@ -37,5 +37,5 @@ desc table t3;
 
 drop table t3;
 
-Affected Rows: 1
+Affected Rows: 0
 
diff --git a/tests/cases/standalone/limit/limit.result b/tests/cases/standalone/limit/limit.result
index 0f58f3c0bf49..0d39bee3e1c2 100644
--- a/tests/cases/standalone/limit/limit.result
+++ b/tests/cases/standalone/limit/limit.result
@@ -8,7 +8,7 @@ EXPLAIN SELECT * FROM (SELECT SUM(number) FROM numbers LIMIT 100000000000) LIMIT
 +---------------+----------------------------------+
 | plan_type     | plan                             |
 +---------------+----------------------------------+
-| logical_plan  | EmptyRelation                    |
+| logical_plan  | MergeScan [is_placeholder=false] |
 | physical_plan | EmptyExec: produce_one_row=false |
 |               |                                  |
 +---------------+----------------------------------+
@@ -18,7 +18,7 @@ EXPLAIN SELECT * FROM (SELECT SUM(number) FROM numbers LIMIT 100000000000) WHERE
 +---------------+----------------------------------+
 | plan_type     | plan                             |
 +---------------+----------------------------------+
-| logical_plan  | EmptyRelation                    |
+| logical_plan  | MergeScan [is_placeholder=false] |
 | physical_plan | EmptyExec: produce_one_row=false |
 |               |                                  |
 +---------------+----------------------------------+
diff --git a/tests/cases/standalone/optimizer/filter_push_down.result b/tests/cases/standalone/optimizer/filter_push_down.result
index 3480044d3b4f..5deb4641b926 100644
--- a/tests/cases/standalone/optimizer/filter_push_down.result
+++ b/tests/cases/standalone/optimizer/filter_push_down.result
@@ -196,5 +196,5 @@ Error: 3001(EngineExecuteQuery), Error during planning: Attempted to create Filt
 
 DROP TABLE integers;
 
-Affected Rows: 1
+Affected Rows: 0
 
diff --git a/tests/cases/standalone/optimizer/order_by.result b/tests/cases/standalone/optimizer/order_by.result
index c1d5b7171a73..129743b39d63 100644
--- a/tests/cases/standalone/optimizer/order_by.result
+++ b/tests/cases/standalone/optimizer/order_by.result
@@ -13,8 +13,7 @@ explain select * from numbers order by number desc;
 +---------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
 | plan_type     | plan                                                                                                                                                                                    |
 +---------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
-| logical_plan  | Sort: numbers.number DESC NULLS FIRST                                                                                                                                                   |
-|               |   MergeScan [is_placeholder=false]                                                                                                                                                      |
+| logical_plan  | MergeScan [is_placeholder=false]                                                                                                                                                        |
 | physical_plan | SortExec: expr=[number@0 DESC]                                                                                                                                                          |
 |               |   StreamScanAdapter { stream: "<SendableRecordBatchStream>", schema: [Field { name: "number", data_type: UInt32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} }] } |
 |               |                                                                                                                                                                                         |
@@ -25,8 +24,7 @@ explain select * from numbers order by number asc;
 +---------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
 | plan_type     | plan                                                                                                                                                                                    |
 +---------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
-| logical_plan  | Sort: numbers.number ASC NULLS LAST                                                                                                                                                     |
-|               |   MergeScan [is_placeholder=false]                                                                                                                                                      |
+| logical_plan  | MergeScan [is_placeholder=false]                                                                                                                                                        |
 | physical_plan | SortExec: expr=[number@0 ASC NULLS LAST]                                                                                                                                                |
 |               |   StreamScanAdapter { stream: "<SendableRecordBatchStream>", schema: [Field { name: "number", data_type: UInt32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} }] } |
 |               |                                                                                                                                                                                         |
@@ -37,9 +35,7 @@ explain select * from numbers order by number desc limit 10;
 +---------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
 | plan_type     | plan                                                                                                                                                                                      |
 +---------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
-| logical_plan  | Limit: skip=0, fetch=10                                                                                                                                                                   |
-|               |   Sort: numbers.number DESC NULLS FIRST, fetch=10                                                                                                                                         |
-|               |     MergeScan [is_placeholder=false]                                                                                                                                                      |
+| logical_plan  | MergeScan [is_placeholder=false]                                                                                                                                                          |
 | physical_plan | GlobalLimitExec: skip=0, fetch=10                                                                                                                                                         |
 |               |   SortExec: fetch=10, expr=[number@0 DESC]                                                                                                                                                |
 |               |     StreamScanAdapter { stream: "<SendableRecordBatchStream>", schema: [Field { name: "number", data_type: UInt32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} }] } |
@@ -51,9 +47,7 @@ explain select * from numbers order by number asc limit 10;
 +---------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
 | plan_type     | plan                                                                                                                                                                                      |
 +---------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
-| logical_plan  | Limit: skip=0, fetch=10                                                                                                                                                                   |
-|               |   Sort: numbers.number ASC NULLS LAST, fetch=10                                                                                                                                           |
-|               |     MergeScan [is_placeholder=false]                                                                                                                                                      |
+| logical_plan  | MergeScan [is_placeholder=false]                                                                                                                                                          |
 | physical_plan | GlobalLimitExec: skip=0, fetch=10                                                                                                                                                         |
 |               |   SortExec: fetch=10, expr=[number@0 ASC NULLS LAST]                                                                                                                                      |
 |               |     StreamScanAdapter { stream: "<SendableRecordBatchStream>", schema: [Field { name: "number", data_type: UInt32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} }] } |
diff --git a/tests/cases/standalone/show/show_create.result b/tests/cases/standalone/show/show_create.result
index ac3c9f6ea3bb..e9e22f060cf3 100644
--- a/tests/cases/standalone/show/show_create.result
+++ b/tests/cases/standalone/show/show_create.result
@@ -40,7 +40,7 @@ SHOW CREATE TABLE system_metrics;
 
 DROP TABLE system_metrics;
 
-Affected Rows: 1
+Affected Rows: 0
 
 CREATE TABLE not_supported_table_options_keys (
   id INT UNSIGNED,
diff --git a/tests/cases/standalone/tql-explain-analyze/analyze.result b/tests/cases/standalone/tql-explain-analyze/analyze.result
index 32bb5223d6c9..1c35d346c58f 100644
--- a/tests/cases/standalone/tql-explain-analyze/analyze.result
+++ b/tests/cases/standalone/tql-explain-analyze/analyze.result
@@ -20,14 +20,15 @@ TQL ANALYZE (0, 10, '5s') test;
 +-+-+
 | Plan with Metrics | PromInstantManipulateExec: range=[0..10000], lookback=[300000], interval=[5000], time index=[j], REDACTED
 |_|_RepartitionExec: partitioning=REDACTED
+|_|_RepartitionExec: partitioning=REDACTED
 |_|_PromSeriesNormalizeExec: offset=[0], time index=[j], filter NaN: [false], REDACTED
 |_|_PromSeriesDivideExec: tags=["k"], REDACTED
-|_|_SortExec: expr=[k@2 DESC NULLS LAST,j@1 DESC NULLS LAST], REDACTED
+|_|_SortExec: expr=[k@2 ASC NULLS LAST], REDACTED
 |_|_MergeScanExec: REDACTED
 |_|_|
 +-+-+
 
 DROP TABLE test;
 
-Affected Rows: 1
+Affected Rows: 0
 
diff --git a/tests/cases/standalone/tql-explain-analyze/explain.result b/tests/cases/standalone/tql-explain-analyze/explain.result
index edc07a6a7f8e..d0ebadd06bb7 100644
--- a/tests/cases/standalone/tql-explain-analyze/explain.result
+++ b/tests/cases/standalone/tql-explain-analyze/explain.result
@@ -18,17 +18,17 @@ TQL EXPLAIN (0, 10, '5s') test;
 | logical_plan  | PromInstantManipulate: range=[0..0], lookback=[300000], interval=[300000], time index=[j]     |
 |               |   PromSeriesNormalize: offset=[0], time index=[j], filter NaN: [false]                        |
 |               |     PromSeriesDivide: tags=["k"]                                                              |
-|               |       Sort: test.k DESC NULLS LAST, test.j DESC NULLS LAST                                    |
-|               |         MergeScan [is_placeholder=false]                                                      |
+|               |       MergeScan [is_placeholder=false]                                                        |
 | physical_plan | PromInstantManipulateExec: range=[0..0], lookback=[300000], interval=[300000], time index=[j] |
-|               |   PromSeriesNormalizeExec: offset=[0], time index=[j], filter NaN: [false]                    |
-|               |     PromSeriesDivideExec: tags=["k"]                                                          |
-|               |       SortExec: expr=[k@2 DESC NULLS LAST,j@1 DESC NULLS LAST]                                |
-|               |         MergeScanExec: REDACTED
+|               |   RepartitionExec: partitioning=REDACTED
+|               |     PromSeriesNormalizeExec: offset=[0], time index=[j], filter NaN: [false]                  |
+|               |       PromSeriesDivideExec: tags=["k"]                                                        |
+|               |         SortExec: expr=[k@2 ASC NULLS LAST]                                                   |
+|               |           MergeScanExec: REDACTED
 |               |                                                                                               |
 +---------------+-----------------------------------------------------------------------------------------------+
 
 DROP TABLE test;
 
-Affected Rows: 1
+Affected Rows: 0