From 65fffe974b6d881dabbf6600fe034f1e5ae809f0 Mon Sep 17 00:00:00 2001 From: Beto Dealmeida Date: Mon, 13 Feb 2023 11:20:38 -0800 Subject: [PATCH] chore: remove db_engines (#22444) --- superset/db_engine_specs/hive.py | 54 +++++++++++++++++++++++-- superset/db_engines/__init__.py | 16 -------- superset/db_engines/hive.py | 67 -------------------------------- 3 files changed, 51 insertions(+), 86 deletions(-) delete mode 100644 superset/db_engines/__init__.py delete mode 100644 superset/db_engines/hive.py diff --git a/superset/db_engine_specs/hive.py b/superset/db_engine_specs/hive.py index c36e9ccba2bc9..c049ee652eee4 100644 --- a/superset/db_engine_specs/hive.py +++ b/superset/db_engine_specs/hive.py @@ -48,6 +48,9 @@ if TYPE_CHECKING: # prevent circular imports + from pyhive.hive import Cursor + from TCLIService.ttypes import TFetchOrientation + from superset.models.core import Database logger = logging.getLogger(__name__) @@ -139,12 +142,10 @@ def patch(cls) -> None: ttypes as patched_ttypes, ) - from superset.db_engines import hive as patched_hive - hive.TCLIService = patched_TCLIService hive.constants = patched_constants hive.ttypes = patched_ttypes - hive.Cursor.fetch_logs = patched_hive.fetch_logs + hive.Cursor.fetch_logs = fetch_logs @classmethod def fetch_data( @@ -611,3 +612,50 @@ def get_view_names( cursor.execute(sql) results = cursor.fetchall() return {row[0] for row in results} + + +# TODO: contribute back to pyhive. +def fetch_logs( # pylint: disable=protected-access + self: "Cursor", + _max_rows: int = 1024, + orientation: Optional["TFetchOrientation"] = None, +) -> str: + """Mocked. Retrieve the logs produced by the execution of the query. + Can be called multiple times to fetch the logs produced after + the previous call. + :returns: list + :raises: ``ProgrammingError`` when no query has been started + .. note:: + This is not a part of DB-API. + """ + # pylint: disable=import-outside-toplevel + from pyhive import hive + from TCLIService import ttypes + from thrift import Thrift + + orientation = orientation or ttypes.TFetchOrientation.FETCH_NEXT + try: + req = ttypes.TGetLogReq(operationHandle=self._operationHandle) + logs = self._connection.client.GetLog(req).log + return logs + # raised if Hive is used + except (ttypes.TApplicationException, Thrift.TApplicationException) as ex: + if self._state == self._STATE_NONE: + raise hive.ProgrammingError("No query yet") from ex + logs = [] + while True: + req = ttypes.TFetchResultsReq( + operationHandle=self._operationHandle, + orientation=ttypes.TFetchOrientation.FETCH_NEXT, + maxRows=self.arraysize, + fetchType=1, # 0: results, 1: logs + ) + response = self._connection.client.FetchResults(req) + hive._check_status(response) + assert not response.results.rows, "expected data in columnar format" + assert len(response.results.columns) == 1, response.results.columns + new_logs = hive._unwrap_column(response.results.columns[0]) + logs += new_logs + if not new_logs: + break + return "\n".join(logs) diff --git a/superset/db_engines/__init__.py b/superset/db_engines/__init__.py deleted file mode 100644 index 13a83393a9124..0000000000000 --- a/superset/db_engines/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. diff --git a/superset/db_engines/hive.py b/superset/db_engines/hive.py deleted file mode 100644 index 0c4094fe3ef2c..0000000000000 --- a/superset/db_engines/hive.py +++ /dev/null @@ -1,67 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -from typing import Optional, TYPE_CHECKING - -if TYPE_CHECKING: - from pyhive.hive import Cursor - from TCLIService.ttypes import TFetchOrientation - -# TODO: contribute back to pyhive. -def fetch_logs( # pylint: disable=protected-access - self: "Cursor", - _max_rows: int = 1024, - orientation: Optional["TFetchOrientation"] = None, -) -> str: - """Mocked. Retrieve the logs produced by the execution of the query. - Can be called multiple times to fetch the logs produced after - the previous call. - :returns: list - :raises: ``ProgrammingError`` when no query has been started - .. note:: - This is not a part of DB-API. - """ - # pylint: disable=import-outside-toplevel - from pyhive import hive - from TCLIService import ttypes - from thrift import Thrift - - orientation = orientation or ttypes.TFetchOrientation.FETCH_NEXT - try: - req = ttypes.TGetLogReq(operationHandle=self._operationHandle) - logs = self._connection.client.GetLog(req).log - return logs - # raised if Hive is used - except (ttypes.TApplicationException, Thrift.TApplicationException) as ex: - if self._state == self._STATE_NONE: - raise hive.ProgrammingError("No query yet") from ex - logs = [] - while True: - req = ttypes.TFetchResultsReq( - operationHandle=self._operationHandle, - orientation=ttypes.TFetchOrientation.FETCH_NEXT, - maxRows=self.arraysize, - fetchType=1, # 0: results, 1: logs - ) - response = self._connection.client.FetchResults(req) - hive._check_status(response) - assert not response.results.rows, "expected data in columnar format" - assert len(response.results.columns) == 1, response.results.columns - new_logs = hive._unwrap_column(response.results.columns[0]) - logs += new_logs - if not new_logs: - break - return "\n".join(logs)