diff --git a/.kokoro/presubmit/snippets-3.12.cfg b/.kokoro/presubmit/snippets-3.12.cfg deleted file mode 100644 index 1381e83..0000000 --- a/.kokoro/presubmit/snippets-3.12.cfg +++ /dev/null @@ -1,7 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -# Only run this nox session. -env_vars: { - key: "NOX_SESSION" - value: "snippets-3.12" -} diff --git a/.kokoro/presubmit/snippets-3.8.cfg b/.kokoro/presubmit/system-3.11.cfg similarity index 80% rename from .kokoro/presubmit/snippets-3.8.cfg rename to .kokoro/presubmit/system-3.11.cfg index 840d9e7..90dc133 100644 --- a/.kokoro/presubmit/snippets-3.8.cfg +++ b/.kokoro/presubmit/system-3.11.cfg @@ -3,5 +3,5 @@ # Only run this nox session. env_vars: { key: "NOX_SESSION" - value: "snippets-3.8" -} + value: "system-3.11" +} \ No newline at end of file diff --git a/bigquery_magics/__init__.py b/bigquery_magics/__init__.py index d228a35..627ca98 100644 --- a/bigquery_magics/__init__.py +++ b/bigquery_magics/__init__.py @@ -12,9 +12,27 @@ # See the License for the specific language governing permissions and # limitations under the License. -from google.cloud.bigquery.magics.magics import context +import bigquery_magics.config +import bigquery_magics.version +context = bigquery_magics.config.context +__version__ = bigquery_magics.version.__version__ -# For backwards compatibility we need to make the context available in the path -# google.cloud.bigquery.magics.context -__all__ = ("context",) + +def load_ipython_extension(ipython): + """Called by IPython when this module is loaded as an IPython extension.""" + # Import here to avoid circular imports. + from bigquery_magics.bigquery import _cell_magic + + ipython.register_magic_function( + _cell_magic, magic_kind="cell", magic_name="bigquery" + ) + + +__all__ = ( + # For backwards compatibility we need to make the context available in + # the path google.cloud.bigquery.magics.context. + "context", + "__version__", + "load_ipython_extension", +) diff --git a/bigquery_magics/_versions_helpers.py b/bigquery_magics/_versions_helpers.py new file mode 100644 index 0000000..192011a --- /dev/null +++ b/bigquery_magics/_versions_helpers.py @@ -0,0 +1,91 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Shared helper functions for verifying versions of installed modules.""" + +from typing import Any + +from google.cloud.bigquery import exceptions +import packaging.version + +_MIN_BQ_STORAGE_VERSION = packaging.version.Version("2.0.0") + + +class BQStorageVersions: + """Version comparisons for google-cloud-bigqueyr-storage package.""" + + def __init__(self): + self._installed_version = None + + @property + def installed_version(self) -> packaging.version.Version: + """Return the parsed version of google-cloud-bigquery-storage.""" + if self._installed_version is None: + from google.cloud import bigquery_storage + + self._installed_version = packaging.version.parse( + # Use 0.0.0, since it is earlier than any released version. + # Legacy versions also have the same property, but + # creating a LegacyVersion has been deprecated. + # https://github.com/pypa/packaging/issues/321 + getattr(bigquery_storage, "__version__", "0.0.0") + ) + + return self._installed_version # type: ignore + + def try_import(self, raise_if_error: bool = False) -> Any: + """Tries to import the bigquery_storage module, and returns results + accordingly. It also verifies the module version is recent enough. + + If the import succeeds, returns the ``bigquery_storage`` module. + + If the import fails, + returns ``None`` when ``raise_if_error == False``, + raises Error when ``raise_if_error == True``. + + Returns: + The ``bigquery_storage`` module or ``None``. + + Raises: + exceptions.BigQueryStorageNotFoundError: + If google-cloud-bigquery-storage is not installed + exceptions.LegacyBigQueryStorageError: + If google-cloud-bigquery-storage package is outdated + """ + try: + from google.cloud import bigquery_storage # type: ignore + except ImportError: + if raise_if_error: + msg = ( + "Package google-cloud-bigquery-storage not found. " + "Install google-cloud-bigquery-storage version >= " + f"{_MIN_BQ_STORAGE_VERSION}." + ) + raise exceptions.BigQueryStorageNotFoundError(msg) + return None + + if self.installed_version < _MIN_BQ_STORAGE_VERSION: + if raise_if_error: + msg = ( + "Dependency google-cloud-bigquery-storage is outdated, " + f"please upgrade it to version >= {_MIN_BQ_STORAGE_VERSION} " + f"(version found: {self.installed_version})." + ) + raise exceptions.LegacyBigQueryStorageError(msg) + return None + + return bigquery_storage + + +BQ_STORAGE_VERSIONS = BQStorageVersions() diff --git a/bigquery_magics/magics.py b/bigquery_magics/bigquery.py similarity index 76% rename from bigquery_magics/magics.py rename to bigquery_magics/bigquery.py index 2a3583c..839e104 100644 --- a/bigquery_magics/magics.py +++ b/bigquery_magics/bigquery.py @@ -82,205 +82,35 @@ from __future__ import print_function -import re import ast +from concurrent import futures import copy import functools +import re import sys import time import warnings -from concurrent import futures - -try: - import IPython # type: ignore - from IPython import display # type: ignore - from IPython.core import magic_arguments # type: ignore -except ImportError: # pragma: NO COVER - raise ImportError("This module can only be loaded in IPython.") +import IPython # type: ignore +from IPython import display # type: ignore +from IPython.core import magic_arguments # type: ignore from google.api_core import client_info -from google.api_core import client_options from google.api_core.exceptions import NotFound -import google.auth # type: ignore from google.cloud import bigquery -import google.cloud.bigquery.dataset -from google.cloud.bigquery import _versions_helpers from google.cloud.bigquery import exceptions from google.cloud.bigquery.dbapi import _helpers -from google.cloud.bigquery.magics import line_arg_parser as lap - - -IPYTHON_USER_AGENT = "ipython-{}".format(IPython.__version__) - - -class Context(object): - """Storage for objects to be used throughout an IPython notebook session. - A Context object is initialized when the ``magics`` module is imported, - and can be found at ``google.cloud.bigquery.magics.context``. - """ +from bigquery_magics import line_arg_parser as lap +import bigquery_magics.config +import bigquery_magics.line_arg_parser.exceptions - def __init__(self): - self._credentials = None - self._project = None - self._connection = None - self._default_query_job_config = bigquery.QueryJobConfig() - self._bigquery_client_options = client_options.ClientOptions() - self._bqstorage_client_options = client_options.ClientOptions() - self._progress_bar_type = "tqdm_notebook" - - @property - def credentials(self): - """google.auth.credentials.Credentials: Credentials to use for queries - performed through IPython magics. - - Note: - These credentials do not need to be explicitly defined if you are - using Application Default Credentials. If you are not using - Application Default Credentials, manually construct a - :class:`google.auth.credentials.Credentials` object and set it as - the context credentials as demonstrated in the example below. See - `auth docs`_ for more information on obtaining credentials. - - Example: - Manually setting the context credentials: - - >>> from google.cloud.bigquery import magics - >>> from google.oauth2 import service_account - >>> credentials = (service_account - ... .Credentials.from_service_account_file( - ... '/path/to/key.json')) - >>> magics.context.credentials = credentials - - - .. _auth docs: http://google-auth.readthedocs.io - /en/latest/user-guide.html#obtaining-credentials - """ - if self._credentials is None: - self._credentials, _ = google.auth.default() - return self._credentials - - @credentials.setter - def credentials(self, value): - self._credentials = value - - @property - def project(self): - """str: Default project to use for queries performed through IPython - magics. - - Note: - The project does not need to be explicitly defined if you have an - environment default project set. If you do not have a default - project set in your environment, manually assign the project as - demonstrated in the example below. - - Example: - Manually setting the context project: - - >>> from google.cloud.bigquery import magics - >>> magics.context.project = 'my-project' - """ - if self._project is None: - _, self._project = google.auth.default() - return self._project - - @project.setter - def project(self, value): - self._project = value - - @property - def bigquery_client_options(self): - """google.api_core.client_options.ClientOptions: client options to be - used through IPython magics. - - Note:: - The client options do not need to be explicitly defined if no - special network connections are required. Normally you would be - using the https://bigquery.googleapis.com/ end point. - - Example: - Manually setting the endpoint: - - >>> from google.cloud.bigquery import magics - >>> client_options = {} - >>> client_options['api_endpoint'] = "https://some.special.url" - >>> magics.context.bigquery_client_options = client_options - """ - return self._bigquery_client_options - - @bigquery_client_options.setter - def bigquery_client_options(self, value): - self._bigquery_client_options = value - - @property - def bqstorage_client_options(self): - """google.api_core.client_options.ClientOptions: client options to be - used through IPython magics for the storage client. - - Note:: - The client options do not need to be explicitly defined if no - special network connections are required. Normally you would be - using the https://bigquerystorage.googleapis.com/ end point. - - Example: - Manually setting the endpoint: - - >>> from google.cloud.bigquery import magics - >>> client_options = {} - >>> client_options['api_endpoint'] = "https://some.special.url" - >>> magics.context.bqstorage_client_options = client_options - """ - return self._bqstorage_client_options - - @bqstorage_client_options.setter - def bqstorage_client_options(self, value): - self._bqstorage_client_options = value - - @property - def default_query_job_config(self): - """google.cloud.bigquery.job.QueryJobConfig: Default job - configuration for queries. - - The context's :class:`~google.cloud.bigquery.job.QueryJobConfig` is - used for queries. Some properties can be overridden with arguments to - the magics. - - Example: - Manually setting the default value for ``maximum_bytes_billed`` - to 100 MB: - - >>> from google.cloud.bigquery import magics - >>> magics.context.default_query_job_config.maximum_bytes_billed = 100000000 - """ - return self._default_query_job_config - - @default_query_job_config.setter - def default_query_job_config(self, value): - self._default_query_job_config = value - - @property - def progress_bar_type(self): - """str: Default progress bar type to use to display progress bar while - executing queries through IPython magics. - - Note:: - Install the ``tqdm`` package to use this feature. - - Example: - Manually setting the progress_bar_type: - - >>> from google.cloud.bigquery import magics - >>> magics.context.progress_bar_type = "tqdm_notebook" - """ - return self._progress_bar_type +try: + from google.cloud import bigquery_storage # type: ignore +except ImportError: + bigquery_storage = None - @progress_bar_type.setter - def progress_bar_type(self, value): - self._progress_bar_type = value - - -context = Context() +IPYTHON_USER_AGENT = "ipython-{}".format(IPython.__version__) +context = bigquery_magics.config.context def _handle_error(error, destination_var=None): @@ -288,7 +118,7 @@ def _handle_error(error, destination_var=None): Args: error (Exception): - An exception that ocurred during the query execution. + An exception that occurred during the query execution. destination_var (Optional[str]): The name of the IPython session variable to store the query job. """ @@ -508,6 +338,15 @@ def _create_dataset_if_necessary(client, dataset_id): "Defaults to use tqdm_notebook. Install the ``tqdm`` package to use this feature." ), ) +@magic_arguments.argument( + "--location", + type=str, + default=None, + help=( + "Set the location to execute query." + "Defaults to location set in query setting in console." + ), +) def _cell_magic(line, query): """Underlying function for bigquery cell magic @@ -550,7 +389,8 @@ def _cell_magic(line, query): "Storage API is already used by default.", category=DeprecationWarning, ) - use_bqstorage_api = not args.use_rest_api + use_bqstorage_api = not args.use_rest_api and (bigquery_storage is not None) + location = args.location params = [] if params_option_value: @@ -579,6 +419,7 @@ def _cell_magic(line, query): default_query_job_config=context.default_query_job_config, client_info=client_info.ClientInfo(user_agent=IPYTHON_USER_AGENT), client_options=bigquery_client_options, + location=location, ) if context._connection: client._connection = context._connection @@ -769,7 +610,9 @@ def _make_bqstorage_client(client, use_bqstorage_api, client_options): return None try: - _versions_helpers.BQ_STORAGE_VERSIONS.try_import(raise_if_error=True) + bigquery_magics._versions_helpers.BQ_STORAGE_VERSIONS.try_import( + raise_if_error=True + ) except exceptions.BigQueryStorageNotFoundError as err: customized_error = ImportError( "The default BigQuery Storage API client cannot be used, install " @@ -778,8 +621,6 @@ def _make_bqstorage_client(client, use_bqstorage_api, client_options): "the --use_rest_api magic option." ) raise customized_error from err - except exceptions.LegacyBigQueryStorageError: - pass try: from google.api_core.gapic_v1 import client_info as gapic_client_info diff --git a/bigquery_magics/config.py b/bigquery_magics/config.py new file mode 100644 index 0000000..7d354f0 --- /dev/null +++ b/bigquery_magics/config.py @@ -0,0 +1,187 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import google.api_core.client_options as client_options +import google.auth # type: ignore +import google.cloud.bigquery as bigquery + + +class Context(object): + """Storage for objects to be used throughout an IPython notebook session. + + A Context object is initialized when the ``bigquery_magics`` module is imported, + and can be found at ``bigquery_magics.context``. + """ + + def __init__(self): + self._credentials = None + self._project = None + self._connection = None + self._default_query_job_config = bigquery.QueryJobConfig() + self._bigquery_client_options = client_options.ClientOptions() + self._bqstorage_client_options = client_options.ClientOptions() + self._progress_bar_type = "tqdm_notebook" + + @property + def credentials(self): + """google.auth.credentials.Credentials: Credentials to use for queries + performed through IPython magics. + + Note: + These credentials do not need to be explicitly defined if you are + using Application Default Credentials. If you are not using + Application Default Credentials, manually construct a + :class:`google.auth.credentials.Credentials` object and set it as + the context credentials as demonstrated in the example below. See + `auth docs`_ for more information on obtaining credentials. + + Example: + Manually setting the context credentials: + + >>> from google.cloud.bigquery import magics + >>> from google.oauth2 import service_account + >>> credentials = (service_account + ... .Credentials.from_service_account_file( + ... '/path/to/key.json')) + >>> bigquery_magics.context.credentials = credentials + + + .. _auth docs: http://google-auth.readthedocs.io + /en/latest/user-guide.html#obtaining-credentials + """ + if self._credentials is None: + self._credentials, _ = google.auth.default() + return self._credentials + + @credentials.setter + def credentials(self, value): + self._credentials = value + + @property + def project(self): + """str: Default project to use for queries performed through IPython + magics. + + Note: + The project does not need to be explicitly defined if you have an + environment default project set. If you do not have a default + project set in your environment, manually assign the project as + demonstrated in the example below. + + Example: + Manually setting the context project: + + >>> from google.cloud.bigquery import magics + >>> bigquery_magics.context.project = 'my-project' + """ + if self._project is None: + _, self._project = google.auth.default() + return self._project + + @project.setter + def project(self, value): + self._project = value + + @property + def bigquery_client_options(self): + """google.api_core.client_options.ClientOptions: client options to be + used through IPython magics. + + Note:: + The client options do not need to be explicitly defined if no + special network connections are required. Normally you would be + using the https://bigquery.googleapis.com/ end point. + + Example: + Manually setting the endpoint: + + >>> from google.cloud.bigquery import magics + >>> client_options = {} + >>> client_options['api_endpoint'] = "https://some.special.url" + >>> bigquery_magics.context.bigquery_client_options = client_options + """ + return self._bigquery_client_options + + @bigquery_client_options.setter + def bigquery_client_options(self, value): + self._bigquery_client_options = value + + @property + def bqstorage_client_options(self): + """google.api_core.client_options.ClientOptions: client options to be + used through IPython magics for the storage client. + + Note:: + The client options do not need to be explicitly defined if no + special network connections are required. Normally you would be + using the https://bigquerystorage.googleapis.com/ end point. + + Example: + Manually setting the endpoint: + + >>> from google.cloud.bigquery import magics + >>> client_options = {} + >>> client_options['api_endpoint'] = "https://some.special.url" + >>> bigquery_magics.context.bqstorage_client_options = client_options + """ + return self._bqstorage_client_options + + @bqstorage_client_options.setter + def bqstorage_client_options(self, value): + self._bqstorage_client_options = value + + @property + def default_query_job_config(self): + """google.cloud.bigquery.job.QueryJobConfig: Default job + configuration for queries. + + The context's :class:`~google.cloud.bigquery.job.QueryJobConfig` is + used for queries. Some properties can be overridden with arguments to + the magics. + + Example: + Manually setting the default value for ``maximum_bytes_billed`` + to 100 MB: + + >>> from google.cloud.bigquery import magics + >>> bigquery_magics.context.default_query_job_config.maximum_bytes_billed = 100000000 + """ + return self._default_query_job_config + + @default_query_job_config.setter + def default_query_job_config(self, value): + self._default_query_job_config = value + + @property + def progress_bar_type(self): + """str: Default progress bar type to use to display progress bar while + executing queries through IPython magics. + + Note:: + Install the ``tqdm`` package to use this feature. + + Example: + Manually setting the progress_bar_type: + + >>> from google.cloud.bigquery import magics + >>> bigquery_magics.context.progress_bar_type = "tqdm_notebook" + """ + return self._progress_bar_type + + @progress_bar_type.setter + def progress_bar_type(self, value): + self._progress_bar_type = value + + +context = Context() diff --git a/bigquery_magics/line_arg_parser/__init__.py b/bigquery_magics/line_arg_parser/__init__.py index 9471446..dfa9ec6 100644 --- a/bigquery_magics/line_arg_parser/__init__.py +++ b/bigquery_magics/line_arg_parser/__init__.py @@ -12,16 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -from google.cloud.bigquery.magics.line_arg_parser.exceptions import ParseError -from google.cloud.bigquery.magics.line_arg_parser.exceptions import ( +from bigquery_magics.line_arg_parser.exceptions import ( DuplicateQueryParamsError, + ParseError, QueryParamsParseError, ) -from google.cloud.bigquery.magics.line_arg_parser.lexer import Lexer -from google.cloud.bigquery.magics.line_arg_parser.lexer import TokenType -from google.cloud.bigquery.magics.line_arg_parser.parser import Parser -from google.cloud.bigquery.magics.line_arg_parser.visitors import QueryParamsExtractor - +from bigquery_magics.line_arg_parser.lexer import Lexer, TokenType +from bigquery_magics.line_arg_parser.parser import Parser +from bigquery_magics.line_arg_parser.visitors import QueryParamsExtractor __all__ = ( "DuplicateQueryParamsError", diff --git a/bigquery_magics/line_arg_parser/lexer.py b/bigquery_magics/line_arg_parser/lexer.py index 71b287d..6e8b4cc 100644 --- a/bigquery_magics/line_arg_parser/lexer.py +++ b/bigquery_magics/line_arg_parser/lexer.py @@ -12,14 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -from collections import namedtuple -from collections import OrderedDict +from collections import OrderedDict, namedtuple +import enum import itertools import re -import enum - - Token = namedtuple("Token", ("type_", "lexeme", "pos")) StateTransition = namedtuple("StateTransition", ("new_state", "total_offset")) diff --git a/bigquery_magics/line_arg_parser/parser.py b/bigquery_magics/line_arg_parser/parser.py index b9da20c..390b7bb 100644 --- a/bigquery_magics/line_arg_parser/parser.py +++ b/bigquery_magics/line_arg_parser/parser.py @@ -12,10 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from google.cloud.bigquery.magics.line_arg_parser import DuplicateQueryParamsError -from google.cloud.bigquery.magics.line_arg_parser import ParseError -from google.cloud.bigquery.magics.line_arg_parser import QueryParamsParseError -from google.cloud.bigquery.magics.line_arg_parser import TokenType +import bigquery_magics.line_arg_parser.exceptions as lap_exceptions +import bigquery_magics.line_arg_parser.lexer as lap_lexer class ParseNode(object): @@ -154,11 +152,11 @@ def get_next_token(self): token = next(self._tokens_iter) self._current_token = token - def consume(self, expected_type, exc_type=ParseError): + def consume(self, expected_type, exc_type=lap_exceptions.ParseError): """Move to the next token in token stream if it matches the expected type. Args: - expected_type (lexer.TokenType): The expected token type to be consumed. + expected_type (bigquery_magics.line_arg_parser.lexer.TokenType): The expected token type to be consumed. exc_type (Optional[ParseError]): The type of the exception to raise. Should be the ``ParseError`` class or one of its subclasses. Defaults to ``ParseError``. @@ -167,10 +165,10 @@ def consume(self, expected_type, exc_type=ParseError): ParseError: If the current token does not match the expected type. """ if self._current_token.type_ == expected_type: - if expected_type != TokenType.EOL: + if expected_type != lap_lexer.TokenType.EOL: self.get_next_token() else: - if self._current_token.type_ == TokenType.EOL: + if self._current_token.type_ == lap_lexer.TokenType.EOL: msg = "Unexpected end of input, expected {}.".format(expected_type) else: msg = "Expected token type {}, but found {} at position {}.".format( @@ -178,11 +176,11 @@ def consume(self, expected_type, exc_type=ParseError): ) self.error(message=msg, exc_type=exc_type) - def error(self, message="Syntax error.", exc_type=ParseError): + def error(self, message="Syntax error.", exc_type=lap_exceptions.ParseError): """Raise an error with the given message. Args: - expected_type (lexer.TokenType): The expected token type to be consumed. + expected_type (bigquery_magics.line_arg_parser.lexer.TokenType): The expected token type to be consumed. exc_type (Optional[ParseError]): The type of the exception to raise. Should be the ``ParseError`` class or one of its subclasses. Defaults to ``ParseError``. @@ -204,7 +202,7 @@ def input_line(self): token = self._current_token - if token.type_ != TokenType.EOL: + if token.type_ != lap_lexer.TokenType.EOL: msg = "Unexpected input at position {}: {}".format(token.pos, token.lexeme) self.error(msg) @@ -219,10 +217,10 @@ def destination_var(self): """ token = self._current_token - if token.type_ == TokenType.DEST_VAR: - self.consume(TokenType.DEST_VAR) + if token.type_ == lap_lexer.TokenType.DEST_VAR: + self.consume(lap_lexer.TokenType.DEST_VAR) result = DestinationVar(token) - elif token.type_ == TokenType.UNKNOWN: + elif token.type_ == lap_lexer.TokenType.UNKNOWN: msg = "Unknown input at position {}: {}".format(token.pos, token.lexeme) self.error(msg) else: @@ -242,15 +240,15 @@ def option_list(self): all_options = [] def parse_nonparams_options(): - while self._current_token.type_ == TokenType.OPTION_SPEC: + while self._current_token.type_ == lap_lexer.TokenType.OPTION_SPEC: token = self._current_token - self.consume(TokenType.OPTION_SPEC) + self.consume(lap_lexer.TokenType.OPTION_SPEC) opt_name = token.lexeme[2:] # cut off the "--" prefix # skip the optional "=" character - if self._current_token.type_ == TokenType.OPTION_EQ: - self.consume(TokenType.OPTION_EQ) + if self._current_token.type_ == lap_lexer.TokenType.OPTION_EQ: + self.consume(lap_lexer.TokenType.OPTION_EQ) opt_value = self.option_value() option = CmdOption(opt_name, opt_value) @@ -260,15 +258,16 @@ def parse_nonparams_options(): token = self._current_token - if token.type_ == TokenType.PARAMS_OPT_SPEC: + if token.type_ == lap_lexer.TokenType.PARAMS_OPT_SPEC: option = self.params_option() all_options.append(option) parse_nonparams_options() - if self._current_token.type_ == TokenType.PARAMS_OPT_SPEC: + if self._current_token.type_ == lap_lexer.TokenType.PARAMS_OPT_SPEC: self.error( - message="Duplicate --params option", exc_type=DuplicateQueryParamsError + message="Duplicate --params option", + exc_type=lap_exceptions.DuplicateQueryParamsError, ) return CmdOptionList(all_options) @@ -282,10 +281,10 @@ def option_value(self): """ token = self._current_token - if token.type_ == TokenType.OPT_VAL: - self.consume(TokenType.OPT_VAL) + if token.type_ == lap_lexer.TokenType.OPT_VAL: + self.consume(lap_lexer.TokenType.OPT_VAL) result = CmdOptionValue(token) - elif token.type_ == TokenType.UNKNOWN: + elif token.type_ == lap_lexer.TokenType.UNKNOWN: msg = "Unknown input at position {}: {}".format(token.pos, token.lexeme) self.error(msg) else: @@ -301,19 +300,22 @@ def params_option(self): params_option : PARAMS_OPT_SPEC [PARAMS_OPT_EQ] \ (DOLLAR_PY_ID | PY_STRING | py_dict) """ - self.consume(TokenType.PARAMS_OPT_SPEC) + self.consume(lap_lexer.TokenType.PARAMS_OPT_SPEC) # skip the optional "=" character - if self._current_token.type_ == TokenType.PARAMS_OPT_EQ: - self.consume(TokenType.PARAMS_OPT_EQ) + if self._current_token.type_ == lap_lexer.TokenType.PARAMS_OPT_EQ: + self.consume(lap_lexer.TokenType.PARAMS_OPT_EQ) - if self._current_token.type_ == TokenType.DOLLAR_PY_ID: + if self._current_token.type_ == lap_lexer.TokenType.DOLLAR_PY_ID: token = self._current_token - self.consume(TokenType.DOLLAR_PY_ID) + self.consume(lap_lexer.TokenType.DOLLAR_PY_ID) opt_value = PyVarExpansion(token) - elif self._current_token.type_ == TokenType.PY_STRING: + elif self._current_token.type_ == lap_lexer.TokenType.PY_STRING: token = self._current_token - self.consume(TokenType.PY_STRING, exc_type=QueryParamsParseError) + self.consume( + lap_lexer.TokenType.PY_STRING, + exc_type=lap_exceptions.QueryParamsParseError, + ) opt_value = PyScalarValue(token, token.lexeme) else: opt_value = self.py_dict() @@ -329,9 +331,13 @@ def py_dict(self): py_dict : LCURL dict_items RCURL """ - self.consume(TokenType.LCURL, exc_type=QueryParamsParseError) + self.consume( + lap_lexer.TokenType.LCURL, exc_type=lap_exceptions.QueryParamsParseError + ) dict_items = self.dict_items() - self.consume(TokenType.RCURL, exc_type=QueryParamsParseError) + self.consume( + lap_lexer.TokenType.RCURL, exc_type=lap_exceptions.QueryParamsParseError + ) return PyDict(dict_items) @@ -348,8 +354,10 @@ def dict_items(self): if item is not None: result.append(item) - while self._current_token.type_ == TokenType.COMMA: - self.consume(TokenType.COMMA, exc_type=QueryParamsParseError) + while self._current_token.type_ == lap_lexer.TokenType.COMMA: + self.consume( + lap_lexer.TokenType.COMMA, exc_type=lap_exceptions.QueryParamsParseError + ) item = self.dict_item() if item is not None: result.append(item) @@ -365,14 +373,16 @@ def dict_item(self): """ token = self._current_token - if token.type_ == TokenType.PY_STRING: + if token.type_ == lap_lexer.TokenType.PY_STRING: key = self.dict_key() - self.consume(TokenType.COLON, exc_type=QueryParamsParseError) + self.consume( + lap_lexer.TokenType.COLON, exc_type=lap_exceptions.QueryParamsParseError + ) value = self.py_value() result = PyDictItem(key, value) - elif token.type_ == TokenType.UNKNOWN: + elif token.type_ == lap_lexer.TokenType.UNKNOWN: msg = "Unknown input at position {}: {}".format(token.pos, token.lexeme) - self.error(msg, exc_type=QueryParamsParseError) + self.error(msg, exc_type=lap_exceptions.QueryParamsParseError) else: result = None @@ -386,7 +396,9 @@ def dict_key(self): dict_key : PY_STRING """ token = self._current_token - self.consume(TokenType.PY_STRING, exc_type=QueryParamsParseError) + self.consume( + lap_lexer.TokenType.PY_STRING, exc_type=lap_exceptions.QueryParamsParseError + ) return PyDictKey(token) def py_value(self): @@ -398,29 +410,38 @@ def py_value(self): """ token = self._current_token - if token.type_ == TokenType.PY_BOOL: - self.consume(TokenType.PY_BOOL, exc_type=QueryParamsParseError) + if token.type_ == lap_lexer.TokenType.PY_BOOL: + self.consume( + lap_lexer.TokenType.PY_BOOL, + exc_type=lap_exceptions.QueryParamsParseError, + ) return PyScalarValue(token, token.lexeme) - elif token.type_ == TokenType.PY_NUMBER: - self.consume(TokenType.PY_NUMBER, exc_type=QueryParamsParseError) + elif token.type_ == lap_lexer.TokenType.PY_NUMBER: + self.consume( + lap_lexer.TokenType.PY_NUMBER, + exc_type=lap_exceptions.QueryParamsParseError, + ) return PyScalarValue(token, token.lexeme) - elif token.type_ == TokenType.PY_STRING: - self.consume(TokenType.PY_STRING, exc_type=QueryParamsParseError) + elif token.type_ == lap_lexer.TokenType.PY_STRING: + self.consume( + lap_lexer.TokenType.PY_STRING, + exc_type=lap_exceptions.QueryParamsParseError, + ) return PyScalarValue(token, token.lexeme) - elif token.type_ == TokenType.LPAREN: + elif token.type_ == lap_lexer.TokenType.LPAREN: tuple_node = self.py_tuple() return tuple_node - elif token.type_ == TokenType.LSQUARE: + elif token.type_ == lap_lexer.TokenType.LSQUARE: list_node = self.py_list() return list_node - elif token.type_ == TokenType.LCURL: + elif token.type_ == lap_lexer.TokenType.LCURL: dict_node = self.py_dict() return dict_node else: msg = "Unexpected token type {} at position {}.".format( token.type_, token.pos ) - self.error(msg, exc_type=QueryParamsParseError) + self.error(msg, exc_type=lap_exceptions.QueryParamsParseError) def py_tuple(self): """Implementation of the ``py_tuple`` grammar production rule. @@ -429,9 +450,13 @@ def py_tuple(self): py_tuple : LPAREN collection_items RPAREN """ - self.consume(TokenType.LPAREN, exc_type=QueryParamsParseError) + self.consume( + lap_lexer.TokenType.LPAREN, exc_type=lap_exceptions.QueryParamsParseError + ) items = self.collection_items() - self.consume(TokenType.RPAREN, exc_type=QueryParamsParseError) + self.consume( + lap_lexer.TokenType.RPAREN, exc_type=lap_exceptions.QueryParamsParseError + ) return PyTuple(items) @@ -442,9 +467,13 @@ def py_list(self): py_list : LSQUARE collection_items RSQUARE """ - self.consume(TokenType.LSQUARE, exc_type=QueryParamsParseError) + self.consume( + lap_lexer.TokenType.LSQUARE, exc_type=lap_exceptions.QueryParamsParseError + ) items = self.collection_items() - self.consume(TokenType.RSQUARE, exc_type=QueryParamsParseError) + self.consume( + lap_lexer.TokenType.RSQUARE, exc_type=lap_exceptions.QueryParamsParseError + ) return PyList(items) @@ -461,8 +490,10 @@ def collection_items(self): if item is not None: result.append(item) - while self._current_token.type_ == TokenType.COMMA: - self.consume(TokenType.COMMA, exc_type=QueryParamsParseError) + while self._current_token.type_ == lap_lexer.TokenType.COMMA: + self.consume( + lap_lexer.TokenType.COMMA, exc_type=lap_exceptions.QueryParamsParseError + ) item = self.collection_item() if item is not None: result.append(item) @@ -476,7 +507,10 @@ def collection_item(self): collection_item : py_value | EMPTY """ - if self._current_token.type_ not in {TokenType.RPAREN, TokenType.RSQUARE}: + if self._current_token.type_ not in { + lap_lexer.TokenType.RPAREN, + lap_lexer.TokenType.RSQUARE, + }: result = self.py_value() else: result = None # end of list/tuple items diff --git a/bigquery_magics/version.py b/bigquery_magics/version.py index 72b0b02..4027bb8 100644 --- a/bigquery_magics/version.py +++ b/bigquery_magics/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "0.0.1" \ No newline at end of file +__version__ = "0.0.1" diff --git a/docs/conf.py b/docs/conf.py index d0468e2..073edc7 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -24,9 +24,9 @@ # All configuration values have a default; values that are commented out # serve to show the default. -import sys import os import shlex +import sys # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the diff --git a/noxfile.py b/noxfile.py index 08ee214..f5dd689 100644 --- a/noxfile.py +++ b/noxfile.py @@ -42,17 +42,32 @@ "pytest-cov", "pytest-asyncio", ] -UNIT_TEST_EXTERNAL_DEPENDENCIES: List[str] = [] +UNIT_TEST_EXTERNAL_DEPENDENCIES: List[str] = [ + "google-cloud-testutils", +] UNIT_TEST_LOCAL_DEPENDENCIES: List[str] = [] UNIT_TEST_DEPENDENCIES: List[str] = [] -UNIT_TEST_EXTRAS: List[str] = [ - "tqdm", -] +UNIT_TEST_EXTRAS: List[str] = [] UNIT_TEST_EXTRAS_BY_PYTHON: Dict[str, List[str]] = { - "3.8": [], + "3.7": [ + "bqstorage", + ], + "3.8": [ + "bqstorage", + ], + "3.9": [ + "bqstorage", + ], + "3.10": [ + "bqstorage", + ], + "3.11": [], + "3.12": [ + "bqstorage", + ], } -SYSTEM_TEST_PYTHON_VERSIONS: List[str] = ["3.8", "3.12"] +SYSTEM_TEST_PYTHON_VERSIONS: List[str] = ["3.8", "3.11", "3.12"] SYSTEM_TEST_STANDARD_DEPENDENCIES: List[str] = [ "mock", "pytest", @@ -61,10 +76,25 @@ SYSTEM_TEST_EXTERNAL_DEPENDENCIES: List[str] = [] SYSTEM_TEST_LOCAL_DEPENDENCIES: List[str] = [] SYSTEM_TEST_DEPENDENCIES: List[str] = [] -SYSTEM_TEST_EXTRAS: List[str] = [ - "tqdm", -] -SYSTEM_TEST_EXTRAS_BY_PYTHON: Dict[str, List[str]] = {} +SYSTEM_TEST_EXTRAS: List[str] = [] +SYSTEM_TEST_EXTRAS_BY_PYTHON: Dict[str, List[str]] = { + "3.7": [ + "bqstorage", + ], + "3.8": [ + "bqstorage", + ], + "3.9": [ + "bqstorage", + ], + "3.10": [ + "bqstorage", + ], + "3.11": [], + "3.12": [ + "bqstorage", + ], +} CURRENT_DIRECTORY = pathlib.Path(__file__).parent.absolute() diff --git a/owlbot.py b/owlbot.py index c2de310..ca0d20e 100644 --- a/owlbot.py +++ b/owlbot.py @@ -1,4 +1,4 @@ -# Copyright 2018 Google LLC +# Copyright 2021 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,116 +13,82 @@ # limitations under the License. """This script is used to synthesize generated parts of this library.""" -from pathlib import Path -import textwrap + +import pathlib import synthtool as s from synthtool import gcp from synthtool.languages import python -REPO_ROOT = Path(__file__).parent.absolute() - -default_version = "v2" - -for library in s.get_staging_dirs(default_version): - # Avoid breaking change due to change in field renames. - # https://github.com/googleapis/python-bigquery/issues/319 - s.replace( - library / f"google/cloud/bigquery_{library.name}/types/standard_sql.py", - r"type_ ", - "type ", - ) - # Patch docs issue - s.replace( - library / f"google/cloud/bigquery_{library.name}/types/model.py", - r"""\"predicted_\"""", - """`predicted_`""", - ) - s.move(library / f"google/cloud/bigquery_{library.name}/types") -s.remove_staging_dirs() +REPO_ROOT = pathlib.Path(__file__).parent.absolute() common = gcp.CommonTemplates() # ---------------------------------------------------------------------------- # Add templated files # ---------------------------------------------------------------------------- + +extras = ["bqstorage"] +extras_by_python = { + "3.7": extras, + "3.8": extras, + "3.9": extras, + "3.10": extras, + # Use a middle version of Python to test when no extras are installed. + "3.11": [], + "3.12": extras, +} templated_files = common.py_library( + unit_test_python_versions=["3.7", "3.8", "3.11", "3.12"], + system_test_python_versions=["3.8", "3.11", "3.12"], cov_level=100, - samples=True, - microgenerator=True, - split_system_tests=True, + unit_test_extras_by_python=extras_by_python, + unit_test_external_dependencies=["google-cloud-testutils"], + system_test_extras_by_python=extras_by_python, intersphinx_dependencies={ - "dateutil": "https://dateutil.readthedocs.io/en/latest/", - "geopandas": "https://geopandas.org/", "pandas": "https://pandas.pydata.org/pandas-docs/stable/", + "pydata-google-auth": "https://pydata-google-auth.readthedocs.io/en/latest/", }, ) - -# BigQuery has a custom multiprocessing note s.move( templated_files, excludes=[ - "noxfile.py", + # Multi-processing note isn't relevant, as bigquery-magics is responsible for + # creating clients, not the end user. "docs/multiprocessing.rst", - "docs/index.rst", - ".coveragerc", - ".github/CODEOWNERS", - # Include custom SNIPPETS_TESTS job for performance. - # https://github.com/googleapis/python-bigquery/issues/191 - ".kokoro/presubmit/presubmit.cfg", - ".github/workflows", # exclude gh actions as credentials are needed for tests - "README.rst", + "README.rst", ], ) -python.configure_previous_major_version_branches() # ---------------------------------------------------------------------------- -# Samples templates +# Fixup files # ---------------------------------------------------------------------------- -python.py_samples() +s.replace( + ["noxfile.py"], r"[\"']google[\"']", '"bigquery_magics"', +) + s.replace( - "docs/conf.py", - r'\{"members": True\}', - '{"members": True, "inherited-members": True}', + ["noxfile.py"], "--cov=google", "--cov=bigquery_magics", ) + + +# Workaround for https://github.com/googleapis/synthtool/issues/1317 s.replace( - "docs/conf.py", - r"exclude_patterns = \[", - '\\g<0>\n "google/cloud/bigquery_v2/**", # Legacy proto-based types.', + ["noxfile.py"], r'extras = "\[\]"', 'extras = ""', ) # ---------------------------------------------------------------------------- -# pytype-related changes +# Samples templates # ---------------------------------------------------------------------------- -# Add .pytype to .gitignore -s.replace(".gitignore", r"\.pytest_cache", "\\g<0>\n.pytype") +python.py_samples(skip_readmes=True) -# Add pytype config to setup.cfg -s.replace( - "setup.cfg", - r"universal = 1", - textwrap.dedent( - """ \\g<0> - - [pytype] - python_version = 3.8 - inputs = - google/cloud/ - exclude = - tests/ - google/cloud/bigquery_v2/ # Legacy proto-based types. - output = .pytype/ - disable = - # There's some issue with finding some pyi files, thus disabling. - # The issue https://github.com/google/pytype/issues/150 is closed, but the - # error still occurs for some reason. - pyi-error""" - ), -) +# ---------------------------------------------------------------------------- +# Final cleanup +# ---------------------------------------------------------------------------- -s.shell.run(["nox", "-s", "blacken"], hide_output=False) +s.shell.run(["nox", "-s", "format"], hide_output=False) for noxfile in REPO_ROOT.glob("samples/**/noxfile.py"): - s.shell.run(["nox", "-s", "blacken"], cwd=noxfile.parent, hide_output=False) + s.shell.run(["nox", "-s", "blacken"], cwd=noxfile.parent, hide_output=False) \ No newline at end of file diff --git a/setup.py b/setup.py index 207976d..d96d519 100644 --- a/setup.py +++ b/setup.py @@ -17,7 +17,6 @@ import setuptools - # Package metadata. name = "bigquery-magics" @@ -30,14 +29,31 @@ release_status = "Development Status :: 4 - Beta" dependencies = [ "db-dtypes>=0.3.0,<2.0.0dev", - "google-cloud-bigquery >= 3.0.0, <4.0.0dev", + "google-cloud-bigquery >= 3.13.0, <4.0.0dev", "ipywidgets>=7.7.1", "ipython>=7.23.1", "ipykernel>=6.0.0", + "packaging >= 20.0.0", "pandas>=1.1.0", "pyarrow >= 3.0.0", + "tqdm >= 4.7.4, <5.0.0dev", ] -extras = {} +extras = { + # bqstorage had a period where it was a required dependency, and has been + # moved back to optional due to bloat. See + # https://github.com/googleapis/python-bigquery/issues/1196 for more background. + "bqstorage": [ + "google-cloud-bigquery-storage >= 2.6.0, <3.0.0dev", + # Due to an issue in pip's dependency resolver, the `grpc` extra is not + # installed, even though `google-cloud-bigquery-storage` specifies it + # as `google-api-core[grpc]`. We thus need to explicitly specify it here. + # See: https://github.com/googleapis/python-bigquery/issues/83 The + # grpc.Channel.close() method isn't added until 1.32.0. + # https://github.com/grpc/grpc/pull/15254 + "grpcio >= 1.47.0, < 2.0dev", + "grpcio >= 1.49.1, < 2.0dev; python_version>='3.11'", + ], +} all_extras = [] diff --git a/testing/constraints-3.7.txt b/testing/constraints-3.7.txt index bd915d1..289ccec 100644 --- a/testing/constraints-3.7.txt +++ b/testing/constraints-3.7.txt @@ -6,9 +6,11 @@ # e.g., if setup.py has "foo >= 1.14.0, < 2.0.0dev", # Then this file should have foo==1.14.0 db-dtypes==0.3.0 -google-cloud-bigquery==3.0.0 +google-cloud-bigquery==3.13.0 +google-cloud-bigquery-storage==2.6.0 ipywidgets==7.7.1 ipython==7.23.1 ipykernel==6.0.0 pandas==1.1.0 pyarrow==3.0.0 +tqdm==4.7.4 \ No newline at end of file diff --git a/tests/system/test_bigquery.py b/tests/system/test_bigquery.py index 3d761cd..d7147eb 100644 --- a/tests/system/test_bigquery.py +++ b/tests/system/test_bigquery.py @@ -16,9 +16,8 @@ import re -import pytest import psutil - +import pytest IPython = pytest.importorskip("IPython") io = pytest.importorskip("IPython.utils.io") @@ -50,7 +49,7 @@ def test_bigquery_magic(ipython_interactive): current_process = psutil.Process() conn_count_start = len(current_process.connections()) - ip.extension_manager.load_extension("google.cloud.bigquery") + ip.extension_manager.load_extension("bigquery_magics") sql = """ SELECT CONCAT( diff --git a/tests/unit/line_arg_parser/test_parser.py b/tests/unit/line_arg_parser/test_parser.py index b170d53..2d6c62f 100644 --- a/tests/unit/line_arg_parser/test_parser.py +++ b/tests/unit/line_arg_parser/test_parser.py @@ -36,8 +36,7 @@ def test_consume_expected_eol(parser_class): def test_consume_unexpected_eol(parser_class): - from google.cloud.bigquery.magics.line_arg_parser import ParseError - from google.cloud.bigquery.magics.line_arg_parser import TokenType + from google.cloud.bigquery.magics.line_arg_parser import ParseError, TokenType from google.cloud.bigquery.magics.line_arg_parser.lexer import Token # A simple iterable of Tokens is sufficient. @@ -49,8 +48,7 @@ def test_consume_unexpected_eol(parser_class): def test_input_line_unexpected_input(parser_class): - from google.cloud.bigquery.magics.line_arg_parser import ParseError - from google.cloud.bigquery.magics.line_arg_parser import TokenType + from google.cloud.bigquery.magics.line_arg_parser import ParseError, TokenType from google.cloud.bigquery.magics.line_arg_parser.lexer import Token # A simple iterable of Tokens is sufficient. @@ -66,8 +64,7 @@ def test_input_line_unexpected_input(parser_class): def test_destination_var_unexpected_input(parser_class): - from google.cloud.bigquery.magics.line_arg_parser import ParseError - from google.cloud.bigquery.magics.line_arg_parser import TokenType + from google.cloud.bigquery.magics.line_arg_parser import ParseError, TokenType from google.cloud.bigquery.magics.line_arg_parser.lexer import Token # A simple iterable of Tokens is sufficient. @@ -82,8 +79,7 @@ def test_destination_var_unexpected_input(parser_class): def test_option_value_unexpected_input(parser_class): - from google.cloud.bigquery.magics.line_arg_parser import ParseError - from google.cloud.bigquery.magics.line_arg_parser import TokenType + from google.cloud.bigquery.magics.line_arg_parser import ParseError, TokenType from google.cloud.bigquery.magics.line_arg_parser.lexer import Token # A simple iterable of Tokens is sufficient. @@ -133,8 +129,7 @@ def test_dict_items_trailing_comma(parser_class): def test_dict_item_unknown_input(parser_class): - from google.cloud.bigquery.magics.line_arg_parser import ParseError - from google.cloud.bigquery.magics.line_arg_parser import TokenType + from google.cloud.bigquery.magics.line_arg_parser import ParseError, TokenType from google.cloud.bigquery.magics.line_arg_parser.lexer import Token # A simple iterable of Tokens is sufficient. @@ -148,8 +143,7 @@ def test_dict_item_unknown_input(parser_class): def test_pyvalue_list_containing_dict(parser_class): from google.cloud.bigquery.magics.line_arg_parser import TokenType from google.cloud.bigquery.magics.line_arg_parser.lexer import Token - from google.cloud.bigquery.magics.line_arg_parser.parser import PyDict - from google.cloud.bigquery.magics.line_arg_parser.parser import PyList + from google.cloud.bigquery.magics.line_arg_parser.parser import PyDict, PyList # A simple iterable of Tokens is sufficient. fake_lexer = [ @@ -180,8 +174,7 @@ def test_pyvalue_list_containing_dict(parser_class): def test_pyvalue_invalid_token(parser_class): - from google.cloud.bigquery.magics.line_arg_parser import ParseError - from google.cloud.bigquery.magics.line_arg_parser import TokenType + from google.cloud.bigquery.magics.line_arg_parser import ParseError, TokenType from google.cloud.bigquery.magics.line_arg_parser.lexer import Token # A simple iterable of Tokens is sufficient. diff --git a/tests/unit/test__versions_helpers.py b/tests/unit/test__versions_helpers.py new file mode 100644 index 0000000..80a690e --- /dev/null +++ b/tests/unit/test__versions_helpers.py @@ -0,0 +1,108 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from unittest import mock + +import pytest + +try: + from google.cloud import bigquery_storage # type: ignore +except ImportError: + bigquery_storage = None + +from google.cloud.bigquery import exceptions + +from bigquery_magics import _versions_helpers + + +@pytest.mark.skipif( + bigquery_storage is None, reason="Requires `google-cloud-bigquery-storage`" +) +def test_raises_no_error_w_recent_bqstorage(): + with mock.patch("google.cloud.bigquery_storage.__version__", new="2.0.0"): + try: + bqstorage_versions = _versions_helpers.BQStorageVersions() + bqstorage_versions.try_import(raise_if_error=True) + except exceptions.LegacyBigQueryStorageError: # pragma: NO COVER + raise ("Legacy error raised with a non-legacy dependency version.") + + +@pytest.mark.skipif( + bigquery_storage is None, reason="Requires `google-cloud-bigquery-storage`" +) +def test_raises_error_w_legacy_bqstorage(): + with mock.patch("google.cloud.bigquery_storage.__version__", new="1.9.9"): + with pytest.raises(exceptions.LegacyBigQueryStorageError): + bqstorage_versions = _versions_helpers.BQStorageVersions() + bqstorage_versions.try_import(raise_if_error=True) + + +@pytest.mark.skipif( + bigquery_storage is None, reason="Requires `google-cloud-bigquery-storage`" +) +def test_returns_none_with_legacy_bqstorage(): + with mock.patch("google.cloud.bigquery_storage.__version__", new="1.9.9"): + try: + bqstorage_versions = _versions_helpers.BQStorageVersions() + bq_storage = bqstorage_versions.try_import() + except exceptions.LegacyBigQueryStorageError: # pragma: NO COVER + raise ("Legacy error raised when raise_if_error == False.") + assert bq_storage is None + + +@pytest.mark.skipif( + bigquery_storage is not None, + reason="Tests behavior when `google-cloud-bigquery-storage` isn't installed", +) +def test_returns_none_with_bqstorage_uninstalled(): + try: + bqstorage_versions = _versions_helpers.BQStorageVersions() + bq_storage = bqstorage_versions.try_import() + except exceptions.LegacyBigQueryStorageError: # pragma: NO COVER + raise ("NotFound error raised when raise_if_error == False.") + assert bq_storage is None + + +@pytest.mark.skipif( + bigquery_storage is None, reason="Requires `google-cloud-bigquery-storage`" +) +def test_raises_error_w_unknown_bqstorage_version(): + with mock.patch("google.cloud.bigquery_storage", autospec=True) as fake_module: + del fake_module.__version__ + error_pattern = r"version found: 0.0.0" + with pytest.raises(exceptions.LegacyBigQueryStorageError, match=error_pattern): + bqstorage_versions = _versions_helpers.BQStorageVersions() + bqstorage_versions.try_import(raise_if_error=True) + + +@pytest.mark.skipif( + bigquery_storage is None, reason="Requires `google-cloud-bigquery-storage`" +) +def test_installed_bqstorage_version_returns_cached(): + bqstorage_versions = _versions_helpers.BQStorageVersions() + bqstorage_versions._installed_version = object() + assert bqstorage_versions.installed_version is bqstorage_versions._installed_version + + +@pytest.mark.skipif( + bigquery_storage is None, reason="Requires `google-cloud-bigquery-storage`" +) +def test_installed_bqstorage_version_returns_parsed_version(): + bqstorage_versions = _versions_helpers.BQStorageVersions() + with mock.patch("google.cloud.bigquery_storage.__version__", new="1.2.3"): + bqstorage_versions = bqstorage_versions.installed_version + + assert bqstorage_versions.major == 1 + assert bqstorage_versions.minor == 2 + assert bqstorage_versions.micro == 3 diff --git a/tests/unit/test_bigquery.py b/tests/unit/test_bigquery.py index 4b1aaf1..678aa56 100644 --- a/tests/unit/test_bigquery.py +++ b/tests/unit/test_bigquery.py @@ -12,36 +12,44 @@ # See the License for the specific language governing permissions and # limitations under the License. +from concurrent import futures +import contextlib import copy import re -from concurrent import futures from unittest import mock import warnings +import IPython +import IPython.terminal.interactiveshell as interactiveshell +import IPython.testing.tools as tools +import IPython.utils.io as io from google.api_core import exceptions import google.auth.credentials -import pytest -from tests.unit.helpers import make_connection -from test_utils.imports import maybe_fail_import - from google.cloud import bigquery from google.cloud.bigquery import exceptions as bq_exceptions -from google.cloud.bigquery import job -from google.cloud.bigquery import table +from google.cloud.bigquery import job, table +import google.cloud.bigquery._http +import google.cloud.bigquery.exceptions from google.cloud.bigquery.retry import DEFAULT_TIMEOUT +import pandas +import pytest +import test_utils.imports # google-cloud-testutils +import bigquery_magics +import bigquery_magics.bigquery as magics try: - from google.cloud.bigquery.magics import magics + import google.cloud.bigquery_storage as bigquery_storage except ImportError: - magics = None + bigquery_storage = None + -bigquery_storage = pytest.importorskip("google.cloud.bigquery_storage") -IPython = pytest.importorskip("IPython") -interactiveshell = pytest.importorskip("IPython.terminal.interactiveshell") -tools = pytest.importorskip("IPython.testing.tools") -io = pytest.importorskip("IPython.utils.io") -pandas = pytest.importorskip("pandas") +def make_connection(*args): + # TODO(tswast): Remove this in favor of a mock google.cloud.bigquery.Client + # in tests. + conn = mock.create_autospec(google.cloud.bigquery._http.Connection, instance=True) + conn.api_request.side_effect = args + return conn @pytest.fixture(scope="session") @@ -87,7 +95,7 @@ def fail_if(name, globals, locals, fromlist, level): fromlist is not None and "bigquery_storage" in fromlist ) - return maybe_fail_import(predicate=fail_if) + return test_utils.imports.maybe_fail_import(predicate=fail_if) @pytest.fixture(scope="session") @@ -98,7 +106,7 @@ def fail_if(name, globals, locals, fromlist, level): # NOTE: *very* simplified, assuming a straightforward absolute import return "gapic_v1" in name or (fromlist is not None and "gapic_v1" in fromlist) - return maybe_fail_import(predicate=fail_if) + return test_utils.imports.maybe_fail_import(predicate=fail_if) PROJECT_ID = "its-a-project-eh" @@ -136,8 +144,8 @@ def test_context_with_default_credentials(): """When Application Default Credentials are set, the context credentials will be created the first time it is called """ - assert magics.context._credentials is None - assert magics.context._project is None + assert bigquery_magics.context._credentials is None + assert bigquery_magics.context._project is None project = "prahj-ekt" credentials_mock = mock.create_autospec( @@ -147,20 +155,19 @@ def test_context_with_default_credentials(): "google.auth.default", return_value=(credentials_mock, project) ) with default_patch as default_mock: - assert magics.context.credentials is credentials_mock - assert magics.context.project == project + assert bigquery_magics.context.credentials is credentials_mock + assert bigquery_magics.context.project == project assert default_mock.call_count == 2 @pytest.mark.usefixtures("ipython_interactive") -@pytest.mark.skipif(pandas is None, reason="Requires `pandas`") def test_context_with_default_connection(): ip = IPython.get_ipython() - ip.extension_manager.load_extension("google.cloud.bigquery") - magics.context._credentials = None - magics.context._project = None - magics.context._connection = None + ip.extension_manager.load_extension("bigquery_magics") + bigquery_magics.context._credentials = None + bigquery_magics.context._project = None + bigquery_magics.context._connection = None default_credentials = mock.create_autospec( google.auth.credentials.Credentials, instance=True @@ -193,6 +200,7 @@ def test_context_with_default_connection(): path=f"/projects/{PROJECT_ID}/queries/{JOB_ID}", query_params=mock.ANY, timeout=mock.ANY, + headers=mock.ANY, ) default_conn.api_request.assert_has_calls([begin_call, query_results_call]) @@ -207,23 +215,22 @@ def test_context_credentials_and_project_can_be_set_explicitly(): "google.auth.default", return_value=(credentials_mock, project1) ) with default_patch as default_mock: - magics.context.credentials = credentials_mock - magics.context.project = project2 + bigquery_magics.context.credentials = credentials_mock + bigquery_magics.context.project = project2 - assert magics.context.project == project2 - assert magics.context.credentials is credentials_mock + assert bigquery_magics.context.project == project2 + assert bigquery_magics.context.credentials is credentials_mock # default should not be called if credentials & project are explicitly set assert default_mock.call_count == 0 @pytest.mark.usefixtures("ipython_interactive") -@pytest.mark.skipif(pandas is None, reason="Requires `pandas`") def test_context_with_custom_connection(): ip = IPython.get_ipython() - ip.extension_manager.load_extension("google.cloud.bigquery") - magics.context._project = None - magics.context._credentials = None - context_conn = magics.context._connection = make_connection( + ip.extension_manager.load_extension("bigquery_magics") + bigquery_magics.context._project = None + bigquery_magics.context._credentials = None + context_conn = bigquery_magics.context._connection = make_connection( QUERY_RESOURCE, QUERY_RESULTS_RESOURCE ) @@ -257,12 +264,13 @@ def test_context_with_custom_connection(): path=f"/projects/{PROJECT_ID}/queries/{JOB_ID}", query_params=mock.ANY, timeout=mock.ANY, + headers=mock.ANY, ) context_conn.api_request.assert_has_calls([begin_call, query_results_call]) def test__run_query(): - magics.context._credentials = None + bigquery_magics.context._credentials = None job_id = "job_1234" sql = "SELECT 17" @@ -272,9 +280,7 @@ def test__run_query(): [table.Row((17,), {"num": 0})], ] - client_patch = mock.patch( - "google.cloud.bigquery.magics.magics.bigquery.Client", autospec=True - ) + client_patch = mock.patch("bigquery_magics.bigquery.bigquery.Client", autospec=True) with client_patch as client_mock, io.capture_output() as captured: client_mock().query(sql).result.side_effect = responses client_mock().query(sql).job_id = job_id @@ -295,13 +301,11 @@ def test__run_query(): def test__run_query_dry_run_without_errors_is_silent(): - magics.context._credentials = None + bigquery_magics.context._credentials = None sql = "SELECT 17" - client_patch = mock.patch( - "google.cloud.bigquery.magics.magics.bigquery.Client", autospec=True - ) + client_patch = mock.patch("bigquery_magics.bigquery.bigquery.Client", autospec=True) job_config = job.QueryJobConfig() job_config.dry_run = True @@ -372,28 +376,20 @@ def test__make_bqstorage_client_true_obsolete_dependency(): ) patcher = mock.patch( - "google.cloud.bigquery._versions_helpers.BQ_STORAGE_VERSIONS.try_import", + "bigquery_magics._versions_helpers.BQ_STORAGE_VERSIONS.try_import", side_effect=bq_exceptions.LegacyBigQueryStorageError( "google-cloud-bigquery-storage is outdated" ), ) - with patcher, warnings.catch_warnings(record=True) as warned: - got = magics._make_bqstorage_client(test_client, True, {}) - - assert got is None - - matching_warnings = [ - warning - for warning in warned - if "google-cloud-bigquery-storage is outdated" in str(warning) - ] - assert matching_warnings, "Obsolete dependency warning not raised." + with patcher, pytest.raises( + google.cloud.bigquery.exceptions.LegacyBigQueryStorageError + ): + magics._make_bqstorage_client(test_client, True, {}) @pytest.mark.skipif( bigquery_storage is None, reason="Requires `google-cloud-bigquery-storage`" ) -@pytest.mark.skipif(pandas is None, reason="Requires `pandas`") def test__make_bqstorage_client_true_missing_gapic(missing_grpcio_lib): credentials_mock = mock.create_autospec( google.auth.credentials.Credentials, instance=True @@ -410,9 +406,7 @@ def test__create_dataset_if_necessary_exists(): dataset_id = "dataset_id" dataset_reference = bigquery.dataset.DatasetReference(project, dataset_id) dataset = bigquery.Dataset(dataset_reference) - client_patch = mock.patch( - "google.cloud.bigquery.magics.magics.bigquery.Client", autospec=True - ) + client_patch = mock.patch("bigquery_magics.bigquery.bigquery.Client", autospec=True) with client_patch as client_mock: client = client_mock() client.project = project @@ -424,9 +418,7 @@ def test__create_dataset_if_necessary_exists(): def test__create_dataset_if_necessary_not_exist(): project = "project_id" dataset_id = "dataset_id" - client_patch = mock.patch( - "google.cloud.bigquery.magics.magics.bigquery.Client", autospec=True - ) + client_patch = mock.patch("bigquery_magics.bigquery.bigquery.Client", autospec=True) with client_patch as client_mock: client = client_mock() client.location = "us" @@ -439,28 +431,27 @@ def test__create_dataset_if_necessary_not_exist(): @pytest.mark.usefixtures("ipython_interactive") def test_extension_load(): ip = IPython.get_ipython() - ip.extension_manager.load_extension("google.cloud.bigquery") + ip.extension_manager.load_extension("bigquery_magics") # verify that the magic is registered and has the correct source magic = ip.magics_manager.magics["cell"].get("bigquery") - assert magic.__module__ == "google.cloud.bigquery.magics.magics" + assert magic.__module__ == "bigquery_magics.bigquery" @pytest.mark.usefixtures("ipython_interactive") -@pytest.mark.skipif(pandas is None, reason="Requires `pandas`") @pytest.mark.skipif( bigquery_storage is None, reason="Requires `google-cloud-bigquery-storage`" ) def test_bigquery_magic_without_optional_arguments(monkeypatch): ip = IPython.get_ipython() - ip.extension_manager.load_extension("google.cloud.bigquery") + ip.extension_manager.load_extension("bigquery_magics") mock_credentials = mock.create_autospec( google.auth.credentials.Credentials, instance=True ) # Set up the context with monkeypatch so that it's reset for subsequent # tests. - monkeypatch.setattr(magics.context, "_credentials", mock_credentials) + monkeypatch.setattr(bigquery_magics.context, "_credentials", mock_credentials) # Mock out the BigQuery Storage API. bqstorage_mock = mock.create_autospec(bigquery_storage.BigQueryReadClient) @@ -475,9 +466,7 @@ def test_bigquery_magic_without_optional_arguments(monkeypatch): sql = "SELECT 17 AS num" result = pandas.DataFrame([17], columns=["num"]) - run_query_patch = mock.patch( - "google.cloud.bigquery.magics.magics._run_query", autospec=True - ) + run_query_patch = mock.patch("bigquery_magics.bigquery._run_query", autospec=True) query_job_mock = mock.create_autospec( google.cloud.bigquery.job.QueryJob, instance=True ) @@ -496,8 +485,8 @@ def test_bigquery_magic_without_optional_arguments(monkeypatch): @pytest.mark.usefixtures("ipython_interactive") def test_bigquery_magic_default_connection_user_agent(): ip = IPython.get_ipython() - ip.extension_manager.load_extension("google.cloud.bigquery") - magics.context._connection = None + ip.extension_manager.load_extension("bigquery_magics") + bigquery_magics.context._connection = None credentials_mock = mock.create_autospec( google.auth.credentials.Credentials, instance=True @@ -505,9 +494,7 @@ def test_bigquery_magic_default_connection_user_agent(): default_patch = mock.patch( "google.auth.default", return_value=(credentials_mock, "general-project") ) - run_query_patch = mock.patch( - "google.cloud.bigquery.magics.magics._run_query", autospec=True - ) + run_query_patch = mock.patch("bigquery_magics.bigquery._run_query", autospec=True) conn_patch = mock.patch("google.cloud.bigquery.client.Connection", autospec=True) with conn_patch as conn, run_query_patch, default_patch: @@ -521,14 +508,12 @@ def test_bigquery_magic_default_connection_user_agent(): @pytest.mark.usefixtures("ipython_interactive") def test_bigquery_magic_with_legacy_sql(): ip = IPython.get_ipython() - ip.extension_manager.load_extension("google.cloud.bigquery") - magics.context.credentials = mock.create_autospec( + ip.extension_manager.load_extension("bigquery_magics") + bigquery_magics.context.credentials = mock.create_autospec( google.auth.credentials.Credentials, instance=True ) - run_query_patch = mock.patch( - "google.cloud.bigquery.magics.magics._run_query", autospec=True - ) + run_query_patch = mock.patch("bigquery_magics.bigquery._run_query", autospec=True) with run_query_patch as run_query_mock: ip.run_cell_magic("bigquery", "--use_legacy_sql", "SELECT 17 AS num") @@ -537,11 +522,10 @@ def test_bigquery_magic_with_legacy_sql(): @pytest.mark.usefixtures("ipython_interactive") -@pytest.mark.skipif(pandas is None, reason="Requires `pandas`") def test_bigquery_magic_with_result_saved_to_variable(ipython_ns_cleanup): ip = IPython.get_ipython() - ip.extension_manager.load_extension("google.cloud.bigquery") - magics.context.credentials = mock.create_autospec( + ip.extension_manager.load_extension("bigquery_magics") + bigquery_magics.context.credentials = mock.create_autospec( google.auth.credentials.Credentials, instance=True ) @@ -551,9 +535,7 @@ def test_bigquery_magic_with_result_saved_to_variable(ipython_ns_cleanup): result = pandas.DataFrame([17], columns=["num"]) assert "df" not in ip.user_ns - run_query_patch = mock.patch( - "google.cloud.bigquery.magics.magics._run_query", autospec=True - ) + run_query_patch = mock.patch("bigquery_magics.bigquery._run_query", autospec=True) query_job_mock = mock.create_autospec( google.cloud.bigquery.job.QueryJob, instance=True ) @@ -573,18 +555,16 @@ def test_bigquery_magic_with_result_saved_to_variable(ipython_ns_cleanup): @pytest.mark.usefixtures("ipython_interactive") def test_bigquery_magic_does_not_clear_display_in_verbose_mode(): ip = IPython.get_ipython() - ip.extension_manager.load_extension("google.cloud.bigquery") - magics.context.credentials = mock.create_autospec( + ip.extension_manager.load_extension("bigquery_magics") + bigquery_magics.context.credentials = mock.create_autospec( google.auth.credentials.Credentials, instance=True ) clear_patch = mock.patch( - "google.cloud.bigquery.magics.magics.display.clear_output", + "bigquery_magics.bigquery.display.clear_output", autospec=True, ) - run_query_patch = mock.patch( - "google.cloud.bigquery.magics.magics._run_query", autospec=True - ) + run_query_patch = mock.patch("bigquery_magics.bigquery._run_query", autospec=True) with clear_patch as clear_mock, run_query_patch: ip.run_cell_magic("bigquery", "--verbose", "SELECT 17 as num") @@ -594,18 +574,16 @@ def test_bigquery_magic_does_not_clear_display_in_verbose_mode(): @pytest.mark.usefixtures("ipython_interactive") def test_bigquery_magic_clears_display_in_non_verbose_mode(): ip = IPython.get_ipython() - ip.extension_manager.load_extension("google.cloud.bigquery") - magics.context.credentials = mock.create_autospec( + ip.extension_manager.load_extension("bigquery_magics") + bigquery_magics.context.credentials = mock.create_autospec( google.auth.credentials.Credentials, instance=True ) clear_patch = mock.patch( - "google.cloud.bigquery.magics.magics.display.clear_output", + "bigquery_magics.bigquery.display.clear_output", autospec=True, ) - run_query_patch = mock.patch( - "google.cloud.bigquery.magics.magics._run_query", autospec=True - ) + run_query_patch = mock.patch("bigquery_magics.bigquery._run_query", autospec=True) with clear_patch as clear_mock, run_query_patch: ip.run_cell_magic("bigquery", "", "SELECT 17 as num") @@ -618,14 +596,14 @@ def test_bigquery_magic_clears_display_in_non_verbose_mode(): ) def test_bigquery_magic_with_bqstorage_from_argument(monkeypatch): ip = IPython.get_ipython() - ip.extension_manager.load_extension("google.cloud.bigquery") + ip.extension_manager.load_extension("bigquery_magics") mock_credentials = mock.create_autospec( google.auth.credentials.Credentials, instance=True ) # Set up the context with monkeypatch so that it's reset for subsequent # tests. - monkeypatch.setattr(magics.context, "_credentials", mock_credentials) + monkeypatch.setattr(bigquery_magics.context, "_credentials", mock_credentials) # Mock out the BigQuery Storage API. bqstorage_mock = mock.create_autospec(bigquery_storage.BigQueryReadClient) @@ -640,9 +618,7 @@ def test_bigquery_magic_with_bqstorage_from_argument(monkeypatch): sql = "SELECT 17 AS num" result = pandas.DataFrame([17], columns=["num"]) - run_query_patch = mock.patch( - "google.cloud.bigquery.magics.magics._run_query", autospec=True - ) + run_query_patch = mock.patch("bigquery_magics.bigquery._run_query", autospec=True) query_job_mock = mock.create_autospec( google.cloud.bigquery.job.QueryJob, instance=True ) @@ -686,14 +662,14 @@ def test_bigquery_magic_with_rest_client_requested(monkeypatch): pandas = pytest.importorskip("pandas") ip = IPython.get_ipython() - ip.extension_manager.load_extension("google.cloud.bigquery") + ip.extension_manager.load_extension("bigquery_magics") mock_credentials = mock.create_autospec( google.auth.credentials.Credentials, instance=True ) # Set up the context with monkeypatch so that it's reset for subsequent # tests. - monkeypatch.setattr(magics.context, "_credentials", mock_credentials) + monkeypatch.setattr(bigquery_magics.context, "_credentials", mock_credentials) # Mock out the BigQuery Storage API. bqstorage_mock = mock.create_autospec(bigquery_storage.BigQueryReadClient) @@ -703,9 +679,7 @@ def test_bigquery_magic_with_rest_client_requested(monkeypatch): sql = "SELECT 17 AS num" result = pandas.DataFrame([17], columns=["num"]) - run_query_patch = mock.patch( - "google.cloud.bigquery.magics.magics._run_query", autospec=True - ) + run_query_patch = mock.patch("bigquery_magics.bigquery._run_query", autospec=True) query_job_mock = mock.create_autospec( google.cloud.bigquery.job.QueryJob, instance=True ) @@ -728,8 +702,8 @@ def test_bigquery_magic_with_rest_client_requested(monkeypatch): @pytest.mark.usefixtures("ipython_interactive") def test_bigquery_magic_w_max_results_invalid(): ip = IPython.get_ipython() - ip.extension_manager.load_extension("google.cloud.bigquery") - magics.context._project = None + ip.extension_manager.load_extension("bigquery_magics") + bigquery_magics.context._project = None credentials_mock = mock.create_autospec( google.auth.credentials.Credentials, instance=True @@ -750,8 +724,8 @@ def test_bigquery_magic_w_max_results_invalid(): @pytest.mark.usefixtures("ipython_interactive") def test_bigquery_magic_w_max_results_valid_calls_queryjob_result(): ip = IPython.get_ipython() - ip.extension_manager.load_extension("google.cloud.bigquery") - magics.context._project = None + ip.extension_manager.load_extension("bigquery_magics") + bigquery_magics.context._project = None credentials_mock = mock.create_autospec( google.auth.credentials.Credentials, instance=True @@ -784,8 +758,8 @@ def test_bigquery_magic_w_max_results_valid_calls_queryjob_result(): @pytest.mark.usefixtures("ipython_interactive") def test_bigquery_magic_w_max_results_query_job_results_fails(): ip = IPython.get_ipython() - ip.extension_manager.load_extension("google.cloud.bigquery") - magics.context._project = None + ip.extension_manager.load_extension("bigquery_magics") + bigquery_magics.context._project = None credentials_mock = mock.create_autospec( google.auth.credentials.Credentials, instance=True @@ -797,7 +771,7 @@ def test_bigquery_magic_w_max_results_query_job_results_fails(): "google.cloud.bigquery.client.Client.query", autospec=True ) close_transports_patch = mock.patch( - "google.cloud.bigquery.magics.magics._close_transports", + "bigquery_magics.bigquery._close_transports", autospec=True, ) @@ -821,8 +795,8 @@ def test_bigquery_magic_w_max_results_query_job_results_fails(): def test_bigquery_magic_w_table_id_invalid(): ip = IPython.get_ipython() - ip.extension_manager.load_extension("google.cloud.bigquery") - magics.context._project = None + ip.extension_manager.load_extension("bigquery_magics") + bigquery_magics.context._project = None credentials_mock = mock.create_autospec( google.auth.credentials.Credentials, instance=True @@ -832,7 +806,7 @@ def test_bigquery_magic_w_table_id_invalid(): ) list_rows_patch = mock.patch( - "google.cloud.bigquery.magics.magics.bigquery.Client.list_rows", + "bigquery_magics.bigquery.bigquery.Client.list_rows", autospec=True, side_effect=exceptions.BadRequest("Not a valid table ID"), ) @@ -850,8 +824,8 @@ def test_bigquery_magic_w_table_id_invalid(): def test_bigquery_magic_w_missing_query(): ip = IPython.get_ipython() - ip.extension_manager.load_extension("google.cloud.bigquery") - magics.context._project = None + ip.extension_manager.load_extension("bigquery_magics") + bigquery_magics.context._project = None credentials_mock = mock.create_autospec( google.auth.credentials.Credentials, instance=True @@ -872,11 +846,10 @@ def test_bigquery_magic_w_missing_query(): @pytest.mark.usefixtures("ipython_interactive") -@pytest.mark.skipif(pandas is None, reason="Requires `pandas`") def test_bigquery_magic_w_table_id_and_destination_var(ipython_ns_cleanup): ip = IPython.get_ipython() - ip.extension_manager.load_extension("google.cloud.bigquery") - magics.context._project = None + ip.extension_manager.load_extension("bigquery_magics") + bigquery_magics.context._project = None ipython_ns_cleanup.append((ip, "df")) @@ -891,9 +864,7 @@ def test_bigquery_magic_w_table_id_and_destination_var(ipython_ns_cleanup): google.cloud.bigquery.table.RowIterator, instance=True ) - client_patch = mock.patch( - "google.cloud.bigquery.magics.magics.bigquery.Client", autospec=True - ) + client_patch = mock.patch("bigquery_magics.bigquery.bigquery.Client", autospec=True) table_id = "bigquery-public-data.samples.shakespeare" result = pandas.DataFrame([17], columns=["num"]) @@ -914,11 +885,10 @@ def test_bigquery_magic_w_table_id_and_destination_var(ipython_ns_cleanup): @pytest.mark.skipif( bigquery_storage is None, reason="Requires `google-cloud-bigquery-storage`" ) -@pytest.mark.skipif(pandas is None, reason="Requires `pandas`") def test_bigquery_magic_w_table_id_and_bqstorage_client(): ip = IPython.get_ipython() - ip.extension_manager.load_extension("google.cloud.bigquery") - magics.context._project = None + ip.extension_manager.load_extension("bigquery_magics") + bigquery_magics.context._project = None credentials_mock = mock.create_autospec( google.auth.credentials.Credentials, instance=True @@ -931,9 +901,7 @@ def test_bigquery_magic_w_table_id_and_bqstorage_client(): google.cloud.bigquery.table.RowIterator, instance=True ) - client_patch = mock.patch( - "google.cloud.bigquery.magics.magics.bigquery.Client", autospec=True - ) + client_patch = mock.patch("bigquery_magics.bigquery.bigquery.Client", autospec=True) bqstorage_mock = mock.create_autospec(bigquery_storage.BigQueryReadClient) bqstorage_instance_mock = mock.create_autospec( @@ -961,14 +929,12 @@ def test_bigquery_magic_w_table_id_and_bqstorage_client(): @pytest.mark.usefixtures("ipython_interactive") def test_bigquery_magic_dryrun_option_sets_job_config(): ip = IPython.get_ipython() - ip.extension_manager.load_extension("google.cloud.bigquery") - magics.context.credentials = mock.create_autospec( + ip.extension_manager.load_extension("bigquery_magics") + bigquery_magics.context.credentials = mock.create_autospec( google.auth.credentials.Credentials, instance=True ) - run_query_patch = mock.patch( - "google.cloud.bigquery.magics.magics._run_query", autospec=True - ) + run_query_patch = mock.patch("bigquery_magics.bigquery._run_query", autospec=True) sql = "SELECT 17 AS num" @@ -982,16 +948,14 @@ def test_bigquery_magic_dryrun_option_sets_job_config(): @pytest.mark.usefixtures("ipython_interactive") def test_bigquery_magic_dryrun_option_returns_query_job(): ip = IPython.get_ipython() - ip.extension_manager.load_extension("google.cloud.bigquery") - magics.context.credentials = mock.create_autospec( + ip.extension_manager.load_extension("bigquery_magics") + bigquery_magics.context.credentials = mock.create_autospec( google.auth.credentials.Credentials, instance=True ) query_job_mock = mock.create_autospec( google.cloud.bigquery.job.QueryJob, instance=True ) - run_query_patch = mock.patch( - "google.cloud.bigquery.magics.magics._run_query", autospec=True - ) + run_query_patch = mock.patch("bigquery_magics.bigquery._run_query", autospec=True) sql = "SELECT 17 AS num" @@ -1006,15 +970,15 @@ def test_bigquery_magic_dryrun_option_returns_query_job(): @pytest.mark.usefixtures("ipython_interactive") def test_bigquery_magic_dryrun_option_variable_error_message(ipython_ns_cleanup): ip = IPython.get_ipython() - ip.extension_manager.load_extension("google.cloud.bigquery") - magics.context.credentials = mock.create_autospec( + ip.extension_manager.load_extension("bigquery_magics") + bigquery_magics.context.credentials = mock.create_autospec( google.auth.credentials.Credentials, instance=True ) ipython_ns_cleanup.append((ip, "q_job")) run_query_patch = mock.patch( - "google.cloud.bigquery.magics.magics._run_query", + "bigquery_magics.bigquery._run_query", autospec=True, side_effect=exceptions.BadRequest("Syntax error in SQL query"), ) @@ -1033,16 +997,14 @@ def test_bigquery_magic_dryrun_option_variable_error_message(ipython_ns_cleanup) @pytest.mark.usefixtures("ipython_interactive") def test_bigquery_magic_dryrun_option_saves_query_job_to_variable(ipython_ns_cleanup): ip = IPython.get_ipython() - ip.extension_manager.load_extension("google.cloud.bigquery") - magics.context.credentials = mock.create_autospec( + ip.extension_manager.load_extension("bigquery_magics") + bigquery_magics.context.credentials = mock.create_autospec( google.auth.credentials.Credentials, instance=True ) query_job_mock = mock.create_autospec( google.cloud.bigquery.job.QueryJob, instance=True ) - run_query_patch = mock.patch( - "google.cloud.bigquery.magics.magics._run_query", autospec=True - ) + run_query_patch = mock.patch("bigquery_magics.bigquery._run_query", autospec=True) ipython_ns_cleanup.append((ip, "q_job")) @@ -1063,8 +1025,8 @@ def test_bigquery_magic_dryrun_option_saves_query_job_to_variable(ipython_ns_cle @pytest.mark.usefixtures("ipython_interactive") def test_bigquery_magic_saves_query_job_to_variable_on_error(ipython_ns_cleanup): ip = IPython.get_ipython() - ip.extension_manager.load_extension("google.cloud.bigquery") - magics.context.credentials = mock.create_autospec( + ip.extension_manager.load_extension("bigquery_magics") + bigquery_magics.context.credentials = mock.create_autospec( google.auth.credentials.Credentials, instance=True ) @@ -1096,8 +1058,8 @@ def test_bigquery_magic_saves_query_job_to_variable_on_error(ipython_ns_cleanup) @pytest.mark.usefixtures("ipython_interactive") def test_bigquery_magic_w_maximum_bytes_billed_invalid(): ip = IPython.get_ipython() - ip.extension_manager.load_extension("google.cloud.bigquery") - magics.context._project = None + ip.extension_manager.load_extension("bigquery_magics") + bigquery_magics.context._project = None credentials_mock = mock.create_autospec( google.auth.credentials.Credentials, instance=True @@ -1117,14 +1079,13 @@ def test_bigquery_magic_w_maximum_bytes_billed_invalid(): "param_value,expected", [("987654321", "987654321"), ("None", "0")] ) @pytest.mark.usefixtures("ipython_interactive") -@pytest.mark.skipif(pandas is None, reason="Requires `pandas`") def test_bigquery_magic_w_maximum_bytes_billed_overrides_context(param_value, expected): ip = IPython.get_ipython() - ip.extension_manager.load_extension("google.cloud.bigquery") - magics.context._project = None + ip.extension_manager.load_extension("bigquery_magics") + bigquery_magics.context._project = None # Set the default maximum bytes billed, so we know it's overridable by the param. - magics.context.default_query_job_config.maximum_bytes_billed = 1234567 + bigquery_magics.context.default_query_job_config.maximum_bytes_billed = 1234567 project = "test-project" job_reference = copy.deepcopy(JOB_REFERENCE_RESOURCE) @@ -1141,7 +1102,9 @@ def test_bigquery_magic_w_maximum_bytes_billed_overrides_context(param_value, ex default_patch = mock.patch( "google.auth.default", return_value=(credentials_mock, "general-project") ) - conn = magics.context._connection = make_connection(resource, query_results, data) + conn = bigquery_magics.context._connection = make_connection( + resource, query_results, data + ) list_rows_patch = mock.patch( "google.cloud.bigquery.client.Client._list_rows_from_query_results", return_value=google.cloud.bigquery.table._EmptyRowIterator(), @@ -1157,13 +1120,12 @@ def test_bigquery_magic_w_maximum_bytes_billed_overrides_context(param_value, ex @pytest.mark.usefixtures("ipython_interactive") -@pytest.mark.skipif(pandas is None, reason="Requires `pandas`") def test_bigquery_magic_w_maximum_bytes_billed_w_context_inplace(): ip = IPython.get_ipython() - ip.extension_manager.load_extension("google.cloud.bigquery") - magics.context._project = None + ip.extension_manager.load_extension("bigquery_magics") + bigquery_magics.context._project = None - magics.context.default_query_job_config.maximum_bytes_billed = 1337 + bigquery_magics.context.default_query_job_config.maximum_bytes_billed = 1337 project = "test-project" job_reference = copy.deepcopy(JOB_REFERENCE_RESOURCE) @@ -1180,7 +1142,9 @@ def test_bigquery_magic_w_maximum_bytes_billed_w_context_inplace(): default_patch = mock.patch( "google.auth.default", return_value=(credentials_mock, "general-project") ) - conn = magics.context._connection = make_connection(resource, query_results, data) + conn = bigquery_magics.context._connection = make_connection( + resource, query_results, data + ) list_rows_patch = mock.patch( "google.cloud.bigquery.client.Client._list_rows_from_query_results", return_value=google.cloud.bigquery.table._EmptyRowIterator(), @@ -1194,13 +1158,12 @@ def test_bigquery_magic_w_maximum_bytes_billed_w_context_inplace(): @pytest.mark.usefixtures("ipython_interactive") -@pytest.mark.skipif(pandas is None, reason="Requires `pandas`") def test_bigquery_magic_w_maximum_bytes_billed_w_context_setter(): ip = IPython.get_ipython() - ip.extension_manager.load_extension("google.cloud.bigquery") - magics.context._project = None + ip.extension_manager.load_extension("bigquery_magics") + bigquery_magics.context._project = None - magics.context.default_query_job_config = job.QueryJobConfig( + bigquery_magics.context.default_query_job_config = job.QueryJobConfig( maximum_bytes_billed=10203 ) @@ -1219,7 +1182,9 @@ def test_bigquery_magic_w_maximum_bytes_billed_w_context_setter(): default_patch = mock.patch( "google.auth.default", return_value=(credentials_mock, "general-project") ) - conn = magics.context._connection = make_connection(resource, query_results, data) + conn = bigquery_magics.context._connection = make_connection( + resource, query_results, data + ) list_rows_patch = mock.patch( "google.cloud.bigquery.client.Client._list_rows_from_query_results", return_value=google.cloud.bigquery.table._EmptyRowIterator(), @@ -1233,17 +1198,16 @@ def test_bigquery_magic_w_maximum_bytes_billed_w_context_setter(): @pytest.mark.usefixtures("ipython_interactive") -@pytest.mark.skipif(pandas is None, reason="Requires `pandas`") def test_bigquery_magic_with_no_query_cache(monkeypatch): ip = IPython.get_ipython() - ip.extension_manager.load_extension("google.cloud.bigquery") + ip.extension_manager.load_extension("bigquery_magics") conn = make_connection() - monkeypatch.setattr(magics.context, "_connection", conn) - monkeypatch.setattr(magics.context, "project", "project-from-context") + monkeypatch.setattr(bigquery_magics.context, "_connection", conn) + monkeypatch.setattr(bigquery_magics.context, "project", "project-from-context") # --no_query_cache option should override context. monkeypatch.setattr( - magics.context.default_query_job_config, "use_query_cache", True + bigquery_magics.context.default_query_job_config, "use_query_cache", True ) ip.run_cell_magic("bigquery", "--no_query_cache", QUERY_STRING) @@ -1263,15 +1227,14 @@ def test_bigquery_magic_with_no_query_cache(monkeypatch): @pytest.mark.usefixtures("ipython_interactive") -@pytest.mark.skipif(pandas is None, reason="Requires `pandas`") def test_context_with_no_query_cache_from_context(monkeypatch): ip = IPython.get_ipython() - ip.extension_manager.load_extension("google.cloud.bigquery") + ip.extension_manager.load_extension("bigquery_magics") conn = make_connection() - monkeypatch.setattr(magics.context, "_connection", conn) - monkeypatch.setattr(magics.context, "project", "project-from-context") + monkeypatch.setattr(bigquery_magics.context, "_connection", conn) + monkeypatch.setattr(bigquery_magics.context, "project", "project-from-context") monkeypatch.setattr( - magics.context.default_query_job_config, "use_query_cache", False + bigquery_magics.context.default_query_job_config, "use_query_cache", False ) ip.run_cell_magic("bigquery", "", QUERY_STRING) @@ -1291,13 +1254,12 @@ def test_context_with_no_query_cache_from_context(monkeypatch): @pytest.mark.usefixtures("ipython_interactive") -@pytest.mark.skipif(pandas is None, reason="Requires `pandas`") def test_bigquery_magic_w_progress_bar_type_w_context_setter(monkeypatch): ip = IPython.get_ipython() - ip.extension_manager.load_extension("google.cloud.bigquery") - magics.context._project = None + ip.extension_manager.load_extension("bigquery_magics") + bigquery_magics.context._project = None - magics.context.progress_bar_type = "tqdm_gui" + bigquery_magics.context.progress_bar_type = "tqdm_gui" mock_credentials = mock.create_autospec( google.auth.credentials.Credentials, instance=True @@ -1305,19 +1267,21 @@ def test_bigquery_magic_w_progress_bar_type_w_context_setter(monkeypatch): # Set up the context with monkeypatch so that it's reset for subsequent # tests. - monkeypatch.setattr(magics.context, "_credentials", mock_credentials) + monkeypatch.setattr(bigquery_magics.context, "_credentials", mock_credentials) # Mock out the BigQuery Storage API. - bqstorage_mock = mock.create_autospec(bigquery_storage.BigQueryReadClient) - bqstorage_client_patch = mock.patch( - "google.cloud.bigquery_storage.BigQueryReadClient", bqstorage_mock - ) + if bigquery_storage is not None: + bqstorage_mock = mock.create_autospec(bigquery_storage.BigQueryReadClient) + bqstorage_client_patch = mock.patch( + "google.cloud.bigquery_storage.BigQueryReadClient", bqstorage_mock + ) + else: + bqstorage_mock = mock.MagicMock() + bqstorage_client_patch = contextlib.nullcontext() sql = "SELECT 17 AS num" result = pandas.DataFrame([17], columns=["num"]) - run_query_patch = mock.patch( - "google.cloud.bigquery.magics.magics._run_query", autospec=True - ) + run_query_patch = mock.patch("bigquery_magics.bigquery._run_query", autospec=True) query_job_mock = mock.create_autospec( google.cloud.bigquery.job.QueryJob, instance=True ) @@ -1331,7 +1295,7 @@ def test_bigquery_magic_w_progress_bar_type_w_context_setter(monkeypatch): query_job_mock.to_dataframe.assert_called_once_with( bqstorage_client=None, create_bqstorage_client=False, - progress_bar_type=magics.context.progress_bar_type, + progress_bar_type=bigquery_magics.context.progress_bar_type, ) assert isinstance(return_value, pandas.DataFrame) @@ -1340,12 +1304,10 @@ def test_bigquery_magic_w_progress_bar_type_w_context_setter(monkeypatch): @pytest.mark.usefixtures("ipython_interactive") def test_bigquery_magic_with_progress_bar_type(): ip = IPython.get_ipython() - ip.extension_manager.load_extension("google.cloud.bigquery") - magics.context.progress_bar_type = None + ip.extension_manager.load_extension("bigquery_magics") + bigquery_magics.context.progress_bar_type = None - run_query_patch = mock.patch( - "google.cloud.bigquery.magics.magics._run_query", autospec=True - ) + run_query_patch = mock.patch("bigquery_magics.bigquery._run_query", autospec=True) with run_query_patch as run_query_mock: ip.run_cell_magic( "bigquery", "--progress_bar_type=tqdm_gui", "SELECT 17 as num" @@ -1354,14 +1316,14 @@ def test_bigquery_magic_with_progress_bar_type(): progress_bar_used = run_query_mock.mock_calls[1][2]["progress_bar_type"] assert progress_bar_used == "tqdm_gui" # context progress bar type should not change - assert magics.context.progress_bar_type is None + assert bigquery_magics.context.progress_bar_type is None @pytest.mark.usefixtures("ipython_interactive") def test_bigquery_magic_with_project(): ip = IPython.get_ipython() - ip.extension_manager.load_extension("google.cloud.bigquery") - magics.context._project = None + ip.extension_manager.load_extension("bigquery_magics") + bigquery_magics.context._project = None credentials_mock = mock.create_autospec( google.auth.credentials.Credentials, instance=True @@ -1369,27 +1331,23 @@ def test_bigquery_magic_with_project(): default_patch = mock.patch( "google.auth.default", return_value=(credentials_mock, "general-project") ) - run_query_patch = mock.patch( - "google.cloud.bigquery.magics.magics._run_query", autospec=True - ) + run_query_patch = mock.patch("bigquery_magics.bigquery._run_query", autospec=True) with run_query_patch as run_query_mock, default_patch: ip.run_cell_magic("bigquery", "--project=specific-project", "SELECT 17 as num") client_used = run_query_mock.call_args_list[0][0][0] assert client_used.project == "specific-project" # context project should not change - assert magics.context.project == "general-project" + assert bigquery_magics.context.project == "general-project" @pytest.mark.usefixtures("ipython_interactive") def test_bigquery_magic_with_bigquery_api_endpoint(ipython_ns_cleanup): ip = IPython.get_ipython() - ip.extension_manager.load_extension("google.cloud.bigquery") - magics.context._connection = None + ip.extension_manager.load_extension("bigquery_magics") + bigquery_magics.context._connection = None - run_query_patch = mock.patch( - "google.cloud.bigquery.magics.magics._run_query", autospec=True - ) + run_query_patch = mock.patch("bigquery_magics.bigquery._run_query", autospec=True) with run_query_patch as run_query_mock: ip.run_cell_magic( "bigquery", @@ -1400,19 +1358,17 @@ def test_bigquery_magic_with_bigquery_api_endpoint(ipython_ns_cleanup): connection_used = run_query_mock.call_args_list[0][0][0]._connection assert connection_used.API_BASE_URL == "https://bigquery_api.endpoint.com" # context client options should not change - assert magics.context.bigquery_client_options.api_endpoint is None + assert bigquery_magics.context.bigquery_client_options.api_endpoint is None @pytest.mark.usefixtures("ipython_interactive") def test_bigquery_magic_with_bigquery_api_endpoint_context_dict(): ip = IPython.get_ipython() - ip.extension_manager.load_extension("google.cloud.bigquery") - magics.context._connection = None - magics.context.bigquery_client_options = {} + ip.extension_manager.load_extension("bigquery_magics") + bigquery_magics.context._connection = None + bigquery_magics.context.bigquery_client_options = {} - run_query_patch = mock.patch( - "google.cloud.bigquery.magics.magics._run_query", autospec=True - ) + run_query_patch = mock.patch("bigquery_magics.bigquery._run_query", autospec=True) with run_query_patch as run_query_mock: ip.run_cell_magic( "bigquery", @@ -1423,18 +1379,19 @@ def test_bigquery_magic_with_bigquery_api_endpoint_context_dict(): connection_used = run_query_mock.call_args_list[0][0][0]._connection assert connection_used.API_BASE_URL == "https://bigquery_api.endpoint.com" # context client options should not change - assert magics.context.bigquery_client_options == {} + assert bigquery_magics.context.bigquery_client_options == {} @pytest.mark.usefixtures("ipython_interactive") +@pytest.mark.skipif( + bigquery_storage is None, reason="Requires `google-cloud-bigquery-storage`" +) def test_bigquery_magic_with_bqstorage_api_endpoint(ipython_ns_cleanup): ip = IPython.get_ipython() - ip.extension_manager.load_extension("google.cloud.bigquery") - magics.context._connection = None + ip.extension_manager.load_extension("bigquery_magics") + bigquery_magics.context._connection = None - run_query_patch = mock.patch( - "google.cloud.bigquery.magics.magics._run_query", autospec=True - ) + run_query_patch = mock.patch("bigquery_magics.bigquery._run_query", autospec=True) with run_query_patch as run_query_mock: ip.run_cell_magic( "bigquery", @@ -1445,19 +1402,20 @@ def test_bigquery_magic_with_bqstorage_api_endpoint(ipython_ns_cleanup): client_used = run_query_mock.mock_calls[1][2]["bqstorage_client"] assert client_used._transport._host == "https://bqstorage_api.endpoint.com" # context client options should not change - assert magics.context.bqstorage_client_options.api_endpoint is None + assert bigquery_magics.context.bqstorage_client_options.api_endpoint is None @pytest.mark.usefixtures("ipython_interactive") +@pytest.mark.skipif( + bigquery_storage is None, reason="Requires `google-cloud-bigquery-storage`" +) def test_bigquery_magic_with_bqstorage_api_endpoint_context_dict(): ip = IPython.get_ipython() - ip.extension_manager.load_extension("google.cloud.bigquery") - magics.context._connection = None - magics.context.bqstorage_client_options = {} + ip.extension_manager.load_extension("bigquery_magics") + bigquery_magics.context._connection = None + bigquery_magics.context.bqstorage_client_options = {} - run_query_patch = mock.patch( - "google.cloud.bigquery.magics.magics._run_query", autospec=True - ) + run_query_patch = mock.patch("bigquery_magics.bigquery._run_query", autospec=True) with run_query_patch as run_query_mock: ip.run_cell_magic( "bigquery", @@ -1468,14 +1426,14 @@ def test_bigquery_magic_with_bqstorage_api_endpoint_context_dict(): client_used = run_query_mock.mock_calls[1][2]["bqstorage_client"] assert client_used._transport._host == "https://bqstorage_api.endpoint.com" # context client options should not change - assert magics.context.bqstorage_client_options == {} + assert bigquery_magics.context.bqstorage_client_options == {} @pytest.mark.usefixtures("ipython_interactive") def test_bigquery_magic_with_multiple_options(): ip = IPython.get_ipython() - ip.extension_manager.load_extension("google.cloud.bigquery") - magics.context._project = None + ip.extension_manager.load_extension("bigquery_magics") + bigquery_magics.context._project = None credentials_mock = mock.create_autospec( google.auth.credentials.Credentials, instance=True @@ -1483,9 +1441,7 @@ def test_bigquery_magic_with_multiple_options(): default_patch = mock.patch( "google.auth.default", return_value=(credentials_mock, "general-project") ) - run_query_patch = mock.patch( - "google.cloud.bigquery.magics.magics._run_query", autospec=True - ) + run_query_patch = mock.patch("bigquery_magics.bigquery._run_query", autospec=True) with run_query_patch as run_query_mock, default_patch: ip.run_cell_magic( "bigquery", @@ -1503,11 +1459,10 @@ def test_bigquery_magic_with_multiple_options(): @pytest.mark.usefixtures("ipython_interactive") -@pytest.mark.skipif(pandas is None, reason="Requires `pandas`") def test_bigquery_magic_with_string_params(ipython_ns_cleanup): ip = IPython.get_ipython() - ip.extension_manager.load_extension("google.cloud.bigquery") - magics.context.credentials = mock.create_autospec( + ip.extension_manager.load_extension("bigquery_magics") + bigquery_magics.context.credentials = mock.create_autospec( google.auth.credentials.Credentials, instance=True ) @@ -1518,9 +1473,7 @@ def test_bigquery_magic_with_string_params(ipython_ns_cleanup): assert "params_dict_df" not in ip.user_ns - run_query_patch = mock.patch( - "google.cloud.bigquery.magics.magics._run_query", autospec=True - ) + run_query_patch = mock.patch("bigquery_magics.bigquery._run_query", autospec=True) query_job_mock = mock.create_autospec( google.cloud.bigquery.job.QueryJob, instance=True ) @@ -1540,11 +1493,10 @@ def test_bigquery_magic_with_string_params(ipython_ns_cleanup): @pytest.mark.usefixtures("ipython_interactive") -@pytest.mark.skipif(pandas is None, reason="Requires `pandas`") def test_bigquery_magic_with_dict_params(ipython_ns_cleanup): ip = IPython.get_ipython() - ip.extension_manager.load_extension("google.cloud.bigquery") - magics.context.credentials = mock.create_autospec( + ip.extension_manager.load_extension("bigquery_magics") + bigquery_magics.context.credentials = mock.create_autospec( google.auth.credentials.Credentials, instance=True ) @@ -1557,9 +1509,7 @@ def test_bigquery_magic_with_dict_params(ipython_ns_cleanup): assert "params_dict_df" not in ip.user_ns - run_query_patch = mock.patch( - "google.cloud.bigquery.magics.magics._run_query", autospec=True - ) + run_query_patch = mock.patch("bigquery_magics.bigquery._run_query", autospec=True) query_job_mock = mock.create_autospec( google.cloud.bigquery.job.QueryJob, instance=True ) @@ -1584,11 +1534,10 @@ def test_bigquery_magic_with_dict_params(ipython_ns_cleanup): @pytest.mark.usefixtures("ipython_interactive") -@pytest.mark.skipif(pandas is None, reason="Requires `pandas`") def test_bigquery_magic_with_dict_params_nonexisting(): ip = IPython.get_ipython() - ip.extension_manager.load_extension("google.cloud.bigquery") - magics.context.credentials = mock.create_autospec( + ip.extension_manager.load_extension("bigquery_magics") + bigquery_magics.context.credentials = mock.create_autospec( google.auth.credentials.Credentials, instance=True ) @@ -1599,11 +1548,10 @@ def test_bigquery_magic_with_dict_params_nonexisting(): @pytest.mark.usefixtures("ipython_interactive") -@pytest.mark.skipif(pandas is None, reason="Requires `pandas`") def test_bigquery_magic_with_dict_params_incorrect_syntax(): ip = IPython.get_ipython() - ip.extension_manager.load_extension("google.cloud.bigquery") - magics.context.credentials = mock.create_autospec( + ip.extension_manager.load_extension("bigquery_magics") + bigquery_magics.context.credentials = mock.create_autospec( google.auth.credentials.Credentials, instance=True ) @@ -1615,11 +1563,10 @@ def test_bigquery_magic_with_dict_params_incorrect_syntax(): @pytest.mark.usefixtures("ipython_interactive") -@pytest.mark.skipif(pandas is None, reason="Requires `pandas`") def test_bigquery_magic_with_dict_params_duplicate(): ip = IPython.get_ipython() - ip.extension_manager.load_extension("google.cloud.bigquery") - magics.context.credentials = mock.create_autospec( + ip.extension_manager.load_extension("bigquery_magics") + bigquery_magics.context.credentials = mock.create_autospec( google.auth.credentials.Credentials, instance=True ) @@ -1633,11 +1580,10 @@ def test_bigquery_magic_with_dict_params_duplicate(): @pytest.mark.usefixtures("ipython_interactive") -@pytest.mark.skipif(pandas is None, reason="Requires `pandas`") def test_bigquery_magic_with_option_value_incorrect(): ip = IPython.get_ipython() - ip.extension_manager.load_extension("google.cloud.bigquery") - magics.context.credentials = mock.create_autospec( + ip.extension_manager.load_extension("bigquery_magics") + bigquery_magics.context.credentials = mock.create_autospec( google.auth.credentials.Credentials, instance=True ) @@ -1649,11 +1595,10 @@ def test_bigquery_magic_with_option_value_incorrect(): @pytest.mark.usefixtures("ipython_interactive") -@pytest.mark.skipif(pandas is None, reason="Requires `pandas`") def test_bigquery_magic_with_dict_params_negative_value(ipython_ns_cleanup): ip = IPython.get_ipython() - ip.extension_manager.load_extension("google.cloud.bigquery") - magics.context.credentials = mock.create_autospec( + ip.extension_manager.load_extension("bigquery_magics") + bigquery_magics.context.credentials = mock.create_autospec( google.auth.credentials.Credentials, instance=True ) @@ -1664,9 +1609,7 @@ def test_bigquery_magic_with_dict_params_negative_value(ipython_ns_cleanup): assert "params_dict_df" not in ip.user_ns - run_query_patch = mock.patch( - "google.cloud.bigquery.magics.magics._run_query", autospec=True - ) + run_query_patch = mock.patch("bigquery_magics.bigquery._run_query", autospec=True) query_job_mock = mock.create_autospec( google.cloud.bigquery.job.QueryJob, instance=True ) @@ -1689,11 +1632,10 @@ def test_bigquery_magic_with_dict_params_negative_value(ipython_ns_cleanup): @pytest.mark.usefixtures("ipython_interactive") -@pytest.mark.skipif(pandas is None, reason="Requires `pandas`") def test_bigquery_magic_with_dict_params_array_value(ipython_ns_cleanup): ip = IPython.get_ipython() - ip.extension_manager.load_extension("google.cloud.bigquery") - magics.context.credentials = mock.create_autospec( + ip.extension_manager.load_extension("bigquery_magics") + bigquery_magics.context.credentials = mock.create_autospec( google.auth.credentials.Credentials, instance=True ) @@ -1704,9 +1646,7 @@ def test_bigquery_magic_with_dict_params_array_value(ipython_ns_cleanup): assert "params_dict_df" not in ip.user_ns - run_query_patch = mock.patch( - "google.cloud.bigquery.magics.magics._run_query", autospec=True - ) + run_query_patch = mock.patch("bigquery_magics.bigquery._run_query", autospec=True) query_job_mock = mock.create_autospec( google.cloud.bigquery.job.QueryJob, instance=True ) @@ -1729,11 +1669,10 @@ def test_bigquery_magic_with_dict_params_array_value(ipython_ns_cleanup): @pytest.mark.usefixtures("ipython_interactive") -@pytest.mark.skipif(pandas is None, reason="Requires `pandas`") def test_bigquery_magic_with_dict_params_tuple_value(ipython_ns_cleanup): ip = IPython.get_ipython() - ip.extension_manager.load_extension("google.cloud.bigquery") - magics.context.credentials = mock.create_autospec( + ip.extension_manager.load_extension("bigquery_magics") + bigquery_magics.context.credentials = mock.create_autospec( google.auth.credentials.Credentials, instance=True ) @@ -1744,9 +1683,7 @@ def test_bigquery_magic_with_dict_params_tuple_value(ipython_ns_cleanup): assert "params_dict_df" not in ip.user_ns - run_query_patch = mock.patch( - "google.cloud.bigquery.magics.magics._run_query", autospec=True - ) + run_query_patch = mock.patch("bigquery_magics.bigquery._run_query", autospec=True) query_job_mock = mock.create_autospec( google.cloud.bigquery.job.QueryJob, instance=True ) @@ -1769,11 +1706,10 @@ def test_bigquery_magic_with_dict_params_tuple_value(ipython_ns_cleanup): @pytest.mark.usefixtures("ipython_interactive") -@pytest.mark.skipif(pandas is None, reason="Requires `pandas`") def test_bigquery_magic_with_improperly_formatted_params(): ip = IPython.get_ipython() - ip.extension_manager.load_extension("google.cloud.bigquery") - magics.context.credentials = mock.create_autospec( + ip.extension_manager.load_extension("bigquery_magics") + bigquery_magics.context.credentials = mock.create_autospec( google.auth.credentials.Credentials, instance=True ) @@ -1787,20 +1723,17 @@ def test_bigquery_magic_with_improperly_formatted_params(): "raw_sql", ("SELECT answer AS 42", " \t SELECT answer AS 42 \t ") ) @pytest.mark.usefixtures("ipython_interactive") -@pytest.mark.skipif(pandas is None, reason="Requires `pandas`") def test_bigquery_magic_valid_query_in_existing_variable(ipython_ns_cleanup, raw_sql): ip = IPython.get_ipython() - ip.extension_manager.load_extension("google.cloud.bigquery") - magics.context.credentials = mock.create_autospec( + ip.extension_manager.load_extension("bigquery_magics") + bigquery_magics.context.credentials = mock.create_autospec( google.auth.credentials.Credentials, instance=True ) ipython_ns_cleanup.append((ip, "custom_query")) ipython_ns_cleanup.append((ip, "query_results_df")) - run_query_patch = mock.patch( - "google.cloud.bigquery.magics.magics._run_query", autospec=True - ) + run_query_patch = mock.patch("bigquery_magics.bigquery._run_query", autospec=True) query_job_mock = mock.create_autospec( google.cloud.bigquery.job.QueryJob, instance=True ) @@ -1826,17 +1759,14 @@ def test_bigquery_magic_valid_query_in_existing_variable(ipython_ns_cleanup, raw @pytest.mark.usefixtures("ipython_interactive") -@pytest.mark.skipif(pandas is None, reason="Requires `pandas`") def test_bigquery_magic_nonexisting_query_variable(): ip = IPython.get_ipython() - ip.extension_manager.load_extension("google.cloud.bigquery") - magics.context.credentials = mock.create_autospec( + ip.extension_manager.load_extension("bigquery_magics") + bigquery_magics.context.credentials = mock.create_autospec( google.auth.credentials.Credentials, instance=True ) - run_query_patch = mock.patch( - "google.cloud.bigquery.magics.magics._run_query", autospec=True - ) + run_query_patch = mock.patch("bigquery_magics.bigquery._run_query", autospec=True) ip.user_ns.pop("custom_query", None) # Make sure the variable does NOT exist. cell_body = "$custom_query" # Referring to a non-existing variable name. @@ -1850,17 +1780,14 @@ def test_bigquery_magic_nonexisting_query_variable(): @pytest.mark.usefixtures("ipython_interactive") -@pytest.mark.skipif(pandas is None, reason="Requires `pandas`") def test_bigquery_magic_empty_query_variable_name(): ip = IPython.get_ipython() - ip.extension_manager.load_extension("google.cloud.bigquery") - magics.context.credentials = mock.create_autospec( + ip.extension_manager.load_extension("bigquery_magics") + bigquery_magics.context.credentials = mock.create_autospec( google.auth.credentials.Credentials, instance=True ) - run_query_patch = mock.patch( - "google.cloud.bigquery.magics.magics._run_query", autospec=True - ) + run_query_patch = mock.patch("bigquery_magics.bigquery._run_query", autospec=True) cell_body = "$" # Not referring to any variable (name omitted). with pytest.raises( @@ -1872,17 +1799,14 @@ def test_bigquery_magic_empty_query_variable_name(): @pytest.mark.usefixtures("ipython_interactive") -@pytest.mark.skipif(pandas is None, reason="Requires `pandas`") def test_bigquery_magic_query_variable_non_string(ipython_ns_cleanup): ip = IPython.get_ipython() - ip.extension_manager.load_extension("google.cloud.bigquery") - magics.context.credentials = mock.create_autospec( + ip.extension_manager.load_extension("bigquery_magics") + bigquery_magics.context.credentials = mock.create_autospec( google.auth.credentials.Credentials, instance=True ) - run_query_patch = mock.patch( - "google.cloud.bigquery.magics.magics._run_query", autospec=True - ) + run_query_patch = mock.patch("bigquery_magics.bigquery._run_query", autospec=True) ipython_ns_cleanup.append((ip, "custom_query")) @@ -1898,11 +1822,10 @@ def test_bigquery_magic_query_variable_non_string(ipython_ns_cleanup): @pytest.mark.usefixtures("ipython_interactive") -@pytest.mark.skipif(pandas is None, reason="Requires `pandas`") def test_bigquery_magic_query_variable_not_identifier(): ip = IPython.get_ipython() - ip.extension_manager.load_extension("google.cloud.bigquery") - magics.context.credentials = mock.create_autospec( + ip.extension_manager.load_extension("bigquery_magics") + bigquery_magics.context.credentials = mock.create_autospec( google.auth.credentials.Credentials, instance=True ) @@ -1921,11 +1844,10 @@ def test_bigquery_magic_query_variable_not_identifier(): @pytest.mark.usefixtures("ipython_interactive") -@pytest.mark.skipif(pandas is None, reason="Requires `pandas`") def test_bigquery_magic_with_invalid_multiple_option_values(): ip = IPython.get_ipython() - ip.extension_manager.load_extension("google.cloud.bigquery") - magics.context.credentials = mock.create_autospec( + ip.extension_manager.load_extension("bigquery_magics") + bigquery_magics.context.credentials = mock.create_autospec( google.auth.credentials.Credentials, instance=True ) @@ -1941,7 +1863,7 @@ def test_bigquery_magic_with_invalid_multiple_option_values(): @pytest.mark.usefixtures("ipython_interactive") def test_bigquery_magic_omits_tracebacks_from_error_message(): ip = IPython.get_ipython() - ip.extension_manager.load_extension("google.cloud.bigquery") + ip.extension_manager.load_extension("bigquery_magics") credentials_mock = mock.create_autospec( google.auth.credentials.Credentials, instance=True @@ -1951,7 +1873,7 @@ def test_bigquery_magic_omits_tracebacks_from_error_message(): ) run_query_patch = mock.patch( - "google.cloud.bigquery.magics.magics._run_query", + "bigquery_magics.bigquery._run_query", autospec=True, side_effect=exceptions.BadRequest("Syntax error in SQL query"), ) @@ -1968,8 +1890,8 @@ def test_bigquery_magic_omits_tracebacks_from_error_message(): @pytest.mark.usefixtures("ipython_interactive") def test_bigquery_magic_w_destination_table_invalid_format(): ip = IPython.get_ipython() - ip.extension_manager.load_extension("google.cloud.bigquery") - magics.context._project = None + ip.extension_manager.load_extension("bigquery_magics") + bigquery_magics.context._project = None credentials_mock = mock.create_autospec( google.auth.credentials.Credentials, instance=True @@ -1978,9 +1900,7 @@ def test_bigquery_magic_w_destination_table_invalid_format(): "google.auth.default", return_value=(credentials_mock, "general-project") ) - client_patch = mock.patch( - "google.cloud.bigquery.magics.magics.bigquery.Client", autospec=True - ) + client_patch = mock.patch("bigquery_magics.bigquery.bigquery.Client", autospec=True) with client_patch, default_patch, pytest.raises(ValueError) as exc_context: ip.run_cell_magic( @@ -1996,19 +1916,17 @@ def test_bigquery_magic_w_destination_table_invalid_format(): @pytest.mark.usefixtures("ipython_interactive") def test_bigquery_magic_w_destination_table(): ip = IPython.get_ipython() - ip.extension_manager.load_extension("google.cloud.bigquery") - magics.context.credentials = mock.create_autospec( + ip.extension_manager.load_extension("bigquery_magics") + bigquery_magics.context.credentials = mock.create_autospec( google.auth.credentials.Credentials, instance=True ) create_dataset_if_necessary_patch = mock.patch( - "google.cloud.bigquery.magics.magics._create_dataset_if_necessary", + "bigquery_magics.bigquery._create_dataset_if_necessary", autospec=True, ) - run_query_patch = mock.patch( - "google.cloud.bigquery.magics.magics._run_query", autospec=True - ) + run_query_patch = mock.patch("bigquery_magics.bigquery._run_query", autospec=True) with create_dataset_if_necessary_patch, run_query_patch as run_query_mock: ip.run_cell_magic( @@ -2028,18 +1946,18 @@ def test_bigquery_magic_w_destination_table(): @pytest.mark.usefixtures("ipython_interactive") def test_bigquery_magic_create_dataset_fails(): ip = IPython.get_ipython() - ip.extension_manager.load_extension("google.cloud.bigquery") - magics.context.credentials = mock.create_autospec( + ip.extension_manager.load_extension("bigquery_magics") + bigquery_magics.context.credentials = mock.create_autospec( google.auth.credentials.Credentials, instance=True ) create_dataset_if_necessary_patch = mock.patch( - "google.cloud.bigquery.magics.magics._create_dataset_if_necessary", + "bigquery_magics.bigquery._create_dataset_if_necessary", autospec=True, side_effect=OSError, ) close_transports_patch = mock.patch( - "google.cloud.bigquery.magics.magics._close_transports", + "bigquery_magics.bigquery._close_transports", autospec=True, ) @@ -2058,14 +1976,12 @@ def test_bigquery_magic_create_dataset_fails(): @pytest.mark.usefixtures("ipython_interactive") def test_bigquery_magic_with_location(): ip = IPython.get_ipython() - ip.extension_manager.load_extension("google.cloud.bigquery") - magics.context.credentials = mock.create_autospec( + ip.extension_manager.load_extension("bigquery_magics") + bigquery_magics.context.credentials = mock.create_autospec( google.auth.credentials.Credentials, instance=True ) - run_query_patch = mock.patch( - "google.cloud.bigquery.magics.magics._run_query", autospec=True - ) + run_query_patch = mock.patch("bigquery_magics.bigquery._run_query", autospec=True) with run_query_patch as run_query_mock: ip.run_cell_magic("bigquery", "--location=us-east1", "SELECT 17 AS num")