chore: copy magics unit tests from google-cloud-bigquery

.github/workflows/docs.yml

@@ -0,0 +1,38 @@
+on:
+  pull_request:
+    branches:
+      - main
+name: docs
+jobs:
+  docs:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+    - name: Setup Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: "3.9"
+    - name: Install nox
+      run: |
+        python -m pip install --upgrade setuptools pip wheel
+        python -m pip install nox
+    - name: Run docs
+      run: |
+        nox -s docs
+  docfx:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+    - name: Setup Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: "3.10"
+    - name: Install nox
+      run: |
+        python -m pip install --upgrade setuptools pip wheel
+        python -m pip install nox
+    - name: Run docfx
+      run: |
+        nox -s docfx

.github/workflows/lint.yml

@@ -0,0 +1,25 @@
+on:
+  pull_request:
+    branches:
+      - main
+name: lint
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+    - name: Setup Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: "3.8"
+    - name: Install nox
+      run: |
+        python -m pip install --upgrade setuptools pip wheel
+        python -m pip install nox
+    - name: Run lint
+      run: |
+        nox -s lint
+    - name: Run lint_setup_py
+      run: |
+        nox -s lint_setup_py

.github/workflows/unittest.yml

@@ -0,0 +1,57 @@
+on:
+  pull_request:
+    branches:
+      - main
+name: unittest
+jobs:
+  unit:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python: ['3.8', '3.9', '3.10', '3.11', '3.12']
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+    - name: Setup Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python }}
+    - name: Install nox
+      run: |
+        python -m pip install --upgrade setuptools pip wheel
+        python -m pip install nox
+    - name: Run unit tests
+      env:
+        COVERAGE_FILE: .coverage-${{ matrix.python }}
+      run: |
+        nox -s unit-${{ matrix.python }}
+    - name: Upload coverage results
+      uses: actions/upload-artifact@v4
+      with:
+        name: coverage-artifact-${{ matrix.python }}
+        path: .coverage-${{ matrix.python }}
+
+  cover:
+    runs-on: ubuntu-latest
+    needs:
+        - unit
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+    - name: Setup Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: "3.8"
+    - name: Install coverage
+      run: |
+        python -m pip install --upgrade setuptools pip wheel
+        python -m pip install coverage
+    - name: Download coverage results
+      uses: actions/download-artifact@v4
+      with:
+        path: .coverage-results/
+    - name: Report coverage results
+      run: |
+        find .coverage-results -type f -name '*.zip' -exec unzip {} \;
+        coverage combine .coverage-results/**/.coverage*
+        coverage report --show-missing --fail-under=96

bigquery_magics/version.py

@@ -0,0 +1,15 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+__version__ = "0.0.1"

setup.py

@@ -20,70 +20,24 @@
 
 # Package metadata.
 
-name = "google-cloud-bigquery"
-description = "Google BigQuery API client library"
+name = "bigquery-magics"
+description = "Google BigQuery magics for Jupyter and IPython"
 
 # Should be one of:
 # 'Development Status :: 3 - Alpha'
 # 'Development Status :: 4 - Beta'
-# 'Development Status :: 5 - Production/Stable'
-release_status = "Development Status :: 5 - Production/Stable"
+# 'Development Status :: 5 - Production/Stable'``
+release_status = "Development Status :: 4 - Beta"
 dependencies = [
-    # NOTE: Maintainers, please do not require google-api-core>=2.x.x
-    # Until this issue is closed
-    # https://github.com/googleapis/google-cloud-python/issues/10566
-    "google-api-core >= 1.31.5, <3.0.0dev,!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.0",
-    # NOTE: Maintainers, please do not require google-cloud-core>=2.x.x
-    # Until this issue is closed
-    # https://github.com/googleapis/google-cloud-python/issues/10566
-    "google-cloud-core >= 1.6.0, <3.0.0dev",
-    "google-resumable-media >= 0.6.0, < 3.0dev",
-    "packaging >= 20.0.0",
-    "python-dateutil >= 2.7.2, <3.0dev",
-    "requests >= 2.21.0, < 3.0.0dev",
+    "db-dtypes>=0.3.0,<2.0.0dev",
+    "google-cloud-bigquery >= 3.0.0, <4.0.0dev",
+    "ipywidgets>=7.7.1",
+    "ipython>=7.23.1",
+    "ipykernel>=6.0.0",
+    "pandas>=1.1.0",
+    "pyarrow >= 3.0.0",
 ]
-pyarrow_dependency = "pyarrow >= 3.0.0"
-extras = {
-    # Keep the no-op bqstorage extra for backward compatibility.
-    # See: https://github.com/googleapis/python-bigquery/issues/757
-    "bqstorage": [
-        "google-cloud-bigquery-storage >= 2.6.0, <3.0.0dev",
-        # Due to an issue in pip's dependency resolver, the `grpc` extra is not
-        # installed, even though `google-cloud-bigquery-storage` specifies it
-        # as `google-api-core[grpc]`. We thus need to explicitly specify it here.
-        # See: https://github.com/googleapis/python-bigquery/issues/83 The
-        # grpc.Channel.close() method isn't added until 1.32.0.
-        # https://github.com/grpc/grpc/pull/15254
-        "grpcio >= 1.47.0, < 2.0dev",
-        "grpcio >= 1.49.1, < 2.0dev; python_version>='3.11'",
-        pyarrow_dependency,
-    ],
-    "pandas": [
-        "pandas>=1.1.0",
-        pyarrow_dependency,
-        "db-dtypes>=0.3.0,<2.0.0dev",
-        "importlib_metadata>=1.0.0; python_version<'3.8'",
-    ],
-    "ipywidgets": [
-        "ipywidgets>=7.7.0",
-        "ipykernel>=6.0.0",
-    ],
-    "geopandas": ["geopandas>=0.9.0, <1.0dev", "Shapely>=1.8.4, <3.0.0dev"],
-    "ipython": [
-        "ipython>=7.23.1,!=8.1.0",
-        "ipykernel>=6.0.0",
-    ],
-    "tqdm": ["tqdm >= 4.7.4, <5.0.0dev"],
-    "opentelemetry": [
-        "opentelemetry-api >= 1.1.0",
-        "opentelemetry-sdk >= 1.1.0",
-        "opentelemetry-instrumentation >= 0.20b0",
-    ],
-    "bigquery_v2": [
-        "proto-plus >= 1.15.0, <2.0.0dev",
-        "protobuf>=3.19.5,<5.0.0dev,!=3.20.0,!=3.20.1,!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5",  # For the legacy proto-based types.
-    ],
-}
+extras = {}
 
 all_extras = []
 
@@ -101,16 +55,16 @@
     readme = readme_file.read()
 
 version = {}
-with open(os.path.join(package_root, "google/cloud/bigquery/version.py")) as fp:
+with open(os.path.join(package_root, "bigquery_magics/version.py")) as fp:
     exec(fp.read(), version)
 version = version["__version__"]
 
-# Only include packages under the 'google' namespace. Do not include tests,
+# Only include packages under the 'bigquery_magics' namespace. Do not include tests,
 # benchmarks, etc.
 packages = [
     package
     for package in setuptools.find_namespace_packages()
-    if package.startswith("google")
+    if package.startswith("bigquery_magics")
 ]
 
 setuptools.setup(
@@ -121,7 +75,7 @@
     author="Google LLC",
     author_email="googleapis-packages@google.com",
     license="Apache 2.0",
-    url="https://github.com/googleapis/python-bigquery",
+    url="https://github.com/googleapis/python-bigquery-magics",
     classifiers=[
         release_status,
         "Intended Audience :: Developers",

testing/constraints-3.10.txt

@@ -1 +1,2 @@
-grpcio==1.47.0
+# IMPORTANT: When Python 3.9 support is dropped, update these to
+# match the minimums in setup.py.

testing/constraints-3.11.txt

@@ -0,0 +1,2 @@
+# IMPORTANT: When Python 3.10 support is dropped, update these to
+# match the minimums in setup.py.

testing/constraints-3.12.txt

@@ -0,0 +1,2 @@
+# IMPORTANT: When Python 3.11 support is dropped, update these to
+# match the minimums in setup.py.

testing/constraints-3.7.txt

@@ -6,25 +6,9 @@
 # e.g., if setup.py has "foo >= 1.14.0, < 2.0.0dev",
 # Then this file should have foo==1.14.0
 db-dtypes==0.3.0
-geopandas==0.9.0
-google-api-core==1.31.5
-google-cloud-bigquery-storage==2.6.0
-google-cloud-core==1.6.0
-google-resumable-media==0.6.0
-grpcio==1.47.0
+google-cloud-bigquery==3.0.0
 ipywidgets==7.7.1
 ipython==7.23.1
 ipykernel==6.0.0
-opentelemetry-api==1.1.0
-opentelemetry-instrumentation==0.20b0
-opentelemetry-sdk==1.1.0
-packaging==20.0.0
 pandas==1.1.0
-proto-plus==1.22.0
-protobuf==3.19.5
 pyarrow==3.0.0
-python-dateutil==2.7.3
-requests==2.21.0
-Shapely==1.8.4
-six==1.13.0
-tqdm==4.7.4

testing/constraints-3.8.txt

@@ -1,2 +1,8 @@
-grpcio==1.47.0
-pandas==1.2.0
+# IMPORTANT: When Python 3.7 support is dropped, update these to
+# match the minimums in setup.py.
+#
+# We try to test across major versions of our dependencies.
+# This is the last ipython 7.x release
+ipython==7.34.0
+# This is the last pandas 1.5.x release.
+pandas==1.5.3

testing/constraints-3.9.txt

@@ -1,8 +1,7 @@
-# This constraints file is used to make sure that the latest dependency versions
-# we claim to support in setup.py are indeed installed in test sessions in the most
-# recent Python version supported (3.9 at the time of writing - 2021-05-05).
+# IMPORTANT: When Python 3.8 support is dropped, update these to
+# match the minimums in setup.py.
 #
-# NOTE: Not comprehensive yet, will eventually be maintained semi-automatically by
-# the renovate bot.
-grpcio==1.47.0
-pyarrow>=4.0.0
+# We try to test across major versions of our dependencies.
+# This is the last pandas 2.0.x release.
+pandas==2.0.3
+

tests/system/test_bigquery.py

@@ -0,0 +1,82 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""System tests for Jupyter/IPython connector."""
+
+import re
+
+import pytest
+import psutil
+
+
+IPython = pytest.importorskip("IPython")
+io = pytest.importorskip("IPython.utils.io")
+pandas = pytest.importorskip("pandas")
+tools = pytest.importorskip("IPython.testing.tools")
+interactiveshell = pytest.importorskip("IPython.terminal.interactiveshell")
+
+
+@pytest.fixture(scope="session")
+def ipython():
+    config = tools.default_config()
+    config.TerminalInteractiveShell.simple_prompt = True
+    shell = interactiveshell.TerminalInteractiveShell.instance(config=config)
+    return shell
+
+
+@pytest.fixture()
+def ipython_interactive(ipython):
+    """Activate IPython's builtin hooks
+
+    for the duration of the test scope.
+    """
+    with ipython.builtin_trap:
+        yield ipython
+
+
+def test_bigquery_magic(ipython_interactive):
+    ip = IPython.get_ipython()
+    current_process = psutil.Process()
+    conn_count_start = len(current_process.connections())
+
+    ip.extension_manager.load_extension("google.cloud.bigquery")
+    sql = """
+        SELECT
+            CONCAT(
+            'https://stackoverflow.com/questions/',
+            CAST(id as STRING)) as url,
+            view_count
+        FROM `bigquery-public-data.stackoverflow.posts_questions`
+        WHERE tags like '%google-bigquery%'
+        ORDER BY view_count DESC
+        LIMIT 10
+    """
+    with io.capture_output() as captured:
+        result = ip.run_cell_magic("bigquery", "--use_rest_api", sql)
+
+    conn_count_end = len(current_process.connections())
+
+    lines = re.split("\n|\r", captured.stdout)
+    # Removes blanks & terminal code (result of display clearing)
+    updates = list(filter(lambda x: bool(x) and x != "\x1b[2K", lines))
+    assert re.match("Executing query with job ID: .*", updates[0])
+    assert (re.match("Query executing: .*s", line) for line in updates[1:-1])
+    assert isinstance(result, pandas.DataFrame)
+    assert len(result) == 10  # verify row count
+    assert list(result) == ["url", "view_count"]  # verify column names
+
+    # NOTE: For some reason, the number of open sockets is sometimes one *less*
+    # than expected when running system tests on Kokoro, thus using the <= assertion.
+    # That's still fine, however, since the sockets are apparently not leaked.
+    assert conn_count_end <= conn_count_start  # system resources are released