-
Notifications
You must be signed in to change notification settings - Fork 302
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
feat: added system test and sample for dataframe contains array #365
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,40 @@ | ||
# Copyright 2020 Google LLC | ||
# | ||
# Licensed under the Apache License, Version 2.0 (the "License"); | ||
# you may not use this file except in compliance with the License. | ||
# You may obtain a copy of the License at | ||
# | ||
# http://www.apache.org/licenses/LICENSE-2.0 | ||
# | ||
# Unless required by applicable law or agreed to in writing, software | ||
# distributed under the License is distributed on an "AS IS" BASIS, | ||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
# See the License for the specific language governing permissions and | ||
# limitations under the License. | ||
|
||
|
||
def load_table_dataframe_array_contains(table_id): | ||
|
||
# [START bigquery_load_table_dataframe_array_contains] | ||
|
||
from google.cloud import bigquery | ||
import pandas | ||
|
||
# Construct a BigQuery client object. | ||
client = bigquery.Client() | ||
|
||
# TODO(developer): Set table_id to the ID of the table to create. | ||
# table_id = "your-project.your_dataset.your_table_name" | ||
|
||
dataframe = pandas.DataFrame({"A": [[1, 2, 3], [4, 5, 6], [7, 8, 9]]}) | ||
job = client.load_table_from_dataframe(dataframe, table_id) # Make an API request. | ||
job.result() # Wait for the job to complete. | ||
|
||
table = client.get_table(table_id) # Make an API request. | ||
print( | ||
"Loaded {} rows and {} columns to {}".format( | ||
table.num_rows, len(table.schema), table_id | ||
) | ||
) | ||
# [END bigquery_load_table_dataframe_array_contains] | ||
return table |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,34 @@ | ||
# Copyright 2020 Google LLC | ||
# | ||
# Licensed under the Apache License, Version 2.0 (the "License"); | ||
# you may not use this file except in compliance with the License. | ||
# You may obtain a copy of the License at | ||
# | ||
# http://www.apache.org/licenses/LICENSE-2.0 | ||
# | ||
# Unless required by applicable law or agreed to in writing, software | ||
# distributed under the License is distributed on an "AS IS" BASIS, | ||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
# See the License for the specific language governing permissions and | ||
# limitations under the License. | ||
|
||
import pytest | ||
|
||
from .. import load_table_dataframe_array_contains | ||
|
||
|
||
pandas = pytest.importorskip("pandas") | ||
pyarrow = pytest.importorskip("pyarrow", minversion="2.0.0") | ||
|
||
|
||
def test_load_table_dataframe_array_contains(capsys, random_table_id): | ||
|
||
table = load_table_dataframe_array_contains.load_table_dataframe_array_contains( | ||
random_table_id | ||
) | ||
out, _ = capsys.readouterr() | ||
expected_column_names = ["A"] | ||
assert "Loaded 3 rows and {} columns".format(len(expected_column_names)) in out | ||
|
||
column_names = [field.name for field in table.schema] | ||
assert column_names == expected_column_names |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -46,12 +46,12 @@ | |
# grpc.Channel.close() method isn't added until 1.32.0. | ||
# https://github.com/grpc/grpc/pull/15254 | ||
"grpcio >= 1.32.0, < 2.0dev", | ||
"pyarrow >= 1.0.0, < 2.0dev", | ||
"pyarrow >= 2.0.0, < 3.0dev", | ||
], | ||
"pandas": [ | ||
"pandas>=0.23.0", | ||
# pyarrow 1.0.0 is required for the use of timestamp_as_object keyword. | ||
"pyarrow >= 1.0.0, < 2.0dev", | ||
# pyarrow 2.0.0 is required for the use of arrays in dataframe to load the table . | ||
"pyarrow >= 2.0.0, < 3.0dev", | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Let's not bump the minimum version here. Most features do work with 1.0, and pyarrow is a core library that is very useful to have a wide range of support. |
||
], | ||
"tqdm": ["tqdm >= 4.7.4, <5.0.0dev"], | ||
"opentelemetry": [ | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -129,7 +129,7 @@ | |
) | ||
|
||
PANDAS_MINIMUM_VERSION = pkg_resources.parse_version("1.0.0") | ||
PYARROW_MINIMUM_VERSION = pkg_resources.parse_version("0.17.0") | ||
PYARROW_MINIMUM_VERSION = pkg_resources.parse_version("2.0.0") | ||
|
||
if pandas: | ||
PANDAS_INSTALLED_VERSION = pkg_resources.get_distribution("pandas").parsed_version | ||
|
@@ -1086,9 +1086,9 @@ def test_load_table_from_dataframe_w_explicit_schema(self): | |
|
||
@unittest.skipIf( | ||
pyarrow is None or PYARROW_INSTALLED_VERSION < PYARROW_MINIMUM_VERSION, | ||
"Only `pyarrow version >=0.17.0` is supported", | ||
"Only `pyarrow version >=2.0.0` is supported", | ||
) | ||
@unittest.skipIf(pandas is None, "Requires `pandas`") | ||
@unittest.skipIf(pandas is None, "Requires " "`pandas`") | ||
def test_load_table_from_dataframe_w_struct_datatype(self): | ||
"""Test that a DataFrame with struct datatype can be uploaded if a | ||
BigQuery schema is specified. | ||
|
@@ -1126,6 +1126,62 @@ def test_load_table_from_dataframe_w_struct_datatype(self): | |
self.assertEqual(table.schema, table_schema) | ||
self.assertEqual(table.num_rows, 3) | ||
|
||
@unittest.skipIf( | ||
pyarrow is None or PYARROW_INSTALLED_VERSION < PYARROW_MINIMUM_VERSION, | ||
"Only `pyarrow version >=2.0.0` is supported", | ||
) | ||
@unittest.skipIf(pandas is None, "Requires `pandas`") | ||
def test_load_table_from_dataframe_w_array_datatype(self): | ||
"""Test that a DataFrame contains array can be uploaded if a | ||
BigQuery without specifying a schema. | ||
|
||
https://github.com/googleapis/python-bigquery/issues/19 | ||
""" | ||
table_schema = [ | ||
bigquery.SchemaField( | ||
"A", | ||
"RECORD", | ||
"NULLABLE", | ||
None, | ||
( | ||
bigquery.SchemaField( | ||
"list", | ||
"RECORD", | ||
"REPEATED", | ||
None, | ||
( | ||
bigquery.SchemaField( | ||
"item", "INTEGER", "NULLABLE", None, (), None | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Hmm... This is a bit of a surprising schema. It appears to match the behavior we were encountering previously. This feature is not supported if we cannot upload directly to a REPEATED INTEGER column. |
||
), | ||
), | ||
None, | ||
), | ||
), | ||
None, | ||
) | ||
] | ||
dataset_id = _make_dataset_id("bq_load_test") | ||
self.temp_dataset(dataset_id) | ||
table_id = "{}.{}.load_table_from_dataframe_w_array_datatype".format( | ||
Config.CLIENT.project, dataset_id | ||
) | ||
|
||
job_config = bigquery.LoadJobConfig(autodetect=True) | ||
table = retry_403(Config.CLIENT.create_table)(Table(table_id)) | ||
self.to_delete.insert(0, table) | ||
|
||
dataframe = pandas.DataFrame({"A": [[1, 2, 3], [4, 5, 6], [7, 8, 9]]}) | ||
|
||
load_job = Config.CLIENT.load_table_from_dataframe( | ||
dataframe, table_id, job_config=job_config | ||
) | ||
load_job.result() | ||
|
||
table = Config.CLIENT.get_table(table_id) | ||
|
||
self.assertEqual(table.schema, table_schema) | ||
self.assertEqual(table.num_rows, 3) | ||
|
||
def test_load_table_from_json_basic_use(self): | ||
table_schema = ( | ||
bigquery.SchemaField("name", "STRING", mode="REQUIRED"), | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Without an explicit schema, this sample is no different from the generic load_table_from_dataframe sample.
I was imagining system test XOR sample, as they are testing the same behavior.