From e169ec9559de35d62e74fcf0b3cf770cc631289c Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Thu, 20 Jun 2019 14:28:44 -0700 Subject: [PATCH 1/6] Add Routines API. Adds support for managing permanent functions in BigQuery, such as scalar UDFs and stored procedures. At present, only scalar UDF functionality is available. Routines are registered as resources inside of datasets, and allow expected CRUD operations. Currently, routines do not support partial updates. See: https://cloud.google.com/bigquery/docs/reference/rest/v2/routines --- bigquery/docs/reference.rst | 10 + bigquery/google/cloud/bigquery/__init__.py | 7 + bigquery/google/cloud/bigquery/client.py | 220 +++++++- bigquery/google/cloud/bigquery/routine.py | 490 ++++++++++++++++++ bigquery/samples/create_routine.py | 46 ++ bigquery/samples/create_routine_ddl.py | 46 ++ bigquery/samples/delete_routine.py | 30 ++ bigquery/samples/get_routine.py | 39 ++ bigquery/samples/list_routines.py | 34 ++ bigquery/samples/tests/conftest.py | 35 ++ bigquery/samples/tests/test_create_table.py | 1 - .../samples/tests/test_routine_samples.py | 87 ++++ bigquery/samples/update_routine.py | 43 ++ bigquery/tests/unit/test_client.py | 28 + 14 files changed, 1113 insertions(+), 3 deletions(-) create mode 100644 bigquery/google/cloud/bigquery/routine.py create mode 100644 bigquery/samples/create_routine.py create mode 100644 bigquery/samples/create_routine_ddl.py create mode 100644 bigquery/samples/delete_routine.py create mode 100644 bigquery/samples/get_routine.py create mode 100644 bigquery/samples/list_routines.py create mode 100644 bigquery/samples/tests/test_routine_samples.py create mode 100644 bigquery/samples/update_routine.py diff --git a/bigquery/docs/reference.rst b/bigquery/docs/reference.rst index b3f949e3daab..39b3e8407d30 100644 --- a/bigquery/docs/reference.rst +++ b/bigquery/docs/reference.rst @@ -101,6 +101,16 @@ Model model.Model model.ModelReference +Routine +======= + +.. autosummary:: + :toctree: generated + + routine.Routine + routine.RoutineArgument + routine.RoutineReference + Schema ====== diff --git a/bigquery/google/cloud/bigquery/__init__.py b/bigquery/google/cloud/bigquery/__init__.py index 0b972bb7297b..b84051fc6be1 100644 --- a/bigquery/google/cloud/bigquery/__init__.py +++ b/bigquery/google/cloud/bigquery/__init__.py @@ -67,6 +67,9 @@ from google.cloud.bigquery.query import StructQueryParameter from google.cloud.bigquery.query import UDFResource from google.cloud.bigquery.retry import DEFAULT_RETRY +from google.cloud.bigquery.routine import Routine +from google.cloud.bigquery.routine import RoutineArgument +from google.cloud.bigquery.routine import RoutineReference from google.cloud.bigquery.schema import SchemaField from google.cloud.bigquery.table import EncryptionConfiguration from google.cloud.bigquery.table import Table @@ -105,6 +108,10 @@ # Models "Model", "ModelReference", + # Routines + "Routine", + "RoutineArgument", + "RoutineReference", # Shared helpers "SchemaField", "UDFResource", diff --git a/bigquery/google/cloud/bigquery/client.py b/bigquery/google/cloud/bigquery/client.py index 65d6915c7ea2..acb7e3383664 100644 --- a/bigquery/google/cloud/bigquery/client.py +++ b/bigquery/google/cloud/bigquery/client.py @@ -58,6 +58,8 @@ from google.cloud.bigquery.model import ModelReference from google.cloud.bigquery.query import _QueryResults from google.cloud.bigquery.retry import DEFAULT_RETRY +from google.cloud.bigquery.routine import Routine +from google.cloud.bigquery.routine import RoutineReference from google.cloud.bigquery.schema import SchemaField from google.cloud.bigquery.table import _table_arg_to_table from google.cloud.bigquery.table import _table_arg_to_table_ref @@ -374,6 +376,41 @@ def create_dataset(self, dataset, exists_ok=False, retry=DEFAULT_RETRY): raise return self.get_dataset(dataset.reference, retry=retry) + def create_routine(self, routine, exists_ok=False, retry=DEFAULT_RETRY): + """[Beta] Create a routine via a POST request. + + See + https://cloud.google.com/bigquery/docs/reference/rest/v2/routines/insert + + Args: + routine (:class:`~google.cloud.bigquery.routine.Routine`): + A :class:`~google.cloud.bigquery.routine.Routine` to create. + The dataset that the routine belongs to must already exist. + exists_ok (bool): + Defaults to ``False``. If ``True``, ignore "already exists" + errors when creating the routine. + retry (google.api_core.retry.Retry): + Optional. How to retry the RPC. + + Returns: + google.cloud.bigquery.routine.Routine: + A new ``Routine`` returned from the service. + """ + reference = routine.reference + path = "/projects/{}/datasets/{}/routines".format( + reference.project, reference.dataset_id + ) + resource = routine.to_api_repr() + try: + api_response = self._call_api( + retry, method="POST", path=path, data=resource + ) + return Routine.from_api_repr(api_response) + except google.api_core.exceptions.Conflict: + if not exists_ok: + raise + return self.get_routine(routine.reference, retry=retry) + def create_table(self, table, exists_ok=False, retry=DEFAULT_RETRY): """API call: create a table via a PUT request @@ -472,6 +509,34 @@ def get_model(self, model_ref, retry=DEFAULT_RETRY): api_response = self._call_api(retry, method="GET", path=model_ref.path) return Model.from_api_repr(api_response) + def get_routine(self, routine_ref, retry=DEFAULT_RETRY): + """[Beta] Get the routine referenced by ``routine_ref``. + + Args: + routine_ref (Union[ \ + :class:`~google.cloud.bigquery.routine.Routine`, \ + :class:`~google.cloud.bigquery.routine.RoutineReference`, \ + str, \ + ]): + A reference to the routine to fetch from the BigQuery API. If + a string is passed in, this method attempts to create a + reference from a string using + :func:`google.cloud.bigquery.routine.RoutineReference.from_string`. + retry (:class:`google.api_core.retry.Retry`): + (Optional) How to retry the API call. + + Returns: + google.cloud.bigquery.routine.Routine: + A ``Routine`` instance. + """ + if isinstance(routine_ref, str): + routine_ref = RoutineReference.from_string( + routine_ref, default_project=self.project + ) + + api_response = self._call_api(retry, method="GET", path=routine_ref.path) + return Routine.from_api_repr(api_response) + def get_table(self, table, retry=DEFAULT_RETRY): """Fetch the table referenced by ``table``. @@ -537,7 +602,7 @@ def update_model(self, model, fields, retry=DEFAULT_RETRY): Use ``fields`` to specify which fields to update. At least one field must be provided. If a field is listed in ``fields`` and is ``None`` - in ``model``, it will be deleted. + in ``model``, the field value will be deleted. If ``model.etag`` is not ``None``, the update will only succeed if the model on the server has the same ETag. Thus reading a model with @@ -567,12 +632,54 @@ def update_model(self, model, fields, retry=DEFAULT_RETRY): ) return Model.from_api_repr(api_response) + def update_routine(self, routine, fields, retry=DEFAULT_RETRY): + """[Beta] Change some fields of a routine. + + Use ``fields`` to specify which fields to update. At least one field + must be provided. If a field is listed in ``fields`` and is ``None`` + in ``routine``, the field value will be deleted. + + If :attr:`~google.cloud.bigquery.routine.Routine.etag` is not + ``None``, the update will only succeed if the resource on the server + has the same ETag. Thus reading a routine with + :func:`~google.cloud.bigquery.client.Client.get_routine`, changing + its fields, and then passing it to this method will ensure that the + changes will only be saved if no modifications to the resource + occurred since the read. + + Args: + routine (google.cloud.bigquery.routine.Routine): The routine to update. + fields (Sequence[str]): + The fields of ``routine`` to change, spelled as the + :class:`~google.cloud.bigquery.routine.Routine` properties + (e.g. ``type_``). + retry (google.api_core.retry.Retry): + (Optional) A description of how to retry the API call. + + Returns: + google.cloud.bigquery.routine.Routine: + The routine resource returned from the API call. + """ + partial = routine._build_resource(fields) + if routine.etag: + headers = {"If-Match": routine.etag} + else: + headers = None + + # TODO: remove when routines update supports partial requests. + partial["routineReference"] = routine.reference.to_api_repr() + + api_response = self._call_api( + retry, method="PUT", path=routine.path, data=partial, headers=headers + ) + return Routine.from_api_repr(api_response) + def update_table(self, table, fields, retry=DEFAULT_RETRY): """Change some fields of a table. Use ``fields`` to specify which fields to update. At least one field must be provided. If a field is listed in ``fields`` and is ``None`` - in ``table``, it will be deleted. + in ``table``, the field value will be deleted. If ``table.etag`` is not ``None``, the update will only succeed if the table on the server has the same ETag. Thus reading a table with @@ -660,6 +767,64 @@ def list_models( result.dataset = dataset return result + def list_routines( + self, dataset, max_results=None, page_token=None, retry=DEFAULT_RETRY + ): + """[Beta] List routines in the dataset. + + See + https://cloud.google.com/bigquery/docs/reference/rest/v2/routines/list + + Args: + dataset (Union[ \ + :class:`~google.cloud.bigquery.dataset.Dataset`, \ + :class:`~google.cloud.bigquery.dataset.DatasetReference`, \ + str, \ + ]): + A reference to the dataset whose routines to list from the + BigQuery API. If a string is passed in, this method attempts + to create a dataset reference from a string using + :func:`google.cloud.bigquery.dataset.DatasetReference.from_string`. + max_results (int): + (Optional) Maximum number of routines to return. If not passed, + defaults to a value set by the API. + page_token (str): + (Optional) Token representing a cursor into the routines. If + not passed, the API will return the first page of routines. The + token marks the beginning of the iterator to be returned and + the value of the ``page_token`` can be accessed at + ``next_page_token`` of the + :class:`~google.api_core.page_iterator.HTTPIterator`. + retry (:class:`google.api_core.retry.Retry`): + (Optional) How to retry the RPC. + + Returns: + google.api_core.page_iterator.Iterator: + Iterator of all + :class:`~google.cloud.bigquery.routine.Routine`s contained + within the requested dataset, limited by ``max_results``. + """ + if isinstance(dataset, str): + dataset = DatasetReference.from_string( + dataset, default_project=self.project + ) + + if not isinstance(dataset, (Dataset, DatasetReference)): + raise TypeError("dataset must be a Dataset, DatasetReference, or string") + + path = "{}/routines".format(dataset.path) + result = page_iterator.HTTPIterator( + client=self, + api_request=functools.partial(self._call_api, retry), + path=path, + item_to_value=_item_to_routine, + items_key="routines", + page_token=page_token, + max_results=max_results, + ) + result.dataset = dataset + return result + def list_tables( self, dataset, max_results=None, page_token=None, retry=DEFAULT_RETRY ): @@ -800,6 +965,42 @@ def delete_model(self, model, retry=DEFAULT_RETRY, not_found_ok=False): if not not_found_ok: raise + def delete_routine(self, routine, retry=DEFAULT_RETRY, not_found_ok=False): + """[Beta] Delete a routine. + + See + https://cloud.google.com/bigquery/docs/reference/rest/v2/routines/delete + + Args: + model (Union[ \ + :class:`~google.cloud.bigquery.routine.Routine`, \ + :class:`~google.cloud.bigquery.routine.RoutineReference`, \ + str, \ + ]): + A reference to the routine to delete. If a string is passed + in, this method attempts to create a routine reference from a + string using + :func:`google.cloud.bigquery.routine.RoutineReference.from_string`. + retry (:class:`google.api_core.retry.Retry`): + (Optional) How to retry the RPC. + not_found_ok (bool): + Defaults to ``False``. If ``True``, ignore "not found" errors + when deleting the routine. + """ + if isinstance(routine, str): + routine = RoutineReference.from_string( + routine, default_project=self.project + ) + + if not isinstance(routine, (Routine, RoutineReference)): + raise TypeError("routine must be a Routine or a RoutineReference") + + try: + self._call_api(retry, method="DELETE", path=routine.path) + except google.api_core.exceptions.NotFound: + if not not_found_ok: + raise + def delete_table(self, table, retry=DEFAULT_RETRY, not_found_ok=False): """Delete a table @@ -2073,6 +2274,21 @@ def _item_to_model(iterator, resource): return Model.from_api_repr(resource) +def _item_to_routine(iterator, resource): + """Convert a JSON model to the native object. + + Args: + iterator (google.api_core.page_iterator.Iterator): + The iterator that is currently in use. + resource (dict): + An item to be converted to a routine. + + Returns: + google.cloud.bigquery.routine.Routine: The next routine in the page. + """ + return Routine.from_api_repr(resource) + + def _item_to_table(iterator, resource): """Convert a JSON table to the native object. diff --git a/bigquery/google/cloud/bigquery/routine.py b/bigquery/google/cloud/bigquery/routine.py new file mode 100644 index 000000000000..c7136d78c422 --- /dev/null +++ b/bigquery/google/cloud/bigquery/routine.py @@ -0,0 +1,490 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Define resources for the BigQuery Routines API.""" + +from google.protobuf import json_format +import six + +import google.cloud._helpers +from google.cloud.bigquery import _helpers +import google.cloud.bigquery_v2.types + + +class Routine(object): + """Resource representing a user-defined routine. + + See + https://cloud.google.com/bigquery/docs/reference/rest/v2/routines + + Args: + routine_ref (Union[ \ + str, \ + google.cloud.bigquery.routine.RoutineReference, \ + ]): + A pointer to a routine. If ``routine_ref`` is a string, it must + included a project ID, dataset ID, and routine ID, each separated + by ``.``. + ``**kwargs`` (Dict): + Initial property values. + """ + + _PROPERTY_TO_API_FIELD = { + "arguments": "arguments", + "body": "definitionBody", + "created": "creationTime", + "etag": "etag", + "imported_libraries": "importedLibraries", + "language": "language", + "modified": "lastModifiedTime", + "reference": "routineReference", + "return_type": "returnType", + "type_": "routineType", + } + + def __init__(self, routine_ref, **kwargs): + if isinstance(routine_ref, six.string_types): + routine_ref = RoutineReference.from_string(routine_ref) + + self._properties = {"routineReference": routine_ref.to_api_repr()} + for property_name in kwargs: + setattr(self, property_name, kwargs[property_name]) + + @property + def reference(self): + """google.cloud.bigquery.routine.RoutineReference: Reference + describing the ID of this routine. + """ + return RoutineReference.from_api_repr( + self._properties[self._PROPERTY_TO_API_FIELD["reference"]] + ) + + @property + def path(self): + """str: URL path for the routine's APIs.""" + return self.reference.path + + @property + def project(self): + """str: ID of the project containing the routine.""" + return self.reference.project + + @property + def dataset_id(self): + """str: ID of dataset containing the routine.""" + return self.reference.dataset_id + + @property + def routine_id(self): + """str: The routine ID.""" + return self.reference.routine_id + + @property + def etag(self): + """str: ETag for the resource (:data:`None` until set from the + server). + + Read-only. + """ + return self._properties.get(self._PROPERTY_TO_API_FIELD["etag"]) + + @property + def type_(self): + """str: The fine-grained type of the routine. + + See: + https://cloud.google.com/bigquery/docs/reference/rest/v2/routines#RoutineType + """ + return self._properties.get(self._PROPERTY_TO_API_FIELD["type_"]) + + @type_.setter + def type_(self, value): + self._properties[self._PROPERTY_TO_API_FIELD["type_"]] = value + + @property + def created(self): + """Optional[datetime.datetime]: Datetime at which the routine was + created (:data:`None` until set from the server). + + Read-only. + """ + value = self._properties.get(self._PROPERTY_TO_API_FIELD["created"]) + if value is not None and value != 0: + # value will be in milliseconds. + return google.cloud._helpers._datetime_from_microseconds( + 1000.0 * float(value) + ) + + @property + def modified(self): + """Optional[datetime.datetime]: Datetime at which the routine was + last modified (:data:`None` until set from the server). + + Read-only. + """ + value = self._properties.get(self._PROPERTY_TO_API_FIELD["modified"]) + if value is not None and value != 0: + # value will be in milliseconds. + return google.cloud._helpers._datetime_from_microseconds( + 1000.0 * float(value) + ) + + @property + def language(self): + """Optional[str]: The language of the routine. + + Defaults to ``SQL``. + """ + return self._properties.get(self._PROPERTY_TO_API_FIELD["language"]) + + @language.setter + def language(self, value): + self._properties[self._PROPERTY_TO_API_FIELD["language"]] = value + + @property + def arguments(self): + """List[google.cloud.bigquery.routine.RoutineArgument]: Input/output + argument of a function or a stored procedure. + + In-place modification is not supported. To set, replace the entire + property value with the modified list of + :class:`~google.cloud.bigquery.routine.RoutineArgument` objects. + """ + resources = self._properties.get(self._PROPERTY_TO_API_FIELD["arguments"], []) + return [RoutineArgument.from_api_repr(resource) for resource in resources] + + @arguments.setter + def arguments(self, value): + if not value: + resource = [] + else: + resource = [argument.to_api_repr() for argument in value] + self._properties[self._PROPERTY_TO_API_FIELD["arguments"]] = resource + + @property + def return_type(self): + """google.cloud.bigquery_v2.types.StandardSqlDataType: Return type of + the routine. + + If absent, the return type is inferred from + :attr:`~google.cloud.bigquery.routine.Routine.body` at query time in + each query that references this routine. If present, then the + evaluated result will be cast to the specified returned type at query + time. + + See: + https://cloud.google.com/bigquery/docs/reference/rest/v2/routines#resource-routine + """ + resource = self._properties.get(self._PROPERTY_TO_API_FIELD["return_type"]) + output = google.cloud.bigquery_v2.types.StandardSqlDataType() + output = json_format.ParseDict(resource, output, ignore_unknown_fields=True) + return output + + @return_type.setter + def return_type(self, value): + self._properties[ + self._PROPERTY_TO_API_FIELD["return_type"] + ] = json_format.MessageToDict(value) + + @property + def imported_libraries(self): + """List[str]: The path of the imported JavaScript libraries. + + The :attr:`~google.cloud.bigquery.routine.Routine.language` must + equal ``JAVACRIPT``. + """ + return self._properties.get( + self._PROPERTY_TO_API_FIELD["imported_libraries"], [] + ) + + @imported_libraries.setter + def imported_libraries(self, value): + if not value: + resource = [] + else: + resource = value + + self._properties[self._PROPERTY_TO_API_FIELD["return_type"]] = resource + + @property + def body(self): + """str: The body of the routine.""" + return self._properties.get(self._PROPERTY_TO_API_FIELD["body"]) + + @body.setter + def body(self, value): + self._properties[self._PROPERTY_TO_API_FIELD["body"]] = value + + @classmethod + def from_api_repr(cls, resource): + """Factory: construct a routine given its API representation. + + Args: + resource (Dict[str, object]): + Resource, as returned from the API. + + Returns: + google.cloud.bigquery.routine.Routine: + Python object, as parsed from ``resource``. + """ + ref = cls(RoutineReference.from_api_repr(resource["routineReference"])) + ref._properties = resource + return ref + + def to_api_repr(self): + """Construct the API resource representation of this routine. + + Returns: + Dict[str, object]: + Routine represented as an API resource. + """ + return self._properties + + def _build_resource(self, filter_fields): + """Generate a resource for ``update``.""" + return _helpers._build_resource_from_properties(self, filter_fields) + + +class RoutineArgument(object): + """Input/output argument of a function or a stored procedure. + + See + https://cloud.google.com/bigquery/docs/reference/rest/v2/routines + + Args: + ``**kwargs`` (Dict): + Initial property values. + """ + + _PROPERTY_TO_API_FIELD = { + "data_type": "dataType", + "kind": "argumentKind", + # Even though it's not necessary for field mapping to map when the + # property name equals the resource name, we add these here so that we + # have an exhaustive list of all properties. + "name": "name", + "mode": "mode", + } + + def __init__(self, **kwargs): + self._properties = {} + for property_name in kwargs: + setattr(self, property_name, kwargs[property_name]) + + @property + def name(self): + """Optional[str]: Name of this argument. + + Can be absent for function return argument. + """ + return self._properties.get(self._PROPERTY_TO_API_FIELD["name"]) + + @name.setter + def name(self, value): + self._properties[self._PROPERTY_TO_API_FIELD["name"]] = value + + @property + def kind(self): + """Optional[str]: The kind of argument, for example ``FIXED_TYPE`` or + ``ANY_TYPE``. + + See: + https://cloud.google.com/bigquery/docs/reference/rest/v2/routines#ArgumentKind + """ + return self._properties.get(self._PROPERTY_TO_API_FIELD["kind"]) + + @kind.setter + def kind(self, value): + self._properties[self._PROPERTY_TO_API_FIELD["kind"]] = value + + @property + def mode(self): + """Optional[str]: The input/output mode of the argument.""" + return self._properties.get(self._PROPERTY_TO_API_FIELD["mode"]) + + @mode.setter + def mode(self, value): + self._properties[self._PROPERTY_TO_API_FIELD["mode"]] = value + + @property + def data_type(self): + """Optional[google.cloud.bigquery_v2.types.StandardSqlDataType]: Type + of a variable, e.g., a function argument. + + See: + https://cloud.google.com/bigquery/docs/reference/rest/v2/StandardSqlDataType + """ + resource = self._properties.get(self._PROPERTY_TO_API_FIELD["data_type"]) + output = google.cloud.bigquery_v2.types.StandardSqlDataType() + output = json_format.ParseDict(resource, output, ignore_unknown_fields=True) + return output + + @data_type.setter + def data_type(self, value): + self._properties[ + self._PROPERTY_TO_API_FIELD["data_type"] + ] = json_format.MessageToDict(value) + + @classmethod + def from_api_repr(cls, resource): + """Factory: construct a routine argument given its API representation. + + Args: + resource (Dict[str, object]): + Resource, as returned from the API. + + Returns: + google.cloud.bigquery.routine.RoutineArgument: + Python object, as parsed from ``resource``. + """ + ref = cls() + ref._properties = resource + return ref + + def to_api_repr(self): + """Construct the API resource representation of this routine argument. + + Returns: + Dict[str, object]: + Routine argument represented as an API resource. + """ + return self._properties + + def __eq__(self, other): + if not isinstance(other, RoutineArgument): + return NotImplemented + return self._properties == other._properties + + def __ne__(self, other): + return not self == other + + def __repr__(self): + all_properties = [ + "{}={}".format(property_name, repr(getattr(self, property_name))) + for property_name in self._PROPERTY_TO_API_FIELD + ] + return "RoutineArgument({})".format(", ".join(all_properties)) + + +class RoutineReference(object): + """A pointer to a routine. + + See + https://cloud.google.com/bigquery/docs/reference/rest/v2/routines + """ + + def __init__(self): + self._properties = {} + + @property + def project(self): + """str: ID of the project containing the routine.""" + return self._properties["projectId"] + + @property + def dataset_id(self): + """str: ID of dataset containing the routine.""" + return self._properties["datasetId"] + + @property + def routine_id(self): + """str: The routine ID.""" + return self._properties["routineId"] + + @property + def path(self): + """str: URL path for the routine's APIs.""" + return "/projects/%s/datasets/%s/routines/%s" % ( + self.project, + self.dataset_id, + self.routine_id, + ) + + @classmethod + def from_api_repr(cls, resource): + """Factory: construct a routine reference given its API representation. + + Args: + resource (Dict[str, object]): + Routine reference representation returned from the API. + + Returns: + google.cloud.bigquery.routine.RoutineReference: + Routine reference parsed from ``resource``. + """ + ref = cls() + ref._properties = resource + return ref + + @classmethod + def from_string(cls, routine_id, default_project=None): + """Factory: construct a routine reference from routine ID string. + + Args: + routine_id (str): + A routine ID in standard SQL format. If ``default_project`` + is not specified, this must included a project ID, dataset + ID, and routine ID, each separated by ``.``. + default_project (str): + Optional. The project ID to use when ``routine_id`` does not + include a project ID. + + Returns: + google.cloud.bigquery.routine.RoutineReference: + Routine reference parsed from ``routine_id``. + + Raises: + ValueError: + If ``routine_id`` is not a fully-qualified routine ID in + standard SQL format. + """ + proj, dset, routine = _helpers._parse_3_part_id( + routine_id, default_project=default_project, property_name="routine_id" + ) + return cls.from_api_repr( + {"projectId": proj, "datasetId": dset, "routineId": routine} + ) + + def to_api_repr(self): + """Construct the API resource representation of this routine reference. + + Returns: + Dict[str, object]: + Routine reference represented as an API resource. + """ + return self._properties + + def __eq__(self, other): + """Two RoutineReferences are equal if they point to the same routine.""" + if not isinstance(other, RoutineReference): + return NotImplemented + return str(self) == str(other) + + def __hash__(self): + return hash(str(self)) + + def __ne__(self, other): + return not self == other + + def __repr__(self): + return "RoutineReference.from_string('{}')".format(str(self)) + + def __str__(self): + """String representation of the reference. + + This is a fully-qualified ID, including the project ID and dataset ID. + """ + return "{}.{}.{}".format(self.project, self.dataset_id, self.routine_id) diff --git a/bigquery/samples/create_routine.py b/bigquery/samples/create_routine.py new file mode 100644 index 000000000000..18b999980d72 --- /dev/null +++ b/bigquery/samples/create_routine.py @@ -0,0 +1,46 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def main(client, routine_id): + # [START bigquery_create_routine] + from google.cloud import bigquery + from google.cloud import bigquery_v2 + + # TODO(developer): Construct a BigQuery client object. + # client = bigquery.Client() + + # TODO(developer): Choose a fully-qualified ID for the routine. + # routine_id = "my-project.my_dataset.my_routine" + + routine = bigquery.Routine( + routine_id, + type_="SCALAR_FUNCTION", + language="SQL", + body="x * 3", + arguments=[ + bigquery.RoutineArgument( + name="x", + data_type=bigquery_v2.types.StandardSqlDataType( + type_kind=bigquery_v2.enums.StandardSqlDataType.TypeKind.INT64 + ), + ) + ], + ) + + routine = client.create_routine(routine) + + print("Created routine {}".format(routine.reference)) + # [END bigquery_create_routine] + return routine diff --git a/bigquery/samples/create_routine_ddl.py b/bigquery/samples/create_routine_ddl.py new file mode 100644 index 000000000000..50f50f7d635d --- /dev/null +++ b/bigquery/samples/create_routine_ddl.py @@ -0,0 +1,46 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def main(client, routine_id): + # [START bigquery_create_routine_ddl] + # TODO(developer): Import the client library. + # from google.cloud import bigquery + + # TODO(developer): Construct a BigQuery client object. + # client = bigquery.Client() + + # TODO(developer): Choose a fully-qualified ID for the routine. + # routine_id = "my-project.my_dataset.my_routine" + + sql = """ + CREATE FUNCTION `{}`( + arr ARRAY> + ) AS ( + (SELECT SUM(IF(elem.name = "foo",elem.val,null)) FROM UNNEST(arr) AS elem) + ) + """.format( + routine_id + ) + + # Initiate the query to create the routine. + query_job = client.query(sql) + + # Wait for the query to complete. + query_job.result() + + routine = client.get_routine(routine_id) + print("Created routine {}".format(routine.reference)) + # [END bigquery_create_routine_ddl] + return routine diff --git a/bigquery/samples/delete_routine.py b/bigquery/samples/delete_routine.py new file mode 100644 index 000000000000..505faa4780f3 --- /dev/null +++ b/bigquery/samples/delete_routine.py @@ -0,0 +1,30 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def main(client, routine_id): + # [START bigquery_delete_routine] + # TODO(developer): Import the client library. + # from google.cloud import bigquery + + # TODO(developer): Construct a BigQuery client object. + # client = bigquery.Client() + + # TODO(developer): Set the fully-qualified ID for the routine. + # routine_id = "my-project.my_dataset.my_routine" + + client.delete_routine(routine_id) + # [END bigquery_delete_routine] + + print("Deleted routine {}.".format(routine_id)) diff --git a/bigquery/samples/get_routine.py b/bigquery/samples/get_routine.py new file mode 100644 index 000000000000..5850d8d06477 --- /dev/null +++ b/bigquery/samples/get_routine.py @@ -0,0 +1,39 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def main(client, routine_id): + # [START bigquery_get_routine] + # TODO(developer): Import the client library. + # from google.cloud import bigquery + + # TODO(developer): Construct a BigQuery client object. + # client = bigquery.Client() + + # TODO(developer): Set the fully-qualified ID for the routine. + # routine_id = "my-project.my_dataset.my_routine" + + routine = client.get_routine(routine_id) + + print("Routine `{}`:".format(routine.reference)) + print(" Type: '{}'".format(routine.type_)) + print(" Language: '{}'".format(routine.language)) + print(" Arguments:") + + for argument in routine.arguments: + print(" Name: '{}'".format(argument.name)) + print(" Type: '{}'".format(argument.type_)) + + # [END bigquery_get_routine] + return routine diff --git a/bigquery/samples/list_routines.py b/bigquery/samples/list_routines.py new file mode 100644 index 000000000000..9e90c87a3d9c --- /dev/null +++ b/bigquery/samples/list_routines.py @@ -0,0 +1,34 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def main(client, dataset_id): + + # [START bigquery_list_routines] + # TODO(developer): Import the client library. + # from google.cloud import bigquery + + # TODO(developer): Construct a BigQuery client object. + # client = bigquery.Client() + + # TODO(developer): Set dataset_id to the ID of the dataset that contains + # the routines you are listing. + # dataset_id = 'your-project.your_dataset' + + routines = client.list_routines(dataset_id) + + print("Routines contained in dataset {}:".format(dataset_id)) + for routine in routines: + print(routine.reference) + # [END bigquery_list_routines] diff --git a/bigquery/samples/tests/conftest.py b/bigquery/samples/tests/conftest.py index 629b23473b01..fe5391ee8a4d 100644 --- a/bigquery/samples/tests/conftest.py +++ b/bigquery/samples/tests/conftest.py @@ -18,6 +18,7 @@ import pytest from google.cloud import bigquery +from google.cloud import bigquery_v2 @pytest.fixture(scope="module") @@ -44,6 +45,15 @@ def random_dataset_id(client): client.delete_dataset(random_dataset_id, delete_contents=True, not_found_ok=True) +@pytest.fixture +def random_routine_id(client, dataset_id): + now = datetime.datetime.now() + random_routine_id = "example_routine_{}_{}".format( + now.strftime("%Y%m%d%H%M%S"), uuid.uuid4().hex[:8] + ) + return "{}.{}".format(dataset_id, random_routine_id) + + @pytest.fixture def dataset_id(client): now = datetime.datetime.now() @@ -68,6 +78,31 @@ def table_id(client, dataset_id): client.delete_table(table, not_found_ok=True) +@pytest.fixture +def routine_id(client, dataset_id): + now = datetime.datetime.now() + routine_id = "python_samples_{}_{}".format( + now.strftime("%Y%m%d%H%M%S"), uuid.uuid4().hex[:8] + ) + + routine = bigquery.Routine("{}.{}".format(dataset_id, routine_id)) + routine.type_ = "SCALAR_FUNCTION" + routine.language = "SQL" + routine.body = "x * 3" + routine.arguments = [ + bigquery.RoutineArgument( + name="x", + data_type=bigquery_v2.types.StandardSqlDataType( + type_kind=bigquery_v2.enums.StandardSqlDataType.TypeKind.INT64 + ), + ) + ] + + routine = client.create_routine(routine) + yield "{}.{}.{}".format(routine.project, routine.dataset_id, routine.routine_id) + client.delete_routine(routine, not_found_ok=True) + + @pytest.fixture def model_id(client, dataset_id): model_id = "{}.{}".format(dataset_id, uuid.uuid4().hex) diff --git a/bigquery/samples/tests/test_create_table.py b/bigquery/samples/tests/test_create_table.py index 903f76b536ea..093ee6e94277 100644 --- a/bigquery/samples/tests/test_create_table.py +++ b/bigquery/samples/tests/test_create_table.py @@ -16,7 +16,6 @@ def test_create_table(capsys, client, random_table_id): - create_table.create_table(client, random_table_id) out, err = capsys.readouterr() assert "Created table {}".format(random_table_id) in out diff --git a/bigquery/samples/tests/test_routine_samples.py b/bigquery/samples/tests/test_routine_samples.py new file mode 100644 index 000000000000..0f01fd533232 --- /dev/null +++ b/bigquery/samples/tests/test_routine_samples.py @@ -0,0 +1,87 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from google.cloud import bigquery +from google.cloud import bigquery_v2 + + +def test_create_routine(capsys, client, random_routine_id): + from .. import create_routine + + create_routine.main(client, random_routine_id) + out, err = capsys.readouterr() + assert "Created routine {}".format(random_routine_id) in out + + +def test_create_routine_ddl(capsys, client, random_routine_id): + from .. import create_routine_ddl + + routine = create_routine_ddl.main(client, random_routine_id) + out, err = capsys.readouterr() + assert "Created routine {}".format(random_routine_id) in out + + assert routine.type_ == "SCALAR_FUNCTION" + assert routine.language == "SQL" + expected_arguments = [ + bigquery.RoutineArgument( + name="arr", + data_type=bigquery_v2.types.StandardSqlDataType( + type_kind=bigquery_v2.enums.StandardSqlDataType.TypeKind.ARRAY, + array_element_type=bigquery_v2.types.StandardSqlDataType( + type_kind=bigquery_v2.enums.StandardSqlDataType.TypeKind.STRUCT, + struct_type=bigquery_v2.types.StandardSqlStructType( + fields=[ + bigquery_v2.types.StandardSqlField( + name="name", + type=bigquery_v2.types.StandardSqlDataType( + type_kind=bigquery_v2.enums.StandardSqlDataType.TypeKind.STRING + ), + ), + bigquery_v2.types.StandardSqlField( + name="val", + type=bigquery_v2.types.StandardSqlDataType( + type_kind=bigquery_v2.enums.StandardSqlDataType.TypeKind.INT64 + ), + ), + ] + ), + ), + ), + ) + ] + assert routine.arguments == expected_arguments + + +def test_list_routines(capsys, client, dataset_id, routine_id): + from .. import list_routines + + list_routines.main(client, dataset_id) + out, err = capsys.readouterr() + assert "Routines contained in dataset {}:".format(dataset_id) in out + assert routine_id in out + + +def test_delete_routine(capsys, client, routine_id): + from .. import delete_routine + + delete_routine.main(client, routine_id) + out, err = capsys.readouterr() + assert "Deleted routine {}.".format(routine_id) in out + + +def test_update_routine(client, routine_id): + from .. import update_routine + + routine = update_routine.main(client, routine_id) + assert routine.body == "x * 4" diff --git a/bigquery/samples/update_routine.py b/bigquery/samples/update_routine.py new file mode 100644 index 000000000000..9fc3c877120f --- /dev/null +++ b/bigquery/samples/update_routine.py @@ -0,0 +1,43 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def main(client, routine_id): + # [START bigquery_update_routine] + # TODO(developer): Import the client library. + # from google.cloud import bigquery + + # TODO(developer): Construct a BigQuery client object. + # client = bigquery.Client() + + # TODO(developer): Set the fully-qualified ID for the routine. + # routine_id = "my-project.my_dataset.my_routine" + + routine = client.get_routine(routine_id) + + routine.body = "x * 4" + + routine = client.update_routine( + routine, + [ + "body", + # Due to a limitation of the API, all fields are required, not just + # those that have been updated. + "arguments", + "type_", + "return_type", + ], + ) + # [END bigquery_update_routine] + return routine diff --git a/bigquery/tests/unit/test_client.py b/bigquery/tests/unit/test_client.py index ea4aad534a13..0eebdcbe0f50 100644 --- a/bigquery/tests/unit/test_client.py +++ b/bigquery/tests/unit/test_client.py @@ -866,6 +866,34 @@ def test_create_dataset_alreadyexists_w_exists_ok_true(self): ] ) + def test_create_routine_w_minimal_resource(self): + from google.cloud.bigquery.routine import Routine + from google.cloud.bigquery.routine import RoutineReference + + creds = _make_credentials() + resource = { + "routineReference": { + "projectId": "test-routine-project", + "datasetId": "test_routines", + "routineId": "minimal_routine", + } + } + client = self._make_one(project=self.PROJECT, credentials=creds) + conn = client._connection = make_connection(resource) + full_routine_id = "test-routine-project.test_routines.minimal_routine" + routine = Routine(full_routine_id) + + actual_routine = client.create_routine(routine) + + conn.api_request.assert_called_once_with( + method="POST", + path="/projects/test-routine-project/datasets/test_routines/routines", + data=resource, + ) + self.assertEqual( + actual_routine.reference, RoutineReference.from_string(full_routine_id) + ) + def test_create_table_w_day_partition(self): from google.cloud.bigquery.table import Table from google.cloud.bigquery.table import TimePartitioning From af72f27bbea4f88bf5a3e7918f8c02e37f2e87c5 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Thu, 27 Jun 2019 17:17:58 -0700 Subject: [PATCH 2/6] Increase unit test coverage. --- bigquery/google/cloud/bigquery/dataset.py | 26 ++ bigquery/google/cloud/bigquery/routine.py | 27 +- bigquery/samples/update_routine.py | 1 + bigquery/tests/unit/routine/__init__.py | 0 bigquery/tests/unit/routine/test_routine.py | 280 ++++++++++++++++ .../unit/routine/test_routine_argument.py | 78 +++++ .../unit/routine/test_routine_reference.py | 138 ++++++++ bigquery/tests/unit/test_client.py | 310 +++++++++++++++++- bigquery/tests/unit/test_dataset.py | 7 + 9 files changed, 859 insertions(+), 8 deletions(-) create mode 100644 bigquery/tests/unit/routine/__init__.py create mode 100644 bigquery/tests/unit/routine/test_routine.py create mode 100644 bigquery/tests/unit/routine/test_routine_argument.py create mode 100644 bigquery/tests/unit/routine/test_routine_reference.py diff --git a/bigquery/google/cloud/bigquery/dataset.py b/bigquery/google/cloud/bigquery/dataset.py index 8566e183cda0..01260ccc6e68 100644 --- a/bigquery/google/cloud/bigquery/dataset.py +++ b/bigquery/google/cloud/bigquery/dataset.py @@ -22,6 +22,7 @@ import google.cloud._helpers from google.cloud.bigquery import _helpers from google.cloud.bigquery.model import ModelReference +from google.cloud.bigquery.routine import RoutineReference from google.cloud.bigquery.table import TableReference @@ -53,6 +54,25 @@ def _get_model_reference(self, model_id): ) +def _get_routine_reference(self, routine_id): + """Constructs a RoutineReference. + + Args: + routine_id (str): the ID of the routine. + + Returns: + google.cloud.bigquery.routine.RoutineReference: + A RoutineReference for a routine in this dataset. + """ + return RoutineReference.from_api_repr( + { + "projectId": self.project, + "datasetId": self.dataset_id, + "routineId": routine_id, + } + ) + + class AccessEntry(object): """Represents grant of an access role to an entity. @@ -224,6 +244,8 @@ def path(self): model = _get_model_reference + routine = _get_routine_reference + @classmethod def from_api_repr(cls, resource): """Factory: construct a dataset reference given its API representation @@ -591,6 +613,8 @@ def _build_resource(self, filter_fields): model = _get_model_reference + routine = _get_routine_reference + def __repr__(self): return "Dataset({})".format(repr(self.reference)) @@ -672,3 +696,5 @@ def reference(self): table = _get_table_reference model = _get_model_reference + + routine = _get_routine_reference diff --git a/bigquery/google/cloud/bigquery/routine.py b/bigquery/google/cloud/bigquery/routine.py index c7136d78c422..9aec03364b78 100644 --- a/bigquery/google/cloud/bigquery/routine.py +++ b/bigquery/google/cloud/bigquery/routine.py @@ -189,15 +189,19 @@ def return_type(self): https://cloud.google.com/bigquery/docs/reference/rest/v2/routines#resource-routine """ resource = self._properties.get(self._PROPERTY_TO_API_FIELD["return_type"]) + if not resource: + return resource output = google.cloud.bigquery_v2.types.StandardSqlDataType() output = json_format.ParseDict(resource, output, ignore_unknown_fields=True) return output @return_type.setter def return_type(self, value): - self._properties[ - self._PROPERTY_TO_API_FIELD["return_type"] - ] = json_format.MessageToDict(value) + if value: + resource = json_format.MessageToDict(value) + else: + resource = None + self._properties[self._PROPERTY_TO_API_FIELD["return_type"]] = resource @property def imported_libraries(self): @@ -257,6 +261,11 @@ def _build_resource(self, filter_fields): """Generate a resource for ``update``.""" return _helpers._build_resource_from_properties(self, filter_fields) + def __repr__(self): + return "Routine('{}.{}.{}')".format( + self.project, self.dataset_id, self.routine_id + ) + class RoutineArgument(object): """Input/output argument of a function or a stored procedure. @@ -328,15 +337,19 @@ def data_type(self): https://cloud.google.com/bigquery/docs/reference/rest/v2/StandardSqlDataType """ resource = self._properties.get(self._PROPERTY_TO_API_FIELD["data_type"]) + if not resource: + return resource output = google.cloud.bigquery_v2.types.StandardSqlDataType() output = json_format.ParseDict(resource, output, ignore_unknown_fields=True) return output @data_type.setter def data_type(self, value): - self._properties[ - self._PROPERTY_TO_API_FIELD["data_type"] - ] = json_format.MessageToDict(value) + if value: + resource = json_format.MessageToDict(value) + else: + resource = None + self._properties[self._PROPERTY_TO_API_FIELD["data_type"]] = resource @classmethod def from_api_repr(cls, resource): @@ -374,7 +387,7 @@ def __ne__(self, other): def __repr__(self): all_properties = [ "{}={}".format(property_name, repr(getattr(self, property_name))) - for property_name in self._PROPERTY_TO_API_FIELD + for property_name in sorted(self._PROPERTY_TO_API_FIELD) ] return "RoutineArgument({})".format(", ".join(all_properties)) diff --git a/bigquery/samples/update_routine.py b/bigquery/samples/update_routine.py index 9fc3c877120f..8683e761562f 100644 --- a/bigquery/samples/update_routine.py +++ b/bigquery/samples/update_routine.py @@ -35,6 +35,7 @@ def main(client, routine_id): # Due to a limitation of the API, all fields are required, not just # those that have been updated. "arguments", + "language", "type_", "return_type", ], diff --git a/bigquery/tests/unit/routine/__init__.py b/bigquery/tests/unit/routine/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/bigquery/tests/unit/routine/test_routine.py b/bigquery/tests/unit/routine/test_routine.py new file mode 100644 index 000000000000..fded11ef88f7 --- /dev/null +++ b/bigquery/tests/unit/routine/test_routine.py @@ -0,0 +1,280 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import datetime + +import pytest + +import google.cloud._helpers +from google.cloud import bigquery_v2 + + +@pytest.fixture +def target_class(): + from google.cloud.bigquery.routine import Routine + + return Routine + + +@pytest.fixture +def object_under_test(target_class): + return target_class("project-id.dataset_id.routine_id") + + +def test_ctor(target_class): + from google.cloud.bigquery.routine import RoutineReference + + ref = RoutineReference.from_string("my-proj.my_dset.my_routine") + actual_routine = target_class(ref) + assert actual_routine.reference == ref + assert ( + actual_routine.path == "/projects/my-proj/datasets/my_dset/routines/my_routine" + ) + + +def test_ctor_w_string(target_class): + from google.cloud.bigquery.routine import RoutineReference + + routine_id = "my-proj.my_dset.my_routine" + ref = RoutineReference.from_string(routine_id) + actual_routine = target_class(routine_id) + assert actual_routine.reference == ref + + +def test_ctor_w_properties(target_class): + from google.cloud.bigquery.routine import RoutineArgument + from google.cloud.bigquery.routine import RoutineReference + + routine_id = "my-proj.my_dset.my_routine" + arguments = [ + RoutineArgument( + name="x", + data_type=bigquery_v2.types.StandardSqlDataType( + type_kind=bigquery_v2.enums.StandardSqlDataType.TypeKind.INT64 + ), + ) + ] + body = "x * 3" + language = "SQL" + return_type = bigquery_v2.types.StandardSqlDataType( + type_kind=bigquery_v2.enums.StandardSqlDataType.TypeKind.INT64 + ) + type_ = "SCALAR_FUNCTION" + + actual_routine = target_class( + routine_id, + arguments=arguments, + body=body, + language=language, + return_type=return_type, + type_=type_, + ) + + ref = RoutineReference.from_string(routine_id) + assert actual_routine.reference == ref + assert actual_routine.arguments == arguments + assert actual_routine.body == body + assert actual_routine.language == language + assert actual_routine.return_type == return_type + assert actual_routine.type_ == type_ + + +def test_from_api_repr(target_class): + from google.cloud.bigquery.routine import RoutineArgument + from google.cloud.bigquery.routine import RoutineReference + + creation_time = datetime.datetime( + 2010, 5, 19, 16, 0, 0, tzinfo=google.cloud._helpers.UTC + ) + modified_time = datetime.datetime( + 2011, 10, 1, 16, 0, 0, tzinfo=google.cloud._helpers.UTC + ) + resource = { + "routineReference": { + "projectId": "my-project", + "datasetId": "my_dataset", + "routineId": "my_routine", + }, + "etag": "abcdefg", + "creationTime": str(google.cloud._helpers._millis(creation_time)), + "lastModifiedTime": str(google.cloud._helpers._millis(modified_time)), + "definitionBody": "42", + "arguments": [{"name": "x", "dataType": {"typeKind": "INT64"}}], + "language": "SQL", + "returnType": {"typeKind": "INT64"}, + "routineType": "SCALAR_FUNCTION", + "someNewField": "someValue", + } + actual_routine = target_class.from_api_repr(resource) + + assert actual_routine.project == "my-project" + assert actual_routine.dataset_id == "my_dataset" + assert actual_routine.routine_id == "my_routine" + assert ( + actual_routine.path + == "/projects/my-project/datasets/my_dataset/routines/my_routine" + ) + assert actual_routine.reference == RoutineReference.from_string( + "my-project.my_dataset.my_routine" + ) + assert actual_routine.etag == "abcdefg" + assert actual_routine.created == creation_time + assert actual_routine.modified == modified_time + assert actual_routine.arguments == [ + RoutineArgument( + name="x", + data_type=bigquery_v2.types.StandardSqlDataType( + type_kind=bigquery_v2.enums.StandardSqlDataType.TypeKind.INT64 + ), + ) + ] + assert actual_routine.body == "42" + assert actual_routine.language == "SQL" + assert actual_routine.return_type == bigquery_v2.types.StandardSqlDataType( + type_kind=bigquery_v2.enums.StandardSqlDataType.TypeKind.INT64 + ) + assert actual_routine.type_ == "SCALAR_FUNCTION" + assert actual_routine._properties["someNewField"] == "someValue" + + +def test_from_api_repr_w_minimal_resource(target_class): + from google.cloud.bigquery.routine import RoutineReference + + resource = { + "routineReference": { + "projectId": "my-project", + "datasetId": "my_dataset", + "routineId": "my_routine", + } + } + actual_routine = target_class.from_api_repr(resource) + assert actual_routine.reference == RoutineReference.from_string( + "my-project.my_dataset.my_routine" + ) + assert actual_routine.etag is None + assert actual_routine.created is None + assert actual_routine.modified is None + assert actual_routine.arguments == [] + assert actual_routine.body is None + assert actual_routine.language is None + assert actual_routine.return_type is None + assert actual_routine.type_ is None + + +def test_from_api_repr_w_unknown_fields(target_class): + from google.cloud.bigquery.routine import RoutineReference + + resource = { + "routineReference": { + "projectId": "my-project", + "datasetId": "my_dataset", + "routineId": "my_routine", + }, + "thisFieldIsNotInTheProto": "just ignore me", + } + actual_routine = target_class.from_api_repr(resource) + assert actual_routine.reference == RoutineReference.from_string( + "my-project.my_dataset.my_routine" + ) + assert actual_routine._properties is resource + + +@pytest.mark.parametrize( + "resource,filter_fields,expected", + [ + ( + { + "arguments": [{"name": "x", "dataType": {"typeKind": "INT64"}}], + "definitionBody": "x * 3", + "language": "SQL", + "returnType": {"typeKind": "INT64"}, + "routineType": "SCALAR_FUNCTION", + }, + ["arguments"], + {"arguments": [{"name": "x", "dataType": {"typeKind": "INT64"}}]}, + ), + ( + { + "arguments": [{"name": "x", "dataType": {"typeKind": "INT64"}}], + "definitionBody": "x * 3", + "language": "SQL", + "returnType": {"typeKind": "INT64"}, + "routineType": "SCALAR_FUNCTION", + }, + ["body"], + {"definitionBody": "x * 3"}, + ), + ( + { + "arguments": [{"name": "x", "dataType": {"typeKind": "INT64"}}], + "definitionBody": "x * 3", + "language": "SQL", + "returnType": {"typeKind": "INT64"}, + "routineType": "SCALAR_FUNCTION", + }, + ["language"], + {"language": "SQL"}, + ), + ( + { + "arguments": [{"name": "x", "dataType": {"typeKind": "INT64"}}], + "definitionBody": "x * 3", + "language": "SQL", + "returnType": {"typeKind": "INT64"}, + "routineType": "SCALAR_FUNCTION", + }, + ["return_type"], + {"returnType": {"typeKind": "INT64"}}, + ), + ( + { + "arguments": [{"name": "x", "dataType": {"typeKind": "INT64"}}], + "definitionBody": "x * 3", + "language": "SQL", + "returnType": {"typeKind": "INT64"}, + "routineType": "SCALAR_FUNCTION", + }, + ["type_"], + {"routineType": "SCALAR_FUNCTION"}, + ), + ( + {}, + ["arguments", "language", "body", "type_", "return_type"], + { + "arguments": None, + "definitionBody": None, + "language": None, + "returnType": None, + "routineType": None, + }, + ), + ( + {"someNewField": "someValue"}, + ["someNewField"], + {"someNewField": "someValue"}, + ), + ], +) +def test_build_resource(object_under_test, resource, filter_fields, expected): + object_under_test._properties = resource + actual_routine = object_under_test._build_resource(filter_fields) + assert actual_routine == expected + + +def test_repr(target_class): + model = target_class("my-proj.my_dset.my_routine") + actual_routine = repr(model) + assert actual_routine == "Routine('my-proj.my_dset.my_routine')" diff --git a/bigquery/tests/unit/routine/test_routine_argument.py b/bigquery/tests/unit/routine/test_routine_argument.py new file mode 100644 index 000000000000..1d7839afbb99 --- /dev/null +++ b/bigquery/tests/unit/routine/test_routine_argument.py @@ -0,0 +1,78 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + +from google.cloud import bigquery_v2 + + +@pytest.fixture +def target_class(): + from google.cloud.bigquery.routine import RoutineArgument + + return RoutineArgument + + +def test_ctor(target_class): + data_type = bigquery_v2.types.StandardSqlDataType( + type_kind=bigquery_v2.enums.StandardSqlDataType.TypeKind.INT64 + ) + actual_arg = target_class( + name="field_name", kind="FIXED_TYPE", mode="IN", data_type=data_type + ) + assert actual_arg.name == "field_name" + assert actual_arg.kind == "FIXED_TYPE" + assert actual_arg.mode == "IN" + assert actual_arg.data_type == data_type + + +def test_from_api_repr(target_class): + resource = { + "argumentKind": "FIXED_TYPE", + "dataType": {"typeKind": "INT64"}, + "mode": "IN", + "name": "field_name", + } + actual_arg = target_class.from_api_repr(resource) + assert actual_arg.name == "field_name" + assert actual_arg.kind == "FIXED_TYPE" + assert actual_arg.mode == "IN" + assert actual_arg.data_type == bigquery_v2.types.StandardSqlDataType( + type_kind=bigquery_v2.enums.StandardSqlDataType.TypeKind.INT64 + ) + + +def test_from_api_repr_w_minimal_resource(target_class): + resource = {} + actual_arg = target_class.from_api_repr(resource) + assert actual_arg.name is None + assert actual_arg.kind is None + assert actual_arg.mode is None + assert actual_arg.data_type is None + + +def test_from_api_repr_w_unknown_fields(target_class): + resource = {"thisFieldIsNotInTheProto": "just ignore me"} + actual_arg = target_class.from_api_repr(resource) + assert actual_arg._properties is resource + + +def test_repr(target_class): + arg = target_class(name="field_name", kind="FIXED_TYPE", mode="IN", data_type=None) + actual_repr = repr(arg) + assert actual_repr == ( + "RoutineArgument(data_type=None, kind='FIXED_TYPE', mode='IN', name='field_name')" + ) diff --git a/bigquery/tests/unit/routine/test_routine_reference.py b/bigquery/tests/unit/routine/test_routine_reference.py new file mode 100644 index 000000000000..9d3d551a6294 --- /dev/null +++ b/bigquery/tests/unit/routine/test_routine_reference.py @@ -0,0 +1,138 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + + +@pytest.fixture +def target_class(): + from google.cloud.bigquery.routine import RoutineReference + + return RoutineReference + + +def test_from_api_repr(target_class): + resource = { + "projectId": "my-project", + "datasetId": "my_dataset", + "routineId": "my_routine", + } + got = target_class.from_api_repr(resource) + assert got.project == "my-project" + assert got.dataset_id == "my_dataset" + assert got.routine_id == "my_routine" + assert got.path == "/projects/my-project/datasets/my_dataset/routines/my_routine" + + +def test_from_api_repr_w_unknown_fields(target_class): + resource = { + "projectId": "my-project", + "datasetId": "my_dataset", + "routineId": "my_routine", + "thisFieldIsNotInTheProto": "just ignore me", + } + got = target_class.from_api_repr(resource) + assert got.project == "my-project" + assert got.dataset_id == "my_dataset" + assert got.routine_id == "my_routine" + assert got._properties is resource + + +def test_to_api_repr(target_class): + ref = target_class.from_string("my-project.my_dataset.my_routine") + got = ref.to_api_repr() + assert got == { + "projectId": "my-project", + "datasetId": "my_dataset", + "routineId": "my_routine", + } + + +def test_from_string(target_class): + got = target_class.from_string("string-project.string_dataset.string_routine") + assert got.project == "string-project" + assert got.dataset_id == "string_dataset" + assert got.routine_id == "string_routine" + assert got.path == ( + "/projects/string-project/datasets/string_dataset/routines/string_routine" + ) + + +def test_from_string_legacy_string(target_class): + with pytest.raises(ValueError): + target_class.from_string("string-project:string_dataset.string_routine") + + +def test_from_string_not_fully_qualified(target_class): + with pytest.raises(ValueError): + target_class.from_string("string_routine") + + with pytest.raises(ValueError): + target_class.from_string("string_dataset.string_routine") + + with pytest.raises(ValueError): + target_class.from_string("a.b.c.d") + + +def test_from_string_with_default_project(target_class): + got = target_class.from_string( + "string_dataset.string_routine", default_project="default-project" + ) + assert got.project == "default-project" + assert got.dataset_id == "string_dataset" + assert got.routine_id == "string_routine" + + +def test_from_string_ignores_default_project(target_class): + got = target_class.from_string( + "string-project.string_dataset.string_routine", + default_project="default-project", + ) + assert got.project == "string-project" + assert got.dataset_id == "string_dataset" + assert got.routine_id == "string_routine" + + +def test_eq(target_class): + routine = target_class.from_string("my-proj.my_dset.my_routine") + routine_too = target_class.from_string("my-proj.my_dset.my_routine") + assert routine == routine_too + assert not (routine != routine_too) + + other_routine = target_class.from_string("my-proj.my_dset.my_routine2") + assert not (routine == other_routine) + assert routine != other_routine + + notaroutine = object() + assert not (routine == notaroutine) + assert routine != notaroutine + + +def test_hash(target_class): + routine = target_class.from_string("my-proj.my_dset.my_routine") + routine2 = target_class.from_string("my-proj.my_dset.routine2") + got = {routine: "hello", routine2: "world"} + assert got[routine] == "hello" + assert got[routine2] == "world" + + routine_too = target_class.from_string("my-proj.my_dset.my_routine") + assert got[routine_too] == "hello" + + +def test_repr(target_class): + routine = target_class.from_string("my-proj.my_dset.my_routine") + got = repr(routine) + assert got == "RoutineReference.from_string('my-proj.my_dset.my_routine')" diff --git a/bigquery/tests/unit/test_client.py b/bigquery/tests/unit/test_client.py index 0eebdcbe0f50..8ad9dc8858c6 100644 --- a/bigquery/tests/unit/test_client.py +++ b/bigquery/tests/unit/test_client.py @@ -41,8 +41,9 @@ import google.api_core.exceptions from google.api_core.gapic_v1 import client_info import google.cloud._helpers -from tests.unit.helpers import make_connection +from google.cloud import bigquery_v2 from google.cloud.bigquery.dataset import DatasetReference +from tests.unit.helpers import make_connection def _make_credentials(): @@ -894,6 +895,70 @@ def test_create_routine_w_minimal_resource(self): actual_routine.reference, RoutineReference.from_string(full_routine_id) ) + def test_create_routine_w_conflict(self): + from google.cloud.bigquery.routine import Routine + + creds = _make_credentials() + client = self._make_one(project=self.PROJECT, credentials=creds) + conn = client._connection = make_connection( + google.api_core.exceptions.AlreadyExists("routine already exists") + ) + full_routine_id = "test-routine-project.test_routines.minimal_routine" + routine = Routine(full_routine_id) + + with pytest.raises(google.api_core.exceptions.AlreadyExists): + client.create_routine(routine) + + resource = { + "routineReference": { + "projectId": "test-routine-project", + "datasetId": "test_routines", + "routineId": "minimal_routine", + } + } + conn.api_request.assert_called_once_with( + method="POST", + path="/projects/test-routine-project/datasets/test_routines/routines", + data=resource, + ) + + def test_create_routine_w_conflict_exists_ok(self): + from google.cloud.bigquery.routine import Routine + + creds = _make_credentials() + client = self._make_one(project=self.PROJECT, credentials=creds) + resource = { + "routineReference": { + "projectId": "test-routine-project", + "datasetId": "test_routines", + "routineId": "minimal_routine", + } + } + conn = client._connection = make_connection( + google.api_core.exceptions.AlreadyExists("routine already exists"), resource + ) + full_routine_id = "test-routine-project.test_routines.minimal_routine" + routine = Routine(full_routine_id) + + actual_routine = client.create_routine(routine, exists_ok=True) + + self.assertEqual(actual_routine.project, "test-routine-project") + self.assertEqual(actual_routine.dataset_id, "test_routines") + self.assertEqual(actual_routine.routine_id, "minimal_routine") + conn.api_request.assert_has_calls( + [ + mock.call( + method="POST", + path="/projects/test-routine-project/datasets/test_routines/routines", + data=resource, + ), + mock.call( + method="GET", + path="/projects/test-routine-project/datasets/test_routines/routines/minimal_routine", + ), + ] + ) + def test_create_table_w_day_partition(self): from google.cloud.bigquery.table import Table from google.cloud.bigquery.table import TimePartitioning @@ -1326,6 +1391,52 @@ def test_get_model_w_string(self): conn.api_request.assert_called_once_with(method="GET", path="/%s" % path) self.assertEqual(got.model_id, self.MODEL_ID) + def test_get_routine(self): + from google.cloud.bigquery.routine import Routine + from google.cloud.bigquery.routine import RoutineReference + + full_routine_id = "test-routine-project.test_routines.minimal_routine" + routines = [ + full_routine_id, + Routine(full_routine_id), + RoutineReference.from_string(full_routine_id), + ] + for routine in routines: + creds = _make_credentials() + resource = { + "etag": "im-an-etag", + "routineReference": { + "projectId": "test-routine-project", + "datasetId": "test_routines", + "routineId": "minimal_routine", + }, + "routineType": "SCALAR_FUNCTION", + } + client = self._make_one(project=self.PROJECT, credentials=creds) + conn = client._connection = make_connection(resource) + + actual_routine = client.get_routine(routine) + + conn.api_request.assert_called_once_with( + method="GET", + path="/projects/test-routine-project/datasets/test_routines/routines/minimal_routine", + ) + self.assertEqual( + actual_routine.reference, + RoutineReference.from_string(full_routine_id), + msg="routine={}".format(repr(routine)), + ) + self.assertEqual( + actual_routine.etag, + "im-an-etag", + msg="routine={}".format(repr(routine)), + ) + self.assertEqual( + actual_routine.type_, + "SCALAR_FUNCTION", + msg="routine={}".format(repr(routine)), + ) + def test_get_table(self): path = "projects/%s/datasets/%s/tables/%s" % ( self.PROJECT, @@ -1527,6 +1638,66 @@ def test_update_model(self): req = conn.api_request.call_args self.assertEqual(req[1]["headers"]["If-Match"], "etag") + def test_update_routine(self): + from google.cloud.bigquery.routine import Routine + from google.cloud.bigquery.routine import RoutineArgument + + full_routine_id = "routines-project.test_routines.updated_routine" + resource = { + "routineReference": { + "projectId": "routines-project", + "datasetId": "test_routines", + "routineId": "updated_routine", + }, + "routineType": "SCALAR_FUNCTION", + "language": "SQL", + "definitionBody": "x * 3", + "arguments": [{"name": "x", "dataType": {"typeKind": "INT64"}}], + "returnType": None, + "someNewField": "someValue", + } + creds = _make_credentials() + client = self._make_one(project=self.PROJECT, credentials=creds) + conn = client._connection = make_connection(resource, resource) + routine = Routine(full_routine_id) + routine.arguments = [ + RoutineArgument( + name="x", + data_type=bigquery_v2.types.StandardSqlDataType( + type_kind=bigquery_v2.enums.StandardSqlDataType.TypeKind.INT64 + ), + ) + ] + routine.body = "x * 3" + routine.language = "SQL" + routine.type_ = "SCALAR_FUNCTION" + routine._properties["someNewField"] = "someValue" + + actual_routine = client.update_routine( + routine, + ["arguments", "language", "body", "type_", "return_type", "someNewField"], + ) + + # TODO: routineReference isn't needed when the Routines API supports + # partial updates. + sent = resource + conn.api_request.assert_called_once_with( + method="PUT", + data=sent, + path="/projects/routines-project/datasets/test_routines/routines/updated_routine", + headers=None, + ) + self.assertEqual(actual_routine.arguments, routine.arguments) + self.assertEqual(actual_routine.body, routine.body) + self.assertEqual(actual_routine.language, routine.language) + self.assertEqual(actual_routine.type_, routine.type_) + + # ETag becomes If-Match header. + routine._properties["etag"] = "im-an-etag" + client.update_routine(routine, []) + req = conn.api_request.call_args + self.assertEqual(req[1]["headers"]["If-Match"], "im-an-etag") + def test_update_table(self): from google.cloud.bigquery.table import Table, SchemaField @@ -1905,6 +2076,82 @@ def test_list_models_wrong_type(self): with self.assertRaises(TypeError): client.list_models(client.dataset(self.DS_ID).model("foo")) + def test_list_routines_empty(self): + creds = _make_credentials() + client = self._make_one(project=self.PROJECT, credentials=creds) + conn = client._connection = make_connection({}) + + iterator = client.list_routines("test-routines.test_routines") + page = six.next(iterator.pages) + routines = list(page) + token = iterator.next_page_token + + self.assertEqual(routines, []) + self.assertIsNone(token) + conn.api_request.assert_called_once_with( + method="GET", + path="/projects/test-routines/datasets/test_routines/routines", + query_params={}, + ) + + def test_list_routines_defaults(self): + from google.cloud.bigquery.routine import Routine + + project_id = "test-routines" + dataset_id = "test_routines" + path = "/projects/test-routines/datasets/test_routines/routines" + routine_1 = "routine_one" + routine_2 = "routine_two" + token = "TOKEN" + resource = { + "nextPageToken": token, + "routines": [ + { + "routineReference": { + "routineId": routine_1, + "datasetId": dataset_id, + "projectId": project_id, + } + }, + { + "routineReference": { + "routineId": routine_2, + "datasetId": dataset_id, + "projectId": project_id, + } + }, + ], + } + + creds = _make_credentials() + client = self._make_one(project=project_id, credentials=creds) + conn = client._connection = make_connection(resource) + dataset = client.dataset(dataset_id) + + iterator = client.list_routines(dataset) + self.assertIs(iterator.dataset, dataset) + page = six.next(iterator.pages) + routines = list(page) + actual_token = iterator.next_page_token + + self.assertEqual(len(routines), len(resource["routines"])) + for found, expected in zip(routines, resource["routines"]): + self.assertIsInstance(found, Routine) + self.assertEqual( + found.routine_id, expected["routineReference"]["routineId"] + ) + self.assertEqual(actual_token, token) + + conn.api_request.assert_called_once_with( + method="GET", path=path, query_params={} + ) + + def test_list_routines_wrong_type(self): + creds = _make_credentials() + client = self._make_one(project=self.PROJECT, credentials=creds) + with self.assertRaises(TypeError): + client.list_routines(client.dataset(self.DS_ID).table("foo")) + def test_list_tables_defaults(self): from google.cloud.bigquery.table import TableListItem @@ -2154,6 +2401,67 @@ def test_delete_model_w_not_found_ok_true(self): conn.api_request.assert_called_with(method="DELETE", path=path) + def test_delete_routine(self): + from google.cloud.bigquery.routine import Routine + from google.cloud.bigquery.routine import RoutineReference + + full_routine_id = "test-routine-project.test_routines.minimal_routine" + routines = [ + full_routine_id, + Routine(full_routine_id), + RoutineReference.from_string(full_routine_id), + ] + creds = _make_credentials() + http = object() + client = self._make_one(project=self.PROJECT, credentials=creds, _http=http) + conn = client._connection = make_connection(*([{}] * len(routines))) + + for routine in routines: + client.delete_routine(routine) + conn.api_request.assert_called_with( + method="DELETE", + path="/projects/test-routine-project/datasets/test_routines/routines/minimal_routine", + ) + + def test_delete_routine_w_wrong_type(self): + creds = _make_credentials() + client = self._make_one(project=self.PROJECT, credentials=creds) + with self.assertRaises(TypeError): + client.delete_routine(client.dataset(self.DS_ID)) + + def test_delete_routine_w_not_found_ok_false(self): + creds = _make_credentials() + http = object() + client = self._make_one(project=self.PROJECT, credentials=creds, _http=http) + conn = client._connection = make_connection( + google.api_core.exceptions.NotFound("routine not found") + ) + + with self.assertRaises(google.api_core.exceptions.NotFound): + client.delete_routine("routines-project.test_routines.test_routine") + + conn.api_request.assert_called_with( + method="DELETE", + path="/projects/routines-project/datasets/test_routines/routines/test_routine", + ) + + def test_delete_routine_w_not_found_ok_true(self): + creds = _make_credentials() + http = object() + client = self._make_one(project=self.PROJECT, credentials=creds, _http=http) + conn = client._connection = make_connection( + google.api_core.exceptions.NotFound("routine not found") + ) + + client.delete_routine( + "routines-project.test_routines.test_routine", not_found_ok=True + ) + + conn.api_request.assert_called_with( + method="DELETE", + path="/projects/routines-project/datasets/test_routines/routines/test_routine", + ) + def test_delete_table(self): from google.cloud.bigquery.table import Table diff --git a/bigquery/tests/unit/test_dataset.py b/bigquery/tests/unit/test_dataset.py index 96a2ace7da0c..b8805a9c7ce3 100644 --- a/bigquery/tests/unit/test_dataset.py +++ b/bigquery/tests/unit/test_dataset.py @@ -158,6 +158,13 @@ def test_model(self): self.assertEqual(model_ref.dataset_id, "dataset_1") self.assertEqual(model_ref.model_id, "model_1") + def test_routine(self): + dataset_ref = self._make_one("some-project-1", "dataset_1") + routine_ref = dataset_ref.routine("routine_1") + self.assertEqual(routine_ref.project, "some-project-1") + self.assertEqual(routine_ref.dataset_id, "dataset_1") + self.assertEqual(routine_ref.routine_id, "routine_1") + def test_to_api_repr(self): dataset = self._make_one("project_1", "dataset_1") From d131ae292630040dcd18bce36f58ad4b641083f1 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Thu, 27 Jun 2019 17:25:28 -0700 Subject: [PATCH 3/6] Add unit tests for RoutineArgument equality. --- .../unit/routine/test_routine_argument.py | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/bigquery/tests/unit/routine/test_routine_argument.py b/bigquery/tests/unit/routine/test_routine_argument.py index 1d7839afbb99..7d17b5fc703f 100644 --- a/bigquery/tests/unit/routine/test_routine_argument.py +++ b/bigquery/tests/unit/routine/test_routine_argument.py @@ -70,6 +70,28 @@ def test_from_api_repr_w_unknown_fields(target_class): assert actual_arg._properties is resource +def test_eq(target_class): + data_type = bigquery_v2.types.StandardSqlDataType( + type_kind=bigquery_v2.enums.StandardSqlDataType.TypeKind.INT64 + ) + arg = target_class( + name="field_name", kind="FIXED_TYPE", mode="IN", data_type=data_type + ) + arg_too = target_class( + name="field_name", kind="FIXED_TYPE", mode="IN", data_type=data_type + ) + assert arg == arg_too + assert not (arg != arg_too) + + other_arg = target_class() + assert not (arg == other_arg) + assert arg != other_arg + + notanarg = object() + assert not (arg == notanarg) + assert arg != notanarg + + def test_repr(target_class): arg = target_class(name="field_name", kind="FIXED_TYPE", mode="IN", data_type=None) actual_repr = repr(arg) From 356b0985630eafe03cfd00d9ef603f0952997bb2 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Fri, 28 Jun 2019 12:58:44 -0700 Subject: [PATCH 4/6] Fix bug with misspelled property name for imported_libraries. --- bigquery/google/cloud/bigquery/routine.py | 2 +- bigquery/tests/system.py | 35 +++++++++++++++++++++ bigquery/tests/unit/routine/test_routine.py | 19 +++++++++++ 3 files changed, 55 insertions(+), 1 deletion(-) diff --git a/bigquery/google/cloud/bigquery/routine.py b/bigquery/google/cloud/bigquery/routine.py index 9aec03364b78..57f3b8ac6de9 100644 --- a/bigquery/google/cloud/bigquery/routine.py +++ b/bigquery/google/cloud/bigquery/routine.py @@ -221,7 +221,7 @@ def imported_libraries(self, value): else: resource = value - self._properties[self._PROPERTY_TO_API_FIELD["return_type"]] = resource + self._properties[self._PROPERTY_TO_API_FIELD["imported_libraries"]] = resource @property def body(self): diff --git a/bigquery/tests/system.py b/bigquery/tests/system.py index eba4c3b6adef..2213bc7c88da 100644 --- a/bigquery/tests/system.py +++ b/bigquery/tests/system.py @@ -59,6 +59,7 @@ from google.api_core.exceptions import ServiceUnavailable from google.api_core.exceptions import TooManyRequests from google.cloud import bigquery +from google.cloud import bigquery_v2 from google.cloud.bigquery.dataset import Dataset from google.cloud.bigquery.dataset import DatasetReference from google.cloud.bigquery.table import Table @@ -1864,6 +1865,40 @@ def test_insert_rows_nested_nested_dictionary(self): expected_rows = [("Some value", record)] self.assertEqual(row_tuples, expected_rows) + def test_create_routine(self): + routine_name = "test_routine" + dataset = self.temp_dataset(_make_dataset_id("create_routine")) + float64_type = bigquery_v2.types.StandardSqlDataType( + type_kind=bigquery_v2.enums.StandardSqlDataType.TypeKind.FLOAT64 + ) + routine = bigquery.Routine( + dataset.routine(routine_name), + language="JAVASCRIPT", + type_="SCALAR_FUNCTION", + return_type=float64_type, + imported_libraries=["gs://cloud-samples-data/bigquery/udfs/max-value.js"], + ) + routine.arguments = [ + bigquery.RoutineArgument( + name="arr", + data_type=bigquery_v2.types.StandardSqlDataType( + type_kind=bigquery_v2.enums.StandardSqlDataType.TypeKind.ARRAY, + array_element_type=float64_type, + ), + ) + ] + routine.body = "return maxValue(arr)" + query_string = "SELECT `{}`([-100.0, 3.14, 100.0, 42.0]) as max_value;".format( + str(routine.reference) + ) + + routine = retry_403(Config.CLIENT.create_routine)(routine) + query_job = retry_403(Config.CLIENT.query)(query_string) + rows = list(query_job.result()) + + assert len(rows) == 1 + assert rows[0].max_value == 100.0 + def test_create_table_rows_fetch_nested_schema(self): table_name = "test_table" dataset = self.temp_dataset(_make_dataset_id("create_table_nested_schema")) diff --git a/bigquery/tests/unit/routine/test_routine.py b/bigquery/tests/unit/routine/test_routine.py index fded11ef88f7..9ed2e58be796 100644 --- a/bigquery/tests/unit/routine/test_routine.py +++ b/bigquery/tests/unit/routine/test_routine.py @@ -274,6 +274,25 @@ def test_build_resource(object_under_test, resource, filter_fields, expected): assert actual_routine == expected +def test_set_imported_libraries(object_under_test): + imported_libraries = ["gs://cloud-samples-data/bigquery/udfs/max-value.js"] + object_under_test.imported_libraries = imported_libraries + assert object_under_test.imported_libraries == imported_libraries + assert object_under_test._properties["importedLibraries"] == imported_libraries + + +def test_set_imported_libraries_w_none(object_under_test): + object_under_test.imported_libraries = None + assert object_under_test.imported_libraries == [] + assert object_under_test._properties["importedLibraries"] == [] + + +def test_set_return_type_w_none(object_under_test): + object_under_test.return_type = None + assert object_under_test.return_type is None + assert object_under_test._properties["returnType"] is None + + def test_repr(target_class): model = target_class("my-proj.my_dset.my_routine") actual_routine = repr(model) From 79d3f8e202d3007167a0dc4b725d5b64fd527303 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Fri, 28 Jun 2019 13:05:39 -0700 Subject: [PATCH 5/6] Test for null arguments. --- bigquery/google/cloud/bigquery/routine.py | 1 - bigquery/tests/unit/routine/test_routine.py | 6 ++++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/bigquery/google/cloud/bigquery/routine.py b/bigquery/google/cloud/bigquery/routine.py index 57f3b8ac6de9..37335835654a 100644 --- a/bigquery/google/cloud/bigquery/routine.py +++ b/bigquery/google/cloud/bigquery/routine.py @@ -220,7 +220,6 @@ def imported_libraries(self, value): resource = [] else: resource = value - self._properties[self._PROPERTY_TO_API_FIELD["imported_libraries"]] = resource @property diff --git a/bigquery/tests/unit/routine/test_routine.py b/bigquery/tests/unit/routine/test_routine.py index 9ed2e58be796..02d4a2ee2883 100644 --- a/bigquery/tests/unit/routine/test_routine.py +++ b/bigquery/tests/unit/routine/test_routine.py @@ -274,6 +274,12 @@ def test_build_resource(object_under_test, resource, filter_fields, expected): assert actual_routine == expected +def test_set_arguments_w_none(object_under_test): + object_under_test.arguments = None + assert object_under_test.arguments == [] + assert object_under_test._properties["arguments"] == [] + + def test_set_imported_libraries(object_under_test): imported_libraries = ["gs://cloud-samples-data/bigquery/udfs/max-value.js"] object_under_test.imported_libraries = imported_libraries From adb9a1c785718a708c010e145c403fe8eea8c35b Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Fri, 28 Jun 2019 16:04:26 -0700 Subject: [PATCH 6/6] Add QueryJob.ddl_target_routine property. Adjust docstrings. --- bigquery/google/cloud/bigquery/client.py | 4 ++++ bigquery/google/cloud/bigquery/job.py | 16 ++++++++++++- bigquery/google/cloud/bigquery/routine.py | 11 +++++++++ bigquery/samples/create_routine_ddl.py | 4 +--- .../samples/tests/test_routine_samples.py | 6 +++-- bigquery/tests/unit/test_job.py | 24 +++++++++++++++++++ 6 files changed, 59 insertions(+), 6 deletions(-) diff --git a/bigquery/google/cloud/bigquery/client.py b/bigquery/google/cloud/bigquery/client.py index acb7e3383664..b8ce2d5a33f3 100644 --- a/bigquery/google/cloud/bigquery/client.py +++ b/bigquery/google/cloud/bigquery/client.py @@ -639,6 +639,10 @@ def update_routine(self, routine, fields, retry=DEFAULT_RETRY): must be provided. If a field is listed in ``fields`` and is ``None`` in ``routine``, the field value will be deleted. + .. warning:: + During beta, partial updates are not supported. You must provide + all fields in the resource. + If :attr:`~google.cloud.bigquery.routine.Routine.etag` is not ``None``, the update will only succeed if the resource on the server has the same ETag. Thus reading a routine with diff --git a/bigquery/google/cloud/bigquery/job.py b/bigquery/google/cloud/bigquery/job.py index 381ad84f0312..87dab59e339b 100644 --- a/bigquery/google/cloud/bigquery/job.py +++ b/bigquery/google/cloud/bigquery/job.py @@ -34,6 +34,7 @@ from google.cloud.bigquery.query import StructQueryParameter from google.cloud.bigquery.query import UDFResource from google.cloud.bigquery.retry import DEFAULT_RETRY +from google.cloud.bigquery.routine import RoutineReference from google.cloud.bigquery.schema import SchemaField from google.cloud.bigquery.table import _EmptyRowIterator from google.cloud.bigquery.table import EncryptionConfiguration @@ -2666,9 +2667,22 @@ def ddl_operation_performed(self): """ return self._job_statistics().get("ddlOperationPerformed") + @property + def ddl_target_routine(self): + """Optional[google.cloud.bigquery.routine.RoutineReference]: Return the DDL target routine, present + for CREATE/DROP FUNCTION/PROCEDURE queries. + + See: + https://cloud.google.com/bigquery/docs/reference/rest/v2/JobStatistics + """ + prop = self._job_statistics().get("ddlTargetRoutine") + if prop is not None: + prop = RoutineReference.from_api_repr(prop) + return prop + @property def ddl_target_table(self): - """Optional[TableReference]: Return the DDL target table, present + """Optional[google.cloud.bigquery.table.TableReference]: Return the DDL target table, present for CREATE/DROP TABLE/VIEW queries. See: diff --git a/bigquery/google/cloud/bigquery/routine.py b/bigquery/google/cloud/bigquery/routine.py index 37335835654a..d5bb752dfddb 100644 --- a/bigquery/google/cloud/bigquery/routine.py +++ b/bigquery/google/cloud/bigquery/routine.py @@ -209,6 +209,17 @@ def imported_libraries(self): The :attr:`~google.cloud.bigquery.routine.Routine.language` must equal ``JAVACRIPT``. + + Examples: + Set the ``imported_libraries`` to a list of Google Cloud Storage + URIs. + + .. code-block:: python + + routine = bigquery.Routine("proj.dataset.routine_id") + routine.imported_libraries = [ + "gs://cloud-samples-data/bigquery/udfs/max-value.js", + ] """ return self._properties.get( self._PROPERTY_TO_API_FIELD["imported_libraries"], [] diff --git a/bigquery/samples/create_routine_ddl.py b/bigquery/samples/create_routine_ddl.py index 50f50f7d635d..aa6254b1139a 100644 --- a/bigquery/samples/create_routine_ddl.py +++ b/bigquery/samples/create_routine_ddl.py @@ -40,7 +40,5 @@ def main(client, routine_id): # Wait for the query to complete. query_job.result() - routine = client.get_routine(routine_id) - print("Created routine {}".format(routine.reference)) + print("Created routine {}".format(query_job.ddl_target_routine)) # [END bigquery_create_routine_ddl] - return routine diff --git a/bigquery/samples/tests/test_routine_samples.py b/bigquery/samples/tests/test_routine_samples.py index 0f01fd533232..5905d2e69439 100644 --- a/bigquery/samples/tests/test_routine_samples.py +++ b/bigquery/samples/tests/test_routine_samples.py @@ -27,10 +27,12 @@ def test_create_routine(capsys, client, random_routine_id): def test_create_routine_ddl(capsys, client, random_routine_id): from .. import create_routine_ddl - routine = create_routine_ddl.main(client, random_routine_id) + create_routine_ddl.main(client, random_routine_id) + routine = client.get_routine(random_routine_id) out, err = capsys.readouterr() - assert "Created routine {}".format(random_routine_id) in out + assert "Created routine {}".format(random_routine_id) in out + return routine assert routine.type_ == "SCALAR_FUNCTION" assert routine.language == "SQL" expected_arguments = [ diff --git a/bigquery/tests/unit/test_job.py b/bigquery/tests/unit/test_job.py index abb2a2c4ec1e..3561fb857647 100644 --- a/bigquery/tests/unit/test_job.py +++ b/bigquery/tests/unit/test_job.py @@ -3845,6 +3845,30 @@ def test_ddl_operation_performed(self): query_stats["ddlOperationPerformed"] = op self.assertEqual(job.ddl_operation_performed, op) + def test_ddl_target_routine(self): + from google.cloud.bigquery.routine import RoutineReference + + ref_routine = { + "projectId": self.PROJECT, + "datasetId": "ddl_ds", + "routineId": "targetroutine", + } + client = _make_client(project=self.PROJECT) + job = self._make_one(self.JOB_ID, self.QUERY, client) + self.assertIsNone(job.ddl_target_routine) + + statistics = job._properties["statistics"] = {} + self.assertIsNone(job.ddl_target_routine) + + query_stats = statistics["query"] = {} + self.assertIsNone(job.ddl_target_routine) + + query_stats["ddlTargetRoutine"] = ref_routine + self.assertIsInstance(job.ddl_target_routine, RoutineReference) + self.assertEqual(job.ddl_target_routine.routine_id, "targetroutine") + self.assertEqual(job.ddl_target_routine.dataset_id, "ddl_ds") + self.assertEqual(job.ddl_target_routine.project, self.PROJECT) + def test_ddl_target_table(self): from google.cloud.bigquery.table import TableReference