From abdf5ec00261e5500dbdd190c23b0b2b05836799 Mon Sep 17 00:00:00 2001 From: Bu Sun Kim <8822365+busunkim96@users.noreply.github.com> Date: Mon, 3 May 2021 16:49:51 -0600 Subject: [PATCH] feat: add autogenerated snippets (#845) This PR targets iteration 1 and 2 specified in the [Snippet Gen Design](go/snippet-gen-design): Full canonical coverage of simple requests, paginated, LRO, server streaming, and Bidi streaming with empty request objects. Snippet generation is hidden behind a new option `autogen-snippets`. After discussion with folks on different language teams on snippetgen, I decided using "golden" snippet files would be easier than following the unit testing strategy used to check the library surface. I also believe goldens will be be easier for review for other Python DPEs. Other notes: - I've commented out the existing metadata generation code and tests. The new metadata format is still under discussion. - Async samples are excluded as the existing samplegen infrastructure was written pre-async. I will add the async samples in the next PR. Co-authored-by: Dov Shlachter --- .github/snippet-bot.yml | 3 + .github/workflows/tests.yaml | 20 ++ .gitignore | 4 + gapic/generator/generator.py | 123 +++---- gapic/samplegen/samplegen.py | 309 ++++++++++++------ gapic/samplegen_utils/types.py | 3 - gapic/samplegen_utils/utils.py | 2 +- gapic/schema/wrappers.py | 13 + gapic/templates/_base.py.j2 | 2 +- .../docs/%name_%version/services.rst.j2 | 2 +- gapic/templates/examples/feature_fragments.j2 | 43 +-- gapic/templates/examples/sample.py.j2 | 23 +- gapic/templates/noxfile.py.j2 | 6 +- gapic/utils/options.py | 3 + noxfile.py | 48 ++- ...ollusca_v1_snippets_list_resources_grpc.py | 45 +++ ..._v1_snippets_method_bidi_streaming_grpc.py | 45 +++ ..._v1_snippets_method_lro_signatures_grpc.py | 48 +++ ...a_v1_snippets_method_one_signature_grpc.py | 46 +++ ...1_snippets_method_server_streaming_grpc.py | 45 +++ tests/snippetgen/snippets.proto | 106 ++++++ tests/snippetgen/test_snippetgen.py | 81 +++++ tests/unit/generator/test_generator.py | 200 +++++++----- tests/unit/samplegen/common_types.py | 14 +- .../samplegen/golden_snippets/sample_basic.py | 55 ++++ .../sample_basic_unflattenable.py | 55 ++++ tests/unit/samplegen/test_integration.py | 164 +--------- tests/unit/samplegen/test_samplegen.py | 123 ++++++- tests/unit/samplegen/test_template.py | 52 +-- 29 files changed, 1217 insertions(+), 466 deletions(-) create mode 100644 .github/snippet-bot.yml create mode 100644 tests/snippetgen/goldens/mollusca_generated_mollusca_v1_snippets_list_resources_grpc.py create mode 100644 tests/snippetgen/goldens/mollusca_generated_mollusca_v1_snippets_method_bidi_streaming_grpc.py create mode 100644 tests/snippetgen/goldens/mollusca_generated_mollusca_v1_snippets_method_lro_signatures_grpc.py create mode 100644 tests/snippetgen/goldens/mollusca_generated_mollusca_v1_snippets_method_one_signature_grpc.py create mode 100644 tests/snippetgen/goldens/mollusca_generated_mollusca_v1_snippets_method_server_streaming_grpc.py create mode 100644 tests/snippetgen/snippets.proto create mode 100644 tests/snippetgen/test_snippetgen.py create mode 100644 tests/unit/samplegen/golden_snippets/sample_basic.py create mode 100644 tests/unit/samplegen/golden_snippets/sample_basic_unflattenable.py diff --git a/.github/snippet-bot.yml b/.github/snippet-bot.yml new file mode 100644 index 0000000000..77ce8f8255 --- /dev/null +++ b/.github/snippet-bot.yml @@ -0,0 +1,3 @@ +# https://github.com/googleapis/repo-automation-bots/tree/master/packages/snippet-bot +ignoreFiles: + - "**/*.py" diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 74fb502ec4..92ff881185 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -265,6 +265,26 @@ jobs: run: python -m pip install nox - name: Typecheck the generated output. run: nox -s showcase_mypy${{ matrix.variant }} + snippetgen: + runs-on: ubuntu-latest + steps: + - name: Cancel Previous Runs + uses: styfle/cancel-workflow-action@0.7.0 + with: + access_token: ${{ github.token }} + - uses: actions/checkout@v2 + - name: Set up Python 3.8 + uses: actions/setup-python@v2 + with: + python-version: 3.8 + - name: Install system dependencies. + run: | + sudo apt-get update + sudo apt-get install -y curl pandoc unzip gcc + - name: Install nox. + run: python -m pip install nox + - name: Check autogenerated snippets. + run: nox -s snippetgen unit: strategy: matrix: diff --git a/.gitignore b/.gitignore index 5b68f2ed58..2cead4ed7d 100644 --- a/.gitignore +++ b/.gitignore @@ -65,3 +65,7 @@ pylintrc.test # pyenv .python-version + +# Test dependencies and output +api-common-protos +tests/snippetgen/.test_output diff --git a/gapic/generator/generator.py b/gapic/generator/generator.py index 7c081b4722..362753ff2d 100644 --- a/gapic/generator/generator.py +++ b/gapic/generator/generator.py @@ -14,8 +14,10 @@ import jinja2 import yaml +import itertools import re import os +import typing from typing import Any, DefaultDict, Dict, Mapping from hashlib import sha256 from collections import OrderedDict, defaultdict @@ -107,12 +109,12 @@ def get_response( template_name, api_schema=api_schema, opts=opts) ) - sample_output = self._generate_samples_and_manifest( - api_schema, - self._env.get_template(sample_templates[0]), - ) if sample_templates else {} - - output_files.update(sample_output) + if sample_templates: + sample_output = self._generate_samples_and_manifest( + api_schema, self._env.get_template(sample_templates[0]), + opts=opts, + ) + output_files.update(sample_output) # Return the CodeGeneratorResponse output. res = CodeGeneratorResponse( @@ -121,12 +123,13 @@ def get_response( return res def _generate_samples_and_manifest( - self, api_schema: api.API, sample_template: jinja2.Template, - ) -> Dict[str, CodeGeneratorResponse.File]: + self, api_schema: api.API, sample_template: jinja2.Template, *, opts: Options) -> Dict: """Generate samples and samplegen manifest for the API. Arguments: api_schema (api.API): The schema for the API to which the samples belong. + sample_template (jinja2.Template): The template to use to generate samples. + opts (Options): Additional generator options. Returns: Dict[str, CodeGeneratorResponse.File]: A dict mapping filepath to rendered file. @@ -137,56 +140,50 @@ def _generate_samples_and_manifest( id_to_hash_to_spec: DefaultDict[str, Dict[str, Any]] = defaultdict(dict) - STANDALONE_TYPE = "standalone" - for config_fpath in self._sample_configs: - with open(config_fpath) as f: - configs = yaml.safe_load_all(f.read()) - - spec_generator = ( - spec - for cfg in configs - if is_valid_sample_cfg(cfg) - for spec in cfg.get("samples", []) - # If unspecified, assume a sample config describes a standalone. - # If sample_types are specified, standalone samples must be - # explicitly enabled. - if STANDALONE_TYPE in spec.get("sample_type", [STANDALONE_TYPE]) - ) + # Autogenerated sample specs + autogen_specs: typing.List[typing.Dict[str, Any]] = [] + if opts.autogen_snippets: + autogen_specs = list( + samplegen.generate_sample_specs(api_schema, opts=opts)) + + # Also process any handwritten sample specs + handwritten_specs = samplegen.parse_handwritten_specs( + self._sample_configs) + + sample_specs = autogen_specs + list(handwritten_specs) + + for spec in sample_specs: + # Every sample requires an ID. This may be provided + # by a samplegen config author. + # If no ID is provided, fall back to the region tag. + # + # Ideally the sample author should pick a descriptive, unique ID, + # but this may be impractical and can be error-prone. + spec_hash = sha256(str(spec).encode("utf8")).hexdigest()[:8] + sample_id = spec.get("id") or spec.get("region_tag") or spec_hash + spec["id"] = sample_id - for spec in spec_generator: - # Every sample requires an ID, preferably provided by the - # samplegen config author. - # If no ID is provided, fall back to the region tag. - # If there's no region tag, generate a unique ID. - # - # Ideally the sample author should pick a descriptive, unique ID, - # but this may be impractical and can be error-prone. - spec_hash = sha256(str(spec).encode("utf8")).hexdigest()[:8] - sample_id = spec.get("id") or spec.get( - "region_tag") or spec_hash - spec["id"] = sample_id - - hash_to_spec = id_to_hash_to_spec[sample_id] - if spec_hash in hash_to_spec: - raise DuplicateSample( - f"Duplicate samplegen spec found: {spec}") - - hash_to_spec[spec_hash] = spec - - out_dir = "samples" + hash_to_spec = id_to_hash_to_spec[sample_id] + + if spec_hash in hash_to_spec: + raise DuplicateSample( + f"Duplicate samplegen spec found: {spec}") + + hash_to_spec[spec_hash] = spec + + out_dir = "samples/generated_samples" fpath_to_spec_and_rendered = {} for hash_to_spec in id_to_hash_to_spec.values(): for spec_hash, spec in hash_to_spec.items(): id_is_unique = len(hash_to_spec) == 1 - # The ID is used to generate the file name and by sample tester - # to link filenames to invoked samples. It must be globally unique. + # The ID is used to generate the file name. It must be globally unique. if not id_is_unique: spec["id"] += f"_{spec_hash}" sample = samplegen.generate_sample( spec, api_schema, sample_template,) - fpath = spec["id"] + ".py" + fpath = utils.to_snake_case(spec["id"]) + ".py" fpath_to_spec_and_rendered[os.path.join(out_dir, fpath)] = ( spec, sample, @@ -199,20 +196,24 @@ def _generate_samples_and_manifest( for fname, (_, sample) in fpath_to_spec_and_rendered.items() } - # Only generate a manifest if we generated samples. - if output_files: - manifest_fname, manifest_doc = manifest.generate( - ( - (fname, spec) - for fname, (spec, _) in fpath_to_spec_and_rendered.items() - ), - api_schema, - ) - - manifest_fname = os.path.join(out_dir, manifest_fname) - output_files[manifest_fname] = CodeGeneratorResponse.File( - content=manifest_doc.render(), name=manifest_fname - ) + # TODO(busunkim): Re-enable manifest generation once metadata + # format has been formalized. + # https://docs.google.com/document/d/1ghBam8vMj3xdoe4xfXhzVcOAIwrkbTpkMLgKc9RPD9k/edit#heading=h.sakzausv6hue + # + # if output_files: + + # manifest_fname, manifest_doc = manifest.generate( + # ( + # (fname, spec) + # for fname, (spec, _) in fpath_to_spec_and_rendered.items() + # ), + # api_schema, + # ) + + # manifest_fname = os.path.join(out_dir, manifest_fname) + # output_files[manifest_fname] = CodeGeneratorResponse.File( + # content=manifest_doc.render(), name=manifest_fname + # ) return output_files diff --git a/gapic/samplegen/samplegen.py b/gapic/samplegen/samplegen.py index 051419d7db..9cd4987d59 100644 --- a/gapic/samplegen/samplegen.py +++ b/gapic/samplegen/samplegen.py @@ -20,14 +20,17 @@ import os import re import time +import yaml from gapic import utils from gapic.samplegen_utils import types +from gapic.samplegen_utils.utils import is_valid_sample_cfg +from gapic.schema import api from gapic.schema import wrappers -from collections import (defaultdict, namedtuple, ChainMap as chainmap) -from typing import (ChainMap, Dict, FrozenSet, List, Mapping, Optional, Tuple) +from collections import defaultdict, namedtuple, ChainMap as chainmap +from typing import Any, ChainMap, Dict, FrozenSet, Generator, List, Mapping, Optional, Tuple, Sequence # There is no library stub file for this module, so ignore it. from google.api import resource_pb2 # type: ignore @@ -71,6 +74,7 @@ class AttributeRequestSetup: that contains the value for the attribute. """ + value: str field: Optional[str] = None value_is_file: bool = False @@ -98,6 +102,7 @@ class TransformedRequest: The Optional[single]/Optional[body] is workaround for not having tagged unions. """ + base: str single: Optional[AttributeRequestSetup] body: Optional[List[AttributeRequestSetup]] @@ -108,8 +113,14 @@ class TransformedRequest: RESOURCE_RE = re.compile(r"\{([^}/]+)\}") @classmethod - def build(cls, request_type: wrappers.MessageType, api_schema, base: str, - attrs: List[AttributeRequestSetup], is_resource_request: bool): + def build( + cls, + request_type: wrappers.MessageType, + api_schema, + base: str, + attrs: List[AttributeRequestSetup], + is_resource_request: bool, + ): """Build a TransformedRequest based on parsed input. Acts as a factory to hide complicated logic for resource-based requests. @@ -152,34 +163,44 @@ def build(cls, request_type: wrappers.MessageType, api_schema, base: str, # # It's a precondition that the base field is # a valid field of the request message type. - resource_typestr = (request_type. - fields[base]. - options. - Extensions[resource_pb2.resource_reference]. - type) + resource_typestr = ( + request_type.fields[base] + .options.Extensions[resource_pb2.resource_reference] + .type + ) resource_message_descriptor = next( - (msg.options.Extensions[resource_pb2.resource] - for msg in api_schema.messages.values() - if msg.options.Extensions[resource_pb2.resource].type == resource_typestr), - None + ( + msg.options.Extensions[resource_pb2.resource] + for msg in api_schema.messages.values() + if msg.options.Extensions[resource_pb2.resource].type + == resource_typestr + ), + None, ) if not resource_message_descriptor: raise types.NoSuchResource( - f"No message exists for resource: {resource_typestr}") + f"No message exists for resource: {resource_typestr}" + ) # The field is only ever empty for singleton attributes. attr_names: List[str] = [a.field for a in attrs] # type: ignore # A single resource may be found under multiple paths and have many patterns. # We want to find an _exact_ match, if one exists. - pattern = next((p - for p in resource_message_descriptor.pattern - if cls.RESOURCE_RE.findall(p) == attr_names), None) + pattern = next( + ( + p + for p in resource_message_descriptor.pattern + if cls.RESOURCE_RE.findall(p) == attr_names + ), + None, + ) if not pattern: attr_name_str = ", ".join(attr_names) raise types.NoSuchResourcePattern( - f"Resource {resource_typestr} has no pattern with params: {attr_name_str}") + f"Resource {resource_typestr} has no pattern with params: {attr_name_str}" + ) return cls(base=base, body=attrs, single=None, pattern=pattern) @@ -222,10 +243,12 @@ class Validator: EXPRESSION_ATTR_RE = re.compile( r""" (?P\$?\w+)(?:\[(?P\d+)\]|\{["'](?P[^"']+)["']\})?$ - """.strip()) + """.strip() + ) VALID_REQUEST_KWORDS = frozenset( - ("value", "field", "value_is_file", "input_parameter", "comment")) + ("value", "field", "value_is_file", "input_parameter", "comment") + ) # TODO(dovs): make the schema a required param. def __init__(self, method: wrappers.Method, api_schema=None): @@ -234,7 +257,7 @@ def __init__(self, method: wrappers.Method, api_schema=None): self.request_type_ = method.input response_type = method.output if method.paged_result_field: - response_type = method.paged_result_field + response_type = method.paged_result_field.message elif method.lro: response_type = method.lro.response_type @@ -258,21 +281,30 @@ def __init__(self, method: wrappers.Method, api_schema=None): ) @staticmethod - def preprocess_sample(sample, api_schema): + def preprocess_sample(sample, api_schema: api.API, rpc: wrappers.Method): """Modify a sample to set default or missing fields. Args: sample (Any): A definition for a single sample generated from parsed yaml. api_schema (api.API): The schema that defines the API to which the sample belongs. + rpc (wrappers.Method): The rpc method used in the sample. """ sample["package_name"] = api_schema.naming.warehouse_package_name - sample.setdefault("response", [{"print": ["%s", "$resp"]}]) + sample["module_name"] = api_schema.naming.versioned_module_name + sample["module_namespace"] = api_schema.naming.module_namespace + + sample["client_name"] = api_schema.services[sample["service"]].client_name + # the type of the request object passed to the rpc e.g, `ListRequest` + sample["request_type"] = rpc.input.ident.name + + # If no response was specified in the config + # Add reasonable defaults depending on the type of the sample + if not rpc.void: + sample.setdefault("response", [{"print": ["%s", "$resp"]}]) @utils.cached_property def flattenable_fields(self) -> FrozenSet[str]: - return frozenset( - field.name for field in self.method.flattened_fields.values() - ) + return frozenset(field.name for field in self.method.flattened_fields.values()) def var_field(self, var_name: str) -> Optional[wrappers.Field]: return self.var_defs_.get(var_name) @@ -299,7 +331,9 @@ def _normal_request_setup(self, base_param_to_attrs, val, request, field): if not attr: raise types.BadAttributeLookup( "Method request type {} has no attribute: '{}'".format( - self.request_type_, attr_name)) + self.request_type_, attr_name + ) + ) if attr.message: base = attr.message @@ -309,20 +343,23 @@ def _normal_request_setup(self, base_param_to_attrs, val, request, field): witness = any(e.name == val for e in attr.enum.values) if not witness: raise types.InvalidEnumVariant( - "Invalid variant for enum {}: '{}'".format(attr, val)) + "Invalid variant for enum {}: '{}'".format(attr, val) + ) break elif attr.is_primitive: # Only valid if this is the last attribute in the chain. break else: raise TypeError( - f"Could not handle attribute '{attr_name}' of type: {attr.type}") + f"Could not handle attribute '{attr_name}' of type: {attr.type}" + ) if i != len(attr_chain) - 1: # We broke out of the loop after processing an enum or a primitive. extra_attrs = ".".join(attr_chain[i:]) raise types.NonTerminalPrimitiveOrEnum( - f"Attempted to reference attributes of enum value or primitive type: '{extra_attrs}'") + f"Attempted to reference attributes of enum value or primitive type: '{extra_attrs}'" + ) if len(attr_chain) > 1: request["field"] = ".".join(attr_chain[1:]) @@ -333,7 +370,9 @@ def _normal_request_setup(self, base_param_to_attrs, val, request, field): if attr_chain[0] in base_param_to_attrs: raise types.InvalidRequestSetup( "Duplicated top level field in request block: '{}'".format( - attr_chain[0])) + attr_chain[0] + ) + ) del request["field"] if isinstance(request["value"], str): @@ -351,9 +390,9 @@ def _normal_request_setup(self, base_param_to_attrs, val, request, field): # so disable it for the AttributeRequestSetup ctor call. return attr_chain[0], AttributeRequestSetup(**request) # type: ignore - def validate_and_transform_request(self, - calling_form: types.CallingForm, - request: List[Mapping[str, str]]) -> FullRequest: + def validate_and_transform_request( + self, calling_form: types.CallingForm, request: List[Mapping[str, str]] + ) -> FullRequest: """Validates and transforms the "request" block from a sample config. In the initial request, each dict has a "field" key that maps to a dotted @@ -427,61 +466,76 @@ def validate_and_transform_request(self, """ base_param_to_attrs: Dict[str, - RequestEntry] = defaultdict(RequestEntry) + RequestEntry] = defaultdict(RequestEntry) for r in request: r_dup = dict(r) val = r_dup.get("value") if not val: raise types.InvalidRequestSetup( - "Missing keyword in request entry: 'value'") + "Missing keyword in request entry: 'value'" + ) field = r_dup.get("field") if not field: raise types.InvalidRequestSetup( - "Missing keyword in request entry: 'field'") + "Missing keyword in request entry: 'field'" + ) spurious_kwords = set(r_dup.keys()) - self.VALID_REQUEST_KWORDS if spurious_kwords: raise types.InvalidRequestSetup( "Spurious keyword(s) in request entry: {}".format( - ", ".join(f"'{kword}'" for kword in spurious_kwords))) + ", ".join(f"'{kword}'" for kword in spurious_kwords) + ) + ) input_parameter = r_dup.get("input_parameter") if input_parameter: - self._handle_lvalue(input_parameter, wrappers.Field( - field_pb=descriptor_pb2.FieldDescriptorProto())) + self._handle_lvalue( + input_parameter, + wrappers.Field( + field_pb=descriptor_pb2.FieldDescriptorProto()), + ) # The percentage sign is used for setting up resource based requests - percent_idx = field.find('%') + percent_idx = field.find("%") if percent_idx == -1: base_param, attr = self._normal_request_setup( - base_param_to_attrs, val, r_dup, field) + base_param_to_attrs, val, r_dup, field + ) request_entry = base_param_to_attrs.get(base_param) if request_entry and request_entry.is_resource_request: raise types.ResourceRequestMismatch( - f"Request setup mismatch for base: {base_param}") + f"Request setup mismatch for base: {base_param}" + ) base_param_to_attrs[base_param].attrs.append(attr) else: # It's a resource based request. - base_param, resource_attr = (field[:percent_idx], - field[percent_idx + 1:]) + base_param, resource_attr = ( + field[:percent_idx], + field[percent_idx + 1:], + ) request_entry = base_param_to_attrs.get(base_param) if request_entry and not request_entry.is_resource_request: raise types.ResourceRequestMismatch( - f"Request setup mismatch for base: {base_param}") + f"Request setup mismatch for base: {base_param}" + ) if not self.request_type_.fields.get(base_param): raise types.BadAttributeLookup( "Method request type {} has no attribute: '{}'".format( - self.request_type_, base_param)) + self.request_type_, base_param + ) + ) r_dup["field"] = resource_attr request_entry = base_param_to_attrs[base_param] request_entry.is_resource_request = True request_entry.attrs.append( - AttributeRequestSetup(**r_dup)) # type: ignore + AttributeRequestSetup(**r_dup) # type: ignore + ) client_streaming_forms = { types.CallingForm.RequestStreamingClient, @@ -490,7 +544,8 @@ def validate_and_transform_request(self, if len(base_param_to_attrs) > 1 and calling_form in client_streaming_forms: raise types.InvalidRequestSetup( - "Too many base parameters for client side streaming form") + "Too many base parameters for client side streaming form" + ) # We can only flatten a collection of request parameters if they're a # subset of the flattened fields of the method. @@ -502,11 +557,11 @@ def validate_and_transform_request(self, self.api_schema_, key, val.attrs, - val.is_resource_request + val.is_resource_request, ) for key, val in base_param_to_attrs.items() ], - flattenable=flattenable + flattenable=False, ) def validate_response(self, response): @@ -535,7 +590,8 @@ def validate_response(self, response): validater = self.STATEMENT_DISPATCH_TABLE.get(keyword) if not validater: raise types.InvalidStatement( - "Invalid statement keyword: {}".format(keyword)) + "Invalid statement keyword: {}".format(keyword) + ) validater(self, body) @@ -558,34 +614,45 @@ def validate_expression(self, exp: str) -> wrappers.Field: Returns: wrappers.Field: The final field in the chain. """ + def validate_recursively(expression, scope, depth=0): first_dot = expression.find(".") base = expression[:first_dot] if first_dot > 0 else expression match = self.EXPRESSION_ATTR_RE.match(base) if not match: raise types.BadAttributeLookup( - f"Badly formed attribute expression: {expression}") + f"Badly formed attribute expression: {expression}" + ) - name, idxed, mapped = (match.groupdict()["attr_name"], - bool(match.groupdict()["index"]), - bool(match.groupdict()["key"])) + name, idxed, mapped = ( + match.groupdict()["attr_name"], + bool(match.groupdict()["index"]), + bool(match.groupdict()["key"]), + ) field = scope.get(name) + if not field: - exception_class = (types.BadAttributeLookup if depth else - types.UndefinedVariableReference) + exception_class = ( + types.BadAttributeLookup + if depth + else types.UndefinedVariableReference + ) raise exception_class(f"No such variable or attribute: {name}") # Invalid input if (idxed or mapped) and not field.repeated: raise types.BadAttributeLookup( - f"Collection lookup on non-repeated field: {base}") + f"Collection lookup on non-repeated field: {base}" + ) # Can only ignore indexing or mapping in an indexed (or mapped) field # if it is the terminal point in the expression. if field.repeated and not (idxed or mapped) and first_dot != -1: raise types.BadAttributeLookup( - ("Accessing attribute on a non-terminal collection without" - f"indexing into the collection: {base}") + ( + "Accessing attribute on a non-terminal collection without" + f"indexing into the collection: {base}" + ) ) message = field.message @@ -601,12 +668,14 @@ def validate_recursively(expression, scope, depth=0): value_field = message.fields.get("value") if not value_field: raise types.BadAttributeLookup( - f"Mapped attribute has no value field: {base}") + f"Mapped attribute has no value field: {base}" + ) value_message = value_field.message if not value_message: raise types.BadAttributeLookup( - f"Mapped value field is not a message: {base}") + f"Mapped value field is not a message: {base}" + ) if first_dot != -1: scope = value_message.fields @@ -618,11 +687,10 @@ def validate_recursively(expression, scope, depth=0): # Enums and primitives are only allowed at the tail of an expression. if not message: raise types.BadAttributeLookup( - f"Non-terminal attribute is not a message: {base}") + f"Non-terminal attribute is not a message: {base}" + ) - return validate_recursively(expression[first_dot + 1:], - scope, - depth + 1) + return validate_recursively(expression[first_dot + 1:], scope, depth + 1) return validate_recursively(exp, self.var_defs_) @@ -664,9 +732,7 @@ def _validate_format(self, body: List[str]): if num_prints != len(body) - 1: raise types.MismatchedFormatSpecifier( "Expected {} expresssions in format string '{}' but found {}".format( - num_prints, - fmt_str, - len(body) - 1 + num_prints, fmt_str, len(body) - 1 ) ) @@ -714,14 +780,16 @@ def _validate_write_file(self, body): fname_fmt = body.get("filename") if not fname_fmt: raise types.InvalidStatement( - "Missing key in 'write_file' statement: 'filename'") + "Missing key in 'write_file' statement: 'filename'" + ) self._validate_format(fname_fmt) contents_var = body.get("contents") if not contents_var: raise types.InvalidStatement( - "Missing key in 'write_file' statement: 'contents'") + "Missing key in 'write_file' statement: 'contents'" + ) self.validate_expression(contents_var) @@ -775,13 +843,14 @@ def _validate_loop(self, loop): # TODO: resolve the implicit $resp dilemma # if collection_name.startswith("."): # collection_name = "$resp" + collection_name - collection_field = self.validate_expression( - loop[self.COLL_KWORD]) + collection_field = self.validate_expression(loop[self.COLL_KWORD]) if not collection_field.repeated: raise types.BadLoop( "Tried to use a non-repeated field as a collection: {}".format( - tokens[-1])) + tokens[-1] + ) + ) var = loop[self.VAR_KWORD] # The collection_field is repeated, @@ -792,8 +861,8 @@ def _validate_loop(self, loop): field_pb=collection_field.field_pb, message=collection_field.message, enum=collection_field.enum, - meta=collection_field.meta - ) + meta=collection_field.meta, + ), ) elif map_args <= segments: @@ -817,7 +886,8 @@ def _validate_loop(self, loop): if not (key or val): raise types.BadLoop( - "Need at least one of 'key' or 'value' in a map loop") + "Need at least one of 'key' or 'value' in a map loop" + ) else: raise types.BadLoop("Unexpected loop form: {}".format(segments)) @@ -838,17 +908,70 @@ def _validate_loop(self, loop): } -def generate_sample( - sample, - api_schema, - sample_template: jinja2.Template -) -> str: +def parse_handwritten_specs(sample_configs: Sequence[str]) -> Generator[Dict[str, Any], None, None]: + """Parse a handwritten sample spec""" + + STANDALONE_TYPE = "standalone" + + for config_fpath in sample_configs: + with open(config_fpath) as f: + configs = yaml.safe_load_all(f.read()) + + for cfg in configs: + valid = is_valid_sample_cfg(cfg) + if not valid: + raise types.InvalidConfig( + "Sample config is invalid", valid) + for spec in cfg.get("samples", []): + # If unspecified, assume a sample config describes a standalone. + # If sample_types are specified, standalone samples must be + # explicitly enabled. + if STANDALONE_TYPE in spec.get("sample_type", [STANDALONE_TYPE]): + yield spec + + +def generate_sample_specs(api_schema: api.API, *, opts) -> Generator[Dict[str, Any], None, None]: + """Given an API, generate basic sample specs for each method. + + Args: + api_schema (api.API): The schema that defines the API. + + Yields: + Dict[str, Any]: A sample spec. + """ + + gapic_metadata = api_schema.gapic_metadata(opts) + + for service_name, service in gapic_metadata.services.items(): + api_short_name = api_schema.services[f"{api_schema.naming.proto_package}.{service_name}"].shortname + for transport_type, client in service.clients.items(): + if transport_type == "grpc-async": + # TODO(busunkim): Enable generation of async samples + continue + for rpc_name, method_list in client.rpcs.items(): + # Region Tag Format: + # [{START|END} ${apishortname}_generated_${api}_${apiVersion}_${serviceName}_${rpcName}_{sync|async}_${overloadDisambiguation}] + region_tag = f"{api_short_name}_generated_{api_schema.naming.versioned_module_name}_{service_name}_{rpc_name}_{transport_type}" + spec = { + "sample_type": "standalone", + "rpc": rpc_name, + "request": [], + # response is populated in `preprocess_sample` + "service": f"{api_schema.naming.proto_package}.{service_name}", + "region_tag": region_tag, + "description": f"Snippet for {utils.to_snake_case(rpc_name)}" + } + + yield spec + + +def generate_sample(sample, api_schema, sample_template: jinja2.Template) -> str: """Generate a standalone, runnable sample. Writing the rendered output is left for the caller. Args: - sample (Any): A definition for a single sample generated from parsed yaml. + sample (Any): A definition for a single sample. api_schema (api.API): The schema that defines the API to which the sample belongs. sample_template (jinja2.Template): The template representing a generic sample. @@ -871,21 +994,19 @@ def generate_sample( calling_form = types.CallingForm.method_default(rpc) v = Validator(rpc) - # Tweak some small aspects of the sample to set sane defaults for optional + # Tweak some small aspects of the sample to set defaults for optional # fields, add fields that are required for the template, and so forth. - v.preprocess_sample(sample, api_schema) - sample["request"] = v.validate_and_transform_request(calling_form, - sample["request"]) + v.preprocess_sample(sample, api_schema, rpc) + sample["request"] = v.validate_and_transform_request( + calling_form, sample["request"] + ) v.validate_response(sample["response"]) return sample_template.render( sample=sample, - imports=[ - "from google import auth", - "from google.auth import credentials", - ], + imports=[], calling_form=calling_form, calling_form_enum=types.CallingForm, - api=api_schema, - service=service, + trim_blocks=True, + lstrip_blocks=True, ) diff --git a/gapic/samplegen_utils/types.py b/gapic/samplegen_utils/types.py index dfd89c8098..48a086f953 100644 --- a/gapic/samplegen_utils/types.py +++ b/gapic/samplegen_utils/types.py @@ -123,6 +123,3 @@ def method_default(cls, m): return cls.RequestStreamingServer return cls.Request - - def __str__(self): - return to_snake_case(super().__str__().split(".")[-1]) diff --git a/gapic/samplegen_utils/utils.py b/gapic/samplegen_utils/utils.py index a0d9892e9b..7cf0a14a39 100644 --- a/gapic/samplegen_utils/utils.py +++ b/gapic/samplegen_utils/utils.py @@ -47,7 +47,7 @@ def is_valid_sample_cfg( min_version: Tuple[int, int, int] = MIN_SCHEMA_VERSION, config_type: str = VALID_CONFIG_TYPE, ) -> bool: - """Predicate that takes a parsed yaml doc checks if it is a valid sampel config. + """Predicate that takes a parsed yaml doc checks if it is a valid sample config. Arguments: doc (Any): The yaml document to be assessed diff --git a/gapic/schema/wrappers.py b/gapic/schema/wrappers.py index 442528b736..962407aa9c 100644 --- a/gapic/schema/wrappers.py +++ b/gapic/schema/wrappers.py @@ -1089,6 +1089,19 @@ def host(self) -> str: return self.options.Extensions[client_pb2.default_host] return '' + @property + def shortname(self) -> str: + """Return the API short name. DRIFT uses this to identify + APIs. + + Returns: + str: The api shortname. + """ + # Get the shortname from the host + # Real APIs are expected to have format: + # "{api_shortname}.googleapis.com" + return self.host.split(".")[0] + @property def oauth_scopes(self) -> Sequence[str]: """Return a sequence of oauth scopes, if applicable. diff --git a/gapic/templates/_base.py.j2 b/gapic/templates/_base.py.j2 index 133cf7aa58..35d3c9100f 100644 --- a/gapic/templates/_base.py.j2 +++ b/gapic/templates/_base.py.j2 @@ -2,5 +2,5 @@ {% block license %} {% include "_license.j2" %} {% endblock %} -{%- block content %} +{% block content %} {% endblock %} diff --git a/gapic/templates/docs/%name_%version/services.rst.j2 b/gapic/templates/docs/%name_%version/services.rst.j2 index 98ba64f60f..442a48cab6 100644 --- a/gapic/templates/docs/%name_%version/services.rst.j2 +++ b/gapic/templates/docs/%name_%version/services.rst.j2 @@ -3,6 +3,6 @@ Services for {{ api.naming.long_name }} {{ api.naming.version }} API .. toctree:: :maxdepth: 2 - {% for service in api.services.values()|sort(attribute='name') -%} + {% for service in api.services.values()|sort(attribute='name') %} {{ service.name|snake_case }} {% endfor %} diff --git a/gapic/templates/examples/feature_fragments.j2 b/gapic/templates/examples/feature_fragments.j2 index 4702d596b6..2959157a30 100644 --- a/gapic/templates/examples/feature_fragments.j2 +++ b/gapic/templates/examples/feature_fragments.j2 @@ -13,23 +13,17 @@ # See the License for the specific language governing permissions and # limitations under the License. #} - -{# -A careful reader may comment that there is duplication of effort -between the python verification step and the dispatch/rendering here. -There is a little, but not enough for it to be important because -1) Other python artifacts (client libraries, unit tests, and so forth) - are generated using templates, so doing the same for generated samples is consistent. -2) Using jinja for anything requiring real logic or data structures is a bad idea. -#} - {# response handling macros #} {% macro sample_header(sample, calling_form) %} -# DO NOT EDIT! This is a generated sample ("{{ calling_form }}", "{{ sample.id }}") +# Generated code. DO NOT EDIT! # +# Snippet for {{ sample.rpc }} +# NOTE: This snippet has been automatically generated for illustrative purposes only. +# It may require modifications to work in your environment. + # To install the latest published package dependency, execute the following: -# pip3 install {{ sample.package_name }} +# python3 -m pip install {{ sample.package_name }} {% endmacro %} {% macro print_string_formatting(string_list) %} @@ -147,7 +141,14 @@ with open({{ attr.input_parameter }}, "rb") as f: {% endif %} {% endmacro %} -{% macro render_request_setup(full_request) %} + +{% macro render_client_setup(module_name, client_name) %} +# Create a client +client = {{ module_name }}.{{ client_name }}() +{% endmacro %}} + +{% macro render_request_setup(full_request, module_name, request_type) %} +# Initialize request argument(s) {% for parameter_block in full_request.request_list if parameter_block.body %} {% if parameter_block.pattern %} {# This is a resource-name patterned lookup parameter #} @@ -165,15 +166,16 @@ with open({{ attr.input_parameter }}, "rb") as f: {% endif %} {% endfor %} {% if not full_request.flattenable %} -request = { +request = {{ module_name }}.{{ request_type }}( {% for parameter in full_request.request_list %} - '{{ parameter.base }}': {{ parameter.base if parameter.body else parameter.single }}, -{% endfor %}} + {{ parameter.base }}={{ parameter.base if parameter.body else parameter.single }}, +{% endfor %} +) {% endif %} {% endmacro %} {% macro render_request_params(request) %} -{# Provide the top level parameters last and as keyword params #} + {# Provide the top level parameters last and as keyword params #} {% with params = [] %} {% for r in request if r.body %} {% do params.append(r.base) %} @@ -186,13 +188,13 @@ request = { {% endmacro %} {% macro render_request_params_unary(request) %} -{# Provide the top level parameters last and as keyword params #} + {# Provide the top level parameters last and as keyword params #} {% if request.flattenable %} {% with params = [] %} {% for r in request.request_list %} {% do params.append("%s=%s"|format(r.base, r.single.value if r.single else r.base)) %} {% endfor %} -{{ params|join(", ") }} +{{ params|join(", ") -}} {% endwith %} {% else %} request=request @@ -214,8 +216,11 @@ client.{{ sample.rpc|snake_case }}({{ render_request_params_unary(sample.request {# Setting up the method invocation is the responsibility of the caller: #} {# it's just easier to set up client side streaming and other things from outside this macro. #} {% macro render_calling_form(method_invocation_text, calling_form, calling_form_enum, response_statements ) %} +# Make the request {% if calling_form == calling_form_enum.Request %} response = {{ method_invocation_text|trim }} + +# Handle response {% for statement in response_statements %} {{ dispatch_statement(statement)|trim }} {% endfor %} diff --git a/gapic/templates/examples/sample.py.j2 b/gapic/templates/examples/sample.py.j2 index 4cdb81e47c..79614d71a3 100644 --- a/gapic/templates/examples/sample.py.j2 +++ b/gapic/templates/examples/sample.py.j2 @@ -12,7 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -#} + #} {% extends "_base.py.j2" %} {% block content %} @@ -27,23 +27,20 @@ {% for import_statement in imports %} {{ import_statement }} {% endfor %} -from {{ (api.naming.module_namespace + (api.naming.versioned_module_name,) + service.meta.address.subpackage)|join(".") }}.services.{{ service.name|snake_case }} import {{ service.client_name }} +from {{ sample.module_namespace|join(".") }} import {{ sample.module_name }} + {# also need calling form #} def sample_{{ frags.render_method_name(sample.rpc)|trim }}({{ frags.print_input_params(sample.request)|trim }}): """{{ sample.description }}""" - client = {{ service.client_name }}( - credentials=credentials.AnonymousCredentials(), - transport="grpc", - ) - - {{ frags.render_request_setup(sample.request)|indent }} -{% with method_call = frags.render_method_call(sample, calling_form, calling_form_enum) %} - {{ frags.render_calling_form(method_call, calling_form, calling_form_enum, sample.response, )|indent }} -{% endwith %} + {{ frags.render_client_setup(sample.module_name, sample.client_name)|indent }} + {{ frags.render_request_setup(sample.request, sample.module_name, sample.request_type)|indent }} + {% with method_call = frags.render_method_call(sample, calling_form, calling_form_enum) %} + {{ frags.render_calling_form(method_call, calling_form, calling_form_enum, sample.response)|indent -}} + {% endwith %} # [END {{ sample.id }}] - -{{ frags.render_main_block(sample.rpc, sample.request) -}} +{# TODO: Enable main block (or decide to remove main block from python sample) #} +{# {{ frags.render_main_block(sample.rpc, sample.request) }} #} {% endblock %} diff --git a/gapic/templates/noxfile.py.j2 b/gapic/templates/noxfile.py.j2 index 87b5ef2a00..c2d08a4807 100644 --- a/gapic/templates/noxfile.py.j2 +++ b/gapic/templates/noxfile.py.j2 @@ -64,11 +64,11 @@ def mypy(session): session.run( 'mypy', '--explicit-package-bases', - {%- if api.naming.module_namespace %} + {% if api.naming.module_namespace %} '{{ api.naming.module_namespace[0] }}', - {%- else %} + {% else %} '{{ api.naming.versioned_module_name }}', - {%- endif %} + {% endif %} ) diff --git a/gapic/utils/options.py b/gapic/utils/options.py index 826c7734bc..d7bbe2473d 100644 --- a/gapic/utils/options.py +++ b/gapic/utils/options.py @@ -37,6 +37,7 @@ class Options: warehouse_package_name: str = '' retry: Optional[Dict[str, Any]] = None sample_configs: Tuple[str, ...] = dataclasses.field(default=()) + autogen_snippets: bool = False templates: Tuple[str, ...] = dataclasses.field(default=('DEFAULT',)) lazy_import: bool = False old_naming: bool = False @@ -54,6 +55,7 @@ class Options: 'old-naming', # TODO(dovs): Come up with a better comment 'retry-config', # takes a path 'samples', # output dir + 'autogen-snippets', # produce auto-generated snippets # transport type(s) delineated by '+' (i.e. grpc, rest, custom.[something], etc?) 'transport', 'warehouse-package-name', # change the package name on PyPI @@ -141,6 +143,7 @@ def tweak_path(p): for s in sample_paths for cfg_path in samplegen_utils.generate_all_sample_fpaths(s) ), + autogen_snippets=bool(opts.pop("autogen-snippets", False)), templates=tuple(path.expanduser(i) for i in templates), lazy_import=bool(opts.pop('lazy-import', False)), old_naming=bool(opts.pop('old-naming', False)), diff --git a/noxfile.py b/noxfile.py index e17277a487..0924ef5999 100644 --- a/noxfile.py +++ b/noxfile.py @@ -13,13 +13,16 @@ # limitations under the License. from __future__ import absolute_import +from pathlib import Path import os +import sys import tempfile import typing import nox # type: ignore from contextlib import contextmanager from os import path +import shutil showcase_version = "0.11.0" @@ -74,6 +77,9 @@ def showcase_library( # Install gapic-generator-python session.install("-e", ".") + # Install grpcio-tools for protoc + session.install("grpcio-tools") + # Install a client library for Showcase. with tempfile.TemporaryDirectory() as tmp_dir: # Download the Showcase descriptor. @@ -96,7 +102,9 @@ def showcase_library( opts = "--python_gapic_opt=" opts += ",".join(other_opts + (f"{template_opt}",)) cmd_tup = ( - f"protoc", + "python", + "-m", + "grpc_tools.protoc", f"--experimental_allow_proto3_optional", f"--descriptor_set_in={tmp_dir}{path.sep}showcase.desc", opts, @@ -205,11 +213,11 @@ def showcase_unit( with showcase_library(session, templates=templates, other_opts=other_opts) as lib: session.chdir(lib) - + # Unit tests are run twice with different dependencies to exercise # all code paths. # TODO(busunkim): remove when default templates require google-auth>=1.25.0 - + # 1. Run tests at lower bound of dependencies session.install("nox") session.run("nox", "-s", "update_lower_bounds") @@ -217,7 +225,7 @@ def showcase_unit( # Some code paths require an older version of google-auth. # google-auth is a transitive dependency so it isn't in the # lower bound constraints file produced above. - session.install("google-auth==1.21.1") + session.install("google-auth==1.21.1") run_showcase_unit_tests(session, fail_under=0) # 2. Run the tests again with latest version of dependencies @@ -241,7 +249,7 @@ def showcase_unit_add_iam_methods(session): # Unit tests are run twice with different dependencies to exercise # all code paths. # TODO(busunkim): remove when default templates require google-auth>=1.25.0 - + # 1. Run tests at lower bound of dependencies session.install("nox") session.run("nox", "-s", "update_lower_bounds") @@ -249,7 +257,7 @@ def showcase_unit_add_iam_methods(session): # Some code paths require an older version of google-auth. # google-auth is a transitive dependency so it isn't in the # lower bound constraints file produced above. - session.install("google-auth==1.21.1") + session.install("google-auth==1.21.1") run_showcase_unit_tests(session, fail_under=0) # 2. Run the tests again with latest version of dependencies @@ -279,6 +287,34 @@ def showcase_mypy_alternative_templates(session): showcase_mypy(session, templates=ADS_TEMPLATES, other_opts=("old-naming",)) +@nox.session(python="3.8") +def snippetgen(session): + # Clone googleapis/api-common-protos which are referenced by the snippet + # protos + api_common_protos = "api-common-protos" + try: + session.run("git", "-C", api_common_protos, "pull", external=True) + except nox.command.CommandFailed: + session.run( + "git", + "clone", + "--single-branch", + f"https://github.com/googleapis/{api_common_protos}", + external=True, + ) + + # Install gapic-generator-python + session.install("-e", ".") + + session.install("grpcio-tools", "mock", "pytest", "pytest-asyncio") + + session.run( + "py.test", + "--quiet", + "tests/snippetgen" + ) + + @nox.session(python="3.8") def docs(session): """Build the docs.""" diff --git a/tests/snippetgen/goldens/mollusca_generated_mollusca_v1_snippets_list_resources_grpc.py b/tests/snippetgen/goldens/mollusca_generated_mollusca_v1_snippets_list_resources_grpc.py new file mode 100644 index 0000000000..1ea032b5d9 --- /dev/null +++ b/tests/snippetgen/goldens/mollusca_generated_mollusca_v1_snippets_list_resources_grpc.py @@ -0,0 +1,45 @@ +# -*- coding: utf-8 -*- +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Generated code. DO NOT EDIT! +# +# Snippet for ListResources +# NOTE: This snippet has been automatically generated for illustrative purposes only. +# It may require modifications to work in your environment. + +# To install the latest published package dependency, execute the following: +# python3 -m pip install animalia-mollusca + + +# [START mollusca_generated_mollusca_v1_Snippets_ListResources_grpc] +from animalia import mollusca_v1 + + +def sample_list_resources(): + """Snippet for list_resources""" + + # Create a client + client = mollusca_v1.SnippetsClient() + + # Initialize request argument(s) + request = mollusca_v1.ListResourcesRequest( + ) + + # Make the request + page_result = client.list_resources(request=request) + for response in page_result: + print("{}".format(response)) + +# [END mollusca_generated_mollusca_v1_Snippets_ListResources_grpc] diff --git a/tests/snippetgen/goldens/mollusca_generated_mollusca_v1_snippets_method_bidi_streaming_grpc.py b/tests/snippetgen/goldens/mollusca_generated_mollusca_v1_snippets_method_bidi_streaming_grpc.py new file mode 100644 index 0000000000..1c9be7560f --- /dev/null +++ b/tests/snippetgen/goldens/mollusca_generated_mollusca_v1_snippets_method_bidi_streaming_grpc.py @@ -0,0 +1,45 @@ +# -*- coding: utf-8 -*- +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Generated code. DO NOT EDIT! +# +# Snippet for MethodBidiStreaming +# NOTE: This snippet has been automatically generated for illustrative purposes only. +# It may require modifications to work in your environment. + +# To install the latest published package dependency, execute the following: +# python3 -m pip install animalia-mollusca + + +# [START mollusca_generated_mollusca_v1_Snippets_MethodBidiStreaming_grpc] +from animalia import mollusca_v1 + + +def sample_method_bidi_streaming(): + """Snippet for method_bidi_streaming""" + + # Create a client + client = mollusca_v1.SnippetsClient() + + # Initialize request argument(s) + request = mollusca_v1.SignatureRequest( + ) + + # Make the request + stream = client.method_bidi_streaming([]) + for response in stream: + print("{}".format(response)) + +# [END mollusca_generated_mollusca_v1_Snippets_MethodBidiStreaming_grpc] diff --git a/tests/snippetgen/goldens/mollusca_generated_mollusca_v1_snippets_method_lro_signatures_grpc.py b/tests/snippetgen/goldens/mollusca_generated_mollusca_v1_snippets_method_lro_signatures_grpc.py new file mode 100644 index 0000000000..50974d82b3 --- /dev/null +++ b/tests/snippetgen/goldens/mollusca_generated_mollusca_v1_snippets_method_lro_signatures_grpc.py @@ -0,0 +1,48 @@ +# -*- coding: utf-8 -*- +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Generated code. DO NOT EDIT! +# +# Snippet for MethodLroSignatures +# NOTE: This snippet has been automatically generated for illustrative purposes only. +# It may require modifications to work in your environment. + +# To install the latest published package dependency, execute the following: +# python3 -m pip install animalia-mollusca + + +# [START mollusca_generated_mollusca_v1_Snippets_MethodLroSignatures_grpc] +from animalia import mollusca_v1 + + +def sample_method_lro_signatures(): + """Snippet for method_lro_signatures""" + + # Create a client + client = mollusca_v1.SnippetsClient() + + # Initialize request argument(s) + request = mollusca_v1.SignatureRequest( + ) + + # Make the request + operation = client.method_lro_signatures(request=request) + + print("Waiting for operation to complete...") + + response = operation.result() + print("{}".format(response)) + +# [END mollusca_generated_mollusca_v1_Snippets_MethodLroSignatures_grpc] diff --git a/tests/snippetgen/goldens/mollusca_generated_mollusca_v1_snippets_method_one_signature_grpc.py b/tests/snippetgen/goldens/mollusca_generated_mollusca_v1_snippets_method_one_signature_grpc.py new file mode 100644 index 0000000000..9c6192b43f --- /dev/null +++ b/tests/snippetgen/goldens/mollusca_generated_mollusca_v1_snippets_method_one_signature_grpc.py @@ -0,0 +1,46 @@ +# -*- coding: utf-8 -*- +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Generated code. DO NOT EDIT! +# +# Snippet for MethodOneSignature +# NOTE: This snippet has been automatically generated for illustrative purposes only. +# It may require modifications to work in your environment. + +# To install the latest published package dependency, execute the following: +# python3 -m pip install animalia-mollusca + + +# [START mollusca_generated_mollusca_v1_Snippets_MethodOneSignature_grpc] +from animalia import mollusca_v1 + + +def sample_method_one_signature(): + """Snippet for method_one_signature""" + + # Create a client + client = mollusca_v1.SnippetsClient() + + # Initialize request argument(s) + request = mollusca_v1.SignatureRequest( + ) + + # Make the request + response = client.method_one_signature(request=request) + + # Handle response + print("{}".format(response)) + +# [END mollusca_generated_mollusca_v1_Snippets_MethodOneSignature_grpc] diff --git a/tests/snippetgen/goldens/mollusca_generated_mollusca_v1_snippets_method_server_streaming_grpc.py b/tests/snippetgen/goldens/mollusca_generated_mollusca_v1_snippets_method_server_streaming_grpc.py new file mode 100644 index 0000000000..13913a0ed3 --- /dev/null +++ b/tests/snippetgen/goldens/mollusca_generated_mollusca_v1_snippets_method_server_streaming_grpc.py @@ -0,0 +1,45 @@ +# -*- coding: utf-8 -*- +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Generated code. DO NOT EDIT! +# +# Snippet for MethodServerStreaming +# NOTE: This snippet has been automatically generated for illustrative purposes only. +# It may require modifications to work in your environment. + +# To install the latest published package dependency, execute the following: +# python3 -m pip install animalia-mollusca + + +# [START mollusca_generated_mollusca_v1_Snippets_MethodServerStreaming_grpc] +from animalia import mollusca_v1 + + +def sample_method_server_streaming(): + """Snippet for method_server_streaming""" + + # Create a client + client = mollusca_v1.SnippetsClient() + + # Initialize request argument(s) + request = mollusca_v1.SignatureRequest( + ) + + # Make the request + stream = client.method_server_streaming(request=request) + for response in stream: + print("{}".format(response)) + +# [END mollusca_generated_mollusca_v1_Snippets_MethodServerStreaming_grpc] diff --git a/tests/snippetgen/snippets.proto b/tests/snippetgen/snippets.proto new file mode 100644 index 0000000000..6aaa404bcf --- /dev/null +++ b/tests/snippetgen/snippets.proto @@ -0,0 +1,106 @@ +// -*- coding: utf-8 -*- +// Copyright 2021 Google LLC + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + + +syntax = "proto3"; + +package animalia.mollusca.v1; + +import "google/api/annotations.proto"; +import "google/api/client.proto"; +import "google/api/field_behavior.proto"; +import "google/api/resource.proto"; +import "google/longrunning/operations.proto"; +import "google/protobuf/wrappers.proto"; + +service Snippets { + option (google.api.default_host) = "mollusca.example.com"; + + rpc MethodOneSignature(SignatureRequest) returns(Response) { + option (google.api.method_signature) = "a_string,an_int,a_bool"; + } + + rpc MethodLroSignatures(SignatureRequest) returns(google.longrunning.Operation) { + option (google.api.method_signature) = "a_string,an_int,a_bool"; + option (google.longrunning.operation_info) = { + response_type: "LroResponse" + metadata_type: "LroMetadata" + }; + } + + rpc ListResources(ListResourcesRequest) returns (ListResourcesResponse) { + option (google.api.http) = { + get: "/v1/{parent=items/*}/resources" + }; + option (google.api.method_signature) = "parent"; + } + + rpc MethodServerStreaming(SignatureRequest) returns(stream Response) { + option (google.api.method_signature) = "a_string,a_bool"; + option (google.api.method_signature) = ""; + } + + rpc MethodBidiStreaming(stream SignatureRequest) returns (stream Response); +} + +message ListResourcesRequest { + string parent = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + child_type: "snippets.example.com/Resource" + }]; + + int32 page_size = 2; + string page_token = 3; +} + +message ListResourcesResponse { + repeated Resource resources = 1; + string next_page_token = 2; +} + +message ParentResource { + option (google.api.resource) = { + type: "snippets.example.com/ParentResource" + pattern: "items/{item_id}" + }; + string name = 1; +} + +message Resource { + option (google.api.resource) = { + type: "snippets.example.com/Resource" + pattern: "items/{item_id}/parts/{part_id}" + }; + string name = 1; +} + + +message SignatureRequest { + string a_string = 1; + int32 an_int = 2; + bool a_bool = 3; + map map_int_string = 4; +} + +message Response { +} + +message LroResponse { +} + +message LroMetadata { +} + diff --git a/tests/snippetgen/test_snippetgen.py b/tests/snippetgen/test_snippetgen.py new file mode 100644 index 0000000000..389e7c5334 --- /dev/null +++ b/tests/snippetgen/test_snippetgen.py @@ -0,0 +1,81 @@ +# -*- coding: utf-8 -*- +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile + +import pytest + + +CURRENT_DIRECTORY = Path(__file__).parent.absolute() +REPO_ROOT = CURRENT_DIRECTORY.parent.parent + +GOLDEN_SNIPPETS = CURRENT_DIRECTORY / "goldens" +GENERATED_SNIPPETS = CURRENT_DIRECTORY / ".test_output" + + +def setup_module(module): + """Run protoc on modules and copy the output samples into .test_output""" + + # Delete any existing content in .test_output + # We intentionally preserve this directory between test runs to make + # it easier to inspect generated samples. + shutil.rmtree(GENERATED_SNIPPETS, ignore_errors=True) + + protos = {str(p) for p in CURRENT_DIRECTORY.glob("*.proto")} + api_common_protos = Path(REPO_ROOT / "api-common-protos").absolute() + + with tempfile.TemporaryDirectory() as tmp_dir: + # Write out a client library and samples + subprocess.check_output( + [ + "python", + "-m", + "grpc_tools.protoc", + f"--experimental_allow_proto3_optional", + "--python_gapic_opt=autogen-snippets", + f"--proto_path={CURRENT_DIRECTORY}", + f"--proto_path={api_common_protos}", + f"--python_gapic_out={tmp_dir}", + *protos, + ] + ) + + # We only care about the auto-generated samples + generated_samples = Path(tmp_dir) / "samples" / "generated_samples" + + shutil.copytree(generated_samples, GENERATED_SNIPPETS) + + +def test_files_exist(): + # The golden directory and .test_output directory + # should have exactly the same number of entries + golden_files = {p.name for p in GOLDEN_SNIPPETS.glob("*.py")} + test_output_files = {p.name for p in GENERATED_SNIPPETS.glob("*.py")} + + assert golden_files == test_output_files + + +def test_goldens(): + # Loop through the goldens directory and assert that each file + # exists in output directory and has the same code. + golden_files = GOLDEN_SNIPPETS.glob("*.py") + for golden in golden_files: + output_file = GENERATED_SNIPPETS / golden.name + assert output_file.exists() + assert golden.read_text() == output_file.read_text() diff --git a/tests/unit/generator/test_generator.py b/tests/unit/generator/test_generator.py index 7c6d850076..d068250e97 100644 --- a/tests/unit/generator/test_generator.py +++ b/tests/unit/generator/test_generator.py @@ -448,33 +448,36 @@ def test_samplegen_config_to_output_files( expected_response = CodeGeneratorResponse( file=[ CodeGeneratorResponse.File( - name="samples/squid_sample.py", content="\n",), + name="samples/generated_samples/squid_sample.py", content="\n",), CodeGeneratorResponse.File( - name="samples/clam_sample.py", content="\n",), - CodeGeneratorResponse.File( - name="samples/mollusc.v6.python.21120601.131313.manifest.yaml", - content=dedent( - """\ - --- - type: manifest/samples - schema_version: 3 - python: &python - environment: python - bin: python3 - base_path: samples - invocation: '{bin} {path} @args' - samples: - - <<: *python - sample: squid_sample - path: '{base_path}/squid_sample.py' - region_tag: humboldt_tag - - <<: *python - sample: clam_sample - path: '{base_path}/clam_sample.py' - region_tag: clam_sample - """ - ), - ), + name="samples/generated_samples/clam_sample.py", content="\n",), + # TODO(busunkim): Re-enable manifest generation once metadata + # format has been formalized. + # https://docs.google.com/document/d/1ghBam8vMj3xdoe4xfXhzVcOAIwrkbTpkMLgKc9RPD9k/edit#heading=h.sakzausv6hue + # CodeGeneratorResponse.File( + # name="samples/generated_samples/mollusc.v6.python.21120601.131313.manifest.yaml", + # content=dedent( + # """\ + # --- + # type: manifest/samples + # schema_version: 3 + # python: &python + # environment: python + # bin: python3 + # base_path: samples + # invocation: '{bin} {path} @args' + # samples: + # - <<: *python + # sample: squid_sample + # path: '{base_path}/squid_sample.py' + # region_tag: humboldt_tag + # - <<: *python + # sample: clam_sample + # path: '{base_path}/clam_sample.py' + # region_tag: clam_sample + # """ + # ), + # ), ] ) expected_response.supported_features |= ( @@ -484,6 +487,31 @@ def test_samplegen_config_to_output_files( assert actual_response == expected_response +@mock.patch( + "gapic.samplegen.samplegen.generate_sample_specs", return_value=[] +) +@mock.patch( + "gapic.samplegen.samplegen.generate_sample", return_value="", +) +def test_generate_autogen_samples(mock_generate_sample, mock_generate_specs): + opts = Options.build("autogen-snippets") + g = generator.Generator(opts) + # Need to have the sample template visible to the generator. + g._env.loader = jinja2.DictLoader({"sample.py.j2": ""}) + + api_schema = make_api(naming=naming.NewNaming( + name="Mollusc", version="v6")) + + actual_response = g.get_response(api_schema, opts=opts) + + # Just check that generate_sample_specs was called + # Correctness of the spec is tested in samplegen unit tests + mock_generate_specs.assert_called_once_with( + api_schema, + opts=opts + ) + + @mock.patch( "gapic.samplegen.samplegen.generate_sample", return_value="", ) @@ -534,40 +562,43 @@ def test_samplegen_id_disambiguation(mock_gmtime, mock_generate_sample, fs): expected_response = CodeGeneratorResponse( file=[ CodeGeneratorResponse.File( - name="samples/squid_sample_91a465c6.py", content="\n", + name="samples/generated_samples/squid_sample_91a465c6.py", content="\n", ), CodeGeneratorResponse.File( - name="samples/squid_sample_55051b38.py", content="\n", + name="samples/generated_samples/squid_sample_55051b38.py", content="\n", ), - CodeGeneratorResponse.File(name="samples/157884ee.py", + CodeGeneratorResponse.File(name="samples/generated_samples/157884ee.py", content="\n",), - CodeGeneratorResponse.File( - name="samples/mollusc.v6.python.21120601.131313.manifest.yaml", - content=dedent( - """\ - --- - type: manifest/samples - schema_version: 3 - python: &python - environment: python - bin: python3 - base_path: samples - invocation: '{bin} {path} @args' - samples: - - <<: *python - sample: squid_sample_91a465c6 - path: '{base_path}/squid_sample_91a465c6.py' - region_tag: humboldt_tag - - <<: *python - sample: squid_sample_55051b38 - path: '{base_path}/squid_sample_55051b38.py' - region_tag: squid_sample - - <<: *python - sample: 157884ee - path: '{base_path}/157884ee.py' - """ - ), - ), + # TODO(busunkim): Re-enable manifest generation once metadata + # format has been formalized. + # https://docs.google.com/document/d/1ghBam8vMj3xdoe4xfXhzVcOAIwrkbTpkMLgKc9RPD9k/edit#heading=h.sakzausv6hue + # CodeGeneratorResponse.File( + # name="samples/generated_samples/mollusc.v6.python.21120601.131313.manifest.yaml", + # content=dedent( + # """\ + # --- + # type: manifest/samples + # schema_version: 3 + # python: &python + # environment: python + # bin: python3 + # base_path: samples + # invocation: '{bin} {path} @args' + # samples: + # - <<: *python + # sample: squid_sample_91a465c6 + # path: '{base_path}/squid_sample_91a465c6.py' + # region_tag: humboldt_tag + # - <<: *python + # sample: squid_sample_55051b38 + # path: '{base_path}/squid_sample_55051b38.py' + # region_tag: squid_sample + # - <<: *python + # sample: 157884ee + # path: '{base_path}/157884ee.py' + # """ + # ), + # ), ] ) expected_response.supported_features |= ( @@ -675,35 +706,38 @@ def test_dont_generate_in_code_samples(mock_gmtime, mock_generate_sample, fs): expected = CodeGeneratorResponse( file=[ CodeGeneratorResponse.File( - name="samples/squid_sample.py", content="\n",), - CodeGeneratorResponse.File( - name="samples/whelk_sample.py", content="\n",), + name="samples/generated_samples/squid_sample.py", content="\n",), CodeGeneratorResponse.File( - name="samples/octopus_sample.py", content="\n",), + name="samples/generated_samples/whelk_sample.py", content="\n",), CodeGeneratorResponse.File( - name="samples/mollusc.v6.python.21120601.131313.manifest.yaml", - content=dedent( - """ --- - type: manifest/samples - schema_version: 3 - python: &python - environment: python - bin: python3 - base_path: samples - invocation: \'{bin} {path} @args\' - samples: - - <<: *python - sample: squid_sample - path: \'{base_path}/squid_sample.py\' - - <<: *python - sample: whelk_sample - path: \'{base_path}/whelk_sample.py\' - - <<: *python - sample: octopus_sample - path: \'{base_path}/octopus_sample.py\' - """ - ), - ), + name="samples/generated_samples/octopus_sample.py", content="\n",), + # TODO(busunkim): Re-enable manifest generation once metadata + # format has been formalized. + # https://docs.google.com/document/d/1ghBam8vMj3xdoe4xfXhzVcOAIwrkbTpkMLgKc9RPD9k/edit#heading=h.sakzausv6hue + # CodeGeneratorResponse.File( + # name="samples/generated_samples/mollusc.v6.python.21120601.131313.manifest.yaml", + # content=dedent( + # """ --- + # type: manifest/samples + # schema_version: 3 + # python: &python + # environment: python + # bin: python3 + # base_path: samples + # invocation: \'{bin} {path} @args\' + # samples: + # - <<: *python + # sample: squid_sample + # path: \'{base_path}/squid_sample.py\' + # - <<: *python + # sample: whelk_sample + # path: \'{base_path}/whelk_sample.py\' + # - <<: *python + # sample: octopus_sample + # path: \'{base_path}/octopus_sample.py\' + # """ + # ), + # ), ] ) expected.supported_features |= CodeGeneratorResponse.Feature.FEATURE_PROTO3_OPTIONAL diff --git a/tests/unit/samplegen/common_types.py b/tests/unit/samplegen/common_types.py index e073501923..0e9c8129d1 100644 --- a/tests/unit/samplegen/common_types.py +++ b/tests/unit/samplegen/common_types.py @@ -30,13 +30,17 @@ class DummyMethod: input: bool = False output: bool = False lro: bool = False + void: bool = False paged_result_field: bool = False client_streaming: bool = False server_streaming: bool = False flattened_fields: Dict[str, Any] = dataclasses.field(default_factory=dict) -DummyMessage = namedtuple("DummyMessage", ["fields", "type", "options"]) +DummyIdent = namedtuple("DummyIdent", ["name"]) + +DummyMessage = namedtuple( + "DummyMessage", ["fields", "type", "options", "ident"]) DummyMessage.__new__.__defaults__ = (False,) * len(DummyMessage._fields) DummyField = namedtuple("DummyField", @@ -50,20 +54,21 @@ class DummyMethod: "type"]) DummyField.__new__.__defaults__ = (False,) * len(DummyField._fields) -DummyService = namedtuple("DummyService", ["methods"]) +DummyService = namedtuple("DummyService", ["methods", "client_name"]) DummyApiSchema = namedtuple("DummyApiSchema", ["services", "naming", "messages"]) DummyApiSchema.__new__.__defaults__ = (False,) * len(DummyApiSchema._fields) DummyNaming = namedtuple( - "DummyNaming", ["warehouse_package_name", "name", "version"]) + "DummyNaming", ["warehouse_package_name", "name", "version", "versioned_module_name", "module_namespace"]) DummyNaming.__new__.__defaults__ = (False,) * len(DummyNaming._fields) def message_factory(exp: str, repeated_iter=itertools.repeat(False), - enum: Optional[wrappers.EnumType] = None) -> DummyMessage: + enum: Optional[wrappers.EnumType] = None, + ) -> DummyMessage: # This mimics the structure of MessageType in the wrappers module: # A MessageType has a map from field names to Fields, # and a Field has an (optional) MessageType. @@ -81,7 +86,6 @@ def message_factory(exp: str, base.fields[attr_name] = (DummyField(message=field, repeated=repeated_field) if isinstance(field, DummyMessage) else DummyField(enum=field)) - return messages[0] diff --git a/tests/unit/samplegen/golden_snippets/sample_basic.py b/tests/unit/samplegen/golden_snippets/sample_basic.py new file mode 100644 index 0000000000..5310595732 --- /dev/null +++ b/tests/unit/samplegen/golden_snippets/sample_basic.py @@ -0,0 +1,55 @@ +# -*- coding: utf-8 -*- +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Generated code. DO NOT EDIT! +# +# Snippet for Classify +# NOTE: This snippet has been automatically generated for illustrative purposes only. +# It may require modifications to work in your environment. + +# To install the latest published package dependency, execute the following: +# python3 -m pip install molluscs-v1-molluscclient + + +# [START mollusc_classify_sync] +from molluscs.v1 import molluscclient + + +def sample_classify(video, location): + """Determine the full taxonomy of input mollusc""" + + # Create a client + client = molluscclient.MolluscServiceClient() + + # Initialize request argument(s) + classify_target = {} + # video = "path/to/mollusc/video.mkv" + with open(video, "rb") as f: + classify_target["video"] = f.read() + + # location = "New Zealand" + classify_target["location_annotation"] = location + + request = molluscclient.molluscs.v1.ClassifyRequest( + classify_target=classify_target, + ) + + # Make the request + response = client.classify(request=request) + + # Handle response + print("Mollusc is a \"{}\"".format(response.taxonomy)) + +# [END mollusc_classify_sync] diff --git a/tests/unit/samplegen/golden_snippets/sample_basic_unflattenable.py b/tests/unit/samplegen/golden_snippets/sample_basic_unflattenable.py new file mode 100644 index 0000000000..5310595732 --- /dev/null +++ b/tests/unit/samplegen/golden_snippets/sample_basic_unflattenable.py @@ -0,0 +1,55 @@ +# -*- coding: utf-8 -*- +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Generated code. DO NOT EDIT! +# +# Snippet for Classify +# NOTE: This snippet has been automatically generated for illustrative purposes only. +# It may require modifications to work in your environment. + +# To install the latest published package dependency, execute the following: +# python3 -m pip install molluscs-v1-molluscclient + + +# [START mollusc_classify_sync] +from molluscs.v1 import molluscclient + + +def sample_classify(video, location): + """Determine the full taxonomy of input mollusc""" + + # Create a client + client = molluscclient.MolluscServiceClient() + + # Initialize request argument(s) + classify_target = {} + # video = "path/to/mollusc/video.mkv" + with open(video, "rb") as f: + classify_target["video"] = f.read() + + # location = "New Zealand" + classify_target["location_annotation"] = location + + request = molluscclient.molluscs.v1.ClassifyRequest( + classify_target=classify_target, + ) + + # Make the request + response = client.classify(request=request) + + # Handle response + print("Mollusc is a \"{}\"".format(response.taxonomy)) + +# [END mollusc_classify_sync] diff --git a/tests/unit/samplegen/test_integration.py b/tests/unit/samplegen/test_integration.py index 0f1d98de74..c3a2cb7d2c 100644 --- a/tests/unit/samplegen/test_integration.py +++ b/tests/unit/samplegen/test_integration.py @@ -15,6 +15,7 @@ import jinja2 import os.path as path import pytest +from pathlib import Path import gapic.utils as utils @@ -22,7 +23,8 @@ from gapic.samplegen_utils import (types, utils as gapic_utils) from gapic.schema import (naming, wrappers) -from common_types import (DummyField, DummyMessage, DummyMethod, DummyService, +from tests.unit.samplegen.common_types import (DummyField, DummyMessage, + DummyMethod, DummyService, DummyIdent, DummyApiSchema, DummyNaming, enum_factory, message_factory) from collections import namedtuple @@ -43,12 +45,19 @@ env.filters['coerce_response_name'] = gapic_utils.coerce_response_name +def golden_snippet(filename: str) -> str: + """Load the golden snippet with the name provided""" + snippet_path = Path(__file__).parent / "golden_snippets" / filename + return snippet_path.read_text() + + def test_generate_sample_basic(): # Note: the sample integration tests are needfully large # and difficult to eyeball parse. They are intended to be integration tests # that catch errors in behavior that is emergent from combining smaller features # or in features that are sufficiently small and trivial that it doesn't make sense # to have standalone tests. + input_type = DummyMessage( type="REQUEST TYPE", fields={ @@ -65,7 +74,8 @@ def test_generate_sample_basic(): }, ) ) - } + }, + ident=DummyIdent(name="molluscs.v1.ClassifyRequest") ) api_naming = naming.NewNaming( @@ -110,76 +120,7 @@ def test_generate_sample_basic(): env.get_template('examples/sample.py.j2') ) - sample_id = ("mollusc_classify_sync") - expected_str = '''# -*- coding: utf-8 -*- -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# DO NOT EDIT! This is a generated sample ("request", "%s") -# -# To install the latest published package dependency, execute the following: -# pip3 install molluscs-v1-molluscclient - - -# [START %s] -from google import auth -from google.auth import credentials -from molluscs.v1.molluscclient.services.mollusc_service import MolluscServiceClient - -def sample_classify(video, location): - """Determine the full taxonomy of input mollusc""" - - client = MolluscServiceClient( - credentials=credentials.AnonymousCredentials(), - transport="grpc", - ) - - classify_target = {} - # video = "path/to/mollusc/video.mkv" - with open(video, "rb") as f: - classify_target["video"] = f.read() - - # location = "New Zealand" - classify_target["location_annotation"] = location - - - response = client.classify(classify_target=classify_target) - print("Mollusc is a \\"{}\\"".format(response.taxonomy)) - - -# [END %s] - -def main(): - import argparse - - parser = argparse.ArgumentParser() - parser.add_argument("--video", - type=str, - default="path/to/mollusc/video.mkv") - parser.add_argument("--location", - type=str, - default="New Zealand") - args = parser.parse_args() - - sample_classify(args.video, args.location) - - -if __name__ == "__main__": - main() -''' % (sample_id, sample_id, sample_id) - - assert sample_str == expected_str + assert sample_str == golden_snippet("sample_basic.py") def test_generate_sample_basic_unflattenable(): @@ -204,7 +145,8 @@ def test_generate_sample_basic_unflattenable(): }, ) ) - } + }, + ident=DummyIdent(name="molluscs.v1.ClassifyRequest") ) api_naming = naming.NewNaming( @@ -246,79 +188,7 @@ def test_generate_sample_basic_unflattenable(): env.get_template('examples/sample.py.j2') ) - sample_id = ("mollusc_classify_sync") - expected_str = '''# -*- coding: utf-8 -*- -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# DO NOT EDIT! This is a generated sample ("request", "%s") -# -# To install the latest published package dependency, execute the following: -# pip3 install molluscs-v1-molluscclient - - -# [START %s] -from google import auth -from google.auth import credentials -from molluscs.v1.molluscclient.services.mollusc_service import MolluscServiceClient - -def sample_classify(video, location): - """Determine the full taxonomy of input mollusc""" - - client = MolluscServiceClient( - credentials=credentials.AnonymousCredentials(), - transport="grpc", - ) - - classify_target = {} - # video = "path/to/mollusc/video.mkv" - with open(video, "rb") as f: - classify_target["video"] = f.read() - - # location = "New Zealand" - classify_target["location_annotation"] = location - - request = { - 'classify_target': classify_target, - } - - response = client.classify(request=request) - print("Mollusc is a \\"{}\\"".format(response.taxonomy)) - - -# [END %s] - -def main(): - import argparse - - parser = argparse.ArgumentParser() - parser.add_argument("--video", - type=str, - default="path/to/mollusc/video.mkv") - parser.add_argument("--location", - type=str, - default="New Zealand") - args = parser.parse_args() - - sample_classify(args.video, args.location) - - -if __name__ == "__main__": - main() -''' % (sample_id, sample_id, sample_id) - - assert sample_str == expected_str + assert sample_str == golden_snippet("sample_basic_unflattenable.py") def test_generate_sample_service_not_found(): @@ -335,7 +205,7 @@ def test_generate_sample_service_not_found(): def test_generate_sample_rpc_not_found(): schema = DummyApiSchema( - {"Mollusc": DummyService({})}, DummyNaming("pkg_name")) + {"Mollusc": DummyService(methods={}, client_name="ClassifyClient")}, DummyNaming("pkg_name")) sample = {"service": "Mollusc", "rpc": "Classify"} with pytest.raises(types.RpcMethodNotFound): diff --git a/tests/unit/samplegen/test_samplegen.py b/tests/unit/samplegen/test_samplegen.py index 82e4bee1e2..32ef4411f1 100644 --- a/tests/unit/samplegen/test_samplegen.py +++ b/tests/unit/samplegen/test_samplegen.py @@ -15,8 +15,10 @@ import yaml import pytest +from textwrap import dedent from typing import (TypeVar, Sequence) from collections import (OrderedDict, namedtuple) +from google.api import client_pb2 from google.api import resource_pb2 from google.protobuf import descriptor_pb2 @@ -25,9 +27,10 @@ import gapic.samplegen_utils.yaml as gapic_yaml from gapic.schema import (api, metadata, naming) import gapic.schema.wrappers as wrappers +from gapic.utils import Options -from common_types import (DummyApiSchema, DummyField, DummyMessage, - DummyMethod, message_factory, enum_factory) +from common_types import (DummyApiSchema, DummyField, DummyIdent, DummyNaming, DummyMessage, + DummyService, DummyMethod, message_factory, enum_factory) from gapic.samplegen_utils import utils @@ -81,15 +84,18 @@ def test_define_redefinition(): def test_preprocess_sample(): # Verify that the default response is added. - sample = {} - api_schema = api.API( - naming.NewNaming( - namespace=("mollusc", "cephalopod", "teuthida") - ), - all_protos={}, + sample = {"service": "Mollusc", "rpc": "Classify"} + api_schema = DummyApiSchema( + services={"Mollusc": DummyService( + methods={}, client_name="MolluscClient")}, + naming=DummyNaming(warehouse_package_name="mollusc-cephalopod-teuthida-", + versioned_module_name="teuthida_v1", module_namespace="mollusc.cephalopod"), ) - samplegen.Validator.preprocess_sample(sample, api_schema) + rpc = DummyMethod(input=DummyMessage( + ident=DummyIdent(name="ClassifyRequest"))) + + samplegen.Validator.preprocess_sample(sample, api_schema, rpc) response = sample.get("response") assert response == [{"print": ["%s", "$resp"]}] @@ -97,6 +103,36 @@ def test_preprocess_sample(): package_name = sample.get("package_name") assert package_name == "mollusc-cephalopod-teuthida-" + module_name = sample.get("module_name") + assert module_name == "teuthida_v1" + + module_namespace = sample.get("module_namespace") + assert module_namespace == "mollusc.cephalopod" + + client_name = sample.get("client_name") + assert client_name == "MolluscClient" + + request_type = sample.get("request_type") + assert request_type == "ClassifyRequest" + + +def test_preprocess_sample_void_method(): + # Verify no response is added for a void method + sample = {"service": "Mollusc", "rpc": "Classify"} + api_schema = DummyApiSchema( + services={"Mollusc": DummyService( + methods={}, client_name="MolluscClient")}, + naming=DummyNaming(warehouse_package_name="mollusc-cephalopod-teuthida-", + versioned_module_name="teuthida_v1", module_namespace="mollusc.cephalopod"), + ) + + rpc = DummyMethod(void=True, input=DummyMessage( + ident=DummyIdent(name="ClassifyRequest"))) + + samplegen.Validator.preprocess_sample(sample, api_schema, rpc) + + assert "response" not in sample + def test_define_input_param(): v = samplegen.Validator( @@ -1220,7 +1256,8 @@ def test_regular_response_type(): def test_paged_response_type(): OutputType = TypeVar("OutputType") PagedType = TypeVar("PagedType") - method = DummyMethod(output=OutputType, paged_result_field=PagedType) + PagedField = DummyField(message=PagedType) + method = DummyMethod(output=OutputType, paged_result_field=PagedField) v = samplegen.Validator(method) assert v.var_field("$resp").message == PagedType @@ -1821,6 +1858,72 @@ def test_validate_request_non_terminal_primitive_field(): request) +def test_parse_invalid_handwritten_spec(fs): + fpath = "sampledir/sample.yaml" + fs.create_file( + fpath, + # spec is missing type + contents=dedent( + """ + --- + schema_version: 1.2.0 + samples: + - service: google.cloud.language.v1.LanguageService + """ + ), + ) + + with pytest.raises(types.InvalidConfig): + list(samplegen.parse_handwritten_specs(sample_configs=[fpath])) + + +def test_generate_sample_spec_basic(): + service_options = descriptor_pb2.ServiceOptions() + service_options.Extensions[client_pb2.default_host] = "example.googleapis.com" + + api_schema = api.API.build( + file_descriptors=[ + descriptor_pb2.FileDescriptorProto( + name="cephalopod.proto", + package="animalia.mollusca.v1", + message_type=[ + descriptor_pb2.DescriptorProto( + name="MolluscRequest", + ), + descriptor_pb2.DescriptorProto( + name="Mollusc", + ), + ], + service=[ + descriptor_pb2.ServiceDescriptorProto( + name="Squid", + options=service_options, + method=[ + descriptor_pb2.MethodDescriptorProto( + name="Ramshorn", + input_type="animalia.mollusca.v1.MolluscRequest", + output_type="animalia.mollusca.v1.Mollusc", + ), + ], + ), + ], + ) + ] + ) + opts = Options.build("transport=grpc") + specs = list(samplegen.generate_sample_specs(api_schema, opts=opts)) + assert len(specs) == 1 + + assert specs[0] == { + "sample_type": "standalone", + "rpc": "Ramshorn", + "request": [], + "service": "animalia.mollusca.v1.Squid", + "region_tag": "example_generated_mollusca_v1_Squid_Ramshorn_grpc", + "description": "Snippet for ramshorn" + } + + def make_message(name: str, package: str = 'animalia.mollusca.v1', module: str = 'cephalopoda', fields: Sequence[wrappers.Field] = (), meta: metadata.Metadata = None, options: descriptor_pb2.MethodOptions = None, diff --git a/tests/unit/samplegen/test_template.py b/tests/unit/samplegen/test_template.py index bd3e539891..0eabe9e4f0 100644 --- a/tests/unit/samplegen/test_template.py +++ b/tests/unit/samplegen/test_template.py @@ -21,6 +21,7 @@ from gapic.samplegen_utils.types import CallingForm from textwrap import dedent +from tests.unit.samplegen import common_types def check_template(template_fragment, expected_output, **kwargs): @@ -112,6 +113,7 @@ def test_render_request_basic(): {{ frags.render_request_setup(request) }} ''', ''' + # Initialize request argument(s) cephalopod = {} # cephalopod_mass = '10 kg' cephalopod["mantle_mass"] = cephalopod_mass @@ -182,9 +184,10 @@ def test_render_request_unflattened(): check_template( ''' {% import "feature_fragments.j2" as frags %} - {{ frags.render_request_setup(request) }} + {{ frags.render_request_setup(request, "mollusca", "CreateMolluscRequest") }} ''', ''' + # Initialize request argument(s) cephalopod = {} # cephalopod_mass = '10 kg' cephalopod["mantle_mass"] = cephalopod_mass @@ -205,11 +208,11 @@ def test_render_request_unflattened(): with open(movie_path, "rb") as f: gastropod["movie"] = f.read() - request = { - 'cephalopod': cephalopod, - 'gastropod': gastropod, - 'bivalve': "humboldt", - } + request = mollusca.CreateMolluscRequest( + cephalopod=cephalopod, + gastropod=gastropod, + bivalve="humboldt", + ) ''', request=samplegen.FullRequest( request_list=[ @@ -254,7 +257,9 @@ def test_render_request_unflattened(): body=None, single='"humboldt"'), ] - ) + ), + api=common_types.DummyApiSchema(), + ) @@ -265,6 +270,7 @@ def test_render_request_resource_name(): {{ frags.render_request_setup(request) }} ''', ''' + # Initialize request argument(s) taxon = "kingdom/{kingdom}/phylum/{phylum}".format(kingdom="animalia", phylum=mollusca) ''', request=samplegen.FullRequest( @@ -287,7 +293,7 @@ def test_render_request_resource_name(): ), ], flattenable=True - ) + ), ) @@ -538,7 +544,7 @@ def test_dispatch_map_loop(): print("A {} is a {}".format(example, cls)) - + ''', statement={"loop": {"map": "molluscs", "key": "cls", @@ -586,11 +592,11 @@ def test_render_nested_loop_collection(): print("Sucker: {}".format(s)) - - - - - + + + + + """, statement=statement ) @@ -639,11 +645,11 @@ def test_render_nested_loop_map(): print("Example: {}".format(ex)) - - - - - + + + + + """, statement=statement ) @@ -702,7 +708,10 @@ def test_print_input_params(): def test_render_calling_form_request(): check_template(CALLING_FORM_TEMPLATE_TEST_STR, ''' + # Make the request response = TEST_INVOCATION_TXT + + # Handle response print("Test print statement") ''', calling_form_enum=CallingForm, @@ -712,6 +721,7 @@ def test_render_calling_form_request(): def test_render_calling_form_paged_all(): check_template(CALLING_FORM_TEMPLATE_TEST_STR, ''' + # Make the request page_result = TEST_INVOCATION_TXT for response in page_result: print("Test print statement") @@ -723,6 +733,7 @@ def test_render_calling_form_paged_all(): def test_render_calling_form_paged(): check_template(CALLING_FORM_TEMPLATE_TEST_STR, ''' + # Make the request page_result = TEST_INVOCATION_TXT for page in page_result.pages(): for response in page: @@ -735,6 +746,7 @@ def test_render_calling_form_paged(): def test_render_calling_form_streaming_server(): check_template(CALLING_FORM_TEMPLATE_TEST_STR, ''' + # Make the request stream = TEST_INVOCATION_TXT for response in stream: print("Test print statement") @@ -746,6 +758,7 @@ def test_render_calling_form_streaming_server(): def test_render_calling_form_streaming_bidi(): check_template(CALLING_FORM_TEMPLATE_TEST_STR, ''' + # Make the request stream = TEST_INVOCATION_TXT for response in stream: print("Test print statement") @@ -757,6 +770,7 @@ def test_render_calling_form_streaming_bidi(): def test_render_calling_form_longrunning(): check_template(CALLING_FORM_TEMPLATE_TEST_STR, ''' + # Make the request operation = TEST_INVOCATION_TXT print("Waiting for operation to complete...")