Skip to content

Commit

Permalink
feat: add autogenerated snippets (#845)
Browse files Browse the repository at this point in the history
This PR targets iteration 1 and 2 specified in the [Snippet Gen Design](go/snippet-gen-design): Full canonical coverage of simple requests, paginated, LRO, server streaming, and Bidi streaming with empty request objects.

Snippet generation is hidden behind a new option `autogen-snippets`.

After discussion with folks on different language teams on snippetgen, I decided using "golden" snippet files would be easier than following the unit testing strategy used to check the library surface. I also believe goldens will be be easier for review for other Python DPEs.

Other notes:
- I've commented out the existing metadata generation code and tests. The new metadata format is still under discussion.
- Async samples are excluded as the existing samplegen infrastructure was written pre-async. I will add the async samples in the next PR.

Co-authored-by: Dov Shlachter <dovs@google.com>
  • Loading branch information
busunkim96 and software-dov authored May 3, 2021
1 parent 358a702 commit abdf5ec
Show file tree
Hide file tree
Showing 29 changed files with 1,217 additions and 466 deletions.
3 changes: 3 additions & 0 deletions .github/snippet-bot.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
# https://github.com/googleapis/repo-automation-bots/tree/master/packages/snippet-bot
ignoreFiles:
- "**/*.py"
20 changes: 20 additions & 0 deletions .github/workflows/tests.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -265,6 +265,26 @@ jobs:
run: python -m pip install nox
- name: Typecheck the generated output.
run: nox -s showcase_mypy${{ matrix.variant }}
snippetgen:
runs-on: ubuntu-latest
steps:
- name: Cancel Previous Runs
uses: styfle/cancel-workflow-action@0.7.0
with:
access_token: ${{ github.token }}
- uses: actions/checkout@v2
- name: Set up Python 3.8
uses: actions/setup-python@v2
with:
python-version: 3.8
- name: Install system dependencies.
run: |
sudo apt-get update
sudo apt-get install -y curl pandoc unzip gcc
- name: Install nox.
run: python -m pip install nox
- name: Check autogenerated snippets.
run: nox -s snippetgen
unit:
strategy:
matrix:
Expand Down
4 changes: 4 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -65,3 +65,7 @@ pylintrc.test

# pyenv
.python-version

# Test dependencies and output
api-common-protos
tests/snippetgen/.test_output
123 changes: 62 additions & 61 deletions gapic/generator/generator.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,10 @@

import jinja2
import yaml
import itertools
import re
import os
import typing
from typing import Any, DefaultDict, Dict, Mapping
from hashlib import sha256
from collections import OrderedDict, defaultdict
Expand Down Expand Up @@ -107,12 +109,12 @@ def get_response(
template_name, api_schema=api_schema, opts=opts)
)

sample_output = self._generate_samples_and_manifest(
api_schema,
self._env.get_template(sample_templates[0]),
) if sample_templates else {}

output_files.update(sample_output)
if sample_templates:
sample_output = self._generate_samples_and_manifest(
api_schema, self._env.get_template(sample_templates[0]),
opts=opts,
)
output_files.update(sample_output)

# Return the CodeGeneratorResponse output.
res = CodeGeneratorResponse(
Expand All @@ -121,12 +123,13 @@ def get_response(
return res

def _generate_samples_and_manifest(
self, api_schema: api.API, sample_template: jinja2.Template,
) -> Dict[str, CodeGeneratorResponse.File]:
self, api_schema: api.API, sample_template: jinja2.Template, *, opts: Options) -> Dict:
"""Generate samples and samplegen manifest for the API.
Arguments:
api_schema (api.API): The schema for the API to which the samples belong.
sample_template (jinja2.Template): The template to use to generate samples.
opts (Options): Additional generator options.
Returns:
Dict[str, CodeGeneratorResponse.File]: A dict mapping filepath to rendered file.
Expand All @@ -137,56 +140,50 @@ def _generate_samples_and_manifest(
id_to_hash_to_spec: DefaultDict[str,
Dict[str, Any]] = defaultdict(dict)

STANDALONE_TYPE = "standalone"
for config_fpath in self._sample_configs:
with open(config_fpath) as f:
configs = yaml.safe_load_all(f.read())

spec_generator = (
spec
for cfg in configs
if is_valid_sample_cfg(cfg)
for spec in cfg.get("samples", [])
# If unspecified, assume a sample config describes a standalone.
# If sample_types are specified, standalone samples must be
# explicitly enabled.
if STANDALONE_TYPE in spec.get("sample_type", [STANDALONE_TYPE])
)
# Autogenerated sample specs
autogen_specs: typing.List[typing.Dict[str, Any]] = []
if opts.autogen_snippets:
autogen_specs = list(
samplegen.generate_sample_specs(api_schema, opts=opts))

# Also process any handwritten sample specs
handwritten_specs = samplegen.parse_handwritten_specs(
self._sample_configs)

sample_specs = autogen_specs + list(handwritten_specs)

for spec in sample_specs:
# Every sample requires an ID. This may be provided
# by a samplegen config author.
# If no ID is provided, fall back to the region tag.
#
# Ideally the sample author should pick a descriptive, unique ID,
# but this may be impractical and can be error-prone.
spec_hash = sha256(str(spec).encode("utf8")).hexdigest()[:8]
sample_id = spec.get("id") or spec.get("region_tag") or spec_hash
spec["id"] = sample_id

for spec in spec_generator:
# Every sample requires an ID, preferably provided by the
# samplegen config author.
# If no ID is provided, fall back to the region tag.
# If there's no region tag, generate a unique ID.
#
# Ideally the sample author should pick a descriptive, unique ID,
# but this may be impractical and can be error-prone.
spec_hash = sha256(str(spec).encode("utf8")).hexdigest()[:8]
sample_id = spec.get("id") or spec.get(
"region_tag") or spec_hash
spec["id"] = sample_id

hash_to_spec = id_to_hash_to_spec[sample_id]
if spec_hash in hash_to_spec:
raise DuplicateSample(
f"Duplicate samplegen spec found: {spec}")

hash_to_spec[spec_hash] = spec

out_dir = "samples"
hash_to_spec = id_to_hash_to_spec[sample_id]

if spec_hash in hash_to_spec:
raise DuplicateSample(
f"Duplicate samplegen spec found: {spec}")

hash_to_spec[spec_hash] = spec

out_dir = "samples/generated_samples"
fpath_to_spec_and_rendered = {}
for hash_to_spec in id_to_hash_to_spec.values():
for spec_hash, spec in hash_to_spec.items():
id_is_unique = len(hash_to_spec) == 1
# The ID is used to generate the file name and by sample tester
# to link filenames to invoked samples. It must be globally unique.
# The ID is used to generate the file name. It must be globally unique.
if not id_is_unique:
spec["id"] += f"_{spec_hash}"

sample = samplegen.generate_sample(
spec, api_schema, sample_template,)

fpath = spec["id"] + ".py"
fpath = utils.to_snake_case(spec["id"]) + ".py"
fpath_to_spec_and_rendered[os.path.join(out_dir, fpath)] = (
spec,
sample,
Expand All @@ -199,20 +196,24 @@ def _generate_samples_and_manifest(
for fname, (_, sample) in fpath_to_spec_and_rendered.items()
}

# Only generate a manifest if we generated samples.
if output_files:
manifest_fname, manifest_doc = manifest.generate(
(
(fname, spec)
for fname, (spec, _) in fpath_to_spec_and_rendered.items()
),
api_schema,
)

manifest_fname = os.path.join(out_dir, manifest_fname)
output_files[manifest_fname] = CodeGeneratorResponse.File(
content=manifest_doc.render(), name=manifest_fname
)
# TODO(busunkim): Re-enable manifest generation once metadata
# format has been formalized.
# https://docs.google.com/document/d/1ghBam8vMj3xdoe4xfXhzVcOAIwrkbTpkMLgKc9RPD9k/edit#heading=h.sakzausv6hue
#
# if output_files:

# manifest_fname, manifest_doc = manifest.generate(
# (
# (fname, spec)
# for fname, (spec, _) in fpath_to_spec_and_rendered.items()
# ),
# api_schema,
# )

# manifest_fname = os.path.join(out_dir, manifest_fname)
# output_files[manifest_fname] = CodeGeneratorResponse.File(
# content=manifest_doc.render(), name=manifest_fname
# )

return output_files

Expand Down
Loading

0 comments on commit abdf5ec

Please sign in to comment.