From 9bbd5b221e58b56e47773b89a3fe1dc91341cde8 Mon Sep 17 00:00:00 2001 From: t-karasova Date: Mon, 24 Jan 2022 17:57:30 +0100 Subject: [PATCH 01/12] feat: Retail Interactive Tutorials. Product service code samples --- .kokoro/continuous/common.cfg | 8 + .../import_products_big_query_table.py | 105 +++++++ .../product/import_products_bq_test.py | 29 ++ .../product/import_products_gcs.py | 122 ++++++++ .../product/import_products_gcs_test.py | 32 ++ .../product/import_products_inline_source.py | 161 ++++++++++ .../product/import_products_inline_test.py | 29 ++ .../interactive-tutorials/product/noxfile.py | 279 ++++++++++++++++++ .../product/noxfile_config.py | 34 +++ .../product/requirements-test.txt | 2 + .../product/requirements.txt | 4 + .../setup/products_create_bigquery_table.py | 31 ++ .../products_create_bigquery_table_test.py | 43 +++ .../setup/products_create_gcs_bucket.py | 26 ++ .../setup/products_create_gcs_bucket_test.py | 40 +++ .../setup/products_delete_gcs_bucket.py | 25 ++ .../product/setup/setup_cleanup.py | 226 ++++++++++++++ 17 files changed, 1196 insertions(+) create mode 100644 samples/interactive-tutorials/product/import_products_big_query_table.py create mode 100644 samples/interactive-tutorials/product/import_products_bq_test.py create mode 100644 samples/interactive-tutorials/product/import_products_gcs.py create mode 100644 samples/interactive-tutorials/product/import_products_gcs_test.py create mode 100644 samples/interactive-tutorials/product/import_products_inline_source.py create mode 100644 samples/interactive-tutorials/product/import_products_inline_test.py create mode 100644 samples/interactive-tutorials/product/noxfile.py create mode 100644 samples/interactive-tutorials/product/noxfile_config.py create mode 100644 samples/interactive-tutorials/product/requirements-test.txt create mode 100644 samples/interactive-tutorials/product/requirements.txt create mode 100644 samples/interactive-tutorials/product/setup/products_create_bigquery_table.py create mode 100644 samples/interactive-tutorials/product/setup/products_create_bigquery_table_test.py create mode 100644 samples/interactive-tutorials/product/setup/products_create_gcs_bucket.py create mode 100644 samples/interactive-tutorials/product/setup/products_create_gcs_bucket_test.py create mode 100644 samples/interactive-tutorials/product/setup/products_delete_gcs_bucket.py create mode 100644 samples/interactive-tutorials/product/setup/setup_cleanup.py diff --git a/.kokoro/continuous/common.cfg b/.kokoro/continuous/common.cfg index a0f9b6a5..a6520457 100644 --- a/.kokoro/continuous/common.cfg +++ b/.kokoro/continuous/common.cfg @@ -25,3 +25,11 @@ env_vars: { key: "TRAMPOLINE_BUILD_FILE" value: "github/python-retail/.kokoro/build.sh" } +env_vars: { + key: "BUCKET_NAME" + value: "crs-interactive-tutorials-products" +} +env_vars: { + key: "EVENTS_BUCKET_NAME" + value: "crs-interactive-tutorials-events" +} diff --git a/samples/interactive-tutorials/product/import_products_big_query_table.py b/samples/interactive-tutorials/product/import_products_big_query_table.py new file mode 100644 index 00000000..9980bc99 --- /dev/null +++ b/samples/interactive-tutorials/product/import_products_big_query_table.py @@ -0,0 +1,105 @@ +# Copyright 2021 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START retail_import_products_from_big_query] +# Import products into a catalog from big query table using Retail API +# +import os +import time + +from google.api_core.client_options import ClientOptions +from google.cloud.retail import BigQuerySource, ImportProductsRequest, \ + ProductInputConfig, ProductServiceClient + +from setup.setup_cleanup import get_project_id + +project_number = os.getenv('GOOGLE_CLOUD_PROJECT_NUMBER') +project_id = get_project_id() + +endpoint = "retail.googleapis.com" +default_catalog = "projects/{0}/locations/global/catalogs/default_catalog/branches/default_branch".format( + project_number) +dataset_id = "products" +table_id = "products" + + +# TO CHECK ERROR HANDLING USE THE TABLE WITH INVALID PRODUCTS: +# table_id = "products_some_invalid" + + +# get product service client +def get_product_service_client(): + client_options = ClientOptions(endpoint) + return ProductServiceClient(client_options=client_options) + + +# get import products from big query request +def get_import_products_big_query_request(reconciliation_mode): + # TO CHECK ERROR HANDLING PASTE THE INVALID CATALOG NAME HERE: + # default_catalog = "invalid_catalog_name" + big_query_source = BigQuerySource() + big_query_source.project_id = project_id + big_query_source.dataset_id = dataset_id + big_query_source.table_id = table_id + big_query_source.data_schema = "product" + + input_config = ProductInputConfig() + input_config.big_query_source = big_query_source + + import_request = ImportProductsRequest() + import_request.parent = default_catalog + import_request.reconciliation_mode = reconciliation_mode + import_request.input_config = input_config + + print("---import products from big query table request---") + print(import_request) + + return import_request + + +# call the Retail API to import products +def import_products_from_big_query(): + # TRY THE FULL RECONCILIATION MODE HERE: + reconciliation_mode = ImportProductsRequest.ReconciliationMode.INCREMENTAL + + import_big_query_request = get_import_products_big_query_request( + reconciliation_mode) + big_query_operation = get_product_service_client().import_products( + import_big_query_request) + + print("---the operation was started:----") + print(big_query_operation.operation.name) + + while not big_query_operation.done(): + print("---please wait till operation is done---") + time.sleep(5) + print("---import products operation is done---") + + if big_query_operation.metadata is not None: + print("---number of successfully imported products---") + print(big_query_operation.metadata.success_count) + print("---number of failures during the importing---") + print(big_query_operation.metadata.failure_count) + else: + print("---operation.metadata is empty---") + + if big_query_operation.result is not None: + print("---operation result:---") + print(big_query_operation.result()) + else: + print("---operation.result is empty---") + +import_products_from_big_query() + +# [END retail_import_products_from_big_query] diff --git a/samples/interactive-tutorials/product/import_products_bq_test.py b/samples/interactive-tutorials/product/import_products_bq_test.py new file mode 100644 index 00000000..f304dd16 --- /dev/null +++ b/samples/interactive-tutorials/product/import_products_bq_test.py @@ -0,0 +1,29 @@ +# Copyright 2021 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re +import subprocess + + +def test_import_products_bq(): + output = str(subprocess.check_output( + 'python product/import_products_big_query_table.py', shell=True)) + + assert re.match('.*import products from big query table request.*', output) + assert re.match('.*the operation was started.*', output) + assert re.match( + '.*projects/.*/locations/global/catalogs/default_catalog/branches/0/operations/import-products.*', + output) + + assert re.match('.*number of successfully imported products.*316.*', output) diff --git a/samples/interactive-tutorials/product/import_products_gcs.py b/samples/interactive-tutorials/product/import_products_gcs.py new file mode 100644 index 00000000..995304dd --- /dev/null +++ b/samples/interactive-tutorials/product/import_products_gcs.py @@ -0,0 +1,122 @@ +# Copyright 2021 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START retail_import_products_from_gcs] +# Import products into a catalog from gcs using Retail API +# +import os +import re +import shlex +import subprocess +import time + +from google.api_core.client_options import ClientOptions +from google.cloud.retail import GcsSource, ImportErrorsConfig, \ + ImportProductsRequest, ProductInputConfig, ProductServiceClient + + +def get_project_id(): + get_project_command = "gcloud config get-value project --format json" + config = subprocess.check_output(shlex.split(get_project_command)) + project_id = re.search('\"(.*?)\"', str(config)).group(1) + return project_id + + +# Read the project number from the environment variable +project_number = os.getenv('GOOGLE_CLOUD_PROJECT_NUMBER') +project_id = get_project_id() +endpoint = "retail.googleapis.com" +# You can change the branch here. The "default_branch" is set to point to the branch "0" +default_catalog = "projects/{0}/locations/global/catalogs/default_catalog/branches/default_branch".format( + project_number) + +# Read bucket name from the environment variable +gcs_bucket = "gs://{}".format(os.getenv("BUCKET_NAME")) +gcs_errors_bucket = "{}/error".format(gcs_bucket) +gcs_products_object = "products.json" + + +# TO CHECK ERROR HANDLING USE THE JSON WITH INVALID PRODUCT +# gcs_products_object = "products_some_invalid.json" + + +# get product service client +def get_product_service_client(): + client_options = ClientOptions(endpoint) + return ProductServiceClient(client_options=client_options) + + +# get import products from gcs request +def get_import_products_gcs_request(gcs_object_name: str): + # TO CHECK ERROR HANDLING PASTE THE INVALID CATALOG NAME HERE: + # default_catalog = "invalid_catalog_name" + gcs_source = GcsSource() + gcs_source.input_uris = ["{0}/{1}".format(gcs_bucket, gcs_object_name)] + + input_config = ProductInputConfig() + input_config.gcs_source = gcs_source + print("GRS source:") + print(gcs_source.input_uris) + + errors_config = ImportErrorsConfig() + errors_config.gcs_prefix = gcs_errors_bucket + + import_request = ImportProductsRequest() + import_request.parent = default_catalog + import_request.reconciliation_mode = ImportProductsRequest.ReconciliationMode.INCREMENTAL + import_request.input_config = input_config + import_request.errors_config = errors_config + + print("---import products from google cloud source request---") + print(import_request) + + return import_request + + +# call the Retail API to import products +def import_products_from_gcs(): + import_gcs_request = get_import_products_gcs_request(gcs_products_object) + gcs_operation = get_product_service_client().import_products( + import_gcs_request) + + print("---the operation was started:----") + print(gcs_operation.operation.name) + + while not gcs_operation.done(): + print("---please wait till operation is done---") + time.sleep(5) + print("---import products operation is done---") + + if gcs_operation.metadata is not None: + print("---number of successfully imported products---") + print(gcs_operation.metadata.success_count) + print("---number of failures during the importing---") + print(gcs_operation.metadata.failure_count) + else: + print("---operation.metadata is empty---") + + if gcs_operation.result is not None: + print("---operation result:---") + print(gcs_operation.result()) + else: + print("---operation.result is empty---") + + # The imported products needs to be indexed in the catalog before they become available for search. + print( + "Wait 2 -5 minutes till products become indexed in the catalog, after that they will be available for search") + + +import_products_from_gcs() + +# [END retail_import_products_from_gcs] diff --git a/samples/interactive-tutorials/product/import_products_gcs_test.py b/samples/interactive-tutorials/product/import_products_gcs_test.py new file mode 100644 index 00000000..f5b441c2 --- /dev/null +++ b/samples/interactive-tutorials/product/import_products_gcs_test.py @@ -0,0 +1,32 @@ +# Copyright 2021 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re +import subprocess + + +def test_import_products_gcs(): + output = str( + subprocess.check_output('python product/import_products_gcs.py', + shell=True)) + + assert re.match('.*import products from google cloud source request.*', + output) + assert re.match('.*input_uris: "gs://.*/products.json".*', output) + assert re.match('.*the operation was started.*', output) + assert re.match( + '.*projects/.*/locations/global/catalogs/default_catalog/branches/0/operations/import-products.*', + output) + + assert re.match('.*number of successfully imported products.*316.*', output) diff --git a/samples/interactive-tutorials/product/import_products_inline_source.py b/samples/interactive-tutorials/product/import_products_inline_source.py new file mode 100644 index 00000000..4933a154 --- /dev/null +++ b/samples/interactive-tutorials/product/import_products_inline_source.py @@ -0,0 +1,161 @@ +# Copyright 2021 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START retail_import_products_from_inline_source] +# Import products into a catalog from inline source using Retail API +# +import os +import random +import string +import time + +from google.api_core.client_options import ClientOptions +from google.cloud.retail import ColorInfo, FulfillmentInfo, \ + ImportProductsRequest, PriceInfo, Product, ProductInlineSource, \ + ProductInputConfig, ProductServiceClient +from google.protobuf.field_mask_pb2 import FieldMask + +project_number = os.getenv('GOOGLE_CLOUD_PROJECT_NUMBER') + +endpoint = "retail.googleapis.com" +default_catalog = "projects/{0}/locations/global/catalogs/default_catalog/branches/default_branch".format( + project_number) + + +# prepare product to import as inline source +def get_products(): + products = [] + product1 = Product() + product2 = Product() + + price_info1 = PriceInfo() + price_info1.price = 16.0 + price_info1.original_price = 45.0 + price_info1.cost = 12.0 + price_info1.currency_code = "USD" + + color_info1 = ColorInfo() + color_info1.color_families = ["Blue"] + color_info1.colors = ["Light blue", + "Blue", + "Dark blue"] + + fulfillment_info1 = FulfillmentInfo() + fulfillment_info1.type_ = "pickup-in-store" + fulfillment_info1.place_ids = ["store1", "store2"] + + field_mask1 = FieldMask( + paths=["title", "categories", "price_info", "color_info"]) + + # TO CHECK ERROR HANDLING COMMENT OUT THE PRODUCT TITLE HERE: + product1.title = "#IamRemarkable Pen" + product1.id = ''.join(random.sample(string.ascii_lowercase, 8)) + product1.categories = ["Office"] + product1.uri = "https://shop.googlemerchandisestore.com/Google+Redesign/Office/IamRemarkable+Pen" + product1.brands = ["#IamRemarkable"] + product1.price_info = price_info1 + product1.color_info = color_info1 + product1.fulfillment_info = [fulfillment_info1] + product1.retrievable_fields = field_mask1 + + price_info2 = PriceInfo() + price_info2.price = 35.0 + price_info2.original_price = 45.0 + price_info2.cost = 12.0 + price_info2.currency_code = "USD" + + color_info2 = ColorInfo() + color_info2.color_families = ["Blue"] + color_info2.colors = ["Sky blue"] + + fulfillment_info2 = FulfillmentInfo() + fulfillment_info2.type_ = "pickup-in-store" + fulfillment_info2.place_ids = ["store2", "store3"] + + field_mask2 = FieldMask( + paths=["title", "categories", "price_info", "color_info"]) + + product2.title = "Android Embroidered Crewneck Sweater" + product2.id = ''.join(random.sample(string.ascii_lowercase, 8)) + product2.categories = ["Apparel"] + product2.uri = "https://shop.googlemerchandisestore.com/Google+Redesign/Apparel/Android+Embroidered+Crewneck+Sweater" + product2.brands = ["Android"] + product2.price_info = price_info2 + product2.color_info = color_info2 + product2.fulfillment_info = [fulfillment_info2] + product2.retrievable_fields = field_mask2 + + products.append(product1) + products.append(product2) + return products + + +# get product service client +def get_product_service_client(): + client_options = ClientOptions(endpoint) + return ProductServiceClient(client_options=client_options) + + +# get import products from inline source request +def get_import_products_inline_request(products_to_import): + # TO CHECK ERROR HANDLING PASTE THE INVALID CATALOG NAME HERE: + # default_catalog = "invalid_catalog_name" + inline_source = ProductInlineSource() + inline_source.products = products_to_import + + input_config = ProductInputConfig() + input_config.product_inline_source = inline_source + + import_request = ImportProductsRequest() + import_request.parent = default_catalog + import_request.input_config = input_config + + print("---import products from inline source request---") + print(import_request) + + return import_request + + +# call the Retail API to import products +def import_products_from_inline_source(): + import_request = get_import_products_inline_request(get_products()) + import_operation = get_product_service_client().import_products( + import_request) + + print("---the operation was started:----") + print(import_operation.operation.name) + + while not import_operation.done(): + print("---please wait till operation is done---") + time.sleep(5) + print("---import products operation is done---") + + if import_operation.metadata is not None: + print("---number of successfully imported products---") + print(import_operation.metadata.success_count) + print("---number of failures during the importing---") + print(import_operation.metadata.failure_count) + else: + print("---operation.metadata is empty---") + + if import_operation.result is not None: + print("---operation result:---") + print(import_operation.result()) + else: + print("---operation.result is empty---") + + +import_products_from_inline_source() + +# [END retail_import_products_from_inline_source] diff --git a/samples/interactive-tutorials/product/import_products_inline_test.py b/samples/interactive-tutorials/product/import_products_inline_test.py new file mode 100644 index 00000000..96aa0193 --- /dev/null +++ b/samples/interactive-tutorials/product/import_products_inline_test.py @@ -0,0 +1,29 @@ +# Copyright 2021 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re +import subprocess + + +def test_import_products_gcs(): + output = str(subprocess.check_output( + 'python product/import_products_inline_source.py', shell=True)) + + assert re.match('.*import products from inline source request.*', output) + assert re.match('.*the operation was started.*', output) + assert re.match( + '.*projects/.*/locations/global/catalogs/default_catalog/branches/0/operations/import-products.*', + output) + + assert re.match('.*number of successfully imported products.*2.*', output) diff --git a/samples/interactive-tutorials/product/noxfile.py b/samples/interactive-tutorials/product/noxfile.py new file mode 100644 index 00000000..20cdfc62 --- /dev/null +++ b/samples/interactive-tutorials/product/noxfile.py @@ -0,0 +1,279 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function + +import glob +import os +from pathlib import Path +import sys +from typing import Callable, Dict, List, Optional + +import nox + + +# WARNING - WARNING - WARNING - WARNING - WARNING +# WARNING - WARNING - WARNING - WARNING - WARNING +# DO NOT EDIT THIS FILE EVER! +# WARNING - WARNING - WARNING - WARNING - WARNING +# WARNING - WARNING - WARNING - WARNING - WARNING + +BLACK_VERSION = "black==19.10b0" + +# Copy `noxfile_config.py` to your directory and modify it instead. + +# `TEST_CONFIG` dict is a configuration hook that allows users to +# modify the test configurations. The values here should be in sync +# with `noxfile_config.py`. Users will copy `noxfile_config.py` into +# their directory and modify it. + +TEST_CONFIG = { + # You can opt out from the test for specific Python versions. + "ignored_versions": [], + # Old samples are opted out of enforcing Python type hints + # All new samples should feature them + "enforce_type_hints": False, + # An envvar key for determining the project id to use. Change it + # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a + # build specific Cloud project. You can also use your own string + # to use your own Cloud project. + "gcloud_project_env": "GOOGLE_CLOUD_PROJECT", + # 'gcloud_project_env': 'BUILD_SPECIFIC_GCLOUD_PROJECT', + # If you need to use a specific version of pip, + # change pip_version_override to the string representation + # of the version number, for example, "20.2.4" + "pip_version_override": None, + # A dictionary you want to inject into your test. Don't put any + # secrets here. These values will override predefined values. + "envs": {}, +} + + +try: + # Ensure we can import noxfile_config in the project's directory. + sys.path.append(".") + from noxfile_config import TEST_CONFIG_OVERRIDE +except ImportError as e: + print("No user noxfile_config found: detail: {}".format(e)) + TEST_CONFIG_OVERRIDE = {} + +# Update the TEST_CONFIG with the user supplied values. +TEST_CONFIG.update(TEST_CONFIG_OVERRIDE) + + +def get_pytest_env_vars() -> Dict[str, str]: + """Returns a dict for pytest invocation.""" + ret = {} + + # Override the GCLOUD_PROJECT and the alias. + env_key = TEST_CONFIG["gcloud_project_env"] + # This should error out if not set. + ret["GOOGLE_CLOUD_PROJECT"] = os.environ[env_key] + + # Apply user supplied envs. + ret.update(TEST_CONFIG["envs"]) + return ret + + +# DO NOT EDIT - automatically generated. +# All versions used to test samples. +ALL_VERSIONS = ["3.6", "3.7", "3.8", "3.9", "3.10"] + +# Any default versions that should be ignored. +IGNORED_VERSIONS = TEST_CONFIG["ignored_versions"] + +TESTED_VERSIONS = sorted([v for v in ALL_VERSIONS if v not in IGNORED_VERSIONS]) + +INSTALL_LIBRARY_FROM_SOURCE = os.environ.get("INSTALL_LIBRARY_FROM_SOURCE", False) in ( + "True", + "true", +) + +# Error if a python version is missing +nox.options.error_on_missing_interpreters = True + +# +# Style Checks +# + + +def _determine_local_import_names(start_dir: str) -> List[str]: + """Determines all import names that should be considered "local". + + This is used when running the linter to insure that import order is + properly checked. + """ + file_ext_pairs = [os.path.splitext(path) for path in os.listdir(start_dir)] + return [ + basename + for basename, extension in file_ext_pairs + if extension == ".py" + or os.path.isdir(os.path.join(start_dir, basename)) + and basename not in ("__pycache__") + ] + + +# Linting with flake8. +# +# We ignore the following rules: +# E203: whitespace before ‘:’ +# E266: too many leading ‘#’ for block comment +# E501: line too long +# I202: Additional newline in a section of imports +# +# We also need to specify the rules which are ignored by default: +# ['E226', 'W504', 'E126', 'E123', 'W503', 'E24', 'E704', 'E121'] +FLAKE8_COMMON_ARGS = [ + "--show-source", + "--builtin=gettext", + "--max-complexity=20", + "--import-order-style=google", + "--exclude=.nox,.cache,env,lib,generated_pb2,*_pb2.py,*_pb2_grpc.py", + "--ignore=E121,E123,E126,E203,E226,E24,E266,E501,E704,W503,W504,I202", + "--max-line-length=88", +] + + +@nox.session +def lint(session: nox.sessions.Session) -> None: + if not TEST_CONFIG["enforce_type_hints"]: + session.install("flake8", "flake8-import-order") + else: + session.install("flake8", "flake8-import-order", "flake8-annotations") + + local_names = _determine_local_import_names(".") + args = FLAKE8_COMMON_ARGS + [ + "--application-import-names", + ",".join(local_names), + ".", + ] + session.run("flake8", *args) + + +# +# Black +# + + +@nox.session +def blacken(session: nox.sessions.Session) -> None: + session.install(BLACK_VERSION) + python_files = [path for path in os.listdir(".") if path.endswith(".py")] + + session.run("black", *python_files) + + +# +# Sample Tests +# + + +PYTEST_COMMON_ARGS = ["--junitxml=sponge_log.xml"] + + +def _session_tests( + session: nox.sessions.Session, post_install: Callable = None +) -> None: + # check for presence of tests + test_list = glob.glob("*_test.py") + glob.glob("test_*.py") + test_list.extend(glob.glob("tests")) + if len(test_list) == 0: + print("No tests found, skipping directory.") + else: + if TEST_CONFIG["pip_version_override"]: + pip_version = TEST_CONFIG["pip_version_override"] + session.install(f"pip=={pip_version}") + """Runs py.test for a particular project.""" + if os.path.exists("requirements.txt"): + if os.path.exists("constraints.txt"): + session.install("-r", "requirements.txt", "-c", "constraints.txt") + else: + session.install("-r", "requirements.txt") + + if os.path.exists("requirements-test.txt"): + if os.path.exists("constraints-test.txt"): + session.install( + "-r", "requirements-test.txt", "-c", "constraints-test.txt" + ) + else: + session.install("-r", "requirements-test.txt") + + if INSTALL_LIBRARY_FROM_SOURCE: + session.install("-e", _get_repo_root()) + + if post_install: + post_install(session) + + session.run( + "pytest", + *(PYTEST_COMMON_ARGS + session.posargs), + # Pytest will return 5 when no tests are collected. This can happen + # on travis where slow and flaky tests are excluded. + # See http://doc.pytest.org/en/latest/_modules/_pytest/main.html + success_codes=[0, 5], + env=get_pytest_env_vars(), + ) + + +@nox.session(python=ALL_VERSIONS) +def py(session: nox.sessions.Session) -> None: + """Runs py.test for a sample using the specified version of Python.""" + if session.python in TESTED_VERSIONS: + _session_tests(session) + else: + session.skip( + "SKIPPED: {} tests are disabled for this sample.".format(session.python) + ) + + +# +# Readmegen +# + + +def _get_repo_root() -> Optional[str]: + """ Returns the root folder of the project. """ + # Get root of this repository. Assume we don't have directories nested deeper than 10 items. + p = Path(os.getcwd()) + for i in range(10): + if p is None: + break + if Path(p / ".git").exists(): + return str(p) + # .git is not available in repos cloned via Cloud Build + # setup.py is always in the library's root, so use that instead + # https://github.com/googleapis/synthtool/issues/792 + if Path(p / "setup.py").exists(): + return str(p) + p = p.parent + raise Exception("Unable to detect repository root.") + + +GENERATED_READMES = sorted([x for x in Path(".").rglob("*.rst.in")]) + + +@nox.session +@nox.parametrize("path", GENERATED_READMES) +def readmegen(session: nox.sessions.Session, path: str) -> None: + """(Re-)generates the readme for a sample.""" + session.install("jinja2", "pyyaml") + dir_ = os.path.dirname(path) + + if os.path.exists(os.path.join(dir_, "requirements.txt")): + session.install("-r", os.path.join(dir_, "requirements.txt")) + + in_file = os.path.join(dir_, "README.rst.in") + session.run( + "python", _get_repo_root() + "/scripts/readme-gen/readme_gen.py", in_file + ) diff --git a/samples/interactive-tutorials/product/noxfile_config.py b/samples/interactive-tutorials/product/noxfile_config.py new file mode 100644 index 00000000..8eb8d16f --- /dev/null +++ b/samples/interactive-tutorials/product/noxfile_config.py @@ -0,0 +1,34 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default TEST_CONFIG_OVERRIDE for python repos. + +# The source of truth: +# https://github.com/GoogleCloudPlatform/python-docs-samples/blob/master/noxfile_config.py + +TEST_CONFIG_OVERRIDE = { + # You can opt out from the test for specific Python versions. + "ignored_versions": ["2.7", "3.6"], + # An envvar key for determining the project id to use. Change it + # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a + # build specific Cloud project. You can also use your own string + # to use your own Cloud project. + "gcloud_project_env": "GOOGLE_CLOUD_PROJECT", + # 'gcloud_project_env': 'BUILD_SPECIFIC_GCLOUD_PROJECT', + # A dictionary you want to inject into your test. Don't put any + # secrets here. These values will override predefined values. + "envs": { + "BUCKET_NAME": "retail-interactive-tutorials", + }, +} diff --git a/samples/interactive-tutorials/product/requirements-test.txt b/samples/interactive-tutorials/product/requirements-test.txt new file mode 100644 index 00000000..bbf73145 --- /dev/null +++ b/samples/interactive-tutorials/product/requirements-test.txt @@ -0,0 +1,2 @@ +pytest==6.2.5 +pytest-xdist==2.5.0 diff --git a/samples/interactive-tutorials/product/requirements.txt b/samples/interactive-tutorials/product/requirements.txt new file mode 100644 index 00000000..0ba6ea71 --- /dev/null +++ b/samples/interactive-tutorials/product/requirements.txt @@ -0,0 +1,4 @@ +google==3.0.0 +google-cloud-retail==1.1.0 +google-cloud-storage==1.43.0 +google-cloud-bigquery==2.30.1 \ No newline at end of file diff --git a/samples/interactive-tutorials/product/setup/products_create_bigquery_table.py b/samples/interactive-tutorials/product/setup/products_create_bigquery_table.py new file mode 100644 index 00000000..ec14f16c --- /dev/null +++ b/samples/interactive-tutorials/product/setup/products_create_bigquery_table.py @@ -0,0 +1,31 @@ +# Copyright 2021 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from setup_cleanup import create_bq_dataset, create_bq_table, \ + upload_data_to_bq_table + +dataset = "products" +valid_products_table = "products" +invalid_products_table = "products_some_invalid" +product_schema = "resources/product_schema.json" +valid_products_source_file = "resources/products.json" +invalid_products_source_file = "resources/products_some_invalid.json" + +create_bq_dataset(dataset) +create_bq_table(dataset, valid_products_table, product_schema) +upload_data_to_bq_table(dataset, valid_products_table, + valid_products_source_file, product_schema) +create_bq_table(dataset, invalid_products_table, product_schema) +upload_data_to_bq_table(dataset, invalid_products_table, + invalid_products_source_file, product_schema) diff --git a/samples/interactive-tutorials/product/setup/products_create_bigquery_table_test.py b/samples/interactive-tutorials/product/setup/products_create_bigquery_table_test.py new file mode 100644 index 00000000..71a45c3c --- /dev/null +++ b/samples/interactive-tutorials/product/setup/products_create_bigquery_table_test.py @@ -0,0 +1,43 @@ +# Copyright 2021 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re +import subprocess + + +def test_create_bigquery_table(): + output = str( + subprocess.check_output( + 'python product/setup/products_create_bigquery_table.py', + shell=True)) + assert re.match( + '.*Creating dataset products.*', output) + assert re.match( + '(.*dataset products already exists.*|.*dataset is created.*)', output) + assert re.match( + '.*Creating BigQuery table products.*', output) + assert re.match( + '(.*table products already exists.*|.*table is created.*)', output) + assert re.match( + '.*Uploading data form resources/products.json to the table products.products.*', output) + + assert re.match( + '.*Creating BigQuery table products_some_invalid.*', + output) + assert re.match( + '(.*table products_some_invalid already exists.*|.*table is created.*)', + output) + assert re.match( + '.*Uploading data form resources/products_some_invalid.json to the table products.products_some_invalid.*', + output) diff --git a/samples/interactive-tutorials/product/setup/products_create_gcs_bucket.py b/samples/interactive-tutorials/product/setup/products_create_gcs_bucket.py new file mode 100644 index 00000000..bbe07a26 --- /dev/null +++ b/samples/interactive-tutorials/product/setup/products_create_gcs_bucket.py @@ -0,0 +1,26 @@ +# Copyright 2021 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import datetime + +from setup_cleanup import create_bucket, get_project_id, upload_blob + +timestamp_ = datetime.datetime.now().timestamp().__round__() +bucket_name = "{}_products_{}".format(get_project_id(), timestamp_) + +create_bucket(bucket_name) +upload_blob(bucket_name, "resources/products.json") +upload_blob(bucket_name, "resources/products_some_invalid.json") + +print("\nThe gcs bucket {} was created".format(bucket_name)) diff --git a/samples/interactive-tutorials/product/setup/products_create_gcs_bucket_test.py b/samples/interactive-tutorials/product/setup/products_create_gcs_bucket_test.py new file mode 100644 index 00000000..5846ab1c --- /dev/null +++ b/samples/interactive-tutorials/product/setup/products_create_gcs_bucket_test.py @@ -0,0 +1,40 @@ +# Copyright 2021 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re +import subprocess +from products_delete_gcs_bucket import delete_bucket_by_name + + +def test_create_gcs_bucket(): + output = str( + subprocess.check_output( + 'python product/setup/products_create_gcs_bucket.py', + shell=True)) + + bucket_name = re.search('The gcs bucket (.+?) was created', output).group(1) + delete_bucket_by_name(bucket_name) + + print("bucket_name = {}".format(bucket_name)) + + assert re.match( + '.*Creating new bucket.*', output) + assert re.match( + '(.*The gcs bucket.*?was created.*|.*Bucket.*?already exists.*)', output) + assert re.match( + '.*Uploading data form resources/products.json to the bucket.*', output) + assert re.match( + '.*Uploading data form resources/products_some_invalid.json to the bucket.*', + output) + diff --git a/samples/interactive-tutorials/product/setup/products_delete_gcs_bucket.py b/samples/interactive-tutorials/product/setup/products_delete_gcs_bucket.py new file mode 100644 index 00000000..6e76c110 --- /dev/null +++ b/samples/interactive-tutorials/product/setup/products_delete_gcs_bucket.py @@ -0,0 +1,25 @@ +# Copyright 2021 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from setup_cleanup import delete_bucket + + +def delete_bucket_by_name(name: str): + if name is None: + bucket_name = os.getenv("BUCKET_NAME") + delete_bucket(bucket_name) + else: + delete_bucket(name) diff --git a/samples/interactive-tutorials/product/setup/setup_cleanup.py b/samples/interactive-tutorials/product/setup/setup_cleanup.py new file mode 100644 index 00000000..0f69a072 --- /dev/null +++ b/samples/interactive-tutorials/product/setup/setup_cleanup.py @@ -0,0 +1,226 @@ +# Copyright 2021 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import re +import shlex +import subprocess + +from google.api_core.client_options import ClientOptions +from google.api_core.exceptions import NotFound + +from google.cloud import storage +from google.cloud.retail_v2 import CreateProductRequest, DeleteProductRequest, \ + FulfillmentInfo, GetProductRequest, PriceInfo, Product, ProductServiceClient + +project_number = os.getenv('GOOGLE_CLOUD_PROJECT_NUMBER') +endpoint = "retail.googleapis.com" +default_catalog = "projects/{0}/locations/global/catalogs/default_catalog".format( + project_number) +default_branch_name = "projects/" + project_number + "/locations/global/catalogs/default_catalog/branches/default_branch" + + +def get_product_service_client(): + client_options = ClientOptions(endpoint) + return ProductServiceClient(client_options=client_options) + + +def generate_product() -> Product: + price_info = PriceInfo() + price_info.price = 30.0 + price_info.original_price = 35.5 + price_info.currency_code = "USD" + fulfillment_info = FulfillmentInfo() + fulfillment_info.type_ = "pickup-in-store" + fulfillment_info.place_ids = ["store0", "store1"] + return Product( + title='Nest Mini', + type_=Product.Type.PRIMARY, + categories=['Speakers and displays'], + brands=['Google'], + price_info=price_info, + fulfillment_info=[fulfillment_info], + availability="IN_STOCK", + ) + + +def create_product(product_id: str) -> object: + create_product_request = CreateProductRequest() + create_product_request.product = generate_product() + create_product_request.product_id = product_id + create_product_request.parent = default_branch_name + + created_product = get_product_service_client().create_product( + create_product_request) + print("---product is created:---") + print(created_product) + + return created_product + + +def delete_product(product_name: str): + delete_product_request = DeleteProductRequest() + delete_product_request.name = product_name + get_product_service_client().delete_product(delete_product_request) + + print("---product " + product_name + " was deleted:---") + + +def get_product(product_name: str): + get_product_request = GetProductRequest() + get_product_request.name = product_name + try: + product = get_product_service_client().get_product(get_product_request) + print("---get product response:---") + print(product) + return product + except NotFound as e: + print(e.message) + return e.message + + +def try_to_delete_product_if_exists(product_name: str): + get_product_request = GetProductRequest() + get_product_request.name = product_name + delete_product_request = DeleteProductRequest() + delete_product_request.name = product_name + print( + "---delete product from the catalog, if the product already exists---") + try: + product = get_product_service_client().get_product(get_product_request) + get_product_service_client().delete_product(product.name) + except NotFound as e: + print(e.message) + + +def get_project_id(): + get_project_command = "gcloud config get-value project --format json" + config = subprocess.check_output(shlex.split(get_project_command)) + project_id = re.search('\"(.*?)\"', str(config)).group(1) + return project_id + + +def create_bucket(bucket_name: str): + """Create a new bucket in Cloud Storage""" + print("Creating new bucket:" + bucket_name) + buckets_in_your_project = str(list_buckets()) + if bucket_name in buckets_in_your_project: + print("Bucket {} already exists".format(bucket_name)) + else: + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + bucket.storage_class = "STANDARD" + new_bucket = storage_client.create_bucket(bucket, location="us") + print( + "Created bucket {} in {} with storage class {}".format( + new_bucket.name, new_bucket.location, new_bucket.storage_class + ) + ) + return new_bucket + + +def delete_bucket(bucket_name: str): + """Delete a bucket from Cloud Storage""" + storage_client = storage.Client() + print("Deleting bucket name:" + bucket_name) + buckets_in_your_project = str(list_buckets()) + if bucket_name in buckets_in_your_project: + blobs = storage_client.list_blobs(bucket_name) + for blob in blobs: + blob.delete() + bucket = storage_client.get_bucket(bucket_name) + bucket.delete() + print("Bucket {} is deleted".format(bucket.name)) + else: + print("Bucket {} is not found".format(bucket_name)) + + +def list_buckets(): + """Lists all buckets""" + bucket_list = [] + storage_client = storage.Client() + buckets = storage_client.list_buckets() + for bucket in buckets: + bucket_list.append(str(bucket)) + return bucket_list + + +def upload_blob(bucket_name, source_file_name): + """Uploads a file to the bucket.""" + # The path to your file to upload + # source_file_name = "local/path/to/file" + print("Uploading data form {} to the bucket {}".format(source_file_name, + bucket_name)) + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + object_name = re.search('resources/(.*?)$', source_file_name).group(1) + blob = bucket.blob(object_name) + blob.upload_from_filename(source_file_name) + + print( + "File {} uploaded to {}.".format( + source_file_name, object_name + ) + ) + + +def create_bq_dataset(dataset_name): + """Create a BigQuery dataset""" + print("Creating dataset {}".format(dataset_name)) + if dataset_name not in list_bq_datasets(): + create_dataset_command = 'bq --location=US mk -d --default_table_expiration 3600 --description "This is my dataset." {}:{}'.format( + get_project_id(), dataset_name) + subprocess.check_output(shlex.split(create_dataset_command)) + print("dataset is created") + else: + print("dataset {} already exists".format(dataset_name)) + + +def list_bq_datasets(): + """List BigQuery datasets in the project""" + list_dataset_command = "bq ls --project_id {}".format(get_project_id()) + datasets = subprocess.check_output(shlex.split(list_dataset_command)) + return str(datasets) + + +def create_bq_table(dataset, table_name, schema): + """Create a BigQuery table""" + print("Creating BigQuery table {}".format(table_name)) + if table_name not in list_bq_tables(dataset): + create_table_command = "bq mk --table {}:{}.{} {}".format( + get_project_id(), + dataset, + table_name, schema) + output = subprocess.check_output(shlex.split(create_table_command)) + print(output) + print("table is created") + else: + print("table {} already exists".format(table_name)) + + +def list_bq_tables(dataset): + """List BigQuery tables in the dataset""" + list_tables_command = "bq ls {}:{}".format(get_project_id(), dataset) + tables = subprocess.check_output(shlex.split(list_tables_command)) + return str(tables) + + +def upload_data_to_bq_table(dataset, table_name, source, schema): + """Upload data to the table from specified source file""" + print("Uploading data form {} to the table {}.{}".format(source, dataset, + table_name)) + upload_data_command = "bq load --source_format=NEWLINE_DELIMITED_JSON {}:{}.{} {} {}".format( + get_project_id(), dataset, table_name, source, schema) + output = subprocess.check_output(shlex.split(upload_data_command)) + print(output) From 6424ac85b74cd06fcac225079ff3d714963aa0be Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Fri, 28 Jan 2022 21:19:48 +0000 Subject: [PATCH 02/12] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- .kokoro/continuous/common.cfg | 8 ----- .../import_products_big_query_table.py | 20 +++++++---- .../product/import_products_bq_test.py | 18 ++++++---- .../product/import_products_gcs.py | 26 +++++++++----- .../product/import_products_gcs_test.py | 16 ++++----- .../product/import_products_inline_source.py | 35 ++++++++++--------- .../product/import_products_inline_test.py | 18 ++++++---- .../product/noxfile_config.py | 4 +-- .../search/noxfile_config.py | 4 +-- .../search/search_simple_query_test.py | 4 +-- .../search/search_with_filtering_test.py | 4 +-- .../search/search_with_ordering_test.py | 4 +-- 12 files changed, 85 insertions(+), 76 deletions(-) diff --git a/.kokoro/continuous/common.cfg b/.kokoro/continuous/common.cfg index a6520457..a0f9b6a5 100644 --- a/.kokoro/continuous/common.cfg +++ b/.kokoro/continuous/common.cfg @@ -25,11 +25,3 @@ env_vars: { key: "TRAMPOLINE_BUILD_FILE" value: "github/python-retail/.kokoro/build.sh" } -env_vars: { - key: "BUCKET_NAME" - value: "crs-interactive-tutorials-products" -} -env_vars: { - key: "EVENTS_BUCKET_NAME" - value: "crs-interactive-tutorials-events" -} diff --git a/samples/interactive-tutorials/product/import_products_big_query_table.py b/samples/interactive-tutorials/product/import_products_big_query_table.py index 9980bc99..0ecddf05 100644 --- a/samples/interactive-tutorials/product/import_products_big_query_table.py +++ b/samples/interactive-tutorials/product/import_products_big_query_table.py @@ -19,17 +19,22 @@ import time from google.api_core.client_options import ClientOptions -from google.cloud.retail import BigQuerySource, ImportProductsRequest, \ - ProductInputConfig, ProductServiceClient +from google.cloud.retail import ( + BigQuerySource, + ImportProductsRequest, + ProductInputConfig, + ProductServiceClient, +) from setup.setup_cleanup import get_project_id -project_number = os.getenv('GOOGLE_CLOUD_PROJECT_NUMBER') +project_number = os.getenv("GOOGLE_CLOUD_PROJECT_NUMBER") project_id = get_project_id() endpoint = "retail.googleapis.com" default_catalog = "projects/{0}/locations/global/catalogs/default_catalog/branches/default_branch".format( - project_number) + project_number +) dataset_id = "products" table_id = "products" @@ -74,9 +79,11 @@ def import_products_from_big_query(): reconciliation_mode = ImportProductsRequest.ReconciliationMode.INCREMENTAL import_big_query_request = get_import_products_big_query_request( - reconciliation_mode) + reconciliation_mode + ) big_query_operation = get_product_service_client().import_products( - import_big_query_request) + import_big_query_request + ) print("---the operation was started:----") print(big_query_operation.operation.name) @@ -100,6 +107,7 @@ def import_products_from_big_query(): else: print("---operation.result is empty---") + import_products_from_big_query() # [END retail_import_products_from_big_query] diff --git a/samples/interactive-tutorials/product/import_products_bq_test.py b/samples/interactive-tutorials/product/import_products_bq_test.py index f304dd16..7507b4b2 100644 --- a/samples/interactive-tutorials/product/import_products_bq_test.py +++ b/samples/interactive-tutorials/product/import_products_bq_test.py @@ -17,13 +17,17 @@ def test_import_products_bq(): - output = str(subprocess.check_output( - 'python product/import_products_big_query_table.py', shell=True)) + output = str( + subprocess.check_output( + "python product/import_products_big_query_table.py", shell=True + ) + ) - assert re.match('.*import products from big query table request.*', output) - assert re.match('.*the operation was started.*', output) + assert re.match(".*import products from big query table request.*", output) + assert re.match(".*the operation was started.*", output) assert re.match( - '.*projects/.*/locations/global/catalogs/default_catalog/branches/0/operations/import-products.*', - output) + ".*projects/.*/locations/global/catalogs/default_catalog/branches/0/operations/import-products.*", + output, + ) - assert re.match('.*number of successfully imported products.*316.*', output) + assert re.match(".*number of successfully imported products.*316.*", output) diff --git a/samples/interactive-tutorials/product/import_products_gcs.py b/samples/interactive-tutorials/product/import_products_gcs.py index 995304dd..7674d789 100644 --- a/samples/interactive-tutorials/product/import_products_gcs.py +++ b/samples/interactive-tutorials/product/import_products_gcs.py @@ -22,24 +22,30 @@ import time from google.api_core.client_options import ClientOptions -from google.cloud.retail import GcsSource, ImportErrorsConfig, \ - ImportProductsRequest, ProductInputConfig, ProductServiceClient +from google.cloud.retail import ( + GcsSource, + ImportErrorsConfig, + ImportProductsRequest, + ProductInputConfig, + ProductServiceClient, +) def get_project_id(): get_project_command = "gcloud config get-value project --format json" config = subprocess.check_output(shlex.split(get_project_command)) - project_id = re.search('\"(.*?)\"', str(config)).group(1) + project_id = re.search('"(.*?)"', str(config)).group(1) return project_id # Read the project number from the environment variable -project_number = os.getenv('GOOGLE_CLOUD_PROJECT_NUMBER') +project_number = os.getenv("GOOGLE_CLOUD_PROJECT_NUMBER") project_id = get_project_id() endpoint = "retail.googleapis.com" # You can change the branch here. The "default_branch" is set to point to the branch "0" default_catalog = "projects/{0}/locations/global/catalogs/default_catalog/branches/default_branch".format( - project_number) + project_number +) # Read bucket name from the environment variable gcs_bucket = "gs://{}".format(os.getenv("BUCKET_NAME")) @@ -74,7 +80,9 @@ def get_import_products_gcs_request(gcs_object_name: str): import_request = ImportProductsRequest() import_request.parent = default_catalog - import_request.reconciliation_mode = ImportProductsRequest.ReconciliationMode.INCREMENTAL + import_request.reconciliation_mode = ( + ImportProductsRequest.ReconciliationMode.INCREMENTAL + ) import_request.input_config = input_config import_request.errors_config = errors_config @@ -87,8 +95,7 @@ def get_import_products_gcs_request(gcs_object_name: str): # call the Retail API to import products def import_products_from_gcs(): import_gcs_request = get_import_products_gcs_request(gcs_products_object) - gcs_operation = get_product_service_client().import_products( - import_gcs_request) + gcs_operation = get_product_service_client().import_products(import_gcs_request) print("---the operation was started:----") print(gcs_operation.operation.name) @@ -114,7 +121,8 @@ def import_products_from_gcs(): # The imported products needs to be indexed in the catalog before they become available for search. print( - "Wait 2 -5 minutes till products become indexed in the catalog, after that they will be available for search") + "Wait 2 -5 minutes till products become indexed in the catalog, after that they will be available for search" + ) import_products_from_gcs() diff --git a/samples/interactive-tutorials/product/import_products_gcs_test.py b/samples/interactive-tutorials/product/import_products_gcs_test.py index f5b441c2..9edbde9b 100644 --- a/samples/interactive-tutorials/product/import_products_gcs_test.py +++ b/samples/interactive-tutorials/product/import_products_gcs_test.py @@ -18,15 +18,15 @@ def test_import_products_gcs(): output = str( - subprocess.check_output('python product/import_products_gcs.py', - shell=True)) + subprocess.check_output("python product/import_products_gcs.py", shell=True) + ) - assert re.match('.*import products from google cloud source request.*', - output) + assert re.match(".*import products from google cloud source request.*", output) assert re.match('.*input_uris: "gs://.*/products.json".*', output) - assert re.match('.*the operation was started.*', output) + assert re.match(".*the operation was started.*", output) assert re.match( - '.*projects/.*/locations/global/catalogs/default_catalog/branches/0/operations/import-products.*', - output) + ".*projects/.*/locations/global/catalogs/default_catalog/branches/0/operations/import-products.*", + output, + ) - assert re.match('.*number of successfully imported products.*316.*', output) + assert re.match(".*number of successfully imported products.*316.*", output) diff --git a/samples/interactive-tutorials/product/import_products_inline_source.py b/samples/interactive-tutorials/product/import_products_inline_source.py index 4933a154..26f6c055 100644 --- a/samples/interactive-tutorials/product/import_products_inline_source.py +++ b/samples/interactive-tutorials/product/import_products_inline_source.py @@ -21,16 +21,24 @@ import time from google.api_core.client_options import ClientOptions -from google.cloud.retail import ColorInfo, FulfillmentInfo, \ - ImportProductsRequest, PriceInfo, Product, ProductInlineSource, \ - ProductInputConfig, ProductServiceClient +from google.cloud.retail import ( + ColorInfo, + FulfillmentInfo, + ImportProductsRequest, + PriceInfo, + Product, + ProductInlineSource, + ProductInputConfig, + ProductServiceClient, +) from google.protobuf.field_mask_pb2 import FieldMask -project_number = os.getenv('GOOGLE_CLOUD_PROJECT_NUMBER') +project_number = os.getenv("GOOGLE_CLOUD_PROJECT_NUMBER") endpoint = "retail.googleapis.com" default_catalog = "projects/{0}/locations/global/catalogs/default_catalog/branches/default_branch".format( - project_number) + project_number +) # prepare product to import as inline source @@ -47,20 +55,17 @@ def get_products(): color_info1 = ColorInfo() color_info1.color_families = ["Blue"] - color_info1.colors = ["Light blue", - "Blue", - "Dark blue"] + color_info1.colors = ["Light blue", "Blue", "Dark blue"] fulfillment_info1 = FulfillmentInfo() fulfillment_info1.type_ = "pickup-in-store" fulfillment_info1.place_ids = ["store1", "store2"] - field_mask1 = FieldMask( - paths=["title", "categories", "price_info", "color_info"]) + field_mask1 = FieldMask(paths=["title", "categories", "price_info", "color_info"]) # TO CHECK ERROR HANDLING COMMENT OUT THE PRODUCT TITLE HERE: product1.title = "#IamRemarkable Pen" - product1.id = ''.join(random.sample(string.ascii_lowercase, 8)) + product1.id = "".join(random.sample(string.ascii_lowercase, 8)) product1.categories = ["Office"] product1.uri = "https://shop.googlemerchandisestore.com/Google+Redesign/Office/IamRemarkable+Pen" product1.brands = ["#IamRemarkable"] @@ -83,11 +88,10 @@ def get_products(): fulfillment_info2.type_ = "pickup-in-store" fulfillment_info2.place_ids = ["store2", "store3"] - field_mask2 = FieldMask( - paths=["title", "categories", "price_info", "color_info"]) + field_mask2 = FieldMask(paths=["title", "categories", "price_info", "color_info"]) product2.title = "Android Embroidered Crewneck Sweater" - product2.id = ''.join(random.sample(string.ascii_lowercase, 8)) + product2.id = "".join(random.sample(string.ascii_lowercase, 8)) product2.categories = ["Apparel"] product2.uri = "https://shop.googlemerchandisestore.com/Google+Redesign/Apparel/Android+Embroidered+Crewneck+Sweater" product2.brands = ["Android"] @@ -130,8 +134,7 @@ def get_import_products_inline_request(products_to_import): # call the Retail API to import products def import_products_from_inline_source(): import_request = get_import_products_inline_request(get_products()) - import_operation = get_product_service_client().import_products( - import_request) + import_operation = get_product_service_client().import_products(import_request) print("---the operation was started:----") print(import_operation.operation.name) diff --git a/samples/interactive-tutorials/product/import_products_inline_test.py b/samples/interactive-tutorials/product/import_products_inline_test.py index 96aa0193..b2821dae 100644 --- a/samples/interactive-tutorials/product/import_products_inline_test.py +++ b/samples/interactive-tutorials/product/import_products_inline_test.py @@ -17,13 +17,17 @@ def test_import_products_gcs(): - output = str(subprocess.check_output( - 'python product/import_products_inline_source.py', shell=True)) + output = str( + subprocess.check_output( + "python product/import_products_inline_source.py", shell=True + ) + ) - assert re.match('.*import products from inline source request.*', output) - assert re.match('.*the operation was started.*', output) + assert re.match(".*import products from inline source request.*", output) + assert re.match(".*the operation was started.*", output) assert re.match( - '.*projects/.*/locations/global/catalogs/default_catalog/branches/0/operations/import-products.*', - output) + ".*projects/.*/locations/global/catalogs/default_catalog/branches/0/operations/import-products.*", + output, + ) - assert re.match('.*number of successfully imported products.*2.*', output) + assert re.match(".*number of successfully imported products.*2.*", output) diff --git a/samples/interactive-tutorials/product/noxfile_config.py b/samples/interactive-tutorials/product/noxfile_config.py index 8eb8d16f..401ff7f2 100644 --- a/samples/interactive-tutorials/product/noxfile_config.py +++ b/samples/interactive-tutorials/product/noxfile_config.py @@ -28,7 +28,5 @@ # 'gcloud_project_env': 'BUILD_SPECIFIC_GCLOUD_PROJECT', # A dictionary you want to inject into your test. Don't put any # secrets here. These values will override predefined values. - "envs": { - "BUCKET_NAME": "retail-interactive-tutorials", - }, + "envs": {"BUCKET_NAME": "retail-interactive-tutorials",}, } diff --git a/samples/interactive-tutorials/search/noxfile_config.py b/samples/interactive-tutorials/search/noxfile_config.py index 8eb8d16f..401ff7f2 100644 --- a/samples/interactive-tutorials/search/noxfile_config.py +++ b/samples/interactive-tutorials/search/noxfile_config.py @@ -28,7 +28,5 @@ # 'gcloud_project_env': 'BUILD_SPECIFIC_GCLOUD_PROJECT', # A dictionary you want to inject into your test. Don't put any # secrets here. These values will override predefined values. - "envs": { - "BUCKET_NAME": "retail-interactive-tutorials", - }, + "envs": {"BUCKET_NAME": "retail-interactive-tutorials",}, } diff --git a/samples/interactive-tutorials/search/search_simple_query_test.py b/samples/interactive-tutorials/search/search_simple_query_test.py index 32749e2a..fc6d0c69 100644 --- a/samples/interactive-tutorials/search/search_simple_query_test.py +++ b/samples/interactive-tutorials/search/search_simple_query_test.py @@ -21,9 +21,7 @@ def test_search_simple_query_pass(): - output = str( - subprocess.check_output("python search_simple_query.py", shell=True) - ) + output = str(subprocess.check_output("python search_simple_query.py", shell=True)) assert re.match(".*search request.*", output) assert re.match(".*search response.*", output) diff --git a/samples/interactive-tutorials/search/search_with_filtering_test.py b/samples/interactive-tutorials/search/search_with_filtering_test.py index 270ff193..5b10ae34 100644 --- a/samples/interactive-tutorials/search/search_with_filtering_test.py +++ b/samples/interactive-tutorials/search/search_with_filtering_test.py @@ -21,9 +21,7 @@ def test_search_with_filtering_pass(): - output = str( - subprocess.check_output("python search_with_filtering.py", shell=True) - ) + output = str(subprocess.check_output("python search_with_filtering.py", shell=True)) assert re.match(".*search request.*", output) assert re.match(".*search response.*", output) diff --git a/samples/interactive-tutorials/search/search_with_ordering_test.py b/samples/interactive-tutorials/search/search_with_ordering_test.py index 0de2c979..ec8fcfb9 100644 --- a/samples/interactive-tutorials/search/search_with_ordering_test.py +++ b/samples/interactive-tutorials/search/search_with_ordering_test.py @@ -21,9 +21,7 @@ def test_search_with_ordering_pass(): - output = str( - subprocess.check_output("python search_with_ordering.py", shell=True) - ) + output = str(subprocess.check_output("python search_with_ordering.py", shell=True)) assert re.match(".*search request.*", output) assert re.match(".*search response.*", output) From aff17c751abc8593f6f2ee16ffc1f38a8cb955e4 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Fri, 28 Jan 2022 21:33:50 +0000 Subject: [PATCH 03/12] fail if GOOGLE_CLOUD_PROJECT_NUMBER is not set --- .../product/import_products_big_query_table.py | 2 +- samples/interactive-tutorials/product/import_products_gcs.py | 2 +- .../product/import_products_inline_source.py | 2 +- samples/interactive-tutorials/product/setup/setup_cleanup.py | 2 +- samples/snippets/create_test_resources.py | 2 +- samples/snippets/remove_test_resources.py | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/samples/interactive-tutorials/product/import_products_big_query_table.py b/samples/interactive-tutorials/product/import_products_big_query_table.py index 0ecddf05..b74d314a 100644 --- a/samples/interactive-tutorials/product/import_products_big_query_table.py +++ b/samples/interactive-tutorials/product/import_products_big_query_table.py @@ -28,7 +28,7 @@ from setup.setup_cleanup import get_project_id -project_number = os.getenv("GOOGLE_CLOUD_PROJECT_NUMBER") +project_number = os.environ["GOOGLE_CLOUD_PROJECT_NUMBER"] project_id = get_project_id() endpoint = "retail.googleapis.com" diff --git a/samples/interactive-tutorials/product/import_products_gcs.py b/samples/interactive-tutorials/product/import_products_gcs.py index 7674d789..ca81db51 100644 --- a/samples/interactive-tutorials/product/import_products_gcs.py +++ b/samples/interactive-tutorials/product/import_products_gcs.py @@ -39,7 +39,7 @@ def get_project_id(): # Read the project number from the environment variable -project_number = os.getenv("GOOGLE_CLOUD_PROJECT_NUMBER") +project_number = os.environ["GOOGLE_CLOUD_PROJECT_NUMBER"] project_id = get_project_id() endpoint = "retail.googleapis.com" # You can change the branch here. The "default_branch" is set to point to the branch "0" diff --git a/samples/interactive-tutorials/product/import_products_inline_source.py b/samples/interactive-tutorials/product/import_products_inline_source.py index 26f6c055..41a7b46a 100644 --- a/samples/interactive-tutorials/product/import_products_inline_source.py +++ b/samples/interactive-tutorials/product/import_products_inline_source.py @@ -33,7 +33,7 @@ ) from google.protobuf.field_mask_pb2 import FieldMask -project_number = os.getenv("GOOGLE_CLOUD_PROJECT_NUMBER") +project_number = os.environ["GOOGLE_CLOUD_PROJECT_NUMBER"] endpoint = "retail.googleapis.com" default_catalog = "projects/{0}/locations/global/catalogs/default_catalog/branches/default_branch".format( diff --git a/samples/interactive-tutorials/product/setup/setup_cleanup.py b/samples/interactive-tutorials/product/setup/setup_cleanup.py index 0f69a072..2e6ff3b2 100644 --- a/samples/interactive-tutorials/product/setup/setup_cleanup.py +++ b/samples/interactive-tutorials/product/setup/setup_cleanup.py @@ -24,7 +24,7 @@ from google.cloud.retail_v2 import CreateProductRequest, DeleteProductRequest, \ FulfillmentInfo, GetProductRequest, PriceInfo, Product, ProductServiceClient -project_number = os.getenv('GOOGLE_CLOUD_PROJECT_NUMBER') +project_number = os.environ["GOOGLE_CLOUD_PROJECT_NUMBER"] endpoint = "retail.googleapis.com" default_catalog = "projects/{0}/locations/global/catalogs/default_catalog".format( project_number) diff --git a/samples/snippets/create_test_resources.py b/samples/snippets/create_test_resources.py index e56dd575..88ee6759 100644 --- a/samples/snippets/create_test_resources.py +++ b/samples/snippets/create_test_resources.py @@ -23,7 +23,7 @@ ImportProductsRequest, ProductInputConfig from google.cloud.retail_v2 import ProductServiceClient -project_number = os.getenv('GOOGLE_CLOUD_PROJECT_NUMBER') +project_number = os.environ["GOOGLE_CLOUD_PROJECT_NUMBER"] bucket_name = os.getenv('BUCKET_NAME') storage_client = storage.Client() resource_file = "resources/products.json" diff --git a/samples/snippets/remove_test_resources.py b/samples/snippets/remove_test_resources.py index 26891705..5e191f5b 100644 --- a/samples/snippets/remove_test_resources.py +++ b/samples/snippets/remove_test_resources.py @@ -21,7 +21,7 @@ from google.cloud.retail import DeleteProductRequest, ListProductsRequest, \ ProductServiceClient -project_number = os.getenv('GOOGLE_CLOUD_PROJECT_NUMBER') +project_number = os.environ["GOOGLE_CLOUD_PROJECT_NUMBER"] bucket_name = os.getenv('BUCKET_NAME') default_catalog = "projects/{0}/locations/global/catalogs/default_catalog/branches/default_branch".format( From d65fac4421cf9524e8c689cb59b5ab0f3b99a746 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Fri, 28 Jan 2022 21:36:12 +0000 Subject: [PATCH 04/12] remove endpoint --- .../product/import_products_big_query_table.py | 9 +-------- .../product/import_products_gcs.py | 10 ++-------- .../product/import_products_inline_source.py | 9 +-------- .../product/setup/setup_cleanup.py | 16 +++++----------- 4 files changed, 9 insertions(+), 35 deletions(-) diff --git a/samples/interactive-tutorials/product/import_products_big_query_table.py b/samples/interactive-tutorials/product/import_products_big_query_table.py index b74d314a..8ddcff3f 100644 --- a/samples/interactive-tutorials/product/import_products_big_query_table.py +++ b/samples/interactive-tutorials/product/import_products_big_query_table.py @@ -31,7 +31,6 @@ project_number = os.environ["GOOGLE_CLOUD_PROJECT_NUMBER"] project_id = get_project_id() -endpoint = "retail.googleapis.com" default_catalog = "projects/{0}/locations/global/catalogs/default_catalog/branches/default_branch".format( project_number ) @@ -43,12 +42,6 @@ # table_id = "products_some_invalid" -# get product service client -def get_product_service_client(): - client_options = ClientOptions(endpoint) - return ProductServiceClient(client_options=client_options) - - # get import products from big query request def get_import_products_big_query_request(reconciliation_mode): # TO CHECK ERROR HANDLING PASTE THE INVALID CATALOG NAME HERE: @@ -81,7 +74,7 @@ def import_products_from_big_query(): import_big_query_request = get_import_products_big_query_request( reconciliation_mode ) - big_query_operation = get_product_service_client().import_products( + big_query_operation = ProductServiceClient().import_products( import_big_query_request ) diff --git a/samples/interactive-tutorials/product/import_products_gcs.py b/samples/interactive-tutorials/product/import_products_gcs.py index ca81db51..3345b0b8 100644 --- a/samples/interactive-tutorials/product/import_products_gcs.py +++ b/samples/interactive-tutorials/product/import_products_gcs.py @@ -41,7 +41,7 @@ def get_project_id(): # Read the project number from the environment variable project_number = os.environ["GOOGLE_CLOUD_PROJECT_NUMBER"] project_id = get_project_id() -endpoint = "retail.googleapis.com" + # You can change the branch here. The "default_branch" is set to point to the branch "0" default_catalog = "projects/{0}/locations/global/catalogs/default_catalog/branches/default_branch".format( project_number @@ -57,12 +57,6 @@ def get_project_id(): # gcs_products_object = "products_some_invalid.json" -# get product service client -def get_product_service_client(): - client_options = ClientOptions(endpoint) - return ProductServiceClient(client_options=client_options) - - # get import products from gcs request def get_import_products_gcs_request(gcs_object_name: str): # TO CHECK ERROR HANDLING PASTE THE INVALID CATALOG NAME HERE: @@ -95,7 +89,7 @@ def get_import_products_gcs_request(gcs_object_name: str): # call the Retail API to import products def import_products_from_gcs(): import_gcs_request = get_import_products_gcs_request(gcs_products_object) - gcs_operation = get_product_service_client().import_products(import_gcs_request) + gcs_operation = ProductServiceClient().import_products(import_gcs_request) print("---the operation was started:----") print(gcs_operation.operation.name) diff --git a/samples/interactive-tutorials/product/import_products_inline_source.py b/samples/interactive-tutorials/product/import_products_inline_source.py index 41a7b46a..71d338a0 100644 --- a/samples/interactive-tutorials/product/import_products_inline_source.py +++ b/samples/interactive-tutorials/product/import_products_inline_source.py @@ -35,7 +35,6 @@ project_number = os.environ["GOOGLE_CLOUD_PROJECT_NUMBER"] -endpoint = "retail.googleapis.com" default_catalog = "projects/{0}/locations/global/catalogs/default_catalog/branches/default_branch".format( project_number ) @@ -105,12 +104,6 @@ def get_products(): return products -# get product service client -def get_product_service_client(): - client_options = ClientOptions(endpoint) - return ProductServiceClient(client_options=client_options) - - # get import products from inline source request def get_import_products_inline_request(products_to_import): # TO CHECK ERROR HANDLING PASTE THE INVALID CATALOG NAME HERE: @@ -134,7 +127,7 @@ def get_import_products_inline_request(products_to_import): # call the Retail API to import products def import_products_from_inline_source(): import_request = get_import_products_inline_request(get_products()) - import_operation = get_product_service_client().import_products(import_request) + import_operation = ProductServiceClient().import_products(import_request) print("---the operation was started:----") print(import_operation.operation.name) diff --git a/samples/interactive-tutorials/product/setup/setup_cleanup.py b/samples/interactive-tutorials/product/setup/setup_cleanup.py index 2e6ff3b2..ba7b6786 100644 --- a/samples/interactive-tutorials/product/setup/setup_cleanup.py +++ b/samples/interactive-tutorials/product/setup/setup_cleanup.py @@ -25,17 +25,11 @@ FulfillmentInfo, GetProductRequest, PriceInfo, Product, ProductServiceClient project_number = os.environ["GOOGLE_CLOUD_PROJECT_NUMBER"] -endpoint = "retail.googleapis.com" default_catalog = "projects/{0}/locations/global/catalogs/default_catalog".format( project_number) default_branch_name = "projects/" + project_number + "/locations/global/catalogs/default_catalog/branches/default_branch" -def get_product_service_client(): - client_options = ClientOptions(endpoint) - return ProductServiceClient(client_options=client_options) - - def generate_product() -> Product: price_info = PriceInfo() price_info.price = 30.0 @@ -61,7 +55,7 @@ def create_product(product_id: str) -> object: create_product_request.product_id = product_id create_product_request.parent = default_branch_name - created_product = get_product_service_client().create_product( + created_product = ProductServiceClient().create_product( create_product_request) print("---product is created:---") print(created_product) @@ -72,7 +66,7 @@ def create_product(product_id: str) -> object: def delete_product(product_name: str): delete_product_request = DeleteProductRequest() delete_product_request.name = product_name - get_product_service_client().delete_product(delete_product_request) + ProductServiceClient().delete_product(delete_product_request) print("---product " + product_name + " was deleted:---") @@ -81,7 +75,7 @@ def get_product(product_name: str): get_product_request = GetProductRequest() get_product_request.name = product_name try: - product = get_product_service_client().get_product(get_product_request) + product = ProductServiceClient().get_product(get_product_request) print("---get product response:---") print(product) return product @@ -98,8 +92,8 @@ def try_to_delete_product_if_exists(product_name: str): print( "---delete product from the catalog, if the product already exists---") try: - product = get_product_service_client().get_product(get_product_request) - get_product_service_client().delete_product(product.name) + product = ProductServiceClient().get_product(get_product_request) + ProductServiceClient().delete_product(product.name) except NotFound as e: print(e.message) From 8a3e169f3648544136ef707d48ef64e991ff7618 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Fri, 28 Jan 2022 21:40:38 +0000 Subject: [PATCH 05/12] require GOOGLE_CLOUD_PROJECT environment variable --- .../product/import_products_big_query_table.py | 4 +--- .../product/import_products_gcs.py | 9 +-------- .../product/setup/products_create_gcs_bucket.py | 6 ++++-- .../product/setup/setup_cleanup.py | 17 ++++++----------- 4 files changed, 12 insertions(+), 24 deletions(-) diff --git a/samples/interactive-tutorials/product/import_products_big_query_table.py b/samples/interactive-tutorials/product/import_products_big_query_table.py index 8ddcff3f..d8fa5553 100644 --- a/samples/interactive-tutorials/product/import_products_big_query_table.py +++ b/samples/interactive-tutorials/product/import_products_big_query_table.py @@ -26,10 +26,8 @@ ProductServiceClient, ) -from setup.setup_cleanup import get_project_id - project_number = os.environ["GOOGLE_CLOUD_PROJECT_NUMBER"] -project_id = get_project_id() +project_id = os.environ["GOOGLE_CLOUD_PROJECT"] default_catalog = "projects/{0}/locations/global/catalogs/default_catalog/branches/default_branch".format( project_number diff --git a/samples/interactive-tutorials/product/import_products_gcs.py b/samples/interactive-tutorials/product/import_products_gcs.py index 3345b0b8..e1bed6b4 100644 --- a/samples/interactive-tutorials/product/import_products_gcs.py +++ b/samples/interactive-tutorials/product/import_products_gcs.py @@ -31,16 +31,9 @@ ) -def get_project_id(): - get_project_command = "gcloud config get-value project --format json" - config = subprocess.check_output(shlex.split(get_project_command)) - project_id = re.search('"(.*?)"', str(config)).group(1) - return project_id - - # Read the project number from the environment variable project_number = os.environ["GOOGLE_CLOUD_PROJECT_NUMBER"] -project_id = get_project_id() +project_id = os.environ["GOOGLE_CLOUD_PROJECT"] # You can change the branch here. The "default_branch" is set to point to the branch "0" default_catalog = "projects/{0}/locations/global/catalogs/default_catalog/branches/default_branch".format( diff --git a/samples/interactive-tutorials/product/setup/products_create_gcs_bucket.py b/samples/interactive-tutorials/product/setup/products_create_gcs_bucket.py index bbe07a26..e166066e 100644 --- a/samples/interactive-tutorials/product/setup/products_create_gcs_bucket.py +++ b/samples/interactive-tutorials/product/setup/products_create_gcs_bucket.py @@ -13,11 +13,13 @@ # limitations under the License. import datetime +import os -from setup_cleanup import create_bucket, get_project_id, upload_blob +from setup_cleanup import create_bucket, upload_blob +project_id = os.environ["GOOGLE_CLOUD_PROJECT"] timestamp_ = datetime.datetime.now().timestamp().__round__() -bucket_name = "{}_products_{}".format(get_project_id(), timestamp_) +bucket_name = "{}_products_{}".format(project_id, timestamp_) create_bucket(bucket_name) upload_blob(bucket_name, "resources/products.json") diff --git a/samples/interactive-tutorials/product/setup/setup_cleanup.py b/samples/interactive-tutorials/product/setup/setup_cleanup.py index ba7b6786..734d9882 100644 --- a/samples/interactive-tutorials/product/setup/setup_cleanup.py +++ b/samples/interactive-tutorials/product/setup/setup_cleanup.py @@ -25,6 +25,7 @@ FulfillmentInfo, GetProductRequest, PriceInfo, Product, ProductServiceClient project_number = os.environ["GOOGLE_CLOUD_PROJECT_NUMBER"] +project_id = os.environ["GOOGLE_CLOUD_PROJECT"] default_catalog = "projects/{0}/locations/global/catalogs/default_catalog".format( project_number) default_branch_name = "projects/" + project_number + "/locations/global/catalogs/default_catalog/branches/default_branch" @@ -98,12 +99,6 @@ def try_to_delete_product_if_exists(product_name: str): print(e.message) -def get_project_id(): - get_project_command = "gcloud config get-value project --format json" - config = subprocess.check_output(shlex.split(get_project_command)) - project_id = re.search('\"(.*?)\"', str(config)).group(1) - return project_id - def create_bucket(bucket_name: str): """Create a new bucket in Cloud Storage""" @@ -174,7 +169,7 @@ def create_bq_dataset(dataset_name): print("Creating dataset {}".format(dataset_name)) if dataset_name not in list_bq_datasets(): create_dataset_command = 'bq --location=US mk -d --default_table_expiration 3600 --description "This is my dataset." {}:{}'.format( - get_project_id(), dataset_name) + project_id, dataset_name) subprocess.check_output(shlex.split(create_dataset_command)) print("dataset is created") else: @@ -183,7 +178,7 @@ def create_bq_dataset(dataset_name): def list_bq_datasets(): """List BigQuery datasets in the project""" - list_dataset_command = "bq ls --project_id {}".format(get_project_id()) + list_dataset_command = "bq ls --project_id {}".format(project_id) datasets = subprocess.check_output(shlex.split(list_dataset_command)) return str(datasets) @@ -193,7 +188,7 @@ def create_bq_table(dataset, table_name, schema): print("Creating BigQuery table {}".format(table_name)) if table_name not in list_bq_tables(dataset): create_table_command = "bq mk --table {}:{}.{} {}".format( - get_project_id(), + project_id, dataset, table_name, schema) output = subprocess.check_output(shlex.split(create_table_command)) @@ -205,7 +200,7 @@ def create_bq_table(dataset, table_name, schema): def list_bq_tables(dataset): """List BigQuery tables in the dataset""" - list_tables_command = "bq ls {}:{}".format(get_project_id(), dataset) + list_tables_command = "bq ls {}:{}".format(project_id, dataset) tables = subprocess.check_output(shlex.split(list_tables_command)) return str(tables) @@ -215,6 +210,6 @@ def upload_data_to_bq_table(dataset, table_name, source, schema): print("Uploading data form {} to the table {}.{}".format(source, dataset, table_name)) upload_data_command = "bq load --source_format=NEWLINE_DELIMITED_JSON {}:{}.{} {} {}".format( - get_project_id(), dataset, table_name, source, schema) + project_id, dataset, table_name, source, schema) output = subprocess.check_output(shlex.split(upload_data_command)) print(output) From b1b4fbba981b0a76d44fc7adedf55b2291ec6615 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Fri, 28 Jan 2022 21:50:09 +0000 Subject: [PATCH 06/12] adjust filepaths samples testing --- .../interactive-tutorials/product/import_products_bq_test.py | 2 +- samples/interactive-tutorials/product/import_products_gcs.py | 4 ++-- .../interactive-tutorials/product/import_products_gcs_test.py | 2 +- .../product/import_products_inline_test.py | 2 +- samples/interactive-tutorials/product/noxfile_config.py | 4 +++- .../product/setup/products_create_bigquery_table_test.py | 2 +- .../product/setup/products_create_gcs_bucket_test.py | 2 +- .../product/setup/products_delete_gcs_bucket.py | 2 +- samples/snippets/create_test_resources.py | 2 +- samples/snippets/remove_test_resources.py | 2 +- 10 files changed, 13 insertions(+), 11 deletions(-) diff --git a/samples/interactive-tutorials/product/import_products_bq_test.py b/samples/interactive-tutorials/product/import_products_bq_test.py index 7507b4b2..bdc5687a 100644 --- a/samples/interactive-tutorials/product/import_products_bq_test.py +++ b/samples/interactive-tutorials/product/import_products_bq_test.py @@ -19,7 +19,7 @@ def test_import_products_bq(): output = str( subprocess.check_output( - "python product/import_products_big_query_table.py", shell=True + "python import_products_big_query_table.py", shell=True ) ) diff --git a/samples/interactive-tutorials/product/import_products_gcs.py b/samples/interactive-tutorials/product/import_products_gcs.py index e1bed6b4..a851d8f2 100644 --- a/samples/interactive-tutorials/product/import_products_gcs.py +++ b/samples/interactive-tutorials/product/import_products_gcs.py @@ -34,14 +34,14 @@ # Read the project number from the environment variable project_number = os.environ["GOOGLE_CLOUD_PROJECT_NUMBER"] project_id = os.environ["GOOGLE_CLOUD_PROJECT"] +bucket_name = os.environ["BUCKET_NAME"] # You can change the branch here. The "default_branch" is set to point to the branch "0" default_catalog = "projects/{0}/locations/global/catalogs/default_catalog/branches/default_branch".format( project_number ) -# Read bucket name from the environment variable -gcs_bucket = "gs://{}".format(os.getenv("BUCKET_NAME")) +gcs_bucket = "gs://{}".format(bucket_name) gcs_errors_bucket = "{}/error".format(gcs_bucket) gcs_products_object = "products.json" diff --git a/samples/interactive-tutorials/product/import_products_gcs_test.py b/samples/interactive-tutorials/product/import_products_gcs_test.py index 9edbde9b..4e0e023a 100644 --- a/samples/interactive-tutorials/product/import_products_gcs_test.py +++ b/samples/interactive-tutorials/product/import_products_gcs_test.py @@ -18,7 +18,7 @@ def test_import_products_gcs(): output = str( - subprocess.check_output("python product/import_products_gcs.py", shell=True) + subprocess.check_output("python import_products_gcs.py", shell=True) ) assert re.match(".*import products from google cloud source request.*", output) diff --git a/samples/interactive-tutorials/product/import_products_inline_test.py b/samples/interactive-tutorials/product/import_products_inline_test.py index b2821dae..7ece9c6f 100644 --- a/samples/interactive-tutorials/product/import_products_inline_test.py +++ b/samples/interactive-tutorials/product/import_products_inline_test.py @@ -19,7 +19,7 @@ def test_import_products_gcs(): output = str( subprocess.check_output( - "python product/import_products_inline_source.py", shell=True + "python import_products_inline_source.py", shell=True ) ) diff --git a/samples/interactive-tutorials/product/noxfile_config.py b/samples/interactive-tutorials/product/noxfile_config.py index 401ff7f2..edce74e3 100644 --- a/samples/interactive-tutorials/product/noxfile_config.py +++ b/samples/interactive-tutorials/product/noxfile_config.py @@ -28,5 +28,7 @@ # 'gcloud_project_env': 'BUILD_SPECIFIC_GCLOUD_PROJECT', # A dictionary you want to inject into your test. Don't put any # secrets here. These values will override predefined values. - "envs": {"BUCKET_NAME": "retail-interactive-tutorials",}, + "envs": { + "BUCKET_NAME": "retail-interactive-tutorials", + }, } diff --git a/samples/interactive-tutorials/product/setup/products_create_bigquery_table_test.py b/samples/interactive-tutorials/product/setup/products_create_bigquery_table_test.py index 71a45c3c..bc58e8bc 100644 --- a/samples/interactive-tutorials/product/setup/products_create_bigquery_table_test.py +++ b/samples/interactive-tutorials/product/setup/products_create_bigquery_table_test.py @@ -19,7 +19,7 @@ def test_create_bigquery_table(): output = str( subprocess.check_output( - 'python product/setup/products_create_bigquery_table.py', + 'python setup/products_create_bigquery_table.py', shell=True)) assert re.match( '.*Creating dataset products.*', output) diff --git a/samples/interactive-tutorials/product/setup/products_create_gcs_bucket_test.py b/samples/interactive-tutorials/product/setup/products_create_gcs_bucket_test.py index 5846ab1c..90f55ce3 100644 --- a/samples/interactive-tutorials/product/setup/products_create_gcs_bucket_test.py +++ b/samples/interactive-tutorials/product/setup/products_create_gcs_bucket_test.py @@ -20,7 +20,7 @@ def test_create_gcs_bucket(): output = str( subprocess.check_output( - 'python product/setup/products_create_gcs_bucket.py', + 'python setup/products_create_gcs_bucket.py', shell=True)) bucket_name = re.search('The gcs bucket (.+?) was created', output).group(1) diff --git a/samples/interactive-tutorials/product/setup/products_delete_gcs_bucket.py b/samples/interactive-tutorials/product/setup/products_delete_gcs_bucket.py index 6e76c110..010478cf 100644 --- a/samples/interactive-tutorials/product/setup/products_delete_gcs_bucket.py +++ b/samples/interactive-tutorials/product/setup/products_delete_gcs_bucket.py @@ -19,7 +19,7 @@ def delete_bucket_by_name(name: str): if name is None: - bucket_name = os.getenv("BUCKET_NAME") + bucket_name = os.environ["BUCKET_NAME"] delete_bucket(bucket_name) else: delete_bucket(name) diff --git a/samples/snippets/create_test_resources.py b/samples/snippets/create_test_resources.py index 88ee6759..ef07e133 100644 --- a/samples/snippets/create_test_resources.py +++ b/samples/snippets/create_test_resources.py @@ -24,7 +24,7 @@ from google.cloud.retail_v2 import ProductServiceClient project_number = os.environ["GOOGLE_CLOUD_PROJECT_NUMBER"] -bucket_name = os.getenv('BUCKET_NAME') +bucket_name = os.environ['BUCKET_NAME'] storage_client = storage.Client() resource_file = "resources/products.json" object_name = re.search('resources/(.*?)$', resource_file).group(1) diff --git a/samples/snippets/remove_test_resources.py b/samples/snippets/remove_test_resources.py index 5e191f5b..c6faf299 100644 --- a/samples/snippets/remove_test_resources.py +++ b/samples/snippets/remove_test_resources.py @@ -22,7 +22,7 @@ ProductServiceClient project_number = os.environ["GOOGLE_CLOUD_PROJECT_NUMBER"] -bucket_name = os.getenv('BUCKET_NAME') +bucket_name = os.environ['BUCKET_NAME'] default_catalog = "projects/{0}/locations/global/catalogs/default_catalog/branches/default_branch".format( project_number) From 0c74c0ba4cc8fce573e5775ad772d9e4a87b3bfb Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Fri, 28 Jan 2022 21:56:29 +0000 Subject: [PATCH 07/12] lint --- .../product/import_products_big_query_table.py | 1 - samples/interactive-tutorials/product/import_products_gcs.py | 4 ---- .../product/import_products_inline_source.py | 1 - .../product/setup/products_create_gcs_bucket_test.py | 2 +- samples/interactive-tutorials/product/setup/setup_cleanup.py | 2 -- 5 files changed, 1 insertion(+), 9 deletions(-) diff --git a/samples/interactive-tutorials/product/import_products_big_query_table.py b/samples/interactive-tutorials/product/import_products_big_query_table.py index d8fa5553..934e259c 100644 --- a/samples/interactive-tutorials/product/import_products_big_query_table.py +++ b/samples/interactive-tutorials/product/import_products_big_query_table.py @@ -18,7 +18,6 @@ import os import time -from google.api_core.client_options import ClientOptions from google.cloud.retail import ( BigQuerySource, ImportProductsRequest, diff --git a/samples/interactive-tutorials/product/import_products_gcs.py b/samples/interactive-tutorials/product/import_products_gcs.py index a851d8f2..a23272cc 100644 --- a/samples/interactive-tutorials/product/import_products_gcs.py +++ b/samples/interactive-tutorials/product/import_products_gcs.py @@ -16,12 +16,8 @@ # Import products into a catalog from gcs using Retail API # import os -import re -import shlex -import subprocess import time -from google.api_core.client_options import ClientOptions from google.cloud.retail import ( GcsSource, ImportErrorsConfig, diff --git a/samples/interactive-tutorials/product/import_products_inline_source.py b/samples/interactive-tutorials/product/import_products_inline_source.py index 71d338a0..2072e38e 100644 --- a/samples/interactive-tutorials/product/import_products_inline_source.py +++ b/samples/interactive-tutorials/product/import_products_inline_source.py @@ -20,7 +20,6 @@ import string import time -from google.api_core.client_options import ClientOptions from google.cloud.retail import ( ColorInfo, FulfillmentInfo, diff --git a/samples/interactive-tutorials/product/setup/products_create_gcs_bucket_test.py b/samples/interactive-tutorials/product/setup/products_create_gcs_bucket_test.py index 90f55ce3..19301b36 100644 --- a/samples/interactive-tutorials/product/setup/products_create_gcs_bucket_test.py +++ b/samples/interactive-tutorials/product/setup/products_create_gcs_bucket_test.py @@ -14,6 +14,7 @@ import re import subprocess + from products_delete_gcs_bucket import delete_bucket_by_name @@ -37,4 +38,3 @@ def test_create_gcs_bucket(): assert re.match( '.*Uploading data form resources/products_some_invalid.json to the bucket.*', output) - diff --git a/samples/interactive-tutorials/product/setup/setup_cleanup.py b/samples/interactive-tutorials/product/setup/setup_cleanup.py index 734d9882..7fc2b95d 100644 --- a/samples/interactive-tutorials/product/setup/setup_cleanup.py +++ b/samples/interactive-tutorials/product/setup/setup_cleanup.py @@ -17,7 +17,6 @@ import shlex import subprocess -from google.api_core.client_options import ClientOptions from google.api_core.exceptions import NotFound from google.cloud import storage @@ -99,7 +98,6 @@ def try_to_delete_product_if_exists(product_name: str): print(e.message) - def create_bucket(bucket_name: str): """Create a new bucket in Cloud Storage""" print("Creating new bucket:" + bucket_name) From 6b23a7541a45344785091af6cf775f8718088098 Mon Sep 17 00:00:00 2001 From: t-karasova Date: Wed, 2 Feb 2022 15:29:55 +0100 Subject: [PATCH 08/12] update and move the tests resources setup files --- .../TEST_RESOURCES_SETUP_CLEANUP.md | 23 +- .../import_products_big_query_table.py | 103 ------ .../product/import_products_bq_test.py | 33 -- .../product/import_products_gcs.py | 113 ------- .../product/import_products_gcs_test.py | 32 -- .../product/import_products_inline_source.py | 156 --------- .../product/import_products_inline_test.py | 33 -- .../interactive-tutorials/product/noxfile.py | 279 --------------- .../product/noxfile_config.py | 34 -- .../product/requirements-test.txt | 2 - .../product/requirements.txt | 4 - .../setup/products_create_bigquery_table.py | 31 -- .../products_create_bigquery_table_test.py | 43 --- .../setup/products_create_gcs_bucket.py | 28 -- .../setup/products_create_gcs_bucket_test.py | 40 --- .../setup/products_delete_gcs_bucket.py | 25 -- .../product/setup/setup_cleanup.py | 213 ------------ .../resources/events_schema.json | 73 ++++ .../resources/product_schema.json | 317 ++++++++++++++++++ .../resources/products.json | 0 .../resources/products_some_invalid.json | 3 + .../resources/user_events.json | 4 + .../resources/user_events_some_invalid.json | 4 + .../search/noxfile_config.py | 4 +- .../search/search_simple_query_test.py | 4 +- .../search/search_with_filtering_test.py | 4 +- .../search/search_with_ordering_test.py | 4 +- .../create_test_resources.py | 98 +++++- .../remove_test_resources.py | 23 +- 29 files changed, 542 insertions(+), 1188 deletions(-) rename samples/{snippets => interactive-tutorials}/TEST_RESOURCES_SETUP_CLEANUP.md (51%) delete mode 100644 samples/interactive-tutorials/product/import_products_big_query_table.py delete mode 100644 samples/interactive-tutorials/product/import_products_bq_test.py delete mode 100644 samples/interactive-tutorials/product/import_products_gcs.py delete mode 100644 samples/interactive-tutorials/product/import_products_gcs_test.py delete mode 100644 samples/interactive-tutorials/product/import_products_inline_source.py delete mode 100644 samples/interactive-tutorials/product/import_products_inline_test.py delete mode 100644 samples/interactive-tutorials/product/noxfile.py delete mode 100644 samples/interactive-tutorials/product/noxfile_config.py delete mode 100644 samples/interactive-tutorials/product/requirements-test.txt delete mode 100644 samples/interactive-tutorials/product/requirements.txt delete mode 100644 samples/interactive-tutorials/product/setup/products_create_bigquery_table.py delete mode 100644 samples/interactive-tutorials/product/setup/products_create_bigquery_table_test.py delete mode 100644 samples/interactive-tutorials/product/setup/products_create_gcs_bucket.py delete mode 100644 samples/interactive-tutorials/product/setup/products_create_gcs_bucket_test.py delete mode 100644 samples/interactive-tutorials/product/setup/products_delete_gcs_bucket.py delete mode 100644 samples/interactive-tutorials/product/setup/setup_cleanup.py create mode 100644 samples/interactive-tutorials/resources/events_schema.json create mode 100644 samples/interactive-tutorials/resources/product_schema.json rename samples/{snippets => interactive-tutorials}/resources/products.json (100%) create mode 100644 samples/interactive-tutorials/resources/products_some_invalid.json create mode 100644 samples/interactive-tutorials/resources/user_events.json create mode 100644 samples/interactive-tutorials/resources/user_events_some_invalid.json rename samples/{snippets => interactive-tutorials/test_resources_recovery}/create_test_resources.py (53%) rename samples/{snippets => interactive-tutorials/test_resources_recovery}/remove_test_resources.py (76%) diff --git a/samples/snippets/TEST_RESOURCES_SETUP_CLEANUP.md b/samples/interactive-tutorials/TEST_RESOURCES_SETUP_CLEANUP.md similarity index 51% rename from samples/snippets/TEST_RESOURCES_SETUP_CLEANUP.md rename to samples/interactive-tutorials/TEST_RESOURCES_SETUP_CLEANUP.md index 000f12d0..5974753e 100644 --- a/samples/snippets/TEST_RESOURCES_SETUP_CLEANUP.md +++ b/samples/interactive-tutorials/TEST_RESOURCES_SETUP_CLEANUP.md @@ -3,8 +3,10 @@ ## Required environment variables To successfully import the catalog data for tests, the following environment variables should be set: - - PROJECT_NUMBER + - GOOGLE_CLOUD_PROJECT_NUMBER + - GOOGLE_CLOUD_PROJECT_ID - BUCKET_NAME + - EVENTS_BUCKET_NAME These values are stored in the Secret Manager and will be submitted as docker environment variables before the test run. @@ -12,13 +14,20 @@ The Secret Manager name is set in .kokoro/presubmit/common.cfg file, SECRET_MANA ## Import catalog data -There is a JSON file with valid products prepared in the `product` directory: -`resources/products.json`. +There are a JSON files with valid products and user events prepared in `resources` directory: +`samples/resources/products.json` and `samples/resources/user_events.json` respectively. Run the `create_test_resources.py` to perform the following actions: - create the GCS bucket , - - upload the product data from `resources/products.json` file, - - import products to the default branch of the Retail catalog. + - upload the product data from `resources/products.json` file to products bucket, + - import products to the default branch of the Retail catalog, + - create the GCS bucket , + - upload the product data from `resources/user_events.json` file to events bucket, + - create a BigQuery dataset and table `products`, + - insert products from resources/products.json to the created products table, + - create a BigQuery dataset and table `events`, + - insert user events from resources/user_events.json to the created events table + ``` $ python create_test_resources.py @@ -33,6 +42,10 @@ Run the `remove_test_resources.py` to perform the following actions: - remove all objects from the GCS bucket , - remove the bucket, - delete all products from the Retail catalog. + - remove all objects from the GCS bucket , + - remove the bucket, + - remove dataset `products` along with tables + - remove dataset `user_events` along with tables ``` $ python remove_test_resources.py diff --git a/samples/interactive-tutorials/product/import_products_big_query_table.py b/samples/interactive-tutorials/product/import_products_big_query_table.py deleted file mode 100644 index 934e259c..00000000 --- a/samples/interactive-tutorials/product/import_products_big_query_table.py +++ /dev/null @@ -1,103 +0,0 @@ -# Copyright 2021 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# [START retail_import_products_from_big_query] -# Import products into a catalog from big query table using Retail API -# -import os -import time - -from google.cloud.retail import ( - BigQuerySource, - ImportProductsRequest, - ProductInputConfig, - ProductServiceClient, -) - -project_number = os.environ["GOOGLE_CLOUD_PROJECT_NUMBER"] -project_id = os.environ["GOOGLE_CLOUD_PROJECT"] - -default_catalog = "projects/{0}/locations/global/catalogs/default_catalog/branches/default_branch".format( - project_number -) -dataset_id = "products" -table_id = "products" - - -# TO CHECK ERROR HANDLING USE THE TABLE WITH INVALID PRODUCTS: -# table_id = "products_some_invalid" - - -# get import products from big query request -def get_import_products_big_query_request(reconciliation_mode): - # TO CHECK ERROR HANDLING PASTE THE INVALID CATALOG NAME HERE: - # default_catalog = "invalid_catalog_name" - big_query_source = BigQuerySource() - big_query_source.project_id = project_id - big_query_source.dataset_id = dataset_id - big_query_source.table_id = table_id - big_query_source.data_schema = "product" - - input_config = ProductInputConfig() - input_config.big_query_source = big_query_source - - import_request = ImportProductsRequest() - import_request.parent = default_catalog - import_request.reconciliation_mode = reconciliation_mode - import_request.input_config = input_config - - print("---import products from big query table request---") - print(import_request) - - return import_request - - -# call the Retail API to import products -def import_products_from_big_query(): - # TRY THE FULL RECONCILIATION MODE HERE: - reconciliation_mode = ImportProductsRequest.ReconciliationMode.INCREMENTAL - - import_big_query_request = get_import_products_big_query_request( - reconciliation_mode - ) - big_query_operation = ProductServiceClient().import_products( - import_big_query_request - ) - - print("---the operation was started:----") - print(big_query_operation.operation.name) - - while not big_query_operation.done(): - print("---please wait till operation is done---") - time.sleep(5) - print("---import products operation is done---") - - if big_query_operation.metadata is not None: - print("---number of successfully imported products---") - print(big_query_operation.metadata.success_count) - print("---number of failures during the importing---") - print(big_query_operation.metadata.failure_count) - else: - print("---operation.metadata is empty---") - - if big_query_operation.result is not None: - print("---operation result:---") - print(big_query_operation.result()) - else: - print("---operation.result is empty---") - - -import_products_from_big_query() - -# [END retail_import_products_from_big_query] diff --git a/samples/interactive-tutorials/product/import_products_bq_test.py b/samples/interactive-tutorials/product/import_products_bq_test.py deleted file mode 100644 index bdc5687a..00000000 --- a/samples/interactive-tutorials/product/import_products_bq_test.py +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright 2021 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import re -import subprocess - - -def test_import_products_bq(): - output = str( - subprocess.check_output( - "python import_products_big_query_table.py", shell=True - ) - ) - - assert re.match(".*import products from big query table request.*", output) - assert re.match(".*the operation was started.*", output) - assert re.match( - ".*projects/.*/locations/global/catalogs/default_catalog/branches/0/operations/import-products.*", - output, - ) - - assert re.match(".*number of successfully imported products.*316.*", output) diff --git a/samples/interactive-tutorials/product/import_products_gcs.py b/samples/interactive-tutorials/product/import_products_gcs.py deleted file mode 100644 index a23272cc..00000000 --- a/samples/interactive-tutorials/product/import_products_gcs.py +++ /dev/null @@ -1,113 +0,0 @@ -# Copyright 2021 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# [START retail_import_products_from_gcs] -# Import products into a catalog from gcs using Retail API -# -import os -import time - -from google.cloud.retail import ( - GcsSource, - ImportErrorsConfig, - ImportProductsRequest, - ProductInputConfig, - ProductServiceClient, -) - - -# Read the project number from the environment variable -project_number = os.environ["GOOGLE_CLOUD_PROJECT_NUMBER"] -project_id = os.environ["GOOGLE_CLOUD_PROJECT"] -bucket_name = os.environ["BUCKET_NAME"] - -# You can change the branch here. The "default_branch" is set to point to the branch "0" -default_catalog = "projects/{0}/locations/global/catalogs/default_catalog/branches/default_branch".format( - project_number -) - -gcs_bucket = "gs://{}".format(bucket_name) -gcs_errors_bucket = "{}/error".format(gcs_bucket) -gcs_products_object = "products.json" - - -# TO CHECK ERROR HANDLING USE THE JSON WITH INVALID PRODUCT -# gcs_products_object = "products_some_invalid.json" - - -# get import products from gcs request -def get_import_products_gcs_request(gcs_object_name: str): - # TO CHECK ERROR HANDLING PASTE THE INVALID CATALOG NAME HERE: - # default_catalog = "invalid_catalog_name" - gcs_source = GcsSource() - gcs_source.input_uris = ["{0}/{1}".format(gcs_bucket, gcs_object_name)] - - input_config = ProductInputConfig() - input_config.gcs_source = gcs_source - print("GRS source:") - print(gcs_source.input_uris) - - errors_config = ImportErrorsConfig() - errors_config.gcs_prefix = gcs_errors_bucket - - import_request = ImportProductsRequest() - import_request.parent = default_catalog - import_request.reconciliation_mode = ( - ImportProductsRequest.ReconciliationMode.INCREMENTAL - ) - import_request.input_config = input_config - import_request.errors_config = errors_config - - print("---import products from google cloud source request---") - print(import_request) - - return import_request - - -# call the Retail API to import products -def import_products_from_gcs(): - import_gcs_request = get_import_products_gcs_request(gcs_products_object) - gcs_operation = ProductServiceClient().import_products(import_gcs_request) - - print("---the operation was started:----") - print(gcs_operation.operation.name) - - while not gcs_operation.done(): - print("---please wait till operation is done---") - time.sleep(5) - print("---import products operation is done---") - - if gcs_operation.metadata is not None: - print("---number of successfully imported products---") - print(gcs_operation.metadata.success_count) - print("---number of failures during the importing---") - print(gcs_operation.metadata.failure_count) - else: - print("---operation.metadata is empty---") - - if gcs_operation.result is not None: - print("---operation result:---") - print(gcs_operation.result()) - else: - print("---operation.result is empty---") - - # The imported products needs to be indexed in the catalog before they become available for search. - print( - "Wait 2 -5 minutes till products become indexed in the catalog, after that they will be available for search" - ) - - -import_products_from_gcs() - -# [END retail_import_products_from_gcs] diff --git a/samples/interactive-tutorials/product/import_products_gcs_test.py b/samples/interactive-tutorials/product/import_products_gcs_test.py deleted file mode 100644 index 4e0e023a..00000000 --- a/samples/interactive-tutorials/product/import_products_gcs_test.py +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright 2021 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import re -import subprocess - - -def test_import_products_gcs(): - output = str( - subprocess.check_output("python import_products_gcs.py", shell=True) - ) - - assert re.match(".*import products from google cloud source request.*", output) - assert re.match('.*input_uris: "gs://.*/products.json".*', output) - assert re.match(".*the operation was started.*", output) - assert re.match( - ".*projects/.*/locations/global/catalogs/default_catalog/branches/0/operations/import-products.*", - output, - ) - - assert re.match(".*number of successfully imported products.*316.*", output) diff --git a/samples/interactive-tutorials/product/import_products_inline_source.py b/samples/interactive-tutorials/product/import_products_inline_source.py deleted file mode 100644 index 2072e38e..00000000 --- a/samples/interactive-tutorials/product/import_products_inline_source.py +++ /dev/null @@ -1,156 +0,0 @@ -# Copyright 2021 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# [START retail_import_products_from_inline_source] -# Import products into a catalog from inline source using Retail API -# -import os -import random -import string -import time - -from google.cloud.retail import ( - ColorInfo, - FulfillmentInfo, - ImportProductsRequest, - PriceInfo, - Product, - ProductInlineSource, - ProductInputConfig, - ProductServiceClient, -) -from google.protobuf.field_mask_pb2 import FieldMask - -project_number = os.environ["GOOGLE_CLOUD_PROJECT_NUMBER"] - -default_catalog = "projects/{0}/locations/global/catalogs/default_catalog/branches/default_branch".format( - project_number -) - - -# prepare product to import as inline source -def get_products(): - products = [] - product1 = Product() - product2 = Product() - - price_info1 = PriceInfo() - price_info1.price = 16.0 - price_info1.original_price = 45.0 - price_info1.cost = 12.0 - price_info1.currency_code = "USD" - - color_info1 = ColorInfo() - color_info1.color_families = ["Blue"] - color_info1.colors = ["Light blue", "Blue", "Dark blue"] - - fulfillment_info1 = FulfillmentInfo() - fulfillment_info1.type_ = "pickup-in-store" - fulfillment_info1.place_ids = ["store1", "store2"] - - field_mask1 = FieldMask(paths=["title", "categories", "price_info", "color_info"]) - - # TO CHECK ERROR HANDLING COMMENT OUT THE PRODUCT TITLE HERE: - product1.title = "#IamRemarkable Pen" - product1.id = "".join(random.sample(string.ascii_lowercase, 8)) - product1.categories = ["Office"] - product1.uri = "https://shop.googlemerchandisestore.com/Google+Redesign/Office/IamRemarkable+Pen" - product1.brands = ["#IamRemarkable"] - product1.price_info = price_info1 - product1.color_info = color_info1 - product1.fulfillment_info = [fulfillment_info1] - product1.retrievable_fields = field_mask1 - - price_info2 = PriceInfo() - price_info2.price = 35.0 - price_info2.original_price = 45.0 - price_info2.cost = 12.0 - price_info2.currency_code = "USD" - - color_info2 = ColorInfo() - color_info2.color_families = ["Blue"] - color_info2.colors = ["Sky blue"] - - fulfillment_info2 = FulfillmentInfo() - fulfillment_info2.type_ = "pickup-in-store" - fulfillment_info2.place_ids = ["store2", "store3"] - - field_mask2 = FieldMask(paths=["title", "categories", "price_info", "color_info"]) - - product2.title = "Android Embroidered Crewneck Sweater" - product2.id = "".join(random.sample(string.ascii_lowercase, 8)) - product2.categories = ["Apparel"] - product2.uri = "https://shop.googlemerchandisestore.com/Google+Redesign/Apparel/Android+Embroidered+Crewneck+Sweater" - product2.brands = ["Android"] - product2.price_info = price_info2 - product2.color_info = color_info2 - product2.fulfillment_info = [fulfillment_info2] - product2.retrievable_fields = field_mask2 - - products.append(product1) - products.append(product2) - return products - - -# get import products from inline source request -def get_import_products_inline_request(products_to_import): - # TO CHECK ERROR HANDLING PASTE THE INVALID CATALOG NAME HERE: - # default_catalog = "invalid_catalog_name" - inline_source = ProductInlineSource() - inline_source.products = products_to_import - - input_config = ProductInputConfig() - input_config.product_inline_source = inline_source - - import_request = ImportProductsRequest() - import_request.parent = default_catalog - import_request.input_config = input_config - - print("---import products from inline source request---") - print(import_request) - - return import_request - - -# call the Retail API to import products -def import_products_from_inline_source(): - import_request = get_import_products_inline_request(get_products()) - import_operation = ProductServiceClient().import_products(import_request) - - print("---the operation was started:----") - print(import_operation.operation.name) - - while not import_operation.done(): - print("---please wait till operation is done---") - time.sleep(5) - print("---import products operation is done---") - - if import_operation.metadata is not None: - print("---number of successfully imported products---") - print(import_operation.metadata.success_count) - print("---number of failures during the importing---") - print(import_operation.metadata.failure_count) - else: - print("---operation.metadata is empty---") - - if import_operation.result is not None: - print("---operation result:---") - print(import_operation.result()) - else: - print("---operation.result is empty---") - - -import_products_from_inline_source() - -# [END retail_import_products_from_inline_source] diff --git a/samples/interactive-tutorials/product/import_products_inline_test.py b/samples/interactive-tutorials/product/import_products_inline_test.py deleted file mode 100644 index 7ece9c6f..00000000 --- a/samples/interactive-tutorials/product/import_products_inline_test.py +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright 2021 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import re -import subprocess - - -def test_import_products_gcs(): - output = str( - subprocess.check_output( - "python import_products_inline_source.py", shell=True - ) - ) - - assert re.match(".*import products from inline source request.*", output) - assert re.match(".*the operation was started.*", output) - assert re.match( - ".*projects/.*/locations/global/catalogs/default_catalog/branches/0/operations/import-products.*", - output, - ) - - assert re.match(".*number of successfully imported products.*2.*", output) diff --git a/samples/interactive-tutorials/product/noxfile.py b/samples/interactive-tutorials/product/noxfile.py deleted file mode 100644 index 20cdfc62..00000000 --- a/samples/interactive-tutorials/product/noxfile.py +++ /dev/null @@ -1,279 +0,0 @@ -# Copyright 2019 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import print_function - -import glob -import os -from pathlib import Path -import sys -from typing import Callable, Dict, List, Optional - -import nox - - -# WARNING - WARNING - WARNING - WARNING - WARNING -# WARNING - WARNING - WARNING - WARNING - WARNING -# DO NOT EDIT THIS FILE EVER! -# WARNING - WARNING - WARNING - WARNING - WARNING -# WARNING - WARNING - WARNING - WARNING - WARNING - -BLACK_VERSION = "black==19.10b0" - -# Copy `noxfile_config.py` to your directory and modify it instead. - -# `TEST_CONFIG` dict is a configuration hook that allows users to -# modify the test configurations. The values here should be in sync -# with `noxfile_config.py`. Users will copy `noxfile_config.py` into -# their directory and modify it. - -TEST_CONFIG = { - # You can opt out from the test for specific Python versions. - "ignored_versions": [], - # Old samples are opted out of enforcing Python type hints - # All new samples should feature them - "enforce_type_hints": False, - # An envvar key for determining the project id to use. Change it - # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a - # build specific Cloud project. You can also use your own string - # to use your own Cloud project. - "gcloud_project_env": "GOOGLE_CLOUD_PROJECT", - # 'gcloud_project_env': 'BUILD_SPECIFIC_GCLOUD_PROJECT', - # If you need to use a specific version of pip, - # change pip_version_override to the string representation - # of the version number, for example, "20.2.4" - "pip_version_override": None, - # A dictionary you want to inject into your test. Don't put any - # secrets here. These values will override predefined values. - "envs": {}, -} - - -try: - # Ensure we can import noxfile_config in the project's directory. - sys.path.append(".") - from noxfile_config import TEST_CONFIG_OVERRIDE -except ImportError as e: - print("No user noxfile_config found: detail: {}".format(e)) - TEST_CONFIG_OVERRIDE = {} - -# Update the TEST_CONFIG with the user supplied values. -TEST_CONFIG.update(TEST_CONFIG_OVERRIDE) - - -def get_pytest_env_vars() -> Dict[str, str]: - """Returns a dict for pytest invocation.""" - ret = {} - - # Override the GCLOUD_PROJECT and the alias. - env_key = TEST_CONFIG["gcloud_project_env"] - # This should error out if not set. - ret["GOOGLE_CLOUD_PROJECT"] = os.environ[env_key] - - # Apply user supplied envs. - ret.update(TEST_CONFIG["envs"]) - return ret - - -# DO NOT EDIT - automatically generated. -# All versions used to test samples. -ALL_VERSIONS = ["3.6", "3.7", "3.8", "3.9", "3.10"] - -# Any default versions that should be ignored. -IGNORED_VERSIONS = TEST_CONFIG["ignored_versions"] - -TESTED_VERSIONS = sorted([v for v in ALL_VERSIONS if v not in IGNORED_VERSIONS]) - -INSTALL_LIBRARY_FROM_SOURCE = os.environ.get("INSTALL_LIBRARY_FROM_SOURCE", False) in ( - "True", - "true", -) - -# Error if a python version is missing -nox.options.error_on_missing_interpreters = True - -# -# Style Checks -# - - -def _determine_local_import_names(start_dir: str) -> List[str]: - """Determines all import names that should be considered "local". - - This is used when running the linter to insure that import order is - properly checked. - """ - file_ext_pairs = [os.path.splitext(path) for path in os.listdir(start_dir)] - return [ - basename - for basename, extension in file_ext_pairs - if extension == ".py" - or os.path.isdir(os.path.join(start_dir, basename)) - and basename not in ("__pycache__") - ] - - -# Linting with flake8. -# -# We ignore the following rules: -# E203: whitespace before ‘:’ -# E266: too many leading ‘#’ for block comment -# E501: line too long -# I202: Additional newline in a section of imports -# -# We also need to specify the rules which are ignored by default: -# ['E226', 'W504', 'E126', 'E123', 'W503', 'E24', 'E704', 'E121'] -FLAKE8_COMMON_ARGS = [ - "--show-source", - "--builtin=gettext", - "--max-complexity=20", - "--import-order-style=google", - "--exclude=.nox,.cache,env,lib,generated_pb2,*_pb2.py,*_pb2_grpc.py", - "--ignore=E121,E123,E126,E203,E226,E24,E266,E501,E704,W503,W504,I202", - "--max-line-length=88", -] - - -@nox.session -def lint(session: nox.sessions.Session) -> None: - if not TEST_CONFIG["enforce_type_hints"]: - session.install("flake8", "flake8-import-order") - else: - session.install("flake8", "flake8-import-order", "flake8-annotations") - - local_names = _determine_local_import_names(".") - args = FLAKE8_COMMON_ARGS + [ - "--application-import-names", - ",".join(local_names), - ".", - ] - session.run("flake8", *args) - - -# -# Black -# - - -@nox.session -def blacken(session: nox.sessions.Session) -> None: - session.install(BLACK_VERSION) - python_files = [path for path in os.listdir(".") if path.endswith(".py")] - - session.run("black", *python_files) - - -# -# Sample Tests -# - - -PYTEST_COMMON_ARGS = ["--junitxml=sponge_log.xml"] - - -def _session_tests( - session: nox.sessions.Session, post_install: Callable = None -) -> None: - # check for presence of tests - test_list = glob.glob("*_test.py") + glob.glob("test_*.py") - test_list.extend(glob.glob("tests")) - if len(test_list) == 0: - print("No tests found, skipping directory.") - else: - if TEST_CONFIG["pip_version_override"]: - pip_version = TEST_CONFIG["pip_version_override"] - session.install(f"pip=={pip_version}") - """Runs py.test for a particular project.""" - if os.path.exists("requirements.txt"): - if os.path.exists("constraints.txt"): - session.install("-r", "requirements.txt", "-c", "constraints.txt") - else: - session.install("-r", "requirements.txt") - - if os.path.exists("requirements-test.txt"): - if os.path.exists("constraints-test.txt"): - session.install( - "-r", "requirements-test.txt", "-c", "constraints-test.txt" - ) - else: - session.install("-r", "requirements-test.txt") - - if INSTALL_LIBRARY_FROM_SOURCE: - session.install("-e", _get_repo_root()) - - if post_install: - post_install(session) - - session.run( - "pytest", - *(PYTEST_COMMON_ARGS + session.posargs), - # Pytest will return 5 when no tests are collected. This can happen - # on travis where slow and flaky tests are excluded. - # See http://doc.pytest.org/en/latest/_modules/_pytest/main.html - success_codes=[0, 5], - env=get_pytest_env_vars(), - ) - - -@nox.session(python=ALL_VERSIONS) -def py(session: nox.sessions.Session) -> None: - """Runs py.test for a sample using the specified version of Python.""" - if session.python in TESTED_VERSIONS: - _session_tests(session) - else: - session.skip( - "SKIPPED: {} tests are disabled for this sample.".format(session.python) - ) - - -# -# Readmegen -# - - -def _get_repo_root() -> Optional[str]: - """ Returns the root folder of the project. """ - # Get root of this repository. Assume we don't have directories nested deeper than 10 items. - p = Path(os.getcwd()) - for i in range(10): - if p is None: - break - if Path(p / ".git").exists(): - return str(p) - # .git is not available in repos cloned via Cloud Build - # setup.py is always in the library's root, so use that instead - # https://github.com/googleapis/synthtool/issues/792 - if Path(p / "setup.py").exists(): - return str(p) - p = p.parent - raise Exception("Unable to detect repository root.") - - -GENERATED_READMES = sorted([x for x in Path(".").rglob("*.rst.in")]) - - -@nox.session -@nox.parametrize("path", GENERATED_READMES) -def readmegen(session: nox.sessions.Session, path: str) -> None: - """(Re-)generates the readme for a sample.""" - session.install("jinja2", "pyyaml") - dir_ = os.path.dirname(path) - - if os.path.exists(os.path.join(dir_, "requirements.txt")): - session.install("-r", os.path.join(dir_, "requirements.txt")) - - in_file = os.path.join(dir_, "README.rst.in") - session.run( - "python", _get_repo_root() + "/scripts/readme-gen/readme_gen.py", in_file - ) diff --git a/samples/interactive-tutorials/product/noxfile_config.py b/samples/interactive-tutorials/product/noxfile_config.py deleted file mode 100644 index edce74e3..00000000 --- a/samples/interactive-tutorials/product/noxfile_config.py +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Default TEST_CONFIG_OVERRIDE for python repos. - -# The source of truth: -# https://github.com/GoogleCloudPlatform/python-docs-samples/blob/master/noxfile_config.py - -TEST_CONFIG_OVERRIDE = { - # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7", "3.6"], - # An envvar key for determining the project id to use. Change it - # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a - # build specific Cloud project. You can also use your own string - # to use your own Cloud project. - "gcloud_project_env": "GOOGLE_CLOUD_PROJECT", - # 'gcloud_project_env': 'BUILD_SPECIFIC_GCLOUD_PROJECT', - # A dictionary you want to inject into your test. Don't put any - # secrets here. These values will override predefined values. - "envs": { - "BUCKET_NAME": "retail-interactive-tutorials", - }, -} diff --git a/samples/interactive-tutorials/product/requirements-test.txt b/samples/interactive-tutorials/product/requirements-test.txt deleted file mode 100644 index bbf73145..00000000 --- a/samples/interactive-tutorials/product/requirements-test.txt +++ /dev/null @@ -1,2 +0,0 @@ -pytest==6.2.5 -pytest-xdist==2.5.0 diff --git a/samples/interactive-tutorials/product/requirements.txt b/samples/interactive-tutorials/product/requirements.txt deleted file mode 100644 index 0ba6ea71..00000000 --- a/samples/interactive-tutorials/product/requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -google==3.0.0 -google-cloud-retail==1.1.0 -google-cloud-storage==1.43.0 -google-cloud-bigquery==2.30.1 \ No newline at end of file diff --git a/samples/interactive-tutorials/product/setup/products_create_bigquery_table.py b/samples/interactive-tutorials/product/setup/products_create_bigquery_table.py deleted file mode 100644 index ec14f16c..00000000 --- a/samples/interactive-tutorials/product/setup/products_create_bigquery_table.py +++ /dev/null @@ -1,31 +0,0 @@ -# Copyright 2021 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from setup_cleanup import create_bq_dataset, create_bq_table, \ - upload_data_to_bq_table - -dataset = "products" -valid_products_table = "products" -invalid_products_table = "products_some_invalid" -product_schema = "resources/product_schema.json" -valid_products_source_file = "resources/products.json" -invalid_products_source_file = "resources/products_some_invalid.json" - -create_bq_dataset(dataset) -create_bq_table(dataset, valid_products_table, product_schema) -upload_data_to_bq_table(dataset, valid_products_table, - valid_products_source_file, product_schema) -create_bq_table(dataset, invalid_products_table, product_schema) -upload_data_to_bq_table(dataset, invalid_products_table, - invalid_products_source_file, product_schema) diff --git a/samples/interactive-tutorials/product/setup/products_create_bigquery_table_test.py b/samples/interactive-tutorials/product/setup/products_create_bigquery_table_test.py deleted file mode 100644 index bc58e8bc..00000000 --- a/samples/interactive-tutorials/product/setup/products_create_bigquery_table_test.py +++ /dev/null @@ -1,43 +0,0 @@ -# Copyright 2021 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import re -import subprocess - - -def test_create_bigquery_table(): - output = str( - subprocess.check_output( - 'python setup/products_create_bigquery_table.py', - shell=True)) - assert re.match( - '.*Creating dataset products.*', output) - assert re.match( - '(.*dataset products already exists.*|.*dataset is created.*)', output) - assert re.match( - '.*Creating BigQuery table products.*', output) - assert re.match( - '(.*table products already exists.*|.*table is created.*)', output) - assert re.match( - '.*Uploading data form resources/products.json to the table products.products.*', output) - - assert re.match( - '.*Creating BigQuery table products_some_invalid.*', - output) - assert re.match( - '(.*table products_some_invalid already exists.*|.*table is created.*)', - output) - assert re.match( - '.*Uploading data form resources/products_some_invalid.json to the table products.products_some_invalid.*', - output) diff --git a/samples/interactive-tutorials/product/setup/products_create_gcs_bucket.py b/samples/interactive-tutorials/product/setup/products_create_gcs_bucket.py deleted file mode 100644 index e166066e..00000000 --- a/samples/interactive-tutorials/product/setup/products_create_gcs_bucket.py +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright 2021 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import datetime -import os - -from setup_cleanup import create_bucket, upload_blob - -project_id = os.environ["GOOGLE_CLOUD_PROJECT"] -timestamp_ = datetime.datetime.now().timestamp().__round__() -bucket_name = "{}_products_{}".format(project_id, timestamp_) - -create_bucket(bucket_name) -upload_blob(bucket_name, "resources/products.json") -upload_blob(bucket_name, "resources/products_some_invalid.json") - -print("\nThe gcs bucket {} was created".format(bucket_name)) diff --git a/samples/interactive-tutorials/product/setup/products_create_gcs_bucket_test.py b/samples/interactive-tutorials/product/setup/products_create_gcs_bucket_test.py deleted file mode 100644 index 19301b36..00000000 --- a/samples/interactive-tutorials/product/setup/products_create_gcs_bucket_test.py +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright 2021 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import re -import subprocess - -from products_delete_gcs_bucket import delete_bucket_by_name - - -def test_create_gcs_bucket(): - output = str( - subprocess.check_output( - 'python setup/products_create_gcs_bucket.py', - shell=True)) - - bucket_name = re.search('The gcs bucket (.+?) was created', output).group(1) - delete_bucket_by_name(bucket_name) - - print("bucket_name = {}".format(bucket_name)) - - assert re.match( - '.*Creating new bucket.*', output) - assert re.match( - '(.*The gcs bucket.*?was created.*|.*Bucket.*?already exists.*)', output) - assert re.match( - '.*Uploading data form resources/products.json to the bucket.*', output) - assert re.match( - '.*Uploading data form resources/products_some_invalid.json to the bucket.*', - output) diff --git a/samples/interactive-tutorials/product/setup/products_delete_gcs_bucket.py b/samples/interactive-tutorials/product/setup/products_delete_gcs_bucket.py deleted file mode 100644 index 010478cf..00000000 --- a/samples/interactive-tutorials/product/setup/products_delete_gcs_bucket.py +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright 2021 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os - -from setup_cleanup import delete_bucket - - -def delete_bucket_by_name(name: str): - if name is None: - bucket_name = os.environ["BUCKET_NAME"] - delete_bucket(bucket_name) - else: - delete_bucket(name) diff --git a/samples/interactive-tutorials/product/setup/setup_cleanup.py b/samples/interactive-tutorials/product/setup/setup_cleanup.py deleted file mode 100644 index 7fc2b95d..00000000 --- a/samples/interactive-tutorials/product/setup/setup_cleanup.py +++ /dev/null @@ -1,213 +0,0 @@ -# Copyright 2021 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import re -import shlex -import subprocess - -from google.api_core.exceptions import NotFound - -from google.cloud import storage -from google.cloud.retail_v2 import CreateProductRequest, DeleteProductRequest, \ - FulfillmentInfo, GetProductRequest, PriceInfo, Product, ProductServiceClient - -project_number = os.environ["GOOGLE_CLOUD_PROJECT_NUMBER"] -project_id = os.environ["GOOGLE_CLOUD_PROJECT"] -default_catalog = "projects/{0}/locations/global/catalogs/default_catalog".format( - project_number) -default_branch_name = "projects/" + project_number + "/locations/global/catalogs/default_catalog/branches/default_branch" - - -def generate_product() -> Product: - price_info = PriceInfo() - price_info.price = 30.0 - price_info.original_price = 35.5 - price_info.currency_code = "USD" - fulfillment_info = FulfillmentInfo() - fulfillment_info.type_ = "pickup-in-store" - fulfillment_info.place_ids = ["store0", "store1"] - return Product( - title='Nest Mini', - type_=Product.Type.PRIMARY, - categories=['Speakers and displays'], - brands=['Google'], - price_info=price_info, - fulfillment_info=[fulfillment_info], - availability="IN_STOCK", - ) - - -def create_product(product_id: str) -> object: - create_product_request = CreateProductRequest() - create_product_request.product = generate_product() - create_product_request.product_id = product_id - create_product_request.parent = default_branch_name - - created_product = ProductServiceClient().create_product( - create_product_request) - print("---product is created:---") - print(created_product) - - return created_product - - -def delete_product(product_name: str): - delete_product_request = DeleteProductRequest() - delete_product_request.name = product_name - ProductServiceClient().delete_product(delete_product_request) - - print("---product " + product_name + " was deleted:---") - - -def get_product(product_name: str): - get_product_request = GetProductRequest() - get_product_request.name = product_name - try: - product = ProductServiceClient().get_product(get_product_request) - print("---get product response:---") - print(product) - return product - except NotFound as e: - print(e.message) - return e.message - - -def try_to_delete_product_if_exists(product_name: str): - get_product_request = GetProductRequest() - get_product_request.name = product_name - delete_product_request = DeleteProductRequest() - delete_product_request.name = product_name - print( - "---delete product from the catalog, if the product already exists---") - try: - product = ProductServiceClient().get_product(get_product_request) - ProductServiceClient().delete_product(product.name) - except NotFound as e: - print(e.message) - - -def create_bucket(bucket_name: str): - """Create a new bucket in Cloud Storage""" - print("Creating new bucket:" + bucket_name) - buckets_in_your_project = str(list_buckets()) - if bucket_name in buckets_in_your_project: - print("Bucket {} already exists".format(bucket_name)) - else: - storage_client = storage.Client() - bucket = storage_client.bucket(bucket_name) - bucket.storage_class = "STANDARD" - new_bucket = storage_client.create_bucket(bucket, location="us") - print( - "Created bucket {} in {} with storage class {}".format( - new_bucket.name, new_bucket.location, new_bucket.storage_class - ) - ) - return new_bucket - - -def delete_bucket(bucket_name: str): - """Delete a bucket from Cloud Storage""" - storage_client = storage.Client() - print("Deleting bucket name:" + bucket_name) - buckets_in_your_project = str(list_buckets()) - if bucket_name in buckets_in_your_project: - blobs = storage_client.list_blobs(bucket_name) - for blob in blobs: - blob.delete() - bucket = storage_client.get_bucket(bucket_name) - bucket.delete() - print("Bucket {} is deleted".format(bucket.name)) - else: - print("Bucket {} is not found".format(bucket_name)) - - -def list_buckets(): - """Lists all buckets""" - bucket_list = [] - storage_client = storage.Client() - buckets = storage_client.list_buckets() - for bucket in buckets: - bucket_list.append(str(bucket)) - return bucket_list - - -def upload_blob(bucket_name, source_file_name): - """Uploads a file to the bucket.""" - # The path to your file to upload - # source_file_name = "local/path/to/file" - print("Uploading data form {} to the bucket {}".format(source_file_name, - bucket_name)) - storage_client = storage.Client() - bucket = storage_client.bucket(bucket_name) - object_name = re.search('resources/(.*?)$', source_file_name).group(1) - blob = bucket.blob(object_name) - blob.upload_from_filename(source_file_name) - - print( - "File {} uploaded to {}.".format( - source_file_name, object_name - ) - ) - - -def create_bq_dataset(dataset_name): - """Create a BigQuery dataset""" - print("Creating dataset {}".format(dataset_name)) - if dataset_name not in list_bq_datasets(): - create_dataset_command = 'bq --location=US mk -d --default_table_expiration 3600 --description "This is my dataset." {}:{}'.format( - project_id, dataset_name) - subprocess.check_output(shlex.split(create_dataset_command)) - print("dataset is created") - else: - print("dataset {} already exists".format(dataset_name)) - - -def list_bq_datasets(): - """List BigQuery datasets in the project""" - list_dataset_command = "bq ls --project_id {}".format(project_id) - datasets = subprocess.check_output(shlex.split(list_dataset_command)) - return str(datasets) - - -def create_bq_table(dataset, table_name, schema): - """Create a BigQuery table""" - print("Creating BigQuery table {}".format(table_name)) - if table_name not in list_bq_tables(dataset): - create_table_command = "bq mk --table {}:{}.{} {}".format( - project_id, - dataset, - table_name, schema) - output = subprocess.check_output(shlex.split(create_table_command)) - print(output) - print("table is created") - else: - print("table {} already exists".format(table_name)) - - -def list_bq_tables(dataset): - """List BigQuery tables in the dataset""" - list_tables_command = "bq ls {}:{}".format(project_id, dataset) - tables = subprocess.check_output(shlex.split(list_tables_command)) - return str(tables) - - -def upload_data_to_bq_table(dataset, table_name, source, schema): - """Upload data to the table from specified source file""" - print("Uploading data form {} to the table {}.{}".format(source, dataset, - table_name)) - upload_data_command = "bq load --source_format=NEWLINE_DELIMITED_JSON {}:{}.{} {} {}".format( - project_id, dataset, table_name, source, schema) - output = subprocess.check_output(shlex.split(upload_data_command)) - print(output) diff --git a/samples/interactive-tutorials/resources/events_schema.json b/samples/interactive-tutorials/resources/events_schema.json new file mode 100644 index 00000000..a52c0e56 --- /dev/null +++ b/samples/interactive-tutorials/resources/events_schema.json @@ -0,0 +1,73 @@ +[ + { + "fields":[ + { + "mode": "NULLABLE", + "name": "currencyCode", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "revenue", + "type": "FLOAT" + } + ], + "mode": "NULLABLE", + "name": "purchaseTransaction", + "type": "RECORD" + }, + { + "fields":[ + { + "mode": "NULLABLE", + "name": "quantity", + "type": "INTEGER" + }, + { + "fields":[ + { + "mode": "NULLABLE", + "name": "id", + "type": "STRING" + } + ], + "mode": "NULLABLE", + "name": "product", + "type": "RECORD" + } + ], + "mode": "REPEATED", + "name": "productDetails", + "type": "RECORD" + }, + { + "mode": "REQUIRED", + "name": "eventTime", + "type": "STRING" + }, + { + "mode": "REQUIRED", + "name": "visitorId", + "type": "STRING" + }, + { + "mode": "REQUIRED", + "name": "eventType", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "searchQuery", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "cartId", + "type": "STRING" + }, + { + "mode": "REPEATED", + "name": "pageCategories", + "type": "STRING" + } + ] \ No newline at end of file diff --git a/samples/interactive-tutorials/resources/product_schema.json b/samples/interactive-tutorials/resources/product_schema.json new file mode 100644 index 00000000..2dcc79f7 --- /dev/null +++ b/samples/interactive-tutorials/resources/product_schema.json @@ -0,0 +1,317 @@ +[ + { + "name": "name", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "id", + "type": "STRING", + "mode": "REQUIRED" + }, + { + "name": "type", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "primaryProductId", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "collectionMemberIds", + "type": "STRING", + "mode": "REPEATED" + }, + { + "name": "gtin", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "categories", + "type": "STRING", + "mode": "REPEATED" + }, + { + "name": "title", + "type": "STRING", + "mode": "REQUIRED" + }, + { + "name": "brands", + "type": "STRING", + "mode": "REPEATED" + }, + { + "name": "description", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "languageCode", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "attributes", + "type": "RECORD", + "mode": "REPEATED", + "fields": [ + { + "name": "key", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "value", + "type": "RECORD", + "mode": "NULLABLE", + "fields": [ + { + "name": "text", + "type": "STRING", + "mode": "REPEATED" + }, + { + "name": "numbers", + "type": "FLOAT", + "mode": "REPEATED" + }, + { + "name": "searchable", + "type": "BOOLEAN", + "mode": "NULLABLE" + }, + { + "name": "indexable", + "type": "BOOLEAN", + "mode": "NULLABLE" + } + ] + } + ] + }, + { + "name": "tags", + "type": "STRING", + "mode": "REPEATED" + }, + { + "name": "priceInfo", + "type": "RECORD", + "mode": "NULLABLE", + "fields": [ + { + "name": "currencyCode", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "price", + "type": "FLOAT", + "mode": "NULLABLE" + }, + { + "name": "originalPrice", + "type": "FLOAT", + "mode": "NULLABLE" + }, + { + "name": "cost", + "type": "FLOAT", + "mode": "NULLABLE" + }, + { + "name": "priceEffectiveTime", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "priceExpireTime", + "type": "STRING", + "mode": "NULLABLE" + } + ] + }, + { + "name": "rating", + "type": "RECORD", + "mode": "NULLABLE", + "fields": [ + { + "name": "ratingCount", + "type": "INTEGER", + "mode": "NULLABLE" + }, + { + "name": "averageRating", + "type": "FLOAT", + "mode": "NULLABLE" + }, + { + "name": "ratingHistogram", + "type": "INTEGER", + "mode": "REPEATED" + } + ] + }, + { + "name": "expireTime", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "ttl", + "type": "RECORD", + "mode": "NULLABLE", + "fields": [ + { + "name": "seconds", + "type": "INTEGER", + "mode": "NULLABLE" + }, + { + "name": "nanos", + "type": "INTEGER", + "mode": "NULLABLE" + } + ] + }, + { + "name": "availableTime", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "availability", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "availableQuantity", + "type": "INTEGER", + "mode": "NULLABLE" + }, + { + "name": "fulfillmentInfo", + "type": "RECORD", + "mode": "REPEATED", + "fields": [ + { + "name": "type", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "placeIds", + "type": "STRING", + "mode": "REPEATED" + } + ] + }, + { + "name": "uri", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "images", + "type": "RECORD", + "mode": "REPEATED", + "fields": [ + { + "name": "uri", + "type": "STRING", + "mode": "REQUIRED" + }, + { + "name": "height", + "type": "INTEGER", + "mode": "NULLABLE" + }, + { + "name": "width", + "type": "INTEGER", + "mode": "NULLABLE" + } + ] + }, + { + "name": "audience", + "type": "RECORD", + "mode": "NULLABLE", + "fields": [ + { + "name": "genders", + "type": "STRING", + "mode": "REPEATED" + }, + { + "name": "ageGroups", + "type": "STRING", + "mode": "REPEATED" + } + ] + }, + { + "name": "colorInfo", + "type": "RECORD", + "mode": "NULLABLE", + "fields": [ + { + "name": "colorFamilies", + "type": "STRING", + "mode": "REPEATED" + }, + { + "name": "colors", + "type": "STRING", + "mode": "REPEATED" + } + ] + }, + { + "name": "sizes", + "type": "STRING", + "mode": "REPEATED" + }, + { + "name": "materials", + "type": "STRING", + "mode": "REPEATED" + }, + { + "name": "patterns", + "type": "STRING", + "mode": "REPEATED" + }, + { + "name": "conditions", + "type": "STRING", + "mode": "REPEATED" + }, + { + "name": "retrievableFields", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "publishTime", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "promotions", + "type": "RECORD", + "mode": "REPEATED", + "fields": [ + { + "name": "promotionId", + "type": "STRING", + "mode": "NULLABLE" + } + ] + } +] \ No newline at end of file diff --git a/samples/snippets/resources/products.json b/samples/interactive-tutorials/resources/products.json similarity index 100% rename from samples/snippets/resources/products.json rename to samples/interactive-tutorials/resources/products.json diff --git a/samples/interactive-tutorials/resources/products_some_invalid.json b/samples/interactive-tutorials/resources/products_some_invalid.json new file mode 100644 index 00000000..f46dc761 --- /dev/null +++ b/samples/interactive-tutorials/resources/products_some_invalid.json @@ -0,0 +1,3 @@ +{"id": "GGCOGOAC101259","name": "GGCOGOAC101259","title": "#IamRemarkable Pen","brands": ["#IamRemarkable"],"categories": ["Office"],"priceInfo": {"cost": "12.0","currencyCode": "USD","originalPrice": "45.0","priceEffectiveTime": "2020-08-01T12:00:00+00:00","priceExpireTime": "2120-08-01T12:00:00+00:00","price": "16"},"colorInfo": {"colorFamilies": ["Blue"],"colors": ["Light blue","Blue","Dark blue"]},"availability": "INVALID_VALUE","availableQuantity": 50,"availableTime": "2021-10-11T12:00:00+00:00","images": [{"height": "300","width": "400","uri": "https://shop.googlemerchandisestore.com/store/20160512512/assets/items/images/GGCOGOAC101259.jpg"}],"retrievableFields": "name,title,brands,categories,priceInfo,colorInfo,availability,images,attributes.material,attributes.ecofriendly,attributes.style,attributes.collection,uri","attributes":[ {"key":"material", "value": {"indexable": "true","searchable": "true","text": ["Metal","Recycled Plastic"]}}],"uri": "https://shop.googlemerchandisestore.com/Google+Redesign/Office/IamRemarkable+Pen"} +{"id": "GGPRAHPL107110","name": "GGPRAHPL107110","title": "Android Iconic Hat Green","brands": ["Android"],"categories": ["Apparel"],"priceInfo": {"cost": "12.0","currencyCode": "USD","originalPrice": "45.0","priceEffectiveTime": "2020-08-01T12:00:00+00:00","priceExpireTime": "2120-08-01T12:00:00+00:00","price": "16"},"colorInfo": {"colorFamilies": ["Green"],"colors": ["Olive","Grass green","Light green"]},"availability": "IN_STOCK","availableQuantity": 50,"availableTime": "2021-10-11T12:00:00+00:00","images": [{"height": "300","width": "400","uri": "https://shop.googlemerchandisestore.com/store/20160512512/assets/items/images/GGOEAHPL130910.jpg"}],"retrievableFields": "name,title,brands,categories,priceInfo,colorInfo,availability,images,attributes.material,attributes.ecofriendly,attributes.style,attributes.collection,uri","uri": "https://shop.googlemerchandisestore.com/Google+Prize+Portal/Android+Iconic+Hat+Green"} +{"id": "GGOEAAKQ137410","name": "GGOEAAKQ137410","title": "Android Iconic Sock","brands": ["Android"],"categories": ["Apparel"],"priceInfo": {"cost": "12.0","currencyCode": "USD","originalPrice": "45.0","priceEffectiveTime": "2020-08-01T12:00:00+00:00","priceExpireTime": "2120-08-01T12:00:00+00:00","price": "17"},"availability": "IN_STOCK","availableQuantity": 50,"availableTime": "2021-10-11T12:00:00+00:00","images": [{"height": "300","width": "400","uri": "https://shop.googlemerchandisestore.com/store/20160512512/assets/items/images/GGOEAAKQ137410.jpg"}],"sizes": ["XS","S","M","L","XL"],"retrievableFields": "name,title,brands,categories,priceInfo,colorInfo,availability,images,attributes.material,attributes.ecofriendly,attributes.style,attributes.collection,uri","attributes":[ {"key":"collection", "value": {"indexable": "true","numbers": [2022.0]}},{"key":"material", "value": {"indexable": "true","searchable": "true","text": ["Polyester","Membrane"]}},{"key":"ecofriendly", "value": {"indexable": "false","searchable": "false","text": ["Low-impact fabrics","recycled fabrics","recycled packaging","plastic-free packaging","ethically made"]}},{"key":"style", "value": {"indexable": "true","searchable": "true","text": ["Sport","Functional"]}}],"uri": "https://shop.googlemerchandisestore.com/Google+Redesign/Apparel/Android+Iconic+Sock"} \ No newline at end of file diff --git a/samples/interactive-tutorials/resources/user_events.json b/samples/interactive-tutorials/resources/user_events.json new file mode 100644 index 00000000..5360c309 --- /dev/null +++ b/samples/interactive-tutorials/resources/user_events.json @@ -0,0 +1,4 @@ +{"eventType":"home-page-view","visitorId":"bjbs_group1_visitor1","eventTime":"2021-12-12T10:27:42+00:00"} +{"eventType":"search","visitorId":"bjbs_group1_visitor1","eventTime":"2021-12-12T10:27:42+00:00","searchQuery":"RockerJeans teenagers blue jeans"} +{"eventType":"search","visitorId":"bjbs_group1_visitor1","eventTime":"2021-12-12T10:27:42+00:00","searchQuery":"SocksUnlimited teenagers black socks"} +{"eventType":"detail-page-view","visitorId":"bjbs_group1_visitor1","eventTime":"2021-12-12T10:27:42+00:00","productDetails":{"product":{"id":"GGCOGAEC100616"},"quantity":3}} diff --git a/samples/interactive-tutorials/resources/user_events_some_invalid.json b/samples/interactive-tutorials/resources/user_events_some_invalid.json new file mode 100644 index 00000000..c98b1699 --- /dev/null +++ b/samples/interactive-tutorials/resources/user_events_some_invalid.json @@ -0,0 +1,4 @@ +{"eventType":"home-page-view","visitorId":"bjbs_group1_visitor1","eventTime":"2021-12-12T10:27:42+00:00"} +{"eventType":"invalid","visitorId":"bjbs_group1_visitor1","eventTime":"2021-12-12T10:27:42+00:00","searchQuery":"RockerJeans teenagers blue jeans"} +{"eventType":"search","visitorId":"bjbs_group1_visitor1","eventTime":"2021-12-12T10:27:42+00:00","searchQuery":"SocksUnlimited teenagers black socks"} +{"eventType":"detail-page-view","visitorId":"bjbs_group1_visitor1","eventTime":"2021-12-12T10:27:42+00:00","productDetails":{"product":{"id":"GGCOGAEC100616"},"quantity":3}} diff --git a/samples/interactive-tutorials/search/noxfile_config.py b/samples/interactive-tutorials/search/noxfile_config.py index 401ff7f2..8eb8d16f 100644 --- a/samples/interactive-tutorials/search/noxfile_config.py +++ b/samples/interactive-tutorials/search/noxfile_config.py @@ -28,5 +28,7 @@ # 'gcloud_project_env': 'BUILD_SPECIFIC_GCLOUD_PROJECT', # A dictionary you want to inject into your test. Don't put any # secrets here. These values will override predefined values. - "envs": {"BUCKET_NAME": "retail-interactive-tutorials",}, + "envs": { + "BUCKET_NAME": "retail-interactive-tutorials", + }, } diff --git a/samples/interactive-tutorials/search/search_simple_query_test.py b/samples/interactive-tutorials/search/search_simple_query_test.py index fc6d0c69..32749e2a 100644 --- a/samples/interactive-tutorials/search/search_simple_query_test.py +++ b/samples/interactive-tutorials/search/search_simple_query_test.py @@ -21,7 +21,9 @@ def test_search_simple_query_pass(): - output = str(subprocess.check_output("python search_simple_query.py", shell=True)) + output = str( + subprocess.check_output("python search_simple_query.py", shell=True) + ) assert re.match(".*search request.*", output) assert re.match(".*search response.*", output) diff --git a/samples/interactive-tutorials/search/search_with_filtering_test.py b/samples/interactive-tutorials/search/search_with_filtering_test.py index 5b10ae34..270ff193 100644 --- a/samples/interactive-tutorials/search/search_with_filtering_test.py +++ b/samples/interactive-tutorials/search/search_with_filtering_test.py @@ -21,7 +21,9 @@ def test_search_with_filtering_pass(): - output = str(subprocess.check_output("python search_with_filtering.py", shell=True)) + output = str( + subprocess.check_output("python search_with_filtering.py", shell=True) + ) assert re.match(".*search request.*", output) assert re.match(".*search response.*", output) diff --git a/samples/interactive-tutorials/search/search_with_ordering_test.py b/samples/interactive-tutorials/search/search_with_ordering_test.py index ec8fcfb9..0de2c979 100644 --- a/samples/interactive-tutorials/search/search_with_ordering_test.py +++ b/samples/interactive-tutorials/search/search_with_ordering_test.py @@ -21,7 +21,9 @@ def test_search_with_ordering_pass(): - output = str(subprocess.check_output("python search_with_ordering.py", shell=True)) + output = str( + subprocess.check_output("python search_with_ordering.py", shell=True) + ) assert re.match(".*search request.*", output) assert re.match(".*search response.*", output) diff --git a/samples/snippets/create_test_resources.py b/samples/interactive-tutorials/test_resources_recovery/create_test_resources.py similarity index 53% rename from samples/snippets/create_test_resources.py rename to samples/interactive-tutorials/test_resources_recovery/create_test_resources.py index ef07e133..5fbcca0d 100644 --- a/samples/snippets/create_test_resources.py +++ b/samples/interactive-tutorials/test_resources_recovery/create_test_resources.py @@ -14,6 +14,8 @@ import os import re +import shlex +import subprocess import time from google.cloud.storage.bucket import Bucket @@ -25,12 +27,23 @@ project_number = os.environ["GOOGLE_CLOUD_PROJECT_NUMBER"] bucket_name = os.environ['BUCKET_NAME'] -storage_client = storage.Client() -resource_file = "resources/products.json" -object_name = re.search('resources/(.*?)$', resource_file).group(1) +project_id = os.environ["GOOGLE_CLOUD_PROJECT_ID"] + +product_resource_file = "../resources/products.json" +events_source_file = "../resources/user_events.json" + +product_dataset = "products" +product_table = "products" +product_schema = "resources/product_schema.json" +events_dataset = "user_events" +events_table = "events" +events_schema = "resources/events_schema.json" + +object_name = re.search('resources/(.*?)$', product_resource_file).group(1) default_catalog = "projects/{0}/locations/global/catalogs/default_catalog/branches/default_branch".format( project_number) +storage_client = storage.Client() def create_bucket(bucket_name: str) -> Bucket: """Create a new bucket in Cloud Storage""" @@ -65,8 +78,8 @@ def check_if_bucket_exists(new_bucket_name): def upload_data_to_bucket(bucket: Bucket): """Upload data to a GCS bucket""" blob = bucket.blob(object_name) - blob.upload_from_filename(resource_file) - print("Data from {} has being uploaded to {}".format(resource_file, + blob.upload_from_filename(product_resource_file) + print("Data from {} has being uploaded to {}".format(product_resource_file, bucket.name)) @@ -121,7 +134,78 @@ def import_products_from_gcs(): "Wait 2 -5 minutes till products become indexed in the catalog,\ after that they will be available for search") +def create_bq_dataset(dataset_name): + """Create a BigQuery dataset""" + print("Creating dataset {}".format(dataset_name)) + if dataset_name not in list_bq_datasets(): + create_dataset_command = 'bq --location=US mk -d --default_table_expiration 3600 --description "This is my dataset." {}:{}'.format( + project_id, dataset_name) + output = subprocess.check_output(shlex.split(create_dataset_command)) + print(output) + print("dataset is created") + else: + print("dataset {} already exists".format(dataset_name)) + + +def list_bq_datasets(): + """List BigQuery datasets in the project""" + list_dataset_command = "bq ls --project_id {}".format(project_id) + list_output = subprocess.check_output(shlex.split(list_dataset_command)) + datasets = re.split(r'\W+', str(list_output)) + return datasets + + +def create_bq_table(dataset, table_name, schema): + """Create a BigQuery table""" + print("Creating BigQuery table {}".format(table_name)) + if table_name not in list_bq_tables(dataset): + create_table_command = "bq mk --table {}:{}.{} {}".format( + project_id, + dataset, + table_name, schema) + output = subprocess.check_output(shlex.split(create_table_command)) + print(output) + print("table is created") + else: + print("table {} already exists".format(table_name)) + -created_bucket = create_bucket(bucket_name) -upload_data_to_bucket(created_bucket) +def list_bq_tables(dataset): + """List BigQuery tables in the dataset""" + list_tables_command = "bq ls {}:{}".format(project_id, dataset) + tables = subprocess.check_output(shlex.split(list_tables_command)) + return str(tables) + + +def upload_data_to_bq_table(dataset, table_name, source, schema): + """Upload data to the table from specified source file""" + print("Uploading data form {} to the table {}.{}".format(source, dataset, + table_name)) + upload_data_command = "bq load --source_format=NEWLINE_DELIMITED_JSON {}:{}.{} {} {}".format( + project_id, dataset, table_name, source, schema) + output = subprocess.check_output(shlex.split(upload_data_command)) + print(output) + + +# Create a GCS bucket with products.json file +created_products_bucket = create_bucket(bucket_name) +upload_data_to_bucket(created_products_bucket) + +# Create a GCS bucket with user_events.json file +created_events_bucket = create_bucket(bucket_name) +upload_data_to_bucket(created_events_bucket) + +# Import prodcuts from the GCS bucket to the Retail catalog import_products_from_gcs() + +# Create a BigQuery table with products +create_bq_dataset(product_dataset) +create_bq_table(product_dataset, product_table, product_schema) +upload_data_to_bq_table(product_dataset, product_table, + product_resource_file, product_schema) + +# Create a BigQuery table with user events +create_bq_dataset(events_dataset) +create_bq_table(events_dataset, events_table, events_schema) +upload_data_to_bq_table(events_dataset, events_table, events_source_file, + events_schema) diff --git a/samples/snippets/remove_test_resources.py b/samples/interactive-tutorials/test_resources_recovery/remove_test_resources.py similarity index 76% rename from samples/snippets/remove_test_resources.py rename to samples/interactive-tutorials/test_resources_recovery/remove_test_resources.py index c6faf299..41981ba2 100644 --- a/samples/snippets/remove_test_resources.py +++ b/samples/interactive-tutorials/test_resources_recovery/remove_test_resources.py @@ -13,6 +13,8 @@ # limitations under the License. import os +import shlex +import subprocess from google.api_core.exceptions import PermissionDenied from google.cloud.storage.bucket import Bucket @@ -22,7 +24,12 @@ ProductServiceClient project_number = os.environ["GOOGLE_CLOUD_PROJECT_NUMBER"] -bucket_name = os.environ['BUCKET_NAME'] +product_bucket_name = os.environ['BUCKET_NAME'] +events_bucket_name = os.environ['EVENTS_BUCKET_NAME'] +project_id = os.environ["GOOGLE_CLOUD_PROJECT_ID"] + +product_dataset = "products" +events_dataset = "user_events" default_catalog = "projects/{0}/locations/global/catalogs/default_catalog/branches/default_branch".format( project_number) @@ -30,7 +37,7 @@ storage_client = storage.Client() -def delete_bucket(): +def delete_bucket(bucket_name): """Delete bucket""" try: bucket = storage_client.get_bucket(bucket_name) @@ -52,6 +59,7 @@ def delete_object_from_bucket(bucket: Bucket): def delete_all_products(): """Delete all products in the catalog""" + print("Deleting all products, please wait") product_client = ProductServiceClient() list_request = ListProductsRequest() list_request.parent = default_catalog @@ -69,5 +77,14 @@ def delete_all_products(): print(f"{delete_count} products were deleted from {default_catalog}") -delete_bucket() +def delete_bq_dataset_with_tables(dataset): + """Delete a BigQuery dataset with all tables""" + delete_dataset_command = "bq rm -r -d -f {}".format(dataset) + output = subprocess.check_output(shlex.split(delete_dataset_command)) + print(output) + +delete_bucket(product_bucket_name) +delete_bucket(events_bucket_name) delete_all_products() +delete_bq_dataset_with_tables(product_dataset) +delete_bq_dataset_with_tables(events_dataset) From 84b997c5904e222a963a8b9a17f04d3181a4a47e Mon Sep 17 00:00:00 2001 From: t-karasova <91195610+t-karasova@users.noreply.github.com> Date: Thu, 3 Feb 2022 14:58:00 +0100 Subject: [PATCH 09/12] Update create_test_resources.py --- .../test_resources_recovery/create_test_resources.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/samples/interactive-tutorials/test_resources_recovery/create_test_resources.py b/samples/interactive-tutorials/test_resources_recovery/create_test_resources.py index 5fbcca0d..98ac5fe1 100644 --- a/samples/interactive-tutorials/test_resources_recovery/create_test_resources.py +++ b/samples/interactive-tutorials/test_resources_recovery/create_test_resources.py @@ -26,7 +26,8 @@ from google.cloud.retail_v2 import ProductServiceClient project_number = os.environ["GOOGLE_CLOUD_PROJECT_NUMBER"] -bucket_name = os.environ['BUCKET_NAME'] +products_bucket_name = os.environ['BUCKET_NAME'] +events_bucket_name = os.environ['EVENTS_BUCKET_NAME'] project_id = os.environ["GOOGLE_CLOUD_PROJECT_ID"] product_resource_file = "../resources/products.json" @@ -188,11 +189,11 @@ def upload_data_to_bq_table(dataset, table_name, source, schema): # Create a GCS bucket with products.json file -created_products_bucket = create_bucket(bucket_name) +created_products_bucket = create_bucket(products_bucket_name) upload_data_to_bucket(created_products_bucket) # Create a GCS bucket with user_events.json file -created_events_bucket = create_bucket(bucket_name) +created_events_bucket = create_bucket(events_bucket_name) upload_data_to_bucket(created_events_bucket) # Import prodcuts from the GCS bucket to the Retail catalog From ca1f42985f8e36aca543eedb6b2b2ec5761f665e Mon Sep 17 00:00:00 2001 From: t-karasova <91195610+t-karasova@users.noreply.github.com> Date: Thu, 3 Feb 2022 16:03:55 +0100 Subject: [PATCH 10/12] Update create_test_resources.py --- .../test_resources_recovery/create_test_resources.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/interactive-tutorials/test_resources_recovery/create_test_resources.py b/samples/interactive-tutorials/test_resources_recovery/create_test_resources.py index 98ac5fe1..4fa080b3 100644 --- a/samples/interactive-tutorials/test_resources_recovery/create_test_resources.py +++ b/samples/interactive-tutorials/test_resources_recovery/create_test_resources.py @@ -86,7 +86,7 @@ def upload_data_to_bucket(bucket: Bucket): def get_import_products_gcs_request(): """Get import products from gcs request""" - gcs_bucket = "gs://{}".format(bucket_name) + gcs_bucket = "gs://{}".format(products_bucket_name) gcs_errors_bucket = "{}/error".format(gcs_bucket) gcs_source = GcsSource() From c01cfd6a679812d22a362f804ced86df7dd2a757 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Tue, 8 Feb 2022 05:46:43 -0500 Subject: [PATCH 11/12] fix typo --- samples/interactive-tutorials/TEST_RESOURCES_SETUP_CLEANUP.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/interactive-tutorials/TEST_RESOURCES_SETUP_CLEANUP.md b/samples/interactive-tutorials/TEST_RESOURCES_SETUP_CLEANUP.md index 5974753e..f7989d00 100644 --- a/samples/interactive-tutorials/TEST_RESOURCES_SETUP_CLEANUP.md +++ b/samples/interactive-tutorials/TEST_RESOURCES_SETUP_CLEANUP.md @@ -14,7 +14,7 @@ The Secret Manager name is set in .kokoro/presubmit/common.cfg file, SECRET_MANA ## Import catalog data -There are a JSON files with valid products and user events prepared in `resources` directory: +There are JSON files with valid products and user events prepared in `resources` directory: `samples/resources/products.json` and `samples/resources/user_events.json` respectively. Run the `create_test_resources.py` to perform the following actions: From d7268712afb5beb20c214f33646d6d3ffe708661 Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Tue, 8 Feb 2022 10:48:11 +0000 Subject: [PATCH 12/12] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- samples/interactive-tutorials/search/noxfile_config.py | 4 +--- .../interactive-tutorials/search/search_simple_query_test.py | 4 +--- .../search/search_with_filtering_test.py | 4 +--- .../interactive-tutorials/search/search_with_ordering_test.py | 4 +--- 4 files changed, 4 insertions(+), 12 deletions(-) diff --git a/samples/interactive-tutorials/search/noxfile_config.py b/samples/interactive-tutorials/search/noxfile_config.py index 8eb8d16f..401ff7f2 100644 --- a/samples/interactive-tutorials/search/noxfile_config.py +++ b/samples/interactive-tutorials/search/noxfile_config.py @@ -28,7 +28,5 @@ # 'gcloud_project_env': 'BUILD_SPECIFIC_GCLOUD_PROJECT', # A dictionary you want to inject into your test. Don't put any # secrets here. These values will override predefined values. - "envs": { - "BUCKET_NAME": "retail-interactive-tutorials", - }, + "envs": {"BUCKET_NAME": "retail-interactive-tutorials",}, } diff --git a/samples/interactive-tutorials/search/search_simple_query_test.py b/samples/interactive-tutorials/search/search_simple_query_test.py index 32749e2a..fc6d0c69 100644 --- a/samples/interactive-tutorials/search/search_simple_query_test.py +++ b/samples/interactive-tutorials/search/search_simple_query_test.py @@ -21,9 +21,7 @@ def test_search_simple_query_pass(): - output = str( - subprocess.check_output("python search_simple_query.py", shell=True) - ) + output = str(subprocess.check_output("python search_simple_query.py", shell=True)) assert re.match(".*search request.*", output) assert re.match(".*search response.*", output) diff --git a/samples/interactive-tutorials/search/search_with_filtering_test.py b/samples/interactive-tutorials/search/search_with_filtering_test.py index 270ff193..5b10ae34 100644 --- a/samples/interactive-tutorials/search/search_with_filtering_test.py +++ b/samples/interactive-tutorials/search/search_with_filtering_test.py @@ -21,9 +21,7 @@ def test_search_with_filtering_pass(): - output = str( - subprocess.check_output("python search_with_filtering.py", shell=True) - ) + output = str(subprocess.check_output("python search_with_filtering.py", shell=True)) assert re.match(".*search request.*", output) assert re.match(".*search response.*", output) diff --git a/samples/interactive-tutorials/search/search_with_ordering_test.py b/samples/interactive-tutorials/search/search_with_ordering_test.py index 0de2c979..ec8fcfb9 100644 --- a/samples/interactive-tutorials/search/search_with_ordering_test.py +++ b/samples/interactive-tutorials/search/search_with_ordering_test.py @@ -21,9 +21,7 @@ def test_search_with_ordering_pass(): - output = str( - subprocess.check_output("python search_with_ordering.py", shell=True) - ) + output = str(subprocess.check_output("python search_with_ordering.py", shell=True)) assert re.match(".*search request.*", output) assert re.match(".*search response.*", output)