diff --git a/samples/snippets/TEST_RESOURCES_SETUP_CLEANUP.md b/samples/interactive-tutorials/TEST_RESOURCES_SETUP_CLEANUP.md similarity index 51% rename from samples/snippets/TEST_RESOURCES_SETUP_CLEANUP.md rename to samples/interactive-tutorials/TEST_RESOURCES_SETUP_CLEANUP.md index 000f12d0..5974753e 100644 --- a/samples/snippets/TEST_RESOURCES_SETUP_CLEANUP.md +++ b/samples/interactive-tutorials/TEST_RESOURCES_SETUP_CLEANUP.md @@ -3,8 +3,10 @@ ## Required environment variables To successfully import the catalog data for tests, the following environment variables should be set: - - PROJECT_NUMBER + - GOOGLE_CLOUD_PROJECT_NUMBER + - GOOGLE_CLOUD_PROJECT_ID - BUCKET_NAME + - EVENTS_BUCKET_NAME These values are stored in the Secret Manager and will be submitted as docker environment variables before the test run. @@ -12,13 +14,20 @@ The Secret Manager name is set in .kokoro/presubmit/common.cfg file, SECRET_MANA ## Import catalog data -There is a JSON file with valid products prepared in the `product` directory: -`resources/products.json`. +There are a JSON files with valid products and user events prepared in `resources` directory: +`samples/resources/products.json` and `samples/resources/user_events.json` respectively. Run the `create_test_resources.py` to perform the following actions: - create the GCS bucket , - - upload the product data from `resources/products.json` file, - - import products to the default branch of the Retail catalog. + - upload the product data from `resources/products.json` file to products bucket, + - import products to the default branch of the Retail catalog, + - create the GCS bucket , + - upload the product data from `resources/user_events.json` file to events bucket, + - create a BigQuery dataset and table `products`, + - insert products from resources/products.json to the created products table, + - create a BigQuery dataset and table `events`, + - insert user events from resources/user_events.json to the created events table + ``` $ python create_test_resources.py @@ -33,6 +42,10 @@ Run the `remove_test_resources.py` to perform the following actions: - remove all objects from the GCS bucket , - remove the bucket, - delete all products from the Retail catalog. + - remove all objects from the GCS bucket , + - remove the bucket, + - remove dataset `products` along with tables + - remove dataset `user_events` along with tables ``` $ python remove_test_resources.py diff --git a/samples/interactive-tutorials/product/import_products_big_query_table.py b/samples/interactive-tutorials/product/import_products_big_query_table.py deleted file mode 100644 index 934e259c..00000000 --- a/samples/interactive-tutorials/product/import_products_big_query_table.py +++ /dev/null @@ -1,103 +0,0 @@ -# Copyright 2021 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# [START retail_import_products_from_big_query] -# Import products into a catalog from big query table using Retail API -# -import os -import time - -from google.cloud.retail import ( - BigQuerySource, - ImportProductsRequest, - ProductInputConfig, - ProductServiceClient, -) - -project_number = os.environ["GOOGLE_CLOUD_PROJECT_NUMBER"] -project_id = os.environ["GOOGLE_CLOUD_PROJECT"] - -default_catalog = "projects/{0}/locations/global/catalogs/default_catalog/branches/default_branch".format( - project_number -) -dataset_id = "products" -table_id = "products" - - -# TO CHECK ERROR HANDLING USE THE TABLE WITH INVALID PRODUCTS: -# table_id = "products_some_invalid" - - -# get import products from big query request -def get_import_products_big_query_request(reconciliation_mode): - # TO CHECK ERROR HANDLING PASTE THE INVALID CATALOG NAME HERE: - # default_catalog = "invalid_catalog_name" - big_query_source = BigQuerySource() - big_query_source.project_id = project_id - big_query_source.dataset_id = dataset_id - big_query_source.table_id = table_id - big_query_source.data_schema = "product" - - input_config = ProductInputConfig() - input_config.big_query_source = big_query_source - - import_request = ImportProductsRequest() - import_request.parent = default_catalog - import_request.reconciliation_mode = reconciliation_mode - import_request.input_config = input_config - - print("---import products from big query table request---") - print(import_request) - - return import_request - - -# call the Retail API to import products -def import_products_from_big_query(): - # TRY THE FULL RECONCILIATION MODE HERE: - reconciliation_mode = ImportProductsRequest.ReconciliationMode.INCREMENTAL - - import_big_query_request = get_import_products_big_query_request( - reconciliation_mode - ) - big_query_operation = ProductServiceClient().import_products( - import_big_query_request - ) - - print("---the operation was started:----") - print(big_query_operation.operation.name) - - while not big_query_operation.done(): - print("---please wait till operation is done---") - time.sleep(5) - print("---import products operation is done---") - - if big_query_operation.metadata is not None: - print("---number of successfully imported products---") - print(big_query_operation.metadata.success_count) - print("---number of failures during the importing---") - print(big_query_operation.metadata.failure_count) - else: - print("---operation.metadata is empty---") - - if big_query_operation.result is not None: - print("---operation result:---") - print(big_query_operation.result()) - else: - print("---operation.result is empty---") - - -import_products_from_big_query() - -# [END retail_import_products_from_big_query] diff --git a/samples/interactive-tutorials/product/import_products_bq_test.py b/samples/interactive-tutorials/product/import_products_bq_test.py deleted file mode 100644 index bdc5687a..00000000 --- a/samples/interactive-tutorials/product/import_products_bq_test.py +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright 2021 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import re -import subprocess - - -def test_import_products_bq(): - output = str( - subprocess.check_output( - "python import_products_big_query_table.py", shell=True - ) - ) - - assert re.match(".*import products from big query table request.*", output) - assert re.match(".*the operation was started.*", output) - assert re.match( - ".*projects/.*/locations/global/catalogs/default_catalog/branches/0/operations/import-products.*", - output, - ) - - assert re.match(".*number of successfully imported products.*316.*", output) diff --git a/samples/interactive-tutorials/product/import_products_gcs.py b/samples/interactive-tutorials/product/import_products_gcs.py deleted file mode 100644 index a23272cc..00000000 --- a/samples/interactive-tutorials/product/import_products_gcs.py +++ /dev/null @@ -1,113 +0,0 @@ -# Copyright 2021 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# [START retail_import_products_from_gcs] -# Import products into a catalog from gcs using Retail API -# -import os -import time - -from google.cloud.retail import ( - GcsSource, - ImportErrorsConfig, - ImportProductsRequest, - ProductInputConfig, - ProductServiceClient, -) - - -# Read the project number from the environment variable -project_number = os.environ["GOOGLE_CLOUD_PROJECT_NUMBER"] -project_id = os.environ["GOOGLE_CLOUD_PROJECT"] -bucket_name = os.environ["BUCKET_NAME"] - -# You can change the branch here. The "default_branch" is set to point to the branch "0" -default_catalog = "projects/{0}/locations/global/catalogs/default_catalog/branches/default_branch".format( - project_number -) - -gcs_bucket = "gs://{}".format(bucket_name) -gcs_errors_bucket = "{}/error".format(gcs_bucket) -gcs_products_object = "products.json" - - -# TO CHECK ERROR HANDLING USE THE JSON WITH INVALID PRODUCT -# gcs_products_object = "products_some_invalid.json" - - -# get import products from gcs request -def get_import_products_gcs_request(gcs_object_name: str): - # TO CHECK ERROR HANDLING PASTE THE INVALID CATALOG NAME HERE: - # default_catalog = "invalid_catalog_name" - gcs_source = GcsSource() - gcs_source.input_uris = ["{0}/{1}".format(gcs_bucket, gcs_object_name)] - - input_config = ProductInputConfig() - input_config.gcs_source = gcs_source - print("GRS source:") - print(gcs_source.input_uris) - - errors_config = ImportErrorsConfig() - errors_config.gcs_prefix = gcs_errors_bucket - - import_request = ImportProductsRequest() - import_request.parent = default_catalog - import_request.reconciliation_mode = ( - ImportProductsRequest.ReconciliationMode.INCREMENTAL - ) - import_request.input_config = input_config - import_request.errors_config = errors_config - - print("---import products from google cloud source request---") - print(import_request) - - return import_request - - -# call the Retail API to import products -def import_products_from_gcs(): - import_gcs_request = get_import_products_gcs_request(gcs_products_object) - gcs_operation = ProductServiceClient().import_products(import_gcs_request) - - print("---the operation was started:----") - print(gcs_operation.operation.name) - - while not gcs_operation.done(): - print("---please wait till operation is done---") - time.sleep(5) - print("---import products operation is done---") - - if gcs_operation.metadata is not None: - print("---number of successfully imported products---") - print(gcs_operation.metadata.success_count) - print("---number of failures during the importing---") - print(gcs_operation.metadata.failure_count) - else: - print("---operation.metadata is empty---") - - if gcs_operation.result is not None: - print("---operation result:---") - print(gcs_operation.result()) - else: - print("---operation.result is empty---") - - # The imported products needs to be indexed in the catalog before they become available for search. - print( - "Wait 2 -5 minutes till products become indexed in the catalog, after that they will be available for search" - ) - - -import_products_from_gcs() - -# [END retail_import_products_from_gcs] diff --git a/samples/interactive-tutorials/product/import_products_gcs_test.py b/samples/interactive-tutorials/product/import_products_gcs_test.py deleted file mode 100644 index 4e0e023a..00000000 --- a/samples/interactive-tutorials/product/import_products_gcs_test.py +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright 2021 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import re -import subprocess - - -def test_import_products_gcs(): - output = str( - subprocess.check_output("python import_products_gcs.py", shell=True) - ) - - assert re.match(".*import products from google cloud source request.*", output) - assert re.match('.*input_uris: "gs://.*/products.json".*', output) - assert re.match(".*the operation was started.*", output) - assert re.match( - ".*projects/.*/locations/global/catalogs/default_catalog/branches/0/operations/import-products.*", - output, - ) - - assert re.match(".*number of successfully imported products.*316.*", output) diff --git a/samples/interactive-tutorials/product/import_products_inline_source.py b/samples/interactive-tutorials/product/import_products_inline_source.py deleted file mode 100644 index 2072e38e..00000000 --- a/samples/interactive-tutorials/product/import_products_inline_source.py +++ /dev/null @@ -1,156 +0,0 @@ -# Copyright 2021 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# [START retail_import_products_from_inline_source] -# Import products into a catalog from inline source using Retail API -# -import os -import random -import string -import time - -from google.cloud.retail import ( - ColorInfo, - FulfillmentInfo, - ImportProductsRequest, - PriceInfo, - Product, - ProductInlineSource, - ProductInputConfig, - ProductServiceClient, -) -from google.protobuf.field_mask_pb2 import FieldMask - -project_number = os.environ["GOOGLE_CLOUD_PROJECT_NUMBER"] - -default_catalog = "projects/{0}/locations/global/catalogs/default_catalog/branches/default_branch".format( - project_number -) - - -# prepare product to import as inline source -def get_products(): - products = [] - product1 = Product() - product2 = Product() - - price_info1 = PriceInfo() - price_info1.price = 16.0 - price_info1.original_price = 45.0 - price_info1.cost = 12.0 - price_info1.currency_code = "USD" - - color_info1 = ColorInfo() - color_info1.color_families = ["Blue"] - color_info1.colors = ["Light blue", "Blue", "Dark blue"] - - fulfillment_info1 = FulfillmentInfo() - fulfillment_info1.type_ = "pickup-in-store" - fulfillment_info1.place_ids = ["store1", "store2"] - - field_mask1 = FieldMask(paths=["title", "categories", "price_info", "color_info"]) - - # TO CHECK ERROR HANDLING COMMENT OUT THE PRODUCT TITLE HERE: - product1.title = "#IamRemarkable Pen" - product1.id = "".join(random.sample(string.ascii_lowercase, 8)) - product1.categories = ["Office"] - product1.uri = "https://shop.googlemerchandisestore.com/Google+Redesign/Office/IamRemarkable+Pen" - product1.brands = ["#IamRemarkable"] - product1.price_info = price_info1 - product1.color_info = color_info1 - product1.fulfillment_info = [fulfillment_info1] - product1.retrievable_fields = field_mask1 - - price_info2 = PriceInfo() - price_info2.price = 35.0 - price_info2.original_price = 45.0 - price_info2.cost = 12.0 - price_info2.currency_code = "USD" - - color_info2 = ColorInfo() - color_info2.color_families = ["Blue"] - color_info2.colors = ["Sky blue"] - - fulfillment_info2 = FulfillmentInfo() - fulfillment_info2.type_ = "pickup-in-store" - fulfillment_info2.place_ids = ["store2", "store3"] - - field_mask2 = FieldMask(paths=["title", "categories", "price_info", "color_info"]) - - product2.title = "Android Embroidered Crewneck Sweater" - product2.id = "".join(random.sample(string.ascii_lowercase, 8)) - product2.categories = ["Apparel"] - product2.uri = "https://shop.googlemerchandisestore.com/Google+Redesign/Apparel/Android+Embroidered+Crewneck+Sweater" - product2.brands = ["Android"] - product2.price_info = price_info2 - product2.color_info = color_info2 - product2.fulfillment_info = [fulfillment_info2] - product2.retrievable_fields = field_mask2 - - products.append(product1) - products.append(product2) - return products - - -# get import products from inline source request -def get_import_products_inline_request(products_to_import): - # TO CHECK ERROR HANDLING PASTE THE INVALID CATALOG NAME HERE: - # default_catalog = "invalid_catalog_name" - inline_source = ProductInlineSource() - inline_source.products = products_to_import - - input_config = ProductInputConfig() - input_config.product_inline_source = inline_source - - import_request = ImportProductsRequest() - import_request.parent = default_catalog - import_request.input_config = input_config - - print("---import products from inline source request---") - print(import_request) - - return import_request - - -# call the Retail API to import products -def import_products_from_inline_source(): - import_request = get_import_products_inline_request(get_products()) - import_operation = ProductServiceClient().import_products(import_request) - - print("---the operation was started:----") - print(import_operation.operation.name) - - while not import_operation.done(): - print("---please wait till operation is done---") - time.sleep(5) - print("---import products operation is done---") - - if import_operation.metadata is not None: - print("---number of successfully imported products---") - print(import_operation.metadata.success_count) - print("---number of failures during the importing---") - print(import_operation.metadata.failure_count) - else: - print("---operation.metadata is empty---") - - if import_operation.result is not None: - print("---operation result:---") - print(import_operation.result()) - else: - print("---operation.result is empty---") - - -import_products_from_inline_source() - -# [END retail_import_products_from_inline_source] diff --git a/samples/interactive-tutorials/product/import_products_inline_test.py b/samples/interactive-tutorials/product/import_products_inline_test.py deleted file mode 100644 index 7ece9c6f..00000000 --- a/samples/interactive-tutorials/product/import_products_inline_test.py +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright 2021 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import re -import subprocess - - -def test_import_products_gcs(): - output = str( - subprocess.check_output( - "python import_products_inline_source.py", shell=True - ) - ) - - assert re.match(".*import products from inline source request.*", output) - assert re.match(".*the operation was started.*", output) - assert re.match( - ".*projects/.*/locations/global/catalogs/default_catalog/branches/0/operations/import-products.*", - output, - ) - - assert re.match(".*number of successfully imported products.*2.*", output) diff --git a/samples/interactive-tutorials/product/noxfile.py b/samples/interactive-tutorials/product/noxfile.py deleted file mode 100644 index 20cdfc62..00000000 --- a/samples/interactive-tutorials/product/noxfile.py +++ /dev/null @@ -1,279 +0,0 @@ -# Copyright 2019 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import print_function - -import glob -import os -from pathlib import Path -import sys -from typing import Callable, Dict, List, Optional - -import nox - - -# WARNING - WARNING - WARNING - WARNING - WARNING -# WARNING - WARNING - WARNING - WARNING - WARNING -# DO NOT EDIT THIS FILE EVER! -# WARNING - WARNING - WARNING - WARNING - WARNING -# WARNING - WARNING - WARNING - WARNING - WARNING - -BLACK_VERSION = "black==19.10b0" - -# Copy `noxfile_config.py` to your directory and modify it instead. - -# `TEST_CONFIG` dict is a configuration hook that allows users to -# modify the test configurations. The values here should be in sync -# with `noxfile_config.py`. Users will copy `noxfile_config.py` into -# their directory and modify it. - -TEST_CONFIG = { - # You can opt out from the test for specific Python versions. - "ignored_versions": [], - # Old samples are opted out of enforcing Python type hints - # All new samples should feature them - "enforce_type_hints": False, - # An envvar key for determining the project id to use. Change it - # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a - # build specific Cloud project. You can also use your own string - # to use your own Cloud project. - "gcloud_project_env": "GOOGLE_CLOUD_PROJECT", - # 'gcloud_project_env': 'BUILD_SPECIFIC_GCLOUD_PROJECT', - # If you need to use a specific version of pip, - # change pip_version_override to the string representation - # of the version number, for example, "20.2.4" - "pip_version_override": None, - # A dictionary you want to inject into your test. Don't put any - # secrets here. These values will override predefined values. - "envs": {}, -} - - -try: - # Ensure we can import noxfile_config in the project's directory. - sys.path.append(".") - from noxfile_config import TEST_CONFIG_OVERRIDE -except ImportError as e: - print("No user noxfile_config found: detail: {}".format(e)) - TEST_CONFIG_OVERRIDE = {} - -# Update the TEST_CONFIG with the user supplied values. -TEST_CONFIG.update(TEST_CONFIG_OVERRIDE) - - -def get_pytest_env_vars() -> Dict[str, str]: - """Returns a dict for pytest invocation.""" - ret = {} - - # Override the GCLOUD_PROJECT and the alias. - env_key = TEST_CONFIG["gcloud_project_env"] - # This should error out if not set. - ret["GOOGLE_CLOUD_PROJECT"] = os.environ[env_key] - - # Apply user supplied envs. - ret.update(TEST_CONFIG["envs"]) - return ret - - -# DO NOT EDIT - automatically generated. -# All versions used to test samples. -ALL_VERSIONS = ["3.6", "3.7", "3.8", "3.9", "3.10"] - -# Any default versions that should be ignored. -IGNORED_VERSIONS = TEST_CONFIG["ignored_versions"] - -TESTED_VERSIONS = sorted([v for v in ALL_VERSIONS if v not in IGNORED_VERSIONS]) - -INSTALL_LIBRARY_FROM_SOURCE = os.environ.get("INSTALL_LIBRARY_FROM_SOURCE", False) in ( - "True", - "true", -) - -# Error if a python version is missing -nox.options.error_on_missing_interpreters = True - -# -# Style Checks -# - - -def _determine_local_import_names(start_dir: str) -> List[str]: - """Determines all import names that should be considered "local". - - This is used when running the linter to insure that import order is - properly checked. - """ - file_ext_pairs = [os.path.splitext(path) for path in os.listdir(start_dir)] - return [ - basename - for basename, extension in file_ext_pairs - if extension == ".py" - or os.path.isdir(os.path.join(start_dir, basename)) - and basename not in ("__pycache__") - ] - - -# Linting with flake8. -# -# We ignore the following rules: -# E203: whitespace before ‘:’ -# E266: too many leading ‘#’ for block comment -# E501: line too long -# I202: Additional newline in a section of imports -# -# We also need to specify the rules which are ignored by default: -# ['E226', 'W504', 'E126', 'E123', 'W503', 'E24', 'E704', 'E121'] -FLAKE8_COMMON_ARGS = [ - "--show-source", - "--builtin=gettext", - "--max-complexity=20", - "--import-order-style=google", - "--exclude=.nox,.cache,env,lib,generated_pb2,*_pb2.py,*_pb2_grpc.py", - "--ignore=E121,E123,E126,E203,E226,E24,E266,E501,E704,W503,W504,I202", - "--max-line-length=88", -] - - -@nox.session -def lint(session: nox.sessions.Session) -> None: - if not TEST_CONFIG["enforce_type_hints"]: - session.install("flake8", "flake8-import-order") - else: - session.install("flake8", "flake8-import-order", "flake8-annotations") - - local_names = _determine_local_import_names(".") - args = FLAKE8_COMMON_ARGS + [ - "--application-import-names", - ",".join(local_names), - ".", - ] - session.run("flake8", *args) - - -# -# Black -# - - -@nox.session -def blacken(session: nox.sessions.Session) -> None: - session.install(BLACK_VERSION) - python_files = [path for path in os.listdir(".") if path.endswith(".py")] - - session.run("black", *python_files) - - -# -# Sample Tests -# - - -PYTEST_COMMON_ARGS = ["--junitxml=sponge_log.xml"] - - -def _session_tests( - session: nox.sessions.Session, post_install: Callable = None -) -> None: - # check for presence of tests - test_list = glob.glob("*_test.py") + glob.glob("test_*.py") - test_list.extend(glob.glob("tests")) - if len(test_list) == 0: - print("No tests found, skipping directory.") - else: - if TEST_CONFIG["pip_version_override"]: - pip_version = TEST_CONFIG["pip_version_override"] - session.install(f"pip=={pip_version}") - """Runs py.test for a particular project.""" - if os.path.exists("requirements.txt"): - if os.path.exists("constraints.txt"): - session.install("-r", "requirements.txt", "-c", "constraints.txt") - else: - session.install("-r", "requirements.txt") - - if os.path.exists("requirements-test.txt"): - if os.path.exists("constraints-test.txt"): - session.install( - "-r", "requirements-test.txt", "-c", "constraints-test.txt" - ) - else: - session.install("-r", "requirements-test.txt") - - if INSTALL_LIBRARY_FROM_SOURCE: - session.install("-e", _get_repo_root()) - - if post_install: - post_install(session) - - session.run( - "pytest", - *(PYTEST_COMMON_ARGS + session.posargs), - # Pytest will return 5 when no tests are collected. This can happen - # on travis where slow and flaky tests are excluded. - # See http://doc.pytest.org/en/latest/_modules/_pytest/main.html - success_codes=[0, 5], - env=get_pytest_env_vars(), - ) - - -@nox.session(python=ALL_VERSIONS) -def py(session: nox.sessions.Session) -> None: - """Runs py.test for a sample using the specified version of Python.""" - if session.python in TESTED_VERSIONS: - _session_tests(session) - else: - session.skip( - "SKIPPED: {} tests are disabled for this sample.".format(session.python) - ) - - -# -# Readmegen -# - - -def _get_repo_root() -> Optional[str]: - """ Returns the root folder of the project. """ - # Get root of this repository. Assume we don't have directories nested deeper than 10 items. - p = Path(os.getcwd()) - for i in range(10): - if p is None: - break - if Path(p / ".git").exists(): - return str(p) - # .git is not available in repos cloned via Cloud Build - # setup.py is always in the library's root, so use that instead - # https://github.com/googleapis/synthtool/issues/792 - if Path(p / "setup.py").exists(): - return str(p) - p = p.parent - raise Exception("Unable to detect repository root.") - - -GENERATED_READMES = sorted([x for x in Path(".").rglob("*.rst.in")]) - - -@nox.session -@nox.parametrize("path", GENERATED_READMES) -def readmegen(session: nox.sessions.Session, path: str) -> None: - """(Re-)generates the readme for a sample.""" - session.install("jinja2", "pyyaml") - dir_ = os.path.dirname(path) - - if os.path.exists(os.path.join(dir_, "requirements.txt")): - session.install("-r", os.path.join(dir_, "requirements.txt")) - - in_file = os.path.join(dir_, "README.rst.in") - session.run( - "python", _get_repo_root() + "/scripts/readme-gen/readme_gen.py", in_file - ) diff --git a/samples/interactive-tutorials/product/noxfile_config.py b/samples/interactive-tutorials/product/noxfile_config.py deleted file mode 100644 index edce74e3..00000000 --- a/samples/interactive-tutorials/product/noxfile_config.py +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Default TEST_CONFIG_OVERRIDE for python repos. - -# The source of truth: -# https://github.com/GoogleCloudPlatform/python-docs-samples/blob/master/noxfile_config.py - -TEST_CONFIG_OVERRIDE = { - # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7", "3.6"], - # An envvar key for determining the project id to use. Change it - # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a - # build specific Cloud project. You can also use your own string - # to use your own Cloud project. - "gcloud_project_env": "GOOGLE_CLOUD_PROJECT", - # 'gcloud_project_env': 'BUILD_SPECIFIC_GCLOUD_PROJECT', - # A dictionary you want to inject into your test. Don't put any - # secrets here. These values will override predefined values. - "envs": { - "BUCKET_NAME": "retail-interactive-tutorials", - }, -} diff --git a/samples/interactive-tutorials/product/requirements-test.txt b/samples/interactive-tutorials/product/requirements-test.txt deleted file mode 100644 index bbf73145..00000000 --- a/samples/interactive-tutorials/product/requirements-test.txt +++ /dev/null @@ -1,2 +0,0 @@ -pytest==6.2.5 -pytest-xdist==2.5.0 diff --git a/samples/interactive-tutorials/product/requirements.txt b/samples/interactive-tutorials/product/requirements.txt deleted file mode 100644 index 0ba6ea71..00000000 --- a/samples/interactive-tutorials/product/requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -google==3.0.0 -google-cloud-retail==1.1.0 -google-cloud-storage==1.43.0 -google-cloud-bigquery==2.30.1 \ No newline at end of file diff --git a/samples/interactive-tutorials/product/setup/products_create_bigquery_table.py b/samples/interactive-tutorials/product/setup/products_create_bigquery_table.py deleted file mode 100644 index ec14f16c..00000000 --- a/samples/interactive-tutorials/product/setup/products_create_bigquery_table.py +++ /dev/null @@ -1,31 +0,0 @@ -# Copyright 2021 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from setup_cleanup import create_bq_dataset, create_bq_table, \ - upload_data_to_bq_table - -dataset = "products" -valid_products_table = "products" -invalid_products_table = "products_some_invalid" -product_schema = "resources/product_schema.json" -valid_products_source_file = "resources/products.json" -invalid_products_source_file = "resources/products_some_invalid.json" - -create_bq_dataset(dataset) -create_bq_table(dataset, valid_products_table, product_schema) -upload_data_to_bq_table(dataset, valid_products_table, - valid_products_source_file, product_schema) -create_bq_table(dataset, invalid_products_table, product_schema) -upload_data_to_bq_table(dataset, invalid_products_table, - invalid_products_source_file, product_schema) diff --git a/samples/interactive-tutorials/product/setup/products_create_bigquery_table_test.py b/samples/interactive-tutorials/product/setup/products_create_bigquery_table_test.py deleted file mode 100644 index bc58e8bc..00000000 --- a/samples/interactive-tutorials/product/setup/products_create_bigquery_table_test.py +++ /dev/null @@ -1,43 +0,0 @@ -# Copyright 2021 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import re -import subprocess - - -def test_create_bigquery_table(): - output = str( - subprocess.check_output( - 'python setup/products_create_bigquery_table.py', - shell=True)) - assert re.match( - '.*Creating dataset products.*', output) - assert re.match( - '(.*dataset products already exists.*|.*dataset is created.*)', output) - assert re.match( - '.*Creating BigQuery table products.*', output) - assert re.match( - '(.*table products already exists.*|.*table is created.*)', output) - assert re.match( - '.*Uploading data form resources/products.json to the table products.products.*', output) - - assert re.match( - '.*Creating BigQuery table products_some_invalid.*', - output) - assert re.match( - '(.*table products_some_invalid already exists.*|.*table is created.*)', - output) - assert re.match( - '.*Uploading data form resources/products_some_invalid.json to the table products.products_some_invalid.*', - output) diff --git a/samples/interactive-tutorials/product/setup/products_create_gcs_bucket.py b/samples/interactive-tutorials/product/setup/products_create_gcs_bucket.py deleted file mode 100644 index e166066e..00000000 --- a/samples/interactive-tutorials/product/setup/products_create_gcs_bucket.py +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright 2021 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import datetime -import os - -from setup_cleanup import create_bucket, upload_blob - -project_id = os.environ["GOOGLE_CLOUD_PROJECT"] -timestamp_ = datetime.datetime.now().timestamp().__round__() -bucket_name = "{}_products_{}".format(project_id, timestamp_) - -create_bucket(bucket_name) -upload_blob(bucket_name, "resources/products.json") -upload_blob(bucket_name, "resources/products_some_invalid.json") - -print("\nThe gcs bucket {} was created".format(bucket_name)) diff --git a/samples/interactive-tutorials/product/setup/products_create_gcs_bucket_test.py b/samples/interactive-tutorials/product/setup/products_create_gcs_bucket_test.py deleted file mode 100644 index 19301b36..00000000 --- a/samples/interactive-tutorials/product/setup/products_create_gcs_bucket_test.py +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright 2021 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import re -import subprocess - -from products_delete_gcs_bucket import delete_bucket_by_name - - -def test_create_gcs_bucket(): - output = str( - subprocess.check_output( - 'python setup/products_create_gcs_bucket.py', - shell=True)) - - bucket_name = re.search('The gcs bucket (.+?) was created', output).group(1) - delete_bucket_by_name(bucket_name) - - print("bucket_name = {}".format(bucket_name)) - - assert re.match( - '.*Creating new bucket.*', output) - assert re.match( - '(.*The gcs bucket.*?was created.*|.*Bucket.*?already exists.*)', output) - assert re.match( - '.*Uploading data form resources/products.json to the bucket.*', output) - assert re.match( - '.*Uploading data form resources/products_some_invalid.json to the bucket.*', - output) diff --git a/samples/interactive-tutorials/product/setup/products_delete_gcs_bucket.py b/samples/interactive-tutorials/product/setup/products_delete_gcs_bucket.py deleted file mode 100644 index 010478cf..00000000 --- a/samples/interactive-tutorials/product/setup/products_delete_gcs_bucket.py +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright 2021 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os - -from setup_cleanup import delete_bucket - - -def delete_bucket_by_name(name: str): - if name is None: - bucket_name = os.environ["BUCKET_NAME"] - delete_bucket(bucket_name) - else: - delete_bucket(name) diff --git a/samples/interactive-tutorials/product/setup/setup_cleanup.py b/samples/interactive-tutorials/product/setup/setup_cleanup.py deleted file mode 100644 index 7fc2b95d..00000000 --- a/samples/interactive-tutorials/product/setup/setup_cleanup.py +++ /dev/null @@ -1,213 +0,0 @@ -# Copyright 2021 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import re -import shlex -import subprocess - -from google.api_core.exceptions import NotFound - -from google.cloud import storage -from google.cloud.retail_v2 import CreateProductRequest, DeleteProductRequest, \ - FulfillmentInfo, GetProductRequest, PriceInfo, Product, ProductServiceClient - -project_number = os.environ["GOOGLE_CLOUD_PROJECT_NUMBER"] -project_id = os.environ["GOOGLE_CLOUD_PROJECT"] -default_catalog = "projects/{0}/locations/global/catalogs/default_catalog".format( - project_number) -default_branch_name = "projects/" + project_number + "/locations/global/catalogs/default_catalog/branches/default_branch" - - -def generate_product() -> Product: - price_info = PriceInfo() - price_info.price = 30.0 - price_info.original_price = 35.5 - price_info.currency_code = "USD" - fulfillment_info = FulfillmentInfo() - fulfillment_info.type_ = "pickup-in-store" - fulfillment_info.place_ids = ["store0", "store1"] - return Product( - title='Nest Mini', - type_=Product.Type.PRIMARY, - categories=['Speakers and displays'], - brands=['Google'], - price_info=price_info, - fulfillment_info=[fulfillment_info], - availability="IN_STOCK", - ) - - -def create_product(product_id: str) -> object: - create_product_request = CreateProductRequest() - create_product_request.product = generate_product() - create_product_request.product_id = product_id - create_product_request.parent = default_branch_name - - created_product = ProductServiceClient().create_product( - create_product_request) - print("---product is created:---") - print(created_product) - - return created_product - - -def delete_product(product_name: str): - delete_product_request = DeleteProductRequest() - delete_product_request.name = product_name - ProductServiceClient().delete_product(delete_product_request) - - print("---product " + product_name + " was deleted:---") - - -def get_product(product_name: str): - get_product_request = GetProductRequest() - get_product_request.name = product_name - try: - product = ProductServiceClient().get_product(get_product_request) - print("---get product response:---") - print(product) - return product - except NotFound as e: - print(e.message) - return e.message - - -def try_to_delete_product_if_exists(product_name: str): - get_product_request = GetProductRequest() - get_product_request.name = product_name - delete_product_request = DeleteProductRequest() - delete_product_request.name = product_name - print( - "---delete product from the catalog, if the product already exists---") - try: - product = ProductServiceClient().get_product(get_product_request) - ProductServiceClient().delete_product(product.name) - except NotFound as e: - print(e.message) - - -def create_bucket(bucket_name: str): - """Create a new bucket in Cloud Storage""" - print("Creating new bucket:" + bucket_name) - buckets_in_your_project = str(list_buckets()) - if bucket_name in buckets_in_your_project: - print("Bucket {} already exists".format(bucket_name)) - else: - storage_client = storage.Client() - bucket = storage_client.bucket(bucket_name) - bucket.storage_class = "STANDARD" - new_bucket = storage_client.create_bucket(bucket, location="us") - print( - "Created bucket {} in {} with storage class {}".format( - new_bucket.name, new_bucket.location, new_bucket.storage_class - ) - ) - return new_bucket - - -def delete_bucket(bucket_name: str): - """Delete a bucket from Cloud Storage""" - storage_client = storage.Client() - print("Deleting bucket name:" + bucket_name) - buckets_in_your_project = str(list_buckets()) - if bucket_name in buckets_in_your_project: - blobs = storage_client.list_blobs(bucket_name) - for blob in blobs: - blob.delete() - bucket = storage_client.get_bucket(bucket_name) - bucket.delete() - print("Bucket {} is deleted".format(bucket.name)) - else: - print("Bucket {} is not found".format(bucket_name)) - - -def list_buckets(): - """Lists all buckets""" - bucket_list = [] - storage_client = storage.Client() - buckets = storage_client.list_buckets() - for bucket in buckets: - bucket_list.append(str(bucket)) - return bucket_list - - -def upload_blob(bucket_name, source_file_name): - """Uploads a file to the bucket.""" - # The path to your file to upload - # source_file_name = "local/path/to/file" - print("Uploading data form {} to the bucket {}".format(source_file_name, - bucket_name)) - storage_client = storage.Client() - bucket = storage_client.bucket(bucket_name) - object_name = re.search('resources/(.*?)$', source_file_name).group(1) - blob = bucket.blob(object_name) - blob.upload_from_filename(source_file_name) - - print( - "File {} uploaded to {}.".format( - source_file_name, object_name - ) - ) - - -def create_bq_dataset(dataset_name): - """Create a BigQuery dataset""" - print("Creating dataset {}".format(dataset_name)) - if dataset_name not in list_bq_datasets(): - create_dataset_command = 'bq --location=US mk -d --default_table_expiration 3600 --description "This is my dataset." {}:{}'.format( - project_id, dataset_name) - subprocess.check_output(shlex.split(create_dataset_command)) - print("dataset is created") - else: - print("dataset {} already exists".format(dataset_name)) - - -def list_bq_datasets(): - """List BigQuery datasets in the project""" - list_dataset_command = "bq ls --project_id {}".format(project_id) - datasets = subprocess.check_output(shlex.split(list_dataset_command)) - return str(datasets) - - -def create_bq_table(dataset, table_name, schema): - """Create a BigQuery table""" - print("Creating BigQuery table {}".format(table_name)) - if table_name not in list_bq_tables(dataset): - create_table_command = "bq mk --table {}:{}.{} {}".format( - project_id, - dataset, - table_name, schema) - output = subprocess.check_output(shlex.split(create_table_command)) - print(output) - print("table is created") - else: - print("table {} already exists".format(table_name)) - - -def list_bq_tables(dataset): - """List BigQuery tables in the dataset""" - list_tables_command = "bq ls {}:{}".format(project_id, dataset) - tables = subprocess.check_output(shlex.split(list_tables_command)) - return str(tables) - - -def upload_data_to_bq_table(dataset, table_name, source, schema): - """Upload data to the table from specified source file""" - print("Uploading data form {} to the table {}.{}".format(source, dataset, - table_name)) - upload_data_command = "bq load --source_format=NEWLINE_DELIMITED_JSON {}:{}.{} {} {}".format( - project_id, dataset, table_name, source, schema) - output = subprocess.check_output(shlex.split(upload_data_command)) - print(output) diff --git a/samples/interactive-tutorials/resources/events_schema.json b/samples/interactive-tutorials/resources/events_schema.json new file mode 100644 index 00000000..a52c0e56 --- /dev/null +++ b/samples/interactive-tutorials/resources/events_schema.json @@ -0,0 +1,73 @@ +[ + { + "fields":[ + { + "mode": "NULLABLE", + "name": "currencyCode", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "revenue", + "type": "FLOAT" + } + ], + "mode": "NULLABLE", + "name": "purchaseTransaction", + "type": "RECORD" + }, + { + "fields":[ + { + "mode": "NULLABLE", + "name": "quantity", + "type": "INTEGER" + }, + { + "fields":[ + { + "mode": "NULLABLE", + "name": "id", + "type": "STRING" + } + ], + "mode": "NULLABLE", + "name": "product", + "type": "RECORD" + } + ], + "mode": "REPEATED", + "name": "productDetails", + "type": "RECORD" + }, + { + "mode": "REQUIRED", + "name": "eventTime", + "type": "STRING" + }, + { + "mode": "REQUIRED", + "name": "visitorId", + "type": "STRING" + }, + { + "mode": "REQUIRED", + "name": "eventType", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "searchQuery", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "cartId", + "type": "STRING" + }, + { + "mode": "REPEATED", + "name": "pageCategories", + "type": "STRING" + } + ] \ No newline at end of file diff --git a/samples/interactive-tutorials/resources/product_schema.json b/samples/interactive-tutorials/resources/product_schema.json new file mode 100644 index 00000000..2dcc79f7 --- /dev/null +++ b/samples/interactive-tutorials/resources/product_schema.json @@ -0,0 +1,317 @@ +[ + { + "name": "name", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "id", + "type": "STRING", + "mode": "REQUIRED" + }, + { + "name": "type", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "primaryProductId", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "collectionMemberIds", + "type": "STRING", + "mode": "REPEATED" + }, + { + "name": "gtin", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "categories", + "type": "STRING", + "mode": "REPEATED" + }, + { + "name": "title", + "type": "STRING", + "mode": "REQUIRED" + }, + { + "name": "brands", + "type": "STRING", + "mode": "REPEATED" + }, + { + "name": "description", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "languageCode", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "attributes", + "type": "RECORD", + "mode": "REPEATED", + "fields": [ + { + "name": "key", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "value", + "type": "RECORD", + "mode": "NULLABLE", + "fields": [ + { + "name": "text", + "type": "STRING", + "mode": "REPEATED" + }, + { + "name": "numbers", + "type": "FLOAT", + "mode": "REPEATED" + }, + { + "name": "searchable", + "type": "BOOLEAN", + "mode": "NULLABLE" + }, + { + "name": "indexable", + "type": "BOOLEAN", + "mode": "NULLABLE" + } + ] + } + ] + }, + { + "name": "tags", + "type": "STRING", + "mode": "REPEATED" + }, + { + "name": "priceInfo", + "type": "RECORD", + "mode": "NULLABLE", + "fields": [ + { + "name": "currencyCode", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "price", + "type": "FLOAT", + "mode": "NULLABLE" + }, + { + "name": "originalPrice", + "type": "FLOAT", + "mode": "NULLABLE" + }, + { + "name": "cost", + "type": "FLOAT", + "mode": "NULLABLE" + }, + { + "name": "priceEffectiveTime", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "priceExpireTime", + "type": "STRING", + "mode": "NULLABLE" + } + ] + }, + { + "name": "rating", + "type": "RECORD", + "mode": "NULLABLE", + "fields": [ + { + "name": "ratingCount", + "type": "INTEGER", + "mode": "NULLABLE" + }, + { + "name": "averageRating", + "type": "FLOAT", + "mode": "NULLABLE" + }, + { + "name": "ratingHistogram", + "type": "INTEGER", + "mode": "REPEATED" + } + ] + }, + { + "name": "expireTime", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "ttl", + "type": "RECORD", + "mode": "NULLABLE", + "fields": [ + { + "name": "seconds", + "type": "INTEGER", + "mode": "NULLABLE" + }, + { + "name": "nanos", + "type": "INTEGER", + "mode": "NULLABLE" + } + ] + }, + { + "name": "availableTime", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "availability", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "availableQuantity", + "type": "INTEGER", + "mode": "NULLABLE" + }, + { + "name": "fulfillmentInfo", + "type": "RECORD", + "mode": "REPEATED", + "fields": [ + { + "name": "type", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "placeIds", + "type": "STRING", + "mode": "REPEATED" + } + ] + }, + { + "name": "uri", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "images", + "type": "RECORD", + "mode": "REPEATED", + "fields": [ + { + "name": "uri", + "type": "STRING", + "mode": "REQUIRED" + }, + { + "name": "height", + "type": "INTEGER", + "mode": "NULLABLE" + }, + { + "name": "width", + "type": "INTEGER", + "mode": "NULLABLE" + } + ] + }, + { + "name": "audience", + "type": "RECORD", + "mode": "NULLABLE", + "fields": [ + { + "name": "genders", + "type": "STRING", + "mode": "REPEATED" + }, + { + "name": "ageGroups", + "type": "STRING", + "mode": "REPEATED" + } + ] + }, + { + "name": "colorInfo", + "type": "RECORD", + "mode": "NULLABLE", + "fields": [ + { + "name": "colorFamilies", + "type": "STRING", + "mode": "REPEATED" + }, + { + "name": "colors", + "type": "STRING", + "mode": "REPEATED" + } + ] + }, + { + "name": "sizes", + "type": "STRING", + "mode": "REPEATED" + }, + { + "name": "materials", + "type": "STRING", + "mode": "REPEATED" + }, + { + "name": "patterns", + "type": "STRING", + "mode": "REPEATED" + }, + { + "name": "conditions", + "type": "STRING", + "mode": "REPEATED" + }, + { + "name": "retrievableFields", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "publishTime", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "promotions", + "type": "RECORD", + "mode": "REPEATED", + "fields": [ + { + "name": "promotionId", + "type": "STRING", + "mode": "NULLABLE" + } + ] + } +] \ No newline at end of file diff --git a/samples/snippets/resources/products.json b/samples/interactive-tutorials/resources/products.json similarity index 100% rename from samples/snippets/resources/products.json rename to samples/interactive-tutorials/resources/products.json diff --git a/samples/interactive-tutorials/resources/products_some_invalid.json b/samples/interactive-tutorials/resources/products_some_invalid.json new file mode 100644 index 00000000..f46dc761 --- /dev/null +++ b/samples/interactive-tutorials/resources/products_some_invalid.json @@ -0,0 +1,3 @@ +{"id": "GGCOGOAC101259","name": "GGCOGOAC101259","title": "#IamRemarkable Pen","brands": ["#IamRemarkable"],"categories": ["Office"],"priceInfo": {"cost": "12.0","currencyCode": "USD","originalPrice": "45.0","priceEffectiveTime": "2020-08-01T12:00:00+00:00","priceExpireTime": "2120-08-01T12:00:00+00:00","price": "16"},"colorInfo": {"colorFamilies": ["Blue"],"colors": ["Light blue","Blue","Dark blue"]},"availability": "INVALID_VALUE","availableQuantity": 50,"availableTime": "2021-10-11T12:00:00+00:00","images": [{"height": "300","width": "400","uri": "https://shop.googlemerchandisestore.com/store/20160512512/assets/items/images/GGCOGOAC101259.jpg"}],"retrievableFields": "name,title,brands,categories,priceInfo,colorInfo,availability,images,attributes.material,attributes.ecofriendly,attributes.style,attributes.collection,uri","attributes":[ {"key":"material", "value": {"indexable": "true","searchable": "true","text": ["Metal","Recycled Plastic"]}}],"uri": "https://shop.googlemerchandisestore.com/Google+Redesign/Office/IamRemarkable+Pen"} +{"id": "GGPRAHPL107110","name": "GGPRAHPL107110","title": "Android Iconic Hat Green","brands": ["Android"],"categories": ["Apparel"],"priceInfo": {"cost": "12.0","currencyCode": "USD","originalPrice": "45.0","priceEffectiveTime": "2020-08-01T12:00:00+00:00","priceExpireTime": "2120-08-01T12:00:00+00:00","price": "16"},"colorInfo": {"colorFamilies": ["Green"],"colors": ["Olive","Grass green","Light green"]},"availability": "IN_STOCK","availableQuantity": 50,"availableTime": "2021-10-11T12:00:00+00:00","images": [{"height": "300","width": "400","uri": "https://shop.googlemerchandisestore.com/store/20160512512/assets/items/images/GGOEAHPL130910.jpg"}],"retrievableFields": "name,title,brands,categories,priceInfo,colorInfo,availability,images,attributes.material,attributes.ecofriendly,attributes.style,attributes.collection,uri","uri": "https://shop.googlemerchandisestore.com/Google+Prize+Portal/Android+Iconic+Hat+Green"} +{"id": "GGOEAAKQ137410","name": "GGOEAAKQ137410","title": "Android Iconic Sock","brands": ["Android"],"categories": ["Apparel"],"priceInfo": {"cost": "12.0","currencyCode": "USD","originalPrice": "45.0","priceEffectiveTime": "2020-08-01T12:00:00+00:00","priceExpireTime": "2120-08-01T12:00:00+00:00","price": "17"},"availability": "IN_STOCK","availableQuantity": 50,"availableTime": "2021-10-11T12:00:00+00:00","images": [{"height": "300","width": "400","uri": "https://shop.googlemerchandisestore.com/store/20160512512/assets/items/images/GGOEAAKQ137410.jpg"}],"sizes": ["XS","S","M","L","XL"],"retrievableFields": "name,title,brands,categories,priceInfo,colorInfo,availability,images,attributes.material,attributes.ecofriendly,attributes.style,attributes.collection,uri","attributes":[ {"key":"collection", "value": {"indexable": "true","numbers": [2022.0]}},{"key":"material", "value": {"indexable": "true","searchable": "true","text": ["Polyester","Membrane"]}},{"key":"ecofriendly", "value": {"indexable": "false","searchable": "false","text": ["Low-impact fabrics","recycled fabrics","recycled packaging","plastic-free packaging","ethically made"]}},{"key":"style", "value": {"indexable": "true","searchable": "true","text": ["Sport","Functional"]}}],"uri": "https://shop.googlemerchandisestore.com/Google+Redesign/Apparel/Android+Iconic+Sock"} \ No newline at end of file diff --git a/samples/interactive-tutorials/resources/user_events.json b/samples/interactive-tutorials/resources/user_events.json new file mode 100644 index 00000000..5360c309 --- /dev/null +++ b/samples/interactive-tutorials/resources/user_events.json @@ -0,0 +1,4 @@ +{"eventType":"home-page-view","visitorId":"bjbs_group1_visitor1","eventTime":"2021-12-12T10:27:42+00:00"} +{"eventType":"search","visitorId":"bjbs_group1_visitor1","eventTime":"2021-12-12T10:27:42+00:00","searchQuery":"RockerJeans teenagers blue jeans"} +{"eventType":"search","visitorId":"bjbs_group1_visitor1","eventTime":"2021-12-12T10:27:42+00:00","searchQuery":"SocksUnlimited teenagers black socks"} +{"eventType":"detail-page-view","visitorId":"bjbs_group1_visitor1","eventTime":"2021-12-12T10:27:42+00:00","productDetails":{"product":{"id":"GGCOGAEC100616"},"quantity":3}} diff --git a/samples/interactive-tutorials/resources/user_events_some_invalid.json b/samples/interactive-tutorials/resources/user_events_some_invalid.json new file mode 100644 index 00000000..c98b1699 --- /dev/null +++ b/samples/interactive-tutorials/resources/user_events_some_invalid.json @@ -0,0 +1,4 @@ +{"eventType":"home-page-view","visitorId":"bjbs_group1_visitor1","eventTime":"2021-12-12T10:27:42+00:00"} +{"eventType":"invalid","visitorId":"bjbs_group1_visitor1","eventTime":"2021-12-12T10:27:42+00:00","searchQuery":"RockerJeans teenagers blue jeans"} +{"eventType":"search","visitorId":"bjbs_group1_visitor1","eventTime":"2021-12-12T10:27:42+00:00","searchQuery":"SocksUnlimited teenagers black socks"} +{"eventType":"detail-page-view","visitorId":"bjbs_group1_visitor1","eventTime":"2021-12-12T10:27:42+00:00","productDetails":{"product":{"id":"GGCOGAEC100616"},"quantity":3}} diff --git a/samples/interactive-tutorials/search/noxfile.py b/samples/interactive-tutorials/search/noxfile.py deleted file mode 100644 index 20cdfc62..00000000 --- a/samples/interactive-tutorials/search/noxfile.py +++ /dev/null @@ -1,279 +0,0 @@ -# Copyright 2019 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import print_function - -import glob -import os -from pathlib import Path -import sys -from typing import Callable, Dict, List, Optional - -import nox - - -# WARNING - WARNING - WARNING - WARNING - WARNING -# WARNING - WARNING - WARNING - WARNING - WARNING -# DO NOT EDIT THIS FILE EVER! -# WARNING - WARNING - WARNING - WARNING - WARNING -# WARNING - WARNING - WARNING - WARNING - WARNING - -BLACK_VERSION = "black==19.10b0" - -# Copy `noxfile_config.py` to your directory and modify it instead. - -# `TEST_CONFIG` dict is a configuration hook that allows users to -# modify the test configurations. The values here should be in sync -# with `noxfile_config.py`. Users will copy `noxfile_config.py` into -# their directory and modify it. - -TEST_CONFIG = { - # You can opt out from the test for specific Python versions. - "ignored_versions": [], - # Old samples are opted out of enforcing Python type hints - # All new samples should feature them - "enforce_type_hints": False, - # An envvar key for determining the project id to use. Change it - # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a - # build specific Cloud project. You can also use your own string - # to use your own Cloud project. - "gcloud_project_env": "GOOGLE_CLOUD_PROJECT", - # 'gcloud_project_env': 'BUILD_SPECIFIC_GCLOUD_PROJECT', - # If you need to use a specific version of pip, - # change pip_version_override to the string representation - # of the version number, for example, "20.2.4" - "pip_version_override": None, - # A dictionary you want to inject into your test. Don't put any - # secrets here. These values will override predefined values. - "envs": {}, -} - - -try: - # Ensure we can import noxfile_config in the project's directory. - sys.path.append(".") - from noxfile_config import TEST_CONFIG_OVERRIDE -except ImportError as e: - print("No user noxfile_config found: detail: {}".format(e)) - TEST_CONFIG_OVERRIDE = {} - -# Update the TEST_CONFIG with the user supplied values. -TEST_CONFIG.update(TEST_CONFIG_OVERRIDE) - - -def get_pytest_env_vars() -> Dict[str, str]: - """Returns a dict for pytest invocation.""" - ret = {} - - # Override the GCLOUD_PROJECT and the alias. - env_key = TEST_CONFIG["gcloud_project_env"] - # This should error out if not set. - ret["GOOGLE_CLOUD_PROJECT"] = os.environ[env_key] - - # Apply user supplied envs. - ret.update(TEST_CONFIG["envs"]) - return ret - - -# DO NOT EDIT - automatically generated. -# All versions used to test samples. -ALL_VERSIONS = ["3.6", "3.7", "3.8", "3.9", "3.10"] - -# Any default versions that should be ignored. -IGNORED_VERSIONS = TEST_CONFIG["ignored_versions"] - -TESTED_VERSIONS = sorted([v for v in ALL_VERSIONS if v not in IGNORED_VERSIONS]) - -INSTALL_LIBRARY_FROM_SOURCE = os.environ.get("INSTALL_LIBRARY_FROM_SOURCE", False) in ( - "True", - "true", -) - -# Error if a python version is missing -nox.options.error_on_missing_interpreters = True - -# -# Style Checks -# - - -def _determine_local_import_names(start_dir: str) -> List[str]: - """Determines all import names that should be considered "local". - - This is used when running the linter to insure that import order is - properly checked. - """ - file_ext_pairs = [os.path.splitext(path) for path in os.listdir(start_dir)] - return [ - basename - for basename, extension in file_ext_pairs - if extension == ".py" - or os.path.isdir(os.path.join(start_dir, basename)) - and basename not in ("__pycache__") - ] - - -# Linting with flake8. -# -# We ignore the following rules: -# E203: whitespace before ‘:’ -# E266: too many leading ‘#’ for block comment -# E501: line too long -# I202: Additional newline in a section of imports -# -# We also need to specify the rules which are ignored by default: -# ['E226', 'W504', 'E126', 'E123', 'W503', 'E24', 'E704', 'E121'] -FLAKE8_COMMON_ARGS = [ - "--show-source", - "--builtin=gettext", - "--max-complexity=20", - "--import-order-style=google", - "--exclude=.nox,.cache,env,lib,generated_pb2,*_pb2.py,*_pb2_grpc.py", - "--ignore=E121,E123,E126,E203,E226,E24,E266,E501,E704,W503,W504,I202", - "--max-line-length=88", -] - - -@nox.session -def lint(session: nox.sessions.Session) -> None: - if not TEST_CONFIG["enforce_type_hints"]: - session.install("flake8", "flake8-import-order") - else: - session.install("flake8", "flake8-import-order", "flake8-annotations") - - local_names = _determine_local_import_names(".") - args = FLAKE8_COMMON_ARGS + [ - "--application-import-names", - ",".join(local_names), - ".", - ] - session.run("flake8", *args) - - -# -# Black -# - - -@nox.session -def blacken(session: nox.sessions.Session) -> None: - session.install(BLACK_VERSION) - python_files = [path for path in os.listdir(".") if path.endswith(".py")] - - session.run("black", *python_files) - - -# -# Sample Tests -# - - -PYTEST_COMMON_ARGS = ["--junitxml=sponge_log.xml"] - - -def _session_tests( - session: nox.sessions.Session, post_install: Callable = None -) -> None: - # check for presence of tests - test_list = glob.glob("*_test.py") + glob.glob("test_*.py") - test_list.extend(glob.glob("tests")) - if len(test_list) == 0: - print("No tests found, skipping directory.") - else: - if TEST_CONFIG["pip_version_override"]: - pip_version = TEST_CONFIG["pip_version_override"] - session.install(f"pip=={pip_version}") - """Runs py.test for a particular project.""" - if os.path.exists("requirements.txt"): - if os.path.exists("constraints.txt"): - session.install("-r", "requirements.txt", "-c", "constraints.txt") - else: - session.install("-r", "requirements.txt") - - if os.path.exists("requirements-test.txt"): - if os.path.exists("constraints-test.txt"): - session.install( - "-r", "requirements-test.txt", "-c", "constraints-test.txt" - ) - else: - session.install("-r", "requirements-test.txt") - - if INSTALL_LIBRARY_FROM_SOURCE: - session.install("-e", _get_repo_root()) - - if post_install: - post_install(session) - - session.run( - "pytest", - *(PYTEST_COMMON_ARGS + session.posargs), - # Pytest will return 5 when no tests are collected. This can happen - # on travis where slow and flaky tests are excluded. - # See http://doc.pytest.org/en/latest/_modules/_pytest/main.html - success_codes=[0, 5], - env=get_pytest_env_vars(), - ) - - -@nox.session(python=ALL_VERSIONS) -def py(session: nox.sessions.Session) -> None: - """Runs py.test for a sample using the specified version of Python.""" - if session.python in TESTED_VERSIONS: - _session_tests(session) - else: - session.skip( - "SKIPPED: {} tests are disabled for this sample.".format(session.python) - ) - - -# -# Readmegen -# - - -def _get_repo_root() -> Optional[str]: - """ Returns the root folder of the project. """ - # Get root of this repository. Assume we don't have directories nested deeper than 10 items. - p = Path(os.getcwd()) - for i in range(10): - if p is None: - break - if Path(p / ".git").exists(): - return str(p) - # .git is not available in repos cloned via Cloud Build - # setup.py is always in the library's root, so use that instead - # https://github.com/googleapis/synthtool/issues/792 - if Path(p / "setup.py").exists(): - return str(p) - p = p.parent - raise Exception("Unable to detect repository root.") - - -GENERATED_READMES = sorted([x for x in Path(".").rglob("*.rst.in")]) - - -@nox.session -@nox.parametrize("path", GENERATED_READMES) -def readmegen(session: nox.sessions.Session, path: str) -> None: - """(Re-)generates the readme for a sample.""" - session.install("jinja2", "pyyaml") - dir_ = os.path.dirname(path) - - if os.path.exists(os.path.join(dir_, "requirements.txt")): - session.install("-r", os.path.join(dir_, "requirements.txt")) - - in_file = os.path.join(dir_, "README.rst.in") - session.run( - "python", _get_repo_root() + "/scripts/readme-gen/readme_gen.py", in_file - ) diff --git a/samples/interactive-tutorials/search/noxfile_config.py b/samples/interactive-tutorials/search/noxfile_config.py deleted file mode 100644 index 401ff7f2..00000000 --- a/samples/interactive-tutorials/search/noxfile_config.py +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Default TEST_CONFIG_OVERRIDE for python repos. - -# The source of truth: -# https://github.com/GoogleCloudPlatform/python-docs-samples/blob/master/noxfile_config.py - -TEST_CONFIG_OVERRIDE = { - # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7", "3.6"], - # An envvar key for determining the project id to use. Change it - # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a - # build specific Cloud project. You can also use your own string - # to use your own Cloud project. - "gcloud_project_env": "GOOGLE_CLOUD_PROJECT", - # 'gcloud_project_env': 'BUILD_SPECIFIC_GCLOUD_PROJECT', - # A dictionary you want to inject into your test. Don't put any - # secrets here. These values will override predefined values. - "envs": {"BUCKET_NAME": "retail-interactive-tutorials",}, -} diff --git a/samples/interactive-tutorials/search/requirements-test.txt b/samples/interactive-tutorials/search/requirements-test.txt deleted file mode 100644 index bbf73145..00000000 --- a/samples/interactive-tutorials/search/requirements-test.txt +++ /dev/null @@ -1,2 +0,0 @@ -pytest==6.2.5 -pytest-xdist==2.5.0 diff --git a/samples/interactive-tutorials/search/requirements.txt b/samples/interactive-tutorials/search/requirements.txt deleted file mode 100644 index 0ba6ea71..00000000 --- a/samples/interactive-tutorials/search/requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -google==3.0.0 -google-cloud-retail==1.1.0 -google-cloud-storage==1.43.0 -google-cloud-bigquery==2.30.1 \ No newline at end of file diff --git a/samples/interactive-tutorials/search/search_simple_query.py b/samples/interactive-tutorials/search/search_simple_query.py deleted file mode 100644 index 78889209..00000000 --- a/samples/interactive-tutorials/search/search_simple_query.py +++ /dev/null @@ -1,59 +0,0 @@ -# Copyright 2021 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# [START retail_search_for_products_with_query_parameter] -# Call Retail API to search for a products in a catalog using only search query. -# -import os - -from google.cloud.retail import SearchRequest, SearchServiceClient - -project_number = os.environ["GOOGLE_CLOUD_PROJECT_NUMBER"] - - -# get search service request: -def get_search_request(query: str): - default_search_placement = ( - "projects/" - + project_number - + "/locations/global/catalogs/default_catalog/placements/default_search" - ) - - search_request = SearchRequest() - search_request.placement = default_search_placement # Placement is used to identify the Serving Config name. - search_request.query = query - search_request.visitor_id = "123456" # A unique identifier to track visitors - search_request.page_size = 10 - - print("---search request:---") - print(search_request) - - return search_request - - -# call the Retail Search: -def search(): - # TRY DIFFERENT QUERY PHRASES HERE: - query_phrase = "Hoodie" - - search_request = get_search_request(query_phrase) - search_response = SearchServiceClient().search(search_request) - - print("---search response---") - print(search_response) - return search_response - - -search() -# [END retail_search_for_products_with_query_parameter] diff --git a/samples/interactive-tutorials/search/search_simple_query_test.py b/samples/interactive-tutorials/search/search_simple_query_test.py deleted file mode 100644 index fc6d0c69..00000000 --- a/samples/interactive-tutorials/search/search_simple_query_test.py +++ /dev/null @@ -1,37 +0,0 @@ -# Copyright 2021 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Call Retail API to search for a products in a catalog using only search query. -# -import re -import subprocess - -from search_simple_query import search - - -def test_search_simple_query_pass(): - output = str(subprocess.check_output("python search_simple_query.py", shell=True)) - - assert re.match(".*search request.*", output) - assert re.match(".*search response.*", output) - # check the response contains some products - assert re.match(".*results.*id.*", output) - - -def test_search_simple_query_response(): - response = search() - - assert len(response.results) == 10 - product_title = response.results[0].product.title - assert re.match(".*Hoodie.*", product_title) diff --git a/samples/interactive-tutorials/search/search_with_boost_spec.py b/samples/interactive-tutorials/search/search_with_boost_spec.py deleted file mode 100644 index 7b2995d8..00000000 --- a/samples/interactive-tutorials/search/search_with_boost_spec.py +++ /dev/null @@ -1,68 +0,0 @@ -# Copyright 2021 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# [START retail_search_product_with_boost_spec] -# Call Retail API to search for a products in a catalog, rerank the -# results boosting or burying the products that match defined condition. -# -import os - -from google.cloud.retail import SearchRequest, SearchServiceClient - -project_number = os.environ["GOOGLE_CLOUD_PROJECT_NUMBER"] - - -# get search service request: -def get_search_request(query: str, condition: str, boost_strength: float): - default_search_placement = ( - "projects/" - + project_number - + "/locations/global/catalogs/default_catalog/placements/default_search" - ) - - condition_boost_spec = SearchRequest.BoostSpec.ConditionBoostSpec() - condition_boost_spec.condition = condition - condition_boost_spec.boost = boost_strength - - boost_spec = SearchRequest.BoostSpec() - boost_spec.condition_boost_specs = [condition_boost_spec] - - search_request = SearchRequest() - search_request.placement = default_search_placement # Placement is used to identify the Serving Config name. - search_request.query = query - search_request.visitor_id = "123456" # A unique identifier to track visitors - search_request.boost_spec = boost_spec - search_request.page_size = 10 - - print("---search request---") - print(search_request) - - return search_request - - -# call the Retail Search: -def search(): - # TRY DIFFERENT CONDITIONS HERE: - condition = '(colorFamilies: ANY("Blue"))' - boost = 0.0 - - search_request = get_search_request("Tee", condition, boost) - search_response = SearchServiceClient().search(search_request) - print("---search response---") - print(search_response) - return search_response - - -search() -# [END retail_search_product_with_boost_spec] diff --git a/samples/interactive-tutorials/search/search_with_boost_spec_test.py b/samples/interactive-tutorials/search/search_with_boost_spec_test.py deleted file mode 100644 index cf391b2e..00000000 --- a/samples/interactive-tutorials/search/search_with_boost_spec_test.py +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright 2021 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Call Retail API to search for a products in a catalog using only search query. -# -import re -import subprocess - -from search_with_boost_spec import search - - -def test_search_with_boost_spec_pass(): - output = str( - subprocess.check_output("python search_with_boost_spec.py", shell=True) - ) - - assert re.match(".*search request.*", output) - assert re.match(".*search response.*", output) - # check the response contains some products - assert re.match(".*results.*id.*", output) - - -def test_search_with_boost_spec(): - response = search() - - assert len(response.results) == 10 - product_title = response.results[0].product.title - assert re.match(".*Tee.*", product_title) diff --git a/samples/interactive-tutorials/search/search_with_facet_spec.py b/samples/interactive-tutorials/search/search_with_facet_spec.py deleted file mode 100644 index 50a8fb7f..00000000 --- a/samples/interactive-tutorials/search/search_with_facet_spec.py +++ /dev/null @@ -1,66 +0,0 @@ -# Copyright 2021 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# [START retail_search_product_with_facet_spec] -# -import os - -from google.cloud.retail import SearchRequest, SearchServiceClient - -project_number = os.environ["GOOGLE_CLOUD_PROJECT_NUMBER"] - - -# get search service request: -def get_search_request(query: str, facet_key_param: str): - default_search_placement = ( - "projects/" - + project_number - + "/locations/global/catalogs/default_catalog/placements/default_search" - ) - # PUT THE INTERVALS HERE: - facet_key = SearchRequest.FacetSpec().FacetKey() - facet_key.key = facet_key_param - - facet_spec = SearchRequest.FacetSpec() - facet_spec.facet_key = facet_key - - search_request = SearchRequest() - search_request.placement = default_search_placement # Placement is used to identify the Serving Config name. - search_request.query = query - search_request.visitor_id = "123456" # A unique identifier to track visitors - search_request.facet_specs = [facet_spec] - search_request.page_size = 10 - - print("---search request---") - print(search_request) - - return search_request - - -# call the Retail Search: -def search(): - # TRY DIFFERENT FACETS HERE: - facet_key = "colorFamilies" - - search_request = get_search_request("Tee", facet_key) - search_response = SearchServiceClient().search(search_request) - print("---search response---") - print(search_response) - print("---facets:---") - print(search_response.facets) - return search_response - - -search() -# [END retail_search_product_with_facet_spec] diff --git a/samples/interactive-tutorials/search/search_with_facet_spec_test.py b/samples/interactive-tutorials/search/search_with_facet_spec_test.py deleted file mode 100644 index df082ac2..00000000 --- a/samples/interactive-tutorials/search/search_with_facet_spec_test.py +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright 2021 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Call Retail API to search for a products in a catalog using only search query. -# -import re -import subprocess - -from search_with_facet_spec import search - - -def test_search_with_filtering_pass(): - output = str( - subprocess.check_output("python search_with_facet_spec.py", shell=True) - ) - - assert re.match(".*search request.*", output) - assert re.match(".*search response.*", output) - # check the response contains some products - assert re.match(".*results.*id.*", output) - assert re.match(".*facets.*?colorFamilies.*", output) - - -def test_search_with_filtering(): - response = search() - - assert len(response.results) == 10 - product_title = response.results[0].product.title - assert re.match(".*Tee.*", product_title) - assert response.facets[0].key == "colorFamilies" diff --git a/samples/interactive-tutorials/search/search_with_filtering.py b/samples/interactive-tutorials/search/search_with_filtering.py deleted file mode 100644 index e7a8a31d..00000000 --- a/samples/interactive-tutorials/search/search_with_filtering.py +++ /dev/null @@ -1,60 +0,0 @@ -# Copyright 2021 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# [START retail_search_for_products_with_filter] -# Call Retail API to search for a products in a catalog, filter the results by different product fields. -# -import os - -from google.cloud.retail import SearchRequest, SearchServiceClient - -project_number = os.environ["GOOGLE_CLOUD_PROJECT_NUMBER"] - - -# get search service request: -def get_search_request(query: str, _filter: str): - default_search_placement = ( - "projects/" - + project_number - + "/locations/global/catalogs/default_catalog/placements/default_search" - ) - - search_request = SearchRequest() - search_request.placement = default_search_placement # Placement is used to identify the Serving Config name. - search_request.query = query - search_request.filter = _filter - search_request.page_size = 10 - search_request.visitor_id = "123456" # A unique identifier to track visitors - search_request.page_size = 10 - - print("---search request:---") - print(search_request) - - return search_request - - -# call the Retail Search: -def search(): - # TRY DIFFERENT FILTER EXPRESSIONS HERE: - filter_ = '(colorFamilies: ANY("Black"))' - - search_request = get_search_request("Tee", filter_) - search_response = SearchServiceClient().search(search_request) - print("---search response---") - print(search_response) - return search_response - - -search() -# [END retail_search_for_products_with_filter] diff --git a/samples/interactive-tutorials/search/search_with_filtering_test.py b/samples/interactive-tutorials/search/search_with_filtering_test.py deleted file mode 100644 index 5b10ae34..00000000 --- a/samples/interactive-tutorials/search/search_with_filtering_test.py +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright 2021 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Call Retail API to search for a products in a catalog using only search query. -# -import re -import subprocess - -from search_with_filtering import search - - -def test_search_with_filtering_pass(): - output = str(subprocess.check_output("python search_with_filtering.py", shell=True)) - - assert re.match(".*search request.*", output) - assert re.match(".*search response.*", output) - # check the response contains some products - assert re.match(".*results.*id.*", output) - - -def test_search_with_filtering(): - response = search() - - assert len(response.results) == 10 - product_title = response.results[0].product.title - assert re.match(".*Tee.*", product_title) - assert re.match(".*Black.*", product_title) - assert "Black" in response.results[0].product.color_info.color_families - assert response.total_size == 16 diff --git a/samples/interactive-tutorials/search/search_with_ordering.py b/samples/interactive-tutorials/search/search_with_ordering.py deleted file mode 100644 index 118d4202..00000000 --- a/samples/interactive-tutorials/search/search_with_ordering.py +++ /dev/null @@ -1,60 +0,0 @@ -# Copyright 2021 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# [START retail_search_for_products_with_ordering] -# Call Retail API to search for a products in a catalog, order the results by different product fields. -# -import os - -from google.cloud.retail import SearchRequest, SearchServiceClient - -project_number = os.environ["GOOGLE_CLOUD_PROJECT_NUMBER"] - - -# get search service request: -def get_search_request(query: str, order: str): - default_search_placement = ( - "projects/" - + project_number - + "/locations/global/catalogs/default_catalog/placements/default_search" - ) - - search_request = SearchRequest() - search_request.placement = default_search_placement - search_request.query = query - search_request.order_by = order - search_request.visitor_id = "123456" # A unique identifier to track visitors - search_request.page_size = 10 - - print("---search request---") - print(search_request) - - return search_request - - -# call the Retail Search: -def search(): - # TRY DIFFERENT ORDERING EXPRESSIONS HERE: - order = "price desc" - - search_request = get_search_request("Hoodie", order) - search_response = SearchServiceClient().search(search_request) - - print("---search response---") - print(search_response) - return search_response - - -search() -# [END retail_search_for_products_with_ordering] diff --git a/samples/interactive-tutorials/search/search_with_ordering_test.py b/samples/interactive-tutorials/search/search_with_ordering_test.py deleted file mode 100644 index ec8fcfb9..00000000 --- a/samples/interactive-tutorials/search/search_with_ordering_test.py +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright 2021 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Call Retail API to search for a products in a catalog using only search query. -# -import re -import subprocess - -from search_with_ordering import search - - -def test_search_with_ordering_pass(): - output = str(subprocess.check_output("python search_with_ordering.py", shell=True)) - - assert re.match(".*search request.*", output) - assert re.match(".*search response.*", output) - # check the response contains some products - assert re.match(".*results.*id.*", output) - - -def test_search_with_ordering(): - response = search() - - assert len(response.results) == 10 - assert response.results[0].product.price_info.price == 39 diff --git a/samples/interactive-tutorials/search/search_with_pagination.py b/samples/interactive-tutorials/search/search_with_pagination.py deleted file mode 100644 index 727227c2..00000000 --- a/samples/interactive-tutorials/search/search_with_pagination.py +++ /dev/null @@ -1,72 +0,0 @@ -# Copyright 2021 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# [START retail_search_for_products_with_page_size] -# Call Retail API to search for a products in a catalog, -# limit the number of the products per page and go to the next page using "next_page_token" -# or jump to chosen page using "offset". -# -import os - -from google.cloud.retail import SearchRequest, SearchServiceClient - -project_number = os.environ["GOOGLE_CLOUD_PROJECT_NUMBER"] - - -# get search service request: -def get_search_request(query: str, page_size: int, offset: int, next_page_token: str): - default_search_placement = ( - "projects/" - + project_number - + "/locations/global/catalogs/default_catalog/placements/default_search" - ) - - search_request = SearchRequest() - search_request.placement = default_search_placement - search_request.visitor_id = "123456" # A unique identifier to track visitors - search_request.query = query - search_request.page_size = page_size - search_request.offset = offset - search_request.page_token = next_page_token - - print("---search request:---") - print(search_request) - - return search_request - - -# call the Retail Search: -def search(): - # TRY DIFFERENT PAGINATION PARAMETERS HERE: - page_size = 6 - offset = 0 - page_token = "" - - search_request_first_page = get_search_request( - "Hoodie", page_size, offset, page_token - ) - search_response_first_page = SearchServiceClient().search(search_request_first_page) - - print("---search response---") - print(search_response_first_page) - - # PASTE CALL WITH NEXT PAGE TOKEN HERE: - - # PASTE CALL WITH OFFSET HERE: - - return search_response_first_page - - -search() -# [END retail_search_for_products_with_page_size] diff --git a/samples/interactive-tutorials/search/search_with_pagination_test.py b/samples/interactive-tutorials/search/search_with_pagination_test.py deleted file mode 100644 index 49bcd63a..00000000 --- a/samples/interactive-tutorials/search/search_with_pagination_test.py +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright 2021 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Call Retail API to search for a products in a catalog using only search query. -# -import re -import subprocess - -from search_with_pagination import search - - -def test_search_with_pagination_pass(): - output = str( - subprocess.check_output("python search_with_pagination.py", shell=True) - ) - - assert re.match(".*search request.*", output) - assert re.match(".*search response.*", output) - # check the response contains some products - assert re.match(".*results.*id.*", output) - - -def test_search_with_pagination(): - response = search() - - product_title = response.results[0].product.title - assert re.match(".*Hoodie", product_title) - assert len(response.results) == 6 diff --git a/samples/interactive-tutorials/search/search_with_query_expansion_spec.py b/samples/interactive-tutorials/search/search_with_query_expansion_spec.py deleted file mode 100644 index 9455e900..00000000 --- a/samples/interactive-tutorials/search/search_with_query_expansion_spec.py +++ /dev/null @@ -1,66 +0,0 @@ -# Copyright 2021 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# [START retail_search_for_products_with_query_expansion_specification] -# Call Retail API to search for a products in a catalog, -# enabling the query expansion feature to let the Google Retail Search build an automatic query expansion. -# -import os - -from google.cloud.retail import SearchRequest, SearchServiceClient - -project_number = os.environ["GOOGLE_CLOUD_PROJECT_NUMBER"] - - -# get search service request: -def get_search_request( - query: str, condition: SearchRequest.QueryExpansionSpec.Condition -): - default_search_placement = ( - "projects/" - + project_number - + "/locations/global/catalogs/default_catalog/placements/default_search" - ) - - query_expansion_spec = SearchRequest().QueryExpansionSpec() - query_expansion_spec.condition = condition - - search_request = SearchRequest() - search_request.placement = default_search_placement # Placement is used to identify the Serving Config name. - search_request.query = query - search_request.visitor_id = "123456" # A unique identifier to track visitors - search_request.query_expansion_spec = query_expansion_spec - search_request.page_size = 10 - - print("---search request:---") - print(search_request) - - return search_request - - -# call the Retail Search: -def search(): - # TRY DIFFERENT QUERY EXPANSION CONDITION HERE: - condition = SearchRequest.QueryExpansionSpec.Condition.AUTO - - search_request = get_search_request("Google Youth Hero Tee Grey", condition) - search_response = SearchServiceClient().search(search_request) - - print("---search response---") - print(search_response) - return search_response - - -search() -# [END retail_search_for_products_with_query_expansion_specification] diff --git a/samples/interactive-tutorials/search/search_with_query_expansion_spec_test.py b/samples/interactive-tutorials/search/search_with_query_expansion_spec_test.py deleted file mode 100644 index b9c04e3f..00000000 --- a/samples/interactive-tutorials/search/search_with_query_expansion_spec_test.py +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright 2021 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Call Retail API to search for a products in a catalog using only search query. -# -import re -import subprocess - -from search_with_query_expansion_spec import search - - -def test_search_with_query_expansion_spec_pass(): - output = str( - subprocess.check_output( - "python search_with_query_expansion_spec.py", shell=True - ) - ) - - assert re.match(".*search request.*", output) - assert re.match(".*search response.*", output) - # check the response contains some products - assert re.match(".*results.*id.*", output) - - -def test_search_with_query_expansion_spec(): - response = search() - - assert response.results[0].product.title == "Google Youth Hero Tee Grey" - assert response.results[2].product.title != "Google Youth Hero Tee Grey" - assert response.query_expansion_info.expanded_query is True diff --git a/samples/snippets/create_test_resources.py b/samples/interactive-tutorials/test_resources_recovery/create_test_resources.py similarity index 53% rename from samples/snippets/create_test_resources.py rename to samples/interactive-tutorials/test_resources_recovery/create_test_resources.py index ef07e133..5fbcca0d 100644 --- a/samples/snippets/create_test_resources.py +++ b/samples/interactive-tutorials/test_resources_recovery/create_test_resources.py @@ -14,6 +14,8 @@ import os import re +import shlex +import subprocess import time from google.cloud.storage.bucket import Bucket @@ -25,12 +27,23 @@ project_number = os.environ["GOOGLE_CLOUD_PROJECT_NUMBER"] bucket_name = os.environ['BUCKET_NAME'] -storage_client = storage.Client() -resource_file = "resources/products.json" -object_name = re.search('resources/(.*?)$', resource_file).group(1) +project_id = os.environ["GOOGLE_CLOUD_PROJECT_ID"] + +product_resource_file = "../resources/products.json" +events_source_file = "../resources/user_events.json" + +product_dataset = "products" +product_table = "products" +product_schema = "resources/product_schema.json" +events_dataset = "user_events" +events_table = "events" +events_schema = "resources/events_schema.json" + +object_name = re.search('resources/(.*?)$', product_resource_file).group(1) default_catalog = "projects/{0}/locations/global/catalogs/default_catalog/branches/default_branch".format( project_number) +storage_client = storage.Client() def create_bucket(bucket_name: str) -> Bucket: """Create a new bucket in Cloud Storage""" @@ -65,8 +78,8 @@ def check_if_bucket_exists(new_bucket_name): def upload_data_to_bucket(bucket: Bucket): """Upload data to a GCS bucket""" blob = bucket.blob(object_name) - blob.upload_from_filename(resource_file) - print("Data from {} has being uploaded to {}".format(resource_file, + blob.upload_from_filename(product_resource_file) + print("Data from {} has being uploaded to {}".format(product_resource_file, bucket.name)) @@ -121,7 +134,78 @@ def import_products_from_gcs(): "Wait 2 -5 minutes till products become indexed in the catalog,\ after that they will be available for search") +def create_bq_dataset(dataset_name): + """Create a BigQuery dataset""" + print("Creating dataset {}".format(dataset_name)) + if dataset_name not in list_bq_datasets(): + create_dataset_command = 'bq --location=US mk -d --default_table_expiration 3600 --description "This is my dataset." {}:{}'.format( + project_id, dataset_name) + output = subprocess.check_output(shlex.split(create_dataset_command)) + print(output) + print("dataset is created") + else: + print("dataset {} already exists".format(dataset_name)) + + +def list_bq_datasets(): + """List BigQuery datasets in the project""" + list_dataset_command = "bq ls --project_id {}".format(project_id) + list_output = subprocess.check_output(shlex.split(list_dataset_command)) + datasets = re.split(r'\W+', str(list_output)) + return datasets + + +def create_bq_table(dataset, table_name, schema): + """Create a BigQuery table""" + print("Creating BigQuery table {}".format(table_name)) + if table_name not in list_bq_tables(dataset): + create_table_command = "bq mk --table {}:{}.{} {}".format( + project_id, + dataset, + table_name, schema) + output = subprocess.check_output(shlex.split(create_table_command)) + print(output) + print("table is created") + else: + print("table {} already exists".format(table_name)) + -created_bucket = create_bucket(bucket_name) -upload_data_to_bucket(created_bucket) +def list_bq_tables(dataset): + """List BigQuery tables in the dataset""" + list_tables_command = "bq ls {}:{}".format(project_id, dataset) + tables = subprocess.check_output(shlex.split(list_tables_command)) + return str(tables) + + +def upload_data_to_bq_table(dataset, table_name, source, schema): + """Upload data to the table from specified source file""" + print("Uploading data form {} to the table {}.{}".format(source, dataset, + table_name)) + upload_data_command = "bq load --source_format=NEWLINE_DELIMITED_JSON {}:{}.{} {} {}".format( + project_id, dataset, table_name, source, schema) + output = subprocess.check_output(shlex.split(upload_data_command)) + print(output) + + +# Create a GCS bucket with products.json file +created_products_bucket = create_bucket(bucket_name) +upload_data_to_bucket(created_products_bucket) + +# Create a GCS bucket with user_events.json file +created_events_bucket = create_bucket(bucket_name) +upload_data_to_bucket(created_events_bucket) + +# Import prodcuts from the GCS bucket to the Retail catalog import_products_from_gcs() + +# Create a BigQuery table with products +create_bq_dataset(product_dataset) +create_bq_table(product_dataset, product_table, product_schema) +upload_data_to_bq_table(product_dataset, product_table, + product_resource_file, product_schema) + +# Create a BigQuery table with user events +create_bq_dataset(events_dataset) +create_bq_table(events_dataset, events_table, events_schema) +upload_data_to_bq_table(events_dataset, events_table, events_source_file, + events_schema) diff --git a/samples/snippets/remove_test_resources.py b/samples/interactive-tutorials/test_resources_recovery/remove_test_resources.py similarity index 76% rename from samples/snippets/remove_test_resources.py rename to samples/interactive-tutorials/test_resources_recovery/remove_test_resources.py index c6faf299..41981ba2 100644 --- a/samples/snippets/remove_test_resources.py +++ b/samples/interactive-tutorials/test_resources_recovery/remove_test_resources.py @@ -13,6 +13,8 @@ # limitations under the License. import os +import shlex +import subprocess from google.api_core.exceptions import PermissionDenied from google.cloud.storage.bucket import Bucket @@ -22,7 +24,12 @@ ProductServiceClient project_number = os.environ["GOOGLE_CLOUD_PROJECT_NUMBER"] -bucket_name = os.environ['BUCKET_NAME'] +product_bucket_name = os.environ['BUCKET_NAME'] +events_bucket_name = os.environ['EVENTS_BUCKET_NAME'] +project_id = os.environ["GOOGLE_CLOUD_PROJECT_ID"] + +product_dataset = "products" +events_dataset = "user_events" default_catalog = "projects/{0}/locations/global/catalogs/default_catalog/branches/default_branch".format( project_number) @@ -30,7 +37,7 @@ storage_client = storage.Client() -def delete_bucket(): +def delete_bucket(bucket_name): """Delete bucket""" try: bucket = storage_client.get_bucket(bucket_name) @@ -52,6 +59,7 @@ def delete_object_from_bucket(bucket: Bucket): def delete_all_products(): """Delete all products in the catalog""" + print("Deleting all products, please wait") product_client = ProductServiceClient() list_request = ListProductsRequest() list_request.parent = default_catalog @@ -69,5 +77,14 @@ def delete_all_products(): print(f"{delete_count} products were deleted from {default_catalog}") -delete_bucket() +def delete_bq_dataset_with_tables(dataset): + """Delete a BigQuery dataset with all tables""" + delete_dataset_command = "bq rm -r -d -f {}".format(dataset) + output = subprocess.check_output(shlex.split(delete_dataset_command)) + print(output) + +delete_bucket(product_bucket_name) +delete_bucket(events_bucket_name) delete_all_products() +delete_bq_dataset_with_tables(product_dataset) +delete_bq_dataset_with_tables(events_dataset)