Installation scripts

installation_scripts/Create_GDrive_folder.py

@@ -0,0 +1,180 @@
+import google.auth
+import os
+import json
+from googleapiclient.discovery import build
+from googleapiclient.errors import HttpError
+from googleapiclient.http import MediaFileUpload
+import string
+import random
+import argparse
+
+
+parser = argparse.ArgumentParser()
+parser.add_argument("--folder_name", help="Name of the drive folder", type=str)
+parser.add_argument("--service_account_email", help="email of your service account", type=str)
+
+dict_args = parser.parse_args()
+
+print(f"Using arguments: {dict_args}")
+
+folder_name = dict_args.folder_name
+service_account_email = dict_args.service_account_email
+
+def create_folder(folder_name):
+  """Create a folder and prints the folder ID
+  Returns : Folder Id
+
+  Load pre-authorized user credentials from the environment.
+  TODO(developer) - See https://developers.google.com/identity
+  for guides on implementing OAuth2 for the application.
+  """
+  creds, _ = google.auth.default()
+
+  try:
+    # create drive api client
+    service = build("drive", "v3", credentials=creds)
+    file_metadata = {
+        "name": folder_name,
+        "mimeType": "application/vnd.google-apps.folder",
+    }
+
+    # pylint: disable=maybe-no-member
+    file = service.files().create(body=file_metadata, fields="id").execute()
+    print(f'Folder ID: "{file.get("id")}".')
+    return file.get("id")
+
+  except HttpError as error:
+    print(f"An error occurred: {error}")
+    return None
+
+
+# Sharing Folder with DOMAIN
+
+def share_file(real_folder_id, real_user):
+  creds, _ = google.auth.default()
+
+  try:
+    # create drive api client
+    service = build("drive", "v3", credentials=creds)
+    ids = []
+    folder_id = real_folder_id
+
+    def callback(request_id, response, exception):
+      if exception:
+        # Handle error
+        print(exception)
+      else:
+        print(f"Request_Id: {request_id}")
+        print(f'Permission Id: {response.get("id")}')
+        ids.append(response.get("id"))
+
+    # pylint: disable=maybe-no-member
+    batch = service.new_batch_http_request(callback=callback)
+    user_permission = {
+        "type": "user",
+        "role": "writer",
+        "emailAddress": real_user,
+    }
+    batch.add(
+        service.permissions().create(
+            fileId=folder_id,
+            body=user_permission,
+            fields="id",
+        )
+    )
+    batch.execute()
+
+  except HttpError as error:
+    print(f"An error occurred: {error}")
+    ids = None
+
+  return ids
+
+# ----- Files upload ----
+
+def upload_with_conversion(folder_id,source_file,destination_file,file_type):
+  """Upload file with conversion
+  Returns: ID of the file uploaded
+
+  Load pre-authorized user credentials from the environment.
+  TODO(developer) - See https://developers.google.com/identity
+  for guides on implementing OAuth2 for the application.
+  """
+  creds, _ = google.auth.default()
+
+  try:
+    # create drive api client
+    service = build("drive", "v3", credentials=creds)
+    results = service.files().list(q=f"name='{destination_file}'",
+        pageSize=10, fields="nextPageToken, files(id, name)",
+        supportsAllDrives=True, includeItemsFromAllDrives=True).execute()
+
+    query = "'{}' in parents".format(folder_id)
+    children = service.files().list(q=query, 
+                                fields='nextPageToken, files(id, name)').execute()
+    #print(children['files'])
+    fileCheck = next((item for item in children['files'] if item['name'] == destination_file), None)
+    if len(children['files']) == 0 or fileCheck == None:
+
+        file_metadata = {
+            "name": destination_file,
+            "mimeType": "application/vnd.google-apps.spreadsheet",
+            "parents":[folder_id]
+        }
+        media = MediaFileUpload(source_file, mimetype=file_type, resumable=True)
+        # pylint: disable=maybe-no-member
+        file = (
+            service.files()
+            .create(body=file_metadata, media_body=media, fields="id")
+            .execute()
+        )
+        print(f'File with ID: "{file.get("id")}" has been uploaded.')
+        return file.get("id")
+    else:
+        fileData = next((item for item in children['files'] if item['name'] == destination_file), None)
+        print("File already Exists:",fileData["id"])
+        return fileData["id"]
+
+
+  except HttpError as error:
+    print(f"An error occurred: {error}")
+    file = None
+
+
+
+
+
+
+if __name__ == "__main__":
+
+    creds, _ = google.auth.default()
+    service = build("drive", "v3", credentials=creds)
+    print(folder_name)
+    results = service.files().list(q=f"name='{folder_name}'",
+        pageSize=10, fields="nextPageToken, files(id, name)",
+        supportsAllDrives=True, includeItemsFromAllDrives=True).execute()
+
+    if len(results["files"]) == 0: 
+        print(" folder not found")
+        GDRIVE_FOLDER_ID=create_folder(folder_name)
+    else:
+        print(" folder Already Exists, so using the same")
+        GDRIVE_FOLDER_ID=results["files"][0]["id"]
+    share_file(GDRIVE_FOLDER_ID,service_account_email)
+    MarketingExcelID=upload_with_conversion(GDRIVE_FOLDER_ID,"genai-for-marketing/templates/[data source] GenAI for Marketing.xlsx","GenAI for Marketing.xlsx","text/xls")
+    MarketingDocID=upload_with_conversion(GDRIVE_FOLDER_ID,"genai-for-marketing/templates/[template] Gen AI for Marketing Google Doc Template.docx","Gen AI for Marketing Google Doc Template.docx","text/doc")
+    MarketingPptID=upload_with_conversion(GDRIVE_FOLDER_ID,"genai-for-marketing/templates/[template] Marketing Assets.pptx","Marketing Assets.pptx","text/ppt")
+
+    with open("marketingEnvValue.json", "r") as jsonFile:
+        data = json.load(jsonFile)
+        data["GDRIVE_FOLDER_ID"] = GDRIVE_FOLDER_ID
+        data["MarketingExcelID"] = MarketingExcelID
+        data["MarketingDocID"] = MarketingDocID
+        data["MarketingPptID"] = MarketingPptID
+    with open("marketingEnvValue.json", "w") as jsonFile:
+        json.dump(data, jsonFile)
+
+
+
+
+

installation_scripts/README.md

@@ -0,0 +1,24 @@
+# Installation Scripts
+
+Automated installation scripts for Generative AI for Marketing.
+
+Follow the instructions below to install the solution.
+
+## Cloud Shell
+
+Use the Cloud Shell button below to install the solution.
+
+Before running the automation, Navigate to the Vertex AI Search and Conversation in google cloud console and Read and agree to the Terms of Service, then click Continue and activate the API.
+
+[![Open in Cloud Shell](https://gstatic.com/cloudssh/images/open-btn.png)](https://ssh.cloud.google.com/cloudshell/open?cloudshell_git_repo=https://github.com/GoogleCloudPlatform/genai-for-marketing/installation-scripts&cloudshell_tutorial=tutorial.md&cloudshell_git_branch=installation_scripts)
+
+or, in Cloud Shell, clone this repo, `cd installation-scripts` and run:
+
+```
+teachme tutorial.md
+```
+
+
+## Manually
+
+Start with `genai_marketing_automation.sh` shell script and modify the variables at the top of the shell script. This script will call the python scripts.

installation_scripts/genai_marketing_automation.sh

@@ -0,0 +1,180 @@
+#!/bin/bash
+#
+# Performs an automated installation of the Generative AI for Marketing solution accelerator
+# Modify the Globals variables prior to running this script
+#################################
+
+# Global variables
+#################################
+#PROJECT_ID="project-001"          # ID of the project where you want to deploy
+LOCATION="us-central1"            # Name of the region 
+DATASET_NAME="genai_marketing"          # BigQuery Dataset Name for creation
+SEARCH_APP_NAME="genai_marketing"       # Vertex Search App Name for creation
+CHAT_BOT_NAME="genai_marketing"    # Vertex Conversation app Name for creation
+#COMPANY_NAME="genai_marketing"          # Your company name 
+EXISTING_LOOKER_URI=""            # your Existing Looker dashboard URl. leave it empty if you don't have
+SERVICE_ACCOUNT="genai-markting-sa"    # Service account name for creation
+#YOUR_DOMAIN="google.com"               # Your domain name. eg user@company.com then company.com 
+GDRIVE_FOLDER_NAME="genai-marketing-assets"      # Google drive folder name for creation
+
+# do not modify below here
+
+
+read -p "Enter Project ID: " PROJECT_ID
+read -p "Enter Company Name: " COMPANY_NAME
+read -p "Enter your Domain name (example.com): " YOUR_DOMAIN
+
+echo -e "\n \n"
+bold=$(tput bold)
+normal=$(tput sgr0)
+echo -e "Here are the names that will be used for creating resources \n"
+echo -e "BIGQUERY DATASET_NAME: ${bold}${DATASET_NAME}${normal} \nSEARCH_APP: ${bold}${SEARCH_APP_NAME}${normal} \nCHAT_BOT_NAME: ${bold}${CHAT_BOT_NAME}${normal} \nSERVICE_ACCOUNT: ${bold}${SERVICE_ACCOUNT}${normal} \nGOOGLE_DRIVE_FOLDER_NAME: ${bold}${GDRIVE_FOLDER_NAME}${normal}"
+echo -e "\nDo you wish to add postfix ? enter 1 for Yes and 2 for No"
+echo -e "\n Note: If you are reruning the automation, use names like earlier"
+select yn in "Yes" "No"; do
+    case $yn in
+        Yes ) read -p "Enter Postfix: " POSTFIX
+              DATASET_NAME="${DATASET_NAME}_${POSTFIX}"
+              SEARCH_APP_NAME="${SEARCH_APP_NAME}_${POSTFIX}"
+              CHAT_BOT_NAME="${CHAT_BOT_NAME}_${POSTFIX}"
+              SERVICE_ACCOUNT="${SERVICE_ACCOUNT}_${POSTFIX}"
+              GDRIVE_FOLDER_NAME="${GDRIVE_FOLDER_NAME}_${POSTFIX}"
+              break ;;
+        No ) echo "Using same names for setup"
+             break ;;
+    esac
+done
+
+SERVICE_ACCOUNT_EMAIL="${SERVICE_ACCOUNT}@${PROJECT_ID}.iam.gserviceaccount.com"   # Do not modify this
+
+
+gcloud config set project $PROJECT_ID   # Setting the Project in Gcloud
+
+PROJECT_NUMBER=`gcloud projects describe $PROJECT_ID --format="value(projectNumber)"`     # Getting the project Number  
+
+# Enabling the services
+gcloud services enable run.googleapis.com cloudbuild.googleapis.com compute.googleapis.com cloudresourcemanager.googleapis.com iam.googleapis.com container.googleapis.com cloudapis.googleapis.com cloudtrace.googleapis.com containerregistry.googleapis.com iamcredentials.googleapis.com dialogflow.googleapis.com
+gcloud services enable monitoring.googleapis.com logging.googleapis.com notebooks.googleapis.com aiplatform.googleapis.com storage.googleapis.com datacatalog.googleapis.com appengineflex.googleapis.com translate.googleapis.com admin.googleapis.com docs.googleapis.com drive.googleapis.com sheets.googleapis.com slides.googleapis.com
+
+if [ ! -d "genai_for_marketing_automation" ]; then   # Checking the Virtualenv folder exists or not
+   python3 -m venv genai_for_marketing_automation    # Creating virtualenv  
+fi
+
+source genai_for_marketing_automation/bin/activate   # activate Virtualenv
+
+if [ ! -d "genai-for-marketing" ]; then   # Checking the github code folder exists or not
+  git clone https://github.com/GoogleCloudPlatform/genai-for-marketing   # cloning the genai-for-marketing code from github
+else
+  rm -rf genai-for-marketing
+  git clone https://github.com/GoogleCloudPlatform/genai-for-marketing
+
+fi
+
+# Updating the Project and Location details in app config and override files
+sed -i "s|project_id = \"\"|project_id = '${PROJECT_ID}'|" genai-for-marketing/app/app_config.toml
+sed -i "s|location = \"us-central1\"|location = '${LOCATION}'|" genai-for-marketing/app/app_config.toml
+sed -i "s|project_id = \"\"|project_id = '${PROJECT_ID}'|" genai-for-marketing/app/override.toml
+sed -i "s|location = \"us-central1\"|location = '${LOCATION}'|" genai-for-marketing/app/override.toml
+
+
+# installing required python packages
+pip install -U google-cloud-datacatalog google-cloud-storage google-cloud-bigquery numpy google-api-python-client google.cloud google.auth google-cloud-discoveryengine google-cloud-dialogflow-cx
+
+# copy the BigQuery template data to current directory
+if [ ! -d "aux_data" ]; then
+   cp -rf genai-for-marketing/notebooks/aux_data .
+fi
+
+#-----BigQuery Setup -----
+python3 genai_marketing_env_setup.py $PROJECT_ID $LOCATION $DATASET_NAME
+
+# Update the BigQuery Details in config files 
+sed -i "s|dataset_id = \"\"|dataset_id = \"${DATASET_NAME}\"|g" genai-for-marketing/app/app_config.toml
+sed -i "s|dataset_id = \"\"|dataset_id = \"${DATASET_NAME}\"|g" genai-for-marketing/app/override.toml
+sed -i "s|tag_name = \"\"|tag_name = \"llmcdptemplate\"|g" genai-for-marketing/app/app_config.toml
+sed -i "s|tag_name = \"\"|tag_name = \"llmcdptemplate\"|g" genai-for-marketing/app/override.toml
+
+
+# gcloud auth application-default login
+# gcloud auth application-default set-quota-project $PROJECT_ID
+
+python3 genai_marketing_search_app_creation.py --project="${PROJECT_ID}" --app-name="${SEARCH_APP_NAME}" --company-name="${COMPANY_NAME}" --uris="cloud.goole.com/*"
+
+SEARCH_DATASTORE_ID=`jq -r '.SEARCH_DATASTORE_ID' < marketingEnvValue.json`
+
+sed -i "s|# datastores.<datastore ID> = 'default_config'|datastores.${SEARCH_DATASTORE_ID} = 'default_config'|g" genai-for-marketing/app/app_config.toml
+sed -i "s|datastores.example = 'default_config'|datastores.${SEARCH_DATASTORE_ID} = 'default_config'|g" genai-for-marketing/app/override.toml
+
+python3 Create_GDrive_folder.py --folder_name="${GDRIVE_FOLDER_NAME}" --service_account_email="${SERVICE_ACCOUNT_EMAIL}"
+
+SERVICE_ACCOUNT_CHECK=`gcloud iam service-accounts list --format=json | jq .[].email | grep "${SERVICE_ACCOUNT_EMAIL}" | wc -l`
+
+if [[ SERVICE_ACCOUNT_CHECK -eq 0 ]]; then
+    gcloud iam service-accounts create ${SERVICE_ACCOUNT} --display-name="${SERVICE_ACCOUNT}"
+fi
+gcloud projects add-iam-policy-binding ${PROJECT_ID} --member="serviceAccount:${SERVICE_ACCOUNT_EMAIL}" --role="roles/logging.logWriter" --condition=None
+gcloud projects add-iam-policy-binding ${PROJECT_ID} --member="serviceAccount:${SERVICE_ACCOUNT_EMAIL}" --role="roles/artifactregistry.reader" --condition=None
+gcloud projects add-iam-policy-binding ${PROJECT_ID} --member="serviceAccount:${SERVICE_ACCOUNT_EMAIL}" --role="roles/storage.objectViewer" --condition=None
+
+if [ ! -f genai-for-marketing/app/credentials.json ]; then
+  gcloud iam service-accounts keys create genai-for-marketing/app/credentials.json --iam-account=${SERVICE_ACCOUNT_EMAIL}
+fi
+
+if [ -f credentials.json ]; then
+   rm -rf credentials.json
+fi
+gcloud iam service-accounts keys create credentials.json --iam-account="${PROJECT_NUMBER}-compute@developer.gserviceaccount.com"
+
+export GOOGLE_APPLICATION_CREDENTIALS=credentials.json
+python3 genai_marketing_conversation_app_creation.py --project="${PROJECT_ID}" --location="global" --app-name="${CHAT_BOT_NAME}" --company-name="${COMPANY_NAME}" --uris="support.google.com/google-ads/*" --datastore-storage-folder="gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/*"
+
+CHAT_AGENT_ID=`jq -r '.AGENT_ENGINE_NAME' < marketingEnvValue.json | cut -d'/' -f6`
+AGENT_LANGUAGE_CODE=`jq -r '.AGENT_LANGUAGE_CODE' < marketingEnvValue.json`
+
+sed -i "s|  agent-id=\"\"|  agent-id=\"${CHAT_AGENT_ID}\"|g" genai-for-marketing/app/app_config.toml
+sed -i "s|  project-id=\"\"|  project-id=\"${PROJECT_ID}\"|g" genai-for-marketing/app/app_config.toml
+sed -i "s|  language-code=\"\"|  language-code=\"${AGENT_LANGUAGE_CODE}\"|g" genai-for-marketing/app/app_config.toml
+
+sed -i "s|  agent-id=\"\"|  agent-id=\"${CHAT_AGENT_ID}\"|g" genai-for-marketing/app/override.toml
+sed -i "s|  project-id=\"\"|  project-id=\"${PROJECT_ID}\"|g" genai-for-marketing/app/override.toml
+sed -i "s|  language-code=\"\"|  language-code=\"${AGENT_LANGUAGE_CODE}\"|g" genai-for-marketing/app/override.toml
+
+if [[ $EXISTING_LOOKER_URI != "" ]];then
+   sed -i "s|dashboards.Overview = 'https://googledemo.looker.com/embed/dashboards/2131?allow_login_screen=true|dashboards.Overview = '${EXISTING_LOOKER_URI}'|" genai-for-marketing/app/app_config.toml
+   sed -i "s|dashboards.Overview = ''|dashboards.Overview = '${EXISTING_LOOKER_URI}'|" genai-for-marketing/app/app_config.toml
+fi
+
+sed -i "s|domain = \"<YOUR DOMAIN>\"|domain = \"${YOUR_DOMAIN}\"|" genai-for-marketing/app/override.toml
+
+GDRIVE_FOLDER_ID=`jq -r '.GDRIVE_FOLDER_ID' < marketingEnvValue.json`
+MarketingPptID=`jq -r '.MarketingPptID' < marketingEnvValue.json`
+MarketingDocID=`jq -r '.MarketingDocID' < marketingEnvValue.json`
+MarketingExcelID=`jq -r '.MarketingExcelID' < marketingEnvValue.json`
+
+sed -i "s|drive_folder_id = ''|drive_folder_id = '${GDRIVE_FOLDER_ID}'|" genai-for-marketing/app/app_config.toml
+sed -i "s|slides_template_id = ''|slides_template_id = '${MarketingPptID}'|" genai-for-marketing/app/app_config.toml
+sed -i "s|doc_template_id = ''|doc_template_id = '${MarketingDocID}'|" genai-for-marketing/app/app_config.toml
+sed -i "s|sheet_template_id = ''|sheet_template_id = '${MarketingExcelID}'|" genai-for-marketing/app/app_config.toml
+sed -i "s|service_account: <REPLACE WITH YOUR SERVICE ACCOUNT ADDRESS>|service_account: '${PROJECT_NUMBER}-compute@developer.gserviceaccount.com'|" genai-for-marketing/app.yaml
+sed -i "s|service_account_json_key = '/credentials/credentials.json'|service_account_json_key = '/app/credentials.json'|" genai-for-marketing/app/app_config.toml
+sed -i "s|service_account_json_key = \"\"|service_account_json_key = \"/app/credentials.json\"|" genai-for-marketing/app/override.toml
+
+#---override.toml---
+sed -i "s|drive_folder_id = \"\"|drive_folder_id = \"${GDRIVE_FOLDER_ID}\"|" genai-for-marketing/app/override.toml
+sed -i "s|slides_template_id = \"\"|slides_template_id = \"${MarketingPptID}\"|" genai-for-marketing/app/override.toml
+sed -i "s|doc_template_id = \"\"|doc_template_id = \"${MarketingDocID}\"|" genai-for-marketing/app/override.toml
+sed -i "s|sheet_template_id = \"\"|sheet_template_id = \"${MarketingExcelID}\"|" genai-for-marketing/app/override.toml
+
+cat >> genai-for-marketing/app.yaml <<EOL
+
+network:
+  name: projects/${PROJECT_ID}/global/networks/default
+  subnetwork_name: default
+EOL
+
+APP_ENGINE_CHECK=`gcloud app services list --filter="SERVICE: default" --format=json | jq .[].id | wc -l`
+if [[ $APP_ENGINE_CHECK == 0 ]]
+then
+   gcloud app create --region=${LOCATION}
+fi
+cd genai-for-marketing && gcloud app deploy --quiet