From 7f182507a6d832c288531c7597d36cc1e08ede00 Mon Sep 17 00:00:00 2001
From: Carson Dunbar <carsondunbar@google.com>
Date: Tue, 19 Sep 2023 19:49:14 +0000
Subject: [PATCH] Initial commit for startup script warnings based on design
 doc.

---
 .../files/running-script-warning.sh           | 29 +++++++++++++
 .../files/startup-script-stdlib-head.sh       | 41 +++++++++++++++++++
 modules/scripts/startup-script/main.tf        | 11 ++++-
 .../templates/startup-script-custom.tpl       |  3 ++
 4 files changed, 82 insertions(+), 2 deletions(-)
 create mode 100644 modules/scripts/startup-script/files/running-script-warning.sh

diff --git a/modules/scripts/startup-script/files/running-script-warning.sh b/modules/scripts/startup-script/files/running-script-warning.sh
new file mode 100644
index 0000000000..cbd7d26f6d
--- /dev/null
+++ b/modules/scripts/startup-script/files/running-script-warning.sh
@@ -0,0 +1,29 @@
+#!/bin/sh
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+WARNING=$(
+	cat <<EOF
+** NOTICE **: System services may not be running until startup scripts complete.
+The output of the command below will end with "Finished Google Compute Engine 
+Startup Scripts." when they are complete. Please review the output for any 
+errors which may indicate that services are unhealthy. 
+
+sudo journalctl -b 0 -u google-startup-scripts.service
+EOF
+)
+
+echo
+echo "${WARNING}"
+echo
diff --git a/modules/scripts/startup-script/files/startup-script-stdlib-head.sh b/modules/scripts/startup-script/files/startup-script-stdlib-head.sh
index 4b5f7f29d0..01d738b785 100644
--- a/modules/scripts/startup-script/files/startup-script-stdlib-head.sh
+++ b/modules/scripts/startup-script/files/startup-script-stdlib-head.sh
@@ -30,6 +30,28 @@ readonly E_RUN_OR_DIE=5
 readonly E_MISSING_MANDATORY_ARG=9
 readonly E_UNKNOWN_ARG=10
 
+SUCCESS_MESSAGE=$(
+	cat <<-EOF
+		** NOTICE **: The VM startup scripts have finished running successfully.  
+		Systems are configured and running.
+	EOF
+)
+readonly SUCCESS_MESSAGE
+ERROR_MESSAGE=$(
+	cat <<-EOF
+		** ERROR **: The VM startup scripts have finished running, but produced an error.
+		Systems may be in an unhealthy state.
+	EOF
+)
+readonly ERROR_MESSAGE
+WARNING_MESSAGE=$(
+	cat <<-EOF
+		** WARNING **: The VM startup scripts for this machine have started.
+		Systems may not be configured or running.
+	EOF
+)
+readonly WARNING_MESSAGE
+
 stdlib::debug() {
 	[[ -z ${DEBUG:-} ]] && return 0
 	local ds msg
@@ -57,6 +79,25 @@ stdlib::error() {
 	echo -e "${RED}${ds}Error [$$]: ${msg}${NC}" >&2
 }
 
+stdlib::announce_runners_start() {
+	if [ -z "$recursive_proc" ]; then
+		wall -n "$WARNING_MESSAGE"
+	fi
+	export recursive_proc=$((${recursive_proc:=0} + 1))
+}
+
+stdlib::announce_runners_end() {
+	exit_code=$1
+	export recursive_proc=$((${recursive_proc:=0} - 1))
+	if [ "$recursive_proc" -le "0" ]; then
+		if [ "$exit_code" -ne "0" ]; then
+			wall -n "$ERROR_MESSAGE"
+		else
+			wall -n "$SUCCESS_MESSAGE"
+		fi
+	fi
+}
+
 # The main initialization function of this library.  This should be kept to the
 # minimum amount of work required for all functions to operate cleanly.
 stdlib::init() {
diff --git a/modules/scripts/startup-script/main.tf b/modules/scripts/startup-script/main.tf
index b717d0f27a..49aabdda45 100644
--- a/modules/scripts/startup-script/main.tf
+++ b/modules/scripts/startup-script/main.tf
@@ -26,6 +26,14 @@ locals {
     destination = "install_cloud_ops_agent_automatic.sh"
   }] : []
 
+  warnings = [
+    {
+      type        = "data"
+      content     = file("${path.module}/files/running-script-warning.sh")
+      destination = "/etc/profile.d/99-running-script-warning.sh"
+    }
+  ]
+
   configure_ssh = length(var.configure_ssh_host_patterns) > 0
   host_args = {
     host_name_prefix = var.configure_ssh_host_patterns
@@ -57,7 +65,6 @@ locals {
     }
   ] : []
 
-
   has_ansible_runners = anytrue([for r in var.runners : r.type == "ansible-local"]) || local.configure_ssh
   install_ansible     = var.install_ansible == null ? local.has_ansible_runners : var.install_ansible
   ansible_installer = local.install_ansible ? [{
@@ -67,7 +74,7 @@ locals {
     args        = var.ansible_virtualenv_path
   }] : []
 
-  runners = concat(local.ops_agent_installer, local.ansible_installer, local.configure_ssh_runners, var.runners)
+  runners = concat(local.warnings, local.ops_agent_installer, local.ansible_installer, local.configure_ssh_runners, var.runners)
 
   bucket_regex               = "^gs://([^/]*)/*(.*)"
   gcs_bucket_path_trimmed    = var.gcs_bucket_path == null ? null : trimsuffix(var.gcs_bucket_path, "/")
diff --git a/modules/scripts/startup-script/templates/startup-script-custom.tpl b/modules/scripts/startup-script/templates/startup-script-custom.tpl
index 694ef1b24b..436c87aa04 100644
--- a/modules/scripts/startup-script/templates/startup-script-custom.tpl
+++ b/modules/scripts/startup-script/templates/startup-script-custom.tpl
@@ -38,6 +38,7 @@ stdlib::runner() {
   stdlib::info "=== $object finished with exit_code=$exit_code ==="
   if [ "$exit_code" -ne "0" ] ; then
     stdlib::error "=== execution of $object failed, exiting ==="
+    stdlib::announce_runners_end "$exit_code"
     exit $exit_code
   fi
 }
@@ -46,10 +47,12 @@ stdlib::load_runners(){
   tmpdir="$(mktemp -d)"
 
   stdlib::debug "=== BEGIN Running runners ==="
+  stdlib::announce_runners_start
 
   %{for r in runners ~}
   stdlib::runner "${r.type}" "${r.object}" "${r.destination}" $${tmpdir} "${r.args}"
   %{endfor ~}
 
+  stdlib::announce_runners_end "0"
   stdlib::debug "=== END Running runners ==="
 }