feat: installation of the Sumo monitors as part of setup process (#2250)

* feat: installation of the Sumo monitors as part of setup process

* feat: configurable e-mail notifications for monitors

* feat: do not run shellcheck on monitors.sh

* feat: fixed Markdown formatting

* feat: adjust tests after rebase

* feat: changelog

* feat: shellcheck for generated monitors.sh

* feat: rename `sumologic.setup.monitors.monitoringDisabled` to `sumologic.setup.monitors.disabled`

* feat: new line at th end of monitors.sh

* feat: link to monitors in NOTES when installation is disabled

* feat: style improvements

* feat: rename monitors.setupEnabled -> monitors.enabled

* feat: rename monitors.disabled -> monitors.monitorStatus

* Empty lines before EOF

Co-authored-by: Dominik Rosiek <58699848+sumo-drosiek@users.noreply.github.com>

Co-authored-by: Dominik Rosiek <58699848+sumo-drosiek@users.noreply.github.com>

CHANGELOG.md

@@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 
 - feat(fluentd): expose extra configuration for fluentd output plugin [#2244][#2244]
+- feat(monitors): the Sumo Logic monitors installation as part of the setup job [#2250][#2250]
 
 ### Changed
 
@@ -34,6 +35,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 [#2244]: https://github.com/SumoLogic/sumologic-kubernetes-collection/pull/2244
 [#2245]: https://github.com/SumoLogic/sumologic-kubernetes-collection/pull/2245
 [#2246]: https://github.com/SumoLogic/sumologic-kubernetes-collection/pull/2246
+[#2250]: https://github.com/SumoLogic/sumologic-kubernetes-collection/pull/2250
 [#2254]: https://github.com/SumoLogic/sumologic-kubernetes-collection/pull/2254
 [#2255]: https://github.com/SumoLogic/sumologic-kubernetes-collection/pull/2255
 [Unreleased]: https://github.com/SumoLogic/sumologic-kubernetes-collection/compare/v2.6.0...main

ci/shellcheck.sh

@@ -3,7 +3,7 @@
 set -e
 
 echo "Checking the bash scripts with shellcheck..."
-find . ! -path '*deploy/helm/sumologic/conf/setup/setup.sh' ! -path "*/tmp/*" -name '*.sh' -type 'f' -print |
+find . ! -path '*deploy/helm/sumologic/conf/setup/setup.sh' ! -path '*deploy/helm/sumologic/conf/setup/monitors.sh' ! -path "*/tmp/*" -name '*.sh' -type 'f' -print |
     while read -r file; do
         # Run tests in their own context
         echo "Checking ${file} with shellcheck"
@@ -17,6 +17,13 @@ find . -path '*tests/helm/terraform/static/*.output.yaml' -type 'f' -print |
         yq r "${file}" "data[setup.sh]" | shellcheck --enable all --external-sources --exclude SC2155 -
     done
 
+find . -path '*tests/helm/terraform/static/*.output.yaml' -type 'f' -print |
+    while read -r file; do
+        # Run tests in their own context
+        echo "Checking ${file} with shellcheck"
+        yq r "${file}" "data[monitors.sh]" | shellcheck --enable all --external-sources --exclude SC2155 -
+    done
+
 find . -path '*tests/helm/terraform_custom/static/*.output.yaml' ! -path "./tests/helm/terraform_custom/static/empty.output.yaml" -type 'f' -print |
     while read -r file; do
         # Run tests in their own context

deploy/helm/sumologic/conf/setup/monitors.sh

@@ -0,0 +1,70 @@
+#!/bin/bash
+
+SUMOLOGIC_ACCESSID=${SUMOLOGIC_ACCESSID:=""}
+readonly SUMOLOGIC_ACCESSID
+SUMOLOGIC_ACCESSKEY=${SUMOLOGIC_ACCESSKEY:=""}
+readonly SUMOLOGIC_ACCESSKEY
+SUMOLOGIC_BASE_URL=${SUMOLOGIC_BASE_URL:=""}
+readonly SUMOLOGIC_BASE_URL
+
+MONITORS_FOLDER_NAME="Kubernetes"
+{{- if eq .Values.sumologic.setup.monitors.monitorStatus "enabled" }}
+MONITORS_DISABLED="false"
+{{- else }}
+MONITORS_DISABLED="true"
+{{- end}}
+
+# verify if the k8s monitors folder already exists
+MONITORS_RESPONSE="$(curl -XGET -s \
+        -u "${SUMOLOGIC_ACCESSID}:${SUMOLOGIC_ACCESSKEY}" \
+        "${SUMOLOGIC_BASE_URL}"v1/monitors/search?query=type:folder%20"${MONITORS_FOLDER_NAME}" | jq '.[]' )"
+readonly MONITORS_RESPONSE
+MONITORS_ROOT_ID="$(curl -XGET -s \
+        -u "${SUMOLOGIC_ACCESSID}:${SUMOLOGIC_ACCESSKEY}" \
+        "${SUMOLOGIC_BASE_URL}"v1/monitors/root | jq -r '.id' )"
+readonly MONITORS_ROOT_ID
+
+MONITORS_FOLDER_ID="$( echo "${MONITORS_RESPONSE}" | \
+        jq -r "select(.item.name == \"${MONITORS_FOLDER_NAME}\") | select(.item.parentId == \"${MONITORS_ROOT_ID}\") | .item.id" )"
+readonly MONITORS_FOLDER_ID
+
+if [[ -z "${MONITORS_FOLDER_ID}" ]]; then
+  # go to monitors directory
+  cd /monitors || exit 2
+
+  # Fall back to init -upgrade to prevent:
+  # Error: Inconsistent dependency lock file
+  terraform init -input=false || terraform init -input=false -upgrade
+
+  # extract environment from SUMOLOGIC_BASE_URL
+  # see: https://help.sumologic.com/APIs/General-API-Information/Sumo-Logic-Endpoints-by-Deployment-and-Firewall-Security
+  SUMOLOGIC_ENV=$( echo "${SUMOLOGIC_BASE_URL}" | sed -E 's/https:\/\/.*(au|ca|de|eu|fed|in|jp|us2)\.sumologic\.com.*/\1/' )
+  if [[ "${SUMOLOGIC_BASE_URL}" == "${SUMOLOGIC_ENV}" ]] ; then
+    SUMOLOGIC_ENV="us1"
+  fi
+
+{{- if not (.Values.sumologic.setup.monitors.notificationEmails | empty) }}
+
+  NOTIFICATIONS_RECIPIENTS='{{- .Values.sumologic.setup.monitors.notificationEmails | toRawJson }}'
+  NOTIFICATIONS_CONTENT="subject=\"Monitor Alert: {{ printf `{{ TriggerType }}` }} on {{ printf `{{ Name }}` }}\",message_body=\"Triggered {{ printf `{{ TriggerType }}` }} Alert on {{ printf `{{ Name }}` }}: {{ printf `{{ QueryURL }}` }}\""
+  NOTIFICATIONS_SETTINGS="recipients=${NOTIFICATIONS_RECIPIENTS},connection_type=\"Email\",time_zone=\"UTC\""
+{{- end }}
+
+  TF_LOG_PROVIDER=DEBUG terraform apply \
+      -auto-approve \
+      -var="access_id=${SUMOLOGIC_ACCESSID}" \
+      -var="access_key=${SUMOLOGIC_ACCESSKEY}" \
+      -var="environment=${SUMOLOGIC_ENV}" \
+      -var="folder=${MONITORS_FOLDER_NAME}" \
+      -var="monitors_disabled=${MONITORS_DISABLED}" \
+{{- if not (.Values.sumologic.setup.monitors.notificationEmails | empty) }}
+      -var="email_notifications_critical=[{${NOTIFICATIONS_SETTINGS},${NOTIFICATIONS_CONTENT},run_for_trigger_types=[\"Critical\", \"ResolvedCritical\"]}]" \
+      -var="email_notifications_warning=[{${NOTIFICATIONS_SETTINGS},${NOTIFICATIONS_CONTENT},run_for_trigger_types=[\"Warning\", \"ResolvedWarning\"]}]" \
+      -var="email_notifications_missingdata=[{${NOTIFICATIONS_SETTINGS},${NOTIFICATIONS_CONTENT},run_for_trigger_types=[\"MissingData\", \"ResolvedMissingData\"]}]" \
+{{- end }}
+      || { echo "Error during applying Terraform monitors."; exit 1; }
+else
+  echo "The monitors were already installed in ${MONITORS_FOLDER_NAME}."
+  echo "You can (re)install them manually with:"
+  echo "https://github.com/SumoLogic/terraform-sumologic-sumo-logic-monitor/tree/main/monitor_packages/kubernetes"
+fi

deploy/helm/sumologic/conf/setup/setup.sh

@@ -142,6 +142,15 @@ TF_LOG_PROVIDER=DEBUG terraform apply \
     -var="create_fields=${CREATE_FIELDS}" \
     || { echo "Error during applying Terraform changes"; exit 1; }
 
+# Setup Sumo Logic monitors if enabled
+{{- if .Values.sumologic.setup.monitors.enabled }}
+bash /etc/terraform/monitors.sh
+{{- else }}
+echo "Installation of the Sumo Logic monitors is disabled."
+echo "You can install them manually later with:"
+echo "https://github.com/SumoLogic/terraform-sumologic-sumo-logic-monitor/tree/main/monitor_packages/kubernetes"
+{{- end }}
+
 # Cleanup env variables
 export SUMOLOGIC_BASE_URL=
 export SUMOLOGIC_ACCESSKEY=

deploy/helm/sumologic/templates/NOTES.txt

@@ -26,3 +26,9 @@ We recommend turning this property on for production environments by setting met
 We've tried to automatically create fields. In an unlikely scenario that this
 fails please refer to the following to create them manually:
 https://github.com/SumoLogic/sumologic-kubernetes-collection/blob/2b3ca63/deploy/docs/Installation_with_Helm.md#prerequisite
+
+{{- if not (.Values.sumologic.setup.monitors.enabled) }}
+echo "Installation of the Sumo Logic monitors is disabled."
+echo "You can install them with:"
+echo "https://github.com/SumoLogic/terraform-sumologic-sumo-logic-monitor/tree/main/monitor_packages/kubernetes"
+{{- end }}

deploy/helm/sumologic/values.yaml

@@ -122,6 +122,16 @@ sumologic:
     ## uncomment for the debug mode (disables the automatic run of the setup.sh script)
     # debug: true
 
+    monitors:
+      ## If enabled, a pre-install hook will create k8s monitors in Sumo Logic
+      enabled: true
+
+      ## The installed monitors default status: enabled/disabled
+      monitorStatus: enabled
+
+      ## A list of emails to send notifications from monitors
+      notificationEmails: []
+
   collector:
     ## Configuration of additional collector fields
     ## https://help.sumologic.com/Manage/Fields#http-source-fields

tests/helm/terraform/static/all_fields.output.yaml

@@ -145,6 +145,61 @@ data:
         }
       }
     }
+  monitors.sh: |
+    #!/bin/bash
+  
+    SUMOLOGIC_ACCESSID=${SUMOLOGIC_ACCESSID:=""}
+    readonly SUMOLOGIC_ACCESSID
+    SUMOLOGIC_ACCESSKEY=${SUMOLOGIC_ACCESSKEY:=""}
+    readonly SUMOLOGIC_ACCESSKEY
+    SUMOLOGIC_BASE_URL=${SUMOLOGIC_BASE_URL:=""}
+    readonly SUMOLOGIC_BASE_URL
+  
+    MONITORS_FOLDER_NAME="Kubernetes"
+    MONITORS_DISABLED="false"
+  
+    # verify if the k8s monitors folder already exists
+    MONITORS_RESPONSE="$(curl -XGET -s \
+            -u "${SUMOLOGIC_ACCESSID}:${SUMOLOGIC_ACCESSKEY}" \
+            "${SUMOLOGIC_BASE_URL}"v1/monitors/search?query=type:folder%20"${MONITORS_FOLDER_NAME}" | jq '.[]' )"
+    readonly MONITORS_RESPONSE
+    MONITORS_ROOT_ID="$(curl -XGET -s \
+            -u "${SUMOLOGIC_ACCESSID}:${SUMOLOGIC_ACCESSKEY}" \
+            "${SUMOLOGIC_BASE_URL}"v1/monitors/root | jq -r '.id' )"
+    readonly MONITORS_ROOT_ID
+  
+    MONITORS_FOLDER_ID="$( echo "${MONITORS_RESPONSE}" | \
+            jq -r "select(.item.name == \"${MONITORS_FOLDER_NAME}\") | select(.item.parentId == \"${MONITORS_ROOT_ID}\") | .item.id" )"
+    readonly MONITORS_FOLDER_ID
+  
+    if [[ -z "${MONITORS_FOLDER_ID}" ]]; then
+      # go to monitors directory
+      cd /monitors || exit 2
+  
+      # Fall back to init -upgrade to prevent:
+      # Error: Inconsistent dependency lock file
+      terraform init -input=false || terraform init -input=false -upgrade
+  
+      # extract environment from SUMOLOGIC_BASE_URL
+      # see: https://help.sumologic.com/APIs/General-API-Information/Sumo-Logic-Endpoints-by-Deployment-and-Firewall-Security
+      SUMOLOGIC_ENV=$( echo "${SUMOLOGIC_BASE_URL}" | sed -E 's/https:\/\/.*(au|ca|de|eu|fed|in|jp|us2)\.sumologic\.com.*/\1/' )
+      if [[ "${SUMOLOGIC_BASE_URL}" == "${SUMOLOGIC_ENV}" ]] ; then
+        SUMOLOGIC_ENV="us1"
+      fi
+  
+      TF_LOG_PROVIDER=DEBUG terraform apply \
+          -auto-approve \
+          -var="access_id=${SUMOLOGIC_ACCESSID}" \
+          -var="access_key=${SUMOLOGIC_ACCESSKEY}" \
+          -var="environment=${SUMOLOGIC_ENV}" \
+          -var="folder=${MONITORS_FOLDER_NAME}" \
+          -var="monitors_disabled=${MONITORS_DISABLED}" \
+          || { echo "Error during applying Terraform monitors."; exit 1; }
+    else
+      echo "The monitors were already installed in ${MONITORS_FOLDER_NAME}."
+      echo "You can (re)install them manually with:"
+      echo "https://github.com/SumoLogic/terraform-sumologic-sumo-logic-monitor/tree/main/monitor_packages/kubernetes"
+    fi
   providers.tf: |-
     provider "sumologic" {}
   
@@ -426,6 +481,9 @@ data:
         -var="create_fields=${CREATE_FIELDS}" \
         || { echo "Error during applying Terraform changes"; exit 1; }
   
+    # Setup Sumo Logic monitors if enabled
+    bash /etc/terraform/monitors.sh
+  
     # Cleanup env variables
     export SUMOLOGIC_BASE_URL=
     export SUMOLOGIC_ACCESSKEY=

tests/helm/terraform/static/collector_fields.output.yaml

@@ -144,6 +144,61 @@ data:
         }
       }
     }
+  monitors.sh: |
+    #!/bin/bash
+  
+    SUMOLOGIC_ACCESSID=${SUMOLOGIC_ACCESSID:=""}
+    readonly SUMOLOGIC_ACCESSID
+    SUMOLOGIC_ACCESSKEY=${SUMOLOGIC_ACCESSKEY:=""}
+    readonly SUMOLOGIC_ACCESSKEY
+    SUMOLOGIC_BASE_URL=${SUMOLOGIC_BASE_URL:=""}
+    readonly SUMOLOGIC_BASE_URL
+  
+    MONITORS_FOLDER_NAME="Kubernetes"
+    MONITORS_DISABLED="false"
+  
+    # verify if the k8s monitors folder already exists
+    MONITORS_RESPONSE="$(curl -XGET -s \
+            -u "${SUMOLOGIC_ACCESSID}:${SUMOLOGIC_ACCESSKEY}" \
+            "${SUMOLOGIC_BASE_URL}"v1/monitors/search?query=type:folder%20"${MONITORS_FOLDER_NAME}" | jq '.[]' )"
+    readonly MONITORS_RESPONSE
+    MONITORS_ROOT_ID="$(curl -XGET -s \
+            -u "${SUMOLOGIC_ACCESSID}:${SUMOLOGIC_ACCESSKEY}" \
+            "${SUMOLOGIC_BASE_URL}"v1/monitors/root | jq -r '.id' )"
+    readonly MONITORS_ROOT_ID
+  
+    MONITORS_FOLDER_ID="$( echo "${MONITORS_RESPONSE}" | \
+            jq -r "select(.item.name == \"${MONITORS_FOLDER_NAME}\") | select(.item.parentId == \"${MONITORS_ROOT_ID}\") | .item.id" )"
+    readonly MONITORS_FOLDER_ID
+  
+    if [[ -z "${MONITORS_FOLDER_ID}" ]]; then
+      # go to monitors directory
+      cd /monitors || exit 2
+  
+      # Fall back to init -upgrade to prevent:
+      # Error: Inconsistent dependency lock file
+      terraform init -input=false || terraform init -input=false -upgrade
+  
+      # extract environment from SUMOLOGIC_BASE_URL
+      # see: https://help.sumologic.com/APIs/General-API-Information/Sumo-Logic-Endpoints-by-Deployment-and-Firewall-Security
+      SUMOLOGIC_ENV=$( echo "${SUMOLOGIC_BASE_URL}" | sed -E 's/https:\/\/.*(au|ca|de|eu|fed|in|jp|us2)\.sumologic\.com.*/\1/' )
+      if [[ "${SUMOLOGIC_BASE_URL}" == "${SUMOLOGIC_ENV}" ]] ; then
+        SUMOLOGIC_ENV="us1"
+      fi
+  
+      TF_LOG_PROVIDER=DEBUG terraform apply \
+          -auto-approve \
+          -var="access_id=${SUMOLOGIC_ACCESSID}" \
+          -var="access_key=${SUMOLOGIC_ACCESSKEY}" \
+          -var="environment=${SUMOLOGIC_ENV}" \
+          -var="folder=${MONITORS_FOLDER_NAME}" \
+          -var="monitors_disabled=${MONITORS_DISABLED}" \
+          || { echo "Error during applying Terraform monitors."; exit 1; }
+    else
+      echo "The monitors were already installed in ${MONITORS_FOLDER_NAME}."
+      echo "You can (re)install them manually with:"
+      echo "https://github.com/SumoLogic/terraform-sumologic-sumo-logic-monitor/tree/main/monitor_packages/kubernetes"
+    fi
   providers.tf: |-
     provider "sumologic" {}
   
@@ -380,6 +435,9 @@ data:
         -var="create_fields=${CREATE_FIELDS}" \
         || { echo "Error during applying Terraform changes"; exit 1; }
   
+    # Setup Sumo Logic monitors if enabled
+    bash /etc/terraform/monitors.sh
+  
     # Cleanup env variables
     export SUMOLOGIC_BASE_URL=
     export SUMOLOGIC_ACCESSKEY=

tests/helm/terraform/static/conditional_sources.output.yaml

@@ -134,6 +134,61 @@ data:
         }
       }
     }
+  monitors.sh: |
+    #!/bin/bash
+  
+    SUMOLOGIC_ACCESSID=${SUMOLOGIC_ACCESSID:=""}
+    readonly SUMOLOGIC_ACCESSID
+    SUMOLOGIC_ACCESSKEY=${SUMOLOGIC_ACCESSKEY:=""}
+    readonly SUMOLOGIC_ACCESSKEY
+    SUMOLOGIC_BASE_URL=${SUMOLOGIC_BASE_URL:=""}
+    readonly SUMOLOGIC_BASE_URL
+  
+    MONITORS_FOLDER_NAME="Kubernetes"
+    MONITORS_DISABLED="false"
+  
+    # verify if the k8s monitors folder already exists
+    MONITORS_RESPONSE="$(curl -XGET -s \
+            -u "${SUMOLOGIC_ACCESSID}:${SUMOLOGIC_ACCESSKEY}" \
+            "${SUMOLOGIC_BASE_URL}"v1/monitors/search?query=type:folder%20"${MONITORS_FOLDER_NAME}" | jq '.[]' )"
+    readonly MONITORS_RESPONSE
+    MONITORS_ROOT_ID="$(curl -XGET -s \
+            -u "${SUMOLOGIC_ACCESSID}:${SUMOLOGIC_ACCESSKEY}" \
+            "${SUMOLOGIC_BASE_URL}"v1/monitors/root | jq -r '.id' )"
+    readonly MONITORS_ROOT_ID
+  
+    MONITORS_FOLDER_ID="$( echo "${MONITORS_RESPONSE}" | \
+            jq -r "select(.item.name == \"${MONITORS_FOLDER_NAME}\") | select(.item.parentId == \"${MONITORS_ROOT_ID}\") | .item.id" )"
+    readonly MONITORS_FOLDER_ID
+  
+    if [[ -z "${MONITORS_FOLDER_ID}" ]]; then
+      # go to monitors directory
+      cd /monitors || exit 2
+  
+      # Fall back to init -upgrade to prevent:
+      # Error: Inconsistent dependency lock file
+      terraform init -input=false || terraform init -input=false -upgrade
+  
+      # extract environment from SUMOLOGIC_BASE_URL
+      # see: https://help.sumologic.com/APIs/General-API-Information/Sumo-Logic-Endpoints-by-Deployment-and-Firewall-Security
+      SUMOLOGIC_ENV=$( echo "${SUMOLOGIC_BASE_URL}" | sed -E 's/https:\/\/.*(au|ca|de|eu|fed|in|jp|us2)\.sumologic\.com.*/\1/' )
+      if [[ "${SUMOLOGIC_BASE_URL}" == "${SUMOLOGIC_ENV}" ]] ; then
+        SUMOLOGIC_ENV="us1"
+      fi
+  
+      TF_LOG_PROVIDER=DEBUG terraform apply \
+          -auto-approve \
+          -var="access_id=${SUMOLOGIC_ACCESSID}" \
+          -var="access_key=${SUMOLOGIC_ACCESSKEY}" \
+          -var="environment=${SUMOLOGIC_ENV}" \
+          -var="folder=${MONITORS_FOLDER_NAME}" \
+          -var="monitors_disabled=${MONITORS_DISABLED}" \
+          || { echo "Error during applying Terraform monitors."; exit 1; }
+    else
+      echo "The monitors were already installed in ${MONITORS_FOLDER_NAME}."
+      echo "You can (re)install them manually with:"
+      echo "https://github.com/SumoLogic/terraform-sumologic-sumo-logic-monitor/tree/main/monitor_packages/kubernetes"
+    fi
   providers.tf: |-
     provider "sumologic" {}
   
@@ -298,6 +353,9 @@ data:
         -var="create_fields=${CREATE_FIELDS}" \
         || { echo "Error during applying Terraform changes"; exit 1; }
   
+    # Setup Sumo Logic monitors if enabled
+    bash /etc/terraform/monitors.sh
+  
     # Cleanup env variables
     export SUMOLOGIC_BASE_URL=
     export SUMOLOGIC_ACCESSKEY=