[minor] Gitops updates to support junitreporter + timeouts in FVT

image/cli/app-root/src/junit-xml-generator.py

@@ -0,0 +1,44 @@
+#!/usr/bin/env python3
+# *****************************************************************************
+# Copyright (c) 2024 IBM Corporation and other Contributors.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Eclipse Public License v1.0
+# which accompanies this distribution, and is available at
+# http://www.eclipse.org/legal/epl-v10.html
+#
+# *****************************************************************************
+import argparse
+import os
+from junit_xml import TestSuite, TestCase
+
+#
+# The test-cases is a command seperated list of testcasename:timetaken
+# Execute: 
+#  `python junit-xml-generator.py --test-suite-name testsuite1 --test-cases operator-catalog:54,secret-update:9 --output-dir .`
+# 
+#
+if __name__ == "__main__":
+    # Initialize the properties we need
+    parser = argparse.ArgumentParser()
+
+    # Primary Options
+    parser.add_argument("--test-suite-name", required=True)
+    parser.add_argument("--test-cases", required=True)
+    parser.add_argument("--output-dir", required=True)
+
+    args, unknown = parser.parse_known_args()
+
+    test_cases_dict = []
+    test_case_list = args.test_cases.split(',')
+    for test_case in test_case_list:
+        test_case_name = test_case.split(':')[0]
+        test_case_time = int(test_case.split(':')[1])
+        print(f"Adding test_case: {test_case_name} with elapsed time {test_case_time}")
+        test_cases_dict.append(TestCase(test_case_name, test_case_name, test_case_time))
+
+    print(f"Creating testsuite : {args.test_suite_name}")
+    ts = TestSuite(args.test_suite_name, test_cases_dict)
+
+    with open(os.path.join(args.output_dir, 'output.xml'), 'w') as f:
+        TestSuite.to_file(f, [ts], prettyprint=False)

image/cli/app-root/src/register-start.py

@@ -25,11 +25,12 @@
     instanceId = os.getenv("DEVOPS_ENVIRONMENT", "none")
     build = os.getenv("DEVOPS_BUILD_NUMBER")
     suite = os.getenv("DEVOPS_SUITE_NAME", "")
+    productId = os.getenv("PRODUCT_ID", "ibm-mas-devops")
 
-    productId = "ibm-mas-devops"
     channelId = "n/a"
     cliVersion = os.getenv("VERSION", "unknown")
     ansibleDevopsVersion = os.getenv("ANSIBLE_DEVOPS_VERSION", "unknown")
+    gitopsVersion = os.getenv("GITOPS_VERSION", "unknown")
 
     if suite == "":
         print ("Results not recorded because DEVOPS_SUITE_NAME is not defined")
@@ -52,6 +53,7 @@
 
     print(f"CLI Version ............ {cliVersion}")
     print(f"mas_devops Version ..... {ansibleDevopsVersion}")
+    print(f"gitops Version ......... {gitopsVersion}")
 
     print(f"Run ID ................. {runId}")
     print(f"Result ID .............. {resultId}")
@@ -84,11 +86,12 @@
                 }
             },
             '$set': {
-                f"products.ibm-mas-devops.productId": productId,
-                f"products.ibm-mas-devops.channelId": channelId,
-                f"products.ibm-mas-devops.version": cliVersion,
-                f"products.ibm-mas-devops.ansibleDevopsVersion": ansibleDevopsVersion,
-                f"products.ibm-mas-devops.results.{suite}": suiteSummary
+                f"products.{productId}.productId": productId,
+                f"products.{productId}.channelId": channelId,
+                f"products.{productId}.version": cliVersion,
+                f"products.{productId}.ansibleDevopsVersion": ansibleDevopsVersion,
+                f"products.{productId}.gitopsVersion": gitopsVersion,
+                f"products.{productId}.results.{suite}": suiteSummary
             }
         },
         upsert=True

image/cli/app-root/src/save-junit-to-mongo.py

@@ -26,8 +26,8 @@
     build = os.getenv("DEVOPS_BUILD_NUMBER")
     suite = os.getenv("DEVOPS_SUITE_NAME", "")
     junitOutputDir = os.getenv("JUNIT_OUTPUT_DIR", "/tmp")
+    productId = os.getenv("PRODUCT_ID", "ibm-mas-devops")
 
-    productId = "ibm-mas-devops"
     channelId = "n/a"
     cliVersion = os.getenv("VERSION", "unknown")
     ansibleDevopsVersion = os.getenv("ANSIBLE_DEVOPS_VERSION", "unknown")
@@ -71,7 +71,18 @@
         bf = Yahoo(dict_type=dict)
         resultDoc = bf.data(root)
 
-        for testcase in resultDoc["testsuites"]["testsuite"]["testcase"]:
+        if isinstance(resultDoc["testsuites"]["testsuite"]["testcase"], list) :
+            for testcase in resultDoc["testsuites"]["testsuite"]["testcase"]:
+                testcase["name"] = testcase["name"].replace("[localhost] localhost: ", "")
+                # Playbooks don't have ibm/mas_devops in the classname but do have /opt/app-root.
+                # Roles have both ibm/mas_devops and /opt/app-root.
+                # Guard against both and remove when required.
+                if "/opt/app-root/" in testcase["classname"]:
+                    testcase["classname"] = testcase["classname"].split("/opt/app-root/")[1]
+                if "ibm/mas_devops/" in testcase["classname"]:
+                    testcase["classname"] = testcase["classname"].split("ibm/mas_devops/")[1]
+        else:
+            testcase = resultDoc["testsuites"]["testsuite"]["testcase"]
             testcase["name"] = testcase["name"].replace("[localhost] localhost: ", "")
             # Playbooks don't have ibm/mas_devops in the classname but do have /opt/app-root.
             # Roles have both ibm/mas_devops and /opt/app-root.
@@ -121,11 +132,11 @@
                     }
                 },
                 '$set': {
-                    f"products.ibm-mas-devops.productId": productId,
-                    f"products.ibm-mas-devops.channelId": channelId,
-                    f"products.ibm-mas-devops.version": cliVersion,
-                    f"products.ibm-mas-devops.ansibleDevopsVersion": ansibleDevopsVersion,
-                    f"products.ibm-mas-devops.results.{suite}": suiteSummary
+                    f"products.{productId}.productId": productId,
+                    f"products.{productId}.channelId": channelId,
+                    f"products.{productId}.version": cliVersion,
+                    f"products.{productId}.ansibleDevopsVersion": ansibleDevopsVersion,
+                    f"products.{productId}.results.{suite}": suiteSummary
                 }
             },
             upsert=True

image/cli/mascli/functions/gitops_cluster

@@ -58,8 +58,12 @@ Cluster Promotion (Optional):
       --cluster-promotion-target-pr-title ${COLOR_YELLOW}CLUSTER_PROMOTION_TARGET_PR_TITLE${TEXT_RESET}         The title of the PR if a PR is to be created
       --cluster-promotion-cluster-values ${COLOR_YELLOW}CLUSTER_PROMOTION_CLUSTER_VALUES${TEXT_RESET}           A list of values in json format to update in the target git repo.
 
-Notifications:
-      --slack-channel-id ${COLOR_YELLOW}SLACK_CHANNEL_ID${TEXT_RESET} Slack channel for ArgoCD to notify when an app sync has completed or failed
+DevOps Details (Optional):
+      --devops-build-number ${COLOR_YELLOW}DEVOPS_BUILD_NUMBER${TEXT_RESET}                                     The build number to associate with the junitreporter for each argocd app sync
+      --devops-mongo-uri ${COLOR_YELLOW}DEVOPS_MONGO_URI${TEXT_RESET}                                           The Mongo uri used by the junitreporter to store test/run records
+
+Notifications (Optional):
+      --slack-channel-id ${COLOR_YELLOW}SLACK_CHANNEL_ID${TEXT_RESET}                                           Slack channel for ArgoCD to notify when an app sync has completed or failed
 
 Other Commands:
   -h, --help                              Show this help message
@@ -200,6 +204,15 @@ function gitops_cluster_noninteractive() {
         export CLUSTER_PROMOTION_CLUSTER_VALUES=$1 && shift
         ;;
 
+
+      # DevOps Details
+      --devops-build-number)
+        export DEVOPS_BUILD_NUMBER=$1 && shift
+        ;;
+      --devops-mongo-uri)
+        export DEVOPS_MONGO_URI=$1 && shift
+        ;;
+
       # IBM CIS Cert Manager 
       --dns-provider)
         export DNS_PROVIDER=$1 && shift
@@ -241,6 +254,10 @@ function gitops_cluster_noninteractive() {
 
   [[ -z "$SECRETS_PATH" ]] && gitops_cluster_help "SECRETS_PATH is not set"
 
+  if [[ -n "$DEVOPS_MONGO_URI" ]]; then
+    [[ -z "$DEVOPS_BUILD_NUMBER" ]] && gitops_cluster_help "DEVOPS_BUILD_NUMBER is not set"
+  fi
+
   if [[ "$CLUSTER_PROMOTION" == "true" ]]; then
     [[ -z "$CLUSTER_PROMOTION_TARGET_GITHUB_PAT" ]] && gitops_cluster_help "CLUSTER_PROMOTION_TARGET_GITHUB_PAT is not set"
     [[ -z "$CLUSTER_PROMOTION_TARGET_GITHUB_HOST" ]] && gitops_cluster_help "CLUSTER_PROMOTION_TARGET_GITHUB_HOST is not set"
@@ -354,6 +371,12 @@ function gitops_cluster() {
     reset_colors
   fi
 
+  echo "${TEXT_DIM}"
+  echo_h2 "DevOps Details" "    "
+  echo_reset_dim "Devops Build Number ............. ${COLOR_MAGENTA}${DEVOPS_BUILD_NUMBER}"
+  echo_reset_dim "Devops Mongo Uri ................ ${COLOR_MAGENTA}${DEVOPS_MONGO_URI:0:8}<snip>"
+  reset_colors
+
   # Set up secrets
   # ---------------------------------------------------------------------------
   echo
@@ -367,6 +390,15 @@ function gitops_cluster() {
   export SECRET_NAME_CIS=${ACCOUNT_ID}${SM_DELIM}${CLUSTER_ID}${SM_DELIM}cis
   export SECRET_KEY_IBM_APIKEY=${SECRET_NAME_CIS}#ibm_apikey
 
+  export SECRET_NAME_DEVOPS_MONGO=${ACCOUNT_ID}${SM_DELIM}${CLUSTER_ID}${SM_DELIM}devops
+  export SECRET_KEY_DEVOPS_MONGO=${SECRET_NAME_DEVOPS_MONGO}#devops_mongo_uri
+
+  if [ -n "$DEVOPS_MONGO_URI" ];then
+    echo "- Update DEVOPS_MONGO_URI secret"
+    TAGS="[{\"Key\": \"source\", \"Value\": \"gitops_cluster\"}, {\"Key\": \"account\", \"Value\": \"${ACCOUNT_ID}\"}, {\"Key\": \"cluster\", \"Value\": \"${CLUSTER_ID}\"}]"
+    sm_update_secret $SECRET_NAME_DEVOPS_MONGO "{\"devops_mongo_uri\": \"${DEVOPS_MONGO_URI}\"}" "${TAGS}"
+  fi
+
   if [ -z $GIT_SSH ]; then
     export GIT_SSH="false"
   fi

image/cli/mascli/functions/gitops_db2u

@@ -127,7 +127,7 @@ function gitops_db2u_noninteractive() {
   [[ -z "$ACCOUNT_ID" ]] && gitops_db2u_help "ACCOUNT_ID is not set"
   [[ -z "$REGION" ]] && gitops_db2u_help "REGION is not set"
   [[ -z "$CLUSTER_ID" ]] && gitops_db2u_help "CLUSTER_ID is not set"
-  [[ -z "$MAS_INSTANCE_ID" ]] && gitops_cluster_help "MAS_INSTANCE_ID is not set"
+  [[ -z "$MAS_INSTANCE_ID" ]] && gitops_db2u_help "MAS_INSTANCE_ID is not set"
 
 
   if [[ "$GITHUB_PUSH" == "true" ]]; then

image/cli/mascli/functions/gitops_dro

@@ -140,7 +140,7 @@ function gitops_dro_noninteractive() {
   [[ -z "$GITOPS_WORKING_DIR" ]] && gitops_dro_help "GITOPS_WORKING_DIR is not set"
   [[ -z "$ACCOUNT_ID" ]] && gitops_dro_help "ACCOUNT_ID is not set"
   [[ -z "$CLUSTER_ID" ]] && gitops_dro_help "CLUSTER_ID is not set"
-  [[ -z "$REGION_ID" && -z "$SM_AWS_REGION" ]] && gitops_cluster_help "REGION_ID or SM_AWS_REGION is not set"
+  [[ -z "$REGION_ID" && -z "$SM_AWS_REGION" ]] && gitops_dro_help "REGION_ID or SM_AWS_REGION is not set"
 
   if [[ "$GITHUB_PUSH" == "true" ]]; then
     [[ -z "$GITHUB_HOST" ]] && gitops_dro_help "GITHUB_HOST is not set"

image/cli/mascli/functions/gitops_utils

@@ -833,10 +833,14 @@ function check_argo_app_synced() {
 
 function check_argo_app_healthy() {
   APPLICATION=$1
-  CLUSTER_WATCHER=$2
+  MAX_CHECKS=$2
   COUNT=0
   while true; do
-    echo "argo:check_argo_app_healthy : Checking health status for $APPLICATION"
+    if [[ -n "$MAX_CHECKS" ]]; then
+      echo "argo:check_argo_app_healthy : Checking health status for $APPLICATION up to $MAX_CHECKS times"
+    else
+      echo "argo:check_argo_app_healthy : Checking health status for $APPLICATION with no timeout"
+    fi
     HEALTH_STATUS=$(argocd app get $APPLICATION -o json 2> /dev/null | jq -r .status.health.status)
     if [ "$HEALTH_STATUS" == "Healthy" ]; then
       echo "Health Status is Healthy"
@@ -845,10 +849,10 @@ function check_argo_app_healthy() {
       ((COUNT++))
       echo "argo:check_argo_app_healthy : Health Status is $HEALTH_STATUS, Waiting 30s before checking status again - $COUNT"
       sleep 30
-      if ! (( $COUNT % 5 )) ; then
-        if [[ -n "$CLUSTER_WATCHER" ]]; then
-          # sync watcher as sometimes current application health status not reflected rightly (like returned empty status)
-          argocd_sync $CLUSTER_WATCHER
+      if [[ -n "$MAX_CHECKS" ]]; then
+        if [[ $COUNT -eq $MAX_CHECKS ]]; then
+          echo "argo:check_argo_app_healthy : App not healthy after $MAX_CHECKS checks, exiting"
+          exit 1
         fi
       fi
     fi

image/cli/mascli/templates/gitops/appset-configs/cluster/phase1/ibm-mas-cluster-base.yaml.j2

@@ -21,3 +21,9 @@ notifications:
 custom_labels:
   {{ CUSTOM_LABELS | indent(2) }}
 {% endif %}
+
+{% if DEVOPS_MONGO_URI is defined and DEVOPS_MONGO_URI !='' %}
+devops:
+  mongo_uri: "<path:{{ SECRETS_PATH }}:{{ SECRET_KEY_DEVOPS_MONGO }}>"
+  build_number: {{ DEVOPS_BUILD_NUMBER }}
+{% endif %}

image/cli/mascli/templates/gitops/bootstrap/configmap.yaml

@@ -18,6 +18,11 @@ data:
             - sh
             - "-c"
             - "find . -name 'Chart.yaml' && find . -name 'values.yaml'"
+      init:
+        command:
+          - sh
+          - "-c"
+          - helm dependency update;
       generate:
         command:
         - bash