Add remote cypress orchestrator integration with integtest script

.github/workflows/remote-cypress-workflow.yml

@@ -18,7 +18,8 @@ on:
       branch_ref:
         description: 'Test branch name or commit reference id'
         required: true
-
+env:
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 jobs:
   trigger-cypress:
     runs-on: ubuntu-latest
@@ -27,10 +28,9 @@ jobs:
     steps:
       - name: Checkout code
         uses: actions/checkout@v2
-
-      - name: Run Bash Script
-        env:
-          GITHUB_SECRET_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Run Remote Cypress Orchestrator Script
+        if: github.event_name == 'workflow_dispatch'
         run: |
+          echo "Executing remoteCypress script for workflow_dispatch event"
           ./remoteCypress.sh -r "${{ github.event.inputs.repo }}" -w "${{ github.event.inputs.workflow_name }}" -o "${{ github.event.inputs.os_url }}" -d "${{ github.event.inputs.osd_url }}" -b "${{ github.event.inputs.branch_ref }}"
-

integtest.sh

@@ -19,11 +19,12 @@ function usage() {
     echo -e "-t TEST_COMPONENTS\t(OpenSearch-Dashboards reportsDashboards etc.), optional, specify test components, separate with space, else test everything."
     echo -e "-v VERSION\t, no defaults, indicates the OpenSearch version to test."
     echo -e "-o OPTION\t, no defaults, determine the TEST_TYPE value among(default, manifest) in test_finder.sh, optional."
+    echo -e "-r REMOTE_CYPRESS_ENABLED\t(true | false), defaults to true. Feature flag set to specify remote cypress orchestrator runs are enabled or not."
     echo -e "-h\tPrint this message."
     echo "--------------------------------------------------------------------------"
 }
 
-while getopts ":hb:p:s:c:t:v:o:" arg; do
+while getopts ":hb:p:s:c:t:v:o:r:" arg; do
     case $arg in
         h)
             usage
@@ -50,6 +51,9 @@ while getopts ":hb:p:s:c:t:v:o:" arg; do
         o)
             OPTION=$OPTARG
             ;;
+        r)
+            REMOTE_CYPRESS_ENABLED=$OPTARG
+            ;;
         :)
             echo "-${OPTARG} requires an argument"
             usage
@@ -78,6 +82,11 @@ then
   SECURITY_ENABLED="true"
 fi
 
+if [ -z "$REMOTE_CYPRESS_ENABLED" ]
+then
+  REMOTE_CYPRESS_ENABLED="true"
+fi
+
 if [ -z "$CREDENTIAL" ]
 then
   # Starting in 2.12.0, security demo configuration script requires an initial admin password
@@ -102,17 +111,111 @@ echo -e "Test Files List:"
 echo $TEST_FILES | tr ',' '\n'
 echo "BROWSER_PATH: $BROWSER_PATH"
 
+# Can be used as Jenkins environment variable set for the orchestrator feature flag, default: true
+ORCHESTRATOR_FEATURE_FLAG=${ORCHESTRATOR_FEATURE_FLAG:-true}
+
+# Read inputs from the manifest file for remote cypress orchestrator
+REMOTE_MANIFEST_FILE="remote_cypress_manifest.json"
+
+# PID file to store remote cypress workflow background processes IDs when run in parallel
+pid_file="process_ids.txt"
+
+run_remote_cypress() {
+    local repo="$1"
+    local workflow_name="$2"
+    local os_url="$3"
+    local osd_url="$4"
+    local branch_ref="$5"
+
+    # Call the remoteCypress.sh script with the required arguments
+    source remoteCypress.sh -r "$repo" -w "$workflow_name" -o "$os_url" -d "$osd_url" -b "$branch_ref" &
+    bg_process_pid=$!
+    echo "PID for the repo $repo is : $bg_process_pid"
+    echo "$bg_process_pid" >> "$pid_file"
+}
+
+# Helper function to extract information from a component and return a tuple
+get_component_info() {
+    local component="$1"
+    local repo workflow_name os_url osd_url branch_ref release_version os arch
+
+    release_version=$(jq -r '.build.version' "$REMOTE_MANIFEST_FILE")
+    repo=$(echo "$component" | jq -r '.["repository"]')
+    workflow_name=$(echo "$component" | jq -r '.["workflow-name"]')
+    os=$(echo "$component" | jq -r '.["operating-system"]')
+    arch=$(echo "$component" | jq -r '.["arch"]')
+    branch_ref=$(echo "$component" | jq -r '.["ref"]')
+
+
+    # Check if OS and Arch are defined
+    if [ -n "$os" ] && [ -n "$arch" ]; then
+        os_url="https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/$release_version/latest/$os/$arch/tar/dist/opensearch/opensearch-$release_version-$os-$arch.tar.gz"
+        osd_url="https://ci.opensearch.org/ci/dbc/distribution-build-opensearch-dashboards/$release_version/latest/$os/$arch/tar/dist/opensearch-dashboards/opensearch-dashboards-$release_version-$os-$arch.tar.gz"
+    else
+        # Default to linux-x64 if not defined in the manifest
+        os_url="https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/$release_version/latest/linux/x64/tar/dist/opensearch/opensearch-$release_version-linux-x64.tar.gz"
+        osd_url="https://ci.opensearch.org/ci/dbc/distribution-build-opensearch-dashboards/$release_version/latest/linux/x64/tar/dist/opensearch-dashboards/opensearch-dashboards-$release_version-linux-x64.tar.gz"
+    fi
+
+    echo "$repo" "$workflow_name" "$os_url" "$osd_url" "$branch_ref"
+}
+
+# Wait for all processes to finish
+wait_file() {
+    while read -r pid; do
+        wait "$pid"
+    done < "$pid_file"
+}
+
+if [[ $REMOTE_CYPRESS_ENABLED = "true" &&  $ORCHESTRATOR_FEATURE_FLAG = 'true' ]]; then
+    # Parse the JSON file to iterate over the components array
+    components=$(jq -c '.components[]' "$REMOTE_MANIFEST_FILE")
+    echo "Components: $components"
+
+
+    for component in $components; do
+        read -r repo workflow_name os_url osd_url branch_ref <<< "$(get_component_info "$component")"
+
+        echo "Processing for the component: $component"
+        echo "repo: $repo"
+        echo "workflow_name: $workflow_name"
+        echo "os_url: $os_url"
+        echo "osd_url: $osd_url"
+        echo "branch_ref: $branch_ref"
+
+        # Call the function for each component
+        run_remote_cypress "$repo" "$workflow_name" "$os_url" "$osd_url" "$branch_ref" 
+    done
+
+    wait_file
+    log_directory="/tmp/logfiles"
+
+    # Read log files in tmp folder and put the output to CI
+    find "$log_directory" -type f -name "*.txt" | while IFS= read -r log_file; do
+        if [ -f "$log_file" ]; then
+            echo "Log content for file: $log_file"
+            cat "$log_file"
+        else
+            echo "Log file not found: $log_file"
+        fi
+    done
+
+    # Delete the temporary log files and folder after writing to CI
+    rm -rf "$log_directory"
+fi
+rm "$pid_file"
+
 ## WARNING: THIS LOGIC NEEDS TO BE THE LAST IN THIS FILE! ##
 # Cypress returns back the test failure count in the error code
 # The CI outputs the error code as test failure count.
 #
 # We need to ensure the cypress tests are the last execute process to
 # the error code gets passed to the CI.
-if [ $SECURITY_ENABLED = "true" ]
-then
-   echo "run security enabled tests"
-   yarn cypress:run-with-security --browser "$BROWSER_PATH" --spec "$TEST_FILES"
+
+if [ $SECURITY_ENABLED = "true" ]; then
+    echo "Running security enabled tests"
+    yarn cypress:run-with-security --browser "$BROWSER_PATH" --spec "$TEST_FILES"
 else
-   echo "run security disabled tests"
-   yarn cypress:run-without-security --browser "$BROWSER_PATH" --spec "$TEST_FILES"
-fi
+    echo "Running security disabled tests"
+    yarn cypress:run-without-security --browser "$BROWSER_PATH" --spec "$TEST_FILES"
+fi

poll_remote_workflow.sh

@@ -1,10 +1,11 @@
 #!/bin/bash
 
-# Accessing the secret as an environment variable using GitHub actions while invoking this script
-GITHUB_TOKEN=$GITHUB_SECRET_TOKEN
+set -e
+
 REPO="$1"
 UNIQUE_WORKFLOW_ID="$2"
 API_URL="$3"
+exitcode=2
 
 # Function to check the status of the remote github workflow by constantly polling the workflow-run
 check_remote_workflow_status() {
@@ -52,20 +53,22 @@ check_remote_workflow_status() {
                             -H "Accept: application/vnd.github.v3+json" \
                             -H "X-GitHub-Api-Version: 2022-11-28" \
                             "https://api.github.com/repos/$REPO/actions/runs/$run_id")
-            echo "Workflow run details: $run_details" 
-
+
             # Extract status and conclusion from the run details
             status=$(echo "$run_details" | jq -r ".status")
             conclusion=$(echo "$run_details" | jq -r ".conclusion")
 
+            echo "Workflow run status: $status" 
+
             # Check if the status indicates that the workflow is complete
             if [[ "$status" == "completed" ]]; then
                 echo "Workflow completed with status: $status"
 
                 # Check if it was successful
                 if [[ $conclusion == "success" ]]; then
                     echo "Remote workflow completed successfully."
-                    return 0  # Success
+                    exitcode=0  # Success
+                    break;
                 elif [[ $conclusion == "failure" ]]; then
                     echo "Remote workflow completed with errors. Conclusion: $conclusion"
 
@@ -75,13 +78,15 @@ check_remote_workflow_status() {
                             "$jobs_url")
 
                     # Parse the workflow to find any failures in the test
-                    failures=$(echo "$run_details" | jq -r '.jobs[] | select(.conclusion == "failure") | .name')
+                    failures=$(echo "$job_details" | jq -r '.jobs[] | select(.conclusion == "failure") | .name')
                     echo "Test failures: $failures"
 
-                    return 1  # Failure
+                    exitcode=1  # Failure
+                    break;
                 else
                     echo "Remote workflow completed with unexpected conclusion. Conclusion: $conclusion"
-                    return 1  # Failure
+                    exitcode=1  # Failure
+                    break;
                 fi
             else
                 echo "Remote workflow is still running. Waiting..."
@@ -91,13 +96,12 @@ check_remote_workflow_status() {
         done
     else
         echo "No matching workflow run object found even after retries. Exiting..."
-        return 1  # Failure
+        exitcode=1 # Failure  
     fi
 
-    echo "Remote workflow didn't complete within the specified time."
-    return 1  # Failure
+    if [ "$exitcode" -eq 2 ]; then
+        echo "Remote workflow didn't complete within the specified time."
+    fi
 }
 
 check_remote_workflow_status
-
-exit 0

remoteCypress.sh

@@ -53,61 +53,55 @@ while getopts ":h:r:w:o:d:b:i:" opt; do
   esac
 done
 
-# Check if required arguments are provided
-if [[ -z "$REPO" || -z "$WORKFLOW_NAME" || -z "$OS_URL" || -z "$OSD_URL" || -z "$BRANCH_REF" ]]; then
-    echo "Error: Missing required arguments. See usage below."
-    usage
-    exit 1
-fi
+log_directory="/tmp/logfiles/${REPO}" 
+mkdir -p "$log_directory"
+log_file="$log_directory/logfile.txt"
 
-# Accessing the secret as an environment variable using Github actions while invoking this script
-GITHUB_TOKEN=$GITHUB_SECRET_TOKEN
-# This is to uniquely identify each execution workflow. This ID has to be appended to the workflow_run 
-# name in the component repository yaml file for polling purpose. 
-UNIQUE_WORKFLOW_ID=$(uuidgen)
-echo "Unique Execution ID: $UNIQUE_WORKFLOW_ID"
-# For now we are using Github action API to trigger github workflows in plugins component
-# ToDo: We can explore other test runners such as Jenkins to integrate with.
-API_URL="https://api.github.com/repos/$REPO/actions/workflows/$WORKFLOW_NAME"
-PAYLOAD="{\"ref\": \"$BRANCH_REF\",\"inputs\":{\"build_id\":\"$BUILD_ID\", \"OS_URL\":\"$OS_URL\", \"OSD_URL\":\"$OSD_URL\", \"UNIQUE_ID\":\"$UNIQUE_WORKFLOW_ID\"}}" 
-
-# Maximum number of retries for triggering the remote runner
-MAX_RETRIES=3
-
-# Trigger the remote GitHub workflow using curl and the PAT token
-trigger_remote_workflow() {
-    curl -L -X POST -H "Authorization: Bearer $GITHUB_TOKEN" \
-        -H "Accept: application/vnd.github.v3+json" \
-        -H "X-GitHub-Api-Version: 2022-11-28" \
-        -w "%{http_code}" \
-        "$API_URL/dispatches" -d "$PAYLOAD"
-}
-
-echo "Triggering the remote GitHub workflow for Cypress tests in the repository: $REPO"
+# Check if required arguments are provided
+if [[ -n "$REPO" && -n "$WORKFLOW_NAME" && -n "$OS_URL" && -n "$OSD_URL" && -n "$BRANCH_REF" ]]; then
+  # This is to uniquely identify each execution workflow. This ID has to be appended to the workflow_run 
+  # name in the component repository yaml file for polling purpose. 
+  UNIQUE_WORKFLOW_ID=$(uuidgen)
+  echo "Unique Execution ID: $UNIQUE_WORKFLOW_ID"
+  # For now we are using Github action API to trigger github workflows in plugins component
+  # ToDo: We can explore other test runners such as Jenkins to integrate with.
+  API_URL="https://api.github.com/repos/$REPO/actions/workflows/$WORKFLOW_NAME"
+  PAYLOAD="{\"ref\": \"$BRANCH_REF\",\"inputs\":{\"build_id\":\"$BUILD_ID\", \"OS_URL\":\"$OS_URL\", \"OSD_URL\":\"$OSD_URL\", \"UNIQUE_ID\":\"$UNIQUE_WORKFLOW_ID\"}}" 
 
-# Attempt to trigger the remote workflow with retries
-for ((i = 1; i <= MAX_RETRIES; i++)); do
-    echo "Attempting to trigger the remote workflow (Attempt $i)"
-    status_code=$(trigger_remote_workflow)
-    echo "status_code: $status_code"
+  # Trigger the remote GitHub workflow using curl and the PAT token
+  trigger_remote_workflow() {
+      curl -L -X POST -H "Authorization: Bearer $GITHUB_TOKEN" \
+          -H "Accept: application/vnd.github.v3+json" \
+          -H "X-GitHub-Api-Version: 2022-11-28" \
+          -w "%{http_code}" \
+          "$API_URL/dispatches" -d "$PAYLOAD"
+  }
 
-    if [[ $status_code -ge 200 && $status_code -lt 300 ]]; then
-        echo "Remote workflow triggered successfully."
-        break
-    else
-        echo "Failed to trigger the remote workflow. Retrying..."
-        sleep 10  # Adds a delay between retries
-    fi
+  echo "Triggering the remote GitHub workflow for Cypress tests in the repository: $REPO"
 
-    if [ $i -eq $MAX_RETRIES ]; then
-        echo "Maximum number of retries reached. Exiting."
-        exit 1
-    fi
-done
+  status_code=$(trigger_remote_workflow)
+  echo "status_code: $status_code"
 
+  if [[ $status_code -ge 200 && $status_code -lt 300 ]]; then
+      echo "Remote workflow triggered successfully."
+
+      source poll_remote_workflow.sh "$REPO" "$UNIQUE_WORKFLOW_ID" "$API_URL" > "$log_file" 2>&1
+      echo "Return code: $exitcode"
 
-# Check the status of the remote workflow
-source ./poll_remote_workflow.sh "$REPO" "$UNIQUE_WORKFLOW_ID" "$API_URL"
-echo "Return code: $?"
+      if [ "$exitcode" -eq 0 ]; then
+        echo "Remote workflow for the repo $REPO completed successfully. EXIT CODE 0 " >> "$log_file"
+      elif [ "$exitcode" -eq 1 ]; then
+        echo "Remote workflow for the repo $REPO completed with errors. EXIT CODE 1 " >> "$log_file"
+      else
+        echo "Remote workflow for the repo $REPO did not complete within the specified time. EXIT CODE 2 " >> "$log_file"
+      fi
 
-exit 0
+  else
+      echo "Failed to trigger the remote workflow. Exiting."
+      echo "Failed to trigger the remote workflow for repo $REPO : EXIT CODE 1 " >> "$log_file"
+  fi
+else 
+    echo "Error: Missing required arguments. See usage below."
+    usage
+    echo "Remote workflow for the repo $REPO did not start due to missing arguments. EXIT CODE 1 " >> "$log_file"
+fi

remote_cypress_manifest.json

@@ -0,0 +1,31 @@
+{
+  "schema-version": "1.0",
+  "build": {
+    "name": "OpenSearch Dashboards Functional Test",
+    "version": "3.0.0"
+  },
+  "ci": {
+    "image": {
+    }
+  },
+  "components": [
+    {
+      "name": "OpenSearch-Dashboards",
+      "repository": "opensearch-project/OpenSearch-Dashboards",
+      "workflow-name": "",
+      "ref": "",
+      "operating-system": "linux",
+      "arch": "x64",
+      "integ-test": {
+        "test-configs": [
+          "with-security",
+          "without-security"
+        ],
+        "additional-cluster-configs": {
+          "vis_builder.enabled": true,
+          "data_source.enabled": true
+        }
+      }
+    }
+  ]
+}