Fix Metrics Collector error in case of non-existing Process

pkg/metricscollector/v1beta1/common/pns.go

@@ -26,6 +26,7 @@ import (
 	"time"
 
 	psutil "github.com/shirou/gopsutil/process"
+	"k8s.io/klog"
 )
 
 // WaitPidsOpts is the input options for metrics collector
@@ -40,7 +41,7 @@ type WaitPidsOpts struct {
 func WaitMainProcesses(opts WaitPidsOpts) error {
 
 	if runtime.GOOS != "linux" {
-		return fmt.Errorf("Platform '%s' unsupported", runtime.GOOS)
+		return fmt.Errorf("platform '%s' unsupported", runtime.GOOS)
 	}
 
 	pids, mainPid, err := GetMainProcesses(opts.CompletedMarkedDirPath)
@@ -59,21 +60,23 @@ func GetMainProcesses(completedMarkedDirPath string) (map[int]bool, int, error)
 	mainPid := 0
 
 	if err != nil {
-		return nil, 0, fmt.Errorf("Failed to list processes: %v", err)
+		return nil, 0, fmt.Errorf("failed to list processes: %v", err)
 	}
 
 	thisPID := os.Getpid()
 	for _, pid := range allPids {
 		// Create process object from pid
 		proc, err := psutil.NewProcess(pid)
 		if err != nil {
-			return nil, 0, fmt.Errorf("Failed to create new Process from pid %v, error: %v", pid, err)
+			klog.Infof("Unable to create new process from pid: %v, error: %v. Continue to next pid", pid, err)
+			continue
 		}
 
 		// Get parent process
 		ppid, err := proc.Ppid()
 		if err != nil {
-			return nil, 0, fmt.Errorf("Unable to get parent pid for pid: %v, error: %v", ppid, err)
+			klog.Infof("Unable to get parent process for pid: %v, error: %v. Continue to next pid", pid, err)
+			continue
 		}
 
 		// Ignore the pause container, our own pid, and non-root processes (parent pid != 0)
@@ -84,11 +87,12 @@ func GetMainProcesses(completedMarkedDirPath string) (map[int]bool, int, error)
 		// Read the process command line
 		cmdline, err := proc.Cmdline()
 		if err != nil {
-			return nil, 0, fmt.Errorf("Unable to get cmdline from pid %v, error: %v", pid, err)
+			klog.Infof("Unable to get cmdline from pid: %v, error: %v. Continue to next pid", pid, err)
+			continue
 		}
 
 		// By default mainPid is the first process.
-		// Command line contains completed marker for the main pid
+		// In addition to that, command line contains completed marker for the main pid
 		// For example: echo completed > /var/log/katib/$$$$.pid
 		// completedMarkedDirPath is the directory for completed marker, e.g. /var/log/katib
 		if mainPid == 0 ||
@@ -99,6 +103,11 @@ func GetMainProcesses(completedMarkedDirPath string) (map[int]bool, int, error)
 		pids[int(pid)] = true
 	}
 
+	// If mainPid has not been found, return an error.
+	if mainPid == 0 {
+		return nil, 0, fmt.Errorf("unable to find main pid from the process list %v", allPids)
+	}
+
 	return pids, mainPid, nil
 }
 
@@ -132,7 +141,7 @@ func WaitPIDs(pids map[int]bool, mainPid int, opts WaitPidsOpts) error {
 							// Read file with "completed" marker
 							contents, err := ioutil.ReadFile(markFile)
 							if err != nil {
-								return fmt.Errorf("Training container is failed. Unable to read file %v for pid %v, error: %v", markFile, pid, err)
+								return fmt.Errorf("training container is failed. Unable to read file %v for pid %v, error: %v", markFile, pid, err)
 							}
 							// Check if file contains "early-stopped" marker
 							// In that case process is not completed
@@ -141,7 +150,7 @@ func WaitPIDs(pids map[int]bool, mainPid int, opts WaitPidsOpts) error {
 							}
 							// Check if file contains "completed" marker
 							if strings.TrimSpace(string(contents)) != TrainingCompleted {
-								return fmt.Errorf("Unable to find marker: %v in file: %v with contents: %v for pid: %v",
+								return fmt.Errorf("unable to find marker: %v in file: %v with contents: %v for pid: %v",
 									TrainingCompleted, markFile, string(contents), pid)
 							}
 						}
@@ -157,7 +166,7 @@ func WaitPIDs(pids map[int]bool, mainPid int, opts WaitPidsOpts) error {
 					}
 					// We should receive only not exist error when we check /proc/<pid> dir
 				} else {
-					return fmt.Errorf("Fail to check process info: %v", err)
+					return fmt.Errorf("fail to check process info: %v", err)
 				}
 			}
 		}
@@ -167,7 +176,7 @@ func WaitPIDs(pids map[int]bool, mainPid int, opts WaitPidsOpts) error {
 
 	// After main loop notFinishedPids map should be empty
 	if len(notFinishedPids) != 0 {
-		return fmt.Errorf("Timed out waiting for pids to complete")
+		return fmt.Errorf("timed out waiting for pids to complete")
 	}
 	return nil
 }

pkg/metricscollector/v1beta1/common/pns.py

@@ -55,14 +55,18 @@ def GetMainProcesses(completed_marked_dir):
         cmd_lind = " ".join(proc.cmdline())
 
         # By default main_pid is the first process.
-        # Command line contains completed marker for the main pid
+        # In addition to that, command line contains completed marker for the main pid.
         # For example: echo completed > /var/log/katib/$$$$.pid
         # completed_marked_dir is the directory for completed marker, e.g. /var/log/katib
         if main_pid == 0 or ("echo {} > {}".format(const.TRAINING_COMPLETED, completed_marked_dir) in cmd_lind):
             main_pid = pid
 
         pids.add(pid)
 
+    # If mainPid has not been found, return an error.
+    if main_pid == 0:
+        raise Exception("Unable to find main pid from the process list {}".format(pids))
+
     return pids, main_pid