Exit 1 on regression detection

orion.py

@@ -114,7 +114,7 @@ def cmd_analysis(**kwargs):
     logger_instance = SingletonLogger(debug=level, name="Orion")
     logger_instance.info("🏹 Starting Orion in command-line mode")
     kwargs["configMap"] = load_config(kwargs["config"])
-    output = run(**kwargs)
+    output, regression_flag = run(**kwargs)
     if output is None:
         logger_instance.error("Terminating test")
         sys.exit(0)
@@ -126,6 +126,8 @@ def cmd_analysis(**kwargs):
         output_file_name = f"{kwargs['save_output_path'].split('.')[0]}_{test_name}.{kwargs['save_output_path'].split('.')[1]}"
         with open(output_file_name, 'w', encoding="utf-8") as file:
             file.write(str(result_table))
+    if regression_flag:
+        sys.exit(1)
 
 
 

pkg/algorithms/algorithm.py

@@ -30,8 +30,9 @@ def __init__(  # pylint: disable = too-many-arguments
         self.test = test
         self.options = options
         self.metrics_config = metrics_config
+        self.regression_flag = False
 
-    def output_json(self) -> Tuple[str, str]:
+    def output_json(self) -> Tuple[str, str, bool]:
         """Method to output json output
 
         Returns:
@@ -64,9 +65,9 @@ def output_json(self) -> Tuple[str, str]:
                     ] = percentage_change
                     dataframe_json[index]["is_changepoint"] = True
 
-        return self.test["name"], json.dumps(dataframe_json, indent=2)
+        return self.test["name"], json.dumps(dataframe_json, indent=2), self.regression_flag
 
-    def output_text(self) -> Tuple[str,str]:
+    def output_text(self) -> Tuple[str,str, bool]:
         """Outputs the data in text/tabular format"""
         series, change_points_by_metric = self._analyze()
         change_points_by_time = self.group_change_points_by_time(
@@ -76,9 +77,9 @@ def output_text(self) -> Tuple[str,str]:
         output_table = report.produce_report(
             test_name=self.test["name"], report_type=ReportType.LOG
         )
-        return self.test["name"], output_table
+        return self.test["name"], output_table, self.regression_flag
 
-    def output_junit(self) -> Tuple[str,str]:
+    def output_junit(self) -> Tuple[str,str, bool]:
         """Output junit format
 
         Returns:
@@ -92,7 +93,7 @@ def output_junit(self) -> Tuple[str,str]:
             metrics_config=self.metrics_config,
             options=self.options,
         )
-        return test_name, data_junit
+        return test_name, data_junit, self.regression_flag
 
     @abstractmethod
     def _analyze(self):

pkg/algorithms/edivisive/edivisive.py

@@ -25,5 +25,6 @@ def _analyze(self):
                 if ((self.metrics_config[metric]["direction"] == 1 and changepoint_list[i].stats.mean_1 > changepoint_list[i].stats.mean_2) or
                     (self.metrics_config[metric]["direction"] == -1 and changepoint_list[i].stats.mean_1 < changepoint_list[i].stats.mean_2) ):
                     del changepoint_list[i]
-
+        if [val for li in change_points_by_metric.values() for val in li]:
+            self.regression_flag=True
         return series, change_points_by_metric

pkg/algorithms/isolationforest/isolationForest.py

@@ -70,5 +70,6 @@ def _analyze(self):
                                                            pvalue=1
                                                        ))
                             change_points_by_metric[feature].append(change_point)
-
+        if [val for li in change_points_by_metric.values() for val in li]:
+            self.regression_flag=True
         return series, change_points_by_metric

pkg/daemon.py

@@ -53,7 +53,8 @@ async def daemon_changepoint( # pylint: disable = R0913
     filter_changepoints = (
         True if filter_changepoints == "true" else False  # pylint: disable = R1719
     )
-    result = runTest.run(**option_arguments)
+    result, _ = runTest.run(**option_arguments)
+    result = {k:json.loads(v) for k,v in result.items()}
     if result is None:
         return {"Error":"No UUID with given metadata"}
     result = {k:json.loads(v) for k,v in result.items()}
@@ -129,7 +130,8 @@ async def daemon_anomaly( # pylint: disable = R0913, R0914
     filter_points = (
         True if filter_points == "true" else False  # pylint: disable = R1719
     )
-    result = runTest.run(**option_arguments)
+    result, _ = runTest.run(**option_arguments)
+    result = {k:json.loads(v) for k,v in result.items()}
     if result is None:
         return {"Error":"No UUID with given metadata"}
     result = {k:json.loads(v) for k,v in result.items()}

pkg/runTest.py

@@ -10,7 +10,7 @@
 
 
 
-def run(**kwargs: dict[str, Any]) -> dict[str, Any]:
+def run(**kwargs: dict[str, Any]) -> dict[str, Any]: #pylint: disable = R0914
     """run method to start the tests
 
     Args:
@@ -26,6 +26,7 @@ def run(**kwargs: dict[str, Any]) -> dict[str, Any]:
     config_map = kwargs["configMap"]
     datasource = get_datasource(config_map)
     result_output = {}
+    regression_flag = False
     for test in config_map["tests"]:
         # Create fingerprint Matcher
         matcher = Matcher(
@@ -42,7 +43,6 @@ def run(**kwargs: dict[str, Any]) -> dict[str, Any]:
             kwargs,
             start_timestamp,
         )
-
         if fingerprint_matched_df is None:
             return None
 
@@ -62,9 +62,10 @@ def run(**kwargs: dict[str, Any]) -> dict[str, Any]:
                 kwargs,
                 metrics_config,
             )
-        testname, result_data = algorithm.output(kwargs["output_format"])
+        testname, result_data, test_flag = algorithm.output(kwargs["output_format"])
         result_output[testname] = result_data
-    return result_output
+        regression_flag = regression_flag or test_flag
+    return result_output, regression_flag
 
 
 def get_start_timestamp(kwargs: Dict[str, Any]) -> str:

pkg/utils.py

@@ -263,6 +263,7 @@ def process_test(
             else buildUrls[uuid]
         )  # pylint: disable = cell-var-from-loop
     )
+    merged_df=merged_df.reset_index(drop=True)
     #save the dataframe
     output_file_path = f"{options['save_data_path'].split('.')[0]}-{test['name']}.csv"
     match.save_results(merged_df, csv_file_path=output_file_path)