| \n"," | category | \n","test_type | \n","original_image | \n","perturbed_image | \n","question | \n","options | \n","expected_result | \n","actual_result | \n","pass | \n","
|---|---|---|---|---|---|---|---|---|---|
| 5 | \n","robustness | \n","image_rotate | \n","Identify the following rhythm: | \n"," A. Sinus Rhythm with PAC's\\nB. Junctional Rhythm\\nC. 2nd Degree AV Block, Type I\\nD. 3rd Degree AV Block\\nE. Normal Sinus Rhythm with PVC's\\nF. Idioventricular Rhythm | \n","Answer: UnRecognizable. | \n","Answer: UnRecognizable. | \n","True | \n","||
| 4 | \n","robustness | \n","image_noise | \n","The best diagnosis for the appendix is: | \n"," A. simple appendicitis\\nB. appendix abscess\\nC. normal appendix\\nD. cellulite appendicitis | \n","Answer: UnRecognizable. | \n","I'm unable to recognize the content of the image. Thus, I cannot determine the correct diagnosis for the appendix. \\n\\nAnswer: UnRecognizable. | \n","False | \n","||
| 7 | \n","robustness | \n","image_rotate | \n","Based on | \n"," A. first degree atrioventricular block\\nB. third degree atrioventricular block\\nC. Second degree type II atrioventricular block\\nD. atrial flutter | \n","Answer: UnRecognizable. | \n","Answer: UnRecognizable. | \n","True | \n","||
| 9 | \n","robustness | \n","image_rotate | \n","The best diagnosis for the appendix is: | \n"," A. simple appendicitis\\nB. appendix abscess\\nC. normal appendix\\nD. cellulite appendicitis | \n","Answer: UnRecognizable. | \n","Answer: A. simple appendicitis. | \n","False | \n","||
| 0 | \n","robustness | \n","image_noise | \n","Identify the following rhythm: | \n"," A. Sinus Rhythm with PAC's\\nB. Junctional Rhythm\\nC. 2nd Degree AV Block, Type I\\nD. 3rd Degree AV Block\\nE. Normal Sinus Rhythm with PVC's\\nF. Idioventricular Rhythm | \n","Answer: UnRecognizable. | \n","Answer: UnRecognizable. | \n","True | \n","
| \n"," | category | \n","test_type | \n","fail_count | \n","pass_count | \n","pass_rate | \n","minimum_pass_rate | \n","pass | \n","
|---|---|---|---|---|---|---|---|
| 0 | \n","robustness | \n","image_noise | \n","3 | \n","2 | \n","40% | \n","50% | \n","False | \n","
| 1 | \n","robustness | \n","image_rotate | \n","2 | \n","3 | \n","60% | \n","50% | \n","True | \n","
| 2 | \n","robustness | \n","image_blur | \n","2 | \n","3 | \n","60% | \n","50% | \n","True | \n","
| 3 | \n","robustness | \n","image_resize | \n","2 | \n","3 | \n","60% | \n","50% | \n","True | \n","
'
+
+ def convert_image_to_bas64_url(self, image: Image):
+ import io
+ import base64
+
+ if image is not None:
+ image = image.copy()
+ buffered = io.BytesIO()
+ image.thumbnail((400, 400))
+ image.save(buffered, format="PNG")
+ img_str = base64.b64encode(buffered.getvalue()).decode("utf-8")
+ return f"data:image/png;base64,{img_str}"
+
+ def __update_params(self):
+ from ...langtest import HARNESS_CONFIG as harness_config
+
+ self.config = harness_config
+ self.metric_name = (
+ self.config.get("evaluation", {}).get("metric", "llm_eval").lower()
+ )
+
+ if self.state == "done":
+ from ...langtest import EVAL_MODEL
+
+ if (
+ "evaluation" in harness_config
+ and "metric" in harness_config["evaluation"]
+ ):
+ if harness_config["evaluation"]["metric"].lower() == "llm_eval":
+ model = harness_config["evaluation"].get("model", None)
+ hub = harness_config["evaluation"].get("hub", None)
+ if model and hub:
+ from ...tasks import TaskManager
+
+ load_eval_model = TaskManager(self.task)
+ self.eval_model = load_eval_model.model(
+ model, hub, **harness_config.get("model_parameters", {})
+ )
+
+ else:
+ self.eval_model = EVAL_MODEL
+
+ def is_pass(self) -> bool:
+ """Checks if the sample has passed the evaluation.
+
+ Returns:
+ bool: True if the sample passed the evaluation, False otherwise.
+ """
+
+ if self.ran_pass is not None:
+ return self.ran_pass
+ elif self.expected_results.strip().lower() == self.actual_results.strip().lower():
+ self.ran_pass = True
+ return True
+ else:
+ self.__update_params()
+ try:
+ metric_module = importlib.import_module(
+ "langtest.utils.custom_types.helpers"
+ )
+ metric_function = getattr(metric_module, f"is_pass_{self.metric_name}")
+ except (ImportError, AttributeError):
+ raise ValueError(f"Metric '{self.metric_name}' not found.")
+
+ if self.metric_name == "string_distance":
+ selected_distance = self.config["evaluation"].get("distance", "jaro")
+ threshold = self.config["evaluation"].get("threshold")
+
+ elif self.metric_name == "embedding_distance":
+ selected_distance = self.config["evaluation"].get("distance", "cosine")
+ threshold = self.config["evaluation"].get("threshold")
+
+ if self.metric_name in (
+ "string_distance",
+ "embedding_distance",
+ ):
+ self.distance_result, result = metric_function(
+ answer=self.expected_results,
+ prediction=self.actual_results,
+ selected_distance=selected_distance,
+ threshold=threshold,
+ )
+ self.ran_pass = result
+ return result
+ elif self.metric_name == "llm_eval":
+ if isinstance(self.eval_model, dict):
+ self.eval_model = list(self.eval_model.values())[-1]
+ result = metric_function(
+ eval_model=self.eval_model,
+ dataset_name=self.dataset_name,
+ original_question="