From 7b9b3c98a26506d64808bdfb082e1f853f3f4f71 Mon Sep 17 00:00:00 2001
From: Omar Elshehy <41394057+omarelshehy@users.noreply.github.com>
Date: Thu, 12 Dec 2024 00:18:10 +0100
Subject: [PATCH] fix: Add namaa MrTydi reranking dataset (#1573)

* Add dataset class and file requirements

* pass tests

* make lint changes

* adjust meta data and remove load_data

---------

Co-authored-by: Omar Elshehy <omarelshehy@Omars-MacBook-Pro.local>
---
 mteb/tasks/Reranking/__init__.py              |  1 +
 .../Reranking/ara/NamaaMrTydiReranking.py     | 39 +++++++++++++++++++
 mteb/tasks/Reranking/ara/__init__.py          |  0
 3 files changed, 40 insertions(+)
 create mode 100644 mteb/tasks/Reranking/ara/NamaaMrTydiReranking.py
 create mode 100644 mteb/tasks/Reranking/ara/__init__.py

diff --git a/mteb/tasks/Reranking/__init__.py b/mteb/tasks/Reranking/__init__.py
index a4b302a17f..2c3a27919a 100644
--- a/mteb/tasks/Reranking/__init__.py
+++ b/mteb/tasks/Reranking/__init__.py
@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+from .ara.NamaaMrTydiReranking import *
 from .eng.AskUbuntuDupQuestions import *
 from .eng.MindSmallReranking import *
 from .eng.SciDocsReranking import *
diff --git a/mteb/tasks/Reranking/ara/NamaaMrTydiReranking.py b/mteb/tasks/Reranking/ara/NamaaMrTydiReranking.py
new file mode 100644
index 0000000000..4a9d755747
--- /dev/null
+++ b/mteb/tasks/Reranking/ara/NamaaMrTydiReranking.py
@@ -0,0 +1,39 @@
+from __future__ import annotations
+
+from mteb.abstasks.TaskMetadata import TaskMetadata
+
+from ....abstasks.AbsTaskReranking import AbsTaskReranking
+
+
+class NamaaMrTydiReranking(AbsTaskReranking):
+    metadata = TaskMetadata(
+        name="NamaaMrTydiReranking",
+        description="Mr. TyDi is a multi-lingual benchmark dataset built on TyDi, covering eleven typologically diverse languages. It is designed for monolingual retrieval, specifically to evaluate ranking with learned dense representations. This dataset adapts the arabic test split for Reranking evaluation purposes by the addition of multiple (Hard) Negatives to each query and positive",
+        reference="https://huggingface.co/NAMAA-Space",
+        dataset={
+            "path": "NAMAA-Space/mteb-eval-mrtydi",
+            "revision": "502637220a7ad0ecc5c39ff5518d7508d2624af8",
+        },
+        type="Reranking",
+        category="s2s",
+        modalities=["text"],
+        eval_splits=["test"],
+        eval_langs=["ara-Arab"],
+        main_score="map",
+        date=("2023-11-01", "2024-05-15"),
+        domains=["Encyclopaedic", "Written"],
+        task_subtypes=[],
+        license="cc-by-sa-3.0",
+        annotations_creators="human-annotated",
+        dialect=[],
+        sample_creation="found",
+        bibtex_citation="""@article{muennighoff2022mteb,
+  doi = {10.48550/ARXIV.2210.07316},
+  url = {https://arxiv.org/abs/2210.07316},
+  author = {Muennighoff, Niklas and Tazi, Nouamane and Magne, Lo{\"\i}c and Reimers, Nils},
+  title = {MTEB: Massive Text Embedding Benchmark},
+  publisher = {arXiv},
+  journal={arXiv preprint arXiv:2210.07316},  
+  year = {2022}
+}""",
+    )
diff --git a/mteb/tasks/Reranking/ara/__init__.py b/mteb/tasks/Reranking/ara/__init__.py
new file mode 100644
index 0000000000..e69de29bb2