Add law datasets (#311)

* add command * add datasets * reformat dataset * Rephrase description * Update mteb/tasks/Retrieval/law/GerDaLIRRetrieval.py * Update mteb/tasks/Retrieval/law/GerDaLIRRetrieval.py * Update mteb/__init__.py * Update scripts/run_mteb_law.py * Update scripts/run_mteb_law.py * Update mteb/__init__.py * Update mteb/tasks/Retrieval/__init__.py * Update mteb/tasks/Retrieval/law/GerDaLIRRetrieval.py * Update mteb/tasks/Retrieval/law/GerDaLIRRetrieval.py * Update mteb/tasks/Retrieval/law/LegalQuADRetrieval.py * Update mteb/tasks/Retrieval/law/LegalQuADRetrieval.py * Update scripts/run_mteb_law.py * Update mteb/tasks/Retrieval/law/LegalSummarizationRetrieval.py * Update mteb/tasks/Retrieval/law/LegalSummarizationRetrieval.py * Update mteb/tasks/Retrieval/law/LeCaRDv2Retrieval.py * Update mteb/tasks/Retrieval/law/LeCaRDv2Retrieval.py * Rename GerDaLIRRetrieval.py to GerDaLIRSmallRetrieval.py * Update mteb/tasks/Retrieval/__init__.py * Update GerDaLIRSmallRetrieval.py Add metadata * Update GerDaLIRSmallRetrieval.py Update metadata * Update AILACasedocsRetrieval.py Update AILACasedocsRetrieval metadata * Update AILAStatutesRetrieval.py Update AILAStatutesRetrieval metadata * Update LeCaRDv2Retrieval.py Update LeCaRDv2Retrieval metadata * Update LegalBenchConsumerContractsQARetrieval.py Update LegalBenchConsumerContractsQARetrieval metadata * Update LegalBenchCorporateLobbyingRetrieval.py Update LegalBenchCorporateLobbyingRetrieval metadata * Update LegalQuADRetrieval.py Update LegalQuADRetrieval metadata * Update LegalSummarizationRetrieval.py Update LegalSummarizationRetrieval metadata * Update AILACasedocsRetrieval.py Update AILACasedocsRetrieval * Update AILACasedocsRetrieval.py Update AILACasedocsRetrieval metadata * Update AILAStatutesRetrieval.py Update AILAStatutesRetrieval metadata * Update GerDaLIRSmallRetrieval.py Update GerDaLIRSmallRetrieval metadata * Update LeCaRDv2Retrieval.py Update LeCaRDv2Retrieval metadata * Update LegalBenchConsumerContractsQARetrieval.py * Update LegalBenchCorporateLobbyingRetrieval.py * Update LegalQuADRetrieval.py * Update LegalSummarizationRetrieval.py * Update AILACasedocsRetrieval.py * Update AILAStatutesRetrieval.py * Update GerDaLIRSmallRetrieval.py * Update LeCaRDv2Retrieval.py * move dataset language folder * update order --------- Co-authored-by: Niklas Muennighoff <n.muennighoff@gmail.com>
embeddings-benchmark · Apr 6, 2024 · 6e3f419 · 6e3f419
1 parent ca64fc7
commit 6e3f419
Show file tree

Hide file tree

Showing 11 changed files with 328 additions and 0 deletions.
diff --git a/mteb/__init__.py b/mteb/__init__.py
@@ -76,3 +76,14 @@
     "TwitterSemEval2015",
     "TwitterURLCorpus",
 ]
+
+MTEB_RETRIEVAL_LAW = [
+    "LegalSummarization",
+    "LegalBenchConsumerContractsQA",
+    "LegalBenchCorporateLobbying",
+    "AILACasedocs",
+    "AILAStatutes",
+    "LeCaRDv2",
+    "LegalQuAD",
+    "GerDaLIRSmall",
+]
diff --git a/mteb/tasks/Retrieval/__init__.py b/mteb/tasks/Retrieval/__init__.py
@@ -6,6 +6,8 @@
 from .de.GerDaLIRRetrieval import *
 from .de.GermanDPRRetrieval import *
 from .de.GermanQuADRetrieval import *
+from .de.GerDaLIRSmallRetrieval import *
+from .de.LegalQuADRetrieval import *
 from .en.ArguAnaRetrieval import *
 from .en.ClimateFEVERRetrieval import *
 from .en.CQADupstackAndroidRetrieval import *
@@ -35,6 +37,11 @@
 from .en.SciFactRetrieval import *
 from .en.Touche2020Retrieval import *
 from .en.TRECCOVIDRetrieval import *
+from .en.AILACasedocsRetrieval import *
+from .en.AILAStatutesRetrieval import *
+from .en.LegalBenchConsumerContractsQARetrieval import *
+from .en.LegalBenchCorporateLobbyingRetrieval import *
+from .en.LegalSummarizationRetrieval import *
 from .es.SpanishPassageRetrievalS2P import *
 from .es.SpanishPassageRetrievalS2S import *
 from .fr.AlloprofRetrieval import *
@@ -64,3 +71,4 @@
 from .sv.swedn_retrieval import *
 from .sv.swefaq_retrieval import *
 from .zh.CMTEBRetrieval import *
+from .zh.LeCaRDv2Retrieval import *
diff --git a/mteb/tasks/Retrieval/de/GerDaLIRSmallRetrieval.py b/mteb/tasks/Retrieval/de/GerDaLIRSmallRetrieval.py
@@ -0,0 +1,34 @@
+from __future__ import annotations
+
+from mteb.abstasks.TaskMetadata import TaskMetadata
+
+from ....abstasks.AbsTaskRetrieval import AbsTaskRetrieval
+
+
+class GerDaLIRSmall(AbsTaskRetrieval):
+    metadata = TaskMetadata(
+        name="GerDaLIRSmall",
+        description="The dataset consists of documents, passages and relevance labels in German. In contrast to the original dataset, only documents that have corresponding queries in the query set are chosen to create a smaller corpus for evaluation purposes.",
+        reference="https://github.com/lavis-nlp/GerDaLIR",
+        dataset={
+            "path": "mteb/GerDaLIRSmall",
+            "revision": "48327de6ee192e9610f3069789719788957c7abd",
+        },
+        type="Retrieval",
+        category="p2p",
+        eval_splits=["test"],
+        eval_langs=["de"],
+        main_score="ndcg_at_10",
+        date=None,
+        form=["written"],
+        domains=["Legal"],
+        task_subtypes=["Article retrieval"],
+        license="MIT license",
+        socioeconomic_status="high",
+        annotations_creators="derived",
+        dialect=None,
+        text_creation="found",
+        bibtex_citation= None,
+        n_samples=None,
+        avg_character_length=None,
+    )
diff --git a/mteb/tasks/Retrieval/de/LegalQuADRetrieval.py b/mteb/tasks/Retrieval/de/LegalQuADRetrieval.py
@@ -0,0 +1,34 @@
+from __future__ import annotations
+
+from mteb.abstasks.TaskMetadata import TaskMetadata
+
+from ....abstasks.AbsTaskRetrieval import AbsTaskRetrieval
+
+
+class LegalQuAD(AbsTaskRetrieval):
+    metadata = TaskMetadata(
+        name="LegalQuAD",
+        description="The dataset consists of questions and legal documents in German.",
+        reference="https://github.com/Christoph911/AIKE2021_Appendix",
+        dataset={
+            "path": "mteb/LegalQuAD",
+            "revision": "37aa6cfb01d48960b0f8e3f17d6e3d99bf1ebc3e",
+        },
+        type="Retrieval",
+        category="s2p",
+        eval_splits=["test"],
+        eval_langs=["de"],
+        main_score="ndcg_at_10",
+        date=None,
+        form=["written"],
+        domains=["Legal"],
+        task_subtypes=["Question answering"],
+        license="CC BY 4.0",
+        socioeconomic_status="high",
+        annotations_creators="derived",
+        dialect=None,
+        text_creation="found",
+        bibtex_citation= None,
+        n_samples=None,
+        avg_character_length=None,
+    )
diff --git a/mteb/tasks/Retrieval/en/AILACasedocsRetrieval.py b/mteb/tasks/Retrieval/en/AILACasedocsRetrieval.py
@@ -0,0 +1,34 @@
+from __future__ import annotations
+
+from mteb.abstasks.TaskMetadata import TaskMetadata
+
+from ....abstasks.AbsTaskRetrieval import AbsTaskRetrieval
+
+
+class AILACasedocs(AbsTaskRetrieval):
+    metadata = TaskMetadata(
+        name="AILACasedocs",
+        description="The task is to retrieve the case document that most closely matches or is most relevant to the scenario described in the provided query.",
+        reference="https://zenodo.org/records/4063986",
+        dataset={
+            "path": "mteb/AILA_casedocs",
+            "revision": "4106e6bcc72e0698d714ea8b101355e3e238431a",
+        },
+        type="Retrieval",
+        category="p2p",
+        eval_splits=["test"],
+        eval_langs=["en"],
+        main_score="ndcg_at_10",
+        date=None,
+        form=["written"],
+        domains=["Legal"],
+        task_subtypes=["Article retrieval"],
+        license="CC BY 4.0",
+        socioeconomic_status="high",
+        annotations_creators="derived",
+        dialect=None,
+        text_creation="found",
+        bibtex_citation= None,
+        n_samples=None,
+        avg_character_length=None,
+    )
diff --git a/mteb/tasks/Retrieval/en/AILAStatutesRetrieval.py b/mteb/tasks/Retrieval/en/AILAStatutesRetrieval.py
@@ -0,0 +1,34 @@
+from __future__ import annotations
+
+from mteb.abstasks.TaskMetadata import TaskMetadata
+
+from ....abstasks.AbsTaskRetrieval import AbsTaskRetrieval
+
+
+class AILAStatutes(AbsTaskRetrieval):
+    metadata = TaskMetadata(
+        name="AILAStatutes",
+        description="This dataset is structured for the task of identifying the most relevant statutes for a given situation.",
+        reference="https://zenodo.org/records/4063986",
+        dataset={
+            "path": "mteb/AILA_statutes",
+            "revision": "ebfcd844eadd3d667efa3c57fc5c8c87f5c2867e",
+        },
+        type="Retrieval",
+        category="p2p",
+        eval_splits=["test"],
+        eval_langs=["en"],
+        main_score="ndcg_at_10",
+        date=None,
+        form=["written"],
+        domains=["Legal"],
+        task_subtypes=["Article retrieval"],
+        license="CC BY 4.0",
+        socioeconomic_status="high",
+        annotations_creators="derived",
+        dialect=None,
+        text_creation="found",
+        bibtex_citation= None,
+        n_samples=None,
+        avg_character_length=None,
+    )
diff --git a/mteb/tasks/Retrieval/en/LegalBenchConsumerContractsQARetrieval.py b/mteb/tasks/Retrieval/en/LegalBenchConsumerContractsQARetrieval.py
@@ -0,0 +1,34 @@
+from __future__ import annotations
+
+from mteb.abstasks.TaskMetadata import TaskMetadata
+
+from ....abstasks.AbsTaskRetrieval import AbsTaskRetrieval
+
+
+class LegalBenchConsumerContractsQA(AbsTaskRetrieval):
+    metadata = TaskMetadata(
+        name="LegalBenchConsumerContractsQA",
+        description="The dataset includes questions and answers related to contracts.",
+        reference="https://huggingface.co/datasets/nguha/legalbench/viewer/consumer_contracts_qa",
+        dataset={
+            "path": "mteb/legalbench_consumer_contracts_qa",
+            "revision": "b23590301ec94e8087e2850b21d43d4956b1cca9",
+        },
+        type="Retrieval",
+        category="s2p",
+        eval_splits=["test"],
+        eval_langs=["en"],
+        main_score="ndcg_at_10",
+        date=None,
+        form=["written"],
+        domains=["Legal"],
+        task_subtypes=["Question answering"],
+        license="CC BY-NC 4.0",
+        socioeconomic_status="high",
+        annotations_creators="derived",
+        dialect=None,
+        text_creation="found",
+        bibtex_citation= None,
+        n_samples=None,
+        avg_character_length=None,
+    )
diff --git a/mteb/tasks/Retrieval/en/LegalBenchCorporateLobbyingRetrieval.py b/mteb/tasks/Retrieval/en/LegalBenchCorporateLobbyingRetrieval.py
@@ -0,0 +1,34 @@
+from __future__ import annotations
+
+from mteb.abstasks.TaskMetadata import TaskMetadata
+
+from ....abstasks.AbsTaskRetrieval import AbsTaskRetrieval
+
+
+class LegalBenchCorporateLobbying(AbsTaskRetrieval):
+    metadata = TaskMetadata(
+        name="LegalBenchCorporateLobbying",
+        description="The dataset includes bill titles and bill summaries related to corporate lobbying.",
+        reference="https://huggingface.co/datasets/nguha/legalbench/viewer/corporate_lobbying",
+        dataset={
+            "path": "mteb/legalbench_corporate_lobbying",
+            "revision": "f69691c650464e62546d7f2a4536f8f87c891e38",
+        },
+        type="Retrieval",
+        category="s2p",
+        eval_splits=["test"],
+        eval_langs=["en"],
+        main_score="ndcg_at_10",
+        date=None,
+        form=["written"],
+        domains=["Legal"],
+        task_subtypes=["Article retrieval"],
+        license="CC BY 4.0",
+        socioeconomic_status="high",
+        annotations_creators="derived",
+        dialect=None,
+        text_creation="found",
+        bibtex_citation= None,
+        n_samples=None,
+        avg_character_length=None,
+    )
diff --git a/mteb/tasks/Retrieval/en/LegalSummarizationRetrieval.py b/mteb/tasks/Retrieval/en/LegalSummarizationRetrieval.py
@@ -0,0 +1,34 @@
+from __future__ import annotations
+
+from mteb.abstasks.TaskMetadata import TaskMetadata
+
+from ....abstasks.AbsTaskRetrieval import AbsTaskRetrieval
+
+
+class LegalSummarization(AbsTaskRetrieval):
+    metadata = TaskMetadata(
+        name="LegalSummarization",
+        description="The dataset consistes of 439 pairs of contracts and their summarizations from https://tldrlegal.com and https://tosdr.org/.",
+        reference="https://github.com/lauramanor/legal_summarization",
+        dataset={
+            "path": "mteb/legal_summarization",
+            "revision": "3bb1a05c66872889662af04c5691c14489cebd72",
+        },
+        type="Retrieval",
+        category="s2p",
+        eval_splits=["test"],
+        eval_langs=["en"],
+        main_score="ndcg_at_10",
+        date=None,
+        form=["written"],
+        domains=["Legal"],
+        task_subtypes=["Article retrieval"],
+        license="Apache License 2.0",
+        socioeconomic_status="high",
+        annotations_creators="derived",
+        dialect=None,
+        text_creation="found",
+        bibtex_citation= None,
+        n_samples=None,
+        avg_character_length=None,
+    )
diff --git a/mteb/tasks/Retrieval/zh/LeCaRDv2Retrieval.py b/mteb/tasks/Retrieval/zh/LeCaRDv2Retrieval.py
@@ -0,0 +1,34 @@
+from __future__ import annotations
+
+from mteb.abstasks.TaskMetadata import TaskMetadata
+
+from ....abstasks.AbsTaskRetrieval import AbsTaskRetrieval
+
+
+class LeCaRDv2(AbsTaskRetrieval):
+    metadata = TaskMetadata(
+        name="LeCaRDv2",
+        description="The task involves identifying and retrieving the case document that best matches or is most relevant to the scenario described in each of the provided queries.",
+        reference="https://github.com/THUIR/LeCaRDv2",
+        dataset={
+            "path": "mteb/LeCaRDv2",
+            "revision": "b78e18688c3d012a33dc3676597c1d1b2243ce1c",
+        },
+        type="Retrieval",
+        category="p2p",
+        eval_splits=["test"],
+        eval_langs=["zh"],
+        main_score="ndcg_at_10",
+        date=None,
+        form=["written"],
+        domains=["Legal"],
+        task_subtypes=["Article retrieval"],
+        license="MIT license",
+        socioeconomic_status="high",
+        annotations_creators="derived",
+        dialect=None,
+        text_creation="found",
+        bibtex_citation= None,
+        n_samples=None,
+        avg_character_length=None,
+    )
diff --git a/scripts/run_mteb_law.py b/scripts/run_mteb_law.py
@@ -0,0 +1,37 @@
+"""Example script for benchmarking all datasets constituting the Retrieval Law leaderboard & average scores"""
+
+from __future__ import annotations
+
+import logging
+
+from sentence_transformers import SentenceTransformer
+
+from mteb import MTEB
+
+logging.basicConfig(level=logging.INFO)
+
+logger = logging.getLogger("main")
+
+TASK_LIST_RETRIEVAL_LAW = [
+    "LegalSummarization",
+    "LegalBenchConsumerContractsQA",
+    "LegalBenchCorporateLobbying",
+    "AILACasedocs",
+    "AILAStatutes",
+    "LeCaRDv2",
+    "LegalQuAD",
+    "GerDaLIR",
+]
+
+model_name = "average_word_embeddings_komninos"
+model = SentenceTransformer(model_name)
+
+for task in TASK_LIST_RETRIEVAL_LAW:
+    logger.info(f"Running task: {task}")
+    eval_splits = ["test"]
+    evaluation = MTEB(
+        tasks=[task]
+    )
+    evaluation.run(
+        model, output_folder=f"results/{model_name}", eval_splits=eval_splits
+    )