fix: Added tests for checking datasets (#307)

* fix: Fixed hf_hub_name for WikiCitiesClustering * Added points for this PR and a 3 other minor dataset fixes * feat: Added tests which validated that datasets are available * fix: Updated hf references and revisions to multiple datasets * Added points for submission * fix: Added suggestions from the review * Apply suggestions from code review Co-authored-by: Imene Kerboua <33312980+imenelydiaker@users.noreply.github.com> * fix: sped up async test for whether datasets exist * fix: Updated revisions * fix: reuploaded scandeval datasets * fix: Applied formatter --------- Co-authored-by: Imene Kerboua <33312980+imenelydiaker@users.noreply.github.com>
embeddings-benchmark · Apr 3, 2024 · 8d804f4 · 8d804f4
1 parent 74f33f0
commit 8d804f4
Show file tree

Hide file tree

Showing 49 changed files with 212 additions and 84 deletions.
diff --git a/docs/mmteb/points.md b/docs/mmteb/points.md
@@ -2,7 +2,7 @@
 
 | GitHub            | Total points | New dataset | New task | Dataset annotations | (Bug)fixes | Running Models | Review PR |  Paper Writing | Ideation | Coordination |
 | ----------------- | ------------ | ----------- | -------- | ------------------- | ---------- | -------------- |  -------- | -------------- | -------- | ------------- |
-| KennethEnevoldsen |              |   38        |          |                     |         4  |                |           |                |         |               |
+| KennethEnevoldsen |              |   38        |          |                     |         8  |                |           |                |         |               |
 | x-tabdeveloping   |              |    2        |          |                     |            |                |           |                |        |               |
 
 Note that coordination and ideation is not included in the total points, but is used to determine first and last authors. 

diff --git a/mteb/abstasks/TaskMetadata.py b/mteb/abstasks/TaskMetadata.py
@@ -9,7 +9,6 @@
     BeforeValidator,
     TypeAdapter,
     field_validator,
-    model_validator,
 )
 from typing_extensions import Annotated, Literal
 
@@ -172,7 +171,11 @@ def _check_dataset_path_is_specified(cls, dataset):
     @field_validator("dataset")
     def _check_dataset_revision_is_specified(cls, dataset):
         if "revision" not in dataset:
-            raise ValueError("You must explicitly specify a revision for the dataset (either a SHA or None).")
+            raise ValueError(
+                "You must explicitly specify a revision for the dataset (either a SHA or None)."
+            )
         if dataset["revision"] is None:
-            logging.warning("It is encourage to specify a dataset revision for reproducability")
+            logging.warning(
+                "It is encourage to specify a dataset revision for reproducability"
+            )
         return dataset
diff --git a/mteb/evaluation/MTEB.py b/mteb/evaluation/MTEB.py
@@ -304,7 +304,9 @@ def run(
                 # run evaluation
                 task_results = {
                     "mteb_version": version("mteb"),  # noqa: F405
-                    "dataset_revision": task.metadata_dict["dataset"].get("revision", None),
+                    "dataset_revision": task.metadata_dict["dataset"].get(
+                        "revision", None
+                    ),
                     "mteb_dataset_name": task.metadata_dict["name"],
                 }
                 for split in task_eval_splits:

diff --git a/mteb/tasks/BitextMining/nb/norwegian_courts_bitext_mining.py b/mteb/tasks/BitextMining/nb/norwegian_courts_bitext_mining.py
@@ -6,7 +6,7 @@ class NorwegianCourtsBitextMining(AbsTaskBitextMining):
     metadata = TaskMetadata(
         name="NorwegianCourtsBitextMining",
         dataset={
-            "path": "kaedrodrur/norwegian-courts",
+            "path": "kardosdrur/norwegian-courts",
             "revision": "d79af07e969a6678fcbbe819956840425816468f",
         },
         description="Nynorsk and Bokmål parallel corpus from Norwegian courts. ",

diff --git a/mteb/tasks/Classification/multilingual/ScalaClassification.py b/mteb/tasks/Classification/multilingual/ScalaClassification.py
@@ -10,8 +10,8 @@ class ScalaDaClassification(AbsTaskClassification):
         description="A modified version of DDT modified for linguistic acceptability classification",
         reference="https://aclanthology.org/2023.nodalida-1.20/",
         dataset={
-            "path": "ScandEval/scala-da",
-            "revision": "1de08520a7b361e92ffa2a2201ebd41942c54675",
+            "path": "mteb/scala_da_classification",
+            "revision": "e60a77795ed5488fb7a03751cf6f2b026fa27a71",
         },
         type="Classification",
         category="s2s",
@@ -54,8 +54,8 @@ class ScalaNbClassification(AbsTaskClassification):
         description="A Norwegian dataset for linguistic acceptability classification for Bokmål",
         reference="https://aclanthology.org/2023.nodalida-1.20/",
         dataset={
-            "path": "ScandEval/scala-nb",
-            "revision": "237111a078ad5a834a55c57803d40bbe410ed03b",
+            "path": "mteb/scala_nb_classification",
+            "revision": "dda7af4696bd8d5150441908ea8ed6e68a357c13",
         },
         type="Classification",
         category="s2s",
@@ -98,8 +98,8 @@ class ScalaNnClassification(AbsTaskClassification):
         description="A Norwegian dataset for linguistic acceptability classification for Nynorsk",
         reference="https://aclanthology.org/2023.nodalida-1.20/",
         dataset={
-            "path": "ScandEval/scala-nn",
-            "revision": "9d9a2a4092ed3cacf0744592f6d2f32ab8ef4c0b",
+            "path": "mteb/scala_nn_classification",
+            "revision": "d81637ad324afb995ae395a87055380e8118a9c0",
         },
         type="Classification",
         category="s2s",
@@ -142,8 +142,8 @@ class ScalaSvClassification(AbsTaskClassification):
         description="A Swedish dataset for linguistic acceptability classification",
         reference="https://aclanthology.org/2023.nodalida-1.20/",
         dataset={
-            "path": "ScandEval/scala-sv",
-            "revision": "1b48e3dcb02872335ff985ff938a054a4ed99008",
+            "path": "mteb/scala_sv_classification",
+            "revision": "aded78beae37445bf3917102d0b332049cfc8c99",
         },
         type="Classification",
         category="s2s",

diff --git a/mteb/tasks/Classification/nb/NoRecClassification.py b/mteb/tasks/Classification/nb/NoRecClassification.py
@@ -11,8 +11,8 @@ class NoRecClassification(AbsTaskClassification):
         reference="https://aclanthology.org/L18-1661/",
         dataset={
             # using the mini version to keep results ~comparable to the ScandEval benchmark
-            "path": "ScandEval/norec-mini",
-            "revision": "07b99ab3363c2e7f8f87015b01c21f4d9b917ce3",
+            "path": "mteb/norec_classification",
+            "revision": "5b740b7c42c73d586420812a35745fc37118862f",
         },
         type="Classification",
         category="s2s",

diff --git a/mteb/tasks/Classification/pl/PolishClassification.py b/mteb/tasks/Classification/pl/PolishClassification.py
@@ -12,7 +12,7 @@ class CbdClassification(AbsTaskClassification):
         reference="http://2019.poleval.pl/files/poleval2019.pdf",
         dataset={
             "path": "PL-MTEB/cbd",
-            "revision": "59d12749a3c91a186063c7d729ec392fda94681c",
+            "revision": "36ddb419bcffe6a5374c3891957912892916f28d",
         },
         type="Classification",
         category="s2s",
@@ -42,7 +42,7 @@ class PolEmo2InClassification(AbsTaskClassification):
         reference="https://aclanthology.org/K19-1092.pdf",
         dataset={
             "path": "PL-MTEB/polemo2_in",
-            "revision": "9e9b1f8ef51616073f47f306f7f47dd91663f86a",
+            "revision": "d90724373c70959f17d2331ad51fb60c71176b03",
         },
         type="Classification",
         category="s2s",
@@ -73,7 +73,7 @@ class PolEmo2OutClassification(AbsTaskClassification):
         reference="https://aclanthology.org/K19-1092.pdf",
         dataset={
             "path": "PL-MTEB/polemo2_out",
-            "revision": "c99d599f0a6ab9b85b065da6f9d94f9cf731679f",
+            "revision": "6a21ab8716e255ab1867265f8b396105e8aa63d4",
         },
         type="Classification",
         category="s2s",
@@ -102,7 +102,7 @@ class AllegroReviewsClassification(AbsTaskClassification):
         reference="https://aclanthology.org/2020.acl-main.111.pdf",
         dataset={
             "path": "PL-MTEB/allegro-reviews",
-            "revision": "477b8bd4448b5ef8ed01ba82bf9ff67f6e109207",
+            "revision": "b89853e6de927b0e3bfa8ecc0e56fe4e02ceafc6",
         },
         type="Classification",
         category="s2s",
@@ -131,7 +131,7 @@ class PacClassification(AbsTaskClassification):
         reference="https://arxiv.org/pdf/2211.13112.pdf",
         dataset={
             "path": "laugustyniak/abusive-clauses-pl",
-            "revision": "8a04d940a42cd40658986fdd8e3da561533a3646",
+            "revision": "fc69d1c153a8ccdcf1eef52f4e2a27f88782f543",
         },
         type="Classification",
         category="p2p",

diff --git a/mteb/tasks/Classification/sv/SweRecClassification.py b/mteb/tasks/Classification/sv/SweRecClassification.py
@@ -10,8 +10,8 @@ class SweRecClassification(AbsTaskClassification):
         description="A Swedish dataset for sentiment classification on review",
         reference="https://aclanthology.org/2023.nodalida-1.20/",
         dataset={
-            "path": "SweRecClassification",
-            "revision": "3c62f26bafdc4c4e1c16401ad4b32f0a94b46612",
+            "path": "mteb/swerec_classification",
+            "revision": "b07c6ce548f6a7ac8d546e1bbe197a0086409190",
         },
         type="Classification",
         category="s2s",

diff --git a/mteb/tasks/Clustering/de/BlurbsClusteringS2S.py b/mteb/tasks/Clustering/de/BlurbsClusteringS2S.py
@@ -12,7 +12,7 @@ class BlurbsClusteringS2S(AbsTaskClustering):
         reference="https://www.inf.uni-hamburg.de/en/inst/ab/lt/resources/data/germeval-2019-hmc.html",
         dataset={
             "path": "slvnwhrl/blurbs-clustering-s2s",
-            "revision": "5bfd9ee0ae3d2ef0d9a0e0bd0c3b2d0",
+            "revision": "22793b6a6465bf00120ad525e38c51210858132c",
         },
         type="Clustering",
         category="s2s",

diff --git a/mteb/tasks/Clustering/es/FloresClusteringS2S.py b/mteb/tasks/Clustering/es/FloresClusteringS2S.py
@@ -11,8 +11,8 @@ class FloresClusteringS2S(AbsTaskClustering):
         description="Clustering of sentences from various web articles, 32 topics in total.",
         reference="https://huggingface.co/datasets/facebook/flores",
         dataset={
-            "path": "mteb/flores",
-            "revision": "480b580487f53a46f881354a8348335d4edbb2de",
+            "path": "facebook/flores",
+            "revision": "2db78afdeaccaedc3b33a95442a4e55766887e17",
         },
         type="Clustering",
         category="s2s",

diff --git a/mteb/tasks/Clustering/es/SpanishNewsClusteringP2P.py b/mteb/tasks/Clustering/es/SpanishNewsClusteringP2P.py
@@ -11,8 +11,8 @@ class SpanishNewsClusteringP2P(AbsTaskClustering):
         description="Clustering of news articles, 7 topics in total.",
         reference="https://www.kaggle.com/datasets/kevinmorgado/spanish-news-classification",
         dataset={
-            "path": "mteb/spanish_news_clustering",
-            "revision": "b5edc3d3d7c12c7b9f883e9da50f6732f3624142",
+            "path": "jinaai/spanish_news_clustering",
+            "revision": "bf8ca8ddc5b7da4f7004720ddf99bbe0483480e6",
         },
         type="Clustering",
         category="p2p",

diff --git a/mteb/tasks/Clustering/fr/AlloProfClusteringP2P.py b/mteb/tasks/Clustering/fr/AlloProfClusteringP2P.py
@@ -14,7 +14,7 @@ class AlloProfClusteringP2P(AbsTaskClustering):
         description="Clustering of document titles and descriptions from Allo Prof dataset. Clustering of 10 sets on the document topic.",
         reference="https://huggingface.co/datasets/lyon-nlp/alloprof",
         dataset={
-            "path": "mteb/alloprof",
+            "path": "lyon-nlp/alloprof",
             "revision": "392ba3f5bcc8c51f578786c1fc3dae648662cb9b",
             "name": "documents",
         },

diff --git a/mteb/tasks/Clustering/fr/AlloProfClusteringS2S.py b/mteb/tasks/Clustering/fr/AlloProfClusteringS2S.py
@@ -14,7 +14,7 @@ class AlloProfClusteringS2S(AbsTaskClustering):
         description="Clustering of document titles from Allo Prof dataset. Clustering of 10 sets on the document topic.",
         reference="https://huggingface.co/datasets/lyon-nlp/alloprof",
         dataset={
-            "path": "mteb/alloprof",
+            "path": "lyon-nlp/alloprof",
             "revision": "392ba3f5bcc8c51f578786c1fc3dae648662cb9b",
             "name": "documents",
         },

diff --git a/mteb/tasks/Clustering/fr/HALClusteringS2S.py b/mteb/tasks/Clustering/fr/HALClusteringS2S.py
@@ -14,7 +14,7 @@ class HALClusteringS2S(AbsTaskClustering):
         description="Clustering of titles from HAL (https://huggingface.co/datasets/lyon-nlp/clustering-hal-s2s)",
         reference="https://huggingface.co/datasets/lyon-nlp/clustering-hal-s2s",
         dataset={
-            "path": "mteb/hal",
+            "path": "lyon-nlp/clustering-hal-s2s",
             "revision": "e06ebbbb123f8144bef1a5d18796f3dec9ae2915",
         },
         type="Clustering",

diff --git a/mteb/tasks/Clustering/fr/MLSUMClusteringP2P.py b/mteb/tasks/Clustering/fr/MLSUMClusteringP2P.py
@@ -14,7 +14,7 @@ class MLSUMClusteringP2P(AbsTaskClustering):
         description="Clustering of newspaper article contents and titles from MLSUM dataset. Clustering of 10 sets on the newpaper article topics.",
         reference="https://huggingface.co/datasets/mlsum",
         dataset={
-            "path": "mteb/mlsum",
+            "path": "mlsum",
             "revision": "b5d54f8f3b61ae17845046286940f03c6bc79bc7",
             "name": "fr",
             "split": "test",

diff --git a/mteb/tasks/Clustering/fr/MLSUMClusteringS2S.py b/mteb/tasks/Clustering/fr/MLSUMClusteringS2S.py
@@ -14,7 +14,7 @@ class MLSUMClusteringS2S(AbsTaskClustering):
         description="Clustering of newspaper article contents and titles from MLSUM dataset. Clustering of 10 sets on the newpaper article topics.",
         reference="https://huggingface.co/datasets/mlsum",
         dataset={
-            "path": "mteb/mlsum",
+            "path": "mlsum",
             "revision": "b5d54f8f3b61ae17845046286940f03c6bc79bc7",
             "name": "fr",
             "split": "test",

diff --git a/mteb/tasks/Clustering/multilingual/MasakhaNEWSClusteringP2P.py b/mteb/tasks/Clustering/multilingual/MasakhaNEWSClusteringP2P.py
@@ -33,7 +33,7 @@ class MasakhaNEWSClusteringP2P(AbsTaskClustering, MultilingualTask):
         description="Clustering of news article headlines and texts from MasakhaNEWS dataset. Clustering of 10 sets on the news article label.",
         reference="https://huggingface.co/datasets/masakhane/masakhanews",
         dataset={
-            "path": "mteb/masakhanews",
+            "path": "masakhane/masakhanews",
             "revision": "8ccc72e69e65f40c70e117d8b3c08306bb788b60",
         },
         type="Clustering",

diff --git a/mteb/tasks/Clustering/pl/PolishClustering.py b/mteb/tasks/Clustering/pl/PolishClustering.py
@@ -12,8 +12,8 @@ class EightTagsClustering(AbsTaskClustering):
         "food, medicine, motorization, work, sport and technology.",
         reference="https://aclanthology.org/2020.lrec-1.207.pdf",
         dataset={
-            "path": "mteb/polish-clustering",
-            "revision": "e7a26af6f3ae46b30dde8737f02c07b1505bcc73",
+            "path": "PL-MTEB/8tags-clustering",
+            "revision": "78b962b130c6690659c65abf67bf1c2f030606b6",
         },
         type="Clustering",
         category="s2s",

diff --git a/mteb/tasks/Clustering/zh/CMTEBClustering.py b/mteb/tasks/Clustering/zh/CMTEBClustering.py
@@ -11,7 +11,7 @@ class CLSClusteringS2S(AbsTaskClustering):
         description="Clustering of titles from CLS dataset. Clustering of 13 sets on the main category.",
         reference="https://arxiv.org/abs/2209.05034",
         dataset={
-            "path": "mteb/cls",
+            "path": "C-MTEB/CLSClusteringS2S",
             "revision": "e458b3f5414b62b7f9f83499ac1f5497ae2e869f",
         },
         type="Clustering",
@@ -40,7 +40,7 @@ class CLSClusteringP2P(AbsTaskClustering):
         description="Clustering of titles + abstract from CLS dataset. Clustering of 13 sets on the main category.",
         reference="https://arxiv.org/abs/2209.05034",
         dataset={
-            "path": "mteb/cls",
+            "path": "C-MTEB/CLSClusteringP2P",
             "revision": "4b6227591c6c1a73bc76b1055f3b7f3588e72476",
         },
         type="Clustering",

diff --git a/mteb/tasks/PairClassification/pl/PolishPC.py b/mteb/tasks/PairClassification/pl/PolishPC.py
@@ -10,7 +10,7 @@ class SickePLPC(AbsTaskPairClassification):
         name="SICK-E-PL",
         dataset={
             "path": "PL-MTEB/sicke-pl-pairclassification",
-            "revision": "5c59e41555244b7e45c9a6be2d720ab4bafae558",
+            "revision": "71bba34b0ece6c56dfcf46d9758a27f7a90f17e9",
         },
         description="Polish version of SICK dataset for textual entailment.",
         reference="https://aclanthology.org/2020.lrec-1.207",
@@ -39,7 +39,7 @@ class PpcPC(AbsTaskPairClassification):
         name="PpcPC",
         dataset={
             "path": "PL-MTEB/ppc-pairclassification",
-            "revision": "1.0",
+            "revision": "2c7d2df57801a591f6b1e3aaf042e7a04ec7d9f2",
         },
         description="Polish Paraphrase Corpus",
         reference="https://arxiv.org/pdf/2207.12759.pdf",
@@ -68,7 +68,7 @@ class CdscePC(AbsTaskPairClassification):
         name="CDSC-E",
         dataset={
             "path": "PL-MTEB/cdsce-pairclassification",
-            "revision": "1.0",
+            "revision": "0a3d4aa409b22f80eb22cbf59b492637637b536d",
         },
         description="Compositional Distributional Semantics Corpus for textual entailment.",
         reference="https://aclanthology.org/P17-1073.pdf",
@@ -97,7 +97,7 @@ class PscPC(AbsTaskPairClassification):
         name="PSC",
         dataset={
             "path": "PL-MTEB/psc-pairclassification",
-            "revision": "1.0",
+            "revision": "d05a294af9e1d3ff2bfb6b714e08a24a6cabc669",
         },
         description="Polish Summaries Corpus",
         reference="http://www.lrec-conf.org/proceedings/lrec2014/pdf/1211_Paper.pdf",

diff --git a/mteb/tasks/PairClassification/zh/CMTEBPairClassification.py b/mteb/tasks/PairClassification/zh/CMTEBPairClassification.py
@@ -11,7 +11,7 @@ class Ocnli(AbsTaskPairClassification):
         reference="https://arxiv.org/abs/2010.05444",
         dataset={
             "path": "C-MTEB/OCNLI",
-            "revision": "7bd1ee0ae3a820d8a4b6f8a624063f8504a3564d",
+            "revision": "66e76a618a34d6d565d5538088562851e6daa7ec",
         },
         type="PairClassification",
         category="s2s",

diff --git a/mteb/tasks/Reranking/fr/AlloprofReranking.py b/mteb/tasks/Reranking/fr/AlloprofReranking.py
@@ -11,8 +11,8 @@ class AlloprofReranking(AbsTaskReranking):
         description="This dataset was provided by AlloProf, an organisation in Quebec, Canada offering resources and a help forum curated by a large number of teachers to students on all subjects taught from in primary and secondary school",
         reference="https://huggingface.co/datasets/antoinelb7/alloprof",
         dataset={
-            "path": "mteb/alloprof-reranking",
-            "revision": "666fdacebe0291776e86f29345663dfaf80a0db9",
+            "path": "lyon-nlp/mteb-fr-reranking-alloprof-s2p",
+            "revision": "e40c8a63ce02da43200eccb5b0846fcaa888f562",
         },
         type="Reranking",
         category="s2s",

diff --git a/mteb/tasks/Reranking/zh/CMTEBReranking.py b/mteb/tasks/Reranking/zh/CMTEBReranking.py
@@ -69,7 +69,7 @@ class CMedQAv1(AbsTaskReranking):
         reference="https://github.com/zhangsheng93/cMedQA",
         dataset={
             "path": "C-MTEB/CMedQAv1-reranking",
-            "revision": "cd540c506dae1cf9e9a59c3e06f42030d54e7301",
+            "revision": "8d7f1e942507dac42dc58017c1a001c3717da7df",
         },
         type="Reranking",
         category="s2s",

diff --git a/mteb/tasks/Retrieval/de/GermanDPRRetrieval.py b/mteb/tasks/Retrieval/de/GermanDPRRetrieval.py
@@ -17,7 +17,7 @@ class GermanDPR(AbsTaskRetrieval):
         reference="https://www.deepset.ai/germanquad",
         dataset={
             "path": "deepset/germanquad",
-            "revision": "5129d02422a66be600ac89cd3e8531b4f97d347d",
+            "revision": "fff05ceaf2ffbe5b65c7e0c57e678f7b7e1a0581",
         },
         type="Retrieval",
         category="s2p",

diff --git a/mteb/tasks/Retrieval/de/GermanQuADRetrieval.py b/mteb/tasks/Retrieval/de/GermanQuADRetrieval.py
@@ -33,7 +33,7 @@ class GermanQuADRetrieval(AbsTaskRetrieval):
         description="Context Retrieval for German Question Answering",
         reference="https://www.kaggle.com/datasets/GermanQuAD",
         dataset={
-            "path": "mteb/germanquad",
+            "path": "mteb/germanquad-retrieval",
             "revision": "f5c87ae5a2e7a5106606314eef45255f03151bb3",
         },
         type="Retrieval",

diff --git a/mteb/tasks/Retrieval/en/CQADupstackTexRetrieval.py b/mteb/tasks/Retrieval/en/CQADupstackTexRetrieval.py
@@ -12,7 +12,7 @@ class CQADupstackTexRetrieval(AbsTaskRetrieval):
         reference="http://nlp.cis.unimelb.edu.au/resources/cqadupstack/",
         dataset={
             "path": "mteb/cqadupstack-tex",
-            "revision": "6c6430d3a6d36f8d2a829195bc5dc94d7e063e53",
+            "revision": "46989137a86843e03a6195de44b09deda022eec7",
         },
         type="Retrieval",
         category="s2p",

diff --git a/mteb/tasks/Retrieval/en/QuoraRetrieval.py b/mteb/tasks/Retrieval/en/QuoraRetrieval.py
@@ -10,7 +10,7 @@ class QuoraRetrieval(AbsTaskRetrieval):
         name="QuoraRetrieval",
         dataset={
             "path": "mteb/quora",
-            "revision": "0be27e93455051e531182b85e85e425aba12e9d4",
+            "revision": "e4e08e0b7dbe3c8700f0daef558ff32256715259",
         },
         description=(
             "QuoraRetrieval is based on questions that are marked as duplicates on the Quora platform. Given a"

diff --git a/mteb/tasks/Retrieval/en/SCIDOCSRetrieval.py b/mteb/tasks/Retrieval/en/SCIDOCSRetrieval.py
@@ -10,7 +10,7 @@ class SCIDOCS(AbsTaskRetrieval):
         name="SCIDOCS",
         dataset={
             "path": "mteb/scidocs",
-            "revision": "56a6d0140cf6356659e2a7c1413286a774468d44",
+            "revision": "f8c2fcf00f625baaa80f62ec5bd9e1fff3b8ae88",
         },
         description=(
             "SciDocs, a new evaluation benchmark consisting of seven document-level tasks ranging from citation"

diff --git a/mteb/tasks/Retrieval/en/TRECCOVIDRetrieval.py b/mteb/tasks/Retrieval/en/TRECCOVIDRetrieval.py
@@ -12,7 +12,7 @@ class TRECCOVID(AbsTaskRetrieval):
         reference="https://ir.nist.gov/covidSubmit/index.html",
         dataset={
             "path": "mteb/trec-covid",
-            "revision": "1271c7809071a13532e05f25fb53511ffce77117",
+            "revision": "bb9466bac8153a0349341eb1b22e06409e78ef4e",
         },
         type="Retrieval",
         category="s2p",