add a tokenizer option to rouge

metrics/rouge/README.md

@@ -42,6 +42,15 @@ At minimum, this metric takes as input a list of predictions and a list of refer
 {'rouge1': 1.0, 'rouge2': 1.0, 'rougeL': 1.0, 'rougeLsum': 1.0}
 ```
 
+One can also pass a custom tokenizer which is especially useful for non-latin languages.
+```python
+>>> results = rouge.compute(predictions=predictions,
+...                         references=references,
+                            tokenizer=lambda x: x.split())
+>>> print(results)
+{'rouge1': 1.0, 'rouge2': 1.0, 'rougeL': 1.0, 'rougeLsum': 1.0}
+```
+
 It can also deal with lists of references for each predictions:
 ```python
 >>> rouge = evaluate.load('rouge')

metrics/rouge/rouge.py

@@ -80,6 +80,16 @@
 """
 
 
+class Tokenizer:
+    """Helper class to wrap a callable into a class with a `tokenize` method as used by rouge-score."""
+
+    def __init__(self, tokenizer_func):
+        self.tokenizer_func = tokenizer_func
+
+    def tokenize(self, text):
+        return self.tokenizer_func(text)
+
+
 @evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION)
 class Rouge(evaluate.Metric):
     def _info(self):
@@ -108,13 +118,18 @@ def _info(self):
             ],
         )
 
-    def _compute(self, predictions, references, rouge_types=None, use_aggregator=True, use_stemmer=False):
+    def _compute(
+        self, predictions, references, rouge_types=None, use_aggregator=True, use_stemmer=False, tokenizer=None
+    ):
         if rouge_types is None:
             rouge_types = ["rouge1", "rouge2", "rougeL", "rougeLsum"]
 
         multi_ref = isinstance(references[0], list)
 
-        scorer = rouge_scorer.RougeScorer(rouge_types=rouge_types, use_stemmer=use_stemmer)
+        if tokenizer is not None:
+            tokenizer = Tokenizer(tokenizer)
+
+        scorer = rouge_scorer.RougeScorer(rouge_types=rouge_types, use_stemmer=use_stemmer, tokenizer=tokenizer)
         if use_aggregator:
             aggregator = scoring.BootstrapAggregator()
         else: