MacroF MicroF to version 2.0

.gitignore

@@ -3,3 +3,7 @@ data/
 build
 dist
 __pycache__
+**.pyc
+.idea
+.sacrebleu
+

README.md

@@ -215,6 +215,14 @@ TER related arguments (The defaults replicate TERCOM's behavior):
   --ter-asian-support   Enables special treatment of Asian characters (Default: False)
   --ter-no-punct        Removes punctuation. (Default: False)
   --ter-normalized      Applies basic normalization and tokenization. (Default: False)
+
+MacroF and MicroF related arguments (the defaults replicate Gowda et al NAACL 2021):
+  --f-beta F_BETA       Determine the importance of recall w.r.t precision. (Default: 1)
+  --f-lowercase         Enable case-insensitivity. (Default: False)
+  --f-tokenize {none,zh,13a,char,intl,ja-mecab}
+                        Tokenization method to use for Macro and MicroF, same as BLEU --tokenize. (Default: None)
+  --f-smooth-value F_SMOOTH_VALUE
+                        The smoothing value. only add-k smoothing method is supported for MacroF and MicroF (Default: 1)
 ```
 
 ### Version Signatures

sacrebleu/__init__.py

@@ -29,3 +29,4 @@
 from .compat import corpus_bleu, raw_corpus_bleu, sentence_bleu  # noqa: F401
 from .compat import corpus_chrf, sentence_chrf  # noqa: F401
 from .compat import corpus_ter, sentence_ter  # noqa: F401
+from .compat import corpus_f, corpus_macrof, corpus_microf # noqa: F401

sacrebleu/compat.py

@@ -1,6 +1,7 @@
-from typing import Sequence, Optional
+from typing import Sequence, Optional, Union
+from functools import partial
 
-from .metrics import BLEU, CHRF, TER, BLEUScore, CHRFScore, TERScore
+from .metrics import METRICS, BLEU, CHRF, TER, BLEUScore, CHRFScore, TERScore, ClassifierEval, MultiClassMeasure
 
 
 ######################################################################
@@ -196,3 +197,40 @@ def sentence_ter(hypothesis: str,
         asian_support=asian_support,
         case_sensitive=case_sensitive)
     return metric.sentence_score(hypothesis, references)
+
+
+def corpus_f(hypotheses: Union[str, Sequence[str]],
+            references: Union[str, Sequence[Sequence[str]]],
+            average: str,
+            smooth_value=ClassifierEval.DEF_SMOOTH_VAL,
+            beta=ClassifierEval.DEF_F_BETA,
+            force=False,
+            lowercase=False,
+            tokenize=ClassifierEval.TOKENIZER_DEFAULT) -> MultiClassMeasure:
+    """
+    Computes F-measure on a corpus
+    :param average: what kind of averaging to use for obtaining corpus level performance from types.
+         Options: macro, micro
+    :param hypotheses: The system stream (a sequence of segments)
+    :param references: A list of one or more reference streams (each a sequence of segments)
+    :param smooth_value: The smoothing value for `add-k` method. set smooth_value=0 to disable.
+                 Does not influence macro-average
+    :param f_beta: β value that weighs recall in F-measure
+    :param force: Ignore data that looks already tokenized
+    :param lowercase: Lowercase the data
+    :param tokenize: The tokenizer to use
+    :return: a `MultiClassMeasure` object
+    """
+
+    # limiting to these special cases because others are not tested
+    assert average in ('macro', 'micro')
+    args = dict(smooth_value=smooth_value, force=force,
+                lowercase=lowercase, tokenize=tokenize, beta=beta)
+    metric = METRICS[average.upper() + 'F'](**args)
+    return metric.corpus_score(hypotheses, references)
+
+"""Computes Macro-F measure on a corpus. Refer to `corpus_f()` for additional arguments."""
+corpus_macrof = partial(corpus_f, average='macro')
+
+"""Computes Micro-F measure on a corpus. Refer to `corpus_f()` for additional arguments."""
+corpus_microf = partial(corpus_f, average='micro')

sacrebleu/metrics/__init__.py

@@ -1,11 +1,17 @@
 """The implementation of various metrics."""
 
+from functools import partial
 from .bleu import BLEU, BLEUScore   # noqa: F401
 from .chrf import CHRF, CHRFScore   # noqa: F401
 from .ter import TER, TERScore      # noqa: F401
+from .clseval import ClassifierEval, MultiClassMeasure
+
 
 METRICS = {
     'BLEU': BLEU,
     'CHRF': CHRF,
     'TER': TER,
 }
+
+METRICS['MACROF'] = partial(ClassifierEval, average='macro', max_ngram_order=1)
+METRICS['MICROF'] = partial(ClassifierEval, average='micro', max_ngram_order=1)

sacrebleu/metrics/base.py

@@ -17,6 +17,11 @@
 
 
 class Score:
+
+    # https://docs.python.org/3/reference/datamodel.html#slots
+    # __slots__ minimizes memory usage when many objs are created e.g. clseval.ClassMeasure
+    __slots__ = ('name', 'score', '_mean', '_ci', '_verbose')
+
     """A base score class to derive from.
 
     :param name: The name of the underlying metric.