Fix hyperopt tuning via fine_tune method. (#27)

Author: talmago

poetry.lock

@@ -34,6 +34,7 @@ spacy = "^3.4.4"
 sklearn-crfsuite = "^0.3.6"
 joblib = "^1.2.0"
 scikit-learn = "^1.2.0"
+pytest = "^7.2.2"
 
 [tool.poetry.group.dev.dependencies]
 autopep8 = "^2.0.1"

pyproject.toml

@@ -2,16 +2,22 @@
 import joblib
 import warnings
 import sklearn_crfsuite
+import sklearn_crfsuite.metrics as _metrics
 
 from collections import Counter
 from pathlib import Path
 from typing import Dict, Text, Any, Optional, List, Tuple, Union, Callable
 
 from spacy.language import Language
 from spacy.tokens.doc import Doc
-from sklearn.metrics import classification_report, f1_score
-
-from spacy_crfsuite.bilou import entity_name_from_tag, bilou_prefix_from_tag, NO_ENTITY_TAG
+from sklearn.metrics import classification_report
+
+from spacy_crfsuite.bilou import (
+    entity_name_from_tag,
+    bilou_prefix_from_tag,
+    NO_ENTITY_TAG,
+    BILOU_PREFIXES,
+)
 from spacy_crfsuite.compat import CRF
 from spacy_crfsuite.features import CRFToken, Featurizer
 from spacy_crfsuite.tokenizer import Token, SpacyTokenizer
@@ -254,8 +260,13 @@ def fine_tune(
 
         X_train = [self._crf_tokens_to_features(sent) for sent in val_samples]
         y_train = [self._crf_tokens_to_tags(sent) for sent in val_samples]
-        labels = list(set(itertools.chain.from_iterable(y_train)) - {NO_ENTITY_TAG})
-        f1_scorer = make_scorer(f1_score, average="weighted", labels=labels)
+
+        labels = set(itertools.chain.from_iterable(y_train)) - {NO_ENTITY_TAG}
+        labels = list(labels)
+
+        f1_scorer = make_scorer(
+            _metrics.flat_f1_score, average="weighted", labels=labels, zero_division=1
+        )
         rs = RandomizedSearchCV(
             crf,
             params_space,

spacy_crfsuite/crf_extractor.py

@@ -40,10 +40,8 @@ def create_component(nlp, name):
         "George Walker Bush (born July 6, 1946) is an American politician and businessman "
         "who served as the 43rd president of the United States from 2001 to 2009.")
 
-    for ent in doc.ents:
-        print(ent, "-", ent.label_)
-
-    # Output:
-    # George Walker Bush - PER
-    # American - MISC
-    # United States - LOC
+    assert [(ent.text, ent.label_) for ent in doc.ents] == [
+        ('George Walker Bush', 'PER'),
+        ('American', 'MISC'),
+        ('United States', 'LOC')
+    ]

spacy_crfsuite/test.py

@@ -0,0 +1,59 @@
+import pytest
+
+from spacy_crfsuite import CRFExtractor, read_file
+from spacy_crfsuite.tokenizer import SpacyTokenizer
+from spacy_crfsuite.train import gold_example_to_crf_tokens
+
+
+@pytest.fixture()
+def dev_examples(en_core_web_md):
+    tokenizer = SpacyTokenizer(en_core_web_md)
+
+    dev_examples = [
+        gold_example_to_crf_tokens(
+            ex, tokenizer=tokenizer, use_dense_features=False, bilou=True
+        ) for ex in read_file("examples/restaurent_search.md")
+    ]
+
+    return dev_examples
+
+
+def test_hyperparam_optim(dev_examples):
+    crf_extractor = CRFExtractor(component_config={
+        "features": [
+            [
+                "low",
+                "title",
+                "upper",
+                "pos",
+                "pos2"
+            ],
+            [
+                "low",
+                "bias",
+                "prefix5",
+                "prefix2",
+                "suffix5",
+                "suffix3",
+                "suffix2",
+                "upper",
+                "title",
+                "digit",
+                "pos",
+                "pos2"
+            ],
+            [
+                "low",
+                "title",
+                "upper",
+                "pos",
+                "pos2"
+            ],
+        ],
+        "c1": 0.01,
+        "c2": 0.22
+    })
+
+    rs = crf_extractor.fine_tune(dev_examples, cv=5, n_iter=30, random_state=42)
+    assert rs.best_params_ == {'c1': 0.029919384304340338, 'c2': 0.10056154322399698}
+    assert rs.best_score_ == 0.39999999999999997