[python] Bug fix for first_metric_only on earlystopping.

python-package/lightgbm/callback.py

@@ -8,6 +8,7 @@
 from operator import gt, lt
 
 from .compat import range_
+from .basic import LightGBMError
 
 
 class EarlyStopException(Exception):
@@ -214,7 +215,24 @@ def _callback(env):
             _init(env)
         if not enabled[0]:
             return
+        if first_metric_only:
+            eval_metric = None
+            for metric_alias in ['metric', 'metrics', 'metric_types']:
+                if metric_alias in env.params.keys():
+                    if isinstance(env.params[metric_alias], (tuple, list)):
+                        eval_metric = env.params[metric_alias][0]
+                    else:
+                        eval_metric = env.params[metric_alias]
+                    break
+            if eval_metric is None:
+                raise LightGBMError("`metric` should be specified if first_metric_only==True.")
         for i in range_(len(env.evaluation_result_list)):
+            metric_key = env.evaluation_result_list[i][1]
+            if metric_key.split(" ")[0] == "train":
+                continue  # train metric doesn't used on early stopping.
+            if first_metric_only:
+                if metric_key != "valid {}".format(eval_metric) and metric_key != eval_metric and eval_metric != "":
+                    continue
             score = env.evaluation_result_list[i][2]
             if best_score_list[i] is None or cmp_op[i](score, best_score[i]):
                 best_score[i] = score
@@ -224,13 +242,16 @@ def _callback(env):
                 if verbose:
                     print('Early stopping, best iteration is:\n[%d]\t%s' % (
                         best_iter[i] + 1, '\t'.join([_format_eval_result(x) for x in best_score_list[i]])))
+                    if first_metric_only:
+                        print("Evaluating only: {}".format(metric_key))
                 raise EarlyStopException(best_iter[i], best_score_list[i])
             if env.iteration == env.end_iteration - 1:
                 if verbose:
                     print('Did not meet early stopping. Best iteration is:\n[%d]\t%s' % (
                         best_iter[i] + 1, '\t'.join([_format_eval_result(x) for x in best_score_list[i]])))
+                    if first_metric_only:
+                        print("Evaluating only: {}".format(metric_key))
                 raise EarlyStopException(best_iter[i], best_score_list[i])
-            if first_metric_only:  # the only first metric is used for early stopping
-                break
     _callback.order = 30
+    _callback.first_metric_only = first_metric_only
     return _callback

python-package/lightgbm/engine.py

@@ -199,6 +199,8 @@ def train(params, train_set, num_boost_round=100,
         callbacks = set()
     else:
         for i, cb in enumerate(callbacks):
+            if getattr(cb, 'first_metric_only', False) and feval is not None:
+                raise LightGBMError("`first_metric_only` and `feval` are not available at the same time.")
             cb.__dict__.setdefault('order', i - len(callbacks))
         callbacks = set(callbacks)
 
@@ -209,6 +211,8 @@ def train(params, train_set, num_boost_round=100,
         callbacks.add(callback.print_evaluation(verbose_eval))
 
     if early_stopping_rounds is not None:
+        if first_metric_only and feval is not None:
+            raise LightGBMError("`first_metric_only` and `feval` are not available at the same time.")
         callbacks.add(callback.early_stopping(early_stopping_rounds, first_metric_only, verbose=bool(verbose_eval)))
 
     if learning_rates is not None:
@@ -533,6 +537,8 @@ def cv(params, train_set, num_boost_round=100,
         callbacks = set()
     else:
         for i, cb in enumerate(callbacks):
+            if getattr(cb, 'first_metric_only', False) and feval is not None:
+                raise LightGBMError("`first_metric_only` and `feval` are not available at the same time.")
             cb.__dict__.setdefault('order', i - len(callbacks))
         callbacks = set(callbacks)
     if early_stopping_rounds is not None:

tests/python_package_test/test_engine.py

@@ -15,6 +15,7 @@
                               load_iris, load_svmlight_file)
 from sklearn.metrics import log_loss, mean_absolute_error, mean_squared_error, roc_auc_score
 from sklearn.model_selection import train_test_split, TimeSeriesSplit, GroupKFold
+from numpy.testing import assert_raises_regex
 
 try:
     import cPickle as pickle
@@ -1417,13 +1418,10 @@ def test_get_split_value_histogram(self):
         self.assertRaises(lgb.basic.LightGBMError, gbm.get_split_value_histogram, 2)
 
     def test_early_stopping_for_only_first_metric(self):
+        np.random.seed(71)
+        # Regression test
         X, y = load_boston(True)
         X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
-        params = {
-            'objective': 'regression',
-            'metric': 'None',
-            'verbose': -1
-        }
         lgb_train = lgb.Dataset(X_train, y_train)
         lgb_eval = lgb.Dataset(X_test, y_test, reference=lgb_train)
 
@@ -1434,25 +1432,206 @@ def decreasing_metric(preds, train_data):
 
         def constant_metric(preds, train_data):
             return ('constant_metric', 0.0, False)
-
         # test that all metrics are checked (default behaviour)
+        params = {
+            'objective': 'regression',
+            'metric': 'None',
+            'verbose': -1,
+            'seed': 123,
+        }
         gbm = lgb.train(params, lgb_train, num_boost_round=20, valid_sets=[lgb_eval],
                         feval=lambda preds, train_data: [decreasing_metric(preds, train_data),
                                                          constant_metric(preds, train_data)],
                         early_stopping_rounds=5, verbose_eval=False)
         self.assertEqual(gbm.best_iteration, 1)
-
         # test that only the first metric is checked
+        with assert_raises_regex(lgb.basic.LightGBMError,
+                                 '`first_metric_only` and `feval` are not available at the same time.'):
+            lgb.train(dict(params, first_metric_only=True), lgb_train,
+                      num_boost_round=20, valid_sets=[lgb_eval],
+                      feval=lambda preds, train_data: [decreasing_metric(preds, train_data),
+                                                       constant_metric(preds, train_data)],
+                      early_stopping_rounds=5, verbose_eval=False)
+        # test that various combination of metrics.
+        params = {
+            'objective': 'regression',
+            'learning_rate': 0.05,
+            'num_leaves': 5,
+            'metric': 'l2',
+            'verbose': -1,
+            'seed': 123,
+        }
         gbm = lgb.train(dict(params, first_metric_only=True), lgb_train,
-                        num_boost_round=20, valid_sets=[lgb_eval],
+                        num_boost_round=150, valid_sets=[lgb_eval],
+                        early_stopping_rounds=5, verbose_eval=False)
+        self.assertEqual(gbm.best_iteration, 116)
+
+        def metrics_combination_train_regression(metric_list, assumed_iteration, first_metric_only, assertion=True):
+            params = {
+                'objective': 'regression',
+                'learning_rate': 0.05,
+                'num_leaves': 10,
+                'metric': metric_list,
+                'verbose': -1,
+                'seed': 123,
+            }
+            gbm = lgb.train(dict(params, first_metric_only=first_metric_only), lgb_train,
+                            num_boost_round=300, valid_sets=[lgb_eval],
+                            early_stopping_rounds=5, verbose_eval=False)
+            if assertion:
+                self.assertEqual(gbm.best_iteration, assumed_iteration)
+            else:
+                return gbm.best_iteration
+
+        def metrics_combination_cv_regression(metric_list, assumed_iteration, first_metric_only,
+                                              eval_train_metric, assertion=True):
+            params = {
+                'objective': 'regression',
+                'learning_rate': 0.05,
+                'num_leaves': 10,
+                'metric': metric_list,
+                'verbose': -1,
+                'seed': 123,
+            }
+            ret = lgb.cv(dict(params, first_metric_only=first_metric_only),
+                         stratified=False,
+                         train_set=lgb_train,
+                         num_boost_round=300,
+                         early_stopping_rounds=5, verbose_eval=False,
+                         eval_train_metric=eval_train_metric)
+            if assertion:
+                self.assertEqual(len(ret[list(ret.keys())[0]]), assumed_iteration)
+            else:
+                return len(ret[list(ret.keys())[0]])
+
+        n_iter_metric_1 = metrics_combination_train_regression('l2', 0, True, assertion=False)
+        n_iter_metric_2 = metrics_combination_train_regression('l1', 0, True, assertion=False)
+        assert n_iter_metric_1 != n_iter_metric_2, (n_iter_metric_1, n_iter_metric_2)
+        n_iter_metric_min = min([n_iter_metric_1, n_iter_metric_2])
+        metrics_combination_train_regression('l2', n_iter_metric_1, True)
+        metrics_combination_train_regression('l1', n_iter_metric_2, True)
+        metrics_combination_train_regression(['l2', 'l1'], n_iter_metric_1, True)
+        metrics_combination_train_regression(['l1', 'l2'], n_iter_metric_2, True)
+        metrics_combination_train_regression(['l2', 'l1'], n_iter_metric_min, False)
+        metrics_combination_train_regression(['l1', 'l2'], n_iter_metric_min, False)
+
+        n_iter_metric_1 = metrics_combination_cv_regression('l2', 0, True, False, assertion=False)
+        n_iter_metric_2 = metrics_combination_cv_regression('l1', 0, True, False, assertion=False)
+        assert n_iter_metric_1 != n_iter_metric_2, (n_iter_metric_1, n_iter_metric_2)
+        n_iter_metric_min = min([n_iter_metric_1, n_iter_metric_2])
+        metrics_combination_cv_regression('l2', n_iter_metric_1, True, False)
+        metrics_combination_cv_regression('l1', n_iter_metric_2, True, False)
+        metrics_combination_cv_regression(['l2', 'l1'], n_iter_metric_1, True, False)
+        metrics_combination_cv_regression(['l1', 'l2'], n_iter_metric_2, True, False)
+        metrics_combination_cv_regression(['l2', 'l1'], n_iter_metric_min, False, False)
+        metrics_combination_cv_regression(['l1', 'l2'], n_iter_metric_min, False, False)
+        metrics_combination_cv_regression('l2', n_iter_metric_1, True, True)
+        metrics_combination_cv_regression('l1', n_iter_metric_2, True, True)
+        metrics_combination_cv_regression(['l2', 'l1'], n_iter_metric_1, True, True)
+        metrics_combination_cv_regression(['l1', 'l2'], n_iter_metric_2, True, True)
+        metrics_combination_cv_regression(['l2', 'l1'], n_iter_metric_min, False, True)
+        metrics_combination_cv_regression(['l1', 'l2'], n_iter_metric_min, False, True)
+        # Classification test
+        X, y = load_breast_cancer(True)
+        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
+        lgb_train = lgb.Dataset(X_train, y_train)
+        lgb_eval = lgb.Dataset(X_test, y_test, reference=lgb_train)
+        # test that all metrics are checked (default behaviour)
+        params = {
+            'objective': 'binary',
+            'metric': 'None',
+            'verbose': -1,
+            'seed': 123,
+        }
+        gbm = lgb.train(params, lgb_train, num_boost_round=20, valid_sets=[lgb_eval],
                         feval=lambda preds, train_data: [decreasing_metric(preds, train_data),
                                                          constant_metric(preds, train_data)],
                         early_stopping_rounds=5, verbose_eval=False)
-        self.assertEqual(gbm.best_iteration, 20)
-        # ... change the order of metrics
+        self.assertEqual(gbm.best_iteration, 1)
+        # test that only the first metric is checked
+        with assert_raises_regex(lgb.basic.LightGBMError,
+                                 '`first_metric_only` and `feval` are not available at the same time.'):
+            lgb.train(dict(params, first_metric_only=True), lgb_train,
+                      num_boost_round=20, valid_sets=[lgb_eval],
+                      feval=lambda preds, train_data: [decreasing_metric(preds, train_data),
+                                                       constant_metric(preds, train_data)],
+                      early_stopping_rounds=5, verbose_eval=False)
+        # test that various combination of metrics.
+        params = {
+            'objective': 'binary',
+            'learning_rate': 0.05,
+            'num_leaves': 5,
+            'metric': 'binary_logloss',
+            'verbose': -1,
+            'seed': 123,
+        }
         gbm = lgb.train(dict(params, first_metric_only=True), lgb_train,
-                        num_boost_round=20, valid_sets=[lgb_eval],
-                        feval=lambda preds, train_data: [constant_metric(preds, train_data),
-                                                         decreasing_metric(preds, train_data)],
+                        num_boost_round=150, valid_sets=[lgb_eval],
                         early_stopping_rounds=5, verbose_eval=False)
-        self.assertEqual(gbm.best_iteration, 1)
+        self.assertEqual(gbm.best_iteration, 73)
+
+        def metrics_combination_train(metric_list, assumed_iteration, first_metric_only, assertion=True):
+            params = {
+                'objective': 'binary',
+                'learning_rate': 0.05,
+                'num_leaves': 5,
+                'metric': metric_list,
+                'verbose': -1,
+                'seed': 123,
+            }
+            gbm = lgb.train(dict(params, first_metric_only=first_metric_only), lgb_train,
+                            num_boost_round=150, valid_sets=[lgb_eval],
+                            early_stopping_rounds=5, verbose_eval=False)
+            if assertion:
+                self.assertEqual(gbm.best_iteration, assumed_iteration)
+            else:
+                return gbm.best_iteration
+
+        def metrics_combination_cv(metric_list, assumed_iteration, first_metric_only,
+                                   eval_train_metric, assertion=True):
+            params = {
+                'objective': 'binary',
+                'learning_rate': 0.05,
+                'num_leaves': 5,
+                'metric': metric_list,
+                'verbose': -1,
+                'seed': 123,
+            }
+            ret = lgb.cv(dict(params, first_metric_only=first_metric_only),
+                         train_set=lgb_train,
+                         num_boost_round=150,
+                         early_stopping_rounds=5, verbose_eval=False,
+                         eval_train_metric=eval_train_metric)
+            if assertion:
+                self.assertEqual(len(ret[list(ret.keys())[0]]), assumed_iteration)
+            else:
+                return len(ret[list(ret.keys())[0]])
+
+        n_iter_metric_1 = metrics_combination_train('binary_logloss', 0, True, assertion=False)
+        n_iter_metric_2 = metrics_combination_train('auc', 0, True, assertion=False)
+        assert n_iter_metric_1 != n_iter_metric_2, (n_iter_metric_1, n_iter_metric_2)
+        n_iter_metric_min = min([n_iter_metric_1, n_iter_metric_2])
+        metrics_combination_train('binary_logloss', n_iter_metric_1, True)
+        metrics_combination_train('auc', n_iter_metric_2, True)
+        metrics_combination_train(['binary_logloss', 'auc'], n_iter_metric_1, True)
+        metrics_combination_train(['auc', 'binary_logloss'], n_iter_metric_2, True)
+        metrics_combination_train(['binary_logloss', 'auc'], n_iter_metric_min, False)
+        metrics_combination_train(['auc', 'binary_logloss'], n_iter_metric_min, False)
+
+        n_iter_metric_1 = metrics_combination_cv('binary_logloss', 0, True, False, assertion=False)
+        n_iter_metric_2 = metrics_combination_cv('auc', 0, True, False, assertion=False)
+        assert n_iter_metric_1 != n_iter_metric_2, (n_iter_metric_1, n_iter_metric_2)
+        n_iter_metric_min = min([n_iter_metric_1, n_iter_metric_2])
+        metrics_combination_cv('binary_logloss', n_iter_metric_1, True, False)
+        metrics_combination_cv('auc', n_iter_metric_2, True, False)
+        metrics_combination_cv(['binary_logloss', 'auc'], n_iter_metric_1, True, False)
+        metrics_combination_cv(['auc', 'binary_logloss'], n_iter_metric_2, True, False)
+        metrics_combination_cv(['binary_logloss', 'auc'], n_iter_metric_min, False, False)
+        metrics_combination_cv(['auc', 'binary_logloss'], n_iter_metric_min, False, False)
+        metrics_combination_cv('binary_logloss', n_iter_metric_1, True, True)
+        metrics_combination_cv('auc', n_iter_metric_2, True, True)
+        metrics_combination_cv(['binary_logloss', 'auc'], n_iter_metric_1, True, True)
+        metrics_combination_cv(['auc', 'binary_logloss'], n_iter_metric_2, True, True)
+        metrics_combination_cv(['binary_logloss', 'auc'], n_iter_metric_min, False, True)
+        metrics_combination_cv(['auc', 'binary_logloss'], n_iter_metric_min, False, True)
+        np.random.seed()