diff --git a/python-package/xgboost/sklearn.py b/python-package/xgboost/sklearn.py
index bf49202e7116..7e466b0a63f1 100644
--- a/python-package/xgboost/sklearn.py
+++ b/python-package/xgboost/sklearn.py
@@ -510,6 +510,8 @@ def fit(self, X, y, sample_weight=None, base_margin=None,
                 raise TypeError('Unexpected input type for `eval_set`')
             if sample_weight_eval_set is None:
                 sample_weight_eval_set = [None] * len(eval_set)
+            else:
+                assert len(eval_set) == len(sample_weight_eval_set)
             evals = list(
                 DMatrix(eval_set[i][0], label=eval_set[i][1], missing=self.missing,
                         weight=sample_weight_eval_set[i], nthread=self.n_jobs)
@@ -788,6 +790,8 @@ def fit(self, X, y, sample_weight=None, base_margin=None,
         if eval_set is not None:
             if sample_weight_eval_set is None:
                 sample_weight_eval_set = [None] * len(eval_set)
+            else:
+                assert len(sample_weight_eval_set) == len(eval_set)
             evals = list(
                 DMatrix(eval_set[i][0],
                         label=self._le.transform(eval_set[i][1]),
diff --git a/tests/python/test_with_sklearn.py b/tests/python/test_with_sklearn.py
index d2b04927e865..504b92fdc414 100644
--- a/tests/python/test_with_sklearn.py
+++ b/tests/python/test_with_sklearn.py
@@ -596,6 +596,17 @@ def test_validation_weights_xgbmodel():
     assert all((logloss_with_weights[i] != logloss_without_weights[i]
                 for i in [0, 1]))
 
+    with pytest.raises(AssertionError):
+        # length of eval set and sample weight doesn't match.
+        clf.fit(X_train, y_train, sample_weight=weights_train,
+                eval_set=[(X_train, y_train), (X_test, y_test)],
+                sample_weight_eval_set=[weights_train])
+
+    with pytest.raises(AssertionError):
+        cls = xgb.XGBClassifier()
+        cls.fit(X_train, y_train, sample_weight=weights_train,
+                eval_set=[(X_train, y_train), (X_test, y_test)],
+                sample_weight_eval_set=[weights_train])
 
 def test_validation_weights_xgbclassifier():
     from sklearn.datasets import make_hastie_10_2