From 12ba7a7f4da1d9d1aeb9dcce50ed519c9bcd4d7d Mon Sep 17 00:00:00 2001 From: Daniel Grady Date: Mon, 22 May 2017 16:00:17 -0700 Subject: [PATCH] BUG: Fix behavior of argmax and argmin with inf (#16449) Closes #13595 The implementations of `nanargmin` and `nanargmax` in `nanops` were forcing the `_get_values` utility function to always mask out infinite values. For example, in `nanargmax`, >>> nanops._get_values(np.array([1, np.nan, np.inf]), True, isfinite=True, fill_value_typ='-inf') (array([ 1., -inf, -inf]), array([False, True, True], dtype=bool), dtype('float64'), numpy.float64) The first element of the result tuple (the masked version of the values array) is used for actually finding the max or min argument. As a result, infinite values could never be correctly recognized as the maximum or minimum values in an array. This also affects the behavior of `Series.idxmax` with string data (or the `object` dtype generally). Previously, `nanargmax` would always attempt to coerce its input to float, even when there were no missing values. Now, it will not, and so will work correctly in the particular case of a `Series` of strings with no missing values. However, because it's difficult to ensure that `nanargmin` and `nanargmax` will behave consistently for arbitrary `Series` of `object` with and without missing values, these functions are now explicitly disallowed for `object`. --- doc/source/whatsnew/v0.21.0.txt | 25 ++++++++++++ pandas/core/nanops.py | 8 ++-- pandas/tests/groupby/test_groupby.py | 3 +- pandas/tests/series/test_operators.py | 55 +++++++++++++++++++++++++++ 4 files changed, 86 insertions(+), 5 deletions(-) diff --git a/doc/source/whatsnew/v0.21.0.txt b/doc/source/whatsnew/v0.21.0.txt index 8b2c4d16f4e1a0..9f9ba140d35d57 100644 --- a/doc/source/whatsnew/v0.21.0.txt +++ b/doc/source/whatsnew/v0.21.0.txt @@ -233,6 +233,30 @@ Dtype Conversions - Inconsistent behavior in ``.where()`` with datetimelikes which would raise rather than coerce to ``object`` (:issue:`16402`) - Bug in assignment against ``int64`` data with ``np.ndarray`` with ``float64`` dtype may keep ``int64`` dtype (:issue:`14001`) +Dropping argmin/argmax support for arbitrary dtypes +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +:func:`Series.argmin` and :func:`Series.argmax` will now raise a ``TypeError`` when used with ``object`` dtypes. Previously, these functions would attempt to coerce their arguments to floats, which usually led to a ``ValueError``. Now, ``object`` dtypes are explicitly disallowed with ``argmax`` and related functions. For example, the old behavior produced a somewhat confusing message: + +.. code-block:: ipython + + In [1]: s = pd.Series(['foo', 'bar']) + + In [2]: s.argmax() + ... + ValueError: could not convert string to float: 'bar' + +This has now changed to + +.. code-block:: ipython + + In [1]: s = pd.Series(['foo', 'bar']) + + In [2]: s.argmax() + ... + TypeError: reduction operation 'argmax' not allowed for this dtype + + .. _whatsnew_0210.api.na_changes: NA naming Changes @@ -369,6 +393,7 @@ Reshaping - Fixes regression from 0.20, :func:`Series.aggregate` and :func:`DataFrame.aggregate` allow dictionaries as return values again (:issue:`16741`) - Fixes dtype of result with integer dtype input, from :func:`pivot_table` when called with ``margins=True`` (:issue:`17013`) - Bug in :func:`crosstab` where passing two ``Series`` with the same name raised a ``KeyError`` (:issue:`13279`) +- :func:`Series.argmin`, :func:`Series.argmax`, and their counterparts on ``DataFrame`` and groupby objects work correctly with floating point data that contains infinite values (:issue:`13595`). Numeric ^^^^^^^ diff --git a/pandas/core/nanops.py b/pandas/core/nanops.py index 2f4e437c0ae616..b2bbf1c75b7ea0 100644 --- a/pandas/core/nanops.py +++ b/pandas/core/nanops.py @@ -486,23 +486,23 @@ def reduction(values, axis=None, skipna=True): nanmax = _nanminmax('max', fill_value_typ='-inf') +@disallow('O') def nanargmax(values, axis=None, skipna=True): """ Returns -1 in the NA case """ - values, mask, dtype, _ = _get_values(values, skipna, fill_value_typ='-inf', - isfinite=True) + values, mask, dtype, _ = _get_values(values, skipna, fill_value_typ='-inf') result = values.argmax(axis) result = _maybe_arg_null_out(result, axis, mask, skipna) return result +@disallow('O') def nanargmin(values, axis=None, skipna=True): """ Returns -1 in the NA case """ - values, mask, dtype, _ = _get_values(values, skipna, fill_value_typ='+inf', - isfinite=True) + values, mask, dtype, _ = _get_values(values, skipna, fill_value_typ='+inf') result = values.argmin(axis) result = _maybe_arg_null_out(result, axis, mask, skipna) return result diff --git a/pandas/tests/groupby/test_groupby.py b/pandas/tests/groupby/test_groupby.py index 0dea1e8447b2b1..9983adc95b68f5 100644 --- a/pandas/tests/groupby/test_groupby.py +++ b/pandas/tests/groupby/test_groupby.py @@ -2339,7 +2339,8 @@ def test_non_cython_api(self): assert_frame_equal(result, expected) # idxmax - expected = DataFrame([[0], [nan]], columns=['B'], index=[1, 3]) + expected = DataFrame([[0.0], [nan]], columns=['B'], + index=[1, 3]) expected.index.name = 'A' result = g.idxmax() assert_frame_equal(result, expected) diff --git a/pandas/tests/series/test_operators.py b/pandas/tests/series/test_operators.py index 991c5ff6255545..0374e5eae0fe79 100644 --- a/pandas/tests/series/test_operators.py +++ b/pandas/tests/series/test_operators.py @@ -1857,3 +1857,58 @@ def test_op_duplicate_index(self): result = s1 + s2 expected = pd.Series([11, 12, np.nan], index=[1, 1, 2]) assert_series_equal(result, expected) + + def test_argminmax(self): + # Series.argmin, Series.argmax are aliased to Series.idxmin, + # Series.idxmax + + # Expected behavior for empty Series + s = pd.Series([]) + + with pytest.raises(ValueError): + s.argmin() + with pytest.raises(ValueError): + s.argmin(skipna=False) + with pytest.raises(ValueError): + s.argmax() + with pytest.raises(ValueError): + s.argmax(skipna=False) + + # For numeric data with NA and Inf (GH #13595) + s = pd.Series([0, -np.inf, np.inf, np.nan]) + + assert s.argmin() == 1 + assert np.isnan(s.argmin(skipna=False)) + + assert s.argmax() == 2 + assert np.isnan(s.argmax(skipna=False)) + + # Using old-style behavior that treats floating point nan, -inf, and + # +inf as missing + s = pd.Series([0, -np.inf, np.inf, np.nan]) + + with pd.option_context('mode.use_inf_as_na', True): + assert s.argmin() == 0 + assert np.isnan(s.argmin(skipna=False)) + assert s.argmax() == 0 + np.isnan(s.argmax(skipna=False)) + + # For strings (or any Series with dtype 'O') + s = pd.Series(['foo', 'bar', 'baz']) + with pytest.raises(TypeError): + s.argmin() + with pytest.raises(TypeError): + s.argmax() + + s = pd.Series([(1,), (2,)]) + with pytest.raises(TypeError): + s.argmin() + with pytest.raises(TypeError): + s.argmax() + + # For mixed data types + s = pd.Series(['foo', 'foo', 'bar', 'bar', None, np.nan, 'baz']) + with pytest.raises(TypeError): + s.argmin() + with pytest.raises(TypeError): + s.argmax()