From 186607bec2e35bbcb4eaf5dac4ae8a2a10d003d0 Mon Sep 17 00:00:00 2001 From: Morgan Stuart Date: Sat, 15 Jul 2017 18:47:23 -0400 Subject: [PATCH 1/3] BUG #16012 - fix isin for large object arrays --- doc/source/whatsnew/v0.21.0.txt | 1 + pandas/core/algorithms.py | 5 ++++- pandas/tests/series/test_analytics.py | 9 +++++++++ 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v0.21.0.txt b/doc/source/whatsnew/v0.21.0.txt index c63d4575bac43..97c9769ae2abc 100644 --- a/doc/source/whatsnew/v0.21.0.txt +++ b/doc/source/whatsnew/v0.21.0.txt @@ -204,3 +204,4 @@ Categorical Other ^^^^^ - Bug in :func:`eval` where the ``inplace`` parameter was being incorrectly handled (:issue:`16732`) +- Bug when using :func:`isin` on a large object series and large comparison array, numpy's in1d is used but doesn't support objects in most conditions (:issue:`16012`) \ No newline at end of file diff --git a/pandas/core/algorithms.py b/pandas/core/algorithms.py index b490bf787a037..4ee2c54000fb6 100644 --- a/pandas/core/algorithms.py +++ b/pandas/core/algorithms.py @@ -402,7 +402,10 @@ def isin(comps, values): # work-around for numpy < 1.8 and comparisions on py3 # faster for larger cases to use np.in1d f = lambda x, y: htable.ismember_object(x, values) - if (_np_version_under1p8 and compat.PY3) or len(comps) > 1000000: + # GH16012 + # Ensure np.in1d doesn't get object types or it *may* throw an exception + if ((_np_version_under1p8 and compat.PY3) or len(comps) > 1000000 and + not is_object_dtype(comps)): f = lambda x, y: np.in1d(x, y) elif is_integer_dtype(comps): try: diff --git a/pandas/tests/series/test_analytics.py b/pandas/tests/series/test_analytics.py index 749af1c56a7f0..ab5a422d08fe0 100644 --- a/pandas/tests/series/test_analytics.py +++ b/pandas/tests/series/test_analytics.py @@ -1092,6 +1092,15 @@ def test_isin(self): expected = Series([True, False, True, False, False, False, True, True]) assert_series_equal(result, expected) + # GH: 16012 + # This specific issue has to have a series over 1e6 in len, but the + # comparison array (in_list) must be large enough so that numpy doesn't + # do a manual masking trick that will avoid this issue altogether + s = Series(list('abcdefghijk' * 10 ** 5)) + in_list = [-1, 'a', 'b', 'G', 'Y', 'Z', 'E', 'K', 'E', 'S', 'I', 'R', 'R']*6 + + assert s.isin(in_list).sum() == 200000 + def test_isin_with_string_scalar(self): # GH4763 s = Series(['A', 'B', 'C', 'a', 'B', 'B', 'A', 'C']) From 4b5974512c98e0005385b0a589a9b64f73b92484 Mon Sep 17 00:00:00 2001 From: Morgan Stuart Date: Mon, 17 Jul 2017 21:33:36 -0400 Subject: [PATCH 2/3] Linting errors; additional test clarification --- pandas/tests/series/test_analytics.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pandas/tests/series/test_analytics.py b/pandas/tests/series/test_analytics.py index ab5a422d08fe0..ab75dbf1b51cc 100644 --- a/pandas/tests/series/test_analytics.py +++ b/pandas/tests/series/test_analytics.py @@ -1097,7 +1097,10 @@ def test_isin(self): # comparison array (in_list) must be large enough so that numpy doesn't # do a manual masking trick that will avoid this issue altogether s = Series(list('abcdefghijk' * 10 ** 5)) - in_list = [-1, 'a', 'b', 'G', 'Y', 'Z', 'E', 'K', 'E', 'S', 'I', 'R', 'R']*6 + # If numpy doesn't do the manual comparison/mask, these + # unorderable mixed types are what cause the exception in numpy + in_list = [-1, 'a', 'b', 'G', 'Y', 'Z', 'E', + 'K', 'E', 'S', 'I', 'R', 'R'] * 6 assert s.isin(in_list).sum() == 200000 From 31cb4b3165798a5316b79d196d78838ad43e4d73 Mon Sep 17 00:00:00 2001 From: Morgan Stuart Date: Tue, 18 Jul 2017 14:40:35 -0400 Subject: [PATCH 3/3] Removed unneeded details from whatsnew description --- doc/source/whatsnew/v0.21.0.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v0.21.0.txt b/doc/source/whatsnew/v0.21.0.txt index 97c9769ae2abc..9f352f907c0c6 100644 --- a/doc/source/whatsnew/v0.21.0.txt +++ b/doc/source/whatsnew/v0.21.0.txt @@ -204,4 +204,4 @@ Categorical Other ^^^^^ - Bug in :func:`eval` where the ``inplace`` parameter was being incorrectly handled (:issue:`16732`) -- Bug when using :func:`isin` on a large object series and large comparison array, numpy's in1d is used but doesn't support objects in most conditions (:issue:`16012`) \ No newline at end of file +- Bug when using :func:`isin` on a large object series and large comparison array (:issue:`16012`)