diff --git a/examples/series/series_combine.py b/examples/series/series_combine.py new file mode 100644 index 000000000..de212677e --- /dev/null +++ b/examples/series/series_combine.py @@ -0,0 +1,39 @@ +# ***************************************************************************** +# Copyright (c) 2020, Intel Corporation All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; +# OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR +# OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, +# EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# ***************************************************************************** + +import pandas as pd +from numba import njit + + +@njit +def series_combine(): + s1 = pd.Series([1, 5, 2]) + s2 = pd.Series([0, 3, 7, 8, 0]) + + return s1.combine(s2, max, fill_value=0) # Expect series of 1, 5, 7, 8, 0 + + +print(series_combine()) diff --git a/sdc/datatypes/hpat_pandas_series_functions.py b/sdc/datatypes/hpat_pandas_series_functions.py index e7d4fa51b..ed5589bb7 100644 --- a/sdc/datatypes/hpat_pandas_series_functions.py +++ b/sdc/datatypes/hpat_pandas_series_functions.py @@ -45,6 +45,7 @@ from numba.typed import List, Dict from numba import prange from numba.np.arraymath import get_isnan +from numba.core.registry import cpu_target from pandas.core.indexing import IndexingError import sdc @@ -4910,3 +4911,78 @@ def sdc_pandas_series_skew_impl(self, axis=None, skipna=None, level=None, numeri return numpy_like.skew(self._data) return sdc_pandas_series_skew_impl + + +@sdc_overload_method(SeriesType, 'combine', jit_options={'error_model': 'numpy'}) +def sdc_pandas_series_combine(self, other, func, fill_value=None): + """ + Intel Scalable Dataframe Compiler User Guide + ******************************************** + + Pandas API: pandas.Series.combine + + Limitations + ----------- + - Resulting series dtype may be wider than in pandas due to + type-stability requirements and depends on fill_value dtype + and result of series indexes alignment. + - Indixes should be strictly ascending, as inside the function + they are sorted in ascending order and the answer becomes + different from the result of the pandas. + + Examples + -------- + .. literalinclude:: ../../../examples/series/series_combine.py + :language: python + :lines: 27- + :caption: Combined the Series with a Series according to func. + :name: ex_series_combine + + .. command-output:: python ./series/series_combine.py + :cwd: ../../../examples + + Intel Scalable Dataframe Compiler Developer Guide + ************************************************* + Pandas Series method :meth:`pandas.Series.combine` implementation. + + .. only:: developer + Test: python -m sdc.runtests -k sdc.tests.test_series.TestSeries.test_series_combine* + """ + _func_name = 'Method Series.combine()' + + ty_checker = TypeChecker(_func_name) + ty_checker.check(self, SeriesType) + + ty_checker.check(other, SeriesType) + + if not isinstance(fill_value, (types.Omitted, types.NoneType, types.Number)) and fill_value is not None: + ty_checker.raise_exc(fill_value, 'number', 'fill_value') + + fill_is_default = isinstance(fill_value, (types.Omitted, types.NoneType)) or fill_value is None + + sig = func.get_call_type(cpu_target.typing_context, [self.dtype, other.dtype], {}) + ret_type = sig.return_type + + fill_dtype = types.float64 if fill_is_default else fill_value + res_dtype = find_common_dtype_from_numpy_dtypes([], [ret_type, fill_dtype]) + + def sdc_pandas_series_combine_impl(self, other, func, fill_value=None): + + _fill_value = numpy.nan if fill_value is None else fill_value + + indexes, self_indexes, other_indexes = sdc_join_series_indexes(self.index, other.index) + len_val = len(indexes) + + result = numpy.empty(len_val, res_dtype) + + for i in prange(len_val): + self_idx, other_idx = self_indexes[i], other_indexes[i] + val_self = _fill_value if self_idx == -1 else self._data[self_idx] + + val_other = _fill_value if other_idx == -1 else other._data[other_idx] + + result[i] = func(val_self, val_other) + + return pandas.Series(result, index=indexes) + + return sdc_pandas_series_combine_impl diff --git a/sdc/tests/test_series.py b/sdc/tests/test_series.py index 64d3e57c1..f421e4362 100644 --- a/sdc/tests/test_series.py +++ b/sdc/tests/test_series.py @@ -2761,7 +2761,6 @@ def test_impl(S1, S2): S2 = pd.Series([6., 7.]) np.testing.assert_array_equal(hpat_func(S1, S2), test_impl(S1, S2)) - @skip_numba_jit def test_series_combine(self): def test_impl(S1, S2): return S1.combine(S2, lambda a, b: 2 * a + b) @@ -2771,99 +2770,131 @@ def test_impl(S1, S2): S2 = pd.Series([6.0, 21., 3.6, 5.]) pd.testing.assert_series_equal(hpat_func(S1, S2), test_impl(S1, S2)) - @skip_numba_jit - def test_series_combine_float3264(self): - def test_impl(S1, S2): - return S1.combine(S2, lambda a, b: 2 * a + b) - hpat_func = self.jit(test_impl) - - S1 = pd.Series([np.float64(1), np.float64(2), - np.float64(3), np.float64(4), np.float64(5)]) - S2 = pd.Series([np.float32(1), np.float32(2), - np.float32(3), np.float32(4), np.float32(5)]) - pd.testing.assert_series_equal(hpat_func(S1, S2), test_impl(S1, S2)) - - @skip_numba_jit - def test_series_combine_assert1(self): + @unittest.expectedFailure + # https://github.com/numba/numba/issues/5792 + def test_series_combine_div(self): def test_impl(S1, S2): - return S1.combine(S2, lambda a, b: 2 * a + b) - hpat_func = self.jit(test_impl) - - S1 = pd.Series([1, 2, 3]) - S2 = pd.Series([6., 21., 3., 5.]) - with self.assertRaises(AssertionError): - hpat_func(S1, S2) + return S1.combine(S2, lambda a, b: a/b, 0) - @skip_numba_jit - def test_series_combine_assert2(self): - def test_impl(S1, S2): - return S1.combine(S2, lambda a, b: 2 * a + b) hpat_func = self.jit(test_impl) - S1 = pd.Series([6., 21., 3., 5.]) - S2 = pd.Series([1, 2, 3]) - with self.assertRaises(AssertionError): - hpat_func(S1, S2) - - @skip_numba_jit - def test_series_combine_integer(self): - def test_impl(S1, S2): - return S1.combine(S2, lambda a, b: 2 * a + b, 16) - hpat_func = self.jit(test_impl) + sizes1 = [2, 4, 5, 6, 8] + sizes2 = [1, 3, 5, 7, 9] + series_dtypes = [None, np.int64, np.float64] - S1 = pd.Series([1, 2, 3, 4, 5]) - S2 = pd.Series([6, 21, 3, 5]) - pd.testing.assert_series_equal(hpat_func(S1, S2), test_impl(S1, S2)) + for n in sizes1: + for k in sizes2: + for dtype1, dtype2 in product(series_dtypes, series_dtypes): + A = np.random.randint(-100, 100, n) + B = np.arange(k) * 2 + 1 + S1 = pd.Series(A, dtype=dtype1) + S2 = pd.Series(B, dtype=dtype2) + with self.subTest(S1=S1, S2=S2): + result = hpat_func(S1, S2) + result_ref = test_impl(S1, S2) + # check_dtype=False due to difference to pandas in some cases + pd.testing.assert_series_equal(result, result_ref, check_dtype=False) - @skip_numba_jit - def test_series_combine_different_types(self): + def test_series_combine_value(self): def test_impl(S1, S2): return S1.combine(S2, lambda a, b: 2 * a + b) hpat_func = self.jit(test_impl) - S1 = pd.Series([6.1, 21.2, 3.3, 5.4, 6.7]) - S2 = pd.Series([1, 2, 3, 4, 5]) - pd.testing.assert_series_equal(hpat_func(S1, S2), test_impl(S1, S2)) - - @skip_numba_jit - def test_series_combine_integer_samelen(self): - def test_impl(S1, S2): - return S1.combine(S2, lambda a, b: 2 * a + b) - hpat_func = self.jit(test_impl) + series_indexes = [[1, 2, 3, 4, 5], + [4, 5, 7, 8, 9], + [0, 1, 7, 13, 25]] + # Only indixes ascending due to difference to pandas in some cases - S1 = pd.Series([1, 2, 3, 4, 5]) - S2 = pd.Series([6, 21, 17, -5, 4]) - pd.testing.assert_series_equal(hpat_func(S1, S2), test_impl(S1, S2)) + n = 5 + np.random.seed(0) + A = np.random.randint(-100, 100, n) + B = np.arange(n) * 2 + 1 + + series_dtypes = [None, np.int64, np.float64] + fill_values = [None, np.nan, 4, 4.2] + for dtype1, dtype2 in product(series_dtypes, series_dtypes): + for series_index1 in series_indexes: + for series_index2 in series_indexes: + S1 = pd.Series(A, index=series_index1, dtype=dtype1) + S2 = pd.Series(B, index=series_index2, dtype=dtype2) + with self.subTest(S1=S1, S2=S2): + result = hpat_func(S1, S2) + result_ref = test_impl(S1, S2) + # check_dtype=False due to difference to pandas in some cases + pd.testing.assert_series_equal(result, result_ref, check_dtype=False) + + def test_series_combine_value_with_fill_value(self): + def test_impl(S1, S2, fill_value): + return S1.combine(S2, lambda a, b: 2 * a + b, fill_value) + hpat_func = self.jit(test_impl) + + series_indexes = [[1, 2, 3, 4, 5], + [4, 5, 7, 8, 9], + [0, 1, 7, 13, 25]] + # Only indixes ascending due to difference to pandas in some cases + + n = 5 + np.random.seed(0) + A = np.random.randint(-100, 100, n) + B = np.arange(n) * 2 + 1 + + series_dtypes = [None, np.int64, np.float64] + fill_values = [None, np.nan, 4, 4.2] + for dtype1, dtype2, fill_value in product(series_dtypes, series_dtypes, fill_values): + for series_index1 in series_indexes: + for series_index2 in series_indexes: + S1 = pd.Series(A, index=series_index1, dtype=dtype1) + S2 = pd.Series(B, index=series_index2, dtype=dtype2) + with self.subTest(S1=S1, S2=S2, fill_value=fill_value): + result = hpat_func(S1, S2, fill_value) + result_ref = test_impl(S1, S2, fill_value) + # check_dtype=False due to difference to pandas in some cases + pd.testing.assert_series_equal(result, result_ref, check_dtype=False) - @skip_numba_jit - def test_series_combine_samelen(self): - def test_impl(S1, S2): - return S1.combine(S2, lambda a, b: 2 * a + b) + def test_series_combine_value_samelen(self): + def test_impl(S1, S2, fill_value): + return S1.combine(S2, lambda a, b: 2 * a + b, fill_value=fill_value) hpat_func = self.jit(test_impl) - S1 = pd.Series([1.0, 2., 3., 4., 5.]) - S2 = pd.Series([6.0, 21., 3.6, 5., 0.0]) - pd.testing.assert_series_equal(hpat_func(S1, S2), test_impl(S1, S2)) + n = 11 + np.random.seed(0) + A = np.random.randint(-100, 100, n) + B = np.arange(n) * 2 + 1 + series_index = 1 + np.arange(n) + + series_dtypes = [None, np.int64, np.float64] + fill_values = [None, np.nan, 4, 4.2] + for dtype1, dtype2, fill_value in product(series_dtypes, series_dtypes, fill_values): + S1 = pd.Series(A, index=series_index, dtype=dtype1) + S2 = pd.Series(B, index=series_index, dtype=dtype2) + with self.subTest(S1=S1, S2=S2, fill_value=fill_value): + result = hpat_func(S1, S2, fill_value) + result_ref = test_impl(S1, S2, fill_value) + # check_dtype=False due to difference to pandas in some cases + pd.testing.assert_series_equal(result, result_ref, check_dtype=False) - @skip_numba_jit - def test_series_combine_value(self): + def test_series_combine_different_types(self): def test_impl(S1, S2): - return S1.combine(S2, lambda a, b: 2 * a + b, 1237.56) + return S1.combine(S2, lambda a, b: 2 * a + b) hpat_func = self.jit(test_impl) - S1 = pd.Series([1.0, 2., 3., 4., 5.]) - S2 = pd.Series([6.0, 21., 3.6, 5.]) - pd.testing.assert_series_equal(hpat_func(S1, S2), test_impl(S1, S2)) + sizes1 = [2, 4, 5, 6, 8] + sizes2 = [1, 3, 5, 7, 9] + series_dtypes = [None, np.int64, np.float64] - @skip_numba_jit - def test_series_combine_value_samelen(self): - def test_impl(S1, S2): - return S1.combine(S2, lambda a, b: 2 * a + b, 1237.56) - hpat_func = self.jit(test_impl) + for n in sizes1: + for k in sizes2: + for dtype1, dtype2 in product(series_dtypes, series_dtypes): + A = np.random.randint(-100, 100, n) + B = np.arange(k) * 2 + 1 + S1 = pd.Series(A, dtype=dtype1) + S2 = pd.Series(B, dtype=dtype2) + with self.subTest(S1=S1, S2=S2): + result = hpat_func(S1, S2) + result_ref = test_impl(S1, S2) + # check_dtype=False due to difference to pandas in some cases + pd.testing.assert_series_equal(result, result_ref, check_dtype=False) - S1 = pd.Series([1.0, 2., 3., 4., 5.]) - S2 = pd.Series([6.0, 21., 3.6, 5., 0.0]) - pd.testing.assert_series_equal(hpat_func(S1, S2), test_impl(S1, S2)) def test_series_abs1(self): def test_impl(S):