Skip to content
This repository has been archived by the owner on Feb 2, 2024. It is now read-only.

Series combine #821

Open
wants to merge 21 commits into
base: master
Choose a base branch
from
Open

Series combine #821

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
7ec05f3
Impl Series.combine
Rubtsowa Apr 28, 2020
b576e03
merge
Rubtsowa Apr 28, 2020
95d233e
change comment
Rubtsowa Apr 28, 2020
2b95e9d
Merge branch 'master' of https://github.com/IntelPython/hpat into ser…
Rubtsowa May 13, 2020
2104aae
change example
Rubtsowa May 13, 2020
aeef026
use sdc_join_series_indexes
Rubtsowa May 13, 2020
f09b792
change
Rubtsowa May 18, 2020
fafcaed
Merge branch 'master' of https://github.com/IntelPython/hpat into ser…
Rubtsowa May 18, 2020
b1ed1b3
for from chunks, change dtype for result array
Rubtsowa May 25, 2020
4fdb369
Merge branch 'master' of https://github.com/IntelPython/hpat into ser…
Rubtsowa May 25, 2020
6d930d8
change 'if' on 'if-else'
Rubtsowa May 26, 2020
c85feff
change for
Rubtsowa May 26, 2020
18771f2
change for
Rubtsowa May 26, 2020
8628e31
change for
Rubtsowa May 26, 2020
6f37f53
add comment in skip test, some change in impl
Rubtsowa May 26, 2020
f66ace3
change if-else in 1 line
Rubtsowa May 27, 2020
9fc55a7
Merge branch 'master' of https://github.com/IntelPython/hpat into ser…
Rubtsowa May 27, 2020
e7dc1f5
change test
Rubtsowa May 27, 2020
6c78b8e
change tests
Rubtsowa Jun 1, 2020
d623b89
fix problem with PEP8
Rubtsowa Jun 1, 2020
ecd3dba
Merge branch 'master' of https://github.com/IntelPython/hpat into ser…
Rubtsowa Jun 1, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 40 additions & 0 deletions examples/series/series_combine.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
# *****************************************************************************
# Copyright (c) 2020, Intel Corporation All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
#
# Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
# OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
# OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
# EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
# *****************************************************************************

import numpy as np
densmirn marked this conversation as resolved.
Show resolved Hide resolved
import pandas as pd
from numba import njit


@njit
def series_copy():
densmirn marked this conversation as resolved.
Show resolved Hide resolved
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Wrong name function

s1 = pd.Series([1, 5, 2])
s2 = pd.Series([0, 3, 7, 8, 0])

return s1.combine(s2, max, fill_value=0) # Expect series of 1, 5, 7, 8, 0


print(series_copy())
62 changes: 62 additions & 0 deletions sdc/datatypes/hpat_pandas_series_functions.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4882,3 +4882,65 @@ def sdc_pandas_series_skew_impl(self, axis=None, skipna=None, level=None, numeri
return numpy_like.skew(self._data)

return sdc_pandas_series_skew_impl


@sdc_overload_method(SeriesType, 'combine')
def sdc_pandas_series_combine(self, other, func, fill_value=None):
"""
Intel Scalable Dataframe Compiler User Guide
********************************************

Pandas API: pandas.Series.combine

Limitations
-----------
- Only supports the case when data in series of the same type

Examples
--------
.. literalinclude:: ../../../examples/series/series_combine.py
:language: python
:lines: 27-
:caption: Combined the Series with a Series according to func.
:name: ex_series_combine

.. command-output:: python ./series/series_combine.py
:cwd: ../../../examples

Intel Scalable Dataframe Compiler Developer Guide
*************************************************
Pandas Series method :meth:`pandas.Series.combine` implementation.

.. only:: developer

Tests: python -m sdc.runtests -k sdc.tests.test_series.TestSeries.test_series_combine*
"""
_func_name = 'Method Series.combine().'

ty_checker = TypeChecker(_func_name)
ty_checker.check(self, SeriesType)

ty_checker.check(other, SeriesType)

if not isinstance(fill_value, (types.Omitted, types.NoneType, types.Number)) and fill_value is not None:
ty_checker.raise_exc(fill_value, 'number', 'fill_value')

def sdc_pandas_series_combine_impl(self, other, func, fill_value=None):

if fill_value is None:
fill_value = numpy.nan
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This will make fill_value type undefined at compile time. You can probably use the same approach as in operators:

sdc/sdc/sdc_function_templates.py

Line 144 in e87095a

_fill_value = numpy.nan if fill_value_is_none == True else fill_value # noqa


len_val = max(len(self), len(other))
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

And what if all indexes are different? I think we should use sdc_join_series_indexes to find len of result series

result = numpy.empty(len_val, self._data.dtype)
Copy link
Contributor

@kozlov-alexey kozlov-alexey May 21, 2020 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is actually wrong, result dtype should be common dtype for result dtype of func(a, b) where a,b are series values and dtype of _fill_value. Provided tests do not cover this, but e.g. this (where fill_value is float and series are integers) won't pass:

    def test_series_combine_integer_new(self):
        def test_impl(S1, S2):
            return S1.combine(S2, lambda a, b: 2 * a + b, 16.2)
        hpat_func = self.jit(test_impl)

        S1 = pd.Series([1, 2, 3, 4, 5])
        S2 = pd.Series([6, 21, 3, 5])
        result = hpat_func(S1, S2)
        result_ref = test_impl(S1, S2)
        print(f"DEBUG: result:\n{result},\nresult_ref:\n{result_ref}")
        pd.testing.assert_series_equal(result, result_ref)

for ind in range(len_val):
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can we parallel the method based on chunks?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It is case for non-indexes series. Also, it should rewrite with prange

Copy link
Contributor

@densmirn densmirn May 13, 2020 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

+ usage of chunks to predict scalability

val_self = self._data[ind]
val_other = other._data[ind]
if len(self) < ind + 1:
val_self = fill_value
if len(other) < ind + 1:
val_other = fill_value
result[ind] = func(val_self, val_other)

return pandas.Series(result)

return sdc_pandas_series_combine_impl
8 changes: 0 additions & 8 deletions sdc/tests/test_series.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2760,7 +2760,6 @@ def test_impl(S1, S2):
S2 = pd.Series([6., 7.])
np.testing.assert_array_equal(hpat_func(S1, S2), test_impl(S1, S2))

@skip_numba_jit
def test_series_combine(self):
def test_impl(S1, S2):
return S1.combine(S2, lambda a, b: 2 * a + b)
Expand All @@ -2770,7 +2769,6 @@ def test_impl(S1, S2):
S2 = pd.Series([6.0, 21., 3.6, 5.])
pd.testing.assert_series_equal(hpat_func(S1, S2), test_impl(S1, S2))

@skip_numba_jit
def test_series_combine_float3264(self):
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This test has incorrect code, which should be corrected probably:

        S1 = pd.Series([np.float64(1), np.float64(2),
                        np.float64(3), np.float64(4), np.float64(5)])
        S2 = pd.Series([np.float32(1), np.float32(2),
                        np.float32(3), np.float32(4), np.float32(5)])

S2.dtype will be float64 on Win, not float32. Moreover, series dtype should be specified this way:

        S1 = pd.Series([1, 2, 3, 4, 5], dtype=np.int64)
        S2 = pd.Series([1, 2, 3, 4, 5], dtype=np.int32)

def test_impl(S1, S2):
return S1.combine(S2, lambda a, b: 2 * a + b)
Expand Down Expand Up @@ -2804,7 +2802,6 @@ def test_impl(S1, S2):
with self.assertRaises(AssertionError):
hpat_func(S1, S2)

@skip_numba_jit
def test_series_combine_integer(self):
def test_impl(S1, S2):
return S1.combine(S2, lambda a, b: 2 * a + b, 16)
Expand All @@ -2814,7 +2811,6 @@ def test_impl(S1, S2):
S2 = pd.Series([6, 21, 3, 5])
pd.testing.assert_series_equal(hpat_func(S1, S2), test_impl(S1, S2))

@skip_numba_jit
def test_series_combine_different_types(self):
def test_impl(S1, S2):
return S1.combine(S2, lambda a, b: 2 * a + b)
Expand All @@ -2824,7 +2820,6 @@ def test_impl(S1, S2):
S2 = pd.Series([1, 2, 3, 4, 5])
pd.testing.assert_series_equal(hpat_func(S1, S2), test_impl(S1, S2))

@skip_numba_jit
def test_series_combine_integer_samelen(self):
def test_impl(S1, S2):
return S1.combine(S2, lambda a, b: 2 * a + b)
Expand All @@ -2834,7 +2829,6 @@ def test_impl(S1, S2):
S2 = pd.Series([6, 21, 17, -5, 4])
pd.testing.assert_series_equal(hpat_func(S1, S2), test_impl(S1, S2))

@skip_numba_jit
def test_series_combine_samelen(self):
def test_impl(S1, S2):
return S1.combine(S2, lambda a, b: 2 * a + b)
Expand All @@ -2844,7 +2838,6 @@ def test_impl(S1, S2):
S2 = pd.Series([6.0, 21., 3.6, 5., 0.0])
pd.testing.assert_series_equal(hpat_func(S1, S2), test_impl(S1, S2))

@skip_numba_jit
def test_series_combine_value(self):
def test_impl(S1, S2):
return S1.combine(S2, lambda a, b: 2 * a + b, 1237.56)
Expand All @@ -2854,7 +2847,6 @@ def test_impl(S1, S2):
S2 = pd.Series([6.0, 21., 3.6, 5.])
pd.testing.assert_series_equal(hpat_func(S1, S2), test_impl(S1, S2))

@skip_numba_jit
def test_series_combine_value_samelen(self):
def test_impl(S1, S2):
return S1.combine(S2, lambda a, b: 2 * a + b, 1237.56)
Expand Down