Skip to content

Commit

Permalink
BUG: fix tz-aware datetime convert to DatetimeIndex (GH 14088)
Browse files Browse the repository at this point in the history 
closes pandas-dev#14088

Author: John Liekezer <cool.Bakov@yandex.ru>

Closes pandas-dev#14090 from conquistador1492/issue_14088 and squashes the following commits:

c91425b [John Liekezer] BUG: fix tz-aware datetime convert to DatetimeIndex (GH 14088)
  • Loading branch information
@conquistador1492 @jreback
conquistador1492 authored and jreback committed Sep 8, 2016
1 parent ff435ba commit d8cd33b
Show file tree
Hide file tree
Showing 3 changed files with 94 additions and 82 deletions.
1 change: 1 addition & 0 deletions doc/source/whatsnew/v0.19.0.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1549,6 +1549,7 @@ Bug Fixes
- Bug where ``pd.read_gbq()`` could throw ``ImportError: No module named discovery`` as a result of a naming conflict with another python package called apiclient (:issue:`13454`)
- Bug in ``Index.union`` returns an incorrect result with a named empty index (:issue:`13432`)
- Bugs in ``Index.difference`` and ``DataFrame.join`` raise in Python3 when using mixed-integer indexes (:issue:`13432`, :issue:`12814`)
- Bug in subtract tz-aware ``datetime.datetime`` from tz-aware ``datetime64`` series (:issue:`14088`)
- Bug in ``.to_excel()`` when DataFrame contains a MultiIndex which contains a label with a NaN value (:issue:`13511`)
- Bug in invalid frequency offset string like "D1", "-2-3H" may not raise ``ValueError (:issue:`13930`)
- Bug in ``concat`` and ``groupby`` for hierarchical frames with ``RangeIndex`` levels (:issue:`13542`).
Expand Down
2 changes: 1 addition & 1 deletion pandas/core/ops.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -453,7 +453,7 @@ def _convert_to_array(self, values, name=None, other=None):
values = values.to_series()
# datetime with tz
elif (isinstance(ovalues, datetime.datetime) and
hasattr(ovalues, 'tz')):
hasattr(ovalues, 'tzinfo')):
values = pd.DatetimeIndex(values)
# datetime array with tz
elif is_datetimetz(values):
Expand Down
173 changes: 92 additions & 81 deletions pandas/tests/series/test_operators.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,8 @@

from pandas.compat import range, zip
from pandas import compat
from pandas.util.testing import assert_series_equal, assert_almost_equal
from pandas.util.testing import (assert_series_equal, assert_almost_equal,
assert_frame_equal)
import pandas.util.testing as tm

from .common import TestData
Expand All @@ -45,8 +46,8 @@ def test_comparisons(self):

# it works!
exp = Series([False, False, False])
tm.assert_series_equal(s == s2, exp)
tm.assert_series_equal(s2 == s, exp)
assert_series_equal(s == s2, exp)
assert_series_equal(s2 == s, exp)

def test_op_method(self):
def check(series, other, check_reverse=False):
Expand All @@ -64,12 +65,12 @@ def check(series, other, check_reverse=False):

result = op(series, other)
expected = alt(series, other)
tm.assert_almost_equal(result, expected)
assert_almost_equal(result, expected)
if check_reverse:
rop = getattr(Series, "r" + opname)
result = rop(series, other)
expected = alt(other, series)
tm.assert_almost_equal(result, expected)
assert_almost_equal(result, expected)

check(self.ts, self.ts * 2)
check(self.ts, self.ts[::2])
Expand Down Expand Up @@ -149,8 +150,8 @@ def _check_op(series, other, op, pos_only=False,

cython_or_numpy = op(left, right)
python = left.combine(right, op)
tm.assert_series_equal(cython_or_numpy, python,
check_dtype=check_dtype)
assert_series_equal(cython_or_numpy, python,
check_dtype=check_dtype)

def check(series, other):
simple_ops = ['add', 'sub', 'mul', 'truediv', 'floordiv', 'mod']
Expand Down Expand Up @@ -187,7 +188,7 @@ def check_comparators(series, other, check_dtype=True):
def test_operators_empty_int_corner(self):
s1 = Series([], [], dtype=np.int32)
s2 = Series({'x': 0.})
tm.assert_series_equal(s1 * s2, Series([np.nan], index=['x']))
assert_series_equal(s1 * s2, Series([np.nan], index=['x']))

def test_operators_timedelta64(self):

Expand Down Expand Up @@ -668,6 +669,16 @@ def run_ops(ops, get_ser, test_ser):
self.assertRaises(TypeError, lambda: td1 - dt1)
self.assertRaises(TypeError, lambda: td2 - dt2)

def test_sub_datetime_compat(self):
# GH 14088
tm._skip_if_no_pytz()
import pytz
s = Series([datetime(2016, 8, 23, 12, tzinfo=pytz.utc), pd.NaT])
dt = datetime(2016, 8, 22, 12, tzinfo=pytz.utc)
exp = Series([Timedelta('1 days'), pd.NaT])
assert_series_equal(s - dt, exp)
assert_series_equal(s - Timestamp(dt), exp)

def test_sub_single_tz(self):
# GH12290
s1 = Series([pd.Timestamp('2016-02-10', tz='America/Sao_Paulo')])
Expand Down Expand Up @@ -1175,21 +1186,21 @@ def test_comparison_flex_basic(self):
left = pd.Series(np.random.randn(10))
right = pd.Series(np.random.randn(10))

tm.assert_series_equal(left.eq(right), left == right)
tm.assert_series_equal(left.ne(right), left != right)
tm.assert_series_equal(left.le(right), left < right)
tm.assert_series_equal(left.lt(right), left <= right)
tm.assert_series_equal(left.gt(right), left > right)
tm.assert_series_equal(left.ge(right), left >= right)
assert_series_equal(left.eq(right), left == right)
assert_series_equal(left.ne(right), left != right)
assert_series_equal(left.le(right), left < right)
assert_series_equal(left.lt(right), left <= right)
assert_series_equal(left.gt(right), left > right)
assert_series_equal(left.ge(right), left >= right)

# axis
for axis in [0, None, 'index']:
tm.assert_series_equal(left.eq(right, axis=axis), left == right)
tm.assert_series_equal(left.ne(right, axis=axis), left != right)
tm.assert_series_equal(left.le(right, axis=axis), left < right)
tm.assert_series_equal(left.lt(right, axis=axis), left <= right)
tm.assert_series_equal(left.gt(right, axis=axis), left > right)
tm.assert_series_equal(left.ge(right, axis=axis), left >= right)
assert_series_equal(left.eq(right, axis=axis), left == right)
assert_series_equal(left.ne(right, axis=axis), left != right)
assert_series_equal(left.le(right, axis=axis), left < right)
assert_series_equal(left.lt(right, axis=axis), left <= right)
assert_series_equal(left.gt(right, axis=axis), left > right)
assert_series_equal(left.ge(right, axis=axis), left >= right)

#
msg = 'No axis named 1 for object type'
Expand All @@ -1202,44 +1213,44 @@ def test_comparison_flex_alignment(self):
right = Series([2, 2, 2], index=list('bcd'))

exp = pd.Series([False, False, True, False], index=list('abcd'))
tm.assert_series_equal(left.eq(right), exp)
assert_series_equal(left.eq(right), exp)

exp = pd.Series([True, True, False, True], index=list('abcd'))
tm.assert_series_equal(left.ne(right), exp)
assert_series_equal(left.ne(right), exp)

exp = pd.Series([False, False, True, False], index=list('abcd'))
tm.assert_series_equal(left.le(right), exp)
assert_series_equal(left.le(right), exp)

exp = pd.Series([False, False, False, False], index=list('abcd'))
tm.assert_series_equal(left.lt(right), exp)
assert_series_equal(left.lt(right), exp)

exp = pd.Series([False, True, True, False], index=list('abcd'))
tm.assert_series_equal(left.ge(right), exp)
assert_series_equal(left.ge(right), exp)

exp = pd.Series([False, True, False, False], index=list('abcd'))
tm.assert_series_equal(left.gt(right), exp)
assert_series_equal(left.gt(right), exp)

def test_comparison_flex_alignment_fill(self):
left = Series([1, 3, 2], index=list('abc'))
right = Series([2, 2, 2], index=list('bcd'))

exp = pd.Series([False, False, True, True], index=list('abcd'))
tm.assert_series_equal(left.eq(right, fill_value=2), exp)
assert_series_equal(left.eq(right, fill_value=2), exp)

exp = pd.Series([True, True, False, False], index=list('abcd'))
tm.assert_series_equal(left.ne(right, fill_value=2), exp)
assert_series_equal(left.ne(right, fill_value=2), exp)

exp = pd.Series([False, False, True, True], index=list('abcd'))
tm.assert_series_equal(left.le(right, fill_value=0), exp)
assert_series_equal(left.le(right, fill_value=0), exp)

exp = pd.Series([False, False, False, True], index=list('abcd'))
tm.assert_series_equal(left.lt(right, fill_value=0), exp)
assert_series_equal(left.lt(right, fill_value=0), exp)

exp = pd.Series([True, True, True, False], index=list('abcd'))
tm.assert_series_equal(left.ge(right, fill_value=0), exp)
assert_series_equal(left.ge(right, fill_value=0), exp)

exp = pd.Series([True, True, False, False], index=list('abcd'))
tm.assert_series_equal(left.gt(right, fill_value=0), exp)
assert_series_equal(left.gt(right, fill_value=0), exp)

def test_operators_bitwise(self):
# GH 9016: support bitwise op for integer types
Expand Down Expand Up @@ -1426,27 +1437,27 @@ def test_arith_ops_df_compat(self):

exp = pd.Series([3.0, 4.0, np.nan, np.nan],
index=list('ABCD'), name='x')
tm.assert_series_equal(s1 + s2, exp)
tm.assert_series_equal(s2 + s1, exp)
assert_series_equal(s1 + s2, exp)
assert_series_equal(s2 + s1, exp)

exp = pd.DataFrame({'x': [3.0, 4.0, np.nan, np.nan]},
index=list('ABCD'))
tm.assert_frame_equal(s1.to_frame() + s2.to_frame(), exp)
tm.assert_frame_equal(s2.to_frame() + s1.to_frame(), exp)
assert_frame_equal(s1.to_frame() + s2.to_frame(), exp)
assert_frame_equal(s2.to_frame() + s1.to_frame(), exp)

# different length
s3 = pd.Series([1, 2, 3], index=list('ABC'), name='x')
s4 = pd.Series([2, 2, 2, 2], index=list('ABCD'), name='x')

exp = pd.Series([3, 4, 5, np.nan],
index=list('ABCD'), name='x')
tm.assert_series_equal(s3 + s4, exp)
tm.assert_series_equal(s4 + s3, exp)
assert_series_equal(s3 + s4, exp)
assert_series_equal(s4 + s3, exp)

exp = pd.DataFrame({'x': [3, 4, 5, np.nan]},
index=list('ABCD'))
tm.assert_frame_equal(s3.to_frame() + s4.to_frame(), exp)
tm.assert_frame_equal(s4.to_frame() + s3.to_frame(), exp)
assert_frame_equal(s3.to_frame() + s4.to_frame(), exp)
assert_frame_equal(s4.to_frame() + s3.to_frame(), exp)

def test_comp_ops_df_compat(self):
# GH 1134
Expand Down Expand Up @@ -1485,56 +1496,56 @@ def test_bool_ops_df_compat(self):

exp = pd.Series([True, False, False, False],
index=list('ABCD'), name='x')
tm.assert_series_equal(s1 & s2, exp)
tm.assert_series_equal(s2 & s1, exp)
assert_series_equal(s1 & s2, exp)
assert_series_equal(s2 & s1, exp)

# True | np.nan => True
exp = pd.Series([True, True, True, False],
index=list('ABCD'), name='x')
tm.assert_series_equal(s1 | s2, exp)
assert_series_equal(s1 | s2, exp)
# np.nan | True => np.nan, filled with False
exp = pd.Series([True, True, False, False],
index=list('ABCD'), name='x')
tm.assert_series_equal(s2 | s1, exp)
assert_series_equal(s2 | s1, exp)

# DataFrame doesn't fill nan with False
exp = pd.DataFrame({'x': [True, False, np.nan, np.nan]},
index=list('ABCD'))
tm.assert_frame_equal(s1.to_frame() & s2.to_frame(), exp)
tm.assert_frame_equal(s2.to_frame() & s1.to_frame(), exp)
assert_frame_equal(s1.to_frame() & s2.to_frame(), exp)
assert_frame_equal(s2.to_frame() & s1.to_frame(), exp)

exp = pd.DataFrame({'x': [True, True, np.nan, np.nan]},
index=list('ABCD'))
tm.assert_frame_equal(s1.to_frame() | s2.to_frame(), exp)
tm.assert_frame_equal(s2.to_frame() | s1.to_frame(), exp)
assert_frame_equal(s1.to_frame() | s2.to_frame(), exp)
assert_frame_equal(s2.to_frame() | s1.to_frame(), exp)

# different length
s3 = pd.Series([True, False, True], index=list('ABC'), name='x')
s4 = pd.Series([True, True, True, True], index=list('ABCD'), name='x')

exp = pd.Series([True, False, True, False],
index=list('ABCD'), name='x')
tm.assert_series_equal(s3 & s4, exp)
tm.assert_series_equal(s4 & s3, exp)
assert_series_equal(s3 & s4, exp)
assert_series_equal(s4 & s3, exp)

# np.nan | True => np.nan, filled with False
exp = pd.Series([True, True, True, False],
index=list('ABCD'), name='x')
tm.assert_series_equal(s3 | s4, exp)
assert_series_equal(s3 | s4, exp)
# True | np.nan => True
exp = pd.Series([True, True, True, True],
index=list('ABCD'), name='x')
tm.assert_series_equal(s4 | s3, exp)
assert_series_equal(s4 | s3, exp)

exp = pd.DataFrame({'x': [True, False, True, np.nan]},
index=list('ABCD'))
tm.assert_frame_equal(s3.to_frame() & s4.to_frame(), exp)
tm.assert_frame_equal(s4.to_frame() & s3.to_frame(), exp)
assert_frame_equal(s3.to_frame() & s4.to_frame(), exp)
assert_frame_equal(s4.to_frame() & s3.to_frame(), exp)

exp = pd.DataFrame({'x': [True, True, True, np.nan]},
index=list('ABCD'))
tm.assert_frame_equal(s3.to_frame() | s4.to_frame(), exp)
tm.assert_frame_equal(s4.to_frame() | s3.to_frame(), exp)
assert_frame_equal(s3.to_frame() | s4.to_frame(), exp)
assert_frame_equal(s4.to_frame() | s3.to_frame(), exp)

def test_series_frame_radd_bug(self):
# GH 353
Expand All @@ -1546,7 +1557,7 @@ def test_series_frame_radd_bug(self):
frame = DataFrame({'vals': vals})
result = 'foo_' + frame
expected = DataFrame({'vals': vals.map(lambda x: 'foo_' + x)})
tm.assert_frame_equal(result, expected)
assert_frame_equal(result, expected)

# really raise this time
with tm.assertRaises(TypeError):
Expand All @@ -1571,26 +1582,26 @@ def test_series_radd_more(self):
for dtype in [None, object]:
res = 1 + pd.Series([1, 2, 3], dtype=dtype)
exp = pd.Series([2, 3, 4], dtype=dtype)
tm.assert_series_equal(res, exp)
assert_series_equal(res, exp)
res = pd.Series([1, 2, 3], dtype=dtype) + 1
tm.assert_series_equal(res, exp)
assert_series_equal(res, exp)

res = np.nan + pd.Series([1, 2, 3], dtype=dtype)
exp = pd.Series([np.nan, np.nan, np.nan], dtype=dtype)
tm.assert_series_equal(res, exp)
assert_series_equal(res, exp)
res = pd.Series([1, 2, 3], dtype=dtype) + np.nan
tm.assert_series_equal(res, exp)
assert_series_equal(res, exp)

s = pd.Series([pd.Timedelta('1 days'), pd.Timedelta('2 days'),
pd.Timedelta('3 days')], dtype=dtype)
exp = pd.Series([pd.Timedelta('4 days'), pd.Timedelta('5 days'),
pd.Timedelta('6 days')])
tm.assert_series_equal(pd.Timedelta('3 days') + s, exp)
tm.assert_series_equal(s + pd.Timedelta('3 days'), exp)
assert_series_equal(pd.Timedelta('3 days') + s, exp)
assert_series_equal(s + pd.Timedelta('3 days'), exp)

s = pd.Series(['x', np.nan, 'x'])
tm.assert_series_equal('a' + s, pd.Series(['ax', np.nan, 'ax']))
tm.assert_series_equal(s + 'a', pd.Series(['xa', np.nan, 'xa']))
assert_series_equal('a' + s, pd.Series(['ax', np.nan, 'ax']))
assert_series_equal(s + 'a', pd.Series(['xa', np.nan, 'xa']))

def test_frame_radd_more(self):
data = [[1, 2, 3],
Expand All @@ -1608,32 +1619,32 @@ def test_frame_radd_more(self):
for dtype in [None, object]:
res = 1 + pd.DataFrame([1, 2, 3], dtype=dtype)
exp = pd.DataFrame([2, 3, 4], dtype=dtype)
tm.assert_frame_equal(res, exp)
assert_frame_equal(res, exp)
res = pd.DataFrame([1, 2, 3], dtype=dtype) + 1
tm.assert_frame_equal(res, exp)
assert_frame_equal(res, exp)

res = np.nan + pd.DataFrame([1, 2, 3], dtype=dtype)
exp = pd.DataFrame([np.nan, np.nan, np.nan], dtype=dtype)
tm.assert_frame_equal(res, exp)
assert_frame_equal(res, exp)
res = pd.DataFrame([1, 2, 3], dtype=dtype) + np.nan
tm.assert_frame_equal(res, exp)
assert_frame_equal(res, exp)

df = pd.DataFrame(['x', np.nan, 'x'])
tm.assert_frame_equal('a' + df, pd.DataFrame(['ax', np.nan, 'ax']))
tm.assert_frame_equal(df + 'a', pd.DataFrame(['xa', np.nan, 'xa']))
assert_frame_equal('a' + df, pd.DataFrame(['ax', np.nan, 'ax']))
assert_frame_equal(df + 'a', pd.DataFrame(['xa', np.nan, 'xa']))

def test_operators_frame(self):
# rpow does not work with DataFrame
df = DataFrame({'A': self.ts})

tm.assert_series_equal(self.ts + self.ts, self.ts + df['A'],
check_names=False)
tm.assert_series_equal(self.ts ** self.ts, self.ts ** df['A'],
check_names=False)
tm.assert_series_equal(self.ts < self.ts, self.ts < df['A'],
check_names=False)
tm.assert_series_equal(self.ts / self.ts, self.ts / df['A'],
check_names=False)
assert_series_equal(self.ts + self.ts, self.ts + df['A'],
check_names=False)
assert_series_equal(self.ts ** self.ts, self.ts ** df['A'],
check_names=False)
assert_series_equal(self.ts < self.ts, self.ts < df['A'],
check_names=False)
assert_series_equal(self.ts / self.ts, self.ts / df['A'],
check_names=False)

def test_operators_combine(self):
def _check_fill(meth, op, a, b, fill_value=0):
Expand Down Expand Up @@ -1729,12 +1740,12 @@ def test_divide_decimal(self):
s = Series([Decimal(10)])
s = s / Decimal(2)

tm.assert_series_equal(expected, s)
assert_series_equal(expected, s)

s = Series([Decimal(10)])
s = s // Decimal(2)

tm.assert_series_equal(expected, s)
assert_series_equal(expected, s)

def test_datetime64_with_index(self):

Expand Down

0 comments on commit d8cd33b

Please sign in to comment.