From a52db10a0ff50d075b68b1bee6b01831377e5f32 Mon Sep 17 00:00:00 2001 From: Monson Shao Date: Sun, 22 Jul 2018 13:20:28 +0800 Subject: [PATCH 1/4] core: try coerce result back to DatetimeBlock --- doc/source/whatsnew/v0.24.0.txt | 1 + pandas/core/internals/__init__.py | 10 +++++++++- pandas/tests/frame/test_arithmetic.py | 10 ++++++++++ 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v0.24.0.txt b/doc/source/whatsnew/v0.24.0.txt index 8fe3023e9537c..f143cbd13c22c 100644 --- a/doc/source/whatsnew/v0.24.0.txt +++ b/doc/source/whatsnew/v0.24.0.txt @@ -436,6 +436,7 @@ Datetimelike - Fixed bug where two :class:`DateOffset` objects with different ``normalize`` attributes could evaluate as equal (:issue:`21404`) - Fixed bug where :meth:`Timestamp.resolution` incorrectly returned 1-microsecond ``timedelta`` instead of 1-nanosecond :class:`Timedelta` (:issue:`21336`,:issue:`21365`) +- Fixed bug where :class:`DataFrame` with ``dtype='datetime64[ns]'`` operating with :class:`DateOffset` could cast to ``dtype='object'`` (:issue:`21610`) Timedelta ^^^^^^^^^ diff --git a/pandas/core/internals/__init__.py b/pandas/core/internals/__init__.py index fde3aaa14ac5d..06c8bc19e2dd9 100644 --- a/pandas/core/internals/__init__.py +++ b/pandas/core/internals/__init__.py @@ -2737,7 +2737,7 @@ def _try_coerce_args(self, values, other): def _try_coerce_result(self, result): """ reverse of try_coerce_args """ - if isinstance(result, np.ndarray): + if isinstance(result, (np.ndarray, Block)): if result.dtype.kind in ['i', 'f', 'O']: try: result = result.astype('M8[ns]') @@ -2785,6 +2785,12 @@ def set(self, locs, values, check=False): self.values[locs] = values + def eval(self, try_cast=False, *args, **kwargs): + blocks = super().eval(try_cast=try_cast, *args, **kwargs) + if try_cast: + blocks = [self._try_coerce_result(block) for block in blocks] + return blocks + class DatetimeTZBlock(NonConsolidatableMixIn, DatetimeBlock): """ implement a datetime64 block with a tz attribute """ @@ -2920,6 +2926,8 @@ def _try_coerce_result(self, result): if isinstance(result, np.ndarray): if result.dtype.kind in ['i', 'f', 'O']: result = result.astype('M8[ns]') + elif isinstance(result, Block): + result = self.make_block_same_class(result.values.flat) elif isinstance(result, (np.integer, np.float, np.datetime64)): result = tslibs.Timestamp(result, tz=self.values.tz) if isinstance(result, np.ndarray): diff --git a/pandas/tests/frame/test_arithmetic.py b/pandas/tests/frame/test_arithmetic.py index fb381a5640519..0886740e7359c 100644 --- a/pandas/tests/frame/test_arithmetic.py +++ b/pandas/tests/frame/test_arithmetic.py @@ -211,6 +211,16 @@ def test_df_sub_datetime64_not_ns(self): pd.Timedelta(days=2)]) tm.assert_frame_equal(res, expected) + def test_timestamp_df_add_dateoffset(self): + expected = pd.DataFrame([pd.Timestamp('2019')]) + result = pd.DataFrame([pd.Timestamp('2018')]) + pd.DateOffset(years=1) + tm.assert_frame_equal(expected, result) + + expected = pd.DataFrame([pd.Timestamp('2019', tz='Asia/Shanghai')]) + result = (pd.DataFrame([pd.Timestamp('2018', tz='Asia/Shanghai')]) + + pd.DateOffset(years=1)) + tm.assert_frame_equal(expected, result) + @pytest.mark.parametrize('data', [ [1, 2, 3], [1.1, 2.2, 3.3], From 36794ed50963e3bfa777b15204497c589b9b1455 Mon Sep 17 00:00:00 2001 From: Monson Shao Date: Sun, 22 Jul 2018 19:07:08 +0800 Subject: [PATCH 2/4] core: cast result to TimeDeltaBlock if needed --- pandas/core/internals/__init__.py | 17 ++++++++++++----- pandas/tests/frame/test_arithmetic.py | 12 ++++++++++++ 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/pandas/core/internals/__init__.py b/pandas/core/internals/__init__.py index 06c8bc19e2dd9..4e5e619ede91c 100644 --- a/pandas/core/internals/__init__.py +++ b/pandas/core/internals/__init__.py @@ -63,7 +63,9 @@ ABCSeries, ABCDatetimeIndex, ABCExtensionArray, - ABCIndexClass) + ABCIndexClass, + ABCDateOffset, +) import pandas.core.common as com import pandas.core.algorithms as algos @@ -2785,11 +2787,16 @@ def set(self, locs, values, check=False): self.values[locs] = values - def eval(self, try_cast=False, *args, **kwargs): - blocks = super().eval(try_cast=try_cast, *args, **kwargs) + def eval(self, func, other, try_cast=False, **kwargs): + block = super().eval(func, other, try_cast=try_cast, **kwargs)[0] if try_cast: - blocks = [self._try_coerce_result(block) for block in blocks] - return blocks + if isinstance(other, + (tslibs.Timestamp, np.datetime64, datetime, date)): + block = TimeDeltaBlock(block.values, block.mgr_locs, + ndim=block.ndim) + elif isinstance(other, ABCDateOffset): + block = self._try_coerce_result(block) + return [block] class DatetimeTZBlock(NonConsolidatableMixIn, DatetimeBlock): diff --git a/pandas/tests/frame/test_arithmetic.py b/pandas/tests/frame/test_arithmetic.py index 0886740e7359c..eff5357b0722d 100644 --- a/pandas/tests/frame/test_arithmetic.py +++ b/pandas/tests/frame/test_arithmetic.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +import datetime import pytest import numpy as np @@ -221,6 +222,17 @@ def test_timestamp_df_add_dateoffset(self): + pd.DateOffset(years=1)) tm.assert_frame_equal(expected, result) + @pytest.mark.parametrize('other', [ + pd.Timestamp('2017'), + np.datetime64('2017'), + datetime.datetime(2017, 1, 1), + datetime.date(2017, 1, 1), + ]) + def test_timestamp_df_sub_timestamp(self, other): + expected = pd.DataFrame([pd.Timedelta('365d')]) + result = pd.DataFrame([pd.Timestamp('2018')]) - other + tm.assert_frame_equal(expected, result) + @pytest.mark.parametrize('data', [ [1, 2, 3], [1.1, 2.2, 3.3], From 98c3ac964250090a3ace6d3ae34dd0329ccd0059 Mon Sep 17 00:00:00 2001 From: Monson Shao Date: Sun, 22 Jul 2018 19:53:07 +0800 Subject: [PATCH 3/4] internals: remove invalid test ' Date: Mon, 23 Jul 2018 13:43:27 +0800 Subject: [PATCH 4/4] revert style change --- pandas/tests/frame/test_arithmetic.py | 2 ++ pandas/tests/internals/test_internals.py | 18 ++++++++---------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/pandas/tests/frame/test_arithmetic.py b/pandas/tests/frame/test_arithmetic.py index eff5357b0722d..9a41360f4b7bd 100644 --- a/pandas/tests/frame/test_arithmetic.py +++ b/pandas/tests/frame/test_arithmetic.py @@ -213,6 +213,7 @@ def test_df_sub_datetime64_not_ns(self): tm.assert_frame_equal(res, expected) def test_timestamp_df_add_dateoffset(self): + # GH 21610 expected = pd.DataFrame([pd.Timestamp('2019')]) result = pd.DataFrame([pd.Timestamp('2018')]) + pd.DateOffset(years=1) tm.assert_frame_equal(expected, result) @@ -229,6 +230,7 @@ def test_timestamp_df_add_dateoffset(self): datetime.date(2017, 1, 1), ]) def test_timestamp_df_sub_timestamp(self, other): + # GH 8554 12437 expected = pd.DataFrame([pd.Timedelta('365d')]) result = pd.DataFrame([pd.Timestamp('2018')]) - other tm.assert_frame_equal(expected, result) diff --git a/pandas/tests/internals/test_internals.py b/pandas/tests/internals/test_internals.py index 1645f9069fa26..f85978e2e3a58 100644 --- a/pandas/tests/internals/test_internals.py +++ b/pandas/tests/internals/test_internals.py @@ -1248,16 +1248,14 @@ class TestCanHoldElement(object): operator.pow, ], ids=lambda x: x.__name__) def test_binop_other(self, op, value, dtype): - skip = { - (operator.add, 'bool'), - (operator.sub, 'bool'), - (operator.mul, 'bool'), - (operator.truediv, 'bool'), - (operator.mod, 'i8'), - (operator.mod, 'complex128'), - (operator.mod, '