diff --git a/doc/source/whatsnew/v0.22.0.txt b/doc/source/whatsnew/v0.22.0.txt index 5549ba4e8f735..09c8367ff9747 100644 --- a/doc/source/whatsnew/v0.22.0.txt +++ b/doc/source/whatsnew/v0.22.0.txt @@ -77,6 +77,7 @@ Other API Changes - :func:`Series.truncate` and :func:`DataFrame.truncate` will raise a ``ValueError`` if the index is not sorted instead of an unhelpful ``KeyError`` (:issue:`17935`) - :func:`Dataframe.unstack` will now default to filling with ``np.nan`` for ``object`` columns. (:issue:`12815`) - :class:`IntervalIndex` constructor will raise if the ``closed`` parameter conflicts with how the input data is inferred to be closed (:issue:`18421`) +- Inserting missing values into indexes will work for all types of indexes and automatically insert the correct type of missing value (``NaN``, ``NaT``, etc.) regardless of the type passed in (:issue:`18295`) .. _whatsnew_0220.deprecations: diff --git a/pandas/core/indexes/base.py b/pandas/core/indexes/base.py index af9e29a84b472..d0356fa78f13b 100644 --- a/pandas/core/indexes/base.py +++ b/pandas/core/indexes/base.py @@ -3734,6 +3734,10 @@ def insert(self, loc, item): ------- new_index : Index """ + if is_scalar(item) and isna(item): + # GH 18295 + item = self._na_value + _self = np.asarray(self) item = self._coerce_scalar_to_index(item)._values idx = np.concatenate((_self[:loc], item, _self[loc:])) diff --git a/pandas/core/indexes/category.py b/pandas/core/indexes/category.py index d09e5447431ce..26ffb01b9577f 100644 --- a/pandas/core/indexes/category.py +++ b/pandas/core/indexes/category.py @@ -12,7 +12,7 @@ is_scalar) from pandas.core.common import (_asarray_tuplesafe, _values_from_object) -from pandas.core.dtypes.missing import array_equivalent +from pandas.core.dtypes.missing import array_equivalent, isna from pandas.core.algorithms import take_1d @@ -690,7 +690,7 @@ def insert(self, loc, item): """ code = self.categories.get_indexer([item]) - if (code == -1): + if (code == -1) and not (is_scalar(item) and isna(item)): raise TypeError("cannot insert an item into a CategoricalIndex " "that is not already an existing category") diff --git a/pandas/core/indexes/datetimes.py b/pandas/core/indexes/datetimes.py index 111ba0c92aa9b..ee6263a9f0aad 100644 --- a/pandas/core/indexes/datetimes.py +++ b/pandas/core/indexes/datetimes.py @@ -1757,6 +1757,9 @@ def insert(self, loc, item): ------- new_index : Index """ + if is_scalar(item) and isna(item): + # GH 18295 + item = self._na_value freq = None @@ -1773,6 +1776,7 @@ def insert(self, loc, item): elif (loc == len(self)) and item - self.freq == self[-1]: freq = self.freq item = _to_m8(item, tz=self.tz) + try: new_dates = np.concatenate((self[:loc].asi8, [item.view(np.int64)], self[loc:].asi8)) @@ -1780,7 +1784,6 @@ def insert(self, loc, item): new_dates = conversion.tz_convert(new_dates, 'UTC', self.tz) return DatetimeIndex(new_dates, name=self.name, freq=freq, tz=self.tz) - except (AttributeError, TypeError): # fall back to object index diff --git a/pandas/core/indexes/interval.py b/pandas/core/indexes/interval.py index c7c739b766a9f..06843150bf46a 100644 --- a/pandas/core/indexes/interval.py +++ b/pandas/core/indexes/interval.py @@ -1001,14 +1001,21 @@ def delete(self, loc): return self._shallow_copy(new_left, new_right) def insert(self, loc, item): - if not isinstance(item, Interval): - raise ValueError('can only insert Interval objects into an ' - 'IntervalIndex') - if not item.closed == self.closed: - raise ValueError('inserted item must be closed on the same side ' - 'as the index') - new_left = self.left.insert(loc, item.left) - new_right = self.right.insert(loc, item.right) + if isinstance(item, Interval): + if item.closed != self.closed: + raise ValueError('inserted item must be closed on the same ' + 'side as the index') + left_insert = item.left + right_insert = item.right + elif is_scalar(item) and isna(item): + # GH 18295 + left_insert = right_insert = item + else: + raise ValueError('can only insert Interval objects and NA into ' + 'an IntervalIndex') + + new_left = self.left.insert(loc, left_insert) + new_right = self.right.insert(loc, right_insert) return self._shallow_copy(new_left, new_right) def _as_like_interval_index(self, other, error_msg): diff --git a/pandas/core/indexes/timedeltas.py b/pandas/core/indexes/timedeltas.py index 22fb7c255b12c..97f6ca2e5d642 100644 --- a/pandas/core/indexes/timedeltas.py +++ b/pandas/core/indexes/timedeltas.py @@ -852,16 +852,18 @@ def insert(self, loc, item): ------- new_index : Index """ - # try to convert if possible if _is_convertible_to_td(item): try: item = Timedelta(item) except Exception: pass + elif is_scalar(item) and isna(item): + # GH 18295 + item = self._na_value freq = None - if isinstance(item, Timedelta) or item is NaT: + if isinstance(item, Timedelta) or (is_scalar(item) and isna(item)): # check freq can be preserved on edge cases if self.freq is not None: diff --git a/pandas/tests/indexes/datetimes/test_indexing.py b/pandas/tests/indexes/datetimes/test_indexing.py index 4ce9441d87970..b3ce22962d5d4 100644 --- a/pandas/tests/indexes/datetimes/test_indexing.py +++ b/pandas/tests/indexes/datetimes/test_indexing.py @@ -145,6 +145,13 @@ def test_insert(self): assert result.tz == expected.tz assert result.freq is None + # GH 18295 (test missing) + expected = DatetimeIndex( + ['20170101', pd.NaT, '20170102', '20170103', '20170104']) + for na in (np.nan, pd.NaT, None): + result = date_range('20170101', periods=4).insert(1, na) + tm.assert_index_equal(result, expected) + def test_delete(self): idx = date_range(start='2000-01-01', periods=5, freq='M', name='idx') diff --git a/pandas/tests/indexes/period/test_period.py b/pandas/tests/indexes/period/test_period.py index 52558c27ce707..5bb143c5a3cc6 100644 --- a/pandas/tests/indexes/period/test_period.py +++ b/pandas/tests/indexes/period/test_period.py @@ -697,3 +697,11 @@ def test_join_self(self, how): index = period_range('1/1/2000', periods=10) joined = index.join(index, how=how) assert index is joined + + def test_insert(self): + # GH 18295 (test missing) + expected = PeriodIndex( + ['2017Q1', pd.NaT, '2017Q2', '2017Q3', '2017Q4'], freq='Q') + for na in (np.nan, pd.NaT, None): + result = period_range('2017Q1', periods=4, freq='Q').insert(1, na) + tm.assert_index_equal(result, expected) diff --git a/pandas/tests/indexes/test_base.py b/pandas/tests/indexes/test_base.py index 99a99cc5cc3eb..0a6fa97d12170 100644 --- a/pandas/tests/indexes/test_base.py +++ b/pandas/tests/indexes/test_base.py @@ -457,6 +457,12 @@ def test_insert(self): null_index = Index([]) tm.assert_index_equal(Index(['a']), null_index.insert(0, 'a')) + # GH 18295 (test missing) + expected = Index(['a', np.nan, 'b', 'c']) + for na in (np.nan, pd.NaT, None): + result = Index(list('abc')).insert(1, na) + tm.assert_index_equal(result, expected) + def test_delete(self): idx = Index(['a', 'b', 'c', 'd'], name='idx') diff --git a/pandas/tests/indexes/test_category.py b/pandas/tests/indexes/test_category.py index 5e6898f9c8711..a04ea036b2862 100644 --- a/pandas/tests/indexes/test_category.py +++ b/pandas/tests/indexes/test_category.py @@ -344,6 +344,12 @@ def test_insert(self): # invalid pytest.raises(TypeError, lambda: ci.insert(0, 'd')) + # GH 18295 (test missing) + expected = CategoricalIndex(['a', np.nan, 'a', 'b', 'c', 'b']) + for na in (np.nan, pd.NaT, None): + result = CategoricalIndex(list('aabcb')).insert(1, na) + tm.assert_index_equal(result, expected) + def test_delete(self): ci = self.create_index() diff --git a/pandas/tests/indexes/test_interval.py b/pandas/tests/indexes/test_interval.py index 7d6f544f6d533..7df189113247b 100644 --- a/pandas/tests/indexes/test_interval.py +++ b/pandas/tests/indexes/test_interval.py @@ -366,14 +366,50 @@ def test_delete(self, closed): result = self.create_index(closed=closed).delete(0) tm.assert_index_equal(result, expected) - def test_insert(self): - expected = IntervalIndex.from_breaks(range(4)) - actual = self.index.insert(2, Interval(2, 3)) - assert expected.equals(actual) + @pytest.mark.parametrize('data', [ + interval_range(0, periods=10, closed='neither'), + interval_range(1.7, periods=8, freq=2.5, closed='both'), + interval_range(Timestamp('20170101'), periods=12, closed='left'), + interval_range(Timedelta('1 day'), periods=6, closed='right'), + IntervalIndex.from_tuples([('a', 'd'), ('e', 'j'), ('w', 'z')]), + IntervalIndex.from_tuples([(1, 2), ('a', 'z'), (3.14, 6.28)])]) + def test_insert(self, data): + item = data[0] + idx_item = IntervalIndex([item]) + + # start + expected = idx_item.append(data) + result = data.insert(0, item) + tm.assert_index_equal(result, expected) + + # end + expected = data.append(idx_item) + result = data.insert(len(data), item) + tm.assert_index_equal(result, expected) + + # mid + expected = data[:3].append(idx_item).append(data[3:]) + result = data.insert(3, item) + tm.assert_index_equal(result, expected) + + # invalid type + msg = 'can only insert Interval objects and NA into an IntervalIndex' + with tm.assert_raises_regex(ValueError, msg): + data.insert(1, 'foo') - pytest.raises(ValueError, self.index.insert, 0, 1) - pytest.raises(ValueError, self.index.insert, 0, - Interval(2, 3, closed='left')) + # invalid closed + msg = 'inserted item must be closed on the same side as the index' + for closed in {'left', 'right', 'both', 'neither'} - {item.closed}: + with tm.assert_raises_regex(ValueError, msg): + bad_item = Interval(item.left, item.right, closed=closed) + data.insert(1, bad_item) + + # GH 18295 (test missing) + na_idx = IntervalIndex([np.nan], closed=data.closed) + for na in (np.nan, pd.NaT, None): + expected = data[:1].append(na_idx).append(data[1:]) + result = data.insert(1, na) + tm.assert_index_equal(result, expected) def test_take(self, closed): index = self.create_index(closed=closed) diff --git a/pandas/tests/indexes/test_numeric.py b/pandas/tests/indexes/test_numeric.py index 030d688f510b0..cbd819fa9cfb7 100644 --- a/pandas/tests/indexes/test_numeric.py +++ b/pandas/tests/indexes/test_numeric.py @@ -187,6 +187,13 @@ def test_where(self, klass): result = i.where(klass(cond)) tm.assert_index_equal(result, expected) + def test_insert(self): + # GH 18295 (test missing) + expected = Float64Index([0, np.nan, 1, 2, 3, 4]) + for na in (np.nan, pd.NaT, None): + result = self.create_index().insert(1, na) + tm.assert_index_equal(result, expected) + class TestFloat64Index(Numeric): _holder = Float64Index diff --git a/pandas/tests/indexes/test_range.py b/pandas/tests/indexes/test_range.py index b4d1c3760f25a..96d5981abc1bb 100644 --- a/pandas/tests/indexes/test_range.py +++ b/pandas/tests/indexes/test_range.py @@ -295,6 +295,12 @@ def test_insert(self): # test 0th element tm.assert_index_equal(idx[0:4], result.insert(0, idx[0])) + # GH 18295 (test missing) + expected = Float64Index([0, np.nan, 1, 2, 3, 4]) + for na in (np.nan, pd.NaT, None): + result = RangeIndex(5).insert(1, na) + tm.assert_index_equal(result, expected) + def test_delete(self): idx = RangeIndex(5, name='Foo') diff --git a/pandas/tests/indexes/timedeltas/test_indexing.py b/pandas/tests/indexes/timedeltas/test_indexing.py index cb88bac6386f7..e64c4e6ac54a5 100644 --- a/pandas/tests/indexes/timedeltas/test_indexing.py +++ b/pandas/tests/indexes/timedeltas/test_indexing.py @@ -57,6 +57,12 @@ def test_insert(self): assert result.name == expected.name assert result.freq == expected.freq + # GH 18295 (test missing) + expected = TimedeltaIndex(['1day', pd.NaT, '2day', '3day']) + for na in (np.nan, pd.NaT, None): + result = timedelta_range('1day', '3day').insert(1, na) + tm.assert_index_equal(result, expected) + def test_delete(self): idx = timedelta_range(start='1 Days', periods=5, freq='D', name='idx')