Support pandas >=1.0.0

.github/workflows/master.yml

@@ -17,7 +17,7 @@ jobs:
       # The name of the directory '.cache' is for Travis CI. Once we remove Travis CI,
       # we should download Spark to a directory with a different name to prevent confusion.
       SPARK_CACHE_DIR: /home/runner/.cache/spark-versions
-      PANDAS_VERSION: 0.23.4
+      PANDAS_VERSION: 0.24.2
       PYARROW_VERSION: 0.10.0
       # DISPLAY=0.0 does not work in Github Actions with Python 3.5. Here we work around wtih xvfb-run
       PYTHON_EXECUTABLE: xvfb-run python
@@ -73,12 +73,12 @@ jobs:
         include:
           - python-version: 3.6
             spark-version: 2.4.5
-            pandas-version: 0.24.2
+            pandas-version: 0.25.3
             pyarrow-version: 0.13.0
             logger: databricks.koalas.usage_logging.usage_logger
           - python-version: 3.7
             spark-version: 2.4.5
-            pandas-version: 0.25.3
+            pandas-version: 1.0.1
             pyarrow-version: 0.14.1
     env:
       PYTHON_VERSION: ${{ matrix.python-version }}

databricks/koalas/frame.py

@@ -4151,8 +4151,7 @@ def from_records(data: Union[np.array, List[tuple], dict, pd.DataFrame],
         return DataFrame(pd.DataFrame.from_records(data, index, exclude, columns, coerce_float,
                                                    nrows))
 
-    def to_records(self, index=True, convert_datetime64=None,
-                   column_dtypes=None, index_dtypes=None):
+    def to_records(self, index=True, column_dtypes=None, index_dtypes=None):
         """
         Convert DataFrame to a NumPy record array.
 
@@ -4167,9 +4166,6 @@ def to_records(self, index=True, convert_datetime64=None,
         index : bool, default True
             Include index in resulting record array, stored in 'index'
             field or using the index label, if set.
-        convert_datetime64 : bool, default None
-            Whether to convert the index to datetime.datetime if it is a
-            DatetimeIndex.
         column_dtypes : str, type, dict, default None
             If a string or type, the data type to store all columns. If
             a dictionary, a mapping of column names and indices (zero-indexed)
@@ -8361,9 +8357,11 @@ def info(
         <class 'databricks.koalas.frame.DataFrame'>
         Index: 5 entries, 0 to 4
         Data columns (total 3 columns):
-        int_col      5 non-null int64
-        text_col     5 non-null object
-        float_col    5 non-null float64
+         #   Column     Non-Null Count  Dtype
+        ---  ------     --------------  -----
+         0   int_col    5 non-null      int64
+         1   text_col   5 non-null      object
+         2   float_col  5 non-null      float64
         dtypes: float64(1), int64(1), object(1)
 
         Prints a summary of columns count and its dtypes but not per column
@@ -8386,13 +8384,15 @@ def info(
         ...           encoding="utf-8") as f:
         ...     _ = f.write(s)
         >>> with open('%s/info.txt' % path) as f:
-        ...     f.readlines()  # doctest: +NORMALIZE_WHITESPACE, +ELLIPSIS
-        [...databricks.koalas.frame.DataFrame...,
+        ...     f.readlines()  # doctest: +SKIP
+        ["<class 'databricks.koalas.frame.DataFrame'>\\n",
         'Index: 5 entries, 0 to 4\\n',
         'Data columns (total 3 columns):\\n',
-        'int_col      5 non-null int64\\n',
-        'text_col     5 non-null object\\n',
-        'float_col    5 non-null float64\\n',
+        ' #   Column     Non-Null Count  Dtype  \\n',
+        '---  ------     --------------  -----  \\n',
+        ' 0   int_col    5 non-null      int64  \\n',
+        ' 1   text_col   5 non-null      object \\n',
+        ' 2   float_col  5 non-null      float64\\n',
         'dtypes: float64(1), int64(1), object(1)']
         """
         # To avoid pandas' existing config affects Koalas.

databricks/koalas/generic.py

@@ -297,6 +297,8 @@ def cumprod(self, skipna: bool = True):
         """
         return self._apply_series_op(lambda kser: kser._cumprod(skipna))  # type: ignore
 
+    # TODO: Although this has removed pandas >= 1.0.0, but we're keeping this as deprecated
+    # since we're using this for `DataFrame.info` internally.
     def get_dtype_counts(self):
         """
         Return counts of unique dtypes in this object.

databricks/koalas/indexes.py

@@ -1924,13 +1924,13 @@ def symmetric_difference(self, other, result_name=None, sort=None):
         return result
 
     # TODO: ADD error parameter
-    def drop(self, labels, level=None):
+    def drop(self, codes, level=None):
         """
         Make new MultiIndex with passed list of labels deleted
 
         Parameters
         ----------
-        labels : array-like
+        codes : array-like
             Must be a list of tuples
         level : int or level name, default None
 
@@ -1962,7 +1962,7 @@ def drop(self, labels, level=None):
             scol = index_scols[0]
         else:
             scol = index_scols[level] if isinstance(level, int) else sdf[level]
-        sdf = sdf[~scol.isin(labels)]
+        sdf = sdf[~scol.isin(codes)]
         return MultiIndex(DataFrame(_InternalFrame(sdf=sdf,
                                                    index_map=self._kdf._internal.index_map)))
 

databricks/koalas/missing/frame.py

@@ -29,12 +29,6 @@ def unsupported_property(property_name, deprecated=False, reason=""):
 
 class _MissingPandasLikeDataFrame(object):
 
-    # Deprecated properties
-    blocks = unsupported_property('blocks', deprecated=True)
-    ftypes = unsupported_property('ftypes', deprecated=True)
-    is_copy = unsupported_property('is_copy', deprecated=True)
-    ix = unsupported_property('ix', deprecated=True)
-
     # Functions
     align = unsupported_function('align')
     asfreq = unsupported_function('asfreq')
@@ -82,27 +76,18 @@ class _MissingPandasLikeDataFrame(object):
     to_sql = unsupported_function('to_sql')
     to_stata = unsupported_function('to_stata')
     to_timestamp = unsupported_function('to_timestamp')
+    to_markdown = unsupported_function('to_markdown')
     truncate = unsupported_function('truncate')
     tshift = unsupported_function('tshift')
     tz_convert = unsupported_function('tz_convert')
     tz_localize = unsupported_function('tz_localize')
     unstack = unsupported_function('unstack')
 
     # Deprecated functions
-    as_blocks = unsupported_function('as_blocks', deprecated=True)
-    as_matrix = unsupported_function('as_matrix', deprecated=True)
-    clip_lower = unsupported_function('clip_lower', deprecated=True)
-    clip_upper = unsupported_function('clip_upper', deprecated=True)
     convert_objects = unsupported_function('convert_objects', deprecated=True)
-    get_ftype_counts = unsupported_function('get_ftype_counts', deprecated=True)
-    get_value = unsupported_function('get_value', deprecated=True)
     select = unsupported_function('select', deprecated=True)
-    set_value = unsupported_function('set_value', deprecated=True)
     to_panel = unsupported_function('to_panel', deprecated=True)
     get_values = unsupported_function('get_values', deprecated=True)
-    to_dense = unsupported_function('to_dense', deprecated=True)
-    to_sparse = unsupported_function('to_sparse', deprecated=True)
-    to_msgpack = unsupported_function('to_msgpack', deprecated=True)
     compound = unsupported_function('compound', deprecated=True)
     reindex_axis = unsupported_function('reindex_axis', deprecated=True)
 

databricks/koalas/missing/indexes.py

@@ -32,13 +32,6 @@ class _MissingPandasLikeIndex(object):
     # Properties
     nbytes = unsupported_property('nbytes')
 
-    # Deprecated properties
-    strides = unsupported_property('strides', deprecated=True)
-    data = unsupported_property('data', deprecated=True)
-    itemsize = unsupported_property('itemsize', deprecated=True)
-    base = unsupported_property('base', deprecated=True)
-    flags = unsupported_property('flags', deprecated=True)
-
     # Functions
     argsort = unsupported_function('argsort')
     asof = unsupported_function('asof')
@@ -70,7 +63,6 @@ class _MissingPandasLikeIndex(object):
     reindex = unsupported_function('reindex')
     repeat = unsupported_function('repeat')
     searchsorted = unsupported_function('searchsorted')
-    set_value = unsupported_function('set_value')
     slice_indexer = unsupported_function('slice_indexer')
     slice_locs = unsupported_function('slice_locs')
     sortlevel = unsupported_function('sortlevel')
@@ -82,11 +74,9 @@ class _MissingPandasLikeIndex(object):
     where = unsupported_function('where')
 
     # Deprecated functions
-    get_duplicates = unsupported_function('get_duplicates', deprecated=True)
-    summary = unsupported_function('summary', deprecated=True)
     get_values = unsupported_function('get_values', deprecated=True)
     item = unsupported_function('item', deprecated=True)
-    contains = unsupported_function('contains', deprecated=True)
+    set_value = unsupported_function('set_value')
 
     # Properties we won't support.
     values = common.values(unsupported_property)
@@ -105,10 +95,7 @@ class _MissingPandasLikeMultiIndex(object):
     # Deprecated properties
     strides = unsupported_property('strides', deprecated=True)
     data = unsupported_property('data', deprecated=True)
-    base = unsupported_property('base', deprecated=True)
     itemsize = unsupported_property('itemsize', deprecated=True)
-    labels = unsupported_property('labels', deprecated=True)
-    flags = unsupported_property('flags', deprecated=True)
 
     # Functions
     argsort = unsupported_function('argsort')
@@ -148,9 +135,7 @@ class _MissingPandasLikeMultiIndex(object):
     repeat = unsupported_function('repeat')
     searchsorted = unsupported_function('searchsorted')
     set_codes = unsupported_function('set_codes')
-    set_labels = unsupported_function('set_labels')
     set_levels = unsupported_function('set_levels')
-    set_value = unsupported_function('set_value')
     slice_indexer = unsupported_function('slice_indexer')
     slice_locs = unsupported_function('slice_locs')
     sortlevel = unsupported_function('sortlevel')
@@ -164,11 +149,9 @@ class _MissingPandasLikeMultiIndex(object):
 
     # Deprecated functions
     get_duplicates = unsupported_function('get_duplicates', deprecated=True)
-    summary = unsupported_function('summary', deprecated=True)
-    to_hierarchical = unsupported_function('to_hierarchical', deprecated=True)
     get_values = unsupported_function('get_values', deprecated=True)
-    contains = unsupported_function('contains', deprecated=True)
     item = unsupported_function('item', deprecated=True)
+    set_value = unsupported_function('set_value', deprecated=True)
 
     # Functions we won't support.
     values = common.values(unsupported_property)

databricks/koalas/missing/series.py

@@ -29,20 +29,6 @@ def unsupported_property(property_name, deprecated=False, reason=""):
 
 class _MissingPandasLikeSeries(object):
 
-    # Deprecated properties
-    blocks = unsupported_property('blocks', deprecated=True)
-    ftypes = unsupported_property('ftypes', deprecated=True)
-    ftype = unsupported_property('ftype', deprecated=True)
-    is_copy = unsupported_property('is_copy', deprecated=True)
-    ix = unsupported_property('ix', deprecated=True)
-    asobject = unsupported_property('asobject', deprecated=True)
-    strides = unsupported_property('strides', deprecated=True)
-    imag = unsupported_property('imag', deprecated=True)
-    itemsize = unsupported_property('itemsize', deprecated=True)
-    data = unsupported_property('data', deprecated=True)
-    base = unsupported_property('base', deprecated=True)
-    flags = unsupported_property('flags', deprecated=True)
-
     # Functions
     align = unsupported_function('align')
     argsort = unsupported_function('argsort')
@@ -65,6 +51,7 @@ class _MissingPandasLikeSeries(object):
     first = unsupported_function('first')
     infer_objects = unsupported_function('infer_objects')
     interpolate = unsupported_function('interpolate')
+    item = unsupported_function('item')
     items = unsupported_function('items')
     iteritems = unsupported_function('iteritems')
     last = unsupported_function('last')
@@ -99,37 +86,16 @@ class _MissingPandasLikeSeries(object):
     view = unsupported_function('view')
 
     # Deprecated functions
-    as_blocks = unsupported_function('as_blocks', deprecated=True)
-    as_matrix = unsupported_function('as_matrix', deprecated=True)
-    clip_lower = unsupported_function('clip_lower', deprecated=True)
-    clip_upper = unsupported_function('clip_upper', deprecated=True)
-    compress = unsupported_function('compress', deprecated=True)
     convert_objects = unsupported_function('convert_objects', deprecated=True)
-    get_ftype_counts = unsupported_function('get_ftype_counts', deprecated=True)
-    get_value = unsupported_function('get_value', deprecated=True)
     nonzero = unsupported_function('nonzero', deprecated=True)
     reindex_axis = unsupported_function('reindex_axis', deprecated=True)
     select = unsupported_function('select', deprecated=True)
-    set_value = unsupported_function('set_value', deprecated=True)
-    valid = unsupported_function('valid', deprecated=True)
     get_values = unsupported_function('get_values', deprecated=True)
-    to_dense = unsupported_function('to_dense', deprecated=True)
-    to_sparse = unsupported_function('to_sparse', deprecated=True)
-    to_msgpack = unsupported_function('to_msgpack', deprecated=True)
-    compound = unsupported_function('compound', deprecated=True)
-    put = unsupported_function('put', deprecated=True)
-    item = unsupported_function('item', deprecated=True)
-    ptp = unsupported_function('ptp', deprecated=True)
-    argmax = unsupported_function('argmax', deprecated=True)
-    argmin = unsupported_function('argmin', deprecated=True)
 
     # Properties we won't support.
     values = common.values(unsupported_property)
     array = common.array(unsupported_property)
     duplicated = common.duplicated(unsupported_property)
-    real = unsupported_property(
-        'real',
-        reason="If you want to collect your data as an NumPy array, use 'to_numpy()' instead.")
     nbytes = unsupported_property(
         'nbytes',
         reason="'nbytes' requires to compute whole dataset. You can calculate manually it, "

databricks/koalas/namespace.py

@@ -26,7 +26,7 @@
 
 import numpy as np
 import pandas as pd
-
+from pandas.api.types import is_list_like
 from pyspark import sql as spark
 from pyspark.sql import functions as F
 from pyspark.sql.types import ByteType, ShortType, IntegerType, LongType, FloatType, \
@@ -1266,6 +1266,10 @@ def get_dummies(data, prefix=None, prefix_sep='_', dummy_na=False, columns=None,
     if sparse is not False:
         raise NotImplementedError("get_dummies currently does not support sparse")
 
+    if columns is not None:
+        if not is_list_like(columns):
+            raise TypeError("Input must be a list-like for parameter `columns`")
+
     if dtype is None:
         dtype = 'byte'
 
@@ -1307,7 +1311,9 @@ def get_dummies(data, prefix=None, prefix_sep='_', dummy_na=False, columns=None,
                                  for label in kdf._internal.column_labels
                                  if label == key or label[0] == key]
         if len(column_labels) == 0:
-            return kdf
+            if columns is None:
+                return kdf
+            raise KeyError("{} not in index".format(columns))
 
         if prefix is None:
             prefix = [str(label) if len(label) > 1 else label[0] for label in column_labels]

databricks/koalas/tests/test_dataframe_conversion.py

@@ -181,8 +181,9 @@ def test_to_latex(self):
         self.assert_eq(kdf.to_latex(sparsify=False), pdf.to_latex(sparsify=False))
         self.assert_eq(kdf.to_latex(index_names=False), pdf.to_latex(index_names=False))
         self.assert_eq(kdf.to_latex(bold_rows=True), pdf.to_latex(bold_rows=True))
-        self.assert_eq(kdf.to_latex(encoding='ascii'), pdf.to_latex(encoding='ascii'))
         self.assert_eq(kdf.to_latex(decimal=','), pdf.to_latex(decimal=','))
+        if LooseVersion(pd.__version__) < LooseVersion("1.0.0"):
+            self.assert_eq(kdf.to_latex(encoding='ascii'), pdf.to_latex(encoding='ascii'))
 
     def test_to_records(self):
         if LooseVersion(pd.__version__) >= LooseVersion("0.24.0"):

databricks/koalas/tests/test_expanding.py

@@ -96,16 +96,14 @@ def _test_groupby_expanding_func(self, f):
             repr(getattr(kser.groupby(kser).expanding(2), f)().sort_index()),
             repr(getattr(pser.groupby(pser).expanding(2), f)().sort_index()))
 
-        kdf = ks.DataFrame({'a': [1, 2, 3, 2], 'b': [4.0, 2.0, 3.0, 1.0]},
-                           index=np.random.rand(4))
+        kdf = ks.DataFrame({'a': [1, 2, 3, 2], 'b': [4.0, 2.0, 3.0, 1.0]})
         pdf = kdf.to_pandas()
         self.assert_eq(
             repr(getattr(kdf.groupby(kdf.a).expanding(2), f)().sort_index()),
             repr(getattr(pdf.groupby(pdf.a).expanding(2), f)().sort_index()))
 
         # Multiindex column
-        kdf = ks.DataFrame({'a': [1, 2, 3, 2], 'b': [4.0, 2.0, 3.0, 1.0]},
-                           index=np.random.rand(4))
+        kdf = ks.DataFrame({'a': [1, 2, 3, 2], 'b': [4.0, 2.0, 3.0, 1.0]})
         kdf.columns = pd.MultiIndex.from_tuples([('a', 'x'), ('a', 'y')])
         pdf = kdf.to_pandas()
         self.assert_eq(