Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Split test_excel.py (#24472) #24749

Closed
wants to merge 6 commits into from
Closed

Split test_excel.py (#24472) #24749

Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
f6309d9
Split test_excel.py (#24472)
stevenbw Jan 13, 2019
0f9d7f6
Split test_excel.py (#24472)
stevenbw Jan 13, 2019
b21b2aa
Split test_excel.py (#24472)
stevenbw Jan 13, 2019
11ee17e
Split test_excel.py (#24472)
stevenbw Jan 13, 2019
1c2cd2c
Split test_excel.py (#24472)
stevenbw Jan 13, 2019
05467ea
TST: Split test_excel.py (#24472)
stevenbw Jan 20, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Empty file added pandas/tests/io/excel/__init__.py
View file
Open in desktop
Empty file.
255 changes: 1 addition & 254 deletions pandas/tests/io/test_excel.py → pandas/tests/io/excel/test_base.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
from collections import OrderedDict
import contextlib
from datetime import date, datetime, time, timedelta
from distutils.version import LooseVersion
from functools import partial
Expand Down Expand Up @@ -36,21 +35,6 @@
_mixed_frame['foo'] = 'bar'


@contextlib.contextmanager
def ignore_xlrd_time_clock_warning():
"""
Context manager to ignore warnings raised by the xlrd library,
regarding the deprecation of `time.clock` in Python 3.7.
"""
with warnings.catch_warnings():
warnings.filterwarnings(
action='ignore',
message='time.clock has been deprecated',
category=DeprecationWarning)
yield


@td.skip_if_no('xlrd', '1.0.0')
class SharedItems(object):

@pytest.fixture(autouse=True)
Expand Down Expand Up @@ -80,46 +64,11 @@ def get_csv_refdf(self, basename):
dfref = read_csv(pref, index_col=0, parse_dates=True, engine='python')
return dfref

def get_excelfile(self, basename, ext):
"""
Return test data ExcelFile instance.

Parameters
----------

basename : str
File base name, excluding file extension.

Returns
-------

excel : io.excel.ExcelFile
"""
return ExcelFile(os.path.join(self.dirpath, basename + ext))

def get_exceldf(self, basename, ext, *args, **kwds):
"""
Return test data DataFrame.

Parameters
----------

basename : str
File base name, excluding file extension.

Returns
-------

df : DataFrame
"""
pth = os.path.join(self.dirpath, basename + ext)
return read_excel(pth, *args, **kwds)


class ReadingTestsBase(SharedItems):
# This is based on ExcelWriterBase

@pytest.fixture(autouse=True, params=['xlrd', None])
@pytest.fixture(autouse=True, params=[None])
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

maybe a better name for test_base would be test_common or test_all_engines. this split is creating too much code duplication. maybe a split on reader/writer tests first before splitting out xlrd

def set_engine(self, request):
func_name = "get_exceldf"
old_func = getattr(self, func_name)
Expand All @@ -128,96 +77,6 @@ def set_engine(self, request):
yield
setattr(self, func_name, old_func)

@td.skip_if_no("xlrd", "1.0.1") # see gh-22682
def test_usecols_int(self, ext):
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

you'll probably just want to split by complete classes in this PR and leave refactoring the classes to a follow-on PR.


df_ref = self.get_csv_refdf("test1")
df_ref = df_ref.reindex(columns=["A", "B", "C"])

# usecols as int
with tm.assert_produces_warning(FutureWarning,
check_stacklevel=False):
with ignore_xlrd_time_clock_warning():
df1 = self.get_exceldf("test1", ext, "Sheet1",
index_col=0, usecols=3)

# usecols as int
with tm.assert_produces_warning(FutureWarning,
check_stacklevel=False):
with ignore_xlrd_time_clock_warning():
df2 = self.get_exceldf("test1", ext, "Sheet2", skiprows=[1],
index_col=0, usecols=3)

# parse_cols instead of usecols, usecols as int
with tm.assert_produces_warning(FutureWarning,
check_stacklevel=False):
with ignore_xlrd_time_clock_warning():
df3 = self.get_exceldf("test1", ext, "Sheet2", skiprows=[1],
index_col=0, parse_cols=3)

# TODO add index to xls file)
tm.assert_frame_equal(df1, df_ref, check_names=False)
tm.assert_frame_equal(df2, df_ref, check_names=False)
tm.assert_frame_equal(df3, df_ref, check_names=False)

@td.skip_if_no('xlrd', '1.0.1') # GH-22682
def test_usecols_list(self, ext):

dfref = self.get_csv_refdf('test1')
dfref = dfref.reindex(columns=['B', 'C'])
df1 = self.get_exceldf('test1', ext, 'Sheet1', index_col=0,
usecols=[0, 2, 3])
df2 = self.get_exceldf('test1', ext, 'Sheet2', skiprows=[1],
index_col=0, usecols=[0, 2, 3])

with tm.assert_produces_warning(FutureWarning):
with ignore_xlrd_time_clock_warning():
df3 = self.get_exceldf('test1', ext, 'Sheet2', skiprows=[1],
index_col=0, parse_cols=[0, 2, 3])

# TODO add index to xls file)
tm.assert_frame_equal(df1, dfref, check_names=False)
tm.assert_frame_equal(df2, dfref, check_names=False)
tm.assert_frame_equal(df3, dfref, check_names=False)

@td.skip_if_no('xlrd', '1.0.1') # GH-22682
def test_usecols_str(self, ext):

dfref = self.get_csv_refdf('test1')

df1 = dfref.reindex(columns=['A', 'B', 'C'])
df2 = self.get_exceldf('test1', ext, 'Sheet1', index_col=0,
usecols='A:D')
df3 = self.get_exceldf('test1', ext, 'Sheet2', skiprows=[1],
index_col=0, usecols='A:D')

with tm.assert_produces_warning(FutureWarning):
with ignore_xlrd_time_clock_warning():
df4 = self.get_exceldf('test1', ext, 'Sheet2', skiprows=[1],
index_col=0, parse_cols='A:D')

# TODO add index to xls, read xls ignores index name ?
tm.assert_frame_equal(df2, df1, check_names=False)
tm.assert_frame_equal(df3, df1, check_names=False)
tm.assert_frame_equal(df4, df1, check_names=False)

df1 = dfref.reindex(columns=['B', 'C'])
df2 = self.get_exceldf('test1', ext, 'Sheet1', index_col=0,
usecols='A,C,D')
df3 = self.get_exceldf('test1', ext, 'Sheet2', skiprows=[1],
index_col=0, usecols='A,C,D')
# TODO add index to xls file
tm.assert_frame_equal(df2, df1, check_names=False)
tm.assert_frame_equal(df3, df1, check_names=False)

df1 = dfref.reindex(columns=['B', 'C'])
df2 = self.get_exceldf('test1', ext, 'Sheet1', index_col=0,
usecols='A,C:D')
df3 = self.get_exceldf('test1', ext, 'Sheet2', skiprows=[1],
index_col=0, usecols='A,C:D')
tm.assert_frame_equal(df2, df1, check_names=False)
tm.assert_frame_equal(df3, df1, check_names=False)

@pytest.mark.parametrize("usecols", [
[0, 1, 3], [0, 3, 1],
[1, 0, 3], [1, 3, 0],
Expand Down Expand Up @@ -342,47 +201,6 @@ def test_excel_passes_na(self, ext):
columns=['Test'])
tm.assert_frame_equal(parsed, expected)

@td.skip_if_no('xlrd', '1.0.1') # GH-22682
def test_deprecated_sheetname(self, ext):
# gh-17964
excel = self.get_excelfile('test1', ext)

with tm.assert_produces_warning(FutureWarning, check_stacklevel=False):
read_excel(excel, sheetname='Sheet1')

with pytest.raises(TypeError):
read_excel(excel, sheet='Sheet1')

@td.skip_if_no('xlrd', '1.0.1') # GH-22682
def test_excel_table_sheet_by_index(self, ext):

excel = self.get_excelfile('test1', ext)
dfref = self.get_csv_refdf('test1')

df1 = read_excel(excel, 0, index_col=0)
df2 = read_excel(excel, 1, skiprows=[1], index_col=0)
tm.assert_frame_equal(df1, dfref, check_names=False)
tm.assert_frame_equal(df2, dfref, check_names=False)

df1 = excel.parse(0, index_col=0)
df2 = excel.parse(1, skiprows=[1], index_col=0)
tm.assert_frame_equal(df1, dfref, check_names=False)
tm.assert_frame_equal(df2, dfref, check_names=False)

df3 = read_excel(excel, 0, index_col=0, skipfooter=1)
tm.assert_frame_equal(df3, df1.iloc[:-1])

with tm.assert_produces_warning(FutureWarning, check_stacklevel=False):
df4 = read_excel(excel, 0, index_col=0, skip_footer=1)
tm.assert_frame_equal(df3, df4)

df3 = excel.parse(0, index_col=0, skipfooter=1)
tm.assert_frame_equal(df3, df1.iloc[:-1])

import xlrd
with pytest.raises(xlrd.XLRDError):
read_excel(excel, 'asdf')

def test_excel_table(self, ext):

dfref = self.get_csv_refdf('test1')
Expand Down Expand Up @@ -644,35 +462,6 @@ def test_date_conversion_overflow(self, ext):
result = self.get_exceldf('testdateoverflow', ext)
tm.assert_frame_equal(result, expected)

@td.skip_if_no("xlrd", "1.0.1") # see gh-22682
def test_sheet_name_and_sheetname(self, ext):
# gh-10559: Minor improvement: Change "sheet_name" to "sheetname"
# gh-10969: DOC: Consistent var names (sheetname vs sheet_name)
# gh-12604: CLN GH10559 Rename sheetname variable to sheet_name
# gh-20920: ExcelFile.parse() and pd.read_xlsx() have different
# behavior for "sheetname" argument
filename = "test1"
sheet_name = "Sheet1"

df_ref = self.get_csv_refdf(filename)
df1 = self.get_exceldf(filename, ext,
sheet_name=sheet_name, index_col=0) # doc
with tm.assert_produces_warning(FutureWarning, check_stacklevel=False):
with ignore_xlrd_time_clock_warning():
df2 = self.get_exceldf(filename, ext, index_col=0,
sheetname=sheet_name) # backward compat

excel = self.get_excelfile(filename, ext)
df1_parse = excel.parse(sheet_name=sheet_name, index_col=0) # doc
with tm.assert_produces_warning(FutureWarning, check_stacklevel=False):
df2_parse = excel.parse(index_col=0,
sheetname=sheet_name) # backward compat

tm.assert_frame_equal(df1, df_ref, check_names=False)
tm.assert_frame_equal(df2, df_ref, check_names=False)
tm.assert_frame_equal(df1_parse, df_ref, check_names=False)
tm.assert_frame_equal(df2_parse, df_ref, check_names=False)

def test_sheet_name_both_raises(self, ext):
with pytest.raises(TypeError, match="Cannot specify both"):
self.get_exceldf('test1', ext, sheetname='Sheet1',
Expand Down Expand Up @@ -1142,34 +931,6 @@ def test_read_excel_squeeze(self, ext):
tm.assert_series_equal(actual, expected)


@pytest.mark.parametrize("ext", ['.xls', '.xlsx', '.xlsm'])
class TestXlrdReader(ReadingTestsBase):
"""
This is the base class for the xlrd tests, and 3 different file formats
are supported: xls, xlsx, xlsm
"""

@td.skip_if_no("xlwt")
def test_read_xlrd_book(self, ext):
import xlrd
df = self.frame

engine = "xlrd"
sheet_name = "SheetA"

with ensure_clean(ext) as pth:
df.to_excel(pth, sheet_name)
book = xlrd.open_workbook(pth)

with ExcelFile(book, engine=engine) as xl:
result = read_excel(xl, sheet_name, index_col=0)
tm.assert_frame_equal(df, result)

result = read_excel(book, sheet_name=sheet_name,
engine=engine, index_col=0)
tm.assert_frame_equal(df, result)


class _WriterBase(SharedItems):

@pytest.fixture(autouse=True)
Expand Down Expand Up @@ -1213,20 +974,6 @@ class and any subclasses, on account of the `autouse=True`
class TestExcelWriter(_WriterBase):
# Base class for test cases to run with different Excel writers.

def test_excel_sheet_by_name_raise(self, *_):
import xlrd

gt = DataFrame(np.random.randn(10, 2))
gt.to_excel(self.path)

xl = ExcelFile(self.path)
df = read_excel(xl, 0, index_col=0)

tm.assert_frame_equal(gt, df)

with pytest.raises(xlrd.XLRDError):
read_excel(xl, "0")

def test_excel_writer_context_manager(self, *_):
with ExcelWriter(self.path) as writer:
self.frame.to_excel(writer, "Data1")
Expand Down
Loading