From 017171d358dad41e4835221c79325f51db3b3a40 Mon Sep 17 00:00:00 2001 From: phackstock Date: Mon, 12 Sep 2022 11:52:50 +0200 Subject: [PATCH 1/8] Add xlsxwriter as dependency --- setup.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.cfg b/setup.cfg index 6e229c6a7..7d42238a1 100644 --- a/setup.cfg +++ b/setup.cfg @@ -44,6 +44,7 @@ install_requires = setuptools_scm # required explicitly for Python 3.7 importlib_metadata + xlsxwriter setup_requires = setuptools >= 41 setuptools_scm From 6fa970b31ba300d4cf1ea8513360f05836091d70 Mon Sep 17 00:00:00 2001 From: phackstock Date: Mon, 12 Sep 2022 13:31:46 +0200 Subject: [PATCH 2/8] Update RELEASE_NOTES --- RELEASE_NOTES.md | 1 + 1 file changed, 1 insertion(+) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index e982accc7..31e8f1568 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -2,6 +2,7 @@ ## Individual updates +- [#701](https://github.com/IAMconsortium/pyam/pull/701) Add xlsxwriter to improve to_excel performance - [#699](https://github.com/IAMconsortium/pyam/pull/699) Add filter options to IIASA API `index()`, `meta()` and `properties()` methods - [#697](https://github.com/IAMconsortium/pyam/pull/697) Add warning if IIASA API returns empty result - [#695](https://github.com/IAMconsortium/pyam/pull/695) Remove unused meta levels during initialization From 1be680eebbcc86dfeace4fa3c2634cdd1b31127f Mon Sep 17 00:00:00 2001 From: Philip Hackstock Date: Tue, 13 Sep 2022 14:13:10 +0200 Subject: [PATCH 3/8] Apply suggestions from code review Co-authored-by: Daniel Huppmann --- RELEASE_NOTES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 31e8f1568..d7e320c53 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -2,7 +2,7 @@ ## Individual updates -- [#701](https://github.com/IAMconsortium/pyam/pull/701) Add xlsxwriter to improve to_excel performance +- [#701](https://github.com/IAMconsortium/pyam/pull/701) Add **xlsxwriter** as dependency to improve `to_excel()` performance - [#699](https://github.com/IAMconsortium/pyam/pull/699) Add filter options to IIASA API `index()`, `meta()` and `properties()` methods - [#697](https://github.com/IAMconsortium/pyam/pull/697) Add warning if IIASA API returns empty result - [#695](https://github.com/IAMconsortium/pyam/pull/695) Remove unused meta levels during initialization From 68abc8877f56789f5e95e655879f502bc39e125e Mon Sep 17 00:00:00 2001 From: phackstock Date: Tue, 13 Sep 2022 17:03:32 +0200 Subject: [PATCH 4/8] Change write excel engine to xlsxwriter --- pyam/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyam/core.py b/pyam/core.py index bda872503..94d8099c4 100755 --- a/pyam/core.py +++ b/pyam/core.py @@ -2343,7 +2343,7 @@ def to_excel( close = False if not isinstance(excel_writer, pd.ExcelWriter): close = True - excel_writer = pd.ExcelWriter(excel_writer, engine="openpyxl") + excel_writer = pd.ExcelWriter(excel_writer, engine="xlsxwriter") # write data table write_sheet(excel_writer, sheet_name, self._to_file_format(iamc_index)) From 0f09847da014fbc0e605671de6df31508fbf1649 Mon Sep 17 00:00:00 2001 From: phackstock Date: Wed, 14 Sep 2022 11:04:07 +0200 Subject: [PATCH 5/8] Make xlsxwriter usage explicit --- pyam/core.py | 5 ++++- tests/test_io.py | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/pyam/core.py b/pyam/core.py index 94d8099c4..ba43f73a6 100755 --- a/pyam/core.py +++ b/pyam/core.py @@ -2374,7 +2374,10 @@ def export_meta(self, excel_writer, sheet_name="meta"): """ close = False if not isinstance(excel_writer, pd.ExcelWriter): - excel_writer, close = pd.ExcelWriter(excel_writer), True + excel_writer, close = ( + pd.ExcelWriter(excel_writer, engine="xlsxwriter"), + True, + ) write_sheet(excel_writer, sheet_name, self.meta, index=True) if close: excel_writer.close() diff --git a/tests/test_io.py b/tests/test_io.py index e3952cd36..933fb1910 100644 --- a/tests/test_io.py +++ b/tests/test_io.py @@ -79,7 +79,7 @@ def test_io_xlsx(test_df, meta_args, tmpdir): def test_io_xlsx_multiple_data_sheets(test_df, sheets, sheetname, tmpdir): # write data to separate sheets in excel file file = tmpdir / "testing_io_write_read.xlsx" - xl = pd.ExcelWriter(file) + xl = pd.ExcelWriter(file, engine="xlsxwriter") for i, (model, scenario) in enumerate(test_df.index): test_df.filter(scenario=scenario).to_excel(xl, sheet_name=sheets[i]) test_df.export_meta(xl) From 0db57517d65521c8736f2a664934688ec38a416e Mon Sep 17 00:00:00 2001 From: Philip Hackstock Date: Wed, 14 Sep 2022 11:11:40 +0200 Subject: [PATCH 6/8] Update pyam/core.py Co-authored-by: Daniel Huppmann --- pyam/core.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/pyam/core.py b/pyam/core.py index ba43f73a6..bd03f0838 100755 --- a/pyam/core.py +++ b/pyam/core.py @@ -2374,10 +2374,8 @@ def export_meta(self, excel_writer, sheet_name="meta"): """ close = False if not isinstance(excel_writer, pd.ExcelWriter): - excel_writer, close = ( - pd.ExcelWriter(excel_writer, engine="xlsxwriter"), - True, - ) + excel_writer = pd.ExcelWriter(excel_writer, engine="xlsxwriter") + close = True write_sheet(excel_writer, sheet_name, self.meta, index=True) if close: excel_writer.close() From 715c65d814c66b8aa27b87fb72dd7672586532f4 Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Wed, 14 Sep 2022 11:28:35 +0200 Subject: [PATCH 7/8] Use xlsxwriter width specification --- pyam/utils.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/pyam/utils.py b/pyam/utils.py index 51c966c7d..fdaee70fb 100644 --- a/pyam/utils.py +++ b/pyam/utils.py @@ -124,7 +124,6 @@ def write_sheet(writer, name, df, index=False): if index: df = df.reset_index() df.to_excel(writer, name, index=False) - worksheet = writer.sheets[name] for i, col in enumerate(df.columns): if df.dtypes[col].name.startswith(("float", "int")): width = len(str(col)) + 2 @@ -132,11 +131,7 @@ def write_sheet(writer, name, df, index=False): width = ( max([df[col].map(lambda x: len(str(x or "None"))).max(), len(col)]) + 2 ) - # this line fails if using an xlsx-engine other than openpyxl - try: - worksheet.column_dimensions[NUMERIC_TO_STR[i]].width = width - except AttributeError: - pass + writer.sheets[name].set_column(i, i, width) # assumes xlsxwriter as engine def read_pandas(path, sheet_name=["data*", "Data*"], *args, **kwargs): From d30b932ebb02cd9a0654399882fb77630246550b Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Wed, 14 Sep 2022 11:30:20 +0200 Subject: [PATCH 8/8] Add dependency change to the release notes --- RELEASE_NOTES.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index d7e320c53..6a4de3cba 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,5 +1,10 @@ # Next Release +## Dependency changes + +PR [#701](https://github.com/IAMconsortium/pyam/pull/701) added `xlsxwriter` as a +dependency for better performance. + ## Individual updates - [#701](https://github.com/IAMconsortium/pyam/pull/701) Add **xlsxwriter** as dependency to improve `to_excel()` performance