From 4f3fbadd0278d32e81b55bbce7a9a721b4b0d948 Mon Sep 17 00:00:00 2001 From: sebastian-swob Date: Tue, 12 Nov 2024 14:23:35 +0100 Subject: [PATCH] formatting and pipeline fixes --- dev-tools/dev_tools.py | 81 ++++++++++++++++++------- doc/conf.py | 10 ++- pytrnsys_process/monthly.py | 12 +++- pytrnsys_process/plotters.py | 23 +++++-- pytrnsys_process/readers.py | 6 +- tests/pytrnsys_process/constants.py | 2 - tests/pytrnsys_process/test_headers.py | 5 +- tests/pytrnsys_process/test_monthly.py | 8 ++- tests/pytrnsys_process/test_plotters.py | 9 +-- tests/pytrnsys_process/test_reader.py | 15 +++-- 10 files changed, 118 insertions(+), 53 deletions(-) diff --git a/dev-tools/dev_tools.py b/dev-tools/dev_tools.py index 89cff96..168e8da 100644 --- a/dev-tools/dev_tools.py +++ b/dev-tools/dev_tools.py @@ -28,7 +28,9 @@ def main(): arguments = _parse_arguments() test_results_dir_path = pl.Path("test-results") - _prepare_test_results_directory(test_results_dir_path, arguments.shallKeepResults) + _prepare_test_results_directory( + test_results_dir_path, arguments.shallKeepResults + ) _maybe_run_mypy(arguments) @@ -61,7 +63,14 @@ def _parse_arguments() -> ap.Namespace: dest="shallPerformStaticChecks", ) parser.add_argument( - "-l", "--lint", help="Perform linting", type=str, default=None, const="", nargs="?", dest="lintArguments" + "-l", + "--lint", + help="Perform linting", + type=str, + default=None, + const="", + nargs="?", + dest="lintArguments", ) parser.add_argument( "-b", @@ -121,9 +130,14 @@ def _parse_arguments() -> ap.Namespace: return arguments -def _prepare_test_results_directory(test_results_dir_path: pl.Path, shall_keep_results: bool) -> None: +def _prepare_test_results_directory( + test_results_dir_path: pl.Path, shall_keep_results: bool +) -> None: if test_results_dir_path.exists() and not test_results_dir_path.is_dir(): - print("ERROR: `test-results` exists but is not a directory", file=sys.stderr) + print( + "ERROR: `test-results` exists but is not a directory", + file=sys.stderr, + ) sys.exit(2) if not shall_keep_results and test_results_dir_path.is_dir(): @@ -138,14 +152,22 @@ def _prepare_test_results_directory(test_results_dir_path: pl.Path, shall_keep_r def _maybe_run_mypy(arguments): - if arguments.shallRunAll or arguments.shallPerformStaticChecks or arguments.mypyArguments is not None: + if ( + arguments.shallRunAll + or arguments.shallPerformStaticChecks + or arguments.mypyArguments is not None + ): cmd = _create_static_checker_command("mypy", "--show-error-codes") additional_args = arguments.mypyArguments or "" _print_and_run([*cmd, *additional_args.split()]) def _maybe_run_pylint(arguments): - if arguments.shallRunAll or arguments.shallPerformStaticChecks or arguments.lintArguments is not None: + if ( + arguments.shallRunAll + or arguments.shallPerformStaticChecks + or arguments.lintArguments is not None + ): cmd = _create_static_checker_command("pylint") additional_args = arguments.lintArguments or "" @@ -153,14 +175,24 @@ def _maybe_run_pylint(arguments): def _maybe_run_black(arguments): - if arguments.shallRunAll or arguments.shallPerformStaticChecks or arguments.blackArguments is not None: + if ( + arguments.shallRunAll + or arguments.shallPerformStaticChecks + or arguments.blackArguments is not None + ): cmd = _create_static_checker_command("black", "-l 79") - additional_args = "--check" if arguments.blackArguments is None else arguments.blackArguments + additional_args = ( + "--check" + if arguments.blackArguments is None + else arguments.blackArguments + ) _print_and_run([*cmd, *additional_args.split()]) -def _create_static_checker_command(static_checker_name: str, *args: str) -> tp.Sequence[str]: +def _create_static_checker_command( + static_checker_name: str, *args: str +) -> tp.Sequence[str]: cmd = [ f"{_SCRIPTS_DIR / static_checker_name}", *args, @@ -171,35 +203,40 @@ def _create_static_checker_command(static_checker_name: str, *args: str) -> tp.S def _maybe_create_diagrams(arguments): if arguments.shallRunAll or arguments.diagramsFormat: - diagrams_format = arguments.diagramsFormat if arguments.diagramsFormat else "pdf" - cmd = ( - f"{_SCRIPTS_DIR / 'pyreverse'} -k -o {diagrams_format} -p pytrnsys_process -d test-results pytrnsys_process" + diagrams_format = ( + arguments.diagramsFormat if arguments.diagramsFormat else "pdf" ) + cmd = f"{_SCRIPTS_DIR / 'pyreverse'} -k -o {diagrams_format} -p pytrnsys_process -d test-results pytrnsys_process" _print_and_run(cmd.split()) def _maybe_run_pytest(arguments, test_results_dir_path): was_called_without_arguments = ( - not arguments.shallPerformStaticChecks - and arguments.mypyArguments is None - and arguments.lintArguments is None - and arguments.blackArguments is None - and arguments.diagramsFormat is None - and not arguments.shallCreateDocumentation + not arguments.shallPerformStaticChecks + and arguments.mypyArguments is None + and arguments.lintArguments is None + and arguments.blackArguments is None + and arguments.diagramsFormat is None + and not arguments.shallCreateDocumentation ) - if arguments.shallRunAll or arguments.pytestMarkersExpression is not None or was_called_without_arguments: + if ( + arguments.shallRunAll + or arguments.pytestMarkersExpression is not None + or was_called_without_arguments + ): _run_unit_tests_with_pytest(arguments, test_results_dir_path) _run_doctests_with_pytest() def _run_unit_tests_with_pytest(arguments, test_results_dir_path): - marker_expressions = _get_marker_expressions(arguments.pytestMarkersExpression) + marker_expressions = _get_marker_expressions( + arguments.pytestMarkersExpression + ) additional_args = ["-m", marker_expressions] cmd = [ _SCRIPTS_DIR / "pytest", "-v", - "--benchmark-skip" - "--cov=pytrnsys_process", + "--benchmark-skip" "--cov=pytrnsys_process", f"--cov-report=html:{test_results_dir_path / 'coverage-html'}", f"--cov-report=lcov:{test_results_dir_path / 'coverage.lcov'}", "--cov-report=term", diff --git a/doc/conf.py b/doc/conf.py index 9e185c5..b300097 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -15,12 +15,18 @@ project = "pytrnsys_process" copyright = "2024, SPF Institute of Solar Technology, OST University of Applied Sciences" -author = "SPF Institute of Solar Technology, OST University of Applied Sciences" +author = ( + "SPF Institute of Solar Technology, OST University of Applied Sciences" +) # -- General configuration --------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration -extensions = ["sphinx.ext.napoleon", "sphinx.ext.autodoc", "sphinx.ext.doctest"] +extensions = [ + "sphinx.ext.napoleon", + "sphinx.ext.autodoc", + "sphinx.ext.doctest", +] doctest_test_doctest_blocks = "yes" diff --git a/pytrnsys_process/monthly.py b/pytrnsys_process/monthly.py index b61b819..c7b4707 100644 --- a/pytrnsys_process/monthly.py +++ b/pytrnsys_process/monthly.py @@ -8,7 +8,9 @@ _N_ROWS_USED = 12 -def read_monthly_file(prt_file_path: _pl.Path, starting_year: int = 2001) -> _pd.DataFrame: +def read_monthly_file( + prt_file_path: _pl.Path, starting_year: int = 2001 +) -> _pd.DataFrame: """Load monthly data written by Type 46 as a `pandas` `DataFrame`. Parameters @@ -54,14 +56,18 @@ def read_monthly_file(prt_file_path: _pl.Path, starting_year: int = 2001) -> _pd Notice how the time stamps are given **at the end of a month**. """ - df = _pd.read_csv(prt_file_path, header=1, delimiter=r"\s+", nrows=_N_ROWS_USED) + df = _pd.read_csv( + prt_file_path, header=1, delimiter=r"\s+", nrows=_N_ROWS_USED + ) df = df.rename(columns=lambda x: x.strip()) hours = _dt.timedelta(hours=1) * df["Time"] # type: ignore start_of_year = _dt.datetime(day=1, month=1, year=starting_year) actual_ends_of_month = start_of_year + hours - expected_ends_of_months = _pd.date_range(start_of_year, periods=12, freq="ME") + _dt.timedelta(days=1) + expected_ends_of_months = _pd.date_range( + start_of_year, periods=12, freq="ME" + ) + _dt.timedelta(days=1) if (actual_ends_of_month != expected_ends_of_months).any(): raise ValueError( diff --git a/pytrnsys_process/plotters.py b/pytrnsys_process/plotters.py index 39a56c9..4bf3fda 100644 --- a/pytrnsys_process/plotters.py +++ b/pytrnsys_process/plotters.py @@ -1,6 +1,7 @@ from abc import abstractmethod from typing import Tuple +import matplotlib.dates as _mpd import matplotlib.pyplot as _plt import pandas as _pd @@ -24,7 +25,6 @@ class ChartBase: LEGEND_FONT_SIZE = 8 TITLE_FONT_SIZE = 12 - def __init__( self, df, x_label=X_LABEL, y_label=Y_LABEL, title=TITLE, size=SIZE_A4 ): @@ -39,8 +39,6 @@ def configure(self): self.ax.set_xlabel(self.x_label, fontsize=self.LABEL_FONT_SIZE) self.ax.set_ylabel(self.y_label, fontsize=self.LABEL_FONT_SIZE) self.ax.set_title(self.title, fontsize=self.TITLE_FONT_SIZE) - #TODO This line does not seem to work as excpected, figure out why # pylint: disable=fixme - # self.ax.xaxis.set_major_formatter(_mpd.DateFormatter(self.DATE_FORMAT)) _plt.tight_layout() @abstractmethod @@ -68,7 +66,20 @@ def plot(self, columns: list[str]) -> Tuple[_plt.Figure | None, _plt.Axes]: self.configure() return self.fig, self.ax - #TODO Idea for what an energy balance plot method could look like # pylint: disable=fixme + def plotWithoutPandas( + self, columns: list[str] + ) -> Tuple[_plt.Figure | None, _plt.Axes]: + """The matplot date formatter does not work when using df.plot func. + This is an example to plot a stacked bar chart without df.plot""" + bottom = None + for col in columns: + self.ax.bar(self.df.index, self.df[col], label=col, bottom=bottom) + bottom = self.df[col] if bottom is None else bottom + self.df[col] + self.ax.xaxis.set_major_formatter(_mpd.DateFormatter(self.DATE_FORMAT)) + self.configure() + return self.fig, self.ax + + # TODO Idea for what an energy balance plot method could look like # pylint: disable=fixme @staticmethod def create_energy_balance_monthly( df: _pd.DataFrame, @@ -83,7 +94,9 @@ class HourlyCurvePlot(ChartBase): PLOT_KIND = "line" - def plot(self, columns: list[str], use_legend: bool = True) -> Tuple[_plt.Figure | None, _plt.Axes]: + def plot( + self, columns: list[str], use_legend: bool = True + ) -> Tuple[_plt.Figure | None, _plt.Axes]: self.df[columns].plot( kind=self.PLOT_KIND, colormap=self.COLOR_MAP, diff --git a/pytrnsys_process/readers.py b/pytrnsys_process/readers.py index ebe286d..438798e 100644 --- a/pytrnsys_process/readers.py +++ b/pytrnsys_process/readers.py @@ -20,7 +20,7 @@ def read_hourly( header=Reader.HEADER, delimiter=Reader.DELIMITER, ) - hours = _dt.timedelta(hours=1) * df["TIME"] # type: ignore + hours = _dt.timedelta(hours=1) * df["TIME"] # type: ignore start_of_year = _dt.datetime(day=1, month=1, year=starting_year) actual_ends_of_month = start_of_year + hours df = df.drop(columns=["Period", "TIME"]) @@ -54,10 +54,10 @@ def read_monthly( f"The time stamps of the supposedly monthly file '{monthly_file}' don't fall on the end of each month." ) df = df.drop(columns=["Month", "TIME"]) - actual_ends_of_month = [ + formatted_ends_of_month = [ timestamp.strftime("%m-%y") for timestamp in actual_ends_of_month ] - df["Timestamp"] = actual_ends_of_month + df["Timestamp"] = formatted_ends_of_month df = df.set_index("Timestamp") return df diff --git a/tests/pytrnsys_process/constants.py b/tests/pytrnsys_process/constants.py index bb411ba..01c8d7a 100644 --- a/tests/pytrnsys_process/constants.py +++ b/tests/pytrnsys_process/constants.py @@ -4,5 +4,3 @@ REPO_ROOT = _pl.Path(pp.__file__).parents[1] DATA_FOLDER = REPO_ROOT / "tests" / "pytrnsys_process" / "data" - - diff --git a/tests/pytrnsys_process/test_headers.py b/tests/pytrnsys_process/test_headers.py index 9c68375..1059efd 100644 --- a/tests/pytrnsys_process/test_headers.py +++ b/tests/pytrnsys_process/test_headers.py @@ -19,6 +19,8 @@ def test_init_headers(self): assert headers.header_index.get("QSnk417PauxEvap_kW") == [ ("sim-2", "ENERGY_BALANCE_MO_HP_417.Prt") ] + + class TestBenchmarkHeaders: """Initial test have shown multi threading is slower than single thread for this problem. Rerunning these tests requires a more comprehensive data set""" @@ -28,10 +30,11 @@ class TestBenchmarkHeaders: def test_init_headers_benchmark(self, benchmark): def init_headers(): Headers(self.PATH_TO_RESULTS).init_headers() + benchmark(init_headers) def test_init_headers_benchmark_multi_thread(self, benchmark): def init_headers(): Headers(self.PATH_TO_RESULTS).init_headers_multi_thread() - benchmark(init_headers) + benchmark(init_headers) diff --git a/tests/pytrnsys_process/test_monthly.py b/tests/pytrnsys_process/test_monthly.py index 401fbd4..51fdec2 100644 --- a/tests/pytrnsys_process/test_monthly.py +++ b/tests/pytrnsys_process/test_monthly.py @@ -9,11 +9,15 @@ def test(): monthly_file_path = DATA_DIR_PATH / "BUILDING_MO.Prt" - actual_df = _monthly.read_monthly_file(monthly_file_path, starting_year=1990) + actual_df = _monthly.read_monthly_file( + monthly_file_path, starting_year=1990 + ) actual_file_path = DATA_DIR_PATH / "actual.csv" actual_df.to_csv(actual_file_path, encoding="UTF8") expected_file_path = DATA_DIR_PATH / "expected.csv" - assert actual_file_path.read_text(encoding="UTF8") == expected_file_path.read_text(encoding="UTF8") + assert actual_file_path.read_text( + encoding="UTF8" + ) == expected_file_path.read_text(encoding="UTF8") diff --git a/tests/pytrnsys_process/test_plotters.py b/tests/pytrnsys_process/test_plotters.py index bfb1944..6bc6e30 100644 --- a/tests/pytrnsys_process/test_plotters.py +++ b/tests/pytrnsys_process/test_plotters.py @@ -58,19 +58,15 @@ def test_create_stacked_bar_chart_for_monthly(self): def test_create_curve_plot_for_hourly(self): result_data = const.DATA_FOLDER / "hourly/Src_hr.Prt" - expected_fig = ( - const.DATA_FOLDER / "plots/curve-plot/expected.png" - ) - actual_fig = (const.DATA_FOLDER / "plots/curve-plot/actual.png") + expected_fig = const.DATA_FOLDER / "plots/curve-plot/expected.png" + actual_fig = const.DATA_FOLDER / "plots/curve-plot/actual.png" df = Reader.read_hourly(result_data) columns = ["QSrc1TIn", "QSrc1TOut"] - curve_plot = HourlyCurvePlot(df) fig, ax = curve_plot.plot(columns) fig.savefig(actual_fig) - assert ( _mpltc.compare_images( str(actual_fig), str(expected_fig), tol=0.001 @@ -78,7 +74,6 @@ def test_create_curve_plot_for_hourly(self): is None ) - # def test_create_energy_balance_monthly(self): # manual = False # diff --git a/tests/pytrnsys_process/test_reader.py b/tests/pytrnsys_process/test_reader.py index 47acb65..3291e67 100644 --- a/tests/pytrnsys_process/test_reader.py +++ b/tests/pytrnsys_process/test_reader.py @@ -8,7 +8,6 @@ class TestReader: HOURLY_DIR_PATH = _pl.Path(__file__).parent / "data/hourly" MONTHLY_DIR_PATH = _pl.Path(__file__).parent / "data/monthly" - def test_read_hourly(self): hourly_file_path = self.HOURLY_DIR_PATH / "Src_Hr.Prt" @@ -19,18 +18,22 @@ def test_read_hourly(self): expected_file_path = self.HOURLY_DIR_PATH / "expected.csv" - assert actual_file_path.read_text(encoding="UTF8") == expected_file_path.read_text(encoding="UTF8") + assert actual_file_path.read_text( + encoding="UTF8" + ) == expected_file_path.read_text(encoding="UTF8") def test_read_monthly(self): hourly_file_path = self.MONTHLY_DIR_PATH / "PCM_MO.Prt" - actual_df = Reader.read_monthly(hourly_file_path, starting_month=11, periods=14) + actual_df = Reader.read_monthly( + hourly_file_path, starting_month=11, periods=14 + ) actual_file_path = self.MONTHLY_DIR_PATH / "actual.csv" actual_df.to_csv(actual_file_path, encoding="UTF8") expected_file_path = self.MONTHLY_DIR_PATH / "expected.csv" - assert actual_file_path.read_text(encoding="UTF8") == expected_file_path.read_text(encoding="UTF8") - - + assert actual_file_path.read_text( + encoding="UTF8" + ) == expected_file_path.read_text(encoding="UTF8")