diff --git a/action.yml b/action.yml index b88f4edc..2ea99517 100644 --- a/action.yml +++ b/action.yml @@ -119,10 +119,10 @@ runs: --config_file ${{ inputs.snakemake_config }} \ --main_command "${{ inputs.main_command }}" \ --pre_command "${{ inputs.pre_command }}" - + shell: bash - + - name: Upload artifacts (logs) if: ${{ inputs.step == 'run-self-hosted-validation' }} uses: actions/upload-artifact@v4 @@ -300,7 +300,6 @@ runs: mkdir -p _validation-images/main # Copy plots - echo "Plots: ${plots_array[@]}" for plotpath in "${plots_array[@]}" do subpath="${plotpath%/*}" @@ -315,6 +314,23 @@ runs: cp "$HOME/artifacts/results/feature/results/${PREFIX_FEATURE}/${subpath}/${plot}" "_validation-images/feature/${subpath}/" || true # ignore if run failed done + # Get benchmark plot list (from benchmark script) + read -a plots_array_benchmark <<< "$(python scripts/plot_benchmarks.py)" + + mkdir -p _validation-images/benchmarks + + # Copy benchmark plots + for plot in "${plots_array_benchmark[@]}" + do + echo "Copying benchmark plot: ${plot} + + # Create directories + mkdir -p "_validation-images/benchmarks + + cp "${plot}" "_validation-images/benchmarks" || true # ignore if run failed + cp "${plot}" "_validation-images/benchmarks" || true # ignore if run failed + done + # Add plots to repo branch echo "Adding plots to repo branch" git add _validation-images diff --git a/requirements.txt b/requirements.txt index 5ac7c153..e6bb3f11 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,5 @@ numpy pandas -openpyxl \ No newline at end of file +openpyxl +matplotlib +seaborn \ No newline at end of file diff --git a/scripts/draft_comment.py b/scripts/draft_comment.py index fd8ac2ad..25dc5763 100644 --- a/scripts/draft_comment.py +++ b/scripts/draft_comment.py @@ -9,12 +9,12 @@ import re from dataclasses import dataclass from pathlib import Path -from typing import Any import numpy as np import pandas as pd from metrics import min_max_normalized_mae, normalized_root_mean_square_error from numpy.typing import ArrayLike +from utils import get_env_var def create_numeric_mask(arr: ArrayLike) -> np.ndarray: @@ -35,17 +35,6 @@ def create_numeric_mask(arr: ArrayLike) -> np.ndarray: return np.vectorize(lambda x: isinstance(x, (int, float)) and np.isfinite(x))(arr) -def get_env_var(var_name: str, default: Any = None) -> Any: - """Get environment variable or raise an error if not set and no default provided.""" - value = os.getenv(var_name, default) - if value == "" and default is None: - msg = f"The environment variable '{var_name}' is not set." - raise OSError(msg) - if str(value).lower() in ["true", "false"]: - return str(value).lower() == "true" - return value - - @dataclass class CommentData: """Class to store data for comment generation.""" @@ -69,6 +58,13 @@ class CommentData: _sucessfull_run = None + def __init__(self): + """Initialize comment data class.""" + self.plots_base_url = ( + f"https://raw.githubusercontent.com/lkstrp/" + f"pypsa-validator/{self.plots_hash}/_validation-images/" + ) + def errors(self, branch_type: str) -> list: """Return errors for branch type.""" if branch_type not in ["main", "feature"]: @@ -115,6 +111,7 @@ def sucessfull_run(self) -> bool: def get_deviation_df(df1: pd.DataFrame, df2: pd.DataFrame) -> pd.DataFrame: + """Calculate deviation dataframe between two dataframes.""" nrmse_series = df1.apply( lambda row: normalized_root_mean_square_error( row.values, @@ -138,11 +135,27 @@ def get_deviation_df(df1: pd.DataFrame, df2: pd.DataFrame) -> pd.DataFrame: return deviation_df +def create_details_block(summary: str, content: str) -> str: + """Wrap content in a details block (if content is not empty).""" + if content: + return ( + f"
\n" + f" {summary}\n" + f"{content}" + f"
\n" + f"\n" + f"\n" + ) + else: + return "" + + class RunSuccessfull(CommentData): """Class to generate successfull run component.""" def __init__(self): - """Initialize class.""" + """Initialize successfull run component.""" + super().__init__() self.dir_main = [ file for file in (self.dir_artifacts / "results/main/results").iterdir() @@ -173,8 +186,6 @@ def __init__(self): self._variables_deviation_df = None - self.plots_base_url = f"https://raw.githubusercontent.com/lkstrp/pypsa-validator/{self.plots_hash}/_validation-images/" - # Status strings for file comparison table STATUS_FILE_MISSING = " :warning: Missing" STATUS_EQUAL = ":white_check_mark: Equal" @@ -191,6 +202,7 @@ def __init__(self): @property def variables_deviation_df(self): + """Get the deviation dataframe for variables.""" if self._variables_deviation_df is not None: return self._variables_deviation_df vars1 = pd.read_excel(self.dir_main / self.VARIABLES_FILE) @@ -216,6 +228,7 @@ def variables_deviation_df(self): @property def variables_plot_strings(self): + """Return list of variable plot strings.""" plots = ( self.variables_deviation_df.index.to_series() .apply(lambda x: re.sub(r"[ |/]", "-", x)) @@ -226,6 +239,7 @@ def variables_plot_strings(self): @property def variables_comparison(self) -> str: + """Return variables comparison table.""" if ( not (self.dir_main / self.VARIABLES_FILE).exists() or not (self.dir_feature / self.VARIABLES_FILE).exists() @@ -247,6 +261,7 @@ def variables_comparison(self) -> str: @property def changed_variables_plots(self) -> str: + """Return plots for variables that have changed significantly.""" if ( not (self.dir_main / self.VARIABLES_FILE).exists() or not (self.dir_feature / self.VARIABLES_FILE).exists() @@ -279,8 +294,8 @@ def plots_table(self) -> str: url_b = self.plots_base_url + "feature/" + plot rows.append( [ - f'Image not found in results', - f'Image not found in results', + f'Image not available', + f'Image not available', ] ) @@ -460,20 +475,6 @@ def files_table(self) -> str: @property def body(self) -> str: """Body text for successfull run.""" - - def create_details_block(summary: str, content: str) -> str: - if content: - return ( - f"
\n" - f" {summary}\n" - f"{content}" - f"
\n" - f"\n" - f"\n" - ) - else: - return "" - if self.variables_comparison and self.changed_variables_plots: if self.variables_deviation_df.empty: variables_txt = ( @@ -489,7 +490,8 @@ def create_details_block(summary: str, content: str) -> str: ) elif self.variables_comparison or self.changed_variables_plots: raise ValueError( - "Both variables_comparison and changed_variables_plots must be set or unset." + "Both variables_comparison and changed_variables_plots must be set or " + "unset." ) else: variables_txt = "" @@ -508,6 +510,10 @@ def __call__(self) -> str: class RunFailed(CommentData): """Class to generate failed run component.""" + def __init__(self): + """Initialize failed run component.""" + super().__init__() + def body(self) -> str: """Body text for failed run.""" main_errors = self.errors("main") @@ -539,9 +545,45 @@ def __call__(self) -> str: return self.body() +class ModelMetrics(CommentData): + """Class to generate model metrics component.""" + + def __init__(self): + """Initialize model metrics component.""" + super().__init__() + + @property + def benchmark_plots(self) -> str: + """Benchmark plots.""" + "execution_time.png", "memory_peak.png", "memory_scatter.png" + return ( + f'\n' + f'\n' + f'\n' + ) + + def body(self) -> str: + """Body text for Model Metrics.""" + return ( + f"**Model Metrics**\n" + f"{create_details_block('Benchmarks', self.benchmark_plots)}\n" + ) + + def __call__(self) -> str: + """Return text for model metrics component.""" + return self.body() + + class Comment(CommentData): """Class to generate pypsa validator comment for GitHub PRs.""" + def __init__(self) -> None: + """Initialize comment class. It will put all text components together.""" + super().__init__() + @property def header(self) -> str: """ @@ -603,7 +645,15 @@ def subtext(self) -> str: f"Last updated on `{time}`." ) - def needed_plots(self): + def dynamic_plots(self) -> str: + """ + Return a list of dynamic results plots needed for the comment. + + Returns + ------- + str: Space separated list of dynamic plots. + + """ if self.sucessfull_run: body_sucessfull = RunSuccessfull() plots_string = " ".join(body_sucessfull.variables_plot_strings) @@ -613,6 +663,7 @@ def needed_plots(self): def __repr__(self) -> str: """Return full formatted comment.""" + body_benchmarks = ModelMetrics() if self.sucessfull_run: body_sucessfull = RunSuccessfull() @@ -620,6 +671,7 @@ def __repr__(self) -> str: f"{self.header}" f"{self.config_diff if self.git_diff_config else ''}" f"{body_sucessfull()}" + f"{body_benchmarks()}" f"{self.subtext}" ) @@ -630,11 +682,19 @@ def __repr__(self) -> str: f"{self.header}" f"{body_failed()}" f"{self.config_diff if self.git_diff_config else ''}" + f"{body_benchmarks()}" f"{self.subtext}" ) def main(): + """ + Run draft comment script. + + Command line interface for the draft comment script. Use no arguments to print the + comment, or use the "plots" argument to print the dynamic plots which will be needed + for the comment. + """ parser = argparse.ArgumentParser(description="Process some comments.") parser.add_argument( "command", nargs="?", default="", help='Command to run, e.g., "plots".' @@ -644,7 +704,7 @@ def main(): comment = Comment() if args.command == "plots": - print(comment.needed_plots()) + print(comment.dynamic_plots()) else: print(comment) # noqa T201 diff --git a/scripts/metrics.py b/scripts/metrics.py index faacd26f..768d1898 100644 --- a/scripts/metrics.py +++ b/scripts/metrics.py @@ -1,6 +1,4 @@ -""" -Helper module for calculating evaluation metrics. -""" +"""Helper module for calculating evaluation metrics.""" import numpy as np from numpy.typing import ArrayLike @@ -64,6 +62,10 @@ def mean_absolute_percentage_error( Predicted values epsilon : float, optional (default=1e-9) Small value to avoid division by zero + aggregate : bool, optional (default=True) + If True, return the mean MAPE. Otherwise, return an array of individual MAPEs + ignore_inf : bool, optional (default=True) + If True, ignore infinite values in the calculation Returns ------- @@ -111,6 +113,8 @@ def normalized_root_mean_square_error( If True, ignore infinite values in the calculation normalization : str, optional (default='min-max') Method of normalization. Options: 'mean', 'range', 'iqr', 'min-max' + fill_na : float, optional (default=0) + Value to replace NaN values epsilon : float, optional (default=1e-9) Small value to add to normalization factor to avoid division by zero diff --git a/scripts/plot_benchmarks.py b/scripts/plot_benchmarks.py new file mode 100644 index 00000000..8a965237 --- /dev/null +++ b/scripts/plot_benchmarks.py @@ -0,0 +1,272 @@ +"""Read benchmark data and generate plots comparing execution time and memory peak.""" + +import os +from pathlib import Path + +import matplotlib.pyplot as plt +import numpy as np +import pandas as pd +import seaborn as sns +from utils import get_env_var + +DIR_ARTIFACTS: Path = Path( + get_env_var("DIR_ARTIFACTS", Path(get_env_var("HOME")) / "artifacts") +) + + +def read_benchmark_dir(directory: Path) -> pd.DataFrame: + """ + Read benchmark data from a directory. + + Parameters + ---------- + directory : Path + Directory containing benchmark data for a single run + + Returns + ------- + pd.DataFrame + DataFrame containing benchmark data for a single run + + """ + data = [] + for root, _, files in os.walk(directory): + for file in files: + if file == ".DS_Store": + continue + filepath = os.path.join(root, file) + df = pd.read_csv(filepath, sep="\t") + df["name"] = "/".join(os.path.relpath(filepath, directory).split("/")[2:]) + data.append(df) + return pd.concat(data, ignore_index=True) + + +def read_benchmarks(directory: Path) -> pd.DataFrame: + """ + Read benchmark data from the main and feature branches. + + Parameters + ---------- + directory : Path + Directory containing benchmark data + + Returns + ------- + pd.DataFrame + Combined DataFrame containing benchmark data from the main and feature branches + + """ + dir_main = DIR_ARTIFACTS / "benchmarks/main/benchmarks/" + dir_feat = DIR_ARTIFACTS / "benchmarks/feature/benchmarks/" + + df_main = read_benchmark_dir(dir_main) + df_feat = read_benchmark_dir(dir_feat) + + # Add a column to identify the run + df_main["run"] = "main" + df_feat["run"] = "feature" + + # Combine the dataframes + df = pd.concat([df_main, df_feat], ignore_index=True) + + return df + + +def create_bar_chart_comparison( + df: pd.DataFrame, + x_column: str, + title: str, + xlabel: str, + filename: str, + ignore_stacked_plot: bool = False, +): + """ + Create a horizontal bar chart comparing execution time or memory peak. + + Parameters + ---------- + df : pd.DataFrame + DataFrame containing benchmark data + x_column : str + Column to use for comparison + title : str + Title of the plot + xlabel : str + Label for the x-axis + filename : str + Output filename + ignore_stacked_plot : bool, optional (default=False) + If True, do not include the stacked bar plot subplot + + Returns + ------- + None + + """ + if ignore_stacked_plot: + fig, ax1 = plt.subplots(figsize=(8, max(6, len(df["name"].unique()) * 0.4))) + else: + fig, (ax1, ax2) = plt.subplots( + 1, + 2, + figsize=(10, max(8, len(df["name"].unique()) * 0.4)), + gridspec_kw={"width_ratios": [3, 1]}, + ) + + unique_jobs = df["name"].unique() + color_palette = plt.get_cmap("tab20")(np.linspace(0, 1, len(df))) + job_colors = dict(zip(unique_jobs, color_palette)) + + # Horizontal bar plot + job_positions = {job: i for i, job in enumerate(unique_jobs)} + bar_width = 0.35 + + for i, run in enumerate(df["run"].unique()): + df_run = df[df["run"] == run] + positions = [ + job_positions[job] + (i - 0.5) * bar_width for job in df_run["name"] + ] + ax1.barh( + positions, + df_run[x_column], + bar_width, + color=[job_colors[job] for job in df_run["name"]], + edgecolor="black", + linewidth=0.8, + label=run, + ) + + ax1.set_yticks([job_positions[job] for job in unique_jobs]) + ax1.set_yticklabels(unique_jobs) + ax1.tick_params( + axis="y", which="major", labelsize=7 + ) # Adjust the size (8) as needed + + ax1.set_title(f"{title} - Detailed Comparison") + ax1.set_xlabel(xlabel) + ax1.set_ylabel("Benchmark") + if not ignore_stacked_plot: + # Two single vertical bars + totals = df.groupby("run")[x_column].sum() + bar_width = 0.8 + index = np.arange(len(totals)) + + for i, run in enumerate(totals.index): + bottom = 0 + for job in unique_jobs: + value = df[(df["run"] == run) & (df["name"] == job)][x_column].values + if len(value) > 0: + ax2.bar( + i, + value, + bar_width, + bottom=bottom, + color=job_colors[job], + edgecolor="black", + linewidth=0.8, + ) + bottom += value[0] + + ax2.set_title(f"{title} - Total") + ax2.set_ylabel(xlabel) + ax2.set_xlabel("Run") + ax2.set_xticks(index) + ax2.set_xticklabels(totals.index) + + # Remove all spines + for spine in ax2.spines.values(): + spine.set_visible(False) + + # Add total value labels on top of the stacked bars + for i, v in enumerate(totals.values): + ax2.text(i, v, f"{v:.2f}", ha="center", va="bottom") + + plt.tight_layout() + plt.savefig(filename, bbox_inches="tight") + plt.close() + + return filename + + +def create_scatter_memory(df: pd.DataFrame, filename: str) -> None: + """ + Create a scatter plot of max_rss vs max_uss. + + Parameters + ---------- + df : pd.DataFrame + DataFrame containing benchmark data + filename : str + Output filename + + Returns + ------- + None + + """ + plt.figure(figsize=(10, 6)) + sns.scatterplot(x="max_rss", y="max_uss", hue="run", style="run", data=df) + plt.title("Memory Usage: max_rss vs max_uss") + plt.xlabel("Maximum Resident Set Size (MB)") + plt.ylabel("Maximum Unique Set Size (MB)") + + # Add explanatory note + note = ( + "RSS (Resident Set Size): Total memory allocated to the process, including " + "shared libraries.\n" + "USS (Unique Set Size): Memory unique to the process, excluding shared " + "libraries." + ) + plt.text( + 0.05, + -0.15, + note, + transform=plt.gca().transAxes, + fontsize=8, + verticalalignment="top", + bbox=dict(boxstyle="round", facecolor="white", alpha=0.8), + ) + + plt.tight_layout() + plt.savefig( + filename, bbox_inches="tight" + ) # Added bbox_inches='tight' to include the text box + plt.close() + + return filename + + +def main(): + """Read benchmark data and generate plots.""" + # Read data from main and feature run + df = read_benchmarks(DIR_ARTIFACTS / "benchmarks") + + plots = [] + # 1. Execution time comparison plots + file_name = create_bar_chart_comparison( + df, "s", "Execution Time", "Time (seconds)", "execution_time.png" + ) + plots.append(file_name) + + # 2. Memory peak comparison plots + file_name = create_bar_chart_comparison( + df, + "max_rss", + "Memory Peak", + "Max RSS (MB)", + "memory_peak.png", + ignore_stacked_plot=True, + ) + plots.append(file_name) + + # 3. Scatter plot of max_rss vs max_uss + file_name = create_scatter_memory(df, "memory_scatter.png") + plots.append(file_name) + + plots_string = " ".join(plots) + + print(plots_string) + + +if __name__ == "__main__": + main() diff --git a/scripts/utils.py b/scripts/utils.py new file mode 100644 index 00000000..a1b16818 --- /dev/null +++ b/scripts/utils.py @@ -0,0 +1,15 @@ +"""Utility functions for the pypsa validator python scripts.""" + +import os +from typing import Any + + +def get_env_var(var_name: str, default: Any = None) -> Any: + """Get environment variable or raise an error if not set and no default provided.""" + value = os.getenv(var_name, default) + if value == "" and default is None: + msg = f"The environment variable '{var_name}' is not set." + raise OSError(msg) + if str(value).lower() in ["true", "false"]: + return str(value).lower() == "true" + return value