added option for saving output as .csv file in case user prefers it o…

…ver excel, updated CI process

Updated CI process to optimize action usage time, thank you to @dostuffthatmatters for the suggestion

.github/workflows/ci_run_qaqc_single_station.yml → .github/workflows/ci.yml

@@ -1,4 +1,4 @@
-name: ci_run_qaqc_single_station
+name: ci_tests
 
 on:
   push:
@@ -9,15 +9,14 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [windows-latest]
         python-version:
           - "3.9"
           - "3.10"
           - "3.11"
           - "3.12"
 
     name: run_tests
-    runs-on: ${{ matrix.os }}
+    runs-on: ubuntu-latest
 
     steps:
       - name: checkout_code
@@ -27,11 +26,14 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python_version }}
+          cache: 'pip'
+          cache-dependency-path: 'pyproject.toml'
 
       - name: install_dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install -r requirements.txt
+        run: pip install -r pyproject.toml
+
+      - name: run_pytest
+        run: python -m pytest
 
       - name: run_qaqc_single_station
         run: echo 0 | python qaqc_single_station.py

agweatherqaqc/agweatherqaqc.py

@@ -744,6 +744,7 @@ def _create_plots(self):
             fig = gridplot(grid_of_plots, toolbar_location='left', sizing_mode='stretch_width')
             save(fig)
 
+
     def _write_outputs(self):
         """
             Creates all the output files
@@ -805,7 +806,7 @@ def _write_outputs(self):
         #     Delta : Magnitude of difference between original data and corrected data
         #     Filled Data : Tracks which data points have been filled by script generated values instead of provided
         # Data that is provided and subsequently corrected by the script do not count as filled values.
-        print("\nSystem: Saving corrected data to .xslx file.")
+        print("\nSystem: Saving corrected data to output file.")
 
         # Create any individually-requested output data
         ws_2m = _wind_height_adjust(uz=self.data_ws, zw=self.ws_anemometer_height)
@@ -865,14 +866,27 @@ def _write_outputs(self):
         output_df.index.name = 'date'
         delta_df.index.name = 'date'
         fill_df.index.name = 'date'
-        # Open up pandas excel writer
-        output_writer = pd.ExcelWriter(self.output_file_path, engine='xlsxwriter')
-        # Convert data frames to xlsxwriter excel objects
-        output_df.to_excel(output_writer, sheet_name='Corrected Data', na_rep=self.missing_fill_value)
-        delta_df.to_excel(output_writer, sheet_name='Delta (Corr - Orig)', na_rep=self.missing_fill_value)
-        fill_df.to_excel(output_writer, sheet_name='Filled Data', na_rep=self.missing_fill_value)
-        # Save output file
-        output_writer.close()
+
+        # Save outputs, either as 1 xlsx or 3 csvs depending on config file choice
+        if str(self.config_dict['output_file_format']).lower() == 'xlsx':
+            # Open up pandas excel writer
+            output_writer = pd.ExcelWriter(self.output_file_path, engine='xlsxwriter')
+            # Convert data frames to xlsxwriter excel objects
+            output_df.to_excel(output_writer, sheet_name='Corrected Data', na_rep=self.missing_fill_value)
+            delta_df.to_excel(output_writer, sheet_name='Delta (Corr - Orig)', na_rep=self.missing_fill_value)
+            fill_df.to_excel(output_writer, sheet_name='Filled Data', na_rep=self.missing_fill_value)
+            # Save output file
+            output_writer.close()
+        elif str(self.config_dict['output_file_format']).lower() == 'csv':
+            # cut off .xlsx from output file name
+            self.output_file_path = self.output_file_path[:-5]
+            output_df.to_csv(f'{self.output_file_path}.csv', na_rep=self.missing_fill_value)
+            delta_df.to_csv(f'{self.output_file_path}_deltas.csv', na_rep=self.missing_fill_value)
+            fill_df.to_csv(f'{self.output_file_path}_filled_data.csv', na_rep=self.missing_fill_value)
+        else:
+            raise ValueError(f"Config file found at {self.config_path} contained an invalid setting"
+                             f" for \'OUTPUT_DATA_FORMAT\', must be one of [\'CSV\',\'XLSX\'], currently set to:"
+                             f" \'{self.config_dict['output_file_format']}\'")
 
         logger = open(self.log_file, 'a')
         if self.fill_mode == 1:

agweatherqaqc/input_functions.py

@@ -42,6 +42,7 @@ def _read_config(config_file_path):
     config_dict['lines_of_footer'] = config_reader['METADATA'].getint('LINES_OF_FOOTER')  # Lines of header to skip
 
     # OPTIONS Section
+    config_dict['output_file_format'] = config_reader['OPTIONS']['OUTPUT_DATA_FORMAT']  # either .csv or .xlsx
     config_dict['auto_flag'] = config_reader['OPTIONS'].getboolean('AUTOMATIC_OPTION')  # auto first iteration of QAQC
     config_dict['fill_flag'] = config_reader['OPTIONS'].getboolean('FILL_OPTION')  # Option to fill in missing data
 

tests/test_files/test_config.ini

@@ -33,7 +33,6 @@ MISSING_OUTPUT_VALUE = nan
 LINES_OF_HEADER = 1
 LINES_OF_FOOTER = 0
 
-
 ############################################################################################################################
 ############################################################################################################################
 [OPTIONS]
@@ -54,6 +53,10 @@ AUTOMATIC_OPTION = 0
 FILL_OPTION = 0
 
 
+# OUTPUT FILE FORMAT FOR CORRECTED DATA - MUST BE EITHER 'CSV' OR 'XLSX'
+OUTPUT_DATA_FORMAT = XLSX
+
+
 ############################################################################################################################
 ############################################################################################################################
 [DATA]