diff --git a/src/__init__.py b/src/__init__.py index 63d7734..4567f6d 100644 --- a/src/__init__.py +++ b/src/__init__.py @@ -14,7 +14,7 @@ def save_dataset_to_netcdf(logger: PbpLogger, ds: xr.Dataset, filename: str) -> wait_secs = 3 # TODO similar re-attempt logic for the CSV (or other) output - for attempt in range(1, max_attempts): + for attempt in range(1, max_attempts + 1): try: ds.to_netcdf( filename, @@ -25,9 +25,14 @@ def save_dataset_to_netcdf(logger: PbpLogger, ds: xr.Dataset, filename: str) -> "sensitivity": {"_FillValue": None}, }, ) + logger.info(f" - saved dataset to: {filename} ({attempt=}") return True except Exception as e: # pylint: disable=broad-exception-caught - error = f"Unable to save {filename}: {e} (attempt {attempt} of {max_attempts})" + error = ( + f"Unable to save {filename}:\n" + f" {e}\n" + f" (attempt {attempt} of {max_attempts})" + ) logger.error(error) print(error) if attempt < max_attempts: diff --git a/src/main_cloud.py b/src/main_cloud.py index a8fffc5..e56f8a2 100644 --- a/src/main_cloud.py +++ b/src/main_cloud.py @@ -161,7 +161,6 @@ def main(): file_helper=file_helper, output_dir=generated_dir, output_prefix=output_prefix, - gen_csv=False, global_attrs_uri=global_attrs_uri, variable_attrs_uri=variable_attrs_uri, voltage_multiplier=voltage_multiplier, diff --git a/src/process_helper.py b/src/process_helper.py index 0b9ed55..b7eb69c 100644 --- a/src/process_helper.py +++ b/src/process_helper.py @@ -37,6 +37,7 @@ def __init__( file_helper: FileHelper, output_dir: str, output_prefix: str, + gen_netcdf: bool = True, gen_csv: bool = False, global_attrs_uri: Optional[str] = None, variable_attrs_uri: Optional[str] = None, @@ -54,6 +55,8 @@ def __init__( Output directory. :param output_prefix: Output filename prefix. + :param gen_netcdf: + True to generate the netCDF file. :param gen_csv: True to also generate CSV version of the result. :param global_attrs_uri: @@ -80,6 +83,7 @@ def __init__( "Creating ProcessHelper:" + f"\n output_dir: {output_dir}" + f"\n output_prefix: {output_prefix}" + + f"\n gen_netcdf: {gen_netcdf}" + f"\n gen_csv: {gen_csv}" + f"\n global_attrs_uri: {global_attrs_uri}" + f"\n variable_attrs_uri: {variable_attrs_uri}" @@ -97,6 +101,7 @@ def __init__( self.file_helper = file_helper self.output_dir = output_dir self.output_prefix = output_prefix + self.gen_netcdf = gen_netcdf self.gen_csv = gen_csv self.metadata_helper = MetadataHelper( @@ -217,10 +222,14 @@ def process_day(self, date: str) -> Optional[ProcessDayResult]: attrs=self._get_global_attributes(year, month, day), ) + generated_filenames = [] basename = f"{self.output_dir}/{self.output_prefix}{year:04}{month:02}{day:02}" - nc_filename = f"{basename}.nc" - save_dataset_to_netcdf(self.logger, ds_result, nc_filename) - generated_filenames = [nc_filename] + + if self.gen_netcdf: + nc_filename = f"{basename}.nc" + save_dataset_to_netcdf(self.logger, ds_result, nc_filename) + generated_filenames.append(nc_filename) + if self.gen_csv: csv_filename = f"{basename}.csv" save_dataset_to_csv(self.logger, ds_result, csv_filename)