CorrDiff: CWB Dataloader cleanups (#593)

* uncomment tests

* assert that v1==v2 before cleaning up

* remove v1 and v2 option. these were identical

* add regression test

* remove unused normalization logic

* remove unused options, img_utils.reshape_fields

* remove unused options, img_utils.reshape_fields

* remove FilterTime

* delete unused reference

* Revert "delete unused reference"

This reverts commit c27c8f7.

* test ZarrDataset.{denormalize/normalize}_{input/output}

Also normalize input was broken, so I needed to fix it.

examples/generative/corrdiff/conf/config_generate.yaml

@@ -82,19 +82,16 @@ dataset:
   train: False
   all_times: True
   data_path: /data/data2/2023-01-24-cwb-4years.zarr
-  n_history: 0
   in_channels: [0, 1, 2, 3, 4, 9, 10, 11, 12, 17, 18, 19]
   out_channels: [0, 17, 18, 19]
   img_shape_x: 448
   img_shape_y: 448
-  roll: false
   add_grid: true
   ds_factor: 4
   min_path: null
   max_path: null
   global_means_path: null
   global_stds_path: null
-  normalization: "v1"
 
 ## Deterministic sampler options
 sigma_min: null

examples/generative/corrdiff/conf/config_generate_patch.yaml

@@ -82,19 +82,16 @@ dataset:
   train: False
   all_times: True
   data_path: /data/data2/2023-01-24-cwb-4years.zarr
-  n_history: 0
   in_channels: [0, 1, 2, 3, 4, 9, 10, 11, 12, 17, 18, 19]
   out_channels: [0, 17, 18, 19]
   img_shape_x: 448
   img_shape_y: 448
-  roll: false
   add_grid: true
   ds_factor: 4
   min_path: null
   max_path: null
   global_means_path: null
   global_stds_path: null
-  normalization: "v1"
 
 
 ## Deterministic sampler options

examples/generative/corrdiff/conf/config_train_base.yaml

@@ -87,19 +87,16 @@ dataset:
   type: cwb
   train_test_split: false
   data_path: /data/data2/2023-01-24-cwb-4years.zarr
-  n_history: 0
   in_channels: [0, 1, 2, 3, 4, 9, 10, 11, 12, 17, 18, 19]
   out_channels: [0, 17, 18, 19]
   img_shape_x: 448
   img_shape_y: 448
-  roll: false
   add_grid: true
   ds_factor: 4
   min_path: null
   max_path: null
   global_means_path: null
   global_stds_path: null
-  normalization: "v1"
 
 # Validation dataset options
 # (need to set dataset.train_test_split == true to have an effect)

examples/generative/corrdiff/conf/references/config_data_ref.yaml

@@ -18,10 +18,8 @@ full_field: &FULL_FIELD
 
   batch_size: 1
   dt: 1
-  n_history: 0
   img_shape_x: 448
   img_shape_y: 448
-  normalization: v1     #minmax
   # era5_data_dir: '/lustre/fsw/sw_climate_fno/cwb-align'  #old
   # cwb_data_dir: '/lustre/fsw/sw_climate_fno/cwb-rwrf-pad-2212/all_ranges'  #old
   # relative to CWB_ROOT environment variable
@@ -40,7 +38,6 @@ full_field_train_crop448_grid_12inchans_fcn_4outchans_4x_6layer: #config for sin
   <<: *FULL_FIELD
   in_channels: [0,1,2,3,4,9,10,11,12,17,18,19] 
   out_channels: [0,17,18,19]   #[17,18,19]
-  roll: !!bool False
   patch_size: 448
   crop_size_x: 448
   crop_size_y: 448
@@ -53,7 +50,6 @@ full_field_val_crop448_grid_12inchans_fcn_4outchans_4x_6layer: #config for singl
   <<: *FULL_FIELD
   in_channels: [0,1,2,3,4,9,10,11,12,17,18,19]
   out_channels: [0,17,18,19]  #[0,9,10,11] 
-  roll: !!bool False
   patch_size: 448
   crop_size_x: 448
   crop_size_y: 448
@@ -65,7 +61,6 @@ out_of_sample:
   <<: *FULL_FIELD
   in_channels: [0,1,2,3,4,9,10,11,12,17,18,19]
   out_channels: [0,17,18,19]  #[0,9,10,11] 
-  roll: !!bool False
   patch_size: 448
   crop_size_x: 448
   crop_size_y: 448
@@ -81,14 +76,12 @@ full_field_train_crop448_grid_12inchans_fcn_4outchans_4x_normv2: #config for sin
   <<: *FULL_FIELD
   in_channels: [0,1,2,3,4,9,10,11,12,17,18,19] 
   out_channels: [0,17,18,19]   #[17,18,19]
-  roll: !!bool False
   patch_size: 448
   crop_size_x: 448
   crop_size_y: 448
   N_grid_channels: 4
   add_grid: True
   ds_factor: 4
-  normalization: v2
 
 
 
@@ -97,7 +90,6 @@ full_field_train_crop448_grid_12inchans_fcn_4outchans_4x: #config for single gpu
   <<: *FULL_FIELD
   in_channels: [0,1,2,3,4,9,10,11,12,17,18,19] 
   out_channels: [0,17,18,19]   #[17,18,19]
-  roll: !!bool False
   patch_size: 448
   crop_size_x: 448
   crop_size_y: 448
@@ -110,7 +102,6 @@ full_field_val_crop448_grid_12inchans_fcn_4outchans_4x:  &validation #config for
   <<: *FULL_FIELD
   in_channels: [0,1,2,3,4,9,10,11,12,17,18,19]
   out_channels: [0,17,18,19]  #[0,9,10,11] 
-  roll: !!bool False
   patch_size: 448
   crop_size_x: 448
   crop_size_y: 448
@@ -124,7 +115,6 @@ full_field_train_crop448_grid_20inchans_4outchans_4x: #config for single gpu tra
   <<: *FULL_FIELD
   in_channels: [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19]
   out_channels: [0,17,18,19]   #[17,18,19]
-  roll: !!bool False
   patch_size: 448
   crop_size_x: 448
   crop_size_y: 448
@@ -137,7 +127,6 @@ full_field_val_crop448_grid_20inchans_4outchans_4x: #config for single gpu train
   <<: *FULL_FIELD
   in_channels: [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19]
   out_channels: [0,17,18,19]   #[17, 18, 19]
-  roll: !!bool False
   patch_size: 448
   crop_size_x: 448
   crop_size_y: 448
@@ -151,7 +140,6 @@ full_field_train_crop112_grid_20inchans_4outchans_4x: #config for single gpu tra
   <<: *FULL_FIELD
   in_channels: [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19]
   out_channels: [0,17,18,19]   #[17,18,19]
-  roll: !!bool False
   patch_size: 112
   crop_size_x: 112
   crop_size_y: 112
@@ -164,7 +152,6 @@ full_field_val_crop112_grid_20inchans_4outchans_4x: #config for single gpu train
   <<: *FULL_FIELD
   in_channels: [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19]
   out_channels: [0,17,18,19]   #[17, 18, 19]
-  roll: !!bool False
   patch_size: 112
   crop_size_x: 448
   crop_size_y: 448
@@ -178,7 +165,6 @@ full_field_train_crop112_grid_20inchans_19outchans_4x: #config for single gpu tr
   <<: *FULL_FIELD
   in_channels: [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19]
   out_channels: [1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19]   #[17,18,19]
-  roll: !!bool False
   patch_size: 112
   crop_size_x: 112
   crop_size_y: 112
@@ -191,7 +177,6 @@ full_field_val_crop112_grid_20inchans_19outchans_4x: #config for single gpu trai
   <<: *FULL_FIELD
   in_channels: [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19]
   out_channels: [1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19]   #[17, 18, 19]
-  roll: !!bool False
   patch_size: 112
   crop_size_x: 448
   crop_size_y: 448
@@ -206,7 +191,6 @@ full_field_train_crop112_grid_20inchans_4x: #config for single gpu training to c
   <<: *FULL_FIELD
   in_channels: [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19]
   out_channels: [17,18,19]  #[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19]
-  roll: !!bool False
   patch_size: 112
   crop_size_x: 112
   crop_size_y: 112
@@ -219,7 +203,6 @@ full_field_val_crop112_grid_20inchans_4x: #config for single gpu training to cat
   <<: *FULL_FIELD
   in_channels: [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19]
   out_channels: [17, 18, 19]     #[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19]
-  roll: !!bool False
   patch_size: 112
   crop_size_x: 448
   crop_size_y: 448
@@ -233,7 +216,6 @@ full_field_train_crop64_grid_20inchans_4x: #config for single gpu training to ca
   <<: *FULL_FIELD
   in_channels: [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19]
   out_channels: [17, 18, 19]     #[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19]
-  roll: !!bool False
   patch_size: 64
   crop_size_x: 64
   crop_size_y: 64
@@ -246,7 +228,6 @@ full_field_val_crop64_grid_20inchans_4x: #config for single gpu training to catc
   <<: *FULL_FIELD
   in_channels: [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19]
   out_channels: [17, 18, 19]     #[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19]
-  roll: !!bool False
   patch_size: 64
   crop_size_x: 448
   crop_size_y: 448
@@ -259,7 +240,6 @@ full_field_train_crop64_grid_20inchans: #config for single gpu training to catch
   <<: *FULL_FIELD
   in_channels: [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19]
   out_channels: [17, 18, 19]     #[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19]
-  roll: !!bool False
   patch_size: 64
   crop_size_x: 64
   crop_size_y: 64
@@ -271,7 +251,6 @@ full_field_val_crop64_grid_20inchans: #config for single gpu training to catch b
   <<: *FULL_FIELD
   in_channels: [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19]
   out_channels: [17, 18, 19]     #[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19]
-  roll: !!bool False
   patch_size: 64
   crop_size_x: 448
   crop_size_y: 448
@@ -284,7 +263,6 @@ full_field_train_crop64_grid: #config for single gpu training to catch bugs and
   <<: *FULL_FIELD
   in_channels: [17, 18, 19]  #[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19]
   out_channels: [17, 18, 19]     #[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19]
-  roll: !!bool False
   patch_size: 64
   crop_size_x: 64
   crop_size_y: 64
@@ -296,7 +274,6 @@ full_field_val_crop64_grid: #config for single gpu training to catch bugs and ov
   <<: *FULL_FIELD
   in_channels: [17, 18, 19]      #[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19]
   out_channels: [17, 18, 19]     #[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19]
-  roll: !!bool False
   patch_size: 64
   crop_size_x: 448
   crop_size_y: 448
@@ -310,7 +287,6 @@ full_field_train_crop64: #config for single gpu training to catch bugs and overf
   <<: *FULL_FIELD
   in_channels: [17.18,19]        #[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19]
   out_channels: [17, 18, 19]     #[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19]
-  roll: !!bool False
   patch_size: 64
   crop_size_x: 64
   crop_size_y: 64
@@ -320,7 +296,6 @@ full_field_val_crop64: #config for single gpu training to catch bugs and overfit
   <<: *FULL_FIELD
   in_channels: [17, 18, 19]      #[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19]
   out_channels: [17, 18, 19]     #[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19]
-  roll: !!bool False
   patch_size: 64
   crop_size_x: 448
   crop_size_y: 448
@@ -331,7 +306,6 @@ full_field_train_crop112: #config for single gpu training to catch bugs and over
   <<: *FULL_FIELD
   in_channels: [17.18,19]        #[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19]
   out_channels: [17, 18, 19]     #[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19]
-  roll: !!bool False
   patch_size: 112
   crop_size_x: 112
   crop_size_y: 112
@@ -341,7 +315,6 @@ full_field_val_crop112: #config for single gpu training to catch bugs and overfi
   <<: *FULL_FIELD
   in_channels: [17, 18, 19]      #[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19]
   out_channels: [17, 18, 19]     #[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19]
-  roll: !!bool False
   patch_size: 112
   crop_size_x: 112
   crop_size_y: 112
@@ -352,7 +325,6 @@ full_field_train_crop224: #config for single gpu training to catch bugs and over
   <<: *FULL_FIELD
   in_channels: [17.18,19]        #[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19]
   out_channels: [17, 18, 19]     #[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19]
-  roll: !!bool False
   patch_size: 224
   crop_size_x: 224
   crop_size_y: 224
@@ -362,7 +334,6 @@ full_field_val_crop224: #config for single gpu training to catch bugs and overfi
   <<: *FULL_FIELD
   in_channels: [17, 18, 19]      #[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19]
   out_channels: [17, 18, 19]     #[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19]
-  roll: !!bool False
   patch_size: 224
   crop_size_x: 224
   crop_size_y: 224