resources_array.cfg

#Step                                   Time        Nodes   Ntasks  Ncpus_per_task  Mem_per_cpu Gres    Comments
run_01_unpack_topo_reference            2:00:00     1       1       48              1G          0       "Need ncpus=num_process_4_topo*OMP_NUM_THREADS, num_process_4_topo=number of bursts (44 bursts for Makran track up to 32N. OMP_NUM_THREADS currently using 4, going to 6 or 1 seems to slow performance. Note we can only use a single node, so max or 32 or 56 CPUs on Caltech HPC). Make sure to scale walltime with size of processing"
run_02_unpack_secondary_slc             2:00:00     1       1       2               500M        0
run_03_average_baseline                 30:00       1       1       2               500M        0
run_04_extract_burst_overlaps           30:00       1       1       1               500M        0       "Scale walltime with processing size"
run_05_overlap_geo2rdr                  30:00       1       1       1               1G          1       "Can use GPU (adding via shell script), just use 1 CPU"
run_06_overlap_resample                 3:00:00     1       1       4               1G          0
run_07_pairs_misreg                     3:00:00     1       1       1               2G          0
run_08_timeseries_misreg                5:00        1       1       1               500M        0
run_09_fullBurst_geo2rdr                30:00       1       1       1               5G          1       "Can use GPU (adding via shell script), just use 1 CPU"
run_10_fullBurst_resample               2:00:00     1       1       4               500M        0
run_11_extract_stack_valid_region       3:00:00     1       1       2               400M        0       "Scale walltime with processing size"
run_12_merge_reference_secondary_slc    30:00       1       1       1               10G         0       "Big difference in resources between different jobs. Consider splitting up"
run_13_generate_burst_igram             3:00:00     1       1       1               3G          0
run_14_merge_burst_igram                30:00       1       1       1               50G         0       "MaxRSS ~17 GB for some pairs"
run_15_filter_coherence                 2:00:00     1       1       1               80G         0       "Some jobs seem to have extremely large memory demands"
run_16_unwrap                           3:00:00     1       1       1               20G         0       "Can only use 1 CPU, memory constraints (need 16GB for 25 N to 32 N track)"
run_17_subband_and_resamp               5:00:00     1       1       8               1G          0       "Seems to be the only stage that scales well with more CPUs, from limited testing"
run_18_generateIgram_ion                3:00:00     1       1       1               20G         0
run_19_mergeBurstsIon                   30:00       1       1       2               20G         0
run_20_unwrap_ion                       3:00:00     1       1       1               20G         0
run_21_look_ion                         25:00       1       1       1               500M        0
run_22_computeIon                       5:00        1       1       1               500M        0
run_23_filtIon                          30:00       1       1       1               30G         0       "Seems to only need about 20 seconds, but one time took ages and used lots of memory"
run_24_invertIon                        1:00:00     1       1       2               2G          0