Update IO formats and add new IO namelist controls

cicecore/cicedyn/analysis/ice_history_shared.F90

@@ -49,18 +49,21 @@ module ice_history_shared
          history_dir   , & ! directory name for history file
          incond_dir        ! directory for snapshot initial conditions
 
-      character (len=char_len_long), public :: &
-         pointer_file      ! input pointer file for restarts
-
       character (len=char_len), public :: &
          version_name
 
       character (len=char_len), public :: &
-         history_format
+         history_format      , & ! history format, cdf1, cdf2, cdf5, etc
+         history_rearranger      ! history file rearranger, box or subset for pio
 
       character (len=char_len), public :: &
          hist_suffix(max_nstrm)  ! appended to 'h' in filename when not 'x'
 
+      integer (kind=int_kind), public :: &
+         history_iotasks     , & ! iotasks, root, stride defines io pes for pio
+         history_root        , & ! iotasks, root, stride defines io pes for pio
+         history_stride          ! iotasks, root, stride defines io pes for pio
+
       !---------------------------------------------------------------
       ! Instructions for adding a field: (search for 'example')
       !     Here or in ice_history_[process].F90:

cicecore/cicedyn/infrastructure/io/io_netcdf/ice_history_write.F90

@@ -61,7 +61,6 @@ subroutine ice_write_hist (ns)
           lont_bounds, latt_bounds, lonu_bounds, latu_bounds, &
           lonn_bounds, latn_bounds, lone_bounds, late_bounds
       use ice_history_shared
-      use ice_restart_shared, only: lcdf64
 #ifdef CESMCOUPLED
       use ice_restart_shared, only: runid
 #endif
@@ -148,8 +147,18 @@ subroutine ice_write_hist (ns)
          endif
 
          ! create file
-         iflag = nf90_clobber
-         if (lcdf64) iflag = ior(iflag,nf90_64bit_offset)
+         if (history_format == 'cdf1') then
+           iflag = nf90_clobber
+         elseif (history_format == 'cdf2') then
+           iflag = ior(nf90_clobber,nf90_64bit_offset)
+         elseif (history_format == 'cdf5') then
+           iflag = ior(nf90_clobber,nf90_64bit_data)
+         elseif (history_format == 'hdf5') then
+           iflag = ior(nf90_clobber,nf90_netcdf4)
+         else
+           call abort_ice(subname//' ERROR: history_format not allowed for '//trim(history_format), &
+              file=__FILE__, line=__LINE__)
+         endif
          status = nf90_create(ncfile(ns), iflag, ncid)
          call ice_check_nc(status, subname// ' ERROR: creating history ncfile '//ncfile(ns), &
                            file=__FILE__, line=__LINE__)

cicecore/cicedyn/infrastructure/io/io_netcdf/ice_restart.F90

@@ -18,7 +18,7 @@ module ice_restart
       use ice_read_write, only: ice_check_nc
       use ice_restart_shared, only: &
           restart_ext, restart_dir, restart_file, pointer_file, &
-          runid, use_restart_time, lcdf64, lenstr, restart_coszen
+          runid, use_restart_time, lenstr, restart_coszen, restart_format
       use ice_fileunits, only: nu_diag, nu_rst_pointer
       use ice_exit, only: abort_ice
       use icepack_intfc, only: icepack_query_parameters
@@ -216,8 +216,18 @@ subroutine init_restart_write(filename_spec)
          write(nu_rst_pointer,'(a)') filename
          close(nu_rst_pointer)
 
-         iflag = 0
-         if (lcdf64) iflag = nf90_64bit_offset
+         if (restart_format == 'cdf1') then
+           iflag = nf90_clobber
+         elseif (restart_format == 'cdf2') then
+           iflag = ior(nf90_clobber,nf90_64bit_offset)
+         elseif (restart_format == 'cdf5') then
+           iflag = ior(nf90_clobber,nf90_64bit_data)
+         elseif (restart_format == 'hdf5') then
+           iflag = ior(nf90_clobber,nf90_netcdf4)
+         else
+           call abort_ice(subname//' ERROR: restart_format not allowed for '//trim(restart_format), &
+              file=__FILE__, line=__LINE__)
+         endif
          status = nf90_create(trim(filename), iflag, ncid)
          call ice_check_nc(status, subname//' ERROR: creating '//trim(filename), file=__FILE__, line=__LINE__)
 

cicecore/cicedyn/infrastructure/io/io_pio2/ice_history_write.F90

@@ -57,7 +57,7 @@ subroutine ice_write_hist (ns)
           lonn_bounds, latn_bounds, lone_bounds, late_bounds
       use ice_history_shared
       use ice_arrays_column, only: hin_max, floe_rad_c
-      use ice_restart_shared, only: runid, lcdf64
+      use ice_restart_shared, only: runid
       use ice_pio
       use pio
 
@@ -78,7 +78,6 @@ subroutine ice_write_hist (ns)
       character (char_len) :: title
       character (char_len) :: time_period_freq = 'none'
       character (char_len_long) :: ncfile(max_nstrm)
-      integer (kind=int_kind) :: iotype
 
       integer (kind=int_kind) :: icategory,ind,i_aice,boundid
 
@@ -145,6 +144,8 @@ subroutine ice_write_hist (ns)
 
       integer (kind=int_kind) :: lprecision
 
+      logical (kind=log_kind), save :: first_call = .true.
+
       character(len=*), parameter :: subname = '(ice_write_hist)'
 
       call icepack_query_parameters(secday_out=secday)
@@ -167,11 +168,10 @@ subroutine ice_write_hist (ns)
       call broadcast_scalar(filename, master_task)
 
       ! create file
-      iotype = PIO_IOTYPE_NETCDF
-      if (history_format == 'pio_pnetcdf') iotype = PIO_IOTYPE_PNETCDF
       File%fh=-1
       call ice_pio_init(mode='write', filename=trim(filename), File=File, &
-        clobber=.true., cdf64=lcdf64, iotype=iotype)
+           clobber=.true., fformat=trim(history_format), rearr=trim(history_rearranger), &
+           iotasks=history_iotasks, root=history_root, stride=history_stride, debug=first_call)
 
       call ice_pio_initdecomp(iodesc=iodesc2d, precision=history_precision)
       call ice_pio_initdecomp(ndim3=ncat_hist, iodesc=iodesc3dc, precision=history_precision)
@@ -741,13 +741,8 @@ subroutine ice_write_hist (ns)
       call ice_pio_check(pio_put_att(File,pio_global,'history',trim(start_time)), &
            subname//' ERROR: defining att history '//trim(start_time),file=__FILE__,line=__LINE__)
 
-      if (history_format == 'pio_pnetcdf') then
-         call ice_pio_check(pio_put_att(File,pio_global,'io_flavor','io_pio pnetcdf'), &
-              subname//' ERROR: defining att io_flavor',file=__FILE__,line=__LINE__)
-      else
-         call ice_pio_check(pio_put_att(File,pio_global,'io_flavor','io_pio netcdf'), &
-              subname//' ERROR: defining att io_flavor',file=__FILE__,line=__LINE__)
-      endif
+      call ice_pio_check(pio_put_att(File,pio_global,'io_flavor','io_pio '//trim(history_format)), &
+           subname//' ERROR: defining att io_flavor',file=__FILE__,line=__LINE__)
 
       !-----------------------------------------------------------------
       ! end define mode
@@ -1342,6 +1337,8 @@ subroutine ice_write_hist (ns)
          write(nu_diag,*) 'Finished writing ',trim(ncfile(ns))
       endif
 
+      first_call = .false.
+
       end subroutine ice_write_hist
 
 !=======================================================================

cicecore/cicedyn/infrastructure/io/io_pio2/ice_pio.F90

@@ -44,7 +44,8 @@ module ice_pio
 !    Initialize the io subsystem
 !    2009-Feb-17 - J. Edwards - initial version
 
-   subroutine ice_pio_init(mode, filename, File, clobber, cdf64, iotype)
+   subroutine ice_pio_init(mode, filename, File, clobber, fformat, &
+                           rearr, iotasks, root, stride, debug)
 
 #ifdef CESMCOUPLED
    use shr_pio_mod, only: shr_pio_getiosys, shr_pio_getiotype
@@ -59,17 +60,21 @@ subroutine ice_pio_init(mode, filename, File, clobber, cdf64, iotype)
    character(len=*)     , intent(in),    optional :: filename
    type(file_desc_t)    , intent(inout), optional :: File
    logical              , intent(in),    optional :: clobber
-   logical              , intent(in),    optional :: cdf64
-   integer              , intent(in),    optional :: iotype
+   character(len=*)     , intent(in),    optional :: fformat
+   character(len=*)     , intent(in),    optional :: rearr
+   integer              , intent(in),    optional :: iotasks
+   integer              , intent(in),    optional :: root
+   integer              , intent(in),    optional :: stride
+   logical              , intent(in),    optional :: debug
 
    ! local variables
 
    integer (int_kind) :: &
       nml_error          ! namelist read error flag
 
-   integer :: nprocs , istride, basetask, numiotasks, rearranger, pio_iotype, status, nmode
-   logical :: lclobber, lcdf64, exists
-   logical, save :: first_call = .true.
+   integer :: nprocs , lstride, lroot, liotasks, rearranger
+   integer :: pio_iotype, status, nmode0, nmode
+   logical :: lclobber, exists, ldebug
    character(len=*), parameter :: subname = '(ice_pio_init)'
 
 #ifdef CESMCOUPLED
@@ -86,32 +91,95 @@ subroutine ice_pio_init(mode, filename, File, clobber, cdf64, iotype)
 #endif
 
    !--- initialize type of io
-   !pio_iotype = PIO_IOTYPE_PNETCDF
-   !pio_iotype = PIO_IOTYPE_NETCDF4C
-   !pio_iotype = PIO_IOTYPE_NETCDF4P
-   pio_iotype = PIO_IOTYPE_NETCDF
-   if (present(iotype)) then
-      pio_iotype = iotype
+
+   lclobber = .false.
+   if (present(clobber)) then
+      lclobber=clobber
+   endif
+
+   ldebug = .false.
+   if (present(debug)) then
+      ldebug = debug
+   endif
+
+   if (present(fformat)) then
+      if (fformat(1:3) == 'cdf') then
+         pio_iotype = PIO_IOTYPE_NETCDF
+      elseif (fformat(1:3) == 'hdf') then
+         pio_iotype = PIO_IOTYPE_NETCDF4P 
+      elseif (fformat(1:7) == 'pnetcdf') then
+         pio_iotype = PIO_IOTYPE_PNETCDF
+      else
+         call abort_ice(subname//' ERROR: format not allowed for '//trim(fformat), &
+            file=__FILE__, line=__LINE__)
+      endif
+
+      if (fformat == 'cdf2' .or. fformat == 'pnetcdf2') then
+         nmode0 = PIO_64BIT_OFFSET
+      elseif (fformat == 'cdf5' .or. fformat == 'pnetcdf5') then
+         nmode0 = PIO_64BIT_DATA
+      else
+         nmode0 = 0
+      endif
+   else
+      pio_iotype = PIO_IOTYPE_NETCDF
+      nmode0 = 0
+   endif
+
+   if (present(rearr)) then
+      if (rearr == 'box' .or. rearr == 'default') then
+         rearranger = PIO_REARR_BOX
+      elseif (rearr == 'subset') then
+         rearranger = PIO_REARR_SUBSET
+      else
+         call abort_ice(subname//' ERROR: rearr not allowed for '//trim(rearr), &
+            file=__FILE__, line=__LINE__)
+      endif
+   else
+      rearranger = PIO_REARR_BOX
    endif
 
-   !--- initialize ice_pio_subsystem
    nprocs = get_num_procs()
-   istride = 4
-   basetask = min(1,nprocs-1)
-   numiotasks = max((nprocs-basetask)/istride,1)
-!--tcraig this should work better but it causes pio2.4.4 to fail for reasons unknown
-!   numiotasks = 1 + (nprocs-basetask-1)/istride
-   rearranger = PIO_REARR_BOX
-   if (my_task == master_task) then
+   lstride = 4
+   lroot = min(1,nprocs-1)
+   liotasks = max(1,(nprocs-lroot)/lstride)
+
+   if (present(iotasks)) then
+      if (iotasks /= -99) liotasks=iotasks
+   endif
+   if (present(root)) then
+      if (root /= -99) lroot=root
+   endif
+   if (present(stride)) then
+      if (stride /= -99) lstride=stride
+   endif
+
+   if (liotasks < 1 .or. lroot < 0 .or. lstride < 1) then
+      call abort_ice(subname//' ERROR: iotasks, root, stride incorrect ', &
+         file=__FILE__, line=__LINE__)
+   endif
+
+   ! adjust to fit in nprocs, preserve root and stride as much as possible
+   lroot = min(lroot,nprocs-1)   ! lroot <= nprocs-1
+   ! tcraig, should work better but aborts in pio2
+   !liotasks = min(liotasks, 1 + (nprocs-lroot-1)/lstride)
+   if (lroot + (liotasks-1)*lstride > nprocs-1) then
+      liotasks = max(1,(nprocs-lroot)/lstride)
+   endif
+
+   !--- initialize ice_pio_subsystem
+
+   if (ldebug .and. my_task == master_task) then
       write(nu_diag,*) subname,' nprocs     = ',nprocs
-      write(nu_diag,*) subname,' istride    = ',istride
-      write(nu_diag,*) subname,' basetask   = ',basetask
-      write(nu_diag,*) subname,' numiotasks = ',numiotasks
       write(nu_diag,*) subname,' pio_iotype = ',pio_iotype
+      write(nu_diag,*) subname,' iotasks    = ',liotasks
+      write(nu_diag,*) subname,' baseroot   = ',lroot
+      write(nu_diag,*) subname,' stride     = ',lstride
+      write(nu_diag,*) subname,' nmode      = ',nmode0
    end if
 
-   call pio_init(my_task, MPI_COMM_ICE, numiotasks, master_task, istride, &
-                 rearranger, ice_pio_subsystem, base=basetask)
+   call pio_init(my_task, MPI_COMM_ICE, liotasks, master_task, lstride, &
+                 rearranger, ice_pio_subsystem, base=lroot)
 
    call pio_seterrorhandling(ice_pio_subsystem, PIO_RETURN_ERROR)
 
@@ -139,19 +207,13 @@ subroutine ice_pio_init(mode, filename, File, clobber, cdf64, iotype)
    if (present(mode) .and. present(filename) .and. present(File)) then
 
       if (trim(mode) == 'write') then
-         lclobber = .false.
-         if (present(clobber)) lclobber=clobber
-
-         lcdf64 = .false.
-         if (present(cdf64)) lcdf64=cdf64
 
          if (File%fh<0) then
             ! filename not open
             inquire(file=trim(filename),exist=exists)
             if (exists) then
                if (lclobber) then
-                  nmode = pio_clobber
-                  if (lcdf64) nmode = ior(nmode,PIO_64BIT_OFFSET)
+                  nmode = ior(PIO_CLOBBER,nmode0)
                   status = pio_createfile(ice_pio_subsystem, File, pio_iotype, trim(filename), nmode)
                   call ice_pio_check(status, subname//' ERROR: Failed to create file '//trim(filename), &
                        file=__FILE__,line=__LINE__)
@@ -168,8 +230,7 @@ subroutine ice_pio_init(mode, filename, File, clobber, cdf64, iotype)
                   end if
                endif
             else
-               nmode = pio_noclobber
-               if (lcdf64) nmode = ior(nmode,PIO_64BIT_OFFSET)
+               nmode = ior(PIO_NOCLOBBER,nmode0)
                status = pio_createfile(ice_pio_subsystem, File, pio_iotype, trim(filename), nmode)
                call ice_pio_check( status, subname//' ERROR: Failed to create file '//trim(filename), &
                     file=__FILE__,line=__LINE__)