Merge pull request ESMCI#1433 from NCAR/ejh_docs_2

documentation improvements

src/clib/pio.h

@@ -34,12 +34,14 @@
 #endif
 
 /** PIO_OFFSET is an integer type of size sufficient to represent the
- * size (in bytes) of the largest file supported by MPI. But this is
- * not used much in the code. Instead, PIO_Offset is used. */
+ * size (in bytes) of the largest file supported by MPI. This is not
+ * actually used by the code. */
 #define PIO_OFFSET MPI_OFFSET
 
-/** MPI_Offset is defined in pio_internal.h as long long. This is what
- * is used in the PIO C code. */
+/** PIO_OFFSET is defined as MPI_Offset, which is defined in
+ * pio_internal.h as long long. This is what is used throughout the C
+ * code. */
+
 #define PIO_Offset MPI_Offset
 
 /** The maximum number of variables allowed in a netCDF file. */
@@ -60,20 +62,21 @@
 /** Holds the maximum length of any task map. */
 #define DECOMP_MAX_MAPLEN_ATT_NAME "max_maplen"
 
-/** Name of title attribute for decomposition files. */
+/** Name of title attribute in decomposition file. */
 #define DECOMP_TITLE_ATT_NAME "title"
 
-/** Name of history attribute for decomposition files. */
+/** Name of history attribute in decomposition file. */
 #define DECOMP_HISTORY_ATT_NAME "history"
 
-/** Name of source attribute for decomposition files. */
+/** Name of source attribute in decomposition file. */
 #define DECOMP_SOURCE_ATT_NAME "source"
 
-/** Name of array order (C or Fortran) attribute for decomposition
- * files. */
+/** Name of array order (C or Fortran) attribute in decomposition
+ * file. */
 #define DECOMP_ORDER_ATT_NAME "array_order"
 
-/** Name of backtrace attribute for decomposition files. */
+/** Name of backtrace attribute in decomposition file. */
+
 #define DECOMP_BACKTRACE_ATT_NAME "backtrace"
 
 /** Name for the dim dim in decomp file. */
@@ -85,10 +88,10 @@
 /** Name for the npes dim in decomp file. */
 #define DECOMP_MAPELEM_DIM_NAME "map_element"
 
-/** Name for the number of dims dimention in decomposition file. */
+/** Name for the number of dimensions dim in decomp file. */
 #define DECOMP_NDIMS "ndims"
 
-/** Name of var in decomposition file that holds global array sizes. */
+/** Name of var in decomp file that holds global array sizes. */
 #define DECOMP_GLOBAL_SIZE_VAR_NAME "global_size"
 
 /** Name of var in decomp file that holds the length of the map for
@@ -214,7 +217,7 @@ enum PIO_REARR_COMM_FC_DIR
     PIO_REARR_COMM_FC_2D_DISABLE
 };
 
-/** Constant to indicate unlimited requests. */
+/** Constant to indicate unlimited requests for the rearranger. */
 #define PIO_REARR_COMM_UNLIMITED_PEND_REQ -1
 
 /**
@@ -764,10 +767,10 @@ enum PIO_ERROR_HANDLERS
 #define PIO_EINDEP  (-203)
 #endif /* _PNETCDF */
 
-/** This is the value of the first error code for PIO. */
+/** The first error code for PIO. */
 #define PIO_FIRST_ERROR_CODE (-500)
 
-/** IOTYPE error. */
+/** Bad IOTYPE error. */
 #define PIO_EBADIOTYPE  (-500)
 
 /** Variable dimensions do not match in a multivar call. */
@@ -825,11 +828,14 @@ extern "C" {
                         int *num_procs_per_comp, int **proc_list, MPI_Comm *io_comm, MPI_Comm *comp_comm,
                         int rearranger, int *iosysidp);
 
-    int PIOc_Init_Intercomm(int component_count, MPI_Comm peer_comm, MPI_Comm *comp_comms,
-                            MPI_Comm io_comm, int *iosysidp);
+    /* How many IO tasks in this iosysid? */
     int PIOc_get_numiotasks(int iosysid, int *numiotasks);
+
+    /* Initialize PIO for intracomm mode. */
     int PIOc_Init_Intracomm(MPI_Comm comp_comm, int num_iotasks, int stride, int base, int rearr,
                             int *iosysidp);
+
+    /* Shut down iosystem and free all associated resources. */
     int PIOc_finalize(int iosysid);
 
     /* Set error handling for entire io system. */
@@ -920,7 +926,6 @@ extern "C" {
                              int deflate_level);
     int PIOc_inq_var_deflate(int ncid, int varid, int *shufflep, int *deflatep,
                              int *deflate_levelp);
-    int PIOc_inq_var_szip(int ncid, int varid, int *options_maskp, int *pixels_per_blockp);
     int PIOc_def_var_chunking(int ncid, int varid, int storage, const PIO_Offset *chunksizesp);
     int PIOc_inq_var_chunking(int ncid, int varid, int *storagep, PIO_Offset *chunksizesp);
     int PIOc_def_var_endian(int ncid, int varid, int endian);

src/clib/pio_darray.c

@@ -12,18 +12,22 @@
 #include <pio.h>
 #include <pio_internal.h>
 #include <uthash.h>
-/* 10MB default limit. */
+
+/** 10MB default limit. */
 PIO_Offset pio_buffer_size_limit = PIO_BUFFER_SIZE;
 
-/* Global buffer pool pointer. */
+/** Global buffer pool pointer. */
 void *CN_bpool = NULL;
 
-/* Maximum buffer usage. */
+/** Maximum buffer usage. */
 PIO_Offset maxusage = 0;
 
-/* For write_darray_multi_serial() and write_darray_multi_par() to
- * indicate whether fill or data are being written. */
+/** For write_darray_multi_serial() and write_darray_multi_par() to
+ * indicate that fill is being written. */
 #define DARRAY_FILL 1
+
+/** For write_darray_multi_serial() and write_darray_multi_par() to
+ * indicate that data are being written. */
 #define DARRAY_DATA 0
 
 /**
@@ -36,7 +40,8 @@ PIO_Offset maxusage = 0;
  * @return The previous limit setting.
  * @author Jim Edwards
  */
-PIO_Offset PIOc_set_buffer_size_limit(PIO_Offset limit)
+PIO_Offset
+PIOc_set_buffer_size_limit(PIO_Offset limit)
 {
     PIO_Offset oldsize = pio_buffer_size_limit;
 
@@ -101,9 +106,10 @@ PIO_Offset PIOc_set_buffer_size_limit(PIO_Offset limit)
  * @ingroup PIO_write_darray
  * @author Jim Edwards, Ed Hartnett
  */
-int PIOc_write_darray_multi(int ncid, const int *varids, int ioid, int nvars,
-                            PIO_Offset arraylen, void *array, const int *frame,
-                            void **fillvalue, bool flushtodisk)
+int
+PIOc_write_darray_multi(int ncid, const int *varids, int ioid, int nvars,
+                        PIO_Offset arraylen, void *array, const int *frame,
+                        void **fillvalue, bool flushtodisk)
 {
     iosystem_desc_t *ios;  /* Pointer to io system information. */
     file_desc_t *file;     /* Pointer to file information. */
@@ -145,8 +151,8 @@ int PIOc_write_darray_multi(int ncid, const int *varids, int ioid, int nvars,
         var_desc_t *vdesc;
         if ((ierr = get_var_desc(varids[v], &file->varlist, &vdesc)))
             return pio_err(ios, file, ierr, __FILE__, __LINE__);
-//        if (vdesc->pio_type != iodesc->piotype)
-//            return pio_err(ios, file, PIO_EINVAL, __FILE__, __LINE__);
+        /* if (vdesc->pio_type != iodesc->piotype)
+           return pio_err(ios, file, PIO_EINVAL, __FILE__, __LINE__);*/
     }
 
     /* Get a pointer to the variable info for the first variable. */
@@ -162,16 +168,17 @@ int PIOc_write_darray_multi(int ncid, const int *varids, int ioid, int nvars,
         if ((ierr = PIOc_inq_varndims(file->pio_ncid, varids[0], &fndims)))
             return check_netcdf(file, ierr, __FILE__, __LINE__);
         LOG((3, "called PIOc_inq_varndims varids[0] = %d fndims = %d", varids[0], fndims));
-	for (int v=1; v < nvars; v++){
-	    if ((ierr = PIOc_inq_varndims(file->pio_ncid, varids[v], &fndims2)))
-		return check_netcdf(file, ierr, __FILE__, __LINE__);
-	    if(fndims != fndims2)
-		return pio_err(ios, file, PIO_EVARDIMMISMATCH, __FILE__, __LINE__);
-	}
+        for (int v=1; v < nvars; v++){
+            if ((ierr = PIOc_inq_varndims(file->pio_ncid, varids[v], &fndims2)))
+                return check_netcdf(file, ierr, __FILE__, __LINE__);
+            if(fndims != fndims2)
+                return pio_err(ios, file, PIO_EVARDIMMISMATCH, __FILE__, __LINE__);
+        }
 
     }
 
-    /* If async is in use, and this is not an IO task, bcast the parameters. */
+    /* If async is in use, and this is not an IO task, bcast the
+     * parameters. */
     if (ios->async)
     {
         if (!ios->ioproc)
@@ -265,7 +272,7 @@ int PIOc_write_darray_multi(int ncid, const int *varids, int ioid, int nvars,
     }
     else if (file->iotype == PIO_IOTYPE_PNETCDF && ios->ioproc)
     {
-	/* this assures that iobuf is allocated on all iotasks thus
+        /* this assures that iobuf is allocated on all iotasks thus
            assuring that the flush_output_buffer call above is called
            collectively (from all iotasks) */
         if (!(file->iobuf = bget(1)))
@@ -274,13 +281,13 @@ int PIOc_write_darray_multi(int ncid, const int *varids, int ioid, int nvars,
     }
     if (iodesc->needssort)
     {
-	if (!(tmparray = malloc(arraylen*nvars*iodesc->piotype_size)))
-	    return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
-	pio_sorted_copy(array, tmparray, iodesc, nvars, 0);
+        if (!(tmparray = malloc(arraylen*nvars*iodesc->piotype_size)))
+            return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+        pio_sorted_copy(array, tmparray, iodesc, nvars, 0);
     }
     else
     {
-	tmparray = array;
+        tmparray = array;
     }
 
     /* Move data from compute to IO tasks. */
@@ -314,7 +321,7 @@ int PIOc_write_darray_multi(int ncid, const int *varids, int ioid, int nvars,
         /* Release resources. */
         if (file->iobuf)
         {
-	    LOG((3,"freeing variable buffer in pio_darray"));
+            LOG((3,"freeing variable buffer in pio_darray"));
             brel(file->iobuf);
             file->iobuf = NULL;
         }
@@ -335,22 +342,22 @@ int PIOc_write_darray_multi(int ncid, const int *varids, int ioid, int nvars,
         LOG((2, "nvars = %d holegridsize = %ld iodesc->needsfill = %d\n", nvars,
              iodesc->holegridsize, iodesc->needsfill));
 
-	pioassert(!vdesc0->fillbuf, "buffer overwrite",__FILE__, __LINE__);
+        pioassert(!vdesc0->fillbuf, "buffer overwrite",__FILE__, __LINE__);
 
         /* Get a buffer. */
-	if (ios->io_rank == 0)
-	    vdesc0->fillbuf = bget(iodesc->maxholegridsize * iodesc->mpitype_size * nvars);
-	else if (iodesc->holegridsize > 0)
-	    vdesc0->fillbuf = bget(iodesc->holegridsize * iodesc->mpitype_size * nvars);
+        if (ios->io_rank == 0)
+            vdesc0->fillbuf = bget(iodesc->maxholegridsize * iodesc->mpitype_size * nvars);
+        else if (iodesc->holegridsize > 0)
+            vdesc0->fillbuf = bget(iodesc->holegridsize * iodesc->mpitype_size * nvars);
 
         /* copying the fill value into the data buffer for the box
          * rearranger. This will be overwritten with data where
          * provided. */
-	if(fillvalue)
-	    for (int nv = 0; nv < nvars; nv++)
-		for (int i = 0; i < iodesc->holegridsize; i++)
-		    memcpy(&((char *)vdesc0->fillbuf)[iodesc->mpitype_size * (i + nv * iodesc->holegridsize)],
-			   &((char *)fillvalue)[iodesc->mpitype_size * nv], iodesc->mpitype_size);
+        if(fillvalue)
+            for (int nv = 0; nv < nvars; nv++)
+                for (int i = 0; i < iodesc->holegridsize; i++)
+                    memcpy(&((char *)vdesc0->fillbuf)[iodesc->mpitype_size * (i + nv * iodesc->holegridsize)],
+                           &((char *)fillvalue)[iodesc->mpitype_size * nv], iodesc->mpitype_size);
 
         /* Write the darray based on the iotype. */
         switch (file->iotype)
@@ -384,7 +391,7 @@ int PIOc_write_darray_multi(int ncid, const int *varids, int ioid, int nvars,
     }
 
     if(iodesc->needssort && tmparray != NULL)
-	free(tmparray);
+        free(tmparray);
 
     /* Flush data to disk for pnetcdf. */
     if (ios->ioproc && file->iotype == PIO_IOTYPE_PNETCDF)
@@ -510,7 +517,8 @@ pio_inq_var_fill_expected(int ncid, int varid, int pio_type, PIO_Offset type_siz
  * @ingroup PIO_write_darray
  * @author Ed Hartnett
  */
-int find_var_fillvalue(file_desc_t *file, int varid, var_desc_t *vdesc)
+int
+find_var_fillvalue(file_desc_t *file, int varid, var_desc_t *vdesc)
 {
     iosystem_desc_t *ios;  /* Pointer to io system information. */
     int pio_type;
@@ -603,8 +611,9 @@ int find_var_fillvalue(file_desc_t *file, int varid, var_desc_t *vdesc)
  * @ingroup PIO_write_darray
  * @author Jim Edwards, Ed Hartnett
  */
-int PIOc_write_darray(int ncid, int varid, int ioid, PIO_Offset arraylen, void *array,
-                      void *fillvalue)
+int
+PIOc_write_darray(int ncid, int varid, int ioid, PIO_Offset arraylen, void *array,
+                  void *fillvalue)
 {
     iosystem_desc_t *ios;  /* Pointer to io system information. */
     file_desc_t *file;     /* Info about file we are writing to. */
@@ -656,10 +665,10 @@ int PIOc_write_darray(int ncid, int varid, int ioid, PIO_Offset arraylen, void *
 
     /* If the type of the var doesn't match the type of the
      * decomposition, return an error. */
-//    if (iodesc->piotype != vdesc->pio_type)
-//        return pio_err(ios, file, PIO_EINVAL, __FILE__, __LINE__);
-//    pioassert(iodesc->mpitype_size == vdesc->mpi_type_size, "wrong mpi info",
-//              __FILE__, __LINE__);
+    /* if (iodesc->piotype != vdesc->pio_type) */
+    /*     return pio_err(ios, file, PIO_EINVAL, __FILE__, __LINE__); */
+    /* pioassert(iodesc->mpitype_size == vdesc->mpi_type_size, "wrong mpi info", */
+    /*           __FILE__, __LINE__); */
 
     /* If we don't know the fill value for this var, get it. */
     if (!vdesc->fillvalue)
@@ -677,7 +686,7 @@ int PIOc_write_darray(int ncid, int varid, int ioid, PIO_Offset arraylen, void *
     hashid = ioid*10 + vdesc->rec_var;
     HASH_FIND_INT( file->buffer, &hashid, wmb);
     if (wmb)
-      LOG((3, "wmb->ioid = %d wmb->recordvar = %d", wmb->ioid, wmb->recordvar));
+        LOG((3, "wmb->ioid = %d wmb->recordvar = %d", wmb->ioid, wmb->recordvar));
 
     /* If we did not find an existing wmb entry, create a new wmb. */
     if (!wmb)
@@ -695,8 +704,8 @@ int PIOc_write_darray(int ncid, int varid, int ioid, PIO_Offset arraylen, void *
         wmb->data = NULL;
         wmb->frame = NULL;
         wmb->fillvalue = NULL;
-	wmb->htid = hashid;
-	HASH_ADD_INT( file->buffer, htid, wmb );
+        wmb->htid = hashid;
+        HASH_ADD_INT( file->buffer, htid, wmb );
     }
     LOG((2, "wmb->num_arrays = %d arraylen = %d iodesc->mpitype_size = %d\n",
          wmb->num_arrays, arraylen, iodesc->mpitype_size));
@@ -732,7 +741,7 @@ int PIOc_write_darray(int ncid, int varid, int ioid, PIO_Offset arraylen, void *
        https://github.com/pmodels/mpich/pull/2888 */
     io_data_size = (1 + wmb->num_arrays) * iodesc->maxiobuflen * iodesc->mpitype_size;
     if(io_data_size > INT_MAX)
-	needsflush = 2;
+        needsflush = 2;
 
     /* Tell all tasks on the computation communicator whether we need
      * to flush data. */
@@ -848,8 +857,9 @@ int PIOc_write_darray(int ncid, int varid, int ioid, PIO_Offset arraylen, void *
  * @ingroup PIO_read_darray
  * @author Jim Edwards, Ed Hartnett
  */
-int PIOc_read_darray(int ncid, int varid, int ioid, PIO_Offset arraylen,
-                     void *array)
+int
+PIOc_read_darray(int ncid, int varid, int ioid, PIO_Offset arraylen,
+                 void *array)
 {
     iosystem_desc_t *ios;  /* Pointer to io system information. */
     file_desc_t *file;     /* Pointer to file information. */
@@ -899,22 +909,22 @@ int PIOc_read_darray(int ncid, int varid, int ioid, PIO_Offset arraylen,
 
     if (iodesc->needssort)
     {
-	if (!(tmparray = malloc(iodesc->piotype_size*iodesc->maplen)))
-	    return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
-	for(int m=0; m<iodesc->maplen;m++)
-	    ((int *) array)[m] = -1;
+        if (!(tmparray = malloc(iodesc->piotype_size*iodesc->maplen)))
+            return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+        for(int m=0; m<iodesc->maplen;m++)
+            ((int *) array)[m] = -1;
     }
     else
-	tmparray = array;
+        tmparray = array;
 
     /* Rearrange the data. */
     if ((ierr = rearrange_io2comp(ios, iodesc, iobuf, tmparray)))
         return pio_err(ios, file, ierr, __FILE__, __LINE__);
 
     if (iodesc->needssort)
     {
-	pio_sorted_copy(tmparray, array, iodesc, 1, 1);
-	free(tmparray);
+        pio_sorted_copy(tmparray, array, iodesc, 1, 1);
+        free(tmparray);
     }
 
     /* Free the buffer. */