Merge pull request ESMCI#1486 from NCAR/ejh_isolate_gptl

isolating GPTL calls

Makefile.am

@@ -14,4 +14,3 @@ SUBDIRS = src tests examples ${DOC} scripts
 
 EXTRA_DIST = CMakeLists.txt set_flags.am COPYRIGHT
 
-CLEANFILES = docs/*

src/clib/pio_darray_int.c

@@ -421,9 +421,10 @@ write_darray_multi_par(file_desc_t *file, int nvars, int fndims, const int *vari
          iodesc->mpitype, iodesc->maxregions, iodesc->llen));
 
 #ifdef TIMING
-    /* Start timing this function. */
-    GPTLstart("PIO:write_darray_multi_par");
-#endif
+    /* Start timer if desired. */
+    if ((ierr = pio_start_timer("PIO:write_darray_multi_par")))
+        return pio_err(NULL, NULL, ierr, __FILE__, __LINE__);
+#endif /* TIMING */
 
     /* Get pointer to iosystem. */
     ios = file->iosystem;
@@ -720,9 +721,9 @@ write_darray_multi_par(file_desc_t *file, int nvars, int fndims, const int *vari
     ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
 
 #ifdef TIMING
-    /* Stop timing this function. */
-    GPTLstop("PIO:write_darray_multi_par");
-#endif
+    if ((ierr = pio_stop_timer("PIO:write_darray_multi_par")))
+        return pio_err(ios, NULL, ierr, __FILE__, __LINE__);
+#endif /* TIMING */
 
     return ierr;
 }
@@ -1089,9 +1090,10 @@ write_darray_multi_serial(file_desc_t *file, int nvars, int fndims, const int *v
     void *iobuf = fill ? vdesc->fillbuf : file->iobuf;
 
 #ifdef TIMING
-    /* Start timing this function. */
-    GPTLstart("PIO:write_darray_multi_serial");
-#endif
+    /* Start timer if desired. */
+    if ((ierr = pio_start_timer("PIO:write_darray_multi_serial")))
+        return pio_err(ios, NULL, ierr, __FILE__, __LINE__);
+#endif /* TIMING */
 
     /* Only IO tasks participate in this code. */
     if (ios->ioproc)
@@ -1127,9 +1129,9 @@ write_darray_multi_serial(file_desc_t *file, int nvars, int fndims, const int *v
     }
 
 #ifdef TIMING
-    /* Stop timing this function. */
-    GPTLstop("PIO:write_darray_multi_serial");
-#endif
+    if ((ierr = pio_stop_timer("PIO:write_darray_multi_serial")))
+        return pio_err(ios, NULL, ierr, __FILE__, __LINE__);
+#endif /* TIMING */
 
     return PIO_NOERR;
 }
@@ -1169,15 +1171,16 @@ pio_read_darray_nc(file_desc_t *file, io_desc_t *iodesc, int vid, void *iobuf)
     pioassert(file && file->iosystem && iodesc && vid <= PIO_MAX_VARS, "invalid input",
               __FILE__, __LINE__);
 
-#ifdef TIMING
-    /* Start timing this function. */
-    GPTLstart("PIO:read_darray_nc");
-#endif
-
     /* Get the IO system info. */
     ios = file->iosystem;
     LOG((3, "pio_read_darray_nc ios->ioproc %d", ios->ioproc));
 
+#ifdef TIMING
+    /* Start timer if desired. */
+    if ((ierr = pio_start_timer("PIO:read_darray_nc")))
+        return pio_err(ios, NULL, ierr, __FILE__, __LINE__);
+#endif /* TIMING */
+
     /* Get the variable info. */
     if ((ierr = get_var_desc(vid, &file->varlist, &vdesc)))
         return pio_err(NULL, file, ierr, __FILE__, __LINE__);
@@ -1398,9 +1401,9 @@ pio_read_darray_nc(file_desc_t *file, io_desc_t *iodesc, int vid, void *iobuf)
     }
 
 #ifdef TIMING
-    /* Stop timing this function. */
-    GPTLstop("PIO:read_darray_nc");
-#endif
+    if ((ierr = pio_stop_timer("PIO:read_darray_nc")))
+        return pio_err(ios, NULL, ierr, __FILE__, __LINE__);
+#endif /* TIMING */
 
     return PIO_NOERR;
 }
@@ -1442,12 +1445,14 @@ pio_read_darray_nc_serial(file_desc_t *file, io_desc_t *iodesc, int vid,
               "invalid input", __FILE__, __LINE__);
 
     LOG((2, "pio_read_darray_nc_serial vid = %d", vid));
-#ifdef TIMING
-    /* Start timing this function. */
-    GPTLstart("PIO:read_darray_nc_serial");
-#endif
     ios = file->iosystem;
 
+#ifdef TIMING
+    /* Start timer if desired. */
+    if ((ierr = pio_start_timer("PIO:read_darray_nc_serial")))
+        return pio_err(ios, NULL, ierr, __FILE__, __LINE__);
+#endif /* TIMING */
+
     /* Get var info for this var. */
     if ((ierr = get_var_desc(vid, &file->varlist, &vdesc)))
         return pio_err(NULL, file, ierr, __FILE__, __LINE__);
@@ -1703,9 +1708,9 @@ pio_read_darray_nc_serial(file_desc_t *file, io_desc_t *iodesc, int vid,
     }
 
 #ifdef TIMING
-    /* Stop timing this function. */
-    GPTLstop("PIO:read_darray_nc_serial");
-#endif
+    if ((ierr = pio_stop_timer("PIO:read_darray_nc_serial")))
+        return pio_err(ios, NULL, ierr, __FILE__, __LINE__);
+#endif /* TIMING */
 
     return PIO_NOERR;
 }

src/clib/pio_internal.h

@@ -363,6 +363,13 @@ extern "C" {
 
     int PIOc_inq_att_eh(int ncid, int varid, const char *name, int eh,
                         nc_type *xtypep, PIO_Offset *lenp);
+
+    /* Start a timer. */
+    int pio_start_timer(const char *name);
+
+    /* Stop a timer. */
+    int pio_stop_timer(const char *name);
+
 #if defined(__cplusplus)
 }
 #endif

src/clib/pio_rearrange.c

@@ -8,12 +8,10 @@
 #include <pio_internal.h>
 #include <pio.h>
 
-
 #if PIO_USE_MPISERIAL
-# define MPI_Type_create_hvector MPI_Type_hvector
+#define MPI_Type_create_hvector MPI_Type_hvector
 #endif
 
-
 /**
  * Convert a 1-D index into a coordinate value in an arbitrary
  * dimension space. E.g., for index 4 into a array defined as a[3][2],
@@ -813,8 +811,10 @@ int rearrange_comp2io(iosystem_desc_t *ios, io_desc_t *iodesc, void *sbuf,
     int ret;
 
 #ifdef TIMING
-    GPTLstart("PIO:rearrange_comp2io");
-#endif
+    /* Start timer if desired. */
+    if ((ret = pio_start_timer("PIO:rearrange_comp2io")))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+#endif /* TIMING */
 
     /* Caller must provide these. */
     pioassert(ios && iodesc && nvars > 0, "invalid input", __FILE__, __LINE__);
@@ -969,8 +969,9 @@ int rearrange_comp2io(iosystem_desc_t *ios, io_desc_t *iodesc, void *sbuf,
     }
 
 #ifdef TIMING
-    GPTLstop("PIO:rearrange_comp2io");
-#endif
+    if ((ret = pio_stop_timer("PIO:rearrange_comp2io")))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+#endif /* TIMING */
 
     return PIO_NOERR;
 }
@@ -999,8 +1000,10 @@ int rearrange_io2comp(iosystem_desc_t *ios, io_desc_t *iodesc, void *sbuf,
     pioassert(ios && iodesc, "invalid input", __FILE__, __LINE__);
 
 #ifdef TIMING
-    GPTLstart("PIO:rearrange_io2comp");
-#endif
+    /* Start timer if desired. */
+    if ((ret = pio_start_timer("PIO:rearrange_io2comp")))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+#endif /* TIMING */
 
     /* Different rearrangers use different communicators and number of
      * IO tasks. */
@@ -1093,7 +1096,8 @@ int rearrange_io2comp(iosystem_desc_t *ios, io_desc_t *iodesc, void *sbuf,
         return pio_err(ios, NULL, ret, __FILE__, __LINE__);
 
 #ifdef TIMING
-    GPTLstop("PIO:rearrange_io2comp");
+    if ((ret = pio_stop_timer("PIO:rearrange_io2comp")))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
 #endif
 
     return PIO_NOERR;
@@ -1200,9 +1204,6 @@ int box_rearrange_create(iosystem_desc_t *ios, int maplen, const PIO_Offset *com
 {
     int ret;
 
-#ifdef TIMING
-    GPTLstart("PIO:box_rearrange_create");
-#endif
     /* Check inputs. */
     pioassert(ios && maplen >= 0 && compmap && gdimlen && ndims > 0 && iodesc,
               "invalid input", __FILE__, __LINE__);
@@ -1227,6 +1228,12 @@ int box_rearrange_create(iosystem_desc_t *ios, int maplen, const PIO_Offset *com
     PIO_Offset sc_info_msg_send[sc_info_msg_sz];
     PIO_Offset sc_info_msg_recv[ios->num_iotasks * sc_info_msg_sz];
 
+#ifdef TIMING
+    /* Start timer if desired. */
+    if ((ret = pio_start_timer("PIO:box_rearrange_create")))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+#endif /* TIMING */
+
     /* This is the box rearranger. */
     iodesc->rearranger = PIO_REARR_BOX;
 
@@ -1479,8 +1486,10 @@ int box_rearrange_create(iosystem_desc_t *ios, int maplen, const PIO_Offset *com
     LOG((3, "iodesc->maxbytes = %d", iodesc->maxbytes));
 
 #ifdef TIMING
-    GPTLstop("PIO:box_rearrange_create");
+    if ((ret = pio_stop_timer("PIO:box_rearrange_create")))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
 #endif
+
     return PIO_NOERR;
 }
 
@@ -1509,8 +1518,11 @@ int box_rearrange_create_with_holes(iosystem_desc_t *ios, int maplen,
     int ret;
 
 #ifdef TIMING
-    GPTLstart("PIO:box_rearrange_create_with_holes");
-#endif
+    /* Start timer if desired. */
+    if ((ret = pio_start_timer("PIO:box_rearrange_create_with_holes")))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+#endif /* TIMING */
+
     /* Check inputs. */
     pioassert(ios && maplen >= 0 && compmap && gdimlen && ndims > 0 && iodesc,
               "invalid input", __FILE__, __LINE__);
@@ -1766,8 +1778,10 @@ int box_rearrange_create_with_holes(iosystem_desc_t *ios, int maplen,
     LOG((3, "iodesc->maxbytes = %d", iodesc->maxbytes));
 
 #ifdef TIMING
-    GPTLstop("PIO:box_rearrange_create_with_holes");
-#endif
+    if ((ret = pio_stop_timer("PIO:box_rearrange_create_with_holes")))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+#endif /* TIMING */
+
     return PIO_NOERR;
 }
 

src/clib/pioc_support.c

@@ -39,6 +39,38 @@ extern int pio_next_ncid;
 /** The default error handler used when iosystem cannot be located. */
 extern int default_error_handler;
 
+/**
+ * Start the PIO timer.
+ *
+ * @param name name of the timer.
+ * @return 0 for success, error code otherwise.
+ * @author Ed Hartnett
+ */
+int
+pio_start_timer(const char *name)
+{
+#ifdef TIMING
+    GPTLstart(name);
+#endif /* TIMING */
+    return PIO_NOERR;
+}
+
+/**
+ * Stop the PIO timer.
+ *
+ * @param name name of the timer.
+ * @return 0 for success, error code otherwise.
+ * @author Ed Hartnett
+ */
+int
+pio_stop_timer(const char *name)
+{
+#ifdef TIMING
+    GPTLstop("PIO:rearrange_comp2io");
+#endif /* TIMING */
+    return PIO_NOERR;
+}
+
 /**
  * Return a string description of an error code. If zero is passed,
  * the errmsg will be "No error".

tests/performance/pioperformance.F90

@@ -103,7 +103,7 @@ program pioperformance
     rearrangers(1)=1
     rearrangers(2)=2
   endif
-  i = pio_set_log_level(2)
+  i = pio_set_log_level(-1)
   do i=1,max_decomp_files
      if(len_trim(decompfile(i))==0) exit
      if(mype == 0) print *, ' Testing decomp: ',trim(decompfile(i))