Skip to content

Commit

Permalink
Merge pull request ESMCI#1489 from NCAR/ejh_mpe
Browse files Browse the repository at this point in the history 
added MPE to test_perf2.c
  • Loading branch information
@edhartnett
edhartnett authored Jun 12, 2019
2 parents 513190a + 9924038 commit 2af631f
Show file tree
Hide file tree
Showing 2 changed files with 179 additions and 12 deletions.
16 changes: 16 additions & 0 deletions configure.ac
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,22 @@ if test "x$enable_logging" = xyes; then
AC_DEFINE([PIO_ENABLE_LOGGING], 1, [If true, turn on logging.])
fi

# Does the user want to use MPE library?
AC_MSG_CHECKING([whether use of MPE library is enabled])
AC_ARG_ENABLE([mpe],
[AS_HELP_STRING([--enable-mpe],
[enable use of MPE library for timing and diagnostic info (may negatively impact performance).])])
test "x$enable_mpe" = xyes || enable_mpe=no
AC_MSG_RESULT([$enable_mpe])
if test "x$enable_mpe" = xyes; then
AC_SEARCH_LIBS([MPE_Log_get_event_number], [mpe], [HAVE_LIBMPE=yes], [HAVE_LIBMPE=no], [-lpthread -lm])
AC_CHECK_HEADERS([mpe.h], [HAVE_MPE=yes], [HAVE_MPE=no])
if test "x$HAVE_LIBMPE" = xno -o "x$HAVE_MPE" = xno; then
AC_MSG_ERROR([MPE not found but --enable-mpe used.])
fi
AC_DEFINE([USE_MPE], 1, [If true, use MPE timing library.])
fi

# Does the user want to enable timing?
AC_MSG_CHECKING([whether GPTL timing library is used])
AC_ARG_ENABLE([timing],
Expand Down
175 changes: 163 additions & 12 deletions tests/cunit/test_perf2.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,9 @@
#include <pio_internal.h>
#include <pio_tests.h>
#include <sys/time.h>
#ifdef USE_MPE
#include <mpe.h>
#endif /* USE_MPE */

/* The number of tasks this test should run on. */
#define TARGET_NTASKS 16
Expand All @@ -30,9 +33,12 @@
#define NDIM3 3

/* The length of our sample data along each dimension. */
#define X_DIM_LEN 1024
#define Y_DIM_LEN 1024
#define Z_DIM_LEN 128
#define X_DIM_LEN 128
#define Y_DIM_LEN 128
#define Z_DIM_LEN 32
/* #define X_DIM_LEN 1024 */
/* #define Y_DIM_LEN 1024 */
/* #define Z_DIM_LEN 128 */

/* The number of timesteps of data to write. */
#define NUM_TIMESTEPS 10
Expand All @@ -59,6 +65,64 @@ int dim_len[NDIM] = {NC_UNLIMITED, X_DIM_LEN, Y_DIM_LEN, Z_DIM_LEN};
/* Run test for each of the rearrangers. */
#define NUM_REARRANGERS_TO_TEST 2

#ifdef USE_MPE
/* These are for the event numbers array used to log various events in
* the program with the MPE library, which produces output for the
* Jumpshot program. */
#define NUM_EVENTS 7
#define START 0
#define END 1
#define INIT 0
#define DECOMP 1
#define CREATE 2
#define DARRAY_WRITE 3
#define CLOSE 4
#define CALCULATE 5
#define INGEST 6

#define ERR_LOGGING 99

/* This array holds even numbers for MPE. */
int event_num[2][NUM_EVENTS];

/* This will set up the MPE logging event numbers. */
int
init_logging(int my_rank, int event_num[][NUM_EVENTS])
{
/* Get a bunch of event numbers. */
event_num[START][INIT] = MPE_Log_get_event_number();
event_num[END][INIT] = MPE_Log_get_event_number();
event_num[START][DECOMP] = MPE_Log_get_event_number();
event_num[END][DECOMP] = MPE_Log_get_event_number();
event_num[START][INGEST] = MPE_Log_get_event_number();
event_num[END][INGEST] = MPE_Log_get_event_number();
event_num[START][CLOSE] = MPE_Log_get_event_number();
event_num[END][CLOSE] = MPE_Log_get_event_number();
event_num[START][CALCULATE] = MPE_Log_get_event_number();
event_num[END][CALCULATE] = MPE_Log_get_event_number();
event_num[START][CREATE] = MPE_Log_get_event_number();
event_num[END][CREATE] = MPE_Log_get_event_number();
event_num[START][DARRAY_WRITE] = MPE_Log_get_event_number();
event_num[END][DARRAY_WRITE] = MPE_Log_get_event_number();

/* You should track at least initialization and partitioning, data
* ingest, update computation, all communications, any memory
* copies (if you do that), any output rendering, and any global
* communications. */
if (!my_rank)
{
MPE_Describe_state(event_num[START][INIT], event_num[END][INIT], "init", "yellow");
MPE_Describe_state(event_num[START][INGEST], event_num[END][INGEST], "ingest", "red");
MPE_Describe_state(event_num[START][DECOMP], event_num[END][DECOMP], "decomposition", "green");
MPE_Describe_state(event_num[START][CALCULATE], event_num[END][CALCULATE], "calculate", "orange");
MPE_Describe_state(event_num[START][CREATE], event_num[END][CREATE], "create", "purple");
MPE_Describe_state(event_num[START][CLOSE], event_num[END][CLOSE], "close file", "blue");
MPE_Describe_state(event_num[START][DARRAY_WRITE], event_num[END][DARRAY_WRITE], "darray write", "pink");
}
return 0;
}
#endif /* USE_MPE */

/* Create the decomposition to divide the 4-dimensional sample data
* between the 4 tasks. For the purposes of decomposition we are only
* concerned with 3 dimensions - we ignore the unlimited dimension.
Expand All @@ -70,7 +134,8 @@ int dim_len[NDIM] = {NC_UNLIMITED, X_DIM_LEN, Y_DIM_LEN, Z_DIM_LEN};
* @param ioid a pointer that gets the ID of this decomposition.
* @returns 0 for success, error code otherwise.
**/
int create_decomposition_3d(int ntasks, int my_rank, int iosysid, int *ioid)
int
create_decomposition_3d(int ntasks, int my_rank, int iosysid, int *ioid)
{
PIO_Offset elements_per_pe; /* Array elements per processing unit. */
PIO_Offset *compdof; /* The decomposition mapping. */
Expand Down Expand Up @@ -115,9 +180,10 @@ int create_decomposition_3d(int ntasks, int my_rank, int iosysid, int *ioid)
* @param rearranger the rearranger in use.
* @returns 0 for success, error code otherwise.
*/
int test_darray(int iosysid, int ioid, int num_flavors, int *flavor,
int my_rank, int ntasks, int num_io_procs, int provide_fill,
int rearranger)
int
test_darray(int iosysid, int ioid, int num_flavors, int *flavor,
int my_rank, int ntasks, int num_io_procs, int provide_fill,
int rearranger)
{
char filename[PIO_MAX_NAME + 1]; /* Name for the output files. */
int dimids[NDIM]; /* The dimension IDs. */
Expand Down Expand Up @@ -147,6 +213,11 @@ int test_darray(int iosysid, int ioid, int num_flavors, int *flavor,
float delta_in_sec;
float mb_per_sec;

#ifdef USE_MPE
if ((ret = MPE_Log_event(event_num[START][CREATE], 0, "start init")))
return ERR_MPI;
#endif /* USE_MPE */

/* Create the filename. Use the same filename for all, so we
* don't waste disk space. */
/* sprintf(filename, "data_%s_iotype_%d_rearr_%d.nc", TEST_NAME, flavor[fmt], */
Expand Down Expand Up @@ -174,6 +245,11 @@ int test_darray(int iosysid, int ioid, int num_flavors, int *flavor,
if ((ret = PIOc_enddef(ncid)))
ERR(ret);

#ifdef USE_MPE
if ((ret = MPE_Log_event(event_num[END][CREATE], 0, "end init")))
MPIERR(ret);
#endif /* USE_MPE */

/* Start the clock. */
gettimeofday(&starttime, NULL);

Expand All @@ -183,6 +259,11 @@ int test_darray(int iosysid, int ioid, int num_flavors, int *flavor,
for (int f = 0; f < arraylen; f++)
test_data[f] = (my_rank * 10 + f) + t * 1000;

#ifdef USE_MPE
if ((ret = MPE_Log_event(event_num[START][DARRAY_WRITE], 0, "start init")))
return ERR_MPI;
#endif /* USE_MPE */

/* Set the value of the record dimension. */
if ((ret = PIOc_setframe(ncid, varid, t)))
ERR(ret);
Expand All @@ -191,13 +272,29 @@ int test_darray(int iosysid, int ioid, int num_flavors, int *flavor,
if ((ret = PIOc_write_darray(ncid, varid, ioid, arraylen, test_data, fillvalue)))
ERR(ret);

#ifdef USE_MPE
if ((ret = MPE_Log_event(event_num[END][DARRAY_WRITE], 0, "end init")))
MPIERR(ret);
#endif /* USE_MPE */

num_megabytes += (X_DIM_LEN * Y_DIM_LEN * Z_DIM_LEN * sizeof(int))/(1024*1024);
}

#ifdef USE_MPE
if ((ret = MPE_Log_event(event_num[START][CLOSE], 0, "start init")))
return ERR_MPI;
#endif /* USE_MPE */

/* Close the netCDF file. */
if ((ret = PIOc_closefile(ncid)))
ERR(ret);

#ifdef USE_MPE
if ((ret = MPE_Log_event(event_num[END][CLOSE], 0, "end init")))
MPIERR(ret);
#endif /* USE_MPE */


/* Stop the clock. */
gettimeofday(&endtime, NULL);

Expand Down Expand Up @@ -319,9 +416,10 @@ test_decomp_read_write(int iosysid, int ioid, int num_flavors, int *flavor,
* @param test_comm the communicator the test is running on.
* @returns 0 for success, error code otherwise.
*/
int test_all_darray(int iosysid, int num_flavors, int *flavor, int my_rank,
int ntasks, int num_io_procs, int rearranger,
MPI_Comm test_comm)
int
test_all_darray(int iosysid, int num_flavors, int *flavor, int my_rank,
int ntasks, int num_io_procs, int rearranger,
MPI_Comm test_comm)
{
int ioid;
int my_test_size;
Expand All @@ -330,6 +428,11 @@ int test_all_darray(int iosysid, int num_flavors, int *flavor, int my_rank,
if ((ret = MPI_Comm_size(test_comm, &my_test_size)))
MPIERR(ret);

#ifdef USE_MPE
if ((ret = MPE_Log_event(event_num[START][DECOMP], 0, "start init")))
return ERR_MPI;
#endif /* USE_MPE */

/* Decompose the data over the tasks. */
if ((ret = create_decomposition_3d(ntasks, my_rank, iosysid, &ioid)))
return ret;
Expand All @@ -339,6 +442,11 @@ int test_all_darray(int iosysid, int num_flavors, int *flavor, int my_rank,
ntasks, rearranger, test_comm)))
return ret;

#ifdef USE_MPE
if ((ret = MPE_Log_event(event_num[END][DECOMP], 0, "end init")))
MPIERR(ret);
#endif /* USE_MPE */

/* Test with/without providing a fill value to PIOc_write_darray(). */
for (int provide_fill = 0; provide_fill < NUM_TEST_CASES_FILLVALUE; provide_fill++)
{
Expand All @@ -348,15 +456,26 @@ int test_all_darray(int iosysid, int num_flavors, int *flavor, int my_rank,
return ret;
}

#ifdef USE_MPE
if ((ret = MPE_Log_event(event_num[START][DECOMP], 0, "start init")))
return ERR_MPI;
#endif /* USE_MPE */

/* Free the PIO decomposition. */
if ((ret = PIOc_freedecomp(iosysid, ioid)))
ERR(ret);

#ifdef USE_MPE
if ((ret = MPE_Log_event(event_num[END][DECOMP], 0, "end init")))
MPIERR(ret);
#endif /* USE_MPE */

return PIO_NOERR;
}

/* Run tests for darray functions. */
int main(int argc, char **argv)
int
main(int argc, char **argv)
{
int my_rank;
int ntasks;
Expand All @@ -377,6 +496,15 @@ int main(int argc, char **argv)
0, -1, &test_comm)))
ERR(ERR_INIT);

#ifdef USE_MPE

if ((ret = MPE_Init_log()))
return ret;
if (init_logging(my_rank, event_num))
return ERR_LOGGING;
#endif /* USE_MPE */


if ((ret = PIOc_set_iosystem_error_handling(PIO_DEFAULT, PIO_RETURN_ERROR, NULL)))
return ret;

Expand Down Expand Up @@ -404,12 +532,22 @@ int main(int argc, char **argv)
/* for (r = 0; r < NUM_REARRANGERS_TO_TEST; r++) */
for (r = 0; r < 1; r++)
{
#ifdef USE_MPE
if ((ret = MPE_Log_event(event_num[START][INIT], 0, "start init")))
return ERR_MPI;
#endif /* USE_MPE */

/* Initialize the PIO IO system. This specifies how
* many and which processors are involved in I/O. */
if ((ret = PIOc_Init_Intracomm(test_comm, num_io_procs[i], ioproc_stride,
ioproc_start, rearranger[r], &iosysid)))
return ret;

#ifdef USE_MPE
if ((ret = MPE_Log_event(event_num[END][INIT], 0, "end init")))
MPIERR(ret);
#endif /* USE_MPE */

/* Run tests. */
if ((ret = test_all_darray(iosysid, num_flavors, flavor, my_rank,
ntasks, num_io_procs[i], rearranger[r], test_comm)))
Expand All @@ -418,10 +556,23 @@ int main(int argc, char **argv)
/* Finalize PIO system. */
if ((ret = PIOc_free_iosystem(iosysid)))
return ret;

} /* next rearranger */
} /* next num io procs */


#ifdef USE_MPE
{
/* This causes problems on my MPICH2 library on Linux, but seems to be
* required for frost. */
char file_name[128];
sprintf(file_name, "chart_%d", 1);
if ((ret = MPE_Finish_log(file_name)))
MPIERR(ret);
}

#endif /* USE_MPE */


if (!my_rank)
printf("finalizing io_test!\n");

Expand Down

0 comments on commit 2af631f

Please sign in to comment.