Skip to content

Commit

Permalink
Minor change to support running in MPI environment
Browse files Browse the repository at this point in the history
when MPI is not used by HPX or the APEX configuration.  This happens
when HPX is configured without a parcel port, and APEX thinks all
ranks are 0.  This change adds a check for MPI environment variables
to validate the MPI rank that was passed in.
  • Loading branch information
khuck committed Jul 23, 2020
1 parent 254b698 commit f459c9b
Show file tree
Hide file tree
Showing 3 changed files with 49 additions and 0 deletions.
2 changes: 2 additions & 0 deletions src/apex/apex.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -350,6 +350,8 @@ uint64_t init(const char * thread_name, uint64_t comm_rank,
// FIRST! make sure APEX thinks this is a worker thread (the main thread
// is always a worker thread)
thread_instance::instance(true);
comm_rank = test_for_MPI_comm_rank(comm_rank);
comm_size = test_for_MPI_comm_size(comm_size);
// protect against multiple initializations
#ifdef APEX_WITH_JUPYTER_SUPPORT
if (_registered || _initialized) {
Expand Down
44 changes: 44 additions & 0 deletions src/apex/utils.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -427,5 +427,49 @@ uint32_t simple_reverse(uint32_t x)
return x;
}

uint64_t test_for_MPI_comm_rank(uint64_t commrank) {
/* Some configurations might use MPI without telling APEX - they can
* call apex::init() with a rank of 0 and size of 1 even though
* they are running in an MPI application. For that reason, we double
* check to make sure that we aren't in an MPI execution by checking
* for some common environment variables. */
// OpenMPI, Spectrum
const char * tmpvar = getenv("OMPI_COMM_WORLD_RANK");
if (tmpvar != NULL) {
commrank = atol(tmpvar);
// printf("Changing openMPI rank to %lu\n", commrank);
return commrank;
}
// Slurm
tmpvar = getenv("SLURM_PROCID");
if (tmpvar != NULL) {
commrank = atol(tmpvar);
return commrank;
}
return commrank;
}

uint64_t test_for_MPI_comm_size(uint64_t commsize) {
/* Some configurations might use MPI without telling APEX - they can
* call apex::init() with a rank of 0 and size of 1 even though
* they are running in an MPI application. For that reason, we double
* check to make sure that we aren't in an MPI execution by checking
* for some common environment variables. */
// OpenMPI, Spectrum
const char * tmpvar = getenv("OMPI_COMM_WORLD_SIZE");
if (tmpvar != NULL) {
commsize = atol(tmpvar);
// printf("Changing openMPI size to %lu\n", commsize);
return commsize;
}
// Slurm
tmpvar = getenv("SLURM_NNODES");
if (tmpvar != NULL) {
commsize = atol(tmpvar);
return commsize;
}
return commsize;
}

} // namespace apex

3 changes: 3 additions & 0 deletions src/apex/utils.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -241,5 +241,8 @@ inline char filesystem_separator()
#endif
}

uint64_t test_for_MPI_comm_rank(uint64_t commrank);
uint64_t test_for_MPI_comm_size(uint64_t commsize);

}

0 comments on commit f459c9b

Please sign in to comment.