diff --git a/src/externals/pio2/CMakeLists.txt b/src/externals/pio2/CMakeLists.txt index 6677ab46eb0..514384a5a9e 100644 --- a/src/externals/pio2/CMakeLists.txt +++ b/src/externals/pio2/CMakeLists.txt @@ -46,29 +46,6 @@ else() set(USE_MPI_SERIAL 0) endif() -#===== Library Variables ===== -set (PIO_FILESYSTEM_HINTS IGNORE CACHE STRING "Filesystem hints (lustre or gpfs)") - -#===== Testing Options ===== -option (PIO_ENABLE_TESTS "Enable the testing builds" ON) -option (PIO_VALGRIND_CHECK "Enable memory leak check using valgrind" OFF) - -#============================================================================== -# BACKWARDS COMPATIBILITY -#============================================================================== - -# Old NETCDF_DIR variable --> NetCDF_PATH -if (DEFINED NETCDF_DIR) - set (NetCDF_PATH ${NETCDF_DIR} - CACHE STRING "Location of the NetCDF library installation") -endif () - -# Old PNETCDF_DIR variable --> PnetCDF_PATH -if (DEFINED PNETCDF_DIR) - set (PnetCDF_PATH ${PNETCDF_DIR} - CACHE STRING "Location of the PnetCDF library installation") -endif () - #============================================================================== # PREPEND TO CMAKE MODULE PATH #============================================================================== @@ -97,6 +74,46 @@ set (USER_CMAKE_MODULE_PATH ${USER_CMAKE_MODULE_PATH} CACHE STRING "Location of the CMake_Fortran_utils") list (APPEND CMAKE_MODULE_PATH ${USER_CMAKE_MODULE_PATH}) +INCLUDE (CheckTypeSize) + +#===== MPI ===== +if (PIO_USE_MPISERIAL) + find_package (MPISERIAL COMPONENTS C REQUIRED) + if (MPISERIAL_C_FOUND) + set (CMAKE_REQUIRED_INCLUDES ${MPISERIAL_C_INCLUDE_DIRS}) + endif () +else () + find_package (MPI REQUIRED) + set (CMAKE_REQUIRED_INCLUDES ${MPI_INCLUDE_PATH}) +endif () + +SET(CMAKE_EXTRA_INCLUDE_FILES "mpi.h") +check_type_size("MPI_Offset" SIZEOF_MPI_OFFSET) +SET(CMAKE_EXTRA_INCLUDE_FILES) + +#===== Library Variables ===== +set (PIO_FILESYSTEM_HINTS IGNORE CACHE STRING "Filesystem hints (lustre or gpfs)") + +#===== Testing Options ===== +option (PIO_ENABLE_TESTS "Enable the testing builds" ON) +option (PIO_VALGRIND_CHECK "Enable memory leak check using valgrind" OFF) + +#============================================================================== +# BACKWARDS COMPATIBILITY +#============================================================================== + +# Old NETCDF_DIR variable --> NetCDF_PATH +if (DEFINED NETCDF_DIR) + set (NetCDF_PATH ${NETCDF_DIR} + CACHE STRING "Location of the NetCDF library installation") +endif () + +# Old PNETCDF_DIR variable --> PnetCDF_PATH +if (DEFINED PNETCDF_DIR) + set (PnetCDF_PATH ${PNETCDF_DIR} + CACHE STRING "Location of the PnetCDF library installation") +endif () + #============================================================================== # HELPFUL GLOBAL VARIABLES #============================================================================== diff --git a/src/externals/pio2/CTestScript.cmake b/src/externals/pio2/CTestScript.cmake index 1d0de3ccdcd..e817f422d15 100644 --- a/src/externals/pio2/CTestScript.cmake +++ b/src/externals/pio2/CTestScript.cmake @@ -18,7 +18,7 @@ else () set (CTEST_DASHBOARD_ROOT "$ENV{HOME}/pio-dashboard") endif () -## -- Compiler ID +## -- Compiler ID if (DEFINED ENV{PIO_COMPILER_ID}) set (compid "$ENV{PIO_COMPILER_ID}") else () @@ -49,7 +49,8 @@ if (HOSTNAME MATCHES "^yslogin" OR HOSTNAME MATCHES "^pronghorn") set (HOSTNAME_ID "nwsc") # New UCAR/NWSC SGI Machines -elseif (HOSTNAME MATCHES "^laramie") +elseif (HOSTNAME MATCHES "^laramie" OR + HOSTNAME MATCHES "^chadmin") set (HOSTNAME_ID "nwscla") # ALCF/Argonne Machines elseif (HOSTNAME MATCHES "^mira" OR @@ -109,9 +110,9 @@ find_program (CTEST_GIT_COMMAND NAMES git) ## -- make command find_program (MAKE NAMES make) -#----------------------------------------------------------- +#----------------------------------------------------------- #-- Generate build-specific information -#----------------------------------------------------------- +#----------------------------------------------------------- ## -- CTest Site Name @@ -124,25 +125,25 @@ set (CTEST_BUILD_NAME "${osname}-${osrel}-${cpu}-${compid}") ## -- SRC Dir (where this script exists) set (CTEST_SOURCE_DIRECTORY "${CTEST_SCRIPT_DIRECTORY}") -## -- BIN Dir +## -- BIN Dir set (CTEST_BINARY_DIRECTORY "${CTEST_DASHBOARD_ROOT}/build-${CTEST_BUILD_NAME}-${CTEST_BUILD_GROUP}") ## -- Add the CTest script directory to the module path set (CTEST_EXTRA_SCRIPT_PATH "${CTEST_SOURCE_DIRECTORY}/ctest") list (APPEND CMAKE_MODULE_PATH ${CTEST_EXTRA_SCRIPT_PATH}) -# ----------------------------------------------------------- +# ----------------------------------------------------------- # -- Store Build-Specific Info (environment variables) -# ----------------------------------------------------------- +# ----------------------------------------------------------- set (ENV{PIO_DASHBOARD_SITE} ${CTEST_SITE}) set (ENV{PIO_DASHBOARD_BUILD_NAME} ${CTEST_BUILD_NAME}) set (ENV{PIO_DASHBOARD_SOURCE_DIR} ${CTEST_SOURCE_DIRECTORY}) set (ENV{PIO_DASHBOARD_BINARY_DIR} ${CTEST_BINARY_DIRECTORY}) -# ----------------------------------------------------------- +# ----------------------------------------------------------- # -- Run CTest -# ----------------------------------------------------------- +# ----------------------------------------------------------- ## -- Empty the binary directory ctest_empty_binary_directory(${CTEST_BINARY_DIRECTORY}) @@ -157,7 +158,7 @@ message (" -- Update source - ${CTEST_BUILD_NAME} --") set (CTEST_UPDATE_COMMAND "${CTEST_GIT_COMMAND}") ctest_update () -## -- Configure +## -- Configure message (" -- Configure build - ${CTEST_BUILD_NAME} --") include (CTestEnvironment-${HOSTNAME_ID}) set (CTEST_CONFIGURE_COMMAND "${CMAKE_COMMAND} ${CTEST_CONFIGURE_OPTIONS} ${CTEST_SOURCE_DIRECTORY}") @@ -179,9 +180,9 @@ message (" -- Submit to dashboard - ${CTEST_BUILD_NAME} --") message ("** -- PIO_DASHBOARD_SITE=$ENV{PIO_DASHBOARD_SITE}") ctest_submit () -# ----------------------------------------------------------- +# ----------------------------------------------------------- # -- Clear environment -# ----------------------------------------------------------- +# ----------------------------------------------------------- unset (ENV{PIO_DASHBOARD_SITE}) unset (ENV{PIO_DASHBOARD_BUILD_NAME}) diff --git a/src/externals/pio2/ctest/CTestEnvironment-anlworkstation.cmake b/src/externals/pio2/ctest/CTestEnvironment-anlworkstation.cmake index 07ba92a2a72..ddf04f063a6 100644 --- a/src/externals/pio2/ctest/CTestEnvironment-anlworkstation.cmake +++ b/src/externals/pio2/ctest/CTestEnvironment-anlworkstation.cmake @@ -23,3 +23,8 @@ endif () if (DEFINED ENV{VALGRIND_CHECK}) set (CTEST_CONFIGURE_OPTIONS "${CTEST_CONFIGURE_OPTIONS} -DPIO_VALGRIND_CHECK=ON") endif () + +# If USE_MALLOC environment variable is set, then use native malloc (instead of bget package) +if (DEFINED ENV{USE_MALLOC}) + set (CTEST_CONFIGURE_OPTIONS "${CTEST_CONFIGURE_OPTIONS} -DPIO_USE_MALLOC=ON") +endif () diff --git a/src/externals/pio2/ctest/CTestEnvironment-nwsc.cmake b/src/externals/pio2/ctest/CTestEnvironment-nwsc.cmake index 356390f933d..4a0d6fd3acd 100644 --- a/src/externals/pio2/ctest/CTestEnvironment-nwsc.cmake +++ b/src/externals/pio2/ctest/CTestEnvironment-nwsc.cmake @@ -14,5 +14,5 @@ set (CTEST_CONFIGURE_OPTIONS "-DCMAKE_VERBOSE_MAKEFILE=TRUE -DPIO_ENABLE_DOC=OFF # If MPISERIAL environment variable is set, then enable MPISERIAL if (DEFINED ENV{MPISERIAL}) - set (CTEST_CONFIGURE_OPTIONS "${CTEST_CONFIGURE_OPTIONS} -DPIO_USE_MPISERIAL=ON") + set (CTEST_CONFIGURE_OPTIONS "${CTEST_CONFIGURE_OPTIONS} -DPIO_USE_MPISERIAL=ON -DPIO_ENABLE_EXAMPLES=OFF ") endif () diff --git a/src/externals/pio2/ctest/runcdash-cgd-gnu-openmpi.sh b/src/externals/pio2/ctest/runcdash-cgd-gnu-openmpi.sh new file mode 100755 index 00000000000..56407d9c425 --- /dev/null +++ b/src/externals/pio2/ctest/runcdash-cgd-gnu-openmpi.sh @@ -0,0 +1,38 @@ +#!/bin/sh + +# Get/Generate the Dashboard Model +if [ $# -eq 0 ]; then + model=Experimental +else + model=$1 +fi + +module purge +module load compiler/gnu/5.4.0 +module load tool/parallel-netcdf/1.8.1/gnu-5.4.0/openmpi + +export CC=mpicc +export FC=mpif90 +export PIO_DASHBOARD_SITE="cgd" +export PIO_DASHBOARD_ROOT=/scratch/cluster/jedwards/dashboard +export CTEST_SCRIPT_DIRECTORY=${PIO_DASHBOARD_ROOT}/src +export PIO_DASHBOARD_SOURCE_DIR=${CTEST_SCRIPT_DIRECTORY} +export PIO_COMPILER_ID=gcc-`gcc --version | head -n 1 | cut -d' ' -f3` + +if [ ! -d "$PIO_DASHBOARD_ROOT" ]; then + mkdir "$PIO_DASHBOARD_ROOT" +fi +cd "$PIO_DASHBOARD_ROOT" + +echo "CTEST_SCRIPT_DIRECTORY="${CTEST_SCRIPT_DIRECTORY} +echo "PIO_DASHBOARD_SOURCE_DIR="${PIO_DASHBOARD_SOURCE_DIR} + +if [ ! -d src ]; then + git clone --branch develop https://github.com/PARALLELIO/ParallelIO src +fi +cd src +git checkout develop +git pull origin develop + + +ctest -S CTestScript.cmake,${model} -VV diff --git a/src/externals/pio2/ctest/runcdash-cgd-nag.sh b/src/externals/pio2/ctest/runcdash-cgd-nag.sh index 55c477b73bc..e413186131a 100755 --- a/src/externals/pio2/ctest/runcdash-cgd-nag.sh +++ b/src/externals/pio2/ctest/runcdash-cgd-nag.sh @@ -14,7 +14,7 @@ module load tool/parallel-netcdf/1.7.0/nag/mvapich2 export CC=mpicc export FC=mpif90 export PIO_DASHBOARD_SITE="cgd" -export PIO_DASHBOARD_ROOT=/scratch/cluster/katec/dashboard +export PIO_DASHBOARD_ROOT=/scratch/cluster/jedwards/dashboard export CTEST_SCRIPT_DIRECTORY=${PIO_DASHBOARD_ROOT}/src export PIO_DASHBOARD_SOURCE_DIR=${CTEST_SCRIPT_DIRECTORY} export PIO_COMPILER_ID=Nag-6.1-gcc-`gcc --version | head -n 1 | cut -d' ' -f3` diff --git a/src/externals/pio2/ctest/runcdash-nwsc-intel-mpiserial.sh b/src/externals/pio2/ctest/runcdash-nwsc-intel-mpiserial.sh index 702427e5a7b..68ac5826be9 100755 --- a/src/externals/pio2/ctest/runcdash-nwsc-intel-mpiserial.sh +++ b/src/externals/pio2/ctest/runcdash-nwsc-intel-mpiserial.sh @@ -14,7 +14,7 @@ module load git/2.3.0 module load cmake/3.0.2 module load netcdf/4.3.3.1 -export MPISERIAL=/glade/p/work/katec/installs/intel_15.0.3 +export MPISERIAL=/glade/u/home/jedwards/mpi-serial/intel15.0.3/ export CC=icc export FC=ifort diff --git a/src/externals/pio2/ctest/runcdash-nwscla-intel.sh b/src/externals/pio2/ctest/runcdash-nwscla-intel.sh index 3812b9d61f0..630ac838085 100755 --- a/src/externals/pio2/ctest/runcdash-nwscla-intel.sh +++ b/src/externals/pio2/ctest/runcdash-nwscla-intel.sh @@ -11,18 +11,18 @@ source /etc/profile.d/modules.sh module reset module unload netcdf -module swap intel intel/16.0.3 -module load git/2.10.0 -module load cmake/3.6.2 -module load netcdf-mpi/4.4.1 +module swap intel intel/17.0.1 +module load cmake/3.7.2 +module load netcdf-mpi/4.4.1.1 module load pnetcdf/1.8.0 +module switch mpt mpt/2.15 echo "MODULE LIST..." module list export CC=mpicc export FC=mpif90 -export PIO_DASHBOARD_ROOT=`pwd`/dashboard +export PIO_DASHBOARD_ROOT=/glade/scratch/jedwards/dashboard export PIO_COMPILER_ID=Intel-`$CC --version | head -n 1 | cut -d' ' -f3` if [ ! -d "$PIO_DASHBOARD_ROOT" ]; then diff --git a/src/externals/pio2/ctest/runctest-cgd.sh b/src/externals/pio2/ctest/runctest-cgd.sh index fccf3396d3b..bbd31ccf5d0 100755 --- a/src/externals/pio2/ctest/runctest-cgd.sh +++ b/src/externals/pio2/ctest/runctest-cgd.sh @@ -29,7 +29,7 @@ echo "\$CTESTCMD -S ${scrdir}/CTestScript-Test.cmake,${model} -V" >> runctest.sh chmod +x runctest.sh # Submit the job to the queue -jobid=`/usr/local/bin/qsub -l nodes=1:ppn=4 runctest.sh -q short` +jobid=`/usr/local/bin/qsub -l nodes=1:ppn=8 runctest.sh -q short` # Wait for the job to complete before exiting while true; do diff --git a/src/externals/pio2/ctest/runctest-nwscla.sh b/src/externals/pio2/ctest/runctest-nwscla.sh index 6b6f4d87f7c..d3e252317d0 100755 --- a/src/externals/pio2/ctest/runctest-nwscla.sh +++ b/src/externals/pio2/ctest/runctest-nwscla.sh @@ -1,8 +1,8 @@ #!/bin/sh #============================================================================== # -# This script defines how to run CTest on the NCAR Wyoming Supercomputing -# Center systems (yellowstone/caldera/geyser). +# This script defines how to run CTest on the NCAR Wyoming Supercomputing +# Center systems (cheyenne/laramie). # # This assumes the CTest model name (e.g., "Nightly") is passed to it when # run. @@ -19,7 +19,8 @@ model=$2 echo "#!/bin/sh" > runctest.sh echo "#PBS -l walltime=01:00:00" >> runctest.sh echo "#PBS -l select=1:ncpus=8:mpiprocs=8" >> runctest.sh -echo "#PBS -A SCSG0002" >> runctest.sh +echo "#PBS -A P93300606" >> runctest.sh +echo "#PBS -q regular" >> runctest.sh echo "export PIO_DASHBOARD_SITE=nwscla-${HOSTNAME}" >> runctest.sh echo "CTESTCMD=`which ctest`" >> runctest.sh echo "\$CTESTCMD -S ${scrdir}/CTestScript-Test.cmake,${model} -V" >> runctest.sh @@ -28,18 +29,16 @@ echo "\$CTESTCMD -S ${scrdir}/CTestScript-Test.cmake,${model} -V" >> runctest.sh chmod +x runctest.sh # Submit the job to the queue -jobid=`qsub runctest.sh` +jobid=`qsub -l walltime=01:00:00 runctest.sh` # Wait for the job to complete before exiting while true; do - status=`qstat $jobid` - echo $status - if [ "$status" == "" ]; then - break + qstat $jobid + if [ $? -eq 0 ]; then + sleep 30 else - sleep 10 + break; fi done exit 0 - diff --git a/src/externals/pio2/doc/source/contributing_code.txt b/src/externals/pio2/doc/source/contributing_code.txt index 6dbd7089826..c811777a02b 100644 --- a/src/externals/pio2/doc/source/contributing_code.txt +++ b/src/externals/pio2/doc/source/contributing_code.txt @@ -13,7 +13,7 @@ * Documents produced by Doxygen are derivative works derived from the * input used in their production; they are not affected by this license. * - */ /*! \page code_style Code Style for Contributors + */ /*! \page contributing_code Guide for Contributors # Introduction # diff --git a/src/externals/pio2/examples/c/CMakeLists.txt b/src/externals/pio2/examples/c/CMakeLists.txt index 74b4837c7f1..e35562b11cc 100644 --- a/src/externals/pio2/examples/c/CMakeLists.txt +++ b/src/externals/pio2/examples/c/CMakeLists.txt @@ -33,12 +33,22 @@ ADD_EXECUTABLE(example1 example1.c) TARGET_LINK_LIBRARIES(example1 pioc) add_dependencies(tests example1) +ADD_EXECUTABLE(darray_no_async darray_no_async.c) +TARGET_LINK_LIBRARIES(darray_no_async pioc) +add_dependencies(tests darray_no_async) + +ADD_EXECUTABLE(darray_async darray_async.c) +TARGET_LINK_LIBRARIES(darray_async pioc) +add_dependencies(tests darray_async) + if (PIO_USE_MPISERIAL) add_test(NAME examplePio COMMAND examplePio) add_test(NAME example1 COMMAND example1) else () add_mpi_test(examplePio EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/examplePio NUMPROCS 4 TIMEOUT 60) add_mpi_test(example1 EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/example1 NUMPROCS 4 TIMEOUT 60) + #add_mpi_test(darray_async EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/darray_async NUMPROCS 5 TIMEOUT 60) + add_mpi_test(darray_no_async EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/darray_no_async NUMPROCS 4 TIMEOUT 60) endif () diff --git a/src/externals/pio2/examples/c/darray_async.c b/src/externals/pio2/examples/c/darray_async.c new file mode 100644 index 00000000000..1b15607f0f6 --- /dev/null +++ b/src/externals/pio2/examples/c/darray_async.c @@ -0,0 +1,387 @@ +/* + * @file + * @brief A simple C example for the ParallelIO Library. + * + * This example creates a netCDF output file with three dimensions + * (one unlimited) and one variable. It first writes, then reads the + * sample file using distributed arrays. + * + * This example can be run in parallel for 4 processors. + */ + +#include +#include +#include +#include +#include +#include +#include +#ifdef TIMING +#include +#endif + +/* The name of this program. */ +#define TEST_NAME "darray_async" + +/* The number of possible output netCDF output flavors available to + * the ParallelIO library. */ +#define NUM_NETCDF_FLAVORS 4 + +/* The number of dimensions in the example data. */ +#define NDIM3 3 + +/* The number of timesteps of data. */ +#define NUM_TIMESTEPS 2 + +/* The length of our sample data in X dimension.*/ +#define DIM_LEN_X 4 + +/* The length of our sample data in Y dimension.*/ +#define DIM_LEN_Y 4 + +/* The name of the variable in the netCDF output file. */ +#define VAR_NAME "foo" + +/* Return code when netCDF output file does not match + * expectations. */ +#define ERR_BAD 1001 + +/* The meaning of life, the universe, and everything. */ +#define START_DATA_VAL 42 + +/* Number of tasks this example runs on. */ +#define TARGET_NTASKS 5 + +/* Logging level. */ +#define LOG_LEVEL 3 + +/* Number of IO processors. */ +#define NUM_IO_TASKS 1 + +/* Number of computation processors. */ +#define NUM_COMP_TASKS 4 + +/* Number of computation components. */ +#define COMPONENT_COUNT 1 + +/* Lengths of dimensions. */ +int dim_len[NDIM3] = {NC_UNLIMITED, DIM_LEN_X, DIM_LEN_Y}; + +/* Names of dimensions. */ +char dim_name[NDIM3][PIO_MAX_NAME + 1] = {"unlimted", "x", "y"}; + +/* Handle MPI errors. This should only be used with MPI library + * function calls. */ +#define MPIERR(e) do { \ + MPI_Error_string(e, err_buffer, &resultlen); \ + printf("MPI error, line %d, file %s: %s\n", __LINE__, __FILE__, err_buffer); \ + MPI_Finalize(); \ + return 2; \ + } while (0) + +/* Handle non-MPI errors by finalizing the MPI library and exiting + * with an exit code. */ +#define ERR(e) do { \ + MPI_Finalize(); \ + return e; \ + } while (0) + +/* Global err buffer for MPI. When there is an MPI error, this buffer + * is used to store the error message that is associated with the MPI + * error. */ +char err_buffer[MPI_MAX_ERROR_STRING]; + +/* This is the length of the most recent MPI error message, stored + * int the global error string. */ +int resultlen; + +/* @brief Check the output file. + * + * Use netCDF to check that the output is as expected. + * + * @param ntasks The number of processors running the example. + * @param filename The name of the example file to check. + * + * @return 0 if example file is correct, non-zero otherwise. */ +/* int check_file(int iosysid, int ntasks, char *filename, int iotype, */ +/* int elements_per_pe, int my_rank, int ioid) */ +/* { */ + +/* int ncid; /\* File ID from netCDF. *\/ */ +/* int ndims; /\* Number of dimensions. *\/ */ +/* int nvars; /\* Number of variables. *\/ */ +/* int ngatts; /\* Number of global attributes. *\/ */ +/* int unlimdimid; /\* ID of unlimited dimension. *\/ */ +/* int natts; /\* Number of variable attributes. *\/ */ +/* nc_type xtype; /\* NetCDF data type of this variable. *\/ */ +/* int ret; /\* Return code for function calls. *\/ */ +/* int dimids[NDIM3]; /\* Dimension ids for this variable. *\/ */ +/* char var_name[NC_MAX_NAME]; /\* Name of the variable. *\/ */ +/* /\* size_t start[NDIM3]; /\\* Zero-based index to start read. *\\/ *\/ */ +/* /\* size_t count[NDIM3]; /\\* Number of elements to read. *\\/ *\/ */ +/* /\* int buffer[DIM_LEN_X]; /\\* Buffer to read in data. *\\/ *\/ */ +/* /\* int expected[DIM_LEN_X]; /\\* Data values we expect to find. *\\/ *\/ */ + +/* /\* Open the file. *\/ */ +/* if ((ret = PIOc_openfile_retry(iosysid, &ncid, &iotype, filename, 0, 0))) */ +/* return ret; */ +/* printf("opened file %s ncid = %d\n", filename, ncid); */ + +/* /\* Check the metadata. *\/ */ +/* if ((ret = PIOc_inq(ncid, &ndims, &nvars, &ngatts, &unlimdimid))) */ +/* return ret; */ + +/* /\* Check the dimensions. *\/ */ +/* if (ndims != NDIM3 || nvars != 1 || ngatts != 0 || unlimdimid != 0) */ +/* return ERR_BAD; */ +/* for (int d = 0; d < NDIM3; d++) */ +/* { */ +/* char my_dim_name[NC_MAX_NAME]; */ +/* PIO_Offset dimlen; */ + +/* if ((ret = PIOc_inq_dim(ncid, d, my_dim_name, &dimlen))) */ +/* return ret; */ +/* if (dimlen != (d ? dim_len[d] : NUM_TIMESTEPS) || strcmp(my_dim_name, dim_name[d])) */ +/* return ERR_BAD; */ +/* } */ + +/* /\* Check the variable. *\/ */ +/* if ((ret = PIOc_inq_var(ncid, 0, var_name, &xtype, &ndims, dimids, &natts))) */ +/* return ret; */ +/* if (xtype != NC_INT || ndims != NDIM3 || dimids[0] != 0 || dimids[1] != 1 || */ +/* dimids[2] != 2 || natts != 0) */ +/* return ERR_BAD; */ + +/* /\* Allocate storage for sample data. *\/ */ +/* int buffer[elements_per_pe]; */ +/* int buffer_in[elements_per_pe]; */ + +/* /\* Check each timestep. *\/ */ +/* for (int t = 0; t < NUM_TIMESTEPS; t++) */ +/* { */ +/* int varid = 0; /\* There's only one var in sample file. *\/ */ + +/* /\* This is the data we expect for this timestep. *\/ */ +/* for (int i = 0; i < elements_per_pe; i++) */ +/* buffer[i] = 100 * t + START_DATA_VAL + my_rank; */ + +/* /\* Read one record. *\/ */ +/* if ((ret = PIOc_setframe(ncid, varid, t))) */ +/* ERR(ret); */ +/* if ((ret = PIOc_read_darray(ncid, varid, ioid, elements_per_pe, buffer_in))) */ +/* return ret; */ + +/* /\* Check the results. *\/ */ +/* for (int i = 0; i < elements_per_pe; i++) */ +/* if (buffer_in[i] != buffer[i]) */ +/* return ERR_BAD; */ +/* } */ + +/* /\* Close the file. *\/ */ +/* if ((ret = PIOc_closefile(ncid))) */ +/* return ret; */ + +/* /\* Everything looks good! *\/ */ +/* return 0; */ +/* } */ + +/* Write, then read, a simple example with darrays. + + The sample file created by this program is a small netCDF file. It + has the following contents (as shown by ncdump): + + 
+netcdf darray_no_async_iotype_1 {
+dimensions:
+	unlimted = UNLIMITED ; // (2 currently)
+	x = 4 ;
+	y = 4 ;
+variables:
+	int foo(unlimted, x, y) ;
+data:
+
+ foo =
+  42, 42, 42, 42,
+  43, 43, 43, 43,
+  44, 44, 44, 44,
+  45, 45, 45, 45,
+  142, 142, 142, 142,
+  143, 143, 143, 143,
+  144, 144, 144, 144,
+  145, 145, 145, 145 ;
+}
+
+ +*/ + int main(int argc, char* argv[]) + { + int my_rank; /* Zero-based rank of processor. */ + int ntasks; /* Number of processors involved in current execution. */ + int iosysid; /* The ID for the parallel I/O system. */ + /* int ncid; /\* The ncid of the netCDF file. *\/ */ + /* int dimid[NDIM3]; /\* The dimension ID. *\/ */ + /* int varid; /\* The ID of the netCDF varable. *\/ */ + /* char filename[NC_MAX_NAME + 1]; /\* Test filename. *\/ */ + /* int num_flavors = 0; /\* Number of iotypes available in this build. *\/ */ + /* int format[NUM_NETCDF_FLAVORS]; /\* Different output flavors. *\/ */ + int ret; /* Return value. */ + +#ifdef TIMING + /* Initialize the GPTL timing library. */ + if ((ret = GPTLinitialize ())) + return ret; +#endif + + /* Initialize MPI. */ + if ((ret = MPI_Init(&argc, &argv))) + MPIERR(ret); + if ((ret = MPI_Comm_set_errhandler(MPI_COMM_WORLD, MPI_ERRORS_RETURN))) + MPIERR(ret); + + /* Learn my rank and the total number of processors. */ + if ((ret = MPI_Comm_rank(MPI_COMM_WORLD, &my_rank))) + MPIERR(ret); + if ((ret = MPI_Comm_size(MPI_COMM_WORLD, &ntasks))) + MPIERR(ret); + + /* Check that a valid number of processors was specified. */ + printf("%d: ParallelIO Library darray_async example running on %d processors.\n", + my_rank, ntasks); + if (ntasks != TARGET_NTASKS) + { + fprintf(stderr, "Number of processors must be %d!\n", TARGET_NTASKS); + return ERR_BAD; + } + + /* Turn on logging. */ + if ((ret = PIOc_set_log_level(LOG_LEVEL))) + return ret; + + /* Num procs for computation. */ + int num_procs2[COMPONENT_COUNT] = {4}; + + /* Is the current process a computation task? */ + int comp_task = my_rank < NUM_IO_TASKS ? 0 : 1; + + /* Initialize the IO system. */ + if ((ret = PIOc_init_async(MPI_COMM_WORLD, NUM_IO_TASKS, NULL, COMPONENT_COUNT, + num_procs2, NULL, NULL, NULL, PIO_REARR_BOX, &iosysid))) + ERR(ret); + + + /* The rest of the code executes on computation tasks only. As + * PIO functions are called on the computation tasks, the + * async system will call them on the IO task. When the + * computation tasks call PIO_finalize(), the IO task will get + * a message to shut itself down. */ + if (comp_task) + { + /* PIO_Offset elements_per_pe; /\* Array elements per processing unit. *\/ */ + /* int ioid; /\* The I/O description ID. *\/ */ + + /* /\* How many elements on each computation task? *\/ */ + /* elements_per_pe = DIM_LEN_X * DIM_LEN_Y / NUM_COMP_TASKS; */ + + /* /\* Allocate and initialize array of decomposition mapping. *\/ */ + /* PIO_Offset compdof[elements_per_pe]; */ + /* for (int i = 0; i < elements_per_pe; i++) */ + /* compdof[i] = my_rank * elements_per_pe + i; */ + + /* /\* Create the PIO decomposition for this example. Since */ + /* this is a variable with an unlimited dimension, we want */ + /* to create a 2-D composition which represents one */ + /* record. *\/ */ + /* printf("rank: %d Creating decomposition...\n", my_rank); */ + /* if ((ret = PIOc_init_decomp(iosysid, PIO_INT, NDIM3 - 1, &dim_len[1], elements_per_pe, */ + /* compdof, &ioid, 0, NULL, NULL))) */ + /* ERR(ret); */ + +/* /\* The number of favors may change with the build parameters. *\/ */ +/* #ifdef _PNETCDF */ +/* format[num_flavors++] = PIO_IOTYPE_PNETCDF; */ +/* #endif */ +/* format[num_flavors++] = PIO_IOTYPE_NETCDF; */ +/* #ifdef _NETCDF4 */ +/* format[num_flavors++] = PIO_IOTYPE_NETCDF4C; */ +/* format[num_flavors++] = PIO_IOTYPE_NETCDF4P; */ +/* #endif */ + +/* /\* Use PIO to create the example file in each of the four */ +/* * available ways. *\/ */ +/* for (int fmt = 0; fmt < num_flavors; fmt++) */ +/* { */ +/* /\* Create a filename. *\/ */ +/* sprintf(filename, "darray_no_async_iotype_%d.nc", format[fmt]); */ + +/* /\* Create the netCDF output file. *\/ */ +/* printf("rank: %d Creating sample file %s with format %d...\n", */ +/* my_rank, filename, format[fmt]); */ +/* if ((ret = PIOc_createfile(iosysid, &ncid, &(format[fmt]), filename, PIO_CLOBBER))) */ +/* ERR(ret); */ + +/* /\* Define netCDF dimension and variable. *\/ */ +/* printf("rank: %d Defining netCDF metadata...\n", my_rank); */ +/* for (int d = 0; d < NDIM3; d++) */ +/* if ((ret = PIOc_def_dim(ncid, dim_name[d], dim_len[d], &dimid[d]))) */ +/* ERR(ret); */ +/* if ((ret = PIOc_def_var(ncid, VAR_NAME, PIO_INT, NDIM3, dimid, &varid))) */ +/* ERR(ret); */ +/* if ((ret = PIOc_enddef(ncid))) */ +/* ERR(ret); */ + +/* /\* Allocate storage for sample data. *\/ */ +/* int buffer[elements_per_pe]; */ + +/* /\* Write each timestep. *\/ */ +/* for (int t = 0; t < NUM_TIMESTEPS; t++) */ +/* { */ +/* /\* Create some data for this timestep. *\/ */ +/* for (int i = 0; i < elements_per_pe; i++) */ +/* buffer[i] = 100 * t + START_DATA_VAL + my_rank; */ + +/* /\* Write data to the file. *\/ */ +/* printf("rank: %d Writing sample data...\n", my_rank); */ +/* if ((ret = PIOc_setframe(ncid, varid, t))) */ +/* ERR(ret); */ +/* if ((ret = PIOc_write_darray(ncid, varid, ioid, elements_per_pe, buffer, NULL))) */ +/* ERR(ret); */ +/* } */ + +/* /\* THis will cause all data to be written to disk. *\/ */ +/* if ((ret = PIOc_sync(ncid))) */ +/* ERR(ret); */ + +/* /\* Close the netCDF file. *\/ */ +/* printf("rank: %d Closing the sample data file...\n", my_rank); */ +/* if ((ret = PIOc_closefile(ncid))) */ +/* ERR(ret); */ + +/* /\* Check the output file. *\/ */ +/* /\* if ((ret = check_file(iosysid, ntasks, filename, format[fmt], elements_per_pe, *\/ */ +/* /\* my_rank, ioid))) *\/ */ +/* /\* ERR(ret); *\/ */ +/* } */ + + /* Free the PIO decomposition. */ + /* printf("rank: %d Freeing PIO decomposition...\n", my_rank); */ + /* if ((ret = PIOc_freedecomp(iosysid, ioid))) */ + /* ERR(ret); */ + + /* Finalize the IO system. Only call this from the computation tasks. */ + printf("%d %s Freeing PIO resources\n", my_rank, TEST_NAME); + if ((ret = PIOc_finalize(iosysid))) + ERR(ret); + } /* endif comp_task */ + + /* Finalize the MPI library. */ + MPI_Finalize(); + +#ifdef TIMING + /* Finalize the GPTL timing library. */ + if ((ret = GPTLfinalize ())) + return ret; +#endif + + printf("rank: %d SUCCESS!\n", my_rank); + return 0; + } diff --git a/src/externals/pio2/examples/c/darray_no_async.c b/src/externals/pio2/examples/c/darray_no_async.c new file mode 100644 index 00000000000..14228ab422c --- /dev/null +++ b/src/externals/pio2/examples/c/darray_no_async.c @@ -0,0 +1,358 @@ +/* + * @file + * @brief A simple C example for the ParallelIO Library. + * + * This example creates a netCDF output file with three dimensions + * (one unlimited) and one variable. It first writes, then reads the + * sample file using distributed arrays. + * + * This example can be run in parallel for 4 processors. + */ + +#include +#include +#include +#include +#include +#include +#include +#ifdef TIMING +#include +#endif + +/* The number of possible output netCDF output flavors available to + * the ParallelIO library. */ +#define NUM_NETCDF_FLAVORS 4 + +/* The number of dimensions in the example data. */ +#define NDIM3 3 + +/* The number of timesteps of data. */ +#define NUM_TIMESTEPS 2 + +/* The length of our sample data in X dimension.*/ +#define DIM_LEN_X 4 + +/* The length of our sample data in Y dimension.*/ +#define DIM_LEN_Y 4 + +/* The name of the variable in the netCDF output file. */ +#define VAR_NAME "foo" + +/* Return code when netCDF output file does not match + * expectations. */ +#define ERR_BAD 1001 + +/* The meaning of life, the universe, and everything. */ +#define START_DATA_VAL 42 + +/* Number of tasks this example runs on. */ +#define TARGET_NTASKS 4 + +/* Logging level. */ +#define LOG_LEVEL 3 + +/* Lengths of dimensions. */ +int dim_len[NDIM3] = {NC_UNLIMITED, DIM_LEN_X, DIM_LEN_Y}; + +/* Names of dimensions. */ +char dim_name[NDIM3][PIO_MAX_NAME + 1] = {"unlimted", "x", "y"}; + +/* Handle MPI errors. This should only be used with MPI library + * function calls. */ +#define MPIERR(e) do { \ + MPI_Error_string(e, err_buffer, &resultlen); \ + printf("MPI error, line %d, file %s: %s\n", __LINE__, __FILE__, err_buffer); \ + MPI_Finalize(); \ + return 2; \ + } while (0) + +/* Handle non-MPI errors by finalizing the MPI library and exiting + * with an exit code. */ +#define ERR(e) do { \ + MPI_Finalize(); \ + return e; \ + } while (0) + +/* Global err buffer for MPI. When there is an MPI error, this buffer + * is used to store the error message that is associated with the MPI + * error. */ +char err_buffer[MPI_MAX_ERROR_STRING]; + +/* This is the length of the most recent MPI error message, stored + * int the global error string. */ +int resultlen; + +/* @brief Check the output file. + * + * Use netCDF to check that the output is as expected. + * + * @param ntasks The number of processors running the example. + * @param filename The name of the example file to check. + * + * @return 0 if example file is correct, non-zero otherwise. */ +int check_file(int iosysid, int ntasks, char *filename, int iotype, + int elements_per_pe, int my_rank, int ioid) +{ + + int ncid; /* File ID from netCDF. */ + int ndims; /* Number of dimensions. */ + int nvars; /* Number of variables. */ + int ngatts; /* Number of global attributes. */ + int unlimdimid; /* ID of unlimited dimension. */ + int natts; /* Number of variable attributes. */ + nc_type xtype; /* NetCDF data type of this variable. */ + int ret; /* Return code for function calls. */ + int dimids[NDIM3]; /* Dimension ids for this variable. */ + char var_name[NC_MAX_NAME]; /* Name of the variable. */ + /* size_t start[NDIM3]; /\* Zero-based index to start read. *\/ */ + /* size_t count[NDIM3]; /\* Number of elements to read. *\/ */ + /* int buffer[DIM_LEN_X]; /\* Buffer to read in data. *\/ */ + /* int expected[DIM_LEN_X]; /\* Data values we expect to find. *\/ */ + + /* Open the file. */ + if ((ret = PIOc_openfile_retry(iosysid, &ncid, &iotype, filename, 0, 0))) + return ret; + printf("opened file %s ncid = %d\n", filename, ncid); + + /* Check the metadata. */ + if ((ret = PIOc_inq(ncid, &ndims, &nvars, &ngatts, &unlimdimid))) + return ret; + + /* Check the dimensions. */ + if (ndims != NDIM3 || nvars != 1 || ngatts != 0 || unlimdimid != 0) + return ERR_BAD; + for (int d = 0; d < NDIM3; d++) + { + char my_dim_name[NC_MAX_NAME]; + PIO_Offset dimlen; + + if ((ret = PIOc_inq_dim(ncid, d, my_dim_name, &dimlen))) + return ret; + if (dimlen != (d ? dim_len[d] : NUM_TIMESTEPS) || strcmp(my_dim_name, dim_name[d])) + return ERR_BAD; + } + + /* Check the variable. */ + if ((ret = PIOc_inq_var(ncid, 0, var_name, &xtype, &ndims, dimids, &natts))) + return ret; + if (xtype != NC_INT || ndims != NDIM3 || dimids[0] != 0 || dimids[1] != 1 || + dimids[2] != 2 || natts != 0) + return ERR_BAD; + + /* Allocate storage for sample data. */ + int buffer[elements_per_pe]; + int buffer_in[elements_per_pe]; + + /* Check each timestep. */ + for (int t = 0; t < NUM_TIMESTEPS; t++) + { + int varid = 0; /* There's only one var in sample file. */ + + /* This is the data we expect for this timestep. */ + for (int i = 0; i < elements_per_pe; i++) + buffer[i] = 100 * t + START_DATA_VAL + my_rank; + + /* Read one record. */ + if ((ret = PIOc_setframe(ncid, varid, t))) + ERR(ret); + if ((ret = PIOc_read_darray(ncid, varid, ioid, elements_per_pe, buffer_in))) + return ret; + + /* Check the results. */ + for (int i = 0; i < elements_per_pe; i++) + if (buffer_in[i] != buffer[i]) + return ERR_BAD; + } + + /* Close the file. */ + if ((ret = PIOc_closefile(ncid))) + return ret; + + /* Everything looks good! */ + return 0; +} + +/* Write, then read, a simple example with darrays. + + The sample file created by this program is a small netCDF file. It + has the following contents (as shown by ncdump): + + 
+netcdf darray_no_async_iotype_1 {
+dimensions:
+	unlimted = UNLIMITED ; // (2 currently)
+	x = 4 ;
+	y = 4 ;
+variables:
+	int foo(unlimted, x, y) ;
+data:
+
+ foo =
+  42, 42, 42, 42,
+  43, 43, 43, 43,
+  44, 44, 44, 44,
+  45, 45, 45, 45,
+  142, 142, 142, 142,
+  143, 143, 143, 143,
+  144, 144, 144, 144,
+  145, 145, 145, 145 ;
+}
+
+ +*/ + int main(int argc, char* argv[]) + { + int my_rank; /* Zero-based rank of processor. */ + int ntasks; /* Number of processors involved in current execution. */ + int ioproc_stride = 1; /* Stride in the mpi rank between io tasks. */ + int ioproc_start = 0; /* Rank of first task to be used for I/O. */ + PIO_Offset elements_per_pe; /* Array elements per processing unit. */ + int iosysid; /* The ID for the parallel I/O system. */ + int ncid; /* The ncid of the netCDF file. */ + int dimid[NDIM3]; /* The dimension ID. */ + int varid; /* The ID of the netCDF varable. */ + int ioid; /* The I/O description ID. */ + char filename[NC_MAX_NAME + 1]; /* Test filename. */ + int num_flavors = 0; /* Number of iotypes available in this build. */ + int format[NUM_NETCDF_FLAVORS]; /* Different output flavors. */ + int ret; /* Return value. */ + +#ifdef TIMING + /* Initialize the GPTL timing library. */ + if ((ret = GPTLinitialize ())) + return ret; +#endif + + /* Initialize MPI. */ + if ((ret = MPI_Init(&argc, &argv))) + MPIERR(ret); + if ((ret = MPI_Comm_set_errhandler(MPI_COMM_WORLD, MPI_ERRORS_RETURN))) + MPIERR(ret); + + /* Learn my rank and the total number of processors. */ + if ((ret = MPI_Comm_rank(MPI_COMM_WORLD, &my_rank))) + MPIERR(ret); + if ((ret = MPI_Comm_size(MPI_COMM_WORLD, &ntasks))) + MPIERR(ret); + + /* Check that a valid number of processors was specified. */ + if (ntasks != TARGET_NTASKS) + fprintf(stderr, "Number of processors must be 4!\n"); + printf("%d: ParallelIO Library darray_no_async example running on %d processors.\n", + my_rank, ntasks); + + /* Turn on logging. */ + if ((ret = PIOc_set_log_level(LOG_LEVEL))) + return ret; + + /* Initialize the PIO IO system. This specifies how many and + * which processors are involved in I/O. */ + if ((ret = PIOc_Init_Intracomm(MPI_COMM_WORLD, 1, ioproc_stride, + ioproc_start, PIO_REARR_BOX, &iosysid))) + ERR(ret); + + /* Describe the decomposition. */ + elements_per_pe = DIM_LEN_X * DIM_LEN_Y / TARGET_NTASKS; + + /* Allocate and initialize array of decomposition mapping. */ + PIO_Offset compdof[elements_per_pe]; + for (int i = 0; i < elements_per_pe; i++) + compdof[i] = my_rank * elements_per_pe + i; + + /* Create the PIO decomposition for this example. Since this + * is a variable with an unlimited dimension, we want to + * create a 2-D composition which represents one record. */ + printf("rank: %d Creating decomposition...\n", my_rank); + if ((ret = PIOc_init_decomp(iosysid, PIO_INT, NDIM3 - 1, &dim_len[1], elements_per_pe, + compdof, &ioid, 0, NULL, NULL))) + ERR(ret); + + /* The number of favors may change with the build parameters. */ +#ifdef _PNETCDF + format[num_flavors++] = PIO_IOTYPE_PNETCDF; +#endif + format[num_flavors++] = PIO_IOTYPE_NETCDF; +#ifdef _NETCDF4 + format[num_flavors++] = PIO_IOTYPE_NETCDF4C; + format[num_flavors++] = PIO_IOTYPE_NETCDF4P; +#endif + + /* Use PIO to create the example file in each of the four + * available ways. */ + for (int fmt = 0; fmt < num_flavors; fmt++) + { + /* Create a filename. */ + sprintf(filename, "darray_no_async_iotype_%d.nc", format[fmt]); + + /* Create the netCDF output file. */ + printf("rank: %d Creating sample file %s with format %d...\n", + my_rank, filename, format[fmt]); + if ((ret = PIOc_createfile(iosysid, &ncid, &(format[fmt]), filename, PIO_CLOBBER))) + ERR(ret); + + /* Define netCDF dimension and variable. */ + printf("rank: %d Defining netCDF metadata...\n", my_rank); + for (int d = 0; d < NDIM3; d++) + if ((ret = PIOc_def_dim(ncid, dim_name[d], dim_len[d], &dimid[d]))) + ERR(ret); + if ((ret = PIOc_def_var(ncid, VAR_NAME, PIO_INT, NDIM3, dimid, &varid))) + ERR(ret); + if ((ret = PIOc_enddef(ncid))) + ERR(ret); + + /* Allocate storage for sample data. */ + int buffer[elements_per_pe]; + + /* Write each timestep. */ + for (int t = 0; t < NUM_TIMESTEPS; t++) + { + /* Create some data for this timestep. */ + for (int i = 0; i < elements_per_pe; i++) + buffer[i] = 100 * t + START_DATA_VAL + my_rank; + + /* Write data to the file. */ + printf("rank: %d Writing sample data...\n", my_rank); + if ((ret = PIOc_setframe(ncid, varid, t))) + ERR(ret); + if ((ret = PIOc_write_darray(ncid, varid, ioid, elements_per_pe, buffer, NULL))) + ERR(ret); + } + + /* THis will cause all data to be written to disk. */ + if ((ret = PIOc_sync(ncid))) + ERR(ret); + + /* Close the netCDF file. */ + printf("rank: %d Closing the sample data file...\n", my_rank); + if ((ret = PIOc_closefile(ncid))) + ERR(ret); + + /* Check the output file. */ + /* if ((ret = check_file(iosysid, ntasks, filename, format[fmt], elements_per_pe, */ + /* my_rank, ioid))) */ + /* ERR(ret); */ + } + + /* Free the PIO decomposition. */ + printf("rank: %d Freeing PIO decomposition...\n", my_rank); + if ((ret = PIOc_freedecomp(iosysid, ioid))) + ERR(ret); + + /* Finalize the IO system. */ + printf("rank: %d Freeing PIO resources...\n", my_rank); + if ((ret = PIOc_finalize(iosysid))) + ERR(ret); + + /* Finalize the MPI library. */ + MPI_Finalize(); + +#ifdef TIMING + /* Finalize the GPTL timing library. */ + if ((ret = GPTLfinalize ())) + return ret; +#endif + + printf("rank: %d SUCCESS!\n", my_rank); + return 0; + } diff --git a/src/externals/pio2/examples/c/example1.c b/src/externals/pio2/examples/c/example1.c index 1fcd8ab222a..ad650f0b1fa 100644 --- a/src/externals/pio2/examples/c/example1.c +++ b/src/externals/pio2/examples/c/example1.c @@ -285,7 +285,7 @@ int check_file(int ntasks, char *filename) { /* Initialize MPI. */ if ((ret = MPI_Init(&argc, &argv))) MPIERR(ret); - if ((ret = MPI_Errhandler_set(MPI_COMM_WORLD, MPI_ERRORS_RETURN))) + if ((ret = MPI_Comm_set_errhandler(MPI_COMM_WORLD, MPI_ERRORS_RETURN))) MPIERR(ret); /* Learn my rank and the total number of processors. */ diff --git a/src/externals/pio2/src/clib/CMakeLists.txt b/src/externals/pio2/src/clib/CMakeLists.txt index d2c1c3bbeb6..7fac8891a8e 100644 --- a/src/externals/pio2/src/clib/CMakeLists.txt +++ b/src/externals/pio2/src/clib/CMakeLists.txt @@ -54,24 +54,19 @@ install (FILES ${CMAKE_CURRENT_SOURCE_DIR}/pio.h DESTINATION include) #============================================================================== # DEFINE THE DEPENDENCIES #============================================================================== - -#===== MPI ===== -if (PIO_USE_MPISERIAL) - find_package (MPISERIAL COMPONENTS C REQUIRED) - if (MPISERIAL_C_FOUND) +# MPI test done at top level +if (MPISERIAL_C_FOUND) target_compile_definitions (pioc PRIVATE MPI_SERIAL) target_include_directories (pioc PUBLIC ${MPISERIAL_C_INCLUDE_DIRS}) target_link_libraries (pioc PUBLIC ${MPISERIAL_C_LIBRARIES}) - set (WITH_PNETCDF FALSE) - endif () -else () - find_package (MPI REQUIRED) endif () + + #===== GPTL ===== if (PIO_ENABLE_TIMING) find_package (GPTL COMPONENTS C QUIET) diff --git a/src/externals/pio2/src/clib/bget.c b/src/externals/pio2/src/clib/bget.c index 726ce5580da..98bfc36893d 100644 --- a/src/externals/pio2/src/clib/bget.c +++ b/src/externals/pio2/src/clib/bget.c @@ -400,7 +400,7 @@ #include #include #include -#ifdef PIO_USE_MALLOC +#if PIO_USE_MALLOC #include #endif @@ -614,7 +614,7 @@ bufsize requested_size; int compactseq = 0; #endif -#ifdef PIO_USE_MALLOC +#if PIO_USE_MALLOC // if(requested_size>maxsize){ // maxsize=requested_size; // printf("%s %d %d\n",__FILE__,__LINE__,maxsize); @@ -847,7 +847,7 @@ bufsize size; bufsize osize; /* Old size of buffer */ struct bhead *b; -#ifdef PIO_USE_MALLOC +#if PIO_USE_MALLOC return(realloc(buf, size)); #endif if ((nbuf = bget(size)) == NULL) { /* Acquire new buffer */ @@ -882,7 +882,7 @@ void *buf; { struct bfhead *b, *bn; -#ifdef PIO_USE_MALLOC +#if PIO_USE_MALLOC // printf("bget free %d %x\n",__LINE__,buf); free(buf); return; diff --git a/src/externals/pio2/src/clib/config.h.in b/src/externals/pio2/src/clib/config.h.in index 1722872c305..8d80d37fd51 100644 --- a/src/externals/pio2/src/clib/config.h.in +++ b/src/externals/pio2/src/clib/config.h.in @@ -22,4 +22,7 @@ /** Set to non-zero to turn on logging. Output may be large. */ #define PIO_ENABLE_LOGGING @ENABLE_LOGGING@ +/** Size of MPI_Offset type. */ +#define SIZEOF_MPI_OFFSET @SIZEOF_MPI_OFFSET@ + #endif /* _PIO_CONFIG_ */ diff --git a/src/externals/pio2/src/clib/pio.h b/src/externals/pio2/src/clib/pio.h index fe234a15593..9dace3782c9 100644 --- a/src/externals/pio2/src/clib/pio.h +++ b/src/externals/pio2/src/clib/pio.h @@ -40,6 +40,9 @@ /** The maximum number of variables allowed in a netCDF file. */ #define PIO_MAX_VARS NC_MAX_VARS +/** The maximum number of dimensions allowed in a netCDF file. */ +#define PIO_MAX_DIMS NC_MAX_DIMS + /** Pass this to PIOc_set_iosystem_error_handling() as the iosysid in * order to set default error handling. */ #define PIO_DEFAULT (-1) @@ -115,7 +118,20 @@ typedef struct var_desc_t /** Number of requests bending with pnetcdf. */ int nreqs; - /** Buffer that contains the fill value for this variable. */ + /* Holds the fill value of this var. */ + void *fillvalue; + + /* The PIO data type (PIO_INT, PIO_FLOAT, etc.) */ + int pio_type; + + /* The size of the data type (2 for PIO_SHORT, 4 for PIO_INT, etc.) */ + PIO_Offset type_size; + + /** Non-zero if fill mode is turned on for this var. */ + int use_fill; + + /** Buffer that contains the holegrid fill values used to fill in + * missing sections of data when using the subset rearranger. */ void *fillbuf; /** Data buffer for this variable. */ @@ -191,10 +207,10 @@ enum PIO_REARR_COMM_FC_DIR typedef struct rearr_comm_fc_opt { /** Enable handshake */ - bool enable_hs; + bool hs; /** Enable isends - if false use blocking sends */ - bool enable_isend; + bool isend; /** Max pending requests * (PIO_REARR_COMM_UNLIMITED_PEND_REQ => unlimited pend req). @@ -217,10 +233,10 @@ typedef struct rearr_opt int fcd; /** flow control opts, comp to io procs */ - rearr_comm_fc_opt_t comm_fc_opts_comp2io; + rearr_comm_fc_opt_t comp2io; /** flow control opts, io to comp procs */ - rearr_comm_fc_opt_t comm_fc_opts_io2comp; + rearr_comm_fc_opt_t io2comp; } rearr_opt_t; /** @@ -245,10 +261,7 @@ typedef struct io_desc_t * io tasks. */ int nrecvs; - /** Local size of the decomposition array on the compute node. On - each compute task the application passes a compmap array of - length ndof. This array describes the arrangement of data in - memory on that task. */ + /** Local size of the decomposition array on the compute node. */ int ndof; /** All vars included in this io_desc_t have the same number of @@ -277,6 +290,9 @@ typedef struct io_desc_t /** The MPI type of the data. */ MPI_Datatype basetype; + /** The size in bytes of a datum of MPI type basetype. */ + int basetype_size; + /** Length of the iobuffer on this task for a single field on the * IO node. The arrays from compute nodes gathered and rearranged * to the io-nodes (which are sometimes collocated with compute @@ -287,24 +303,25 @@ typedef struct io_desc_t /** Maximum llen participating. */ int maxiobuflen; - /** Array of tasks received from in pio_swapm(). */ + /** Array (length nrecvs) of computation tasks received from. */ int *rfrom; - /** Array of counts of data to be received from each task in - * pio_swapm(). */ + /** Array (length nrecvs) of counts of data to be received from + * each computation task by the IO tasks. */ int *rcount; - /** Array of data count to send to each task in the communication - * in pio_swapm(). */ + /** Array (length numiotasks) of data counts to send to each task + * in the communication in pio_swapm(). */ int *scount; - /** Send index. */ + /** Array (length ndof) for the BOX rearranger with the index + * for computation taks (send side during writes). */ PIO_Offset *sindex; - /** Receive index. */ + /** Index for the IO tasks (receive side during writes). */ PIO_Offset *rindex; - /** Array of receive MPI types in pio_swapm() call. */ + /** Array (of length nrecvs) of receive MPI types in pio_swapm() call. */ MPI_Datatype *rtype; /** Array of send MPI types in pio_swapm() call. */ @@ -387,6 +404,11 @@ typedef struct iosystem_desc_t /** The number of tasks in the computation communicator. */ int num_comptasks; + /** The number of tasks in the union communicator (will be + * num_comptasks for non-async, num_comptasks + num_iotasks for + * async). */ + int num_uniontasks; + /** Rank of this task in the union communicator. */ int union_rank; @@ -419,6 +441,10 @@ typedef struct iosystem_desc_t * communicator. */ int *ioranks; + /** An array of the ranks of all computation tasks within the + * union communicator. */ + int *compranks; + /** Controls handling errors. */ int error_handler; @@ -427,11 +453,15 @@ typedef struct iosystem_desc_t int default_rearranger; /** True if asynchronous interface is in use. */ - bool async_interface; + bool async; /** True if this task is a member of the IO communicator. */ bool ioproc; + /** True if this task is a member of a computation + * communicator. */ + bool compproc; + /** MPI Info object. */ MPI_Info info; @@ -455,8 +485,13 @@ typedef struct wmulti_buffer * PIOc_Init_Decomp(). */ int ioid; - /** Number of variables in this multi variable buffer. */ - int validvars; + /** Non-zero if this is a buffer for a record var. */ + int recordvar; + + /** Number of arrays of data in the multibuffer. Each array had + * data for one var or record. When multibuffer is flushed, all + * arrays are written and num_arrays returns to zero. */ + int num_arrays; /** Size of this variables data on local task. All vars in the * multi-buffer have the same size. */ @@ -673,10 +708,19 @@ enum PIO_ERROR_HANDLERS #define PIO_EBADCHUNK NC_EBADCHUNK #define PIO_ENOTBUILT NC_ENOTBUILT #define PIO_EDISKLESS NC_EDISKLESS -#define PIO_FILL_DOUBLE NC_FILL_DOUBLE -#define PIO_FILL_FLOAT NC_FILL_FLOAT -#define PIO_FILL_INT NC_FILL_INT + +/* These are the netCDF default fill values. */ +#define PIO_FILL_BYTE NC_FILL_BYTE #define PIO_FILL_CHAR NC_FILL_CHAR +#define PIO_FILL_SHORT NC_FILL_SHORT +#define PIO_FILL_INT NC_FILL_INT +#define PIO_FILL_FLOAT NC_FILL_FLOAT +#define PIO_FILL_DOUBLE NC_FILL_DOUBLE +#define PIO_FILL_UBYTE NC_FILL_UBYTE +#define PIO_FILL_USHORT NC_FILL_USHORT +#define PIO_FILL_UINT NC_FILL_UINT +#define PIO_FILL_INT64 NC_FILL_INT64 +#define PIO_FILL_UINT64 NC_FILL_UINT64 #endif /* defined( _PNETCDF) || defined(_NETCDF) */ /** Define the extra error codes for the parallel-netcdf library. */ @@ -703,18 +747,20 @@ extern "C" { /* Decomposition. */ /* Init decomposition with 1-based compmap array. */ - int PIOc_InitDecomp(int iosysid, int basetype, int ndims, const int *dims, int maplen, + int PIOc_InitDecomp(int iosysid, int pio_type, int ndims, const int *gdimlen, int maplen, const PIO_Offset *compmap, int *ioidp, const int *rearr, const PIO_Offset *iostart, const PIO_Offset *iocount); - int PIOc_InitDecomp_bc(int iosysid, int basetype, int ndims, const int *dims, + int PIOc_InitDecomp_bc(int iosysid, int basetype, int ndims, const int *gdimlen, const long int *start, const long int *count, int *ioidp); /* Init decomposition with 0-based compmap array. */ - int PIOc_init_decomp(int iosysid, int basetype, int ndims, const int *dims, int maplen, - const PIO_Offset *compmap, int *ioidp, const int *rearranger, + int PIOc_init_decomp(int iosysid, int pio_type, int ndims, const int *gdimlen, int maplen, + const PIO_Offset *compmap, int *ioidp, int rearranger, const PIO_Offset *iostart, const PIO_Offset *iocount); - + + /* Free resources associated with a decomposition. */ int PIOc_freedecomp(int iosysid, int ioid); + int PIOc_readmap(const char *file, int *ndims, int **gdims, PIO_Offset *fmaplen, PIO_Offset **map, MPI_Comm comm); int PIOc_readmap_from_f90(const char *file,int *ndims, int **gdims, PIO_Offset *maplen, @@ -729,16 +775,17 @@ extern "C" { /* Write a decomposition file using netCDF. */ int PIOc_write_nc_decomp(int iosysid, const char *filename, int cmode, int ioid, - MPI_Comm comm, char *title, char *history, int fortran_order); + char *title, char *history, int fortran_order); /* Read a netCDF decomposition file. */ int PIOc_read_nc_decomp(int iosysid, const char *filename, int *ioid, MPI_Comm comm, int pio_type, char *title, char *history, int *fortran_order); - /* Initializing IO system. */ - int PIOc_Init_Async(MPI_Comm world, int num_io_procs, int *io_proc_list, int component_count, + /* Initializing IO system for async. */ + int PIOc_init_async(MPI_Comm world, int num_io_procs, int *io_proc_list, int component_count, int *num_procs_per_comp, int **proc_list, MPI_Comm *io_comm, MPI_Comm *comp_comm, - int *iosysidp); + int rearranger, int *iosysidp); + int PIOc_Init_Intercomm(int component_count, MPI_Comm peer_comm, MPI_Comm *comp_comms, MPI_Comm io_comm, int *iosysidp); int PIOc_get_numiotasks(int iosysid, int *numiotasks); diff --git a/src/externals/pio2/src/clib/pio_darray.c b/src/externals/pio2/src/clib/pio_darray.c index 4250d87ea38..1d6f15c7d30 100644 --- a/src/externals/pio2/src/clib/pio_darray.c +++ b/src/externals/pio2/src/clib/pio_darray.c @@ -49,6 +49,27 @@ PIO_Offset PIOc_set_buffer_size_limit(PIO_Offset limit) * caller to use their own data buffering (instead of using the * buffering implemented in PIOc_write_darray()). * + * When the user calls PIOc_write_darray() one or more times, then + * PIO_write_darray_multi() will be called when the buffer is flushed. + * + * Internally, this function will: + *
    + *
  • Find info about file, decomposition, and variable. + *
  • Do a special flush for pnetcdf if needed. + *
  • Allocates a buffer big enough to hold all the data in the + * multi-buffer, for all tasks. + *
  • Calls rearrange_comp2io() to move data from compute to IO + * tasks. + *
  • For parallel iotypes (pnetcdf and netCDF-4 parallel) call + * pio_write_darray_multi_nc(). + *
  • For serial iotypes (netcdf classic and netCDF-4 serial) call + * write_darray_multi_serial(). + *
  • For subset rearranger, create holegrid to write missing + * data. Then call pio_write_darray_multi_nc() or + * write_darray_multi_serial() to write the holegrid. + *
  • Special buffer flush for pnetcdf. + *
+ * * @param ncid identifies the netCDF file. * @param varids an array of length nvars containing the variable ids to * be written. @@ -65,25 +86,23 @@ PIO_Offset PIOc_set_buffer_size_limit(PIO_Offset limit) * that is on this processor. There are nvars arrays of data, and each * array of data contains one record worth of data for that variable. * @param frame an array of length nvars with the frame or record - * dimension for each of the nvars variables in IOBUF - * @param fillvalue pointer to the fill value to be used for missing - * data. Ignored if NULL. If provided, must be the correct fill value - * for the variable. The correct fill value will be used if NULL is - * passed. + * dimension for each of the nvars variables in IOBUF. NULL if this + * iodesc contains non-record vars. + * @param fillvalue pointer an array (of length nvars) of pointers to + * the fill value to be used for missing data. * @param flushtodisk non-zero to cause buffers to be flushed to disk. * @return 0 for success, error code otherwise. * @ingroup PIO_write_darray */ -int PIOc_write_darray_multi(int ncid, const int *varids, int ioid, int nvars, PIO_Offset arraylen, - void *array, const int *frame, void **fillvalue, bool flushtodisk) +int PIOc_write_darray_multi(int ncid, const int *varids, int ioid, int nvars, + PIO_Offset arraylen, void *array, const int *frame, + void **fillvalue, bool flushtodisk) { iosystem_desc_t *ios; /* Pointer to io system information. */ file_desc_t *file; /* Pointer to file information. */ io_desc_t *iodesc; /* Pointer to IO description information. */ - int vsize; /* size in bytes of the given data. */ int rlen; /* total data buffer size. */ var_desc_t *vdesc0; /* pointer to var_desc structure for each var. */ - int mpierr; /* Return code from MPI functions. */ int ierr; /* Return code. */ /* Get the file info. */ @@ -98,61 +117,69 @@ int PIOc_write_darray_multi(int ncid, const int *varids, int ioid, int nvars, PI if (varids[v] < 0 || varids[v] > PIO_MAX_VARS) return pio_err(ios, file, PIO_EINVAL, __FILE__, __LINE__); - LOG((1, "PIOc_write_darray_multi ncid = %d ioid = %d nvars = %d arraylen = %ld flushtodisk = %d", + LOG((1, "PIOc_write_darray_multi ncid = %d ioid = %d nvars = %d arraylen = %ld " + "flushtodisk = %d", ncid, ioid, nvars, arraylen, flushtodisk)); /* Check that we can write to this file. */ - if (! (file->mode & PIO_WRITE)) + if (!(file->mode & PIO_WRITE)) return pio_err(ios, file, PIO_EPERM, __FILE__, __LINE__); /* Get iodesc. */ if (!(iodesc = pio_get_iodesc_from_id(ioid))) return pio_err(ios, file, PIO_EBADID, __FILE__, __LINE__); + pioassert(iodesc->rearranger == PIO_REARR_BOX || iodesc->rearranger == PIO_REARR_SUBSET, + "unknown rearranger", __FILE__, __LINE__); + + /* Get a pointer to the variable info for the first variable. */ + vdesc0 = &file->varlist[varids[0]]; + + /* if the buffer is already in use in pnetcdf we need to flush first */ + if (file->iotype == PIO_IOTYPE_PNETCDF && vdesc0->iobuf) + flush_output_buffer(file, 1, 0); - /* For netcdf serial writes we collect the data on io nodes and + pioassert(!vdesc0->iobuf, "buffer overwrite",__FILE__, __LINE__); + + /* Determine total size of aggregated data (all vars/records). + * For netcdf serial writes we collect the data on io nodes and * then move that data one node at a time to the io master node * and write (or read). The buffer size on io task 0 must be as * large as the largest used to accommodate this serial io - * method. */ - vdesc0 = file->varlist + varids[0]; - pioassert(!vdesc0->iobuf, "Attempt to overwrite existing io buffer",__FILE__, __LINE__); - - /* ??? */ - /* rlen = iodesc->llen*nvars; */ + * method. */ rlen = 0; if (iodesc->llen > 0) rlen = iodesc->maxiobuflen * nvars; - /* Currently there are two rearrangers box=1 and subset=2. There - * is never a case where rearranger==0. */ - LOG((2, "iodesc->rearranger = %d iodesc->needsfill = %d\n", iodesc->rearranger, - iodesc->needsfill)); - if (iodesc->rearranger > 0) + /* Allocate iobuf. */ + if (rlen > 0) { - if (rlen > 0) - { - if ((mpierr = MPI_Type_size(iodesc->basetype, &vsize))) - return check_mpi(file, mpierr, __FILE__, __LINE__); - LOG((3, "vsize = %d", vsize)); - - /* Allocate memory for the variable buffer. */ - if (!(vdesc0->iobuf = bget((size_t)vsize * (size_t)rlen))) - piomemerror(ios, (size_t)rlen * (size_t)vsize, __FILE__, __LINE__); - LOG((3, "allocated %ld bytes for variable buffer", (size_t)rlen * (size_t)vsize)); - - /* If data are missing for the BOX rearranger, insert fill values. */ - if (iodesc->needsfill && iodesc->rearranger == PIO_REARR_BOX) - for (int nv = 0; nv < nvars; nv++) - for (int i = 0; i < iodesc->maxiobuflen; i++) - memcpy(&((char *)vdesc0->iobuf)[vsize * (i + nv * iodesc->maxiobuflen)], - &((char *)fillvalue)[nv * vsize], vsize); - } - - /* Move data from compute to IO tasks. */ - if ((ierr = rearrange_comp2io(ios, iodesc, array, vdesc0->iobuf, nvars))) - return pio_err(ios, file, ierr, __FILE__, __LINE__); + /* Allocate memory for the buffer for all vars/records. */ + if (!(vdesc0->iobuf = bget(iodesc->basetype_size * (size_t)rlen))) + return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__); + LOG((3, "allocated %lld bytes for variable buffer", (size_t)rlen * iodesc->basetype_size)); + + /* If fill values are desired, and we're using the BOX + * rearranger, insert fill values. */ + if (iodesc->needsfill && iodesc->rearranger == PIO_REARR_BOX) + for (int nv = 0; nv < nvars; nv++) + for (int i = 0; i < iodesc->maxiobuflen; i++) + memcpy(&((char *)vdesc0->iobuf)[iodesc->basetype_size * (i + nv * iodesc->maxiobuflen)], + &((char *)fillvalue)[nv * iodesc->basetype_size], iodesc->basetype_size); } - + else if (file->iotype == PIO_IOTYPE_PNETCDF && ios->ioproc) + { + /* this assures that iobuf is allocated on all iotasks thus + assuring that the flush_output_buffer call above is called + collectively (from all iotasks) */ + if (!(vdesc0->iobuf = bget(1))) + return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__); + LOG((3, "allocated token for variable buffer")); + } + + /* Move data from compute to IO tasks. */ + if ((ierr = rearrange_comp2io(ios, iodesc, array, vdesc0->iobuf, nvars))) + return pio_err(ios, file, ierr, __FILE__, __LINE__); + /* Write the darray based on the iotype. */ LOG((2, "about to write darray for iotype = %d", file->iotype)); switch (file->iotype) @@ -166,9 +193,7 @@ int PIOc_write_darray_multi(int ncid, const int *varids, int ioid, int nvars, PI break; case PIO_IOTYPE_NETCDF4C: case PIO_IOTYPE_NETCDF: - if ((ierr = pio_write_darray_multi_nc_serial(file, nvars, varids, iodesc->ndims, iodesc->basetype, - iodesc->maxregions, iodesc->firstregion, iodesc->llen, - iodesc->num_aiotasks, vdesc0->iobuf, frame))) + if ((ierr = write_darray_multi_serial(file, nvars, varids, iodesc, 0, frame))) return pio_err(ios, file, ierr, __FILE__, __LINE__); break; @@ -182,6 +207,7 @@ int PIOc_write_darray_multi(int ncid, const int *varids, int ioid, int nvars, PI /* Release resources. */ if (vdesc0->iobuf) { + LOG((3,"freeing variable buffer in pio_darray")); brel(vdesc0->iobuf); vdesc0->iobuf = NULL; } @@ -202,22 +228,21 @@ int PIOc_write_darray_multi(int ncid, const int *varids, int ioid, int nvars, PI LOG((2, "nvars = %d holegridsize = %ld iodesc->needsfill = %d\n", nvars, iodesc->holegridsize, iodesc->needsfill)); - if (vdesc0->fillbuf) - piodie("Attempt to overwrite existing buffer",__FILE__,__LINE__); + pioassert(!vdesc0->fillbuf, "buffer overwrite",__FILE__, __LINE__); /* Get a buffer. */ if (ios->io_rank == 0) - vdesc0->fillbuf = bget(iodesc->maxholegridsize * vsize * nvars); + vdesc0->fillbuf = bget(iodesc->maxholegridsize * iodesc->basetype_size * nvars); else if (iodesc->holegridsize > 0) - vdesc0->fillbuf = bget(iodesc->holegridsize * vsize * nvars); + vdesc0->fillbuf = bget(iodesc->holegridsize * iodesc->basetype_size * nvars); /* copying the fill value into the data buffer for the box * rearranger. This will be overwritten with data where * provided. */ for (int nv = 0; nv < nvars; nv++) for (int i = 0; i < iodesc->holegridsize; i++) - memcpy(&((char *)vdesc0->fillbuf)[vsize * (i + nv * iodesc->holegridsize)], - &((char *)fillvalue)[vsize * nv], vsize); + memcpy(&((char *)vdesc0->fillbuf)[iodesc->basetype_size * (i + nv * iodesc->holegridsize)], + &((char *)fillvalue)[iodesc->basetype_size * nv], iodesc->basetype_size); /* Write the darray based on the iotype. */ switch (file->iotype) @@ -232,10 +257,7 @@ int PIOc_write_darray_multi(int ncid, const int *varids, int ioid, int nvars, PI break; case PIO_IOTYPE_NETCDF4C: case PIO_IOTYPE_NETCDF: - if ((ierr = pio_write_darray_multi_nc_serial(file, nvars, varids, iodesc->ndims, iodesc->basetype, - iodesc->maxfillregions, iodesc->fillregion, - iodesc->holegridsize, - iodesc->num_aiotasks, vdesc0->fillbuf, frame))) + if ((ierr = write_darray_multi_serial(file, nvars, varids, iodesc, 1, frame))) return pio_err(ios, file, ierr, __FILE__, __LINE__); break; default: @@ -254,7 +276,7 @@ int PIOc_write_darray_multi(int ncid, const int *varids, int ioid, int nvars, PI } } - /* Flush data to disk. */ + /* Flush data to disk for pnetcdf. */ if (ios->ioproc && file->iotype == PIO_IOTYPE_PNETCDF) if ((ierr = flush_output_buffer(file, flushtodisk, 0))) return pio_err(ios, file, ierr, __FILE__, __LINE__); @@ -262,6 +284,50 @@ int PIOc_write_darray_multi(int ncid, const int *varids, int ioid, int nvars, PI return PIO_NOERR; } +/** + * Find the fillvalue that should be used for a variable. + * + * @param file Info about file we are writing to. + * @param varid the variable ID. + * @param vdesc pointer to var_desc_t info for this var. + * @returns 0 for success, non-zero error code for failure. + * @ingroup PIO_write_darray + */ +int find_var_fillvalue(file_desc_t *file, int varid, var_desc_t *vdesc) +{ + iosystem_desc_t *ios; /* Pointer to io system information. */ + int no_fill; + int ierr; + + /* Check inputs. */ + pioassert(file && file->iosystem && vdesc, "invalid input", __FILE__, __LINE__); + ios = file->iosystem; + + LOG((3, "find_var_fillvalue file->pio_ncid = %d varid = %d", file->pio_ncid, varid)); + + /* Find out PIO data type of var. */ + if ((ierr = PIOc_inq_vartype(file->pio_ncid, varid, &vdesc->pio_type))) + return pio_err(ios, NULL, ierr, __FILE__, __LINE__); + + /* Find out length of type. */ + if ((ierr = PIOc_inq_type(file->pio_ncid, vdesc->pio_type, NULL, &vdesc->type_size))) + return pio_err(ios, NULL, ierr, __FILE__, __LINE__); + LOG((3, "getting fill value for varid = %d pio_type = %d type_size = %d", + varid, vdesc->pio_type, vdesc->type_size)); + + /* Allocate storage for the fill value. */ + if (!(vdesc->fillvalue = malloc(vdesc->type_size))) + return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); + + /* Get the fill value. */ + if ((ierr = PIOc_inq_var_fill(file->pio_ncid, varid, &no_fill, vdesc->fillvalue))) + return pio_err(ios, NULL, ierr, __FILE__, __LINE__); + vdesc->use_fill = no_fill ? 0 : 1; + LOG((3, "vdesc->use_fill = %d", vdesc->use_fill)); + + return PIO_NOERR; +} + /** * Write a distributed array to the output file. * @@ -269,20 +335,43 @@ int PIOc_write_darray_multi(int ncid, const int *varids, int ioid, int nvars, PI * it to the IO nodes when the compute buffer is full or when a flush * is triggered. * + * Internally, this function will: + *
    + *
  • Locate info about this file, decomposition, and variable. + *
  • If we don't have a fillvalue for this variable, determine one + * and remember it for future calls. + *
  • Initialize or find the multi_buffer for this record/var. + *
  • Find out how much free space is available in the multi buffer + * and flush if needed. + *
  • Store the new user data in the mutli buffer. + *
  • If needed (only for subset rearranger), fill in gaps in data + * with fillvalue. + *
  • Remember the frame value (i.e. record number) of this data if + * there is one. + *
+ * + * NOTE: The write multi buffer wmulti_buffer is the cache on compute + * nodes that will collect and store multiple variables before sending + * them to the io nodes. Aggregating variables in this way leads to a + * considerable savings in communication expense. Variables in the wmb + * array must have the same decomposition and base data size and we + * also need to keep track of whether each is a recordvar (has an + * unlimited dimension) or not. + * * @param ncid the ncid of the open netCDF file. * @param varid the ID of the variable that these data will be written * to. * @param ioid the I/O description ID as passed back by * PIOc_InitDecomp(). - * @param arraylen the length of the array to be written. This should + * @param arraylen the length of the array to be written. This should * be at least the length of the local component of the distrubited * array. (Any values beyond length of the local component will be * ignored.) * @param array pointer to an array of length arraylen with the data * to be written. This is a pointer to the distributed portion of the * array that is on this task. - * @param fillvalue pointer to the fill value to be used for - * missing data. + * @param fillvalue pointer to the fill value to be used for missing + * data. * @returns 0 for success, non-zero error code for failure. * @ingroup PIO_write_darray */ @@ -296,8 +385,7 @@ int PIOc_write_darray(int ncid, int varid, int ioid, PIO_Offset arraylen, void * void *bufptr; /* A data buffer. */ MPI_Datatype vtype; /* The MPI type of the variable. */ wmulti_buffer *wmb; /* The write multi buffer for one or more vars. */ - int tsize; /* Size of MPI type. */ - bool recordvar; /* True if this is a record variable. */ + int recordvar; /* Non-zero if this is a record variable. */ int needsflush = 0; /* True if we need to flush buffer. */ bufsize totfree; /* Amount of free space in the buffer. */ bufsize maxfree; /* Max amount of free space in buffer. */ @@ -320,95 +408,59 @@ int PIOc_write_darray(int ncid, int varid, int ioid, PIO_Offset arraylen, void * if (!(iodesc = pio_get_iodesc_from_id(ioid))) return pio_err(ios, file, PIO_EBADID, __FILE__, __LINE__); - /* Get var description. */ - vdesc = file->varlist + varid; - LOG((2, "vdesc record %d ndims %d nreqs %d", vdesc->record, vdesc->ndims, vdesc->nreqs)); - - /* Is this a record variable? */ - recordvar = vdesc->record >= 0 ? true : false; - LOG((3, "recordvar = %d", recordvar)); - /* Check that the local size of the variable passed in matches the - * size expected by the io descriptor. */ + * size expected by the io descriptor. Fail if arraylen is too + * small, just put a warning in the log if it is too big (the + * excess values will be ignored.) */ if (arraylen < iodesc->ndof) return pio_err(ios, file, PIO_EINVAL, __FILE__, __LINE__); + LOG((2, "%s arraylen = %d iodesc->ndof = %d", + (iodesc->ndof != arraylen) ? "WARNING: iodesc->ndof != arraylen" : "", + arraylen, iodesc->ndof)); - if (iodesc->ndof != arraylen) - LOG((1, "User supplied array is larger than expected, arraylen != iodesc->ndof")); - - /* Get a pointer to the buffer space for this file. It will hold - * data from one or more variables that fit the same - * description. */ - wmb = &file->buffer; + /* Get var description. */ + vdesc = &(file->varlist[varid]); + LOG((2, "vdesc record %d ndims %d nreqs %d", vdesc->record, vdesc->ndims, + vdesc->nreqs)); + + /* If we don't know the fill value for this var, get it. */ + if (!vdesc->fillvalue) + if ((ierr = find_var_fillvalue(file, varid, vdesc))) + return pio_err(ios, file, PIO_EBADID, __FILE__, __LINE__); + + /* Is this a record variable? The user must set the vdesc->record + * value by calling PIOc_setframe() before calling this + * function. */ + recordvar = vdesc->record >= 0 ? 1 : 0; + LOG((3, "recordvar = %d", recordvar)); - /* If the ioid is not initialized, set it. For non record vars, - * use the negative ??? */ - if (wmb->ioid == -1) - wmb->ioid = recordvar ? ioid : -ioid; - else - { - /* Handle record and non-record variables differently. */ - if (recordvar) - { - /* Moving to the end of the wmb linked list to add the - * current variable. ??? */ - while(wmb->next && wmb->ioid != ioid) - if (wmb->next) - wmb = wmb->next; -#ifdef _PNETCDF - /* Do we still need the commented code below? ??? */ - /* flush the previous record before starting a new one. this is collective */ - /* if (vdesc->request != NULL && (vdesc->request[0] != NC_REQ_NULL) || - (wmb->frame != NULL && vdesc->record != wmb->frame[0])){ - needsflush = 2; // flush to disk - } */ -#endif - } - else - { - /* Move to end of list. */ - while(wmb->next && wmb->ioid != -(ioid)) - if (wmb->next) - wmb = wmb->next; - } - } + /* Move to end of list or the entry that matches this ioid. */ + for (wmb = &file->buffer; wmb->next; wmb = wmb->next) + if (wmb->ioid == ioid && wmb->recordvar == recordvar) + break; - /* The write multi buffer wmulti_buffer is the cache on compute - nodes that will collect and store multiple variables before - sending them to the io nodes. Aggregating variables in this way - leads to a considerable savings in communication - expense. Variables in the wmb array must have the same - decomposition and base data size and we also need to keep track - of whether each is a recordvar (has an unlimited dimension) or - not. */ - if ((recordvar && wmb->ioid != ioid) || (!recordvar && wmb->ioid != -(ioid))) + /* If we did not find an existing wmb entry, create a new wmb. */ + if (wmb->ioid != ioid || wmb->recordvar != recordvar) { /* Allocate a buffer. */ if (!(wmb->next = bget((bufsize)sizeof(wmulti_buffer)))) - piomemerror(ios, sizeof(wmulti_buffer), __FILE__, __LINE__); + return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__); LOG((3, "allocated multi-buffer")); /* Set pointer to newly allocated buffer and initialize.*/ wmb = wmb->next; + wmb->recordvar = recordvar; wmb->next = NULL; - wmb->ioid = recordvar ? ioid : -ioid; - wmb->validvars = 0; + wmb->ioid = ioid; + wmb->num_arrays = 0; wmb->arraylen = arraylen; wmb->vid = NULL; wmb->data = NULL; wmb->frame = NULL; wmb->fillvalue = NULL; } - - /* Get the size of the MPI type. */ - if ((mpierr = MPI_Type_size(iodesc->basetype, &tsize))) - return check_mpi(file, mpierr, __FILE__, __LINE__); - - LOG((2, "wmb->validvars = %d arraylen = %d tsize = %d\n", wmb->validvars, - arraylen, tsize)); - - /* At this point wmb should be pointing to a new or existing buffer - so we can add the data. */ + LOG((2, "wmb->num_arrays = %d arraylen = %d iodesc->basetype_size = %d\n", + wmb->num_arrays, arraylen, iodesc->basetype_size)); /* Find out how much free, contiguous space is available. */ bfreespace(&totfree, &maxfree); @@ -416,23 +468,24 @@ int PIOc_write_darray(int ncid, int varid, int ioid, PIO_Offset arraylen, void * /* maxfree is the available memory. If that is < 10% greater than * the size of the current request needsflush is true. */ if (needsflush == 0) - needsflush = (maxfree <= 1.1 * (1 + wmb->validvars) * arraylen * tsize); + needsflush = (maxfree <= 1.1 * (1 + wmb->num_arrays) * arraylen * iodesc->basetype_size); /* Tell all tasks on the computation communicator whether we need * to flush data. */ - if ((mpierr = MPI_Allreduce(MPI_IN_PLACE, &needsflush, 1, MPI_INT, MPI_MAX, ios->comp_comm))) + if ((mpierr = MPI_Allreduce(MPI_IN_PLACE, &needsflush, 1, MPI_INT, MPI_MAX, + ios->comp_comm))) return check_mpi(file, mpierr, __FILE__, __LINE__); LOG((2, "needsflush = %d", needsflush)); /* Flush data if needed. */ if (needsflush > 0) { - LOG((2, "maxfree = %ld wmb->validvars = %d (1 + wmb->validvars) * arraylen * tsize = %ld totfree = %ld\n", - maxfree, wmb->validvars, (1 + wmb->validvars) * arraylen * tsize, totfree)); - #ifdef PIO_ENABLE_LOGGING /* Collect a debug report about buffer. */ cn_buffer_report(ios, true); + LOG((2, "maxfree = %ld wmb->num_arrays = %d (1 + wmb->num_arrays) *" + " arraylen * iodesc->basetype_size = %ld totfree = %ld\n", maxfree, wmb->num_arrays, + (1 + wmb->num_arrays) * arraylen * iodesc->basetype_size, totfree)); #endif /* PIO_ENABLE_LOGGING */ /* If needsflush == 2 flush to disk otherwise just flush to io node. */ @@ -443,91 +496,119 @@ int PIOc_write_darray(int ncid, int varid, int ioid, PIO_Offset arraylen, void * /* Get memory for data. */ if (arraylen > 0) { - if (!(wmb->data = bgetr(wmb->data, (1 + wmb->validvars) * arraylen * tsize))) - piomemerror(ios, (1 + wmb->validvars) * arraylen * tsize, __FILE__, __LINE__); - LOG((2, "got %ld bytes for data", (1 + wmb->validvars) * arraylen * tsize)); + if (!(wmb->data = bgetr(wmb->data, (1 + wmb->num_arrays) * arraylen * iodesc->basetype_size))) + return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__); + LOG((2, "got %ld bytes for data", (1 + wmb->num_arrays) * arraylen * iodesc->basetype_size)); } /* vid is an array of variable ids in the wmb list, grow the list * and add the new entry. */ - if (!(wmb->vid = bgetr(wmb->vid, sizeof(int) * (1 + wmb->validvars)))) - piomemerror(ios, (1 + wmb->validvars) * sizeof(int), __FILE__, __LINE__); + if (!(wmb->vid = bgetr(wmb->vid, sizeof(int) * (1 + wmb->num_arrays)))) + return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__); /* wmb->frame is the record number, we assume that the variables * in the wmb list may not all have the same unlimited dimension * value although they usually do. */ if (vdesc->record >= 0) - if (!(wmb->frame = bgetr(wmb->frame, sizeof(int) * (1 + wmb->validvars)))) - piomemerror(ios, (1 + wmb->validvars) * sizeof(int), __FILE__, __LINE__); + if (!(wmb->frame = bgetr(wmb->frame, sizeof(int) * (1 + wmb->num_arrays)))) + return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__); - /* If we need a fill value, get it. */ + /* If we need a fill value, get it. If we are using the subset + * rearranger and not using the netcdf fill mode then we need to + * do an extra write to fill in the holes with the fill value. */ if (iodesc->needsfill) { /* Get memory to hold fill value. */ - if (!(wmb->fillvalue = bgetr(wmb->fillvalue, tsize * (1 + wmb->validvars)))) - piomemerror(ios, (1 + wmb->validvars) * tsize, __FILE__, __LINE__); + if (!(wmb->fillvalue = bgetr(wmb->fillvalue, iodesc->basetype_size * (1 + wmb->num_arrays)))) + return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__); /* If the user passed a fill value, use that, otherwise use * the default fill value of the netCDF type. Copy the fill * value to the buffer. */ if (fillvalue) { - memcpy((char *)wmb->fillvalue + tsize * wmb->validvars, fillvalue, tsize); - LOG((3, "copied user-provided fill value tsize = %d", tsize)); + memcpy((char *)wmb->fillvalue + iodesc->basetype_size * wmb->num_arrays, + fillvalue, iodesc->basetype_size); + LOG((3, "copied user-provided fill value iodesc->basetype_size = %d", + iodesc->basetype_size)); } else { + void *fill; + signed char byte_fill = PIO_FILL_BYTE; + char char_fill = PIO_FILL_CHAR; + short short_fill = PIO_FILL_SHORT; + int int_fill = PIO_FILL_INT; + float float_fill = PIO_FILL_FLOAT; + double double_fill = PIO_FILL_DOUBLE; +#ifdef _NETCDF4 + unsigned char ubyte_fill = PIO_FILL_UBYTE; + unsigned short ushort_fill = PIO_FILL_USHORT; + unsigned int uint_fill = PIO_FILL_UINT; + long long int64_fill = PIO_FILL_INT64; + long long uint64_fill = PIO_FILL_UINT64; +#endif /* _NETCDF4 */ vtype = (MPI_Datatype)iodesc->basetype; LOG((3, "caller did not provide fill value vtype = %d", vtype)); - if (vtype == MPI_INT) - { - int fill = PIO_FILL_INT; - memcpy((char *)wmb->fillvalue + tsize * wmb->validvars, &fill, tsize); - } + + /* This must be done with an if statement, not a case, or + * openmpi will not build. */ + if (vtype == MPI_BYTE) + fill = &byte_fill; + else if (vtype == MPI_CHAR) + fill = &char_fill; + else if (vtype == MPI_SHORT) + fill = &short_fill; + else if (vtype == MPI_INT) + fill = &int_fill; else if (vtype == MPI_FLOAT) - { - float fill = PIO_FILL_FLOAT; - memcpy((char *)wmb->fillvalue + tsize * wmb->validvars, &fill, tsize); - } + fill = &float_fill; else if (vtype == MPI_DOUBLE) - { - double fill = PIO_FILL_DOUBLE; - memcpy((char *)wmb->fillvalue + tsize * wmb->validvars, &fill, tsize); - } - else if (vtype == MPI_CHARACTER) - { - char fill = PIO_FILL_CHAR; - memcpy((char *)wmb->fillvalue + tsize * wmb->validvars, &fill, tsize); - } + fill = &double_fill; +#ifdef _NETCDF4 + else if (vtype == MPI_UNSIGNED_CHAR) + fill = &ubyte_fill; + else if (vtype == MPI_UNSIGNED_SHORT) + fill = &ushort_fill; + else if (vtype == MPI_UNSIGNED) + fill = &uint_fill; + else if (vtype == MPI_LONG_LONG) + fill = &int64_fill; + else if (vtype == MPI_UNSIGNED_LONG_LONG) + fill = &uint64_fill; +#endif /* _NETCDF4 */ else return pio_err(ios, file, PIO_EBADTYPE, __FILE__, __LINE__); + + memcpy((char *)wmb->fillvalue + iodesc->basetype_size * wmb->num_arrays, + fill, iodesc->basetype_size); + LOG((3, "copied fill value")); } } /* Tell the buffer about the data it is getting. */ wmb->arraylen = arraylen; - wmb->vid[wmb->validvars] = varid; + wmb->vid[wmb->num_arrays] = varid; + LOG((3, "wmb->num_arrays = %d wmb->vid[wmb->num_arrays] = %d", wmb->num_arrays, + wmb->vid[wmb->num_arrays])); /* Copy the user-provided data to the buffer. */ - bufptr = (void *)((char *)wmb->data + arraylen * tsize * wmb->validvars); + bufptr = (void *)((char *)wmb->data + arraylen * iodesc->basetype_size * wmb->num_arrays); if (arraylen > 0) { - memcpy(bufptr, array, arraylen * tsize); - LOG((3, "copied %ld bytes of user data", arraylen * tsize)); + memcpy(bufptr, array, arraylen * iodesc->basetype_size); + LOG((3, "copied %ld bytes of user data", arraylen * iodesc->basetype_size)); } /* Add the unlimited dimension value of this variable to the frame * array in wmb. */ if (wmb->frame) - wmb->frame[wmb->validvars] = vdesc->record; - wmb->validvars++; + wmb->frame[wmb->num_arrays] = vdesc->record; + wmb->num_arrays++; - LOG((2, "wmb->validvars = %d iodesc->maxbytes / tsize = %d iodesc->ndof = %d iodesc->llen = %d", - wmb->validvars, iodesc->maxbytes / tsize, iodesc->ndof, iodesc->llen)); - - /* Call the sync when ??? */ - if (wmb->validvars >= iodesc->maxbytes / tsize) - PIOc_sync(ncid); + LOG((2, "wmb->num_arrays = %d iodesc->maxbytes / iodesc->basetype_size = %d " + "iodesc->ndof = %d iodesc->llen = %d", wmb->num_arrays, + iodesc->maxbytes / iodesc->basetype_size, iodesc->ndof, iodesc->llen)); return PIO_NOERR; } @@ -556,8 +637,6 @@ int PIOc_read_darray(int ncid, int varid, int ioid, PIO_Offset arraylen, io_desc_t *iodesc; /* Pointer to IO description information. */ void *iobuf = NULL; /* holds the data as read on the io node. */ size_t rlen = 0; /* the length of data in iobuf. */ - int tsize; /* Total size. */ - int mpierr; /* Return code from MPI functions. */ int ierr; /* Return code. */ /* Get the file info. */ @@ -568,6 +647,8 @@ int PIOc_read_darray(int ncid, int varid, int ioid, PIO_Offset arraylen, /* Get the iodesc. */ if (!(iodesc = pio_get_iodesc_from_id(ioid))) return pio_err(ios, file, PIO_EBADID, __FILE__, __LINE__); + pioassert(iodesc->rearranger == PIO_REARR_BOX || iodesc->rearranger == PIO_REARR_SUBSET, + "unknown rearranger", __FILE__, __LINE__); /* ??? */ if (ios->iomaster == MPI_ROOT) @@ -575,24 +656,10 @@ int PIOc_read_darray(int ncid, int varid, int ioid, PIO_Offset arraylen, else rlen = iodesc->llen; - /* Is a rearranger in use? */ - if (iodesc->rearranger > 0) - { - if (ios->ioproc && rlen > 0) - { - /* Get the MPI type size. */ - if ((mpierr = MPI_Type_size(iodesc->basetype, &tsize))) - return check_mpi(file, mpierr, __FILE__, __LINE__); - - /* Allocate a buffer for one record. */ - if (!(iobuf = bget((size_t)tsize * rlen))) - piomemerror(ios, rlen * (size_t)tsize, __FILE__, __LINE__); - } - } - else - { - iobuf = array; - } + /* Allocate a buffer for one record. */ + if (ios->ioproc && rlen > 0) + if (!(iobuf = bget(iodesc->basetype_size * rlen))) + return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__); /* Call the correct darray read function based on iotype. */ switch (file->iotype) @@ -611,16 +678,13 @@ int PIOc_read_darray(int ncid, int varid, int ioid, PIO_Offset arraylen, return pio_err(NULL, NULL, PIO_EBADIOTYPE, __FILE__, __LINE__); } - /* If a rearranger was specified, rearrange the data. */ - if (iodesc->rearranger > 0) - { - if ((ierr = rearrange_io2comp(ios, iodesc, iobuf, array))) - return pio_err(ios, file, ierr, __FILE__, __LINE__); + /* Rearrange the data. */ + if ((ierr = rearrange_io2comp(ios, iodesc, iobuf, array))) + return pio_err(ios, file, ierr, __FILE__, __LINE__); - /* Free the buffer. */ - if (rlen > 0) - brel(iobuf); - } + /* Free the buffer. */ + if (rlen > 0) + brel(iobuf); return PIO_NOERR; } diff --git a/src/externals/pio2/src/clib/pio_darray_int.c b/src/externals/pio2/src/clib/pio_darray_int.c index ac19b37e198..79572219b91 100644 --- a/src/externals/pio2/src/clib/pio_darray_int.c +++ b/src/externals/pio2/src/clib/pio_darray_int.c @@ -25,6 +25,16 @@ extern void *CN_bpool; /* Maximum buffer usage. */ extern PIO_Offset maxusage; +/* handler for freeing the memory buffer pool */ +void bpool_free(void *p) +{ + free(p); + if(p == CN_bpool){ + CN_bpool = NULL; + } +} + + /** * Initialize the compute buffer to size pio_cnbuffer_limit. * @@ -49,10 +59,10 @@ int compute_buffer_init(iosystem_desc_t *ios) if (!CN_bpool) return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); - bectl(NULL, malloc, free, pio_cnbuffer_limit); + bectl(NULL, malloc, bpool_free, pio_cnbuffer_limit); } #endif - LOG((2, "compute_buffer_init CN_bpool = %d", CN_bpool)); + LOG((2, "compute_buffer_init complete")); return PIO_NOERR; } @@ -60,7 +70,7 @@ int compute_buffer_init(iosystem_desc_t *ios) /** * Write a set of one or more aggregated arrays to output file. This * function is only used with parallel-netcdf and netcdf-4 parallel - * iotypes. Serial io types use pio_write_darray_multi_nc_serial(). + * iotypes. Serial io types use write_darray_multi_serial(). * * @param file a pointer to the open file descriptor for the file * that will be written to @@ -85,7 +95,7 @@ int compute_buffer_init(iosystem_desc_t *ios) * less than blocksize*numiotasks then some iotasks will have a NULL * iobuf. * @param frame the frame or record dimension for each of the nvars - * variables in iobuf. + * variables in iobuf. NULL if this iodesc contains non-record vars. * @return 0 for success, error code otherwise. * @ingroup PIO_write_darray */ @@ -122,7 +132,7 @@ int pio_write_darray_multi_nc(file_desc_t *file, int nvars, const int *vid, int vdesc = file->varlist + vid[0]; /* If async is in use, send message to IO master task. */ - if (ios->async_interface) + if (ios->async) { if (!ios->ioproc) { @@ -350,6 +360,298 @@ int pio_write_darray_multi_nc(file_desc_t *file, int nvars, const int *vid, int return ierr; } +/** + * Fill the tmp_start and tmp_count arrays, which contain the start + * and count arrays for all regions. + * + * This is an internal function which is only called on io tasks. It + * is called by write_darray_multi_serial(). + * + * @param region pointer to the first in a linked list of regions. + * @param maxregions the number of regions in the list. + * @param fndims the number of dimensions in the file. + * @param iodesc_ndims the number of dimensions in the decomposition. + * @param vdesc pointer to an array of var_desc_t for the vars being + * written. + * @param tmp_start pointer to an already allocaed array of length + * fndims * maxregions. This array will get the start values for all + * regions. + * @param tmp_count pointer to an already allocaed array of length + * fndims * maxregions. This array will get the count values for all + * regions. + * @returns 0 for success, error code otherwise. + * @ingroup PIO_read_darray + **/ +int find_all_start_count(io_region *region, int maxregions, int fndims, + int iodesc_ndims, var_desc_t *vdesc, size_t *tmp_start, + size_t *tmp_count) +{ + /* Check inputs. */ + pioassert(maxregions >= 0 && fndims > 0 && iodesc_ndims >= 0 && vdesc && + tmp_start && tmp_count, "invalid input", __FILE__, __LINE__); + + /* Find the start/count arrays for each region in the list. */ + for (int r = 0; r < maxregions; r++) + { + /* Initialize the start/count arrays for this region to 0. */ + for (int d = 0; d < fndims; d++) + { + tmp_start[d + r * fndims] = 0; + tmp_count[d + r * fndims] = 0; + } + + if (region) + { + if (vdesc->record >= 0) + { + /* This is a record based multidimensional + * array. Copy start/count for non-record + * dimensions. */ + for (int i = fndims - iodesc_ndims; i < fndims; i++) + { + tmp_start[i + r * fndims] = region->start[i - (fndims - iodesc_ndims)]; + tmp_count[i + r * fndims] = region->count[i - (fndims - iodesc_ndims)]; + LOG((3, "tmp_start[%d] = %d tmp_count[%d] = %d", i + r * fndims, + tmp_start[i + r * fndims], i + r * fndims, + tmp_count[i + r * fndims])); + } + } + else + { + /* This is not a record based multidimensional array. */ + for (int i = 0; i < iodesc_ndims; i++) + { + tmp_start[i + r * fndims] = region->start[i]; + tmp_count[i + r * fndims] = region->count[i]; + LOG((3, "tmp_start[%d] = %d tmp_count[%d] = %d", i + r * fndims, + tmp_start[i + r * fndims], i + r * fndims, + tmp_count[i + r * fndims])); + } + } + + /* Move to next region. */ + region = region->next; + + } /* endif region */ + } /* next r */ + + return PIO_NOERR; +} + +/** + * Internal function called by IO tasks other than IO task 0 to send + * their tmp_start/tmp_count arrays to IO task 0. + * + * This is an internal function which is only called on io tasks other + * than IO task 0. It is called by write_darray_multi_serial(). + * + * @return 0 for success, error code otherwise. + * @ingroup PIO_write_darray + **/ +int send_all_start_count(iosystem_desc_t *ios, io_desc_t *iodesc, PIO_Offset llen, + int maxregions, int nvars, int fndims, size_t *tmp_start, + size_t *tmp_count, void *iobuf) +{ + MPI_Status status; /* Recv status for MPI. */ + int mpierr; /* Return code from MPI function codes. */ + int ierr; /* Return code. */ + + /* Check inputs. */ + pioassert(ios && ios->ioproc && ios->io_rank > 0 && maxregions >= 0, + "invalid inputs", __FILE__, __LINE__); + + /* Do a handshake. */ + if ((mpierr = MPI_Recv(&ierr, 1, MPI_INT, 0, 0, ios->io_comm, &status))) + return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); + + /* Send local length of iobuffer for each field (all + * fields are the same length). */ + if ((mpierr = MPI_Send((void *)&llen, 1, MPI_OFFSET, 0, ios->io_rank, ios->io_comm))) + return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); + LOG((3, "sent llen = %d", llen)); + + /* Send the number of data regions, the start/count for + * all regions, and the data buffer with all the data. */ + if (llen > 0) + { + if ((mpierr = MPI_Send((void *)&maxregions, 1, MPI_INT, 0, ios->io_rank + ios->num_iotasks, + ios->io_comm))) + return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); + if ((mpierr = MPI_Send(tmp_start, maxregions * fndims, MPI_OFFSET, 0, + ios->io_rank + 2 * ios->num_iotasks, ios->io_comm))) + return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); + if ((mpierr = MPI_Send(tmp_count, maxregions * fndims, MPI_OFFSET, 0, + ios->io_rank + 3 * ios->num_iotasks, ios->io_comm))) + return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); + if ((mpierr = MPI_Send(iobuf, nvars * llen, iodesc->basetype, 0, + ios->io_rank + 4 * ios->num_iotasks, ios->io_comm))) + return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); + LOG((3, "sent data for maxregions = %d", maxregions)); + } + + return PIO_NOERR; +} + +/** + * This is an internal function that is run only on IO proc 0. It + * receives data from all the other IO tasks, and write that data to + * disk. This is called from write_darray_multi_serial(). + * + * @param file a pointer to the open file descriptor for the file + * that will be written to. + * @param vid an array of the variable ids to be written + * @param frame the record dimension for each of the nvars variables + * in iobuf. NULL if this iodesc contains non-record vars. + * @param iodesc pointer to the decomposition info. + * @param llen length of the iobuffer on this task for a single + * field. + * @param maxregions max number of blocks to be written from this + * iotask. + * @param nvars the number of variables to be written with this + * decomposition. + * @param fndims the number of dimensions in the file. + * @param tmp_start pointer to an already allocaed array of length + * fndims * maxregions. This array will get the start values for all + * regions. + * @param tmp_count pointer to an already allocaed array of length + * fndims * maxregions. This array will get the count values for all + * regions. + * @param iobuf the buffer to be written from this mpi task. May be + * null. for example we have 8 ionodes and a distributed array with + * global size 4, then at least 4 nodes will have a null iobuf. In + * practice the box rearranger trys to have at least blocksize bytes + * on each io task and so if the total number of bytes to write is + * less than blocksize*numiotasks then some iotasks will have a NULL + * iobuf. + * @return 0 for success, error code otherwise. + * @ingroup PIO_write_darray + */ +int recv_and_write_data(file_desc_t *file, const int *vid, const int *frame, + io_desc_t *iodesc, PIO_Offset llen, int maxregions, int nvars, + int fndims, size_t *tmp_start, size_t *tmp_count, void *iobuf) +{ + iosystem_desc_t *ios; /* Pointer to io system information. */ + size_t rlen; /* Length of IO buffer on this task. */ + int rregions; /* Number of regions in buffer for this task. */ + size_t start[fndims], count[fndims]; + size_t loffset; + void *bufptr; + var_desc_t *vdesc; /* Contains info about the variable. */ + MPI_Status status; /* Recv status for MPI. */ + int mpierr; /* Return code from MPI function codes. */ + int ierr; /* Return code. */ + + ios = file->iosystem; + + /* For each of the other tasks that are using this task + * for IO. */ + for (int rtask = 0; rtask < ios->num_iotasks; rtask++) + { + /* From the remote tasks, we send information about + * the data regions. and also the data. */ + if (rtask) + { + /* handshake - tell the sending task I'm ready */ + if ((mpierr = MPI_Send(&ierr, 1, MPI_INT, rtask, 0, ios->io_comm))) + return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); + + /* Get length of iobuffer for each field on this + * task (all fields are the same length). */ + if ((mpierr = MPI_Recv(&rlen, 1, MPI_OFFSET, rtask, rtask, ios->io_comm, + &status))) + return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); + LOG((3, "received rlen = %d", rlen)); + + /* Get the number of regions, the start/count + * values for all regions, and the data buffer. */ + if (rlen > 0) + { + if ((mpierr = MPI_Recv(&rregions, 1, MPI_INT, rtask, rtask + ios->num_iotasks, + ios->io_comm, &status))) + return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); + if ((mpierr = MPI_Recv(tmp_start, rregions * fndims, MPI_OFFSET, rtask, + rtask + 2 * ios->num_iotasks, ios->io_comm, &status))) + return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); + if ((mpierr = MPI_Recv(tmp_count, rregions * fndims, MPI_OFFSET, rtask, + rtask + 3 * ios->num_iotasks, ios->io_comm, &status))) + return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); + if ((mpierr = MPI_Recv(iobuf, nvars * rlen, iodesc->basetype, rtask, + rtask + 4 * ios->num_iotasks, ios->io_comm, &status))) + return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); + LOG((3, "received data rregions = %d fndims = %d", rregions, fndims)); + } + } + else /* task 0 */ + { + rlen = llen; + rregions = maxregions; + } + LOG((3, "rtask = %d rlen = %d rregions = %d", rtask, rlen, rregions)); + + /* If there is data from this task, write it. */ + if (rlen > 0) + { + loffset = 0; + for (int regioncnt = 0; regioncnt < rregions; regioncnt++) + { + LOG((3, "writing data for region with regioncnt = %d", regioncnt)); + + /* Get the start/count arrays for this region. */ + for (int i = 0; i < fndims; i++) + { + start[i] = tmp_start[i + regioncnt * fndims]; + count[i] = tmp_count[i + regioncnt * fndims]; + LOG((3, "start[%d] = %d count[%d] = %d", i, start[i], i, count[i])); + } + + /* Process each variable in the buffer. */ + for (int nv = 0; nv < nvars; nv++) + { + LOG((3, "writing buffer var %d", nv)); + vdesc = file->varlist + vid[0]; + + /* Get a pointer to the correct part of the buffer. */ + bufptr = (void *)((char *)iobuf + iodesc->basetype_size * (nv * rlen + loffset)); + + /* If this var has an unlimited dim, set + * the start on that dim to the frame + * value for this variable. */ + if (vdesc->record >= 0) + { + if (fndims > 1 && iodesc->ndims < fndims && count[1] > 0) + { + count[0] = 1; + start[0] = frame[nv]; + } + else if (fndims == iodesc->ndims) + { + start[0] += vdesc->record; + } + } + + /* Call the netCDF functions to write the data. */ + if ((ierr = nc_put_vara(file->fh, vid[nv], start, count, bufptr))) + return check_netcdf2(ios, NULL, ierr, __FILE__, __LINE__); + + } /* next var */ + + /* Calculate the total size. */ + size_t tsize = 1; + for (int i = 0; i < fndims; i++) + tsize *= count[i]; + + /* Keep track of where we are in the buffer. */ + loffset += tsize; + + LOG((3, " at bottom of loop regioncnt = %d tsize = %d loffset = %d", regioncnt, + tsize, loffset)); + } /* next regioncnt */ + } /* endif (rlen > 0) */ + } /* next rtask */ + + return PIO_NOERR; +} + /** * Write a set of one or more aggregated arrays to output file in * serial mode. This function is called for netCDF classic and @@ -361,16 +663,13 @@ int pio_write_darray_multi_nc(file_desc_t *file, int nvars, const int *vid, int * @param nvars the number of variables to be written with this * decomposition. * @param vid an array of the variable ids to be written - * @param iodesc_ndims the number of dimensions explicitly in the - * iodesc. - * @param basetype the basic type of the minimal data unit + * @param iodesc pointer to the decomposition info. * @param maxregions max number of blocks to be written from this * iotask. * @param firstregion pointer to the first element of a linked * list of region descriptions. May be NULL. * @param llen length of the iobuffer on this task for a single * field. - * @param num_aiotasks actual number of iotasks participating * @param iobuf the buffer to be written from this mpi task. May be * null. for example we have 8 ionodes and a distributed array with * global size 4, then at least 4 nodes will have a null iobuf. In @@ -379,48 +678,47 @@ int pio_write_darray_multi_nc(file_desc_t *file, int nvars, const int *vid, int * less than blocksize*numiotasks then some iotasks will have a NULL * iobuf. * @param frame the record dimension for each of the nvars variables - * in iobuf. + * in iobuf. NULL if this iodesc contains non-record vars. * @return 0 for success, error code otherwise. * @ingroup PIO_write_darray */ -int pio_write_darray_multi_nc_serial(file_desc_t *file, int nvars, const int *vid, int iodesc_ndims, - MPI_Datatype basetype, int maxregions, io_region *firstregion, - PIO_Offset llen, int num_aiotasks, void *iobuf, - const int *frame) +int write_darray_multi_serial(file_desc_t *file, int nvars, const int *vid, + io_desc_t *iodesc, int fill, const int *frame) { iosystem_desc_t *ios; /* Pointer to io system information. */ var_desc_t *vdesc; /* Contains info about the variable. */ - int dsize; /* Size in bytes of one element of basetype. */ int fndims; /* Number of dims in the var in the file. */ - int tsize; /* Size of the MPI type, in bytes. */ - MPI_Status status; /* Recv status for MPI. */ int mpierr = MPI_SUCCESS, mpierr2; /* Return code from MPI function codes. */ int ierr; /* Return code. */ /* Check inputs. */ - pioassert(file && file->iosystem && vid && vid[0] >= 0 && vid[0] <= PIO_MAX_VARS, - "invalid input", __FILE__, __LINE__); + pioassert(file && file->iosystem && file->varlist && vid && vid[0] >= 0 && + vid[0] <= PIO_MAX_VARS && iodesc, "invalid input", __FILE__, __LINE__); - LOG((1, "pio_write_darray_multi_nc_serial nvars = %d iodesc_ndims = %d basetype = %d " - "maxregions = %d llen = %d num_aiotasks = %d", nvars, iodesc_ndims, - basetype, maxregions, llen, num_aiotasks)); - -#ifdef TIMING - /* Start timing this function. */ - GPTLstart("PIO:write_darray_multi_nc_serial"); -#endif + LOG((1, "write_darray_multi_serial nvars = %d iodesc->ndims = %d iodesc->basetype = %d", + nvars, iodesc->ndims, iodesc->basetype)); /* Get the iosystem info. */ ios = file->iosystem; /* Get the var info. */ vdesc = file->varlist + vid[0]; + LOG((2, "vdesc record %d ndims %d nreqs %d ios->async = %d", vdesc->record, + vdesc->ndims, vdesc->nreqs, ios->async)); - LOG((2, "vdesc record %d ndims %d nreqs %d ios->async_interface = %d", vdesc->record, - vdesc->ndims, vdesc->nreqs, ios->async_interface)); + /* Set these differently for data and fill writing. */ + int num_regions = fill ? iodesc->maxfillregions: iodesc->maxregions; + io_region *region = fill ? iodesc->fillregion : iodesc->firstregion; + PIO_Offset llen = fill ? iodesc->holegridsize : iodesc->llen; + void *iobuf = fill ? vdesc->fillbuf : vdesc->iobuf; + +#ifdef TIMING + /* Start timing this function. */ + GPTLstart("PIO:write_darray_multi_nc_serial"); +#endif /* If async is in use, and this is not an IO task, bcast the parameters. */ - if (ios->async_interface) + if (ios->async) { if (!ios->ioproc) { @@ -444,220 +742,36 @@ int pio_write_darray_multi_nc_serial(file_desc_t *file, int nvars, const int *vi if ((ierr = PIOc_inq_varndims(file->pio_ncid, vid[0], &fndims))) return pio_err(ios, file, ierr, __FILE__, __LINE__); - /* Get the size of the type. */ - if ((mpierr = MPI_Type_size(basetype, &tsize))) - return check_mpi(file, mpierr, __FILE__, __LINE__); - LOG((3, "fndims = %d tsize = %d", fndims, tsize)); - /* Only IO tasks participate in this code. */ if (ios->ioproc) { - io_region *region; - void *bufptr; - size_t tmp_start[fndims * maxregions]; /* A start array for each region. */ - size_t tmp_count[fndims * maxregions]; /* A count array for each region. */ + size_t tmp_start[fndims * num_regions]; /* A start array for each region. */ + size_t tmp_count[fndims * num_regions]; /* A count array for each region. */ - LOG((3, "maxregions = %d", maxregions)); + LOG((3, "num_regions = %d", num_regions)); /* Fill the tmp_start and tmp_count arrays, which contain the * start and count arrays for all regions. */ - region = firstregion; - for (int regioncnt = 0; regioncnt < maxregions; regioncnt++) - { - /* Initialize the start/count arrays for this region to 0. */ - for (int i = 0; i < fndims; i++) - { - tmp_start[i + regioncnt * fndims] = 0; - tmp_count[i + regioncnt * fndims] = 0; - } - - if (region) - { - if (vdesc->record >= 0) - { - /* This is a record based multidimensional - * array. Copy start/count for non-record - * dimensions. */ - for (int i = fndims - iodesc_ndims; i < fndims; i++) - { - tmp_start[i + regioncnt * fndims] = region->start[i - (fndims - iodesc_ndims)]; - tmp_count[i + regioncnt * fndims] = region->count[i - (fndims - iodesc_ndims)]; - LOG((3, "tmp_start[%d] = %d tmp_count[%d] = %d", i + regioncnt * fndims, - tmp_start[i + regioncnt * fndims], i + regioncnt * fndims, - tmp_count[i + regioncnt * fndims])); - } - } - else - { - /* This is not a record based multidimensional array. */ - for (int i = 0; i < iodesc_ndims; i++) - { - tmp_start[i + regioncnt * fndims] = region->start[i]; - tmp_count[i + regioncnt * fndims] = region->count[i]; - LOG((3, "tmp_start[%d] = %d tmp_count[%d] = %d", i + regioncnt * fndims, - tmp_start[i + regioncnt * fndims], i + regioncnt * fndims, - tmp_count[i + regioncnt * fndims])); - } - } - - /* Move to next region. */ - region = region->next; - - } /* endif region */ - } /* next regioncnt */ + if ((ierr = find_all_start_count(region, num_regions, fndims, iodesc->ndims, vdesc, + tmp_start, tmp_count))) + return pio_err(ios, file, ierr, __FILE__, __LINE__); /* Tasks other than 0 will send their data to task 0. */ if (ios->io_rank > 0) { - /* Do a handshake. */ - if ((mpierr = MPI_Recv(&ierr, 1, MPI_INT, 0, 0, ios->io_comm, &status))) - return check_mpi(file, mpierr, __FILE__, __LINE__); - - /* Send local length of iobuffer for each field (all - * fields are the same length). */ - if ((mpierr = MPI_Send((void *)&llen, 1, MPI_OFFSET, 0, ios->io_rank, ios->io_comm))) - return check_mpi(file, mpierr, __FILE__, __LINE__); - LOG((3, "sent llen = %d", llen)); - - /* Send the number of data regions, the start/count for - * all regions, and the data buffer with all the data. */ - if (llen > 0) - { - if ((mpierr = MPI_Send((void *)&maxregions, 1, MPI_INT, 0, ios->io_rank + ios->num_iotasks, ios->io_comm))) - return check_mpi(file, mpierr, __FILE__, __LINE__); - if ((mpierr = MPI_Send(tmp_start, maxregions * fndims, MPI_OFFSET, 0, ios->io_rank + 2 * ios->num_iotasks, - ios->io_comm))) - return check_mpi(file, mpierr, __FILE__, __LINE__); - if ((mpierr = MPI_Send(tmp_count, maxregions * fndims, MPI_OFFSET, 0, ios->io_rank + 3 * ios->num_iotasks, - ios->io_comm))) - return check_mpi(file, mpierr, __FILE__, __LINE__); - if ((mpierr = MPI_Send(iobuf, nvars * llen, basetype, 0, ios->io_rank + 4 * ios->num_iotasks, ios->io_comm))) - return check_mpi(file, mpierr, __FILE__, __LINE__); - LOG((3, "sent data for maxregions = %d", maxregions)); - } + /* Send the tmp_start and tmp_count arrays from this IO task + * to task 0. */ + if ((ierr = send_all_start_count(ios, iodesc, llen, num_regions, nvars, fndims, + tmp_start, tmp_count, iobuf))) + return pio_err(ios, file, ierr, __FILE__, __LINE__); } else { /* Task 0 will receive data from all other IO tasks. */ - size_t rlen; /* Length of IO buffer on this task. */ - int rregions; /* Number of regions in buffer for this task. */ - size_t start[fndims], count[fndims]; - size_t loffset; - - /* Get the size of the MPI data type. */ - if ((mpierr = MPI_Type_size(basetype, &dsize))) - return check_mpi(file, mpierr, __FILE__, __LINE__); - LOG((3, "dsize = %d", dsize)); - - /* For each of the other tasks that are using this task - * for IO. */ - for (int rtask = 0; rtask < ios->num_iotasks; rtask++) - { - /* From the remote tasks, we send information about - * the data regions. and also the data. */ - if (rtask) - { - /* handshake - tell the sending task I'm ready */ - if ((mpierr = MPI_Send(&ierr, 1, MPI_INT, rtask, 0, ios->io_comm))) - return check_mpi(file, mpierr, __FILE__, __LINE__); - - /* Get length of iobuffer for each field on this - * task (all fields are the same length). */ - if ((mpierr = MPI_Recv(&rlen, 1, MPI_OFFSET, rtask, rtask, ios->io_comm, &status))) - return check_mpi(file, mpierr, __FILE__, __LINE__); - LOG((3, "received rlen = %d", rlen)); - /* Get the number of regions, the start/count - * values for all regions, and the data buffer. */ - if (rlen > 0) - { - if ((mpierr = MPI_Recv(&rregions, 1, MPI_INT, rtask, rtask + ios->num_iotasks, - ios->io_comm, &status))) - return check_mpi(file, mpierr, __FILE__, __LINE__); - if ((mpierr = MPI_Recv(tmp_start, rregions * fndims, MPI_OFFSET, rtask, rtask + 2 * ios->num_iotasks, - ios->io_comm, &status))) - return check_mpi(file, mpierr, __FILE__, __LINE__); - if ((mpierr = MPI_Recv(tmp_count, rregions * fndims, MPI_OFFSET, rtask, rtask + 3 * ios->num_iotasks, - ios->io_comm, &status))) - return check_mpi(file, mpierr, __FILE__, __LINE__); - if ((mpierr = MPI_Recv(iobuf, nvars * rlen, basetype, rtask, rtask + 4 * ios->num_iotasks, ios->io_comm, - &status))) - return check_mpi(file, mpierr, __FILE__, __LINE__); - LOG((3, "received data rregions = %d fndims = %d", rregions, fndims)); - } - } - else /* task 0 */ - { - rlen = llen; - rregions = maxregions; - } - LOG((3, "rtask = %d rlen = %d rregions = %d", rtask, rlen, rregions)); - - /* If there is data from this task, write it. */ - if (rlen > 0) - { - loffset = 0; - for (int regioncnt = 0; regioncnt < rregions; regioncnt++) - { - LOG((3, "writing data for region with regioncnt = %d", regioncnt)); - - /* Get the start/count arrays for this region. */ - for (int i = 0; i < fndims; i++) - { - start[i] = tmp_start[i + regioncnt * fndims]; - count[i] = tmp_count[i + regioncnt * fndims]; - LOG((3, "start[%d] = %d count[%d] = %d", i, start[i], i, count[i])); - } - - /* Process each variable in the buffer. */ - for (int nv = 0; nv < nvars; nv++) - { - LOG((3, "writing buffer var %d", nv)); - - /* Get a pointer to the correct part of the buffer. */ - bufptr = (void *)((char *)iobuf + tsize * (nv * rlen + loffset)); - - /* If this var has an unlimited dim, set - * the start on that dim to the frame - * value for this variable. */ - if (vdesc->record >= 0) - { - if (fndims > 1 && iodesc_ndims < fndims && count[1] > 0) - { - count[0] = 1; - start[0] = frame[nv]; - } - else if (fndims == iodesc_ndims) - { - start[0] += vdesc->record; - } - } - - /* Call the netCDF functions to write the data. */ - ierr = nc_put_vara(file->fh, vid[nv], start, count, bufptr); - - if (ierr) - { - for (int i = 0; i < fndims; i++) - fprintf(stderr, "vid %d dim %d start %ld count %ld \n", vid[nv], i, - start[i], count[i]); - return check_netcdf(file, ierr, __FILE__, __LINE__); - } - } /* next var */ - - /* Calculate the total size. */ - size_t tsize = 1; - for (int i = 0; i < fndims; i++) - tsize *= count[i]; - - /* Keep track of where we are in the buffer. */ - loffset += tsize; - - LOG((3, " at bottom of loop regioncnt = %d tsize = %d loffset = %d", regioncnt, - tsize, loffset)); - } /* next regioncnt */ - } /* endif (rlen > 0) */ - } /* next rtask */ + if ((ierr = recv_and_write_data(file, vid, frame, iodesc, llen, num_regions, nvars, fndims, + tmp_start, tmp_count, iobuf))) + return pio_err(ios, file, ierr, __FILE__, __LINE__); } } @@ -692,7 +806,6 @@ int pio_read_darray_nc(file_desc_t *file, io_desc_t *iodesc, int vid, void *iobu var_desc_t *vdesc; /* Information about the variable. */ int ndims; /* Number of dims in decomposition. */ int fndims; /* Number of dims for this var in file. */ - int mpierr; /* Return code from MPI functions. */ int ierr; /* Return code from netCDF functions. */ /* Check inputs. */ @@ -729,7 +842,6 @@ int pio_read_darray_nc(file_desc_t *file, io_desc_t *iodesc, int vid, void *iobu size_t count[fndims]; size_t tmp_bufsize = 1; void *bufptr; - int tsize; int rrlen = 0; PIO_Offset *startlist[iodesc->maxregions]; PIO_Offset *countlist[iodesc->maxregions]; @@ -739,10 +851,6 @@ int pio_read_darray_nc(file_desc_t *file, io_desc_t *iodesc, int vid, void *iobu the basetype. */ region = iodesc->firstregion; - /* Get the size of the MPI type. */ - if ((mpierr = MPI_Type_size(iodesc->basetype, &tsize))) - return check_mpi(file, mpierr, __FILE__, __LINE__); - /* ??? */ if (fndims > ndims) { @@ -771,7 +879,7 @@ int pio_read_darray_nc(file_desc_t *file, io_desc_t *iodesc, int vid, void *iobu if (regioncnt == 0 || region == NULL) bufptr = iobuf; else - bufptr=(void *)((char *)iobuf + tsize * region->loffset); + bufptr=(void *)((char *)iobuf + iodesc->basetype_size * region->loffset); LOG((2, "%d %d %d", iodesc->llen - region->loffset, iodesc->llen, region->loffset)); @@ -932,17 +1040,12 @@ int pio_read_darray_nc_serial(file_desc_t *file, io_desc_t *iodesc, int vid, size_t tmp_count[fndims * iodesc->maxregions]; size_t tmp_bufsize; void *bufptr; - int tsize; /* buffer is incremented by byte and loffset is in terms of the iodessc->basetype so we need to multiply by the size of the basetype. */ region = iodesc->firstregion; - /* Get the size of the MPI type. */ - if ((mpierr = MPI_Type_size(iodesc->basetype, &tsize))) - return check_mpi(file, mpierr, __FILE__, __LINE__); - if (fndims > ndims) { if (vdesc->record < 0) @@ -1074,7 +1177,7 @@ int pio_read_darray_nc_serial(file_desc_t *file, io_desc_t *iodesc, int vid, for (int regioncnt = 0; regioncnt < maxregions; regioncnt++) { /* Get pointer where data should go. */ - bufptr = (void *)((char *)iobuf + tsize * loffset); + bufptr = (void *)((char *)iobuf + iodesc->basetype_size * loffset); regionsize = 1; /* ??? */ @@ -1151,7 +1254,9 @@ int flush_output_buffer(file_desc_t *file, bool force, PIO_Offset addsize) /* Find out the buffer usage. */ if ((ierr = ncmpi_inq_buffer_usage(file->fh, &usage))) - return pio_err(NULL, file, PIO_EBADID, __FILE__, __LINE__); + /* allow the buffer to be undefined */ + if (ierr != NC_ENULLABUF) + return pio_err(NULL, file, PIO_EBADID, __FILE__, __LINE__); /* If we are not forcing a flush, spread the usage to all IO * tasks. */ @@ -1223,6 +1328,7 @@ int flush_output_buffer(file_desc_t *file, bool force, PIO_Offset addsize) vdesc = file->varlist + i; if (vdesc->iobuf) { + LOG((3,"freeing variable buffer in flush_output_buffer")); brel(vdesc->iobuf); vdesc->iobuf = NULL; } @@ -1298,6 +1404,7 @@ void free_cn_buffer_pool(iosystem_desc_t *ios) { #if !PIO_USE_MALLOC LOG((2, "free_cn_buffer_pool CN_bpool = %d", CN_bpool)); + /* Note: it is possible that CN_bpool has been freed and set to NULL by bpool_free() */ if (CN_bpool) { cn_buffer_report(ios, false); @@ -1312,9 +1419,9 @@ void free_cn_buffer_pool(iosystem_desc_t *ios) /** * Flush the buffer. * - * @param ncid identifies the netCDF file - * @param wmb May be NULL, in which case function returns. - * @param flushtodisk + * @param ncid identifies the netCDF file. + * @param wmb pointer to the wmulti_buffer structure. + * @param flushtodisk if true, then flush data to disk. * @returns 0 for success, error code otherwise. * @ingroup PIO_write_darray */ @@ -1333,14 +1440,14 @@ int flush_buffer(int ncid, wmulti_buffer *wmb, bool flushtodisk) LOG((1, "flush_buffer ncid = %d flushtodisk = %d", ncid, flushtodisk)); /* If there are any variables in this buffer... */ - if (wmb->validvars > 0) + if (wmb->num_arrays > 0) { /* Write any data in the buffer. */ - ret = PIOc_write_darray_multi(ncid, wmb->vid, wmb->ioid, wmb->validvars, + ret = PIOc_write_darray_multi(ncid, wmb->vid, wmb->ioid, wmb->num_arrays, wmb->arraylen, wmb->data, wmb->frame, wmb->fillvalue, flushtodisk); - wmb->validvars = 0; + wmb->num_arrays = 0; /* Release the list of variable IDs. */ brel(wmb->vid); @@ -1368,11 +1475,11 @@ int flush_buffer(int ncid, wmulti_buffer *wmb, bool flushtodisk) } /** - * Compute the maximum aggregate number of bytes. + * Compute the maximum aggregate number of bytes. This is called by + * subset_rearrange_create() and box_rearrange_create(). * - * @param ios the IO system structure - * @param iodesc a pointer to the defined iodescriptor for the - * buffer. If NULL, function returns immediately. + * @param ios pointer to the IO system structure. + * @param iodesc a pointer to decomposition description. * @returns 0 for success, error code otherwise. */ int compute_maxaggregate_bytes(iosystem_desc_t *ios, io_desc_t *iodesc) @@ -1388,19 +1495,27 @@ int compute_maxaggregate_bytes(iosystem_desc_t *ios, io_desc_t *iodesc) LOG((2, "compute_maxaggregate_bytes iodesc->maxiobuflen = %d iodesc->ndof = %d", iodesc->maxiobuflen, iodesc->ndof)); + /* Determine the max bytes that can be held on IO task. */ if (ios->ioproc && iodesc->maxiobuflen > 0) maxbytesoniotask = pio_buffer_size_limit / iodesc->maxiobuflen; + /* Determine the max bytes that can be held on computation task. */ if (ios->comp_rank >= 0 && iodesc->ndof > 0) maxbytesoncomputetask = pio_cnbuffer_limit / iodesc->ndof; + /* Take the min of the max IO and max comp bytes. */ maxbytes = min(maxbytesoniotask, maxbytesoncomputetask); LOG((2, "compute_maxaggregate_bytes maxbytesoniotask = %d maxbytesoncomputetask = %d", maxbytesoniotask, maxbytesoncomputetask)); + /* Get the min value of this on all tasks. */ + LOG((3, "before allreaduce maxbytes = %d", maxbytes)); if ((mpierr = MPI_Allreduce(MPI_IN_PLACE, &maxbytes, 1, MPI_INT, MPI_MIN, ios->union_comm))) return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); + LOG((3, "after allreaduce maxbytes = %d", maxbytes)); + + /* Remember the result. */ iodesc->maxbytes = maxbytes; return PIO_NOERR; diff --git a/src/externals/pio2/src/clib/pio_file.c b/src/externals/pio2/src/clib/pio_file.c index f799e25f57c..c0523997cfd 100644 --- a/src/externals/pio2/src/clib/pio_file.c +++ b/src/externals/pio2/src/clib/pio_file.c @@ -106,8 +106,10 @@ int PIOc_createfile(int iosysid, int *ncidp, int *iotype, const char *filename, return pio_err(ios, NULL, ret, __FILE__, __LINE__); /* Run this on all tasks if async is not in use, but only on - * non-IO tasks if async is in use. */ - if (!ios->async_interface || !ios->ioproc) + * non-IO tasks if async is in use. (Because otherwise, in async + * mode, set_fill would be called twice by each IO task, since + * PIOc_createfile() will already be called on each IO task.) */ + if (!ios->async || !ios->ioproc) { /* Set the fill mode to NOFILL. */ if ((ret = PIOc_set_fill(*ncidp, NC_NOFILL, NULL))) @@ -174,7 +176,7 @@ int PIOc_closefile(int ncid) /* Sync changes before closing on all tasks if async is not in * use, but only on non-IO tasks if async is in use. */ - if (!ios->async_interface || !ios->ioproc) + if (!ios->async || !ios->ioproc) if (file->mode & PIO_WRITE) PIOc_sync(ncid); @@ -182,7 +184,7 @@ int PIOc_closefile(int ncid) * sends a msg to the pio_msg_handler running on the IO master and * waiting for a message. Then broadcast the ncid over the intercomm * to the IO tasks. */ - if (ios->async_interface) + if (ios->async) { if (!ios->ioproc) { @@ -264,7 +266,7 @@ int PIOc_deletefile(int iosysid, const char *filename) return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__); /* If async is in use, send message to IO master task. */ - if (ios->async_interface) + if (ios->async) { if (!ios->ioproc) { @@ -338,7 +340,7 @@ int PIOc_sync(int ncid) ios = file->iosystem; /* If async is in use, send message to IO master tasks. */ - if (ios->async_interface) + if (ios->async) { if (!ios->ioproc) { @@ -366,7 +368,9 @@ int PIOc_sync(int ncid) wmb = &file->buffer; while (wmb) { - if (wmb->validvars > 0) + /* If there are any data arrays waiting in the + * multibuffer, flush it. */ + if (wmb->num_arrays > 0) flush_buffer(ncid, wmb, true); twmb = wmb; wmb = wmb->next; diff --git a/src/externals/pio2/src/clib/pio_get_nc.c b/src/externals/pio2/src/clib/pio_get_nc.c index e8d54211430..2cd85d8f947 100644 --- a/src/externals/pio2/src/clib/pio_get_nc.c +++ b/src/externals/pio2/src/clib/pio_get_nc.c @@ -619,7 +619,7 @@ int PIOc_get_vara_longlong(int ncid, int varid, const PIO_Offset *start, */ int PIOc_get_var_text(int ncid, int varid, char *buf) { - return PIOc_get_vars_tc(ncid, varid, NULL, NULL, NULL, NC_CHAR, buf); + return PIOc_get_var_tc(ncid, varid, NC_CHAR, buf); } /** @@ -635,7 +635,7 @@ int PIOc_get_var_text(int ncid, int varid, char *buf) */ int PIOc_get_var_uchar(int ncid, int varid, unsigned char *buf) { - return PIOc_get_vars_tc(ncid, varid, NULL, NULL, NULL, NC_UBYTE, buf); + return PIOc_get_var_tc(ncid, varid, NC_UBYTE, buf); } /** @@ -651,7 +651,7 @@ int PIOc_get_var_uchar(int ncid, int varid, unsigned char *buf) */ int PIOc_get_var_schar(int ncid, int varid, signed char *buf) { - return PIOc_get_vars_tc(ncid, varid, NULL, NULL, NULL, NC_BYTE, buf); + return PIOc_get_var_tc(ncid, varid, NC_BYTE, buf); } /** @@ -667,7 +667,7 @@ int PIOc_get_var_schar(int ncid, int varid, signed char *buf) */ int PIOc_get_var_ushort(int ncid, int varid, unsigned short *buf) { - return PIOc_get_vars_tc(ncid, varid, NULL, NULL, NULL, NC_USHORT, buf); + return PIOc_get_var_tc(ncid, varid, NC_USHORT, buf); } /** @@ -683,7 +683,7 @@ int PIOc_get_var_ushort(int ncid, int varid, unsigned short *buf) */ int PIOc_get_var_short(int ncid, int varid, short *buf) { - return PIOc_get_vars_tc(ncid, varid, NULL, NULL, NULL, NC_SHORT, buf); + return PIOc_get_var_tc(ncid, varid, NC_SHORT, buf); } /** @@ -699,7 +699,7 @@ int PIOc_get_var_short(int ncid, int varid, short *buf) */ int PIOc_get_var_uint(int ncid, int varid, unsigned int *buf) { - return PIOc_get_vars_tc(ncid, varid, NULL, NULL, NULL, NC_UINT, buf); + return PIOc_get_var_tc(ncid, varid, NC_UINT, buf); } /** @@ -715,7 +715,7 @@ int PIOc_get_var_uint(int ncid, int varid, unsigned int *buf) */ int PIOc_get_var_int(int ncid, int varid, int *buf) { - return PIOc_get_vars_tc(ncid, varid, NULL, NULL, NULL, NC_INT, buf); + return PIOc_get_var_tc(ncid, varid, NC_INT, buf); } /** @@ -731,7 +731,7 @@ int PIOc_get_var_int(int ncid, int varid, int *buf) */ int PIOc_get_var_long (int ncid, int varid, long *buf) { - return PIOc_get_vars_tc(ncid, varid, NULL, NULL, NULL, PIO_LONG_INTERNAL, buf); + return PIOc_get_var_tc(ncid, varid, PIO_LONG_INTERNAL, buf); } /** @@ -747,7 +747,7 @@ int PIOc_get_var_long (int ncid, int varid, long *buf) */ int PIOc_get_var_float(int ncid, int varid, float *buf) { - return PIOc_get_vars_tc(ncid, varid, NULL, NULL, NULL, NC_FLOAT, buf); + return PIOc_get_var_tc(ncid, varid, NC_FLOAT, buf); } /** @@ -763,7 +763,7 @@ int PIOc_get_var_float(int ncid, int varid, float *buf) */ int PIOc_get_var_double(int ncid, int varid, double *buf) { - return PIOc_get_vars_tc(ncid, varid, NULL, NULL, NULL, NC_DOUBLE, buf); + return PIOc_get_var_tc(ncid, varid, NC_DOUBLE, buf); } /** @@ -779,7 +779,7 @@ int PIOc_get_var_double(int ncid, int varid, double *buf) */ int PIOc_get_var_ulonglong(int ncid, int varid, unsigned long long *buf) { - return PIOc_get_vars_tc(ncid, varid, NULL, NULL, NULL, NC_UINT64, buf); + return PIOc_get_var_tc(ncid, varid, NC_UINT64, buf); } /** @@ -795,7 +795,7 @@ int PIOc_get_var_ulonglong(int ncid, int varid, unsigned long long *buf) */ int PIOc_get_var_longlong(int ncid, int varid, long long *buf) { - return PIOc_get_vars_tc(ncid, varid, NULL, NULL, NULL, NC_INT64, buf); + return PIOc_get_var_tc(ncid, varid, NC_INT64, buf); } /** @@ -1044,7 +1044,7 @@ int PIOc_get_var1_longlong(int ncid, int varid, const PIO_Offset *index, */ int PIOc_get_var(int ncid, int varid, void *buf) { - return PIOc_get_vars_tc(ncid, varid, NULL, NULL, NULL, NC_NAT, buf); + return PIOc_get_var_tc(ncid, varid, NC_NAT, buf); } /** diff --git a/src/externals/pio2/src/clib/pio_getput_int.c b/src/externals/pio2/src/clib/pio_getput_int.c index 32f1968186f..187f74d058d 100644 --- a/src/externals/pio2/src/clib/pio_getput_int.c +++ b/src/externals/pio2/src/clib/pio_getput_int.c @@ -52,7 +52,7 @@ int PIOc_put_att_tc(int ncid, int varid, const char *name, nc_type atttype, /* Run these on all tasks if async is not in use, but only on * non-IO tasks if async is in use. */ - if (!ios->async_interface || !ios->ioproc) + if (!ios->async || !ios->ioproc) { /* Get the length (in bytes) of the type in file. */ if ((ierr = PIOc_inq_type(ncid, atttype, NULL, &atttype_len))) @@ -70,7 +70,7 @@ int PIOc_put_att_tc(int ncid, int varid, const char *name, nc_type atttype, } /* If async is in use, and this is not an IO task, bcast the parameters. */ - if (ios->async_interface) + if (ios->async) { if (!ios->ioproc) { @@ -259,7 +259,7 @@ int PIOc_get_att_tc(int ncid, int varid, const char *name, nc_type memtype, void /* Run these on all tasks if async is not in use, but only on * non-IO tasks if async is in use. */ - if (!ios->async_interface || !ios->ioproc) + if (!ios->async || !ios->ioproc) { /* Get the type and length of the attribute. */ if ((ierr = PIOc_inq_att(ncid, varid, name, &atttype, &attlen))) @@ -284,7 +284,7 @@ int PIOc_get_att_tc(int ncid, int varid, const char *name, nc_type memtype, void /* If async is in use, and this is not an IO task, bcast the * parameters and the attribute and type information we fetched. */ - if (ios->async_interface) + if (ios->async) { if (!ios->ioproc) { @@ -485,19 +485,19 @@ int PIOc_get_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off { iosystem_desc_t *ios; /* Pointer to io system information. */ file_desc_t *file; /* Pointer to file information. */ - int ierr = PIO_NOERR; /* Return code from function calls. */ - int mpierr = MPI_SUCCESS, mpierr2; /* Return code from MPI function codes. */ - int ndims; /* The number of dimensions in the variable. */ - PIO_Offset typelen; /* Size (in bytes) of the data type of data in buf. */ + int ndims; /* The number of dimensions in the variable. */ + PIO_Offset typelen; /* Size (in bytes) of the data type of data in buf. */ PIO_Offset num_elem = 1; /* Number of data elements in the buffer. */ + nc_type vartype; /* The type of the var we are reading from. */ char start_present = start ? true : false; char count_present = count ? true : false; char stride_present = stride ? true : false; - PIO_Offset *rstart = NULL, *rcount = NULL; - nc_type vartype; /* The type of the var we are reading from. */ + int mpierr = MPI_SUCCESS, mpierr2; /* Return code from MPI function codes. */ + int ierr; /* Return code. */ - LOG((1, "PIOc_get_vars_tc ncid = %d varid = %d start = %d count = %d " - "stride = %d xtype = %d", ncid, varid, start, count, stride, xtype)); + LOG((1, "PIOc_get_vars_tc ncid = %d varid = %d xtype = %d start_present = %d " + "count_present = %d stride_present = %d", ncid, varid, xtype, start_present, + count_present, stride_present)); /* Find the info about this file. */ if ((ierr = pio_get_file(ncid, &file))) @@ -510,7 +510,7 @@ int PIOc_get_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off /* Run these on all tasks if async is not in use, but only on * non-IO tasks if async is in use. */ - if (!ios->async_interface || !ios->ioproc) + if (!ios->async || !ios->ioproc) { /* Get the type of this var. */ if ((ierr = PIOc_inq_vartype(ncid, varid, &vartype))) @@ -532,63 +532,20 @@ int PIOc_get_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off /* Get the number of dims for this var. */ if ((ierr = PIOc_inq_varndims(ncid, varid, &ndims))) return check_netcdf(file, ierr, __FILE__, __LINE__); + LOG((3, "ndims = %d", ndims)); - PIO_Offset dimlen[ndims]; - - /* If no count array was passed, we need to know the dimlens - * so we can calculate how many data elements are in the - * buf. */ - if (!count) - { - int dimid[ndims]; - - /* Get the dimids for this var. */ - if ((ierr = PIOc_inq_vardimid(ncid, varid, dimid))) - return check_netcdf(file, ierr, __FILE__, __LINE__); - - /* Get the length of each dimension. */ - for (int vd = 0; vd < ndims; vd++) - if ((ierr = PIOc_inq_dimlen(ncid, dimid[vd], &dimlen[vd]))) - return check_netcdf(file, ierr, __FILE__, __LINE__); - } - - /* Figure out the real start, count, and stride arrays. (The - * user may have passed in NULLs.) */ - /* Allocate memory for these arrays, now that we know ndims. */ - if (!(rstart = malloc(ndims * sizeof(PIO_Offset)))) - return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__); - if (!(rcount = malloc(ndims * sizeof(PIO_Offset)))) - return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__); - - PIO_Offset rstride[ndims]; - for (int vd = 0; vd < ndims; vd++) - { - rstart[vd] = start ? start[vd] : 0; - rcount[vd] = count ? count[vd] : dimlen[vd]; - rstride[vd] = stride ? stride[vd] : 1; - LOG((3, "rstart[%d] = %d rcount[%d] = %d rstride[%d] = %d", vd, - rstart[vd], vd, rcount[vd], vd, rstride[vd])); - } + /* Only scalar vars can pass NULL for start/count. */ + pioassert(ndims == 0 || (start && count), "need start/count", __FILE__, __LINE__); - /* How many elements in buf? */ + /* How many elements in buf? (For scalars, ndims is 0 and + * num_elem will remain 1). */ for (int vd = 0; vd < ndims; vd++) - num_elem *= rcount[vd]; + num_elem *= count[vd]; LOG((2, "PIOc_get_vars_tc num_elem = %d", num_elem)); - - /* Free tmp resources. */ - if (start_present) - free(rstart); - else - start = rstart; - - if (count_present) - free(rcount); - else - count = rcount; } /* If async is in use, and this is not an IO task, bcast the parameters. */ - if (ios->async_interface) + if (ios->async) { if (!ios->ioproc) { @@ -606,8 +563,12 @@ int PIOc_get_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off if (!mpierr) mpierr = MPI_Bcast(&ndims, 1, MPI_INT, ios->compmaster, ios->intercomm); if (!mpierr) + mpierr = MPI_Bcast(&start_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + if (!mpierr && start_present) mpierr = MPI_Bcast((PIO_Offset *)start, ndims, MPI_OFFSET, ios->compmaster, ios->intercomm); if (!mpierr) + mpierr = MPI_Bcast(&count_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + if (!mpierr && count_present) mpierr = MPI_Bcast((PIO_Offset *)count, ndims, MPI_OFFSET, ios->compmaster, ios->intercomm); if (!mpierr) mpierr = MPI_Bcast(&stride_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); @@ -619,9 +580,9 @@ int PIOc_get_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off mpierr = MPI_Bcast(&num_elem, 1, MPI_OFFSET, ios->compmaster, ios->intercomm); if (!mpierr) mpierr = MPI_Bcast(&typelen, 1, MPI_OFFSET, ios->compmaster, ios->intercomm); - LOG((2, "PIOc_get_vars_tc ncid = %d varid = %d ndims = %d " - "stride_present = %d xtype = %d num_elem = %d", ncid, varid, - ndims, stride_present, xtype, num_elem)); + LOG((2, "PIOc_get_vars_tc ncid = %d varid = %d ndims = %d start_present = %d " + "count_present = %d stride_present = %d xtype = %d num_elem = %d", ncid, varid, + ndims, start_present, count_present, stride_present, xtype, num_elem)); } /* Handle MPI errors. */ @@ -646,7 +607,10 @@ int PIOc_get_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off #ifdef _PNETCDF if (file->iotype == PIO_IOTYPE_PNETCDF) { - ncmpi_begin_indep_data(file->fh); + LOG((2, "pnetcdf calling ncmpi_get_vars_*() file->fh = %d varid = %d", file->fh, varid)); + /* Turn on independent access for pnetcdf file. */ + if ((ierr = ncmpi_begin_indep_data(file->fh))) + return pio_err(ios, file, ierr, __FILE__, __LINE__); /* Only the IO master does the IO, so we are not really * getting parallel IO here. */ @@ -678,8 +642,11 @@ int PIOc_get_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off default: return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__); } - }; - ncmpi_end_indep_data(file->fh); + } + + /* Turn off independent access for pnetcdf file. */ + if ((ierr = ncmpi_end_indep_data(file->fh))) + return pio_err(ios, file, ierr, __FILE__, __LINE__); } #endif /* _PNETCDF */ @@ -746,15 +713,6 @@ int PIOc_get_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off } } - if (!ios->async_interface || !ios->ioproc) - { - /* Free tmp start/count allocated to account for NULL start/counts */ - if (!start_present) - free(rstart); - if (!count_present) - free(rcount); - } - /* Broadcast and check the return code. */ if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) return check_mpi(file, mpierr, __FILE__, __LINE__); @@ -772,7 +730,8 @@ int PIOc_get_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off } /** - * Get one value of a variable of any type. + * Get one value of a variable of any type. This is an internal + * function. * * This routine is called collectively by all tasks in the * communicator ios.union_comm. @@ -811,6 +770,74 @@ int PIOc_get_var1_tc(int ncid, int varid, const PIO_Offset *index, nc_type xtype return PIOc_get_vars_tc(ncid, varid, index, count, NULL, xtype, buf); } +/** + * Get a complete variable of any type. This is an internal function. + * + * This routine is called collectively by all tasks in the + * communicator ios.union_comm. + * + * @param ncid identifies the netCDF file + * @param varid the variable ID number + * @param index an array of start indicies (must have same number of + * entries as variable has dimensions). If NULL, indices of 0 will be + * used. + * @param xtype the netcdf type of the variable. + * @param buf pointer that will get the data. + * @return PIO_NOERR on success, error code otherwise. + */ +int PIOc_get_var_tc(int ncid, int varid, nc_type xtype, void *buf) +{ + iosystem_desc_t *ios; /* Pointer to io system information. */ + file_desc_t *file; /* Pointer to file information. */ + PIO_Offset *startp = NULL; /* Pointer to start array. */ + PIO_Offset *countp = NULL; /* Pointer to count array. */ + int ndims; /* The number of dimensions in the variable. */ + PIO_Offset my_start[PIO_MAX_DIMS]; + PIO_Offset dimlen[PIO_MAX_DIMS]; + int ierr; /* Return code from function calls. */ + + LOG((1, "PIOc_get_var_tc ncid = %d varid = %d xtype = %d", ncid, varid, + xtype)); + + /* Find the info about this file. We need this for error handling. */ + if ((ierr = pio_get_file(ncid, &file))) + return pio_err(NULL, NULL, ierr, __FILE__, __LINE__); + ios = file->iosystem; + + /* Find the number of dimensions. */ + if ((ierr = PIOc_inq_varndims(ncid, varid, &ndims))) + return pio_err(ios, file, ierr, __FILE__, __LINE__); + + /* Scalar vars (which have ndims == 0) should just pass NULLs for + * start/count. */ + if (ndims) + { + /* Find the dimension IDs. */ + int dimids[ndims]; + if ((ierr = PIOc_inq_vardimid(ncid, varid, dimids))) + return pio_err(ios, file, ierr, __FILE__, __LINE__); + + /* Find the dimension lengths. */ + for (int d = 0; d < ndims; d++) + if ((ierr = PIOc_inq_dimlen(ncid, dimids[d], &dimlen[d]))) + return pio_err(ios, file, ierr, __FILE__, __LINE__); + + /* Set up start array. */ + for (int d = 0; d < ndims; d++) + { + my_start[d] = 0; + LOG((3, "my_start[%d] = %d dimlen[%d] = %d", d, my_start[d], d, + dimlen[d])); + } + + /* Set the start/count arrays. */ + startp = my_start; + countp = dimlen; + } + + return PIOc_get_vars_tc(ncid, varid, startp, countp, NULL, xtype, buf); +} + /** * Internal PIO function which provides a type-neutral interface to * nc_put_vars. @@ -850,22 +877,22 @@ int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off const PIO_Offset *stride, nc_type xtype, const void *buf) { iosystem_desc_t *ios; /* Pointer to io system information. */ - file_desc_t *file; /* Pointer to file information. */ - int ndims; /* The number of dimensions in the variable. */ + file_desc_t *file; /* Pointer to file information. */ + int ndims; /* The number of dimensions in the variable. */ PIO_Offset typelen; /* Size (in bytes) of the data type of data in buf. */ PIO_Offset num_elem = 1; /* Number of data elements in the buffer. */ - char start_present = start ? true : false; /* Is start non-NULL? */ - char count_present = count ? true : false; /* Is count non-NULL? */ - char stride_present = stride ? true : false; /* Is stride non-NULL? */ - PIO_Offset *rstart, *rcount, *rstride; + char start_present = start ? true : false; /* Is start non-NULL? */ + char count_present = count ? true : false; /* Is count non-NULL? */ + char stride_present = stride ? true : false; /* Is stride non-NULL? */ var_desc_t *vdesc; int *request; nc_type vartype; /* The type of the var we are reading from. */ int mpierr = MPI_SUCCESS, mpierr2; /* Return code from MPI function codes. */ int ierr; /* Return code from function calls. */ - LOG((1, "PIOc_put_vars_tc ncid = %d varid = %d start = %d count = %d " - "stride = %d xtype = %d", ncid, varid, start, count, stride, xtype)); + LOG((1, "PIOc_put_vars_tc ncid = %d varid = %d start_present = %d " + "count_present = %d stride_present = %d xtype = %d", ncid, varid, + start_present, count_present, stride_present, xtype)); /* Get file info. */ if ((ierr = pio_get_file(ncid, &file))) @@ -878,7 +905,7 @@ int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off /* Run these on all tasks if async is not in use, but only on * non-IO tasks if async is in use. */ - if (!ios->async_interface || !ios->ioproc) + if (!ios->async || !ios->ioproc) { /* Get the type of this var. */ if ((ierr = PIOc_inq_vartype(ncid, varid, &vartype))) @@ -903,69 +930,15 @@ int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off LOG((2, "ndims = %d typelen = %d", ndims, typelen)); - PIO_Offset dimlen[ndims]; - - /* If no count array was passed, we need to know the dimlens - * so we can calculate how many data elements are in the - * buf. */ - if (!count) - { - int dimid[ndims]; - - /* Get the dimids for this var. */ - if ((ierr = PIOc_inq_vardimid(ncid, varid, dimid))) - return check_netcdf(file, ierr, __FILE__, __LINE__); - - /* Get the length of each dimension. */ + /* How many elements of data? If no count array was passed, + * this is a scalar. */ + if (count) for (int vd = 0; vd < ndims; vd++) - { - if ((ierr = PIOc_inq_dimlen(ncid, dimid[vd], &dimlen[vd]))) - return check_netcdf(file, ierr, __FILE__, __LINE__); - LOG((3, "dimlen[%d] = %d", vd, dimlen[vd])); - } - } - - /* Allocate memory for these arrays, now that we know ndims. */ - if (!(rstart = malloc(ndims * sizeof(PIO_Offset)))) - return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__); - if (!(rcount = malloc(ndims * sizeof(PIO_Offset)))) - return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__); - if (!(rstride = malloc(ndims * sizeof(PIO_Offset)))) - return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__); - - /* Figure out the real start, count, and stride arrays. (The - * user may have passed in NULLs.) */ - for (int vd = 0; vd < ndims; vd++) - { - rstart[vd] = start ? start[vd] : 0; - rcount[vd] = count ? count[vd] : dimlen[vd]; - rstride[vd] = stride ? stride[vd] : 1; - LOG((3, "rstart[%d] = %d rcount[%d] = %d rstride[%d] = %d", vd, - rstart[vd], vd, rcount[vd], vd, rstride[vd])); - } - - /* How many elements in buf? */ - for (int vd = 0; vd < ndims; vd++) - num_elem *= rcount[vd]; - LOG((2, "PIOc_put_vars_tc num_elem = %d", num_elem)); - - /* Free tmp resources. */ - if (start_present) - free(rstart); - else - start = rstart; - - if (count_present) - free(rcount); - else - count = rcount; - - /* Only PNETCDF requires a non-NULL stride, realocate it later if needed */ - free(rstride); + num_elem *= count[vd]; } /* If async is in use, and this is not an IO task, bcast the parameters. */ - if (ios->async_interface) + if (ios->async) { if (!ios->ioproc) { @@ -983,8 +956,12 @@ int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off if (!mpierr) mpierr = MPI_Bcast(&ndims, 1, MPI_INT, ios->compmaster, ios->intercomm); if (!mpierr) + mpierr = MPI_Bcast(&start_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + if (!mpierr && start_present) mpierr = MPI_Bcast((PIO_Offset *)start, ndims, MPI_OFFSET, ios->compmaster, ios->intercomm); if (!mpierr) + mpierr = MPI_Bcast(&count_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + if (!mpierr && count_present) mpierr = MPI_Bcast((PIO_Offset *)count, ndims, MPI_OFFSET, ios->compmaster, ios->intercomm); if (!mpierr) mpierr = MPI_Bcast(&stride_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); @@ -1028,69 +1005,121 @@ int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off #ifdef _PNETCDF if (file->iotype == PIO_IOTYPE_PNETCDF) { - PIO_Offset *fake_stride; - - if (!stride_present) + /* Scalars have to be handled differently. */ + if (ndims == 0) { - LOG((2, "stride not present")); - if (!(fake_stride = malloc(ndims * sizeof(PIO_Offset)))) - return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__); - for (int d = 0; d < ndims; d++) - fake_stride[d] = 1; + /* This is a scalar var. */ + LOG((2, "pnetcdf writing scalar with ncmpi_put_vars_*() file->fh = %d varid = %d", + file->fh, varid)); + pioassert(!start && !count && !stride, "expected NULLs", __FILE__, __LINE__); + + /* Turn on independent access for pnetcdf file. */ + if ((ierr = ncmpi_begin_indep_data(file->fh))) + return pio_err(ios, file, ierr, __FILE__, __LINE__); + + /* Only the IO master does the IO, so we are not really + * getting parallel IO here. */ + if (ios->iomaster == MPI_ROOT) + { + switch(xtype) + { + case NC_BYTE: + ierr = ncmpi_put_vars_schar(file->fh, varid, start, count, stride, buf); + break; + case NC_CHAR: + ierr = ncmpi_put_vars_text(file->fh, varid, start, count, stride, buf); + break; + case NC_SHORT: + ierr = ncmpi_put_vars_short(file->fh, varid, start, count, stride, buf); + break; + case NC_INT: + ierr = ncmpi_put_vars_int(file->fh, varid, start, count, stride, buf); + break; + case PIO_LONG_INTERNAL: + ierr = ncmpi_put_vars_long(file->fh, varid, start, count, stride, buf); + break; + case NC_FLOAT: + ierr = ncmpi_put_vars_float(file->fh, varid, start, count, stride, buf); + break; + case NC_DOUBLE: + ierr = ncmpi_put_vars_double(file->fh, varid, start, count, stride, buf); + break; + default: + return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__); + } + } + + /* Turn off independent access for pnetcdf file. */ + if ((ierr = ncmpi_end_indep_data(file->fh))) + return pio_err(ios, file, ierr, __FILE__, __LINE__); } else - fake_stride = (PIO_Offset *)stride; - - LOG((2, "PIOc_put_vars_tc calling pnetcdf function")); - vdesc = file->varlist + varid; - if (vdesc->nreqs % PIO_REQUEST_ALLOC_CHUNK == 0) - if (!(vdesc->request = realloc(vdesc->request, - sizeof(int) * (vdesc->nreqs + PIO_REQUEST_ALLOC_CHUNK)))) - return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__); - request = vdesc->request + vdesc->nreqs; - LOG((2, "PIOc_put_vars_tc request = %d", vdesc->request)); - - /* Only the IO master actually does the call. */ - if (ios->iomaster == MPI_ROOT) { - switch(xtype) + /* This is not a scalar var. */ + PIO_Offset *fake_stride; + + if (!stride_present) { - case NC_BYTE: - ierr = ncmpi_bput_vars_schar(file->fh, varid, start, count, fake_stride, buf, request); - break; - case NC_CHAR: - ierr = ncmpi_bput_vars_text(file->fh, varid, start, count, fake_stride, buf, request); - break; - case NC_SHORT: - ierr = ncmpi_bput_vars_short(file->fh, varid, start, count, fake_stride, buf, request); - break; - case NC_INT: - ierr = ncmpi_bput_vars_int(file->fh, varid, start, count, fake_stride, buf, request); - break; - case PIO_LONG_INTERNAL: - ierr = ncmpi_bput_vars_long(file->fh, varid, start, count, fake_stride, buf, request); - break; - case NC_FLOAT: - ierr = ncmpi_bput_vars_float(file->fh, varid, start, count, fake_stride, buf, request); - break; - case NC_DOUBLE: - ierr = ncmpi_bput_vars_double(file->fh, varid, start, count, fake_stride, buf, request); - break; - default: - return pio_err(ios, file, PIO_EBADTYPE, __FILE__, __LINE__); + LOG((2, "stride not present")); + if (!(fake_stride = malloc(ndims * sizeof(PIO_Offset)))) + return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__); + for (int d = 0; d < ndims; d++) + fake_stride[d] = 1; } - LOG((2, "PIOc_put_vars_tc io_rank 0 done with pnetcdf call, ierr=%d", ierr)); - } - else - *request = PIO_REQ_NULL; + else + fake_stride = (PIO_Offset *)stride; + + LOG((2, "PIOc_put_vars_tc calling pnetcdf function")); + vdesc = &file->varlist[varid]; + if (vdesc->nreqs % PIO_REQUEST_ALLOC_CHUNK == 0) + if (!(vdesc->request = realloc(vdesc->request, + sizeof(int) * (vdesc->nreqs + PIO_REQUEST_ALLOC_CHUNK)))) + return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__); + request = vdesc->request + vdesc->nreqs; + LOG((2, "PIOc_put_vars_tc request = %d", vdesc->request)); + + /* Only the IO master actually does the call. */ + if (ios->iomaster == MPI_ROOT) + { + switch(xtype) + { + case NC_BYTE: + ierr = ncmpi_bput_vars_schar(file->fh, varid, start, count, fake_stride, buf, request); + break; + case NC_CHAR: + ierr = ncmpi_bput_vars_text(file->fh, varid, start, count, fake_stride, buf, request); + break; + case NC_SHORT: + ierr = ncmpi_bput_vars_short(file->fh, varid, start, count, fake_stride, buf, request); + break; + case NC_INT: + ierr = ncmpi_bput_vars_int(file->fh, varid, start, count, fake_stride, buf, request); + break; + case PIO_LONG_INTERNAL: + ierr = ncmpi_bput_vars_long(file->fh, varid, start, count, fake_stride, buf, request); + break; + case NC_FLOAT: + ierr = ncmpi_bput_vars_float(file->fh, varid, start, count, fake_stride, buf, request); + break; + case NC_DOUBLE: + ierr = ncmpi_bput_vars_double(file->fh, varid, start, count, fake_stride, buf, request); + break; + default: + return pio_err(ios, file, PIO_EBADTYPE, __FILE__, __LINE__); + } + LOG((2, "PIOc_put_vars_tc io_rank 0 done with pnetcdf call, ierr=%d", ierr)); + } + else + *request = PIO_REQ_NULL; - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - LOG((2, "PIOc_put_vars_tc flushed output buffer")); + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + LOG((2, "PIOc_put_vars_tc flushed output buffer")); - /* Free malloced resources. */ - if (!stride_present) - free(fake_stride); + /* Free malloced resources. */ + if (!stride_present) + free(fake_stride); + } /* endif ndims == 0 */ } #endif /* _PNETCDF */ @@ -1161,15 +1190,6 @@ int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off } } - if (!ios->async_interface || !ios->ioproc) - { - /* Free tmp start/count allocated to account for NULL start/counts */ - if (!start_present) - free(rstart); - if (!count_present) - free(rcount); - } - /* Broadcast and check the return code. */ if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) return check_mpi(file, mpierr, __FILE__, __LINE__); @@ -1232,3 +1252,78 @@ int PIOc_put_var1_tc(int ncid, int varid, const PIO_Offset *index, nc_type xtype return PIOc_put_vars_tc(ncid, varid, index, count, NULL, xtype, op); } + +/** + * Internal PIO function which provides a type-neutral interface to + * nc_put_var calls. + * + * Users should not call this function directly. Instead, call one of + * the derived functions, depending on the type of data you are + * writing: PIOc_put_var_text(), PIOc_put_var_uchar(), + * PIOc_put_var_schar(), PIOc_put_var_ushort(), + * PIOc_put_var_short(), PIOc_put_var_uint(), PIOc_put_var_int(), + * PIOc_put_var_long(), PIOc_put_var_float(), + * PIOc_put_var_longlong(), PIOc_put_var_double(), + * PIOc_put_var_ulonglong(). + * + * This routine is called collectively by all tasks in the + * communicator ios.union_comm. + * + * @param ncid identifies the netCDF file + * @param varid the variable ID number + * @param xtype the netCDF type of the data being passed in buf. Data + * will be automatically covnerted from this type to the type of the + * variable being written to. + * @param op pointer to the data to be written. + * + * @return PIO_NOERR on success, error code otherwise. + */ +int PIOc_put_var_tc(int ncid, int varid, nc_type xtype, const void *op) +{ + iosystem_desc_t *ios; /* Pointer to io system information. */ + file_desc_t *file; /* Pointer to file information. */ + PIO_Offset *startp = NULL; /* Pointer to start array. */ + PIO_Offset *countp = NULL; /* Pointer to count array. */ + PIO_Offset start[PIO_MAX_DIMS]; + PIO_Offset count[PIO_MAX_DIMS]; + int ndims; /* The number of dimensions in the variable. */ + int ierr; /* Return code from function calls. */ + + LOG((1, "PIOc_put_var_tc ncid = %d varid = %d xtype = %d", ncid, + varid, xtype)); + + /* Find the info about this file. We need this for error handling. */ + if ((ierr = pio_get_file(ncid, &file))) + return pio_err(NULL, NULL, ierr, __FILE__, __LINE__); + ios = file->iosystem; + + /* Find the number of dimensions. */ + if ((ierr = PIOc_inq_varndims(ncid, varid, &ndims))) + return pio_err(ios, file, ierr, __FILE__, __LINE__); + + /* Scalar vars (which have ndims == 0) should just pass NULLs for + * start/count. */ + if (ndims) + { + int dimid[ndims]; + + /* Set up start array. */ + for (int d = 0; d < ndims; d++) + start[d] = 0; + + /* Get the dimids for this var. */ + if ((ierr = PIOc_inq_vardimid(ncid, varid, dimid))) + return check_netcdf(file, ierr, __FILE__, __LINE__); + + /* Count array are the dimlens. */ + for (int d = 0; d < ndims; d++) + if ((ierr = PIOc_inq_dimlen(ncid, dimid[d], &count[d]))) + return pio_err(ios, file, ierr, __FILE__, __LINE__); + + /* Set the array pointers. */ + startp = start; + countp = count; + } + + return PIOc_put_vars_tc(ncid, varid, startp, countp, NULL, xtype, op); +} diff --git a/src/externals/pio2/src/clib/pio_internal.h b/src/externals/pio2/src/clib/pio_internal.h index e1c891c761e..9f1a4a18d5d 100644 --- a/src/externals/pio2/src/clib/pio_internal.h +++ b/src/externals/pio2/src/clib/pio_internal.h @@ -12,6 +12,15 @@ #include +/* These are the sizes of types in netCDF files. Do not replace these + * constants with sizeof() calls for C types. They are not the + * same. Even on a system where sizeof(short) is 4, the size of a + * short in a netCDF file is 2 bytes. */ +#define NETCDF_CHAR_SIZE 1 +#define NETCDF_SHORT_SIZE 2 +#define NETCDF_INT_FLOAT_SIZE 4 +#define NETCDF_DOUBLE_INT64_SIZE 8 + /* It seems that some versions of openmpi fail to define * MPI_OFFSET. */ #ifdef OMPI_OFFSET_DATATYPE @@ -123,8 +132,8 @@ extern "C" { int check_netcdf2(iosystem_desc_t *ios, file_desc_t *file, int status, const char *fname, int line); - /* Find the MPI type that matches a PIO type. */ - int find_mpi_type(int pio_type, MPI_Datatype *mpi_type); + /* Given PIO type, find MPI type and type size. */ + int find_mpi_type(int pio_type, MPI_Datatype *mpi_type, int *type_size); /* Check whether an IO type is valid for this build. */ int iotype_is_valid(int iotype); @@ -135,30 +144,59 @@ extern "C" { /* Assert that an expression is true. */ void pioassert(bool exp, const char *msg, const char *fname, int line); - int CalcStartandCount(int basetype, int ndims, const int *gdims, int num_io_procs, - int myiorank, PIO_Offset *start, PIO_Offset *kount); + /* Compute start and count values for each io task for a decomposition. */ + int CalcStartandCount(int pio_type, int ndims, const int *gdims, int num_io_procs, + int myiorank, PIO_Offset *start, PIO_Offset *count, int *num_aiotasks); + + /* Completes the mapping for the box rearranger. */ + int compute_counts(iosystem_desc_t *ios, io_desc_t *iodesc, const int *dest_ioproc, + const PIO_Offset *dest_ioindex); + + /* Create the MPI communicators needed by the subset rearranger. */ + int default_subset_partition(iosystem_desc_t *ios, io_desc_t *iodesc); /* Check return from MPI function and print error message. */ void CheckMPIReturn(int ierr, const char *file, int line); /* Like MPI_Alltoallw(), but with flow control. */ - int pio_swapm(void *sndbuf, int *sndlths, int *sdispls, MPI_Datatype *stypes, - void *rcvbuf, int *rcvlths, int *rdispls, MPI_Datatype *rtypes, - MPI_Comm comm, bool handshake, bool isend, int max_requests); + int pio_swapm(void *sendbuf, int *sendcounts, int *sdispls, MPI_Datatype *sendtypes, + void *recvbuf, int *recvcounts, int *rdispls, MPI_Datatype *recvtypes, + MPI_Comm comm, rearr_comm_fc_opt_t *fc); long long lgcd_array(int nain, long long* ain); void PIO_Offset_size(MPI_Datatype *dtype, int *tsize); PIO_Offset GCDblocksize(int arrlen, const PIO_Offset *arr_in); - /* Initialize the rearranger options. */ - void init_rearr_opts(iosystem_desc_t *iosys); + /* Convert an index into dimension values. */ + void idx_to_dim_list(int ndims, const int *gdims, PIO_Offset idx, PIO_Offset *dim_list); + + /* Convert a global coordinate value into a local array index. */ + PIO_Offset coord_to_lindex(int ndims, const PIO_Offset *lcoord, const PIO_Offset *count); + + /* Determine whether fill values are needed. */ + int determine_fill(iosystem_desc_t *ios, io_desc_t *iodesc, const int *gsize, + const PIO_Offset *compmap); + + /* Set start and count so that they describe the first region in map.*/ + PIO_Offset find_region(int ndims, const int *gdims, int maplen, const PIO_Offset *map, + PIO_Offset *start, PIO_Offset *count); + + /* Calculate start and count regions for the subset rearranger. */ + int get_regions(int ndims, const int *gdimlen, int maplen, const PIO_Offset *map, + int *maxregions, io_region *firstregion); + + /* Expand a region along dimension dim, by incrementing count[i] as + * much as possible, consistent with the map. */ + void expand_region(int dim, const int *gdims, int maplen, const PIO_Offset *map, + int region_size, int region_stride, const int *max_size, + PIO_Offset *count); /* Compare sets of rearranger options. */ bool cmp_rearr_opts(const rearr_opt_t *rearr_opts, const rearr_opt_t *exp_rearr_opts); - /* Reset rearranger opts in iosystem to valid values. */ - void check_and_reset_rearr_opts(iosystem_desc_t *ios); + /* Check and reset, if needed, rearranger opts to default values. */ + int check_and_reset_rearr_opts(rearr_opt_t *rearr_opt); /* Compare rearranger flow control options. */ bool cmp_rearr_comm_fc_opts(const rearr_comm_fc_opt_t *opt, @@ -178,15 +216,23 @@ extern "C" { int rearrange_io2comp(iosystem_desc_t *ios, io_desc_t *iodesc, void *sbuf, void *rbuf); /* Move data from compute tasks to IO tasks. */ - int rearrange_comp2io(iosystem_desc_t *ios, io_desc_t *iodesc, void *sbuf, void *rbuf, int nvars); + int rearrange_comp2io(iosystem_desc_t *ios, io_desc_t *iodesc, void *sbuf, void *rbuf, + int nvars); /* Allocate and initialize storage for decomposition information. */ int malloc_iodesc(iosystem_desc_t *ios, int piotype, int ndims, io_desc_t **iodesc); void performance_tune_rearranger(iosystem_desc_t *ios, io_desc_t *iodesc); + /* Flush contents of multi-buffer to disk. */ int flush_output_buffer(file_desc_t *file, bool force, PIO_Offset addsize); + + /* Compute the size that the IO tasks will need to hold the data. */ int compute_maxIObuffersize(MPI_Comm io_comm, io_desc_t *iodesc); - io_region *alloc_region(int ndims); + + /* Allocation memory for a data region. */ + int alloc_region2(iosystem_desc_t *ios, int ndims, io_region **region); + + /* Delete an entry from the lost of open IO systems. */ int pio_delete_iosystem_from_list(int piosysid); /* Find greatest commond divisor. */ @@ -211,7 +257,7 @@ extern "C" { /* Create the derived MPI datatypes used for comp2io and io2comp * transfers. */ - int create_mpi_datatypes(MPI_Datatype basetype, int msgcnt, PIO_Offset dlen, const PIO_Offset *mindex, + int create_mpi_datatypes(MPI_Datatype basetype, int msgcnt, const PIO_Offset *mindex, const int *mcount, int *mfrom, MPI_Datatype *mtype); int compare_offsets(const void *a, const void *b) ; @@ -230,8 +276,9 @@ extern "C" { int compute_maxaggregate_bytes(iosystem_desc_t *ios, io_desc_t *iodesc); - /* Announce a memory error with bget memory, and die. */ - void piomemerror(iosystem_desc_t *ios, size_t req, char *fname, int line); + /* Compute an element of start/count arrays. */ + void compute_one_dim(int gdim, int ioprocs, int rank, PIO_Offset *start, + PIO_Offset *count); /* Check the return code from an MPI function call. */ int check_mpi(file_desc_t *file, int mpierr, const char *filename, int line); @@ -249,10 +296,8 @@ extern "C" { const int *frame); /* Write aggregated arrays to file using serial I/O (netCDF-3/netCDF-4 serial) */ - int pio_write_darray_multi_nc_serial(file_desc_t *file, int nvars, const int *vid, int iodesc_ndims, - MPI_Datatype basetype, int maxregions, io_region *firstregion, - PIO_Offset llen, int num_aiotasks, void *iobuf, - const int *frame); + int write_darray_multi_serial(file_desc_t *file, int nvars, const int *vid, + io_desc_t *iodesc, int fill, const int *frame); int pio_read_darray_nc(file_desc_t *file, io_desc_t *iodesc, int vid, void *iobuf); int pio_read_darray_nc_serial(file_desc_t *file, io_desc_t *iodesc, int vid, void *iobuf); @@ -269,13 +314,16 @@ extern "C" { const PIO_Offset *stride, nc_type xtype, void *buf); int PIOc_get_var1_tc(int ncid, int varid, const PIO_Offset *index, nc_type xtype, void *buf); + int PIOc_get_var_tc(int ncid, int varid, nc_type xtype, void *buf); + /* Generalized put functions. */ int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, const PIO_Offset *stride, nc_type xtype, const void *buf); int PIOc_put_var1_tc(int ncid, int varid, const PIO_Offset *index, nc_type xtype, const void *op); - + int PIOc_put_var_tc(int ncid, int varid, nc_type xtype, const void *op); + /* An internal replacement for a function pnetcdf does not * have. */ int pioc_pnetcdf_inq_type(int ncid, nc_type xtype, char *name, @@ -289,9 +337,9 @@ extern "C" { void pio_finalize_logging(void ); /* Write a netCDF decomp file. */ - int pioc_write_nc_decomp_int(int iosysid, const char *filename, int cmode, int ndims, int *global_dimlen, - int num_tasks, int *task_maplen, int *map, const char *title, - const char *history, int fortran_order); + int pioc_write_nc_decomp_int(iosystem_desc_t *ios, const char *filename, int cmode, int ndims, + int *global_dimlen, int num_tasks, int *task_maplen, int *map, + const char *title, const char *history, int fortran_order); /* Read a netCDF decomp file. */ int pioc_read_nc_decomp_int(int iosysid, const char *filename, int *ndims, int **global_dimlen, diff --git a/src/externals/pio2/src/clib/pio_lists.c b/src/externals/pio2/src/clib/pio_lists.c index 72e774715bc..df79302cfd1 100644 --- a/src/externals/pio2/src/clib/pio_lists.c +++ b/src/externals/pio2/src/clib/pio_lists.c @@ -115,8 +115,14 @@ int pio_delete_file_from_list(int ncid) if (current_file == cfile) current_file = pfile; + /* Free any fill values that were allocated. */ + for (int v = 0; v < PIO_MAX_VARS; v++) + if (cfile->varlist[v].fillvalue) + free(cfile->varlist[v].fillvalue); + /* Free the memory used for this file. */ free(cfile); + return PIO_NOERR; } pfile = cfile; diff --git a/src/externals/pio2/src/clib/pio_msg.c b/src/externals/pio2/src/clib/pio_msg.c index 75d2a8bfea4..b8ca0e9917b 100644 --- a/src/externals/pio2/src/clib/pio_msg.c +++ b/src/externals/pio2/src/clib/pio_msg.c @@ -675,16 +675,18 @@ int put_vars_handler(iosystem_desc_t *ios) { int ncid; int varid; - PIO_Offset typelen; /** Length (in bytes) of this type. */ - nc_type xtype; /** Type of the data being written. */ - char stride_present; /** Zero if user passed a NULL stride. */ + PIO_Offset typelen; /* Length (in bytes) of this type. */ + nc_type xtype; /* Type of the data being written. */ + char start_present; /* Zero if user passed a NULL start. */ + char count_present; /* Zero if user passed a NULL count. */ + char stride_present; /* Zero if user passed a NULL stride. */ PIO_Offset *startp = NULL; PIO_Offset *countp = NULL; PIO_Offset *stridep = NULL; - int ndims; /** Number of dimensions. */ - void *buf; /** Buffer for data storage. */ - PIO_Offset num_elem; /** Number of data elements in the buffer. */ - int mpierr; /** Error code from MPI function calls. */ + int ndims; /* Number of dimensions. */ + void *buf; /* Buffer for data storage. */ + PIO_Offset num_elem; /* Number of data elements in the buffer. */ + int mpierr; /* Error code from MPI function calls. */ LOG((1, "put_vars_handler")); assert(ios); @@ -701,11 +703,17 @@ int put_vars_handler(iosystem_desc_t *ios) /* Now we know how big to make these arrays. */ PIO_Offset start[ndims], count[ndims], stride[ndims]; - if ((mpierr = MPI_Bcast(start, ndims, MPI_OFFSET, 0, ios->intercomm))) + if ((mpierr = MPI_Bcast(&start_present, 1, MPI_CHAR, 0, ios->intercomm))) return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); + if (start_present) + if ((mpierr = MPI_Bcast(start, ndims, MPI_OFFSET, 0, ios->intercomm))) + return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); LOG((1, "put_vars_handler getting start[0] = %d ndims = %d", start[0], ndims)); - if ((mpierr = MPI_Bcast(count, ndims, MPI_OFFSET, 0, ios->intercomm))) + if ((mpierr = MPI_Bcast(&count_present, 1, MPI_CHAR, 0, ios->intercomm))) return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); + if (count_present) + if ((mpierr = MPI_Bcast(count, ndims, MPI_OFFSET, 0, ios->intercomm))) + return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); if ((mpierr = MPI_Bcast(&stride_present, 1, MPI_CHAR, 0, ios->intercomm))) return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); if (stride_present) @@ -718,16 +726,9 @@ int put_vars_handler(iosystem_desc_t *ios) if ((mpierr = MPI_Bcast(&typelen, 1, MPI_OFFSET, 0, ios->intercomm))) return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); LOG((1, "put_vars_handler ncid = %d varid = %d ndims = %d " - "stride_present = %d xtype = %d num_elem = %d typelen = %d", - ncid, varid, ndims, stride_present, xtype, num_elem, typelen)); - - for (int d = 0; d < ndims; d++) - { - LOG((2, "start[%d] = %d", d, start[d])); - LOG((2, "count[%d] = %d", d, count[d])); - if (stride_present) - LOG((2, "stride[%d] = %d", d, stride[d])); - } + "start_present = %d count_present = %d stride_present = %d xtype = %d " + "num_elem = %d typelen = %d", ncid, varid, ndims, start_present, count_present, + stride_present, xtype, num_elem, typelen)); /* Allocate room for our data. */ if (!(buf = malloc(num_elem * typelen))) @@ -735,17 +736,13 @@ int put_vars_handler(iosystem_desc_t *ios) /* Get the data. */ if ((mpierr = MPI_Bcast(buf, num_elem * typelen, MPI_BYTE, 0, ios->intercomm))) - { - free(buf); return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); - } - - /* for (int e = 0; e < num_elem; e++) */ - /* LOG((2, "element %d = %d", e, ((int *)buf)[e])); */ /* Set the non-NULL pointers. */ - startp = start; - countp = count; + if (start_present) + startp = start; + if (count_present) + countp = count; if (stride_present) stridep = stride; @@ -819,6 +816,11 @@ int get_vars_handler(iosystem_desc_t *ios) int mpierr; PIO_Offset typelen; /** Length (in bytes) of this type. */ nc_type xtype; /** Type of the data being written. */ + PIO_Offset *start; + PIO_Offset *count; + PIO_Offset *stride; + char start_present; + char count_present; char stride_present; PIO_Offset *startp = NULL, *countp = NULL, *stridep = NULL; int ndims; /** Number of dimensions. */ @@ -836,20 +838,33 @@ int get_vars_handler(iosystem_desc_t *ios) return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); if ((mpierr = MPI_Bcast(&ndims, 1, MPI_INT, 0, ios->intercomm))) return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); - - /* Now we know how big to make these arrays. */ - PIO_Offset start[ndims], count[ndims], stride[ndims]; - - if ((mpierr = MPI_Bcast(start, ndims, MPI_OFFSET, 0, ios->intercomm))) + if ((mpierr = MPI_Bcast(&start_present, 1, MPI_CHAR, 0, ios->intercomm))) return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); - LOG((1, "put_vars_handler getting start[0] = %d ndims = %d", start[0], ndims)); - if ((mpierr = MPI_Bcast(count, ndims, MPI_OFFSET, 0, ios->intercomm))) + if (start_present) + { + if (!(start = malloc(ndims * sizeof(PIO_Offset)))) + return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); + if ((mpierr = MPI_Bcast(start, ndims, MPI_OFFSET, 0, ios->intercomm))) + return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); + } + if ((mpierr = MPI_Bcast(&count_present, 1, MPI_CHAR, 0, ios->intercomm))) return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); + if (count_present) + { + if (!(count = malloc(ndims * sizeof(PIO_Offset)))) + return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); + if ((mpierr = MPI_Bcast(count, ndims, MPI_OFFSET, 0, ios->intercomm))) + return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); + } if ((mpierr = MPI_Bcast(&stride_present, 1, MPI_CHAR, 0, ios->intercomm))) return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); if (stride_present) + { + if (!(stride = malloc(ndims * sizeof(PIO_Offset)))) + return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); if ((mpierr = MPI_Bcast(stride, ndims, MPI_OFFSET, 0, ios->intercomm))) return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); + } if ((mpierr = MPI_Bcast(&xtype, 1, MPI_INT, 0, ios->intercomm))) return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); if ((mpierr = MPI_Bcast(&num_elem, 1, MPI_OFFSET, 0, ios->intercomm))) @@ -860,21 +875,17 @@ int get_vars_handler(iosystem_desc_t *ios) "stride_present = %d xtype = %d num_elem = %d typelen = %d", ncid, varid, ndims, stride_present, xtype, num_elem, typelen)); - for (int d = 0; d < ndims; d++) - { - LOG((2, "start[%d] = %d", d, start[d])); - LOG((2, "count[%d] = %d", d, count[d])); - if (stride_present) - LOG((2, "stride[%d] = %d", d, stride[d])); - } - /* Allocate room for our data. */ if (!(buf = malloc(num_elem * typelen))) return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); /* Set the non-NULL pointers. */ - startp = start; - countp = count; + if (start_present) + startp = start; + + if (count_present) + countp = count; + if (stride_present) stridep = stride; @@ -928,7 +939,15 @@ int get_vars_handler(iosystem_desc_t *ios) #endif /* _NETCDF4 */ } + /* Free resourses. */ free(buf); + if (start_present) + free(start); + if (count_present) + free(count); + if (stride_present) + free(stride); + LOG((1, "get_vars_handler succeeded!")); return PIO_NOERR; } @@ -1889,7 +1908,7 @@ int delete_file_handler(iosystem_desc_t *ios) int initdecomp_dof_handler(iosystem_desc_t *ios) { int iosysid; - int basetype; + int pio_type; int ndims; int maplen; int ioid; @@ -1910,7 +1929,7 @@ int initdecomp_dof_handler(iosystem_desc_t *ios) * task is broadcasting. */ if ((mpierr = MPI_Bcast(&iosysid, 1, MPI_INT, 0, ios->intercomm))) return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); - if ((mpierr = MPI_Bcast(&basetype, 1, MPI_INT, 0, ios->intercomm))) + if ((mpierr = MPI_Bcast(&pio_type, 1, MPI_INT, 0, ios->intercomm))) return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); if ((mpierr = MPI_Bcast(&ndims, 1, MPI_INT, 0, ios->intercomm))) return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); @@ -1951,9 +1970,9 @@ int initdecomp_dof_handler(iosystem_desc_t *ios) if ((mpierr = MPI_Bcast(iocount, ndims, MPI_OFFSET, 0, ios->intercomm))) return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); - LOG((2, "initdecomp_dof_handler iosysid = %d basetype = %d ndims = %d maplen = %d " + LOG((2, "initdecomp_dof_handler iosysid = %d pio_type = %d ndims = %d maplen = %d " "rearranger_present = %d iostart_present = %d iocount_present = %d ", - iosysid, basetype, ndims, maplen, rearranger_present, iostart_present, iocount_present)); + iosysid, pio_type, ndims, maplen, rearranger_present, iostart_present, iocount_present)); if (rearranger_present) rearrangerp = &rearranger; @@ -1963,7 +1982,7 @@ int initdecomp_dof_handler(iosystem_desc_t *ios) iocountp = iocount; /* Call the function. */ - ret = PIOc_InitDecomp(iosysid, basetype, ndims, dims, maplen, compmap, &ioid, rearrangerp, + ret = PIOc_InitDecomp(iosysid, pio_type, ndims, dims, maplen, compmap, &ioid, rearrangerp, iostartp, iocountp); LOG((1, "PIOc_InitDecomp returned %d", ret)); diff --git a/src/externals/pio2/src/clib/pio_nc.c b/src/externals/pio2/src/clib/pio_nc.c index 8b7fb81ba64..00932bdfd64 100644 --- a/src/externals/pio2/src/clib/pio_nc.c +++ b/src/externals/pio2/src/clib/pio_nc.c @@ -49,7 +49,7 @@ int PIOc_inq(int ncid, int *ndimsp, int *nvarsp, int *ngattsp, int *unlimdimidp) ios = file->iosystem; /* If async is in use, and this is not an IO task, bcast the parameters. */ - if (ios->async_interface) + if (ios->async) { if (!ios->ioproc) { @@ -89,9 +89,7 @@ int PIOc_inq(int ncid, int *ndimsp, int *nvarsp, int *ngattsp, int *unlimdimidp) #ifdef _PNETCDF if (file->iotype == PIO_IOTYPE_PNETCDF) { - LOG((2, "PIOc_inq calling ncmpi_inq unlimdimidp = %d", unlimdimidp)); ierr = ncmpi_inq(file->fh, ndimsp, nvarsp, ngattsp, unlimdimidp); - LOG((2, "PIOc_inq called ncmpi_inq")); if (unlimdimidp) LOG((2, "PIOc_inq returned from ncmpi_inq unlimdimid = %d", *unlimdimidp)); } @@ -205,7 +203,7 @@ int PIOc_inq_natts(int ncid, int *ngattsp) */ int PIOc_inq_unlimdim(int ncid, int *unlimdimidp) { - LOG((1, "PIOc_inq_unlimdim ncid = %d unlimdimidp = %d", ncid, unlimdimidp)); + LOG((1, "PIOc_inq_unlimdim ncid = %d", ncid)); return PIOc_inq(ncid, NULL, NULL, NULL, unlimdimidp); } @@ -234,7 +232,7 @@ int PIOc_inq_type(int ncid, nc_type xtype, char *name, PIO_Offset *sizep) ios = file->iosystem; /* If async is in use, and this is not an IO task, bcast the parameters. */ - if (ios->async_interface) + if (ios->async) { if (!ios->ioproc) { @@ -323,7 +321,7 @@ int PIOc_inq_format(int ncid, int *formatp) ios = file->iosystem; /* If async is in use, and this is not an IO task, bcast the parameters. */ - if (ios->async_interface) + if (ios->async) { if (!ios->ioproc) { @@ -402,7 +400,7 @@ int PIOc_inq_dim(int ncid, int dimid, char *name, PIO_Offset *lenp) ios = file->iosystem; /* If async is in use, and this is not an IO task, bcast the parameters. */ - if (ios->async_interface) + if (ios->async) { if (!ios->ioproc) { @@ -543,7 +541,7 @@ int PIOc_inq_dimid(int ncid, const char *name, int *idp) LOG((1, "PIOc_inq_dimid ncid = %d name = %s", ncid, name)); /* If using async, and not an IO task, then send parameters. */ - if (ios->async_interface) + if (ios->async) { if (!ios->ioproc) { @@ -629,10 +627,9 @@ int PIOc_inq_var(int ncid, int varid, char *name, nc_type *xtypep, int *ndimsp, if ((ierr = pio_get_file(ncid, &file))) return pio_err(NULL, NULL, ierr, __FILE__, __LINE__); ios = file->iosystem; - LOG((2, "got file and iosystem")); /* If async is in use, and this is not an IO task, bcast the parameters. */ - if (ios->async_interface) + if (ios->async) { if (!ios->ioproc) { @@ -689,8 +686,6 @@ int PIOc_inq_var(int ncid, int varid, char *name, nc_type *xtypep, int *ndimsp, if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io) { ierr = nc_inq_varndims(file->fh, varid, &ndims); - LOG((2, "file->fh = %d varid = %d xtypep = %d ndimsp = %d dimidsp = %d nattsp = %d", - file->fh, varid, xtypep, ndimsp, dimidsp, nattsp)); if (!ierr) { char my_name[NC_MAX_NAME + 1]; @@ -871,7 +866,7 @@ int PIOc_inq_varid(int ncid, const char *name, int *varidp) LOG((1, "PIOc_inq_varid ncid = %d name = %s", ncid, name)); - if (ios->async_interface) + if (ios->async) { if (!ios->ioproc) { @@ -957,11 +952,10 @@ int PIOc_inq_att(int ncid, int varid, const char *name, nc_type *xtypep, if (!name || strlen(name) > NC_MAX_NAME) return pio_err(ios, file, PIO_EINVAL, __FILE__, __LINE__); - LOG((1, "PIOc_inq_att ncid = %d varid = %d xtpyep = %d lenp = %d", - ncid, varid, xtypep, lenp)); + LOG((1, "PIOc_inq_att ncid = %d varid = %d", ncid, varid)); /* If async is in use, and this is not an IO task, bcast the parameters. */ - if (ios->async_interface) + if (ios->async) { if (!ios->ioproc) { @@ -1086,7 +1080,7 @@ int PIOc_inq_attname(int ncid, int varid, int attnum, char *name) ios = file->iosystem; /* If async is in use, and this is not an IO task, bcast the parameters. */ - if (ios->async_interface) + if (ios->async) { if (!ios->ioproc) { @@ -1180,7 +1174,7 @@ int PIOc_inq_attid(int ncid, int varid, const char *name, int *idp) LOG((1, "PIOc_inq_attid ncid = %d varid = %d name = %s", ncid, varid, name)); /* If async is in use, and this is not an IO task, bcast the parameters. */ - if (ios->async_interface) + if (ios->async) { if (!ios->ioproc) { @@ -1269,7 +1263,7 @@ int PIOc_rename_dim(int ncid, int dimid, const char *name) LOG((1, "PIOc_rename_dim ncid = %d dimid = %d name = %s", ncid, dimid, name)); /* If async is in use, and this is not an IO task, bcast the parameters. */ - if (ios->async_interface) + if (ios->async) { if (!ios->ioproc) { @@ -1354,7 +1348,7 @@ int PIOc_rename_var(int ncid, int varid, const char *name) LOG((1, "PIOc_rename_var ncid = %d varid = %d name = %s", ncid, varid, name)); /* If async is in use, and this is not an IO task, bcast the parameters. */ - if (ios->async_interface) + if (ios->async) { if (!ios->ioproc) { @@ -1443,7 +1437,7 @@ int PIOc_rename_att(int ncid, int varid, const char *name, ncid, varid, name, newname)); /* If async is in use, and this is not an IO task, bcast the parameters. */ - if (ios->async_interface) + if (ios->async) { if (!ios->ioproc) { @@ -1531,7 +1525,7 @@ int PIOc_del_att(int ncid, int varid, const char *name) LOG((1, "PIOc_del_att ncid = %d varid = %d name = %s", ncid, varid, name)); /* If async is in use, and this is not an IO task, bcast the parameters. */ - if (ios->async_interface) + if (ios->async) { if (!ios->ioproc) { @@ -1601,8 +1595,7 @@ int PIOc_set_fill(int ncid, int fillmode, int *old_modep) int ierr; /* Return code from function calls. */ int mpierr = MPI_SUCCESS, mpierr2; /* Return code from MPI functions. */ - LOG((1, "PIOc_set_fill ncid = %d fillmode = %d old_modep = %d", ncid, fillmode, - old_modep)); + LOG((1, "PIOc_set_fill ncid = %d fillmode = %d", ncid, fillmode)); /* Find the info about this file. */ if ((ierr = pio_get_file(ncid, &file))) @@ -1610,7 +1603,7 @@ int PIOc_set_fill(int ncid, int fillmode, int *old_modep) ios = file->iosystem; /* If async is in use, and this is not an IO task, bcast the parameters. */ - if (ios->async_interface) + if (ios->async) { if (!ios->ioproc) { @@ -1739,7 +1732,7 @@ int PIOc_def_dim(int ncid, const char *name, PIO_Offset len, int *idp) LOG((1, "PIOc_def_dim ncid = %d name = %s len = %d", ncid, name, len)); /* If async is in use, and this is not an IO task, bcast the parameters. */ - if (ios->async_interface) + if (ios->async) { if (!ios->ioproc) { @@ -1831,7 +1824,7 @@ int PIOc_def_var(int ncid, const char *name, nc_type xtype, int ndims, xtype, ndims)); /* If using async, and not an IO task, then send parameters. */ - if (ios->async_interface) + if (ios->async) { if (!ios->ioproc) { @@ -1948,7 +1941,7 @@ int PIOc_def_var_fill(int ncid, int varid, int fill_mode, const void *fill_value /* Run this on all tasks if async is not in use, but only on * non-IO tasks if async is in use. Get the size of this vars * type. */ - if (!ios->async_interface || !ios->ioproc) + if (!ios->async || !ios->ioproc) { if ((ierr = PIOc_inq_vartype(ncid, varid, &xtype))) return check_netcdf(file, ierr, __FILE__, __LINE__); @@ -1958,7 +1951,7 @@ int PIOc_def_var_fill(int ncid, int varid, int fill_mode, const void *fill_value LOG((2, "PIOc_def_var_fill type_size = %d", type_size)); /* If async is in use, and this is not an IO task, bcast the parameters. */ - if (ios->async_interface) + if (ios->async) { if (!ios->ioproc) { @@ -2042,7 +2035,7 @@ int PIOc_def_var_fill(int ncid, int varid, int fill_mode, const void *fill_value * @param ncid the ncid of the open file, obtained from * PIOc_openfile() or PIOc_createfile(). * @param varid the variable ID. - * @param fill_mode a pointer to int that will get the fill + * @param no_fill a pointer to int that will get the fill * mode. Ignored if NULL (except with pnetcdf, which seg-faults with * NULL.) * @param fill_valuep pointer to space that gets the fill value for @@ -2050,7 +2043,7 @@ int PIOc_def_var_fill(int ncid, int varid, int fill_mode, const void *fill_value * @return PIO_NOERR for success, error code otherwise. * @ingroup PIO_inq_var_fill */ -int PIOc_inq_var_fill(int ncid, int varid, int *fill_mode, void *fill_valuep) +int PIOc_inq_var_fill(int ncid, int varid, int *no_fill, void *fill_valuep) { iosystem_desc_t *ios; /* Pointer to io system information. */ file_desc_t *file; /* Pointer to file information. */ @@ -2070,7 +2063,7 @@ int PIOc_inq_var_fill(int ncid, int varid, int *fill_mode, void *fill_valuep) /* Run this on all tasks if async is not in use, but only on * non-IO tasks if async is in use. Get the size of this vars * type. */ - if (!ios->async_interface || !ios->ioproc) + if (!ios->async || !ios->ioproc) { if ((ierr = PIOc_inq_vartype(ncid, varid, &xtype))) return check_netcdf(file, ierr, __FILE__, __LINE__); @@ -2080,12 +2073,12 @@ int PIOc_inq_var_fill(int ncid, int varid, int *fill_mode, void *fill_valuep) } /* If async is in use, and this is not an IO task, bcast the parameters. */ - if (ios->async_interface) + if (ios->async) { if (!ios->ioproc) { int msg = PIO_MSG_INQ_VAR_FILL; - char fill_mode_present = fill_mode ? true : false; + char no_fill_present = no_fill ? true : false; char fill_value_present = fill_valuep ? true : false; LOG((2, "sending msg type_size = %d", type_size)); @@ -2099,11 +2092,11 @@ int PIOc_inq_var_fill(int ncid, int varid, int *fill_mode, void *fill_valuep) if (!mpierr) mpierr = MPI_Bcast(&type_size, 1, MPI_OFFSET, ios->compmaster, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&fill_mode_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&no_fill_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); if (!mpierr) mpierr = MPI_Bcast(&fill_value_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); - LOG((2, "PIOc_inq_var_fill ncid = %d varid = %d type_size = %lld fill_mode_present = %d fill_value_present = %d", - ncid, varid, type_size, fill_mode_present, fill_value_present)); + LOG((2, "PIOc_inq_var_fill ncid = %d varid = %d type_size = %lld no_fill_present = %d fill_value_present = %d", + ncid, varid, type_size, no_fill_present, fill_value_present)); } /* Handle MPI errors. */ @@ -2122,22 +2115,22 @@ int PIOc_inq_var_fill(int ncid, int varid, int *fill_mode, void *fill_valuep) /* If this is an IO task, then call the netCDF function. */ if (ios->ioproc) { - LOG((2, "calling inq_var_fill file->iotype = %d file->fh = %d varid = %d fill_mode = %d", - file->iotype, file->fh, varid, fill_mode)); + LOG((2, "calling inq_var_fill file->iotype = %d file->fh = %d varid = %d", + file->iotype, file->fh, varid)); if (file->iotype == PIO_IOTYPE_PNETCDF) { #ifdef _PNETCDF - ierr = ncmpi_inq_var_fill(file->fh, varid, fill_mode, fill_valuep); + ierr = ncmpi_inq_var_fill(file->fh, varid, no_fill, fill_valuep); #endif /* _PNETCDF */ } - else if (file->iotype == PIO_IOTYPE_NETCDF) + else if (file->iotype == PIO_IOTYPE_NETCDF && file->do_io) { /* Get the file-level fill mode. */ - if (fill_mode) + if (no_fill) { - ierr = nc_set_fill(file->fh, NC_NOFILL, fill_mode); + ierr = nc_set_fill(file->fh, NC_NOFILL, no_fill); if (!ierr) - ierr = nc_set_fill(file->fh, *fill_mode, NULL); + ierr = nc_set_fill(file->fh, *no_fill, NULL); } if (!ierr && fill_valuep) @@ -2183,7 +2176,7 @@ int PIOc_inq_var_fill(int ncid, int varid, int *fill_mode, void *fill_valuep) #ifdef _NETCDF4 /* The inq_var_fill is not supported in classic-only builds. */ if (file->do_io) - ierr = nc_inq_var_fill(file->fh, varid, fill_mode, fill_valuep); + ierr = nc_inq_var_fill(file->fh, varid, no_fill, fill_valuep); #endif /* _NETCDF */ } LOG((2, "after call to inq_var_fill, ierr = %d", ierr)); @@ -2196,8 +2189,8 @@ int PIOc_inq_var_fill(int ncid, int varid, int *fill_mode, void *fill_valuep) return check_netcdf(file, ierr, __FILE__, __LINE__); /* Broadcast results to all tasks. Ignore NULL parameters. */ - if (fill_mode) - if ((mpierr = MPI_Bcast(fill_mode, 1, MPI_INT, ios->ioroot, ios->my_comm))) + if (no_fill) + if ((mpierr = MPI_Bcast(no_fill, 1, MPI_INT, ios->ioroot, ios->my_comm))) check_mpi(file, mpierr, __FILE__, __LINE__); if (fill_valuep) if ((mpierr = MPI_Bcast(fill_valuep, type_size, MPI_CHAR, ios->ioroot, ios->my_comm))) diff --git a/src/externals/pio2/src/clib/pio_nc4.c b/src/externals/pio2/src/clib/pio_nc4.c index b68bba41050..0b1cdd0e6b7 100644 --- a/src/externals/pio2/src/clib/pio_nc4.c +++ b/src/externals/pio2/src/clib/pio_nc4.c @@ -48,7 +48,7 @@ int PIOc_def_var_deflate(int ncid, int varid, int shuffle, int deflate, return pio_err(ios, file, PIO_ENOTNC4, __FILE__, __LINE__); /* If async is in use, and this is not an IO task, bcast the parameters. */ - if (ios->async_interface) + if (ios->async) { if (!ios->ioproc) { @@ -137,7 +137,7 @@ int PIOc_inq_var_deflate(int ncid, int varid, int *shufflep, int *deflatep, return pio_err(ios, file, PIO_ENOTNC4, __FILE__, __LINE__); /* If async is in use, and this is not an IO task, bcast the parameters. */ - if (ios->async_interface) + if (ios->async) { if (!ios->ioproc) { @@ -251,13 +251,13 @@ int PIOc_def_var_chunking(int ncid, int varid, int storage, const PIO_Offset *ch /* Run this on all tasks if async is not in use, but only on * non-IO tasks if async is in use. Get the number of * dimensions. */ - if (!ios->async_interface || !ios->ioproc) + if (!ios->async || !ios->ioproc) if ((ierr = PIOc_inq_varndims(ncid, varid, &ndims))) return check_netcdf(file, ierr, __FILE__, __LINE__); LOG((2, "PIOc_def_var_chunking first ndims = %d", ndims)); /* If async is in use, and this is not an IO task, bcast the parameters. */ - if (ios->async_interface) + if (ios->async) { if (!ios->ioproc) { @@ -370,7 +370,7 @@ int PIOc_inq_var_chunking(int ncid, int varid, int *storagep, PIO_Offset *chunks /* Run these on all tasks if async is not in use, but only on * non-IO tasks if async is in use. */ - if (!ios->async_interface || !ios->ioproc) + if (!ios->async || !ios->ioproc) { /* Find the number of dimensions of this variable. */ if ((ierr = PIOc_inq_varndims(ncid, varid, &ndims))) @@ -379,7 +379,7 @@ int PIOc_inq_var_chunking(int ncid, int varid, int *storagep, PIO_Offset *chunks } /* If async is in use, and this is not an IO task, bcast the parameters. */ - if (ios->async_interface) + if (ios->async) { if (!ios->ioproc) { @@ -495,7 +495,7 @@ int PIOc_def_var_endian(int ncid, int varid, int endian) return pio_err(ios, file, PIO_ENOTNC4, __FILE__, __LINE__); /* If async is in use, and this is not an IO task, bcast the parameters. */ - if (ios->async_interface) + if (ios->async) { if (!ios->ioproc) { @@ -572,7 +572,7 @@ int PIOc_inq_var_endian(int ncid, int varid, int *endianp) return pio_err(ios, file, PIO_ENOTNC4, __FILE__, __LINE__); /* If async is in use, and this is not an IO task, bcast the parameters. */ - if (ios->async_interface) + if (ios->async) { if (!ios->ioproc) { @@ -663,7 +663,7 @@ int PIOc_set_chunk_cache(int iosysid, int iotype, PIO_Offset size, PIO_Offset ne return pio_err(ios, NULL, PIO_ENOTNC4, __FILE__, __LINE__); /* If async is in use, and this is not an IO task, bcast the parameters. */ - if (ios->async_interface) + if (ios->async) { if (!ios->ioproc) { @@ -761,7 +761,7 @@ int PIOc_get_chunk_cache(int iosysid, int iotype, PIO_Offset *sizep, PIO_Offset return pio_err(ios, NULL, PIO_ENOTNC4, __FILE__, __LINE__); /* If async is in use, and this is not an IO task, bcast the parameters. */ - if (ios->async_interface) + if (ios->async) { if (!ios->ioproc) { @@ -879,7 +879,7 @@ int PIOc_set_var_chunk_cache(int ncid, int varid, PIO_Offset size, PIO_Offset ne return pio_err(ios, file, PIO_ENOTNC4, __FILE__, __LINE__); /* If async is in use, and this is not an IO task, bcast the parameters. */ - if (ios->async_interface) + if (ios->async) { if (!ios->ioproc) { @@ -966,7 +966,7 @@ int PIOc_get_var_chunk_cache(int ncid, int varid, PIO_Offset *sizep, PIO_Offset return pio_err(ios, file, PIO_ENOTNC4, __FILE__, __LINE__); /* If async is in use, and this is not an IO task, bcast the parameters. */ - if (ios->async_interface) + if (ios->async) { if (!ios->ioproc) { diff --git a/src/externals/pio2/src/clib/pio_put_nc.c b/src/externals/pio2/src/clib/pio_put_nc.c index 74279c9871e..7c74144da3b 100644 --- a/src/externals/pio2/src/clib/pio_put_nc.c +++ b/src/externals/pio2/src/clib/pio_put_nc.c @@ -860,7 +860,7 @@ int PIOc_put_vara_double(int ncid, int varid, const PIO_Offset *start, */ int PIOc_put_var_text(int ncid, int varid, const char *op) { - return PIOc_put_vars_text(ncid, varid, NULL, NULL, NULL, op); + return PIOc_put_var_tc(ncid, varid, PIO_CHAR, op); } /** @@ -882,7 +882,7 @@ int PIOc_put_var_text(int ncid, int varid, const char *op) */ int PIOc_put_var_uchar(int ncid, int varid, const unsigned char *op) { - return PIOc_put_vars_uchar(ncid, varid, NULL, NULL, NULL, op); + return PIOc_put_var_tc(ncid, varid, PIO_UBYTE, op); } /** @@ -904,7 +904,7 @@ int PIOc_put_var_uchar(int ncid, int varid, const unsigned char *op) */ int PIOc_put_var_schar(int ncid, int varid, const signed char *op) { - return PIOc_put_vars_schar(ncid, varid, NULL, NULL, NULL, op); + return PIOc_put_var_tc(ncid, varid, PIO_BYTE, op); } /** @@ -926,7 +926,7 @@ int PIOc_put_var_schar(int ncid, int varid, const signed char *op) */ int PIOc_put_var_ushort(int ncid, int varid, const unsigned short *op) { - return PIOc_put_vars_tc(ncid, varid, NULL, NULL, NULL, NC_USHORT, op); + return PIOc_put_var_tc(ncid, varid, NC_USHORT, op); } /** @@ -948,7 +948,7 @@ int PIOc_put_var_ushort(int ncid, int varid, const unsigned short *op) */ int PIOc_put_var_short(int ncid, int varid, const short *op) { - return PIOc_put_vars_short(ncid, varid, NULL, NULL, NULL, op); + return PIOc_put_var_tc(ncid, varid, PIO_SHORT, op); } /** @@ -970,7 +970,7 @@ int PIOc_put_var_short(int ncid, int varid, const short *op) */ int PIOc_put_var_uint(int ncid, int varid, const unsigned int *op) { - return PIOc_put_vars_uint(ncid, varid, NULL, NULL, NULL, op); + return PIOc_put_var_tc(ncid, varid, PIO_UINT, op); } /** @@ -992,7 +992,7 @@ int PIOc_put_var_uint(int ncid, int varid, const unsigned int *op) */ int PIOc_put_var_int(int ncid, int varid, const int *op) { - return PIOc_put_vars_int(ncid, varid, NULL, NULL, NULL, op); + return PIOc_put_var_tc(ncid, varid, PIO_INT, op); } /** @@ -1014,7 +1014,7 @@ int PIOc_put_var_int(int ncid, int varid, const int *op) */ int PIOc_put_var_long(int ncid, int varid, const long *op) { - return PIOc_put_vars_long(ncid, varid, NULL, NULL, NULL, op); + return PIOc_put_var_tc(ncid, varid, PIO_LONG_INTERNAL, op); } /** @@ -1036,7 +1036,7 @@ int PIOc_put_var_long(int ncid, int varid, const long *op) */ int PIOc_put_var_float(int ncid, int varid, const float *op) { - return PIOc_put_vars_float(ncid, varid, NULL, NULL, NULL, op); + return PIOc_put_var_tc(ncid, varid, PIO_FLOAT, op); } /** @@ -1058,7 +1058,7 @@ int PIOc_put_var_float(int ncid, int varid, const float *op) */ int PIOc_put_var_ulonglong(int ncid, int varid, const unsigned long long *op) { - return PIOc_put_vars_ulonglong(ncid, varid, NULL, NULL, NULL, op); + return PIOc_put_var_tc(ncid, varid, PIO_UINT64, op); } /** @@ -1080,7 +1080,7 @@ int PIOc_put_var_ulonglong(int ncid, int varid, const unsigned long long *op) */ int PIOc_put_var_longlong(int ncid, int varid, const long long *op) { - return PIOc_put_vars_longlong(ncid, varid, NULL, NULL, NULL, op); + return PIOc_put_var_tc(ncid, varid, PIO_INT64, op); } /** @@ -1102,7 +1102,7 @@ int PIOc_put_var_longlong(int ncid, int varid, const long long *op) */ int PIOc_put_var_double(int ncid, int varid, const double *op) { - return PIOc_put_vars_double(ncid, varid, NULL, NULL, NULL, op); + return PIOc_put_var_tc(ncid, varid, PIO_DOUBLE, op); } /** @@ -1118,7 +1118,7 @@ int PIOc_put_var_double(int ncid, int varid, const double *op) */ int PIOc_put_var(int ncid, int varid, const void *op) { - return PIOc_put_vars_tc(ncid, varid, NULL, NULL, NULL, NC_NAT, op); + return PIOc_put_var_tc(ncid, varid, NC_NAT, op); } /** diff --git a/src/externals/pio2/src/clib/pio_rearrange.c b/src/externals/pio2/src/clib/pio_rearrange.c index 458e7e3844f..74b27d45b06 100644 --- a/src/externals/pio2/src/clib/pio_rearrange.c +++ b/src/externals/pio2/src/clib/pio_rearrange.c @@ -8,50 +8,40 @@ #include /** - * Internal library util function to initialize rearranger options. + * Convert a 1-D index into a coordinate value in an arbitrary + * dimension space. E.g., for index 4 into a array defined as a[3][2], + * will return 2,0. * - * @param iosys pointer to iosystem descriptor - */ -void init_rearr_opts(iosystem_desc_t *iosys) -{ - /* The old default for max pending requests was 64 - we no longer use it*/ - - /* Disable handshake /isend and set max_pend_req = 0 to turn of throttling */ - const rearr_comm_fc_opt_t def_coll_comm_fc_opts = { false, false, 0 }; - - assert(iosys); - - /* Default to coll - i.e., no flow control */ - iosys->rearr_opts.comm_type = PIO_REARR_COMM_COLL; - iosys->rearr_opts.fcd = PIO_REARR_COMM_FC_2D_DISABLE; - iosys->rearr_opts.comm_fc_opts_comp2io = def_coll_comm_fc_opts; - iosys->rearr_opts.comm_fc_opts_io2comp = def_coll_comm_fc_opts; -} - -/** - * Convert an index into a list of dimensions. E.g., for index 4 into a - * array defined as a[3][2], will return 1 1. + * Sometimes (from box_rearranger_create()) this function is called + * with -1 for idx. Not clear if this makes sense. * - * @param ndims number of dimensions - * @param gdims - * @param idx - * @param dim_list - * @returns 0 on success, error code otherwise. + * @param ndims number of dimensions. + * @param gdimlen array of length ndims with the dimension sizes. + * @param idx the index to convert. This is the index into a 1-D array + * of data. + * @param dim_list array of length ndims that will get the dimensions + * corresponding to this index. */ -void idx_to_dim_list(int ndims, const int *gdims, PIO_Offset idx, +void idx_to_dim_list(int ndims, const int *gdimlen, PIO_Offset idx, PIO_Offset *dim_list) { - int i, curr_idx, next_idx; - curr_idx = idx; + /* Check inputs. */ + pioassert(ndims >= 0 && gdimlen && idx >= -1 && dim_list, "invalid input", + __FILE__, __LINE__); + LOG((2, "idx_to_dim_list ndims = %d idx = %d", ndims, idx)); /* Easiest to start from the right and move left. */ - for (i = ndims - 1; i >= 0; --i) + for (int i = ndims - 1; i >= 0; --i) { + int next_idx; + /* This way of doing div/mod is slightly faster than using "/" * and "%". */ - next_idx = curr_idx / gdims[i]; - dim_list[i] = curr_idx - (next_idx * gdims[i]); - curr_idx = next_idx; + next_idx = idx / gdimlen[i]; + dim_list[i] = idx - (next_idx * gdimlen[i]); + LOG((3, "next_idx = %d idx = %d gdimlen[%d] = %d dim_list[%d] = %d", + next_idx, idx, i, gdimlen[i], i, dim_list[i])); + idx = next_idx; } } @@ -64,30 +54,34 @@ void idx_to_dim_list(int ndims, const int *gdims, PIO_Offset idx, * outermost dimension, until the region has been expanded as much as * possible along all dimensions. * - * @param dim - * @param gdims array of dimension ids - * @param maplen the length of the map - * @param map - * @param region_size - * @param region_stride - * @param max_size array of maximum sizes - * @param count array of counts - * @returns 0 on success, error code otherwise. + * Precondition: maplen >= region_size (thus loop runs at least + * once). + * + * @param dim the dimension number to start with. + * @param gdimlen array of global dimension lengths. + * @param maplen the length of the map. + * @param map array (length maplen) with the the 1-based compmap. + * @param region_size ??? + * @param region_stride amount incremented along dimension. + * @param max_size array of size dim + 1 that contains the maximum + * sizes along that dimension. + * @param count array of size dim + 1 that gets the new counts. */ -void expand_region(int dim, const int *gdims, int maplen, const PIO_Offset *map, +void expand_region(int dim, const int *gdimlen, int maplen, const PIO_Offset *map, int region_size, int region_stride, const int *max_size, PIO_Offset *count) { - int i, j, test_idx, expansion_done; - /* Precondition: maplen >= region_size (thus loop runs at least - * once). */ - /* Flag used to signal that we can no longer expand the region along dimension dim. */ - expansion_done = 0; + int expansion_done = 0; + + /* Check inputs. */ + pioassert(dim >= 0 && gdimlen && maplen >= 0 && map && region_size >= 0 && + maplen >= region_size && region_stride >= 0 && max_size && count, + "invalid input", __FILE__, __LINE__); /* Expand no greater than max_size along this dimension. */ - for (i = 1; i <= max_size[dim]; ++i) + for (int i = 1; i <= max_size[dim]; ++i) { /* Count so far is at least i. */ count[dim] = i; @@ -97,8 +91,10 @@ void expand_region(int dim, const int *gdims, int maplen, const PIO_Offset *map, Assuming monotonicity in the map, we could skip this for the innermost dimension, but it's necessary past that because the region does not necessarily comprise contiguous values. */ - for (j = 0; j < region_size; ++j) + for (int j = 0; j < region_size; j++) { + int test_idx; /* Index we are testing. */ + test_idx = j + i * region_size; /* If we have exhausted the map, or the map no longer matches, @@ -116,65 +112,91 @@ void expand_region(int dim, const int *gdims, int maplen, const PIO_Offset *map, /* Move on to next outermost dimension if there are more left, * else return. */ if (dim > 0) - expand_region(dim-1, gdims, maplen, map, region_size * count[dim], - region_stride * gdims[dim], max_size, count); + expand_region(dim - 1, gdimlen, maplen, map, region_size * count[dim], + region_stride * gdimlen[dim], max_size, count); } /** - * Set start and count so that they describe the first region in map. + * Set start and count so that they describe the first region in + * map. * - * @param ndims the number of dimensions - * @param gdims pointer to an array of dimension ids - * @param maplen the length of the map + * This function is used when creating the subset rearranger (it + * is not used for the box rearranger). It is called by get_regions(). + * + * Preconditions: + *
    + *
  • ndims is > 0 + *
  • maplen is > 0 + *
  • All elements of map are inside the bounds specified by gdimlen. + *
+ * + * Note that the map array is 1 based, but calculations are 0 based. + * + * @param ndims the number of dimensions. + * @param gdimlen an array length ndims with the sizes of the global + * dimensions. + * @param maplen the length of the map. * @param map - * @param start array of start indicies - * @param count array of counts - * @returns 0 on success, error code otherwise. + * @param start array (length ndims) that will get start indicies of + * found region. + * @param count array (length ndims) that will get counts of found + * region. + * @returns length of the region found. */ -PIO_Offset find_region(int ndims, const int *gdims, int maplen, const PIO_Offset *map, +PIO_Offset find_region(int ndims, const int *gdimlen, int maplen, const PIO_Offset *map, PIO_Offset *start, PIO_Offset *count) { - int dim; - int max_size[ndims]; PIO_Offset regionlen = 1; - /* Preconditions (which might be useful to check/assert): - ndims is > 0 - maplen is > 0 - all elements of map are inside the bounds specified by gdims - The map array is 1 based, but calculations are 0 based */ - idx_to_dim_list(ndims, gdims, map[0] - 1, start); + /* Check inputs. */ + pioassert(ndims > 0 && gdimlen && maplen > 0 && map && start && count, + "invalid input", __FILE__, __LINE__); + LOG((2, "find_region ndims = %d maplen = %d", ndims, maplen)); + + int max_size[ndims]; + + /* Convert the index which is the first element of map into global + * data space. */ + idx_to_dim_list(ndims, gdimlen, map[0] - 1, start); - /* Can't expand beyond the array edge.*/ - for (dim = 0; dim < ndims; ++dim) - max_size[dim] = gdims[dim] - start[dim]; + /* Can't expand beyond the array edge. Set up max_size array for + * expand_region call below. */ + for (int dim = 0; dim < ndims; ++dim) + { + max_size[dim] = gdimlen[dim] - start[dim]; + LOG((3, "max_size[%d] = %d", max_size[dim])); + } /* For each dimension, figure out how far we can expand in that dimension while staying contiguous in the input array. Start with the innermost dimension (ndims-1), and it will recurse through to the outermost dimensions. */ - expand_region(ndims - 1, gdims, maplen, map, 1, 1, max_size, count); + expand_region(ndims - 1, gdimlen, maplen, map, 1, 1, max_size, count); - for (dim = 0; dim < ndims; dim++) + /* Calculate the number of data elements in this region. */ + for (int dim = 0; dim < ndims; dim++) regionlen *= count[dim]; - return(regionlen); + return regionlen; } /** * Convert a global coordinate value into a local array index. * - * @param ndims the number of dimensions - * @param lcoord pointer to an offset - * @param count array of counts - * @returns 0 on success, error code otherwise. + * @param ndims the number of dimensions. + * @param lcoord pointer to an offset. + * @param count array of counts. + * @returns the local array index. */ PIO_Offset coord_to_lindex(int ndims, const PIO_Offset *lcoord, const PIO_Offset *count) { PIO_Offset lindex = 0; PIO_Offset stride = 1; + /* Check inputs. */ + pioassert(ndims > 0 && lcoord && count, "invalid input", __FILE__, __LINE__); + for (int i = ndims - 1; i >= 0; i--) { lindex += lcoord[i] * stride; @@ -184,7 +206,10 @@ PIO_Offset coord_to_lindex(int ndims, const PIO_Offset *lcoord, const PIO_Offset } /** - * Compute the max io buffersize needed for a given variable + * Compute the max io buffer size needed for an iodesc. It is the + * combined size (in number of data elements) of all the regions of + * data stored in the buffer of this iodesc. The max size is then set + * in the iodesc. * * @param io_comm the IO communicator * @param iodesc a pointer to the io_desc_t struct. @@ -192,38 +217,34 @@ PIO_Offset coord_to_lindex(int ndims, const PIO_Offset *lcoord, const PIO_Offset */ int compute_maxIObuffersize(MPI_Comm io_comm, io_desc_t *iodesc) { - PIO_Offset iosize, totiosize; - int i; - io_region *region; + PIO_Offset totiosize = 0; int mpierr; /* Return code from MPI calls. */ - assert(iodesc); + pioassert(iodesc, "need iodesc", __FILE__, __LINE__); /* compute the max io buffer size, for conveneance it is the * combined size of all regions */ - totiosize = 0; - region = iodesc->firstregion; - while(region) + for (io_region *region = iodesc->firstregion; region; region = region->next) { if (region->count[0] > 0) { - iosize = 1; - for (i = 0; i < iodesc->ndims; i++) + PIO_Offset iosize = 1; + for (int i = 0; i < iodesc->ndims; i++) iosize *= region->count[i]; totiosize += iosize; } - region = region->next; } + LOG((2, "compute_maxIObuffersize got totiosize = %lld", totiosize)); /* Share the max io buffer size with all io tasks. */ if ((mpierr = MPI_Allreduce(MPI_IN_PLACE, &totiosize, 1, MPI_OFFSET, MPI_MAX, io_comm))) return check_mpi(NULL, mpierr, __FILE__, __LINE__); + pioassert(totiosize > 0, "totiosize <= 0", __FILE__, __LINE__); + LOG((2, "after allreduce compute_maxIObuffersize got totiosize = %lld", totiosize)); + /* Remember the result. */ iodesc->maxiobuflen = totiosize; - if (iodesc->maxiobuflen <= 0) - return pio_err(NULL, NULL, PIO_EINVAL, __FILE__, __LINE__); - return PIO_NOERR; } @@ -231,41 +252,47 @@ int compute_maxIObuffersize(MPI_Comm io_comm, io_desc_t *iodesc) * Create the derived MPI datatypes used for comp2io and io2comp * transfers. Used in define_iodesc_datatypes(). * - * @param basetype The type of data (int,real,double). - * @param msgcnt The number of MPI messages/tasks to use. - * @param dlen The length of the data array. - * @param mindex An array of indexes into the data array from the comp - * map - * @param mcount The number of indexes to be put on each mpi - * message/task + * @param basetype The MPI type of data (MPI_INT, etc.). + * @param msgcnt This is the number of MPI types that are created. + * @param mindex An array (length numinds) of indexes into the data + * array from the comp map. Will be NULL when count is zero. + * @param mcount An array (length msgcnt) with the number of indexes + * to be put on each mpi message/task. * @param mfrom A pointer to the previous structure in the read/write - * list - * @param mtype The final data structure sent through MPI to the - * read/write + * list. This is always NULL for the BOX rearranger. + * @param mtype pointer to an array (length msgcnt) which gets the + * created datatypes. Will be NULL when iodesc->nrecvs == 0. * @returns 0 on success, error code otherwise. */ -int create_mpi_datatypes(MPI_Datatype basetype, int msgcnt, PIO_Offset dlen, +int create_mpi_datatypes(MPI_Datatype basetype, int msgcnt, const PIO_Offset *mindex, const int *mcount, int *mfrom, MPI_Datatype *mtype) { - PIO_Offset bsizeT[msgcnt]; int blocksize; int numinds = 0; PIO_Offset *lindex = NULL; int mpierr; /* Return code from MPI functions. */ + /* Check inputs. */ + pioassert(msgcnt > 0 && mcount, "invalid input", __FILE__, __LINE__); + + PIO_Offset bsizeT[msgcnt]; + + LOG((1, "create_mpi_datatypes basetype = %d msgcnt = %d", basetype, msgcnt)); + LOG((2, "MPI_BYTE = %d MPI_CHAR = %d MPI_SHORT = %d MPI_INT = %d MPI_DOUBLE = %d", + MPI_BYTE, MPI_CHAR, MPI_SHORT, MPI_INT, MPI_DOUBLE)); + + /* How many indicies in the array? */ for (int j = 0; j < msgcnt; j++) numinds += mcount[j]; - - pioassert(dlen >= 0, "dlen < 0", __FILE__, __LINE__); - pioassert(numinds >= 0, "num inds < 0", __FILE__, __LINE__); + LOG((2, "numinds = %d", numinds)); if (mindex) { - /* memcpy(lindex, mindex, (size_t) (dlen*sizeof(PIO_Offset)));*/ if (!(lindex = malloc(numinds * sizeof(PIO_Offset)))) - return PIO_ENOMEM; + return pio_err(NULL, NULL, PIO_ENOMEM, __FILE__, __LINE__); memcpy(lindex, mindex, (size_t)(numinds * sizeof(PIO_Offset))); + LOG((3, "allocated lindex, copied mindex")); } bsizeT[0] = 0; @@ -273,82 +300,100 @@ int create_mpi_datatypes(MPI_Datatype basetype, int msgcnt, PIO_Offset dlen, int pos = 0; int ii = 0; - /* If there are no messages don't need to create any datatypes. */ - if (msgcnt > 0) + if (mfrom == NULL) { - if (mfrom == NULL) + LOG((3, "mfrom is NULL")); + for (int i = 0; i < msgcnt; i++) { - for (int i = 0; i < msgcnt; i++) + if (mcount[i] > 0) { - if (mcount[i] > 0) - { - bsizeT[ii] = GCDblocksize(mcount[i], lindex+pos); - ii++; - pos += mcount[i]; - } + /* Look for the largest block of data for io which + * can be expressed in terms of start and + * count. */ + bsizeT[ii] = GCDblocksize(mcount[i], lindex + pos); + ii++; + pos += mcount[i]; } - blocksize = (int)lgcd_array(ii ,bsizeT); - } - else - { - blocksize=1; } + blocksize = (int)lgcd_array(ii, bsizeT); + } + else + { + blocksize = 1; + } + LOG((3, "blocksize = %d", blocksize)); - /* pos is an index to the start of each message block. */ - pos = 0; - for (int i = 0; i < msgcnt; i++) + /* pos is an index to the start of each message block. */ + pos = 0; + for (int i = 0; i < msgcnt; i++) + { + if (mcount[i] > 0) { - if (mcount[i] > 0) + int len = mcount[i] / blocksize; + int displace[len]; + if (blocksize == 1) { - int len = mcount[i] / blocksize; - int displace[len]; - if (blocksize == 1) + if (!mfrom) { - if (!mfrom) - { - for (int j = 0; j < len; j++) - displace[j] = (int)(lindex[pos + j]); - } - else - { - int k = 0; - for (int j = 0; j < numinds; j++) - if (mfrom[j] == i) - displace[k++] = (int)(lindex[j]); - } - + for (int j = 0; j < len; j++) + displace[j] = (int)(lindex[pos + j]); } else { - for (int j = 0; j < mcount[i]; j++) - (lindex + pos)[j]++; - - for (int j = 0; j < len; j++) - displace[j] = ((lindex + pos)[j * blocksize] - 1); + int k = 0; + for (int j = 0; j < numinds; j++) + if (mfrom[j] == i) + displace[k++] = (int)(lindex[j]); } - if ((mpierr = MPI_Type_create_indexed_block(len, blocksize, displace, basetype, mtype + i))) - return check_mpi(NULL, mpierr, __FILE__, __LINE__); - - if (mtype[i] == PIO_DATATYPE_NULL) - return pio_err(NULL, NULL, PIO_EINVAL, __FILE__, __LINE__); + } + else + { + for (int j = 0; j < mcount[i]; j++) + (lindex + pos)[j]++; - /* Commit the MPI data type. */ - if ((mpierr = MPI_Type_commit(mtype + i))) - return check_mpi(NULL, mpierr, __FILE__, __LINE__); - pos += mcount[i]; + for (int j = 0; j < len; j++) + displace[j] = ((lindex + pos)[j * blocksize] - 1); } + +#if PIO_ENABLE_LOGGING + for (int j = 0; j < len; j++) + LOG((3, "displace[%d] = %d", j, displace[j])); +#endif /* PIO_ENABLE_LOGGING */ + + LOG((3, "calling MPI_Type_create_indexed_block len = %d blocksize = %d " + "basetype = %d", len, blocksize, basetype)); + /* Create an indexed datatype with constant-sized blocks. */ + if ((mpierr = MPI_Type_create_indexed_block(len, blocksize, displace, + basetype, &mtype[i]))) + return check_mpi(NULL, mpierr, __FILE__, __LINE__); + + if (mtype[i] == PIO_DATATYPE_NULL) + return pio_err(NULL, NULL, PIO_EINVAL, __FILE__, __LINE__); + + /* Commit the MPI data type. */ + LOG((3, "about to commit type")); + if ((mpierr = MPI_Type_commit(&mtype[i]))) + return check_mpi(NULL, mpierr, __FILE__, __LINE__); + pos += mcount[i]; } - if (lindex) - free(lindex); } + /* Free resources. */ + if (lindex) + free(lindex); + + LOG((3, "done with create_mpi_datatypes()")); return PIO_NOERR; } /** - * Create the derived MPI datatypes used for comp2io and io2comp - * transfers. + * If needed, create the derived MPI datatypes used for comp2io and + * io2comp transfers. + * + * If iodesc->stype and iodesc->rtype arrays already exist, this + * function does nothing. This function is called from + * rearrange_io2comp() and rearrange_comp2io(). * * NOTE from Jim: I am always oriented toward write so recieve * always means io tasks and send always means comp tasks. The @@ -366,142 +411,140 @@ int define_iodesc_datatypes(iosystem_desc_t *ios, io_desc_t *iodesc) { int ret; /* Return value. */ + pioassert(ios && iodesc, "invalid input", __FILE__, __LINE__); + LOG((1, "define_iodesc_datatypes ios->ioproc = %d", ios->ioproc)); + /* Set up the to transfer data to and from the IO tasks. */ if (ios->ioproc) { + /* If the types for the IO tasks have not been created, then + * create them. */ if (!iodesc->rtype) { if (iodesc->nrecvs > 0) { - /* Allocate memory for array of data. */ + /* Allocate memory for array of MPI types for the IO tasks. */ if (!(iodesc->rtype = malloc(iodesc->nrecvs * sizeof(MPI_Datatype)))) return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); + LOG((2, "allocated memory for IO task MPI types iodesc->nrecvs = %d " + "iodesc->rearranger = %d", iodesc->nrecvs, iodesc->rearranger)); /* Initialize data types to NULL. */ for (int i = 0; i < iodesc->nrecvs; i++) iodesc->rtype[i] = PIO_DATATYPE_NULL; - /* Create the datatypes, which will be used both to - * receive and to send data. */ - if (iodesc->rearranger == PIO_REARR_SUBSET) - { - if ((ret = create_mpi_datatypes(iodesc->basetype, iodesc->nrecvs, iodesc->llen, - iodesc->rindex, iodesc->rcount, iodesc->rfrom, - iodesc->rtype))) - return pio_err(ios, NULL, ret, __FILE__, __LINE__); - } - else - { - if ((ret = create_mpi_datatypes(iodesc->basetype, iodesc->nrecvs, iodesc->llen, - iodesc->rindex, iodesc->rcount, NULL, iodesc->rtype))) - return pio_err(ios, NULL, ret, __FILE__, __LINE__); - } + /* The different rearrangers get different values for mfrom. */ + int *mfrom = iodesc->rearranger == PIO_REARR_SUBSET ? iodesc->rfrom : NULL; + + /* Create the MPI datatypes. */ + if ((ret = create_mpi_datatypes(iodesc->basetype, iodesc->nrecvs, iodesc->rindex, + iodesc->rcount, mfrom, iodesc->rtype))) + return pio_err(ios, NULL, ret, __FILE__, __LINE__); } } } - /* Define the send datatypes if they don't exist. */ + /* Define the datatypes for the computation components if they + * don't exist. (These will be the send side in a write + * operation.) */ if (!iodesc->stype) { int ntypes; - int ncnt; - if (iodesc->rearranger == PIO_REARR_SUBSET) - { - /* Subset rearranger gets one type. */ - ntypes = 1; - ncnt = iodesc->scount[0]; - } - else - { - /* Box rearranger gets one type per IO task. */ - ntypes = ios->num_iotasks; - ncnt = iodesc->ndof; - } + /* Subset rearranger gets one type; box rearranger gets one + * type per IO task. */ + ntypes = iodesc->rearranger == PIO_REARR_SUBSET ? 1 : ios->num_iotasks; - /* Allocate memory for array of send types. */ + /* Allocate memory for array of MPI types for the computation tasks. */ if (!(iodesc->stype = malloc(ntypes * sizeof(MPI_Datatype)))) return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); + LOG((3, "allocated memory for computation MPI types ntypes = %d", ntypes)); /* Initialize send types to NULL. */ for (int i = 0; i < ntypes; i++) iodesc->stype[i] = PIO_DATATYPE_NULL; + /* Remember how many types we created for the send side. */ iodesc->num_stypes = ntypes; /* Create the MPI data types. */ - if ((ret = create_mpi_datatypes(iodesc->basetype, ntypes, ncnt, iodesc->sindex, + LOG((3, "about to call create_mpi_datatypes for computation MPI types")); + if ((ret = create_mpi_datatypes(iodesc->basetype, ntypes, iodesc->sindex, iodesc->scount, NULL, iodesc->stype))) return pio_err(ios, NULL, ret, __FILE__, __LINE__); } + LOG((3, "done with define_iodesc_datatypes()")); return PIO_NOERR; } /** - * Completes the mapping for the box rearranger. + * Completes the mapping for the box rearranger. This function is + * called from box_rearrange_create(). It is not used for the subset + * rearranger. + * + * This function: + *
    + *
  • Allocates and inits iodesc->scount, an array (length + * ios->num_iotasks) containing number of data elements sent to each + * IO task from current compute task. + *
  • Uses pio_swapm() to send iodesc->scount array from each + * computation task to all IO tasks. + *
  • On IO tasks, allocates and inits iodesc->rcount and + * iodesc->rfrom arrays (length max(1, nrecvs)) which holds the amount + * of data to expect from each compute task and the rank of that + * task. . + *
  • Allocates and inits iodesc->sindex arrays (length iodesc->ndof) + * which holds indecies for computation tasks. + *
  • On IO tasks, allocates and init iodesc->rindex (length + * totalrecv) with indices of the data to be sent/received from this + * io task to each compute task. + *
  • Uses pio_swapm() to send list of indicies on each compute task + * to the IO tasks. + *
* - * @param ios pointer to the iosystem_desc_t struct + * @param ios pointer to the iosystem_desc_t struct. * @param iodesc a pointer to the io_desc_t struct. - * @param maplen the length of the map - * @param dest_ioproc an array of IO task numbers - * @param dest_ioindex - * @param mycomm an MPI communicator + * @param dest_ioproc an array (length maplen) of IO task numbers. + * @param dest_ioindex an array (length maplen) of IO indicies. * @returns 0 on success, error code otherwise. */ -int compute_counts(iosystem_desc_t *ios, io_desc_t *iodesc, int maplen, - const int *dest_ioproc, const PIO_Offset *dest_ioindex, MPI_Comm mycomm) +int compute_counts(iosystem_desc_t *ios, io_desc_t *iodesc, + const int *dest_ioproc, const PIO_Offset *dest_ioindex) { - int i; - int iorank; - int rank; - int ntasks; - int mpierr; /* Return call from MPI functions. */ - - /* Find size of communicator, and task rank. */ - if ((mpierr = MPI_Comm_rank(mycomm, &rank))) - return check_mpi(NULL, mpierr, __FILE__, __LINE__); - if ((mpierr = MPI_Comm_size(mycomm, &ntasks))) - return check_mpi(NULL, mpierr, __FILE__, __LINE__); - - MPI_Datatype sr_types[ntasks]; - int send_counts[ntasks]; - int send_displs[ntasks]; - int recv_counts[ntasks]; - int recv_displs[ntasks]; int *recv_buf = NULL; - int nrecvs; + int nrecvs = 0; int ierr; - int io_comprank; - int ioindex; - int tsize; - int numiotasks; - PIO_Offset s2rindex[iodesc->ndof]; - pioassert(iodesc, "iodesc must be provided", __FILE__, __LINE__); + /* Check inputs. */ + pioassert(ios && iodesc && dest_ioproc && dest_ioindex && + iodesc->rearranger == PIO_REARR_BOX && ios->num_uniontasks > 0, + "invalid input", __FILE__, __LINE__); + LOG((1, "compute_counts ios->num_uniontasks = %d", ios->num_uniontasks)); - /* Subset rearranger always gets 1 IO task. */ - if (iodesc->rearranger == PIO_REARR_BOX) - numiotasks = ios->num_iotasks; - else - numiotasks = 1; + /* Arrays for swapm all to all gather calls. */ + MPI_Datatype sr_types[ios->num_uniontasks]; + int send_counts[ios->num_uniontasks]; + int send_displs[ios->num_uniontasks]; + int recv_counts[ios->num_uniontasks]; + int recv_displs[ios->num_uniontasks]; - /* Allocate memory for the array of counts. */ - if (!(iodesc->scount = malloc(numiotasks * sizeof(int)))) - return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); + /* The list of indeces on each compute task */ + PIO_Offset s2rindex[iodesc->ndof]; - /* Initialize counts to zero. */ - for (i = 0; i < numiotasks; i++) - iodesc->scount[i] = 0; + /* Allocate memory for the array of counts and init to zero. */ + if (!(iodesc->scount = calloc(ios->num_iotasks, sizeof(int)))) + return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); - /* iodesc->scount is the amount of data sent to each task from the - * current task */ - for (i = 0; i < maplen; i++) + /* iodesc->scount is the number of data elements sent to each IO + * task from the current compute task. dest_ioindex[i] may be + * -1. */ + for (int i = 0; i < iodesc->ndof; i++) if (dest_ioindex[i] >= 0) (iodesc->scount[dest_ioproc[i]])++; - /* Initialize arrays. */ - for (i = 0; i < ntasks; i++) + /* Initialize arrays used in swapm call. */ + for (int i = 0; i < ios->num_uniontasks; i++) { send_counts[i] = 0; send_displs[i] = 0; @@ -510,66 +553,74 @@ int compute_counts(iosystem_desc_t *ios, io_desc_t *iodesc, int maplen, sr_types[i] = MPI_INT; } - /* ??? */ - for (i = 0; i < numiotasks; i++) + /* Setup for the swapm call. iodesc->scount is the amount of data + * this compute task will transfer to/from each iotask. For the + * box rearranger there can be more than one IO task per compute + * task. This provides enough information to know the size of data + * on the iotask, so at line 557 we allocate arrays to hold the + * map on the iotasks. iodesc->rcount is an array of the amount of + * data to expect from each compute task and iodesc->rfrom is the + * rank of that task. */ + if (ios->compproc) { - int io_comprank; - if (iodesc->rearranger == PIO_REARR_SUBSET) - io_comprank = 0; - else - io_comprank = ios->ioranks[i]; - send_counts[io_comprank] = 1; - send_displs[io_comprank] = i * sizeof(int); + for (int i = 0; i < ios->num_iotasks; i++) + { + send_counts[ios->ioranks[i]] = 1; + send_displs[ios->ioranks[i]] = i * sizeof(int); + LOG((3, "send_counts[%d] = %d send_displs[%d] = %d", ios->ioranks[i], + send_counts[ios->ioranks[i]], ios->ioranks[i], send_displs[ios->ioranks[i]])); + } } - - /* ??? */ + + /* IO tasks need to know how many data elements they will receive + * from each compute task. Allocate space for that, and set up + * swapm call. */ if (ios->ioproc) { - /* Allocate memory to hold array of tasks that have recieved - * data ??? */ - if (!(recv_buf = calloc(ntasks, sizeof(int)))) + /* Allocate memory to hold array of the scounts from all + * computation tasks. */ + if (!(recv_buf = calloc(ios->num_comptasks, sizeof(int)))) return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); - /* Initialize arrays that keep track of receives. */ - for (i = 0; i < ntasks; i++) + /* Initialize arrays that keep track of ???. */ + for (int i = 0; i < ios->num_comptasks; i++) { - recv_counts[i] = 1; - recv_displs[i] = i * sizeof(int); + recv_counts[ios->compranks[i]] = 1; + recv_displs[ios->compranks[i]] = i * sizeof(int); } } - /* Share the iodesc->scount from each compute task to all io tasks. */ - ierr = pio_swapm(iodesc->scount, send_counts, send_displs, sr_types, - recv_buf, recv_counts, recv_displs, sr_types, - mycomm, - iodesc->rearr_opts.comm_fc_opts_comp2io.enable_hs, - iodesc->rearr_opts.comm_fc_opts_comp2io.enable_isend, - iodesc->rearr_opts.comm_fc_opts_comp2io.max_pend_req); + LOG((2, "about to share scount from each compute task to all IO tasks.")); + /* Share the iodesc->scount from each compute task to all IO + * tasks. The scounts will end up in array recv_buf. */ + if ((ierr = pio_swapm(iodesc->scount, send_counts, send_displs, sr_types, + recv_buf, recv_counts, recv_displs, sr_types, ios->union_comm, + &iodesc->rearr_opts.comp2io))) + return pio_err(ios, NULL, ierr, __FILE__, __LINE__); - /* ??? */ - nrecvs = 0; + /* On IO tasks, set up data receives. */ if (ios->ioproc) { - for (i = 0; i < ntasks; i++) + /* Count the number of non-zero scounts from the compute + * tasks. */ + for (int i = 0; i < ios->num_comptasks; i++) + { if (recv_buf[i] != 0) nrecvs++; + LOG((3, "recv_buf[%d] = %d", i, recv_buf[i])); + } /* Get memory to hold the count of data receives. */ - if (!(iodesc->rcount = malloc(max(1, nrecvs) * sizeof(int)))) + if (!(iodesc->rcount = calloc(max(1, nrecvs), sizeof(int)))) return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); /* Get memory to hold the list of task data was from. */ - if (!(iodesc->rfrom = malloc(max(1, nrecvs) * sizeof(int)))) + if (!(iodesc->rfrom = calloc(max(1, nrecvs), sizeof(int)))) return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); - - for (i = 0; i < max(1, nrecvs); i++) - { - iodesc->rcount[i] = 0; - iodesc->rfrom[i] = 0; - } + LOG((3, "allocared rfrom max(1, nrecvs) = %d", max(1, nrecvs))); nrecvs = 0; - for (i = 0; i < ntasks; i++) + for (int i = 0; i < ios->num_comptasks; i++) { if (recv_buf[i] != 0) { @@ -583,39 +634,50 @@ int compute_counts(iosystem_desc_t *ios, io_desc_t *iodesc, int maplen, /* ??? */ iodesc->nrecvs = nrecvs; + LOG((3, "iodesc->nrecvs = %d", iodesc->nrecvs)); + + /* Allocate an array for indicies on the computation tasks (the + * send side when writing). */ if (iodesc->sindex == NULL && iodesc->ndof > 0) if (!(iodesc->sindex = malloc(iodesc->ndof * sizeof(PIO_Offset)))) return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); + LOG((2, "iodesc->ndof = %d", iodesc->ndof)); - int tempcount[numiotasks]; - int spos[numiotasks]; + int tempcount[ios->num_iotasks]; + int spos[ios->num_iotasks]; /* ??? */ spos[0] = 0; tempcount[0] = 0; - for (i = 1; i < numiotasks; i++) + for (int i = 1; i < ios->num_iotasks; i++) { spos[i] = spos[i - 1] + iodesc->scount[i - 1]; tempcount[i] = 0; + LOG((3, "spos[%d] = %d tempcount[%d] = %d", i, spos[i], i, tempcount[i])); } - for (i = 0; i < maplen; i++) + /* ??? */ + for (int i = 0; i < iodesc->ndof; i++) { + int iorank; + int ioindex; + iorank = dest_ioproc[i]; ioindex = dest_ioindex[i]; if (iorank > -1) { /* this should be moved to create_box */ - if (iodesc->rearranger == PIO_REARR_BOX) - iodesc->sindex[spos[iorank] + tempcount[iorank]] = i; + iodesc->sindex[spos[iorank] + tempcount[iorank]] = i; s2rindex[spos[iorank] + tempcount[iorank]] = ioindex; (tempcount[iorank])++; + LOG((3, "iorank = %d ioindex = %d tempcount[iorank] = %d", iorank, ioindex, + tempcount[iorank])); } } /* Initialize arrays to zeros. */ - for (i = 0; i < ntasks; i++) + for (int i = 0; i < ios->num_uniontasks; i++) { send_counts[i] = 0; send_displs[i] = 0; @@ -623,97 +685,83 @@ int compute_counts(iosystem_desc_t *ios, io_desc_t *iodesc, int maplen, recv_displs[i] = 0; } - /* Find the size of the offset type. */ - if ((mpierr = MPI_Type_size(MPI_OFFSET, &tsize))) - return check_mpi(NULL, mpierr, __FILE__, __LINE__); - - for (i = 0; i < ntasks; i++) - sr_types[i] = MPI_OFFSET; - - for (i = 0; i < numiotasks; i++) + /* ??? */ + for (int i = 0; i < ios->num_iotasks; i++) { - if (iodesc->rearranger == PIO_REARR_BOX) - io_comprank = ios->ioranks[i]; - else - io_comprank = 0; - - send_counts[io_comprank] = iodesc->scount[i]; - if (send_counts[io_comprank] > 0) - send_displs[io_comprank] = spos[i] * tsize; + /* Subset rearranger needs one type, box rearranger needs one for + * each IO task. */ + send_counts[ios->ioranks[i]] = iodesc->scount[i]; + if (send_counts[ios->ioranks[i]] > 0) + send_displs[ios->ioranks[i]] = spos[i] * SIZEOF_MPI_OFFSET; + LOG((3, "ios->ioranks[i] = %d iodesc->scount[%d] = %d spos[%d] = %d", + ios->ioranks[i], i, iodesc->scount[i], i, spos[i])); } + /* Only do this on IO tasks. */ if (ios->ioproc) { int totalrecv = 0; - for (i = 0; i < nrecvs; i++) + for (int i = 0; i < nrecvs; i++) { recv_counts[iodesc->rfrom[i]] = iodesc->rcount[i]; - totalrecv+=iodesc->rcount[i]; + totalrecv += iodesc->rcount[i]; } - recv_displs[0] = 0; - for (i = 1; i < nrecvs; i++) - recv_displs[iodesc->rfrom[i]] = recv_displs[iodesc->rfrom[i-1]] + iodesc->rcount[i-1] * tsize; + recv_displs[0] = 0; + for (int i = 1; i < nrecvs; i++) + { + recv_displs[iodesc->rfrom[i]] = recv_displs[iodesc->rfrom[i - 1]] + + iodesc->rcount[i - 1] * SIZEOF_MPI_OFFSET; + LOG((3, "iodesc->rfrom[%d] = %d recv_displs[iodesc->rfrom[i]] = %d", i, + iodesc->rfrom[i], recv_displs[iodesc->rfrom[i]])); + } + /* rindex is an array of the indices of the data to be sent from + this io task to each compute task. */ + LOG((3, "totalrecv = %d", totalrecv)); if (totalrecv > 0) { totalrecv = iodesc->llen; /* can reduce memory usage here */ if (!(iodesc->rindex = calloc(totalrecv, sizeof(PIO_Offset)))) return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); + LOG((3, "allocated totalrecv elements in rindex array")); } } + /* For the swapm call below, init the types to MPI_OFFSET. */ + for (int i = 0; i < ios->num_uniontasks; i++) + sr_types[i] = MPI_OFFSET; + + /* Here we are sending the mapping from the index on the compute + * task to the index on the io task. */ /* s2rindex is the list of indeces on each compute task */ - /* - printf("%d s2rindex: ", ios->comp_rank); - for (i = 0;indof;i++) - printf("%ld ",s2rindex[i]); - printf("\n"); - */ - ierr = pio_swapm(s2rindex, send_counts, send_displs, sr_types, iodesc->rindex, - recv_counts, recv_displs, sr_types, mycomm, - iodesc->rearr_opts.comm_fc_opts_comp2io.enable_hs, - iodesc->rearr_opts.comm_fc_opts_comp2io.enable_isend, - iodesc->rearr_opts.comm_fc_opts_comp2io.max_pend_req); - - /* rindex is an array of the indices of the data to be sent from - this io task to each compute task. */ - /* - if (ios->ioproc){ - printf("%d rindex: ",ios->io_rank); - for (int j = 0;jllen;j++) - printf(" %ld ",iodesc->rindex[j]); - printf("\n"); - } - */ - return ierr; + LOG((3, "sending mapping")); + if ((ierr = pio_swapm(s2rindex, send_counts, send_displs, sr_types, iodesc->rindex, + recv_counts, recv_displs, sr_types, ios->union_comm, + &iodesc->rearr_opts.comp2io))) + return pio_err(ios, NULL, ierr, __FILE__, __LINE__); + + return PIO_NOERR; } /** - * Moves data from compute tasks to IO tasks. + * Moves data from compute tasks to IO tasks. This is called from + * PIOc_write_darray_multi(). * - * @param ios pointer to the iosystem_desc_t struct + * @param ios pointer to the iosystem_desc_t struct. * @param iodesc a pointer to the io_desc_t struct. - * @param sbuf send buffer. - * @param rbuf receive buffer. + * @param sbuf send buffer. May be NULL. + * @param rbuf receive buffer. May be NULL. * @param nvars number of variables. * @returns 0 on success, error code otherwise. */ int rearrange_comp2io(iosystem_desc_t *ios, io_desc_t *iodesc, void *sbuf, void *rbuf, int nvars) { - int ntasks; - int niotasks; - int *scount = iodesc->scount; - int i, tsize; - int *sendcounts; - int *recvcounts; - int *sdispls; - int *rdispls; - MPI_Datatype *sendtypes; - MPI_Datatype *recvtypes; - MPI_Comm mycomm; - int mpierr; /* Return code from MPI calls. */ + int ntasks; /* Number of tasks in communicator. */ + int niotasks; /* Number of IO tasks. */ + MPI_Comm mycomm; /* Communicator that data is transferred over. */ + int mpierr; /* Return code from MPI calls. */ int ret; #ifdef TIMING @@ -721,11 +769,12 @@ int rearrange_comp2io(iosystem_desc_t *ios, io_desc_t *iodesc, void *sbuf, #endif /* Caller must provide these. */ - pioassert(iodesc && nvars > 0, "invalid input", __FILE__, __LINE__); + pioassert(ios && iodesc && nvars > 0, "invalid input", __FILE__, __LINE__); - LOG((2, "rearrange_comp2io nvars = %d iodesc->rearranger = %d", nvars, + LOG((1, "rearrange_comp2io nvars = %d iodesc->rearranger = %d", nvars, iodesc->rearranger)); + /* Different rearraangers use different communicators. */ if (iodesc->rearranger == PIO_REARR_BOX) { mycomm = ios->union_comm; @@ -741,83 +790,89 @@ int rearrange_comp2io(iosystem_desc_t *ios, io_desc_t *iodesc, void *sbuf, if ((mpierr = MPI_Comm_size(mycomm, &ntasks))) return check_mpi(NULL, mpierr, __FILE__, __LINE__); - /* Get the size of the MPI type. */ - if ((mpierr = MPI_Type_size(iodesc->basetype, &tsize))) - return check_mpi(NULL, mpierr, __FILE__, __LINE__); - LOG((3, "ntasks = %d tsize = %d", ntasks, tsize)); - - /* Define the MPI data types that will be used for this - * io_desc_t. */ - if ((ret = define_iodesc_datatypes(ios, iodesc))) - return pio_err(ios, NULL, ret, __FILE__, __LINE__); - - /* Allocate arrays needed by the pio_swapm() function. */ - if (!(sendcounts = calloc(ntasks, sizeof(int)))) - return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); - - if (!(recvcounts = calloc(ntasks, sizeof(int)))) - return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); - - if (!(sdispls = calloc(ntasks, sizeof(int)))) - return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); - - if (!(rdispls = calloc(ntasks, sizeof(int)))) - return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); - - if (!(sendtypes = malloc(ntasks * sizeof(MPI_Datatype)))) - return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); - - if (!(recvtypes = malloc(ntasks * sizeof(MPI_Datatype)))) - return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); + /* These are parameters to pio_swapm to send data from compute to + * IO tasks. */ + int sendcounts[ntasks]; + int recvcounts[ntasks]; + int sdispls[ntasks]; + int rdispls[ntasks]; + MPI_Datatype sendtypes[ntasks]; + MPI_Datatype recvtypes[ntasks]; - /* Initialize arrays. */ - for (i = 0; i < ntasks; i++) + /* Initialize pio_swapm parameter arrays. */ + for (int i = 0; i < ntasks; i++) { + sendcounts[i] = 0; + recvcounts[i] = 0; + sdispls[i] = 0; + rdispls[i] = 0; recvtypes[i] = PIO_DATATYPE_NULL; sendtypes[i] = PIO_DATATYPE_NULL; } + LOG((3, "ntasks = %d iodesc->basetype_size = %d niotasks = %d", ntasks, + iodesc->basetype_size, niotasks)); + + /* If it has not already been done, define the MPI data types that + * will be used for this io_desc_t. */ + if ((ret = define_iodesc_datatypes(ios, iodesc))) + return pio_err(ios, NULL, ret, __FILE__, __LINE__); /* If this io proc will exchange data with compute tasks create a * MPI DataType for that exchange. */ + LOG((2, "ios->ioproc %d iodesc->nrecvs = %d", ios->ioproc, iodesc->nrecvs)); if (ios->ioproc && iodesc->nrecvs > 0) { - for (i = 0; i < iodesc->nrecvs; i++) + for (int i = 0; i < iodesc->nrecvs; i++) { if (iodesc->rtype[i] != PIO_DATATYPE_NULL) { + LOG((3, "iodesc->rtype[%d] = %d iodesc->rearranger = %d", i, iodesc->rtype[i], + iodesc->rearranger)); if (iodesc->rearranger == PIO_REARR_SUBSET) { - recvcounts[ i ] = 1; - - /* The stride here is the length of the collected array (llen) */ - - if ((mpierr = MPI_Type_hvector(nvars, 1, (MPI_Aint) iodesc->llen * tsize, iodesc->rtype[i], - recvtypes + i))) + LOG((3, "exchanging data for subset rearranger")); + recvcounts[i] = 1; + + /* Create an MPI derived data type from equally + * spaced blocks of the same size. The block size + * is 1, the stride here is the length of the + * collected array (llen). */ +#if PIO_USE_MPISERIAL + if ((mpierr = MPI_Type_hvector(nvars, 1, (MPI_Aint)iodesc->llen * iodesc->basetype_size, + iodesc->rtype[i], &recvtypes[i]))) return check_mpi(NULL, mpierr, __FILE__, __LINE__); - - if (recvtypes[i] == PIO_DATATYPE_NULL) - return pio_err(NULL, NULL, PIO_EINVAL, __FILE__, __LINE__); - - if ((mpierr = MPI_Type_commit(recvtypes + i))) +#else + if ((mpierr = MPI_Type_create_hvector(nvars, 1, (MPI_Aint)iodesc->llen * iodesc->basetype_size, + iodesc->rtype[i], &recvtypes[i]))) return check_mpi(NULL, mpierr, __FILE__, __LINE__); +#endif /* PIO_USE_MPISERIAL */ + pioassert(recvtypes[i] != PIO_DATATYPE_NULL, "bad mpi type", __FILE__, __LINE__); - /*recvtypes[ i ] = iodesc->rtype[i]; */ + if ((mpierr = MPI_Type_commit(&recvtypes[i]))) + return check_mpi(NULL, mpierr, __FILE__, __LINE__); } else { + LOG((3, "exchanging data for box rearranger")); + LOG((3, "i = %d iodesc->rfrom[i] = %d recvcounts[iodesc->rfrom[i]] = %d", i, + iodesc->rfrom[i], recvcounts[iodesc->rfrom[i]])); recvcounts[iodesc->rfrom[i]] = 1; - if ((mpierr = MPI_Type_hvector(nvars, 1, (MPI_Aint)iodesc->llen * tsize, iodesc->rtype[i], - recvtypes + iodesc->rfrom[i]))) +#if PIO_USE_MPISERIAL + if ((mpierr = MPI_Type_hvector(nvars, 1, (MPI_Aint)iodesc->llen * iodesc->basetype_size, + iodesc->rtype[i], &recvtypes[iodesc->rfrom[i]]))) return check_mpi(NULL, mpierr, __FILE__, __LINE__); +#else + if ((mpierr = MPI_Type_create_hvector(nvars, 1, (MPI_Aint)iodesc->llen * iodesc->basetype_size, + iodesc->rtype[i], &recvtypes[iodesc->rfrom[i]]))) + return check_mpi(NULL, mpierr, __FILE__, __LINE__); +#endif /* PIO_USE_MPISERIAL */ + pioassert(recvtypes[iodesc->rfrom[i]] != PIO_DATATYPE_NULL, "bad mpi type", + __FILE__, __LINE__); - if (recvtypes[iodesc->rfrom[i]] == PIO_DATATYPE_NULL) - return pio_err(NULL, NULL, PIO_EINVAL, __FILE__, __LINE__); - - if ((mpierr = MPI_Type_commit(recvtypes+iodesc->rfrom[i]))) + if ((mpierr = MPI_Type_commit(&recvtypes[iodesc->rfrom[i]]))) return check_mpi(NULL, mpierr, __FILE__, __LINE__); - /* recvtypes[ iodesc->rfrom[i] ] = iodesc->rtype[i]; */ rdispls[iodesc->rfrom[i]] = 0; } } @@ -826,57 +881,57 @@ int rearrange_comp2io(iosystem_desc_t *ios, io_desc_t *iodesc, void *sbuf, /* On compute tasks loop over iotasks and create a data type for * each exchange. */ - for (i = 0; i < niotasks; i++) + for (int i = 0; i < niotasks; i++) { int io_comprank = ios->ioranks[i]; if (iodesc->rearranger == PIO_REARR_SUBSET) - io_comprank=0; + io_comprank = 0; - if (scount[i] > 0 && sbuf != NULL) + LOG((3, "i = %d iodesc->scount[i] = %d", i, iodesc->scount[i])); + if (iodesc->scount[i] > 0 && sbuf) { + LOG((3, "io task %d creating sendtypes[%d]", i, io_comprank)); sendcounts[io_comprank] = 1; - if ((mpierr = MPI_Type_hvector(nvars, 1, (MPI_Aint)iodesc->ndof * tsize, iodesc->stype[i], - sendtypes + io_comprank))) +#if PIO_USE_MPISERIAL + if ((mpierr = MPI_Type_hvector(nvars, 1, (MPI_Aint)iodesc->ndof * iodesc->basetype_size, + iodesc->stype[i], &sendtypes[io_comprank]))) return check_mpi(NULL, mpierr, __FILE__, __LINE__); +#else + if ((mpierr = MPI_Type_create_hvector(nvars, 1, (MPI_Aint)iodesc->ndof * iodesc->basetype_size, + iodesc->stype[i], &sendtypes[io_comprank]))) + return check_mpi(NULL, mpierr, __FILE__, __LINE__); +#endif /* PIO_USE_MPISERIAL */ + pioassert(sendtypes[io_comprank] != PIO_DATATYPE_NULL, "bad mpi type", __FILE__, __LINE__); - if (sendtypes[io_comprank] == PIO_DATATYPE_NULL) - return pio_err(NULL, NULL, PIO_EINVAL, __FILE__, __LINE__); - - if ((mpierr = MPI_Type_commit(sendtypes + io_comprank))) + if ((mpierr = MPI_Type_commit(&sendtypes[io_comprank]))) return check_mpi(NULL, mpierr, __FILE__, __LINE__); } else { - sendcounts[io_comprank]=0; + sendcounts[io_comprank] = 0; } } + /* Data in sbuf on the compute nodes is sent to rbuf on the ionodes */ - pio_swapm(sbuf, sendcounts, sdispls, sendtypes, - rbuf, recvcounts, rdispls, recvtypes, mycomm, - iodesc->rearr_opts.comm_fc_opts_comp2io.enable_hs, - iodesc->rearr_opts.comm_fc_opts_comp2io.enable_isend, - iodesc->rearr_opts.comm_fc_opts_comp2io.max_pend_req); + LOG((2, "about to call pio_swapm for sbuf")); + if ((ret = pio_swapm(sbuf, sendcounts, sdispls, sendtypes, + rbuf, recvcounts, rdispls, recvtypes, mycomm, + &iodesc->rearr_opts.comp2io))) + return pio_err(ios, NULL, ret, __FILE__, __LINE__); /* Free the MPI types. */ - for (i = 0; i < ntasks; i++) + for (int i = 0; i < ntasks; i++) { + LOG((3, "freeing MPI types for task %d", i)); if (sendtypes[i] != PIO_DATATYPE_NULL) - if ((mpierr = MPI_Type_free(sendtypes + i))) + if ((mpierr = MPI_Type_free(&sendtypes[i]))) return check_mpi(NULL, mpierr, __FILE__, __LINE__); if (recvtypes[i] != PIO_DATATYPE_NULL) - if ((mpierr = MPI_Type_free(recvtypes + i))) + if ((mpierr = MPI_Type_free(&recvtypes[i]))) return check_mpi(NULL, mpierr, __FILE__, __LINE__); } - /* Free memory. */ - free(sendcounts); - free(recvcounts); - free(sdispls); - free(rdispls); - free(sendtypes); - free(recvtypes); - #ifdef TIMING GPTLstop("PIO:rearrange_comp2io"); #endif @@ -885,7 +940,8 @@ int rearrange_comp2io(iosystem_desc_t *ios, io_desc_t *iodesc, void *sbuf, } /** - * Moves data from IO tasks to compute tasks. + * Moves data from IO tasks to compute tasks. This function is used in + * PIOc_read_darray(). * * @param ios pointer to the iosystem_desc_t struct. * @param iodesc a pointer to the io_desc_t struct. @@ -899,18 +955,11 @@ int rearrange_io2comp(iosystem_desc_t *ios, io_desc_t *iodesc, void *sbuf, MPI_Comm mycomm; int ntasks; int niotasks; - int *scount = iodesc->scount; - int *sendcounts; - int *recvcounts; - int *sdispls; - int *rdispls; - MPI_Datatype *sendtypes; - MPI_Datatype *recvtypes; - int i; int mpierr; /* Return code from MPI calls. */ int ret; - assert(iodesc); + /* Check inputs. */ + pioassert(ios && iodesc, "invalid input", __FILE__, __LINE__); #ifdef TIMING GPTLstart("PIO:rearrange_io2comp"); @@ -926,8 +975,9 @@ int rearrange_io2comp(iosystem_desc_t *ios, io_desc_t *iodesc, void *sbuf, else { mycomm = iodesc->subset_comm; - niotasks=1; + niotasks = 1; } + LOG((3, "niotasks = %d", niotasks)); /* Get the size of this communicator. */ if ((mpierr = MPI_Comm_size(mycomm, &ntasks))) @@ -939,34 +989,30 @@ int rearrange_io2comp(iosystem_desc_t *ios, io_desc_t *iodesc, void *sbuf, return pio_err(ios, NULL, ret, __FILE__, __LINE__); /* Allocate arrays needed by the pio_swapm() function. */ - if (!(sendcounts = calloc(ntasks, sizeof(int)))) - return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); - - if (!(recvcounts = calloc(ntasks, sizeof(int)))) - return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); - - if (!(sdispls = calloc(ntasks, sizeof(int)))) - return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); - - if (!(rdispls = calloc(ntasks, sizeof(int)))) - return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); - - if (!(sendtypes = malloc(ntasks * sizeof(MPI_Datatype)))) - return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); - - if (!(recvtypes = malloc(ntasks * sizeof(MPI_Datatype)))) - return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); + int sendcounts[ntasks]; + int recvcounts[ntasks]; + int sdispls[ntasks]; + int rdispls[ntasks]; + MPI_Datatype sendtypes[ntasks]; + MPI_Datatype recvtypes[ntasks]; /* Initialize arrays. */ - for (i = 0; i < ntasks; i++) + for (int i = 0; i < ntasks; i++) { + sendcounts[i] = 0; + recvcounts[i] = 0; + sdispls[i] = 0; + rdispls[i] = 0; sendtypes[i] = PIO_DATATYPE_NULL; recvtypes[i] = PIO_DATATYPE_NULL; } - /* In IO tasks ??? */ + /* In IO tasks set up sendcounts/sendtypes for pio_swapm() call + * below. */ if (ios->ioproc) - for (i = 0; i < iodesc->nrecvs; i++) + { + for (int i = 0; i < iodesc->nrecvs; i++) + { if (iodesc->rtype[i] != PIO_DATATYPE_NULL) { if (iodesc->rearranger == PIO_REARR_SUBSET) @@ -983,18 +1029,21 @@ int rearrange_io2comp(iosystem_desc_t *ios, io_desc_t *iodesc, void *sbuf, sendtypes[iodesc->rfrom[i]] = iodesc->rtype[i]; } } + } + } /* In the box rearranger each comp task may communicate with * multiple IO tasks here we are setting the count and data type * of the communication of a given compute task with each io * task. */ - for (i = 0; i < niotasks; i++) + for (int i = 0; i < niotasks; i++) { int io_comprank = ios->ioranks[i]; + if (iodesc->rearranger == PIO_REARR_SUBSET) io_comprank = 0; - if (scount[i] > 0 && iodesc->stype[i] != PIO_DATATYPE_NULL) + if (iodesc->scount[i] > 0 && iodesc->stype[i] != PIO_DATATYPE_NULL) { recvcounts[io_comprank] = 1; recvtypes[io_comprank] = iodesc->stype[i]; @@ -1002,19 +1051,9 @@ int rearrange_io2comp(iosystem_desc_t *ios, io_desc_t *iodesc, void *sbuf, } /* Data in sbuf on the ionodes is sent to rbuf on the compute nodes */ - pio_swapm(sbuf, sendcounts, sdispls, sendtypes, - rbuf, recvcounts, rdispls, recvtypes, mycomm, - iodesc->rearr_opts.comm_fc_opts_io2comp.enable_hs, - iodesc->rearr_opts.comm_fc_opts_io2comp.enable_isend, - iodesc->rearr_opts.comm_fc_opts_io2comp.max_pend_req); - - /* Release memory. */ - free(sendcounts); - free(recvcounts); - free(sdispls); - free(rdispls); - free(sendtypes); - free(recvtypes); + if ((ret = pio_swapm(sbuf, sendcounts, sdispls, sendtypes, rbuf, recvcounts, + rdispls, recvtypes, mycomm, &iodesc->rearr_opts.io2comp))) + return pio_err(ios, NULL, ret, __FILE__, __LINE__); #ifdef TIMING GPTLstop("PIO:rearrange_io2comp"); @@ -1024,44 +1063,53 @@ int rearrange_io2comp(iosystem_desc_t *ios, io_desc_t *iodesc, void *sbuf, } /** - * Determine fill value. + * Determine whether fill values are needed. This function compares + * how much data we have to how much data is in a record (or + * non-record var). If we have enough data to completely fill the + * variable, then fill is not needed. * - * @param ios pointer to the iosystem_desc_t struct + * @param ios pointer to the iosystem_desc_t struct. * @param iodesc a pointer to the io_desc_t struct. - * @param gsize pointer to an array of sizes - * @param compmap + * @param gdimlen pointer to an array length iodesc->ndims with the + * global array sizes for one record (for record vars) or for the + * entire var (for non-record vars). + * @param compmap only used for the box communicator. * @returns 0 on success, error code otherwise. */ -int determine_fill(iosystem_desc_t *ios, io_desc_t *iodesc, const int *gsize, +int determine_fill(iosystem_desc_t *ios, io_desc_t *iodesc, const int *gdimlen, const PIO_Offset *compmap) { PIO_Offset totalllen = 0; PIO_Offset totalgridsize = 1; - int i; int mpierr; /* Return code from MPI calls. */ - assert(iodesc); + /* Check inputs. */ + pioassert(ios && iodesc && gdimlen && compmap, "invalid input", + __FILE__, __LINE__); - for (i = 0; i < iodesc->ndims; i++) - totalgridsize *= gsize[i]; + /* Determine size of data space. */ + for (int i = 0; i < iodesc->ndims; i++) + totalgridsize *= gdimlen[i]; + /* Determine how many values we have locally. */ if (iodesc->rearranger == PIO_REARR_SUBSET) - totalllen=iodesc->llen; + totalllen = iodesc->llen; else - for (i = 0; indof; i++) + for (int i = 0; i < iodesc->ndof; i++) if (compmap[i] > 0) totalllen++; + /* Add results accross communicator. */ + LOG((2, "determine_fill before allreduce totalllen = %d totalgridsize = %d", + totalllen, totalgridsize)); if ((mpierr = MPI_Allreduce(MPI_IN_PLACE, &totalllen, 1, PIO_OFFSET, MPI_SUM, ios->union_comm))) check_mpi(NULL, mpierr, __FILE__, __LINE__); + LOG((2, "after allreduce totalllen = %d", totalllen)); /* If the total size of the data provided to be written is < the * total data size then we need fill values. */ - if (totalllen < totalgridsize) - iodesc->needsfill = true; - else - iodesc->needsfill = false; + iodesc->needsfill = totalllen < totalgridsize; /* TURN OFF FILL for timing test iodesc->needsfill=false; */ @@ -1069,174 +1117,214 @@ int determine_fill(iosystem_desc_t *ios, io_desc_t *iodesc, const int *gsize, return PIO_NOERR; } -/** - * Prints the IO desc information to stdout. - * - * @param iodesc a pointer to the io_desc_t struct. - * @returns 0 on success, error code otherwise. - */ -void iodesc_dump(io_desc_t *iodesc) -{ - assert(iodesc); - - printf("ioid= %d\n", iodesc->ioid); -/* printf("async_id= %d\n",iodesc->async_id);*/ - printf("nrecvs= %d\n", iodesc->nrecvs); - printf("ndof= %d\n", iodesc->ndof); - printf("ndims= %d\n", iodesc->ndims); - printf("num_aiotasks= %d\n", iodesc->num_aiotasks); - printf("rearranger= %d\n", iodesc->rearranger); - printf("maxregions= %d\n", iodesc->maxregions); - printf("needsfill= %d\n", (int)iodesc->needsfill); - - printf("llen= %lld\n", iodesc->llen); - printf("maxiobuflen= %d\n", iodesc->maxiobuflen); - - printf("rindex= "); - for (int j = 0; j < iodesc->llen; j++) - printf(" %lld ", iodesc->rindex[j]); - printf("\n"); -} - /** * The box rearranger computes a mapping between IO tasks and compute - * tasks such that the data on io tasks can be written with a single - * call to the underlying netcdf library. This may involve an all to + * tasks such that the data on IO tasks can be written with a single + * call to the underlying netCDF library. This may involve an all to * all rearrangement in the mapping, but should minimize data movement - * in lower level libraries + * in lower level libraries. * - * @param ios pointer to the iosystem_desc_t struct - * @param maplen the length of the map - * @param compmap - * @param gsize pointer to an array of sizes - * @param ndims the number of dimensions - * @param iodesc a pointer to the io_desc_t struct. + * On each compute task the application program passes a compmap array + * of length ndof. This array describes the arrangement of data in + * memory on that compute task. + * + * These arrays are gathered and rearranged to the IO-tasks (which are + * sometimes collocated with compute tasks), each IO task contains + * data from the compmap of one or more compute tasks in the iomap + * array and the length of that array is llen. + * + * This function: + *
    + *
  • For IO tasks, determines llen. + *
  • Determine whether fill values will be needed. + *
  • Do an allgether of llen values into array iomaplen. + *
  • For each IO task, send starts/counts to all compute tasks. + *
  • Find dest_ioindex and dest_ioproc for each element in the map. + *
  • Call compute_counts(). + *
  • On IO tasks, compute the max IO buffer size. + *
  • Call compute_maxaggregate_bytes(). + *
+ * + * @param ios pointer to the iosystem_desc_t struct. + * @param maplen the length of the map. This is the number of data + * elements on the compute task. + * @param compmap a 1 based array of offsets into the global space. A + * 0 in this array indicates a value which should not be transfered. + * @param gdimlen an array length ndims with the sizes of the global + * dimensions. + * @param ndims the number of dimensions. + * @param iodesc a pointer to the io_desc_t struct, which must be + * allocated before this function is called. * @returns 0 on success, error code otherwise. */ int box_rearrange_create(iosystem_desc_t *ios, int maplen, const PIO_Offset *compmap, - const int *gsize, int ndims, io_desc_t *iodesc) + const int *gdimlen, int ndims, io_desc_t *iodesc) { - int nprocs = ios->num_comptasks; - int nioprocs = ios->num_iotasks; - PIO_Offset gstride[ndims]; - PIO_Offset start[ndims], count[ndims]; - int tsize, i, j, k; - int dest_ioproc[maplen]; - PIO_Offset dest_ioindex[maplen]; - int sndlths[nprocs]; - int sdispls[nprocs]; - int recvlths[nprocs]; - int rdispls[nprocs]; - MPI_Datatype dtypes[nprocs]; - PIO_Offset iomaplen[nioprocs]; - int mpierr; /* Return code from MPI functions. */ int ret; - assert(iodesc); - + /* Check inputs. */ + pioassert(ios && maplen >= 0 && compmap && gdimlen && ndims > 0 && iodesc, + "invalid input", __FILE__, __LINE__); + LOG((1, "box_rearrange_create maplen = %d ndims = %d ios->num_comptasks = %d " + "ios->num_iotasks = %d", maplen, ndims, ios->num_comptasks, ios->num_iotasks)); + + /* Allocate arrays needed for this function. */ + int dest_ioproc[maplen]; /* Destination IO task for each data element on compute task. */ + PIO_Offset dest_ioindex[maplen]; /* Offset into IO task array for each data element. */ + int sendcounts[ios->num_uniontasks]; /* Send counts for swapm call. */ + int sdispls[ios->num_uniontasks]; /* Send displacements for swapm. */ + int recvcounts[ios->num_uniontasks]; /* Receive counts for swapm. */ + int rdispls[ios->num_uniontasks]; /* Receive displacements for swapm. */ + MPI_Datatype dtypes[ios->num_uniontasks]; /* Array of MPI_OFFSET types for swapm. */ + PIO_Offset iomaplen[ios->num_iotasks]; /* Gets the llen of all IO tasks. */ + + /* This is the box rearranger. */ iodesc->rearranger = PIO_REARR_BOX; + /* Number of elements of data on compute node. */ iodesc->ndof = maplen; - gstride[ndims-1] = 1; - for (int i= ndims - 2; i >= 0; i--) - gstride[i] = gstride[i + 1] * gsize[i + 1]; - - if ((mpierr = MPI_Type_size(MPI_OFFSET, &tsize))) - return check_mpi(NULL, mpierr, __FILE__, __LINE__); - for (i = 0; i < maplen; i++) + /* Initialize array values. */ + for (int i = 0; i < maplen; i++) { dest_ioproc[i] = -1; dest_ioindex[i] = -1; } - for (i = 0; i < nprocs; i++) + + /* Initialize arrays used in swapm. */ + for (int i = 0; i < ios->num_uniontasks; i++) { - sndlths[i] = 0; + sendcounts[i] = 0; sdispls[i] = 0; - recvlths[i] = 0; + recvcounts[i] = 0; rdispls[i] = 0; dtypes[i] = MPI_OFFSET; } - iodesc->llen = 0; + + /* For IO tasks, determine llen, the length of the data array on + * the IO task. For computation tasks, llen will remain at 0. Also + * set up arrays for the allgather which will give every IO task a + * complete list of llens for each IO task. */ + LOG((3, "ios->ioproc = %d ios->num_uniontasks = %d", ios->ioproc, + ios->num_uniontasks)); + pioassert(iodesc->llen == 0, "error", __FILE__, __LINE__); if (ios->ioproc) { - for (i = 0; i < nprocs; i++) - sndlths[i] = 1; - - /* llen here is the number that will be read on each io task */ + /* Set up send counts for sending llen in all to all + * gather. We are sending to all tasks, IO and computation. */ + for (int i = 0; i < ios->num_comptasks; i++) + sendcounts[ios->compranks[i]] = 1; + for (int i = 0; i < ios->num_iotasks; i++) + sendcounts[ios->ioranks[i]] = 1; + + /* Determine llen, the lenght of the data array on this IO + * node, by multipliying the counts in the + * iodesc->firstregion. */ iodesc->llen = 1; - for (i = 0; i < ndims; i++) + for (int i = 0; i < ndims; i++) + { iodesc->llen *= iodesc->firstregion->count[i]; + LOG((3, "iodesc->firstregion->start[%d] = %d iodesc->firstregion->count[%d] = %d", + i, iodesc->firstregion->start[i], i, iodesc->firstregion->count[i])); + } + LOG((2, "iodesc->llen = %d", iodesc->llen)); } - if ((ret = determine_fill(ios, iodesc, gsize, compmap))) - return pio_err(ios, NULL, ret, __FILE__, __LINE__); - /* - if (ios->ioproc){ - for (i = 0; ifirstregion->start[i],iodesc->firstregion->count[i]); - } - printf("\n%s %d\n",__FILE__,__LINE__); - } - */ + /* Determine whether fill values will be needed. */ + if ((ret = determine_fill(ios, iodesc, gdimlen, compmap))) + return pio_err(ios, NULL, ret, __FILE__, __LINE__); + LOG((2, "iodesc->needsfill = %d ios->num_iotasks = %d", iodesc->needsfill, + ios->num_iotasks)); - for (i = 0; i < nioprocs; i++) + /* Set up receive counts and displacements to for an AllToAll + * gather of llen. */ + for (int i = 0; i < ios->num_iotasks; i++) { - int io_comprank = ios->ioranks[i]; - recvlths[io_comprank] = 1; - rdispls[io_comprank] = i * tsize; + recvcounts[ios->ioranks[i]] = 1; + rdispls[ios->ioranks[i]] = i * SIZEOF_MPI_OFFSET; + LOG((3, "i = %d ios->ioranks[%d] = %d recvcounts[%d] = %d rdispls[%d] = %d", + i, i, ios->ioranks[i], ios->ioranks[i], recvcounts[ios->ioranks[i]], + ios->ioranks[i], rdispls[ios->ioranks[i]])); } - /* The length of each iomap - iomaplen = calloc(nioprocs, sizeof(PIO_Offset)); */ - pio_swapm(&(iodesc->llen), sndlths, sdispls, dtypes, - iomaplen, recvlths, rdispls, dtypes, - ios->union_comm, - iodesc->rearr_opts.comm_fc_opts_io2comp.enable_hs, - iodesc->rearr_opts.comm_fc_opts_io2comp.enable_isend, - iodesc->rearr_opts.comm_fc_opts_io2comp.max_pend_req); - - for (i = 0; i < nioprocs; i++) + /* All-gather the llen to all tasks into array iomaplen. */ + LOG((3, "calling pio_swapm to allgather llen into array iomaplen, ndims = %d dtypes[0] = %d", + ndims, dtypes)); + if ((ret = pio_swapm(&iodesc->llen, sendcounts, sdispls, dtypes, iomaplen, recvcounts, + rdispls, dtypes, ios->union_comm, &iodesc->rearr_opts.io2comp))) + return pio_err(ios, NULL, ret, __FILE__, __LINE__); + LOG((3, "iodesc->llen = %d", iodesc->llen)); +#if PIO_ENABLE_LOGGING + for (int i = 0; i < ios->num_iotasks; i++) + LOG((3, "iomaplen[%d] = %d", i, iomaplen[i])); +#endif /* PIO_ENABLE_LOGGING */ + + /* For each IO task send starts/counts to all compute tasks. */ + for (int i = 0; i < ios->num_iotasks; i++) { + /* The ipmaplen contains the llen (number of data elements) + * for this IO task. */ + LOG((2, "iomaplen[%d] = %d", i, iomaplen[i])); + /* If there is data for this IO task, send start/count to all + * compute tasks. */ if (iomaplen[i] > 0) { - int io_comprank = ios->ioranks[i]; - for (j = 0; j < nprocs; j++) + PIO_Offset start[ndims]; + PIO_Offset count[ndims]; + + /* Set up send/recv parameters for all to all gather of + * counts and starts. */ + for (int j = 0; j < ios->num_uniontasks; j++) { - sndlths[j] = 0; + sendcounts[j] = 0; sdispls[j] = 0; rdispls[j] = 0; - recvlths[j] = 0; - if (ios->union_rank == io_comprank) - sndlths[j] = ndims; + recvcounts[j] = 0; + if (ios->union_rank == ios->ioranks[i]) + sendcounts[j] = ndims; } - recvlths[io_comprank] = ndims; - - /* The count from iotask i is sent to all compute tasks */ - pio_swapm(iodesc->firstregion->count, sndlths, sdispls, dtypes, - count, recvlths, rdispls, dtypes, - ios->union_comm, - iodesc->rearr_opts.comm_fc_opts_io2comp.enable_hs, - iodesc->rearr_opts.comm_fc_opts_io2comp.enable_isend, - iodesc->rearr_opts.comm_fc_opts_io2comp.max_pend_req); - - /* The start from iotask i is sent to all compute tasks. */ - pio_swapm(iodesc->firstregion->start, sndlths, sdispls, dtypes, - start, recvlths, rdispls, dtypes, - ios->union_comm, - iodesc->rearr_opts.comm_fc_opts_io2comp.enable_hs, - iodesc->rearr_opts.comm_fc_opts_io2comp.enable_isend, - iodesc->rearr_opts.comm_fc_opts_io2comp.max_pend_req); - - for (k = 0; k < maplen; k++) + recvcounts[ios->ioranks[i]] = ndims; + + /* The count array from iotask i is sent to all compute tasks. */ + LOG((3, "about to call pio_swapm with count from iotask %d ndims = %d", + i, ndims)); + if ((ret = pio_swapm(iodesc->firstregion->count, sendcounts, sdispls, dtypes, count, + recvcounts, rdispls, dtypes, ios->union_comm, + &iodesc->rearr_opts.io2comp))) + return pio_err(ios, NULL, ret, __FILE__, __LINE__); + + /* The start array from iotask i is sent to all compute tasks. */ + LOG((3, "about to call pio_swapm with start from iotask %d ndims = %d", + i, ndims)); + if ((ret = pio_swapm(iodesc->firstregion->start, sendcounts, sdispls, dtypes, + start, recvcounts, rdispls, dtypes, ios->union_comm, + &iodesc->rearr_opts.io2comp))) + return pio_err(ios, NULL, ret, __FILE__, __LINE__); + +#if PIO_ENABLE_LOGGING + for (int d = 0; d < ndims; d++) + LOG((3, "start[%d] = %lld count[%d] = %lld", d, start[d], d, count[d])); +#endif /* PIO_ENABLE_LOGGING */ + + /* For each element of the data array on the compute task, + * find the IO task to send the data element to, and its + * offset into the global data array. */ + for (int k = 0; k < maplen; k++) { PIO_Offset gcoord[ndims], lcoord[ndims]; bool found = true; - /* The compmap array is 1 based but calculations are 0 based */ - idx_to_dim_list(ndims, gsize, compmap[k] - 1, gcoord); - for (j = 0; j < ndims; j++) + /* The compmap array is 1 based but calculations are 0 based */ + LOG((3, "about to call idx_to_dim_list ndims = %d ", ndims)); + idx_to_dim_list(ndims, gdimlen, compmap[k] - 1, gcoord); +#if PIO_ENABLE_LOGGING + for (int d = 0; d < ndims; d++) + LOG((3, "gcoord[%d] = %lld", d, gcoord[d])); +#endif /* PIO_ENABLE_LOGGING */ + + /* Find a destination for each entry in the compmap. */ + for (int j = 0; j < ndims; j++) { if (gcoord[j] >= start[j] && gcoord[j] < start[j] + count[j]) { @@ -1248,44 +1336,56 @@ int box_rearrange_create(iosystem_desc_t *ios, int maplen, const PIO_Offset *com break; } } + + /* Did we find a destination IO task for this element + * of the computation task data array? If so, remember + * the destination IO task, and determine the index + * for that element in the IO task data. */ if (found) { dest_ioindex[k] = coord_to_lindex(ndims, lcoord, count); dest_ioproc[k] = i; + LOG((3, "found dest_ioindex[%d] = %d dest_ioproc[%d] = %d", k, dest_ioindex[k], + k, dest_ioproc[k])); } } } } /* Check that a destination is found for each compmap entry. */ - for (k = 0; k < maplen; k++) + for (int k = 0; k < maplen; k++) if (dest_ioproc[k] < 0 && compmap[k] > 0) return pio_err(ios, NULL, PIO_EINVAL, __FILE__, __LINE__); - compute_counts(ios, iodesc, maplen, dest_ioproc, dest_ioindex, ios->union_comm); + /* Completes the mapping for the box rearranger. */ + LOG((2, "calling compute_counts maplen = %d", maplen)); + if ((ret = compute_counts(ios, iodesc, dest_ioproc, dest_ioindex))) + return pio_err(ios, NULL, ret, __FILE__, __LINE__); + /* Compute the max io buffer size needed for an iodesc. */ if (ios->ioproc) + { if ((ret = compute_maxIObuffersize(ios->io_comm, iodesc))) return pio_err(ios, NULL, ret, __FILE__, __LINE__); + LOG((3, "iodesc->maxiobuflen = %d", iodesc->maxiobuflen)); + } - /* Using maxiobuflen compute the maximum number of vars of this type that the io - task buffer can handle. */ + /* Using maxiobuflen compute the maximum number of bytes that the + * io task buffer can handle. */ if ((ret = compute_maxaggregate_bytes(ios, iodesc))) return pio_err(ios, NULL, ret, __FILE__, __LINE__); + LOG((3, "iodesc->maxbytes = %d", iodesc->maxbytes)); -#ifdef DEBUG - iodesc_dump(iodesc); -#endif return PIO_NOERR; } /** - * Compare offsets is used by the sort in the subset rearrange. This + * Compare offsets is used by the sort in the subset rearranger. This * function is passed to qsort. * - * @param a - * @param b - * @returns 0 if offsets are the same. + * @param a pointer to an offset. + * @param b pointer to another offset. + * @returns 0 if offsets are the same or either pointer is NULL. */ int compare_offsets(const void *a, const void *b) { @@ -1297,60 +1397,75 @@ int compare_offsets(const void *a, const void *b) } /** + * Calculate start and count regions for the subset rearranger. This + * function is not used in the box rearranger. + * * Each region is a block of output which can be represented in a * single call to the underlying netcdf library. This can be as small * as a single data point, but we hope we've aggragated better than * that. * * @param ndims the number of dimensions - * @param gdims pointer to an array of dimension ids + * @param gdimlen an array length ndims with the sizes of the global + * dimensions. * @param maplen the length of the map - * @param map + * @param map may be NULL (when ???). * @param maxregions * @param firstregion pointer to the first region. * @returns 0 on success, error code otherwise. */ -void get_start_and_count_regions(int ndims, const int *gdims, int maplen, const PIO_Offset *map, - int *maxregions, io_region *firstregion) +int get_regions(int ndims, const int *gdimlen, int maplen, const PIO_Offset *map, + int *maxregions, io_region *firstregion) { - int i; - int nmaplen; + int nmaplen = 0; int regionlen; io_region *region; + int ret; - assert(maxregions); - assert(firstregion); + /* Check inputs. */ + pioassert(ndims >= 0 && gdimlen && maplen >= 0 && maxregions && firstregion, + "invalid input", __FILE__, __LINE__); + LOG((1, "get_regions ndims = %d maplen = %d", ndims, maplen)); - nmaplen = 0; region = firstregion; if (map) { - while(map[nmaplen++] <= 0); + while (map[nmaplen++] <= 0) + { + LOG((3, "map[%d] = %d", nmaplen, map[nmaplen])); + ; + } nmaplen--; } region->loffset = nmaplen; + LOG((2, "region->loffset = %d", region->loffset)); *maxregions = 1; - while(nmaplen < maplen) + while (nmaplen < maplen) { /* Here we find the largest region from the current offset - into the iomap regionlen is the size of that region and we + into the iomap. regionlen is the size of that region and we step to that point in the map array until we reach the - end */ - for (i = 0; i < ndims; i++) + end. */ + for (int i = 0; i < ndims; i++) region->count[i] = 1; - regionlen = find_region(ndims, gdims, maplen-nmaplen, - map+nmaplen, region->start, region->count); - + /* Set start/count to describe first region in map. */ + regionlen = find_region(ndims, gdimlen, maplen-nmaplen, + &map[nmaplen], region->start, region->count); pioassert(region->start[0] >= 0, "failed to find region", __FILE__, __LINE__); nmaplen = nmaplen + regionlen; + LOG((2, "regionlen = %d nmaplen = %d", regionlen, nmaplen)); + /* If we need to, allocate the next region. */ if (region->next == NULL && nmaplen < maplen) { - region->next = alloc_region(ndims); + LOG((2, "allocating next region")); + if ((ret = alloc_region2(NULL, ndims, ®ion->next))) + return ret; + /* The offset into the local array buffer is the sum of * the sizes of all of the previous regions (loffset) */ region = region->next; @@ -1361,28 +1476,39 @@ void get_start_and_count_regions(int ndims, const int *gdims, int maplen, const maxregions will be the total number of regions on this task. */ (*maxregions)++; + LOG((2, "*maxregions = %d", *maxregions)); } } + + return PIO_NOERR; } /** + * Create the MPI communicators needed by the subset rearranger. + * * The subset rearranger needs a mapping from compute tasks to IO * task, the only requirement is that each compute task map to one and * only one IO task. This mapping groups by mpi task id others are * possible and may be better for certain decompositions * - * @param ios pointer to the iosystem_desc_t struct + * The as yet unrealized vision here is that the user would be able to + * supply an alternative subset partitioning function. Requirements of + * this function are that there be exactly one io task per compute + * task group. + * + * @param ios pointer to the iosystem_desc_t struct. * @param iodesc a pointer to the io_desc_t struct. * @returns 0 on success, error code otherwise. */ int default_subset_partition(iosystem_desc_t *ios, io_desc_t *iodesc) { - int taskratio = ios->num_comptasks/ios->num_iotasks; int color; int key; int mpierr; /* Return value from MPI functions. */ - assert(ios && iodesc); + pioassert(ios && iodesc, "invalid input", __FILE__, __LINE__); + LOG((1, "default_subset_partition ios->ioproc = %d ios->io_rank = %d " + "ios->comp_rank = %d", ios->ioproc, ios->io_rank, ios->comp_rank)); /* Create a new comm for each subset group with the io task in rank 0 and only 1 io task per group */ @@ -1393,10 +1519,13 @@ int default_subset_partition(iosystem_desc_t *ios, io_desc_t *iodesc) } else { + int taskratio = ios->num_comptasks / ios->num_iotasks; key = max(1, ios->comp_rank % taskratio + 1); color = min(ios->num_iotasks - 1, ios->comp_rank / taskratio); } + LOG((3, "key = %d color = %d", key, color)); + /* Create new communicators. */ if ((mpierr = MPI_Comm_split(ios->comp_comm, color, key, &iodesc->subset_comm))) return check_mpi(NULL, mpierr, __FILE__, __LINE__); @@ -1404,34 +1533,72 @@ int default_subset_partition(iosystem_desc_t *ios, io_desc_t *iodesc) } /** + * Create the subset rearranger. + * * The subset rearranger computes a mapping between IO tasks and * compute tasks such that each compute task communicates with one and * only one IO task. * + * This function is called from PIOc_InitDecomp(). + * + * This function: + *
    + *
  • Calls default_subset_partition() to create subset_comm. + *
  • For IO tasks, allocates iodesc->rcount array (length ntasks). + *
  • Allocates iodesc->scount array (length 1) + *
  • Determins value of iodesc->scount[0], the number of data + * elements on this compute task which are read/written. + *
  • Allocated and inits iodesc->sindex (length iodesc->scount[0]), + * init it to contain indicies to data. + *
  • Pass the reduced maplen (without holes) from each compute task + * to its associated IO task. + *
  • On IO tasks, determine llen. + *
  • Determine whether fill values will be needed. + *
  • Pass iodesc->sindex from each compute task to its associated IO + * task. + *
  • Create shrtmap, which is compmap without the holes. + *
  • Gather shrtmaps from each task into iomap. + *
  • On IO tasks, sort the mapping, this will transpose the data + * into IO order. + *
  • On IO tasks, allocate and init iodesc->rindex and iodesc->rfrom + * (length iodesc->llen). + *
  • On IO tasks, handle fill values, if needed. + *
  • On IO tasks, scatter values of srcindex to subset communicator. + *
  • On IO tasks, call get_regions() and distribute the max + * maxregions to all tasks in IO communicator. + *
  • On IO tasks, call compute_maxIObuffersize(). + *
  • Call compute_maxaggregate_bytes(). + *
+ * * @param ios pointer to the iosystem_desc_t struct. * @param maplen the length of the map. - * @param compmap - * @param gsize pointer to an array of sizes. + * @param compmap a 1 based array of offsets into the array record on + * file. A 0 in this array indicates a value which should not be + * transfered. + * @param gdimlen an array length ndims with the sizes of the global + * dimensions. * @param ndims the number of dimensions. * @param iodesc a pointer to the io_desc_t struct. * @returns 0 on success, error code otherwise. */ int subset_rearrange_create(iosystem_desc_t *ios, int maplen, PIO_Offset *compmap, - const int *gsize, int ndims, io_desc_t *iodesc) + const int *gdimlen, int ndims, io_desc_t *iodesc) { int i, j; PIO_Offset *iomap = NULL; - int ierr = PIO_NOERR; mapsort *map = NULL; PIO_Offset totalgridsize; PIO_Offset *srcindex = NULL; PIO_Offset *myfillgrid = NULL; int maxregions; - int rank, ntasks, rcnt; + int rank, ntasks; + int rcnt = 0; int mpierr; /* Return call from MPI function calls. */ int ret; - pioassert(iodesc, "iodesc must be provided", __FILE__, __LINE__); + /* Check inputs. */ + pioassert(ios && maplen >= 0 && compmap && gdimlen && ndims >= 0 && iodesc, + "invalid input", __FILE__, __LINE__); LOG((2, "subset_rearrange_create maplen = %d ndims = %d", maplen, ndims)); @@ -1442,7 +1609,7 @@ int subset_rearrange_create(iosystem_desc_t *ios, int maplen, PIO_Offset *compma return pio_err(ios, NULL, ret, __FILE__, __LINE__); iodesc->rearranger = PIO_REARR_SUBSET; - /* Get size of this subset communicator, and rank in it. */ + /* Get size of this subset communicator and rank of this task in it. */ if ((mpierr = MPI_Comm_rank(iodesc->subset_comm, &rank))) return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); if ((mpierr = MPI_Comm_size(iodesc->subset_comm, &ntasks))) @@ -1455,8 +1622,9 @@ int subset_rearrange_create(iosystem_desc_t *ios, int maplen, PIO_Offset *compma pioassert(rank > 0 && rank < ntasks, "Bad comp rank in subset create", __FILE__, __LINE__); - rcnt = 0; + /* Remember the maplen for this computation task. */ iodesc->ndof = maplen; + if (ios->ioproc) { /* Allocate space to hold count of data to be received in pio_swapm(). */ @@ -1471,10 +1639,15 @@ int subset_rearrange_create(iosystem_desc_t *ios, int maplen, PIO_Offset *compma return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); iodesc->scount[0] = 0; + + /* Find the total size of the global data array. */ totalgridsize = 1; for (i = 0; i < ndims; i++) - totalgridsize *= gsize[i]; + totalgridsize *= gdimlen[i]; + /* Determine scount[0], the number of data elements in the + * computation task that are to be written, by looking at + * compmap. */ for (i = 0; i < maplen; i++) { /* turns out this can be allowed in some cases @@ -1484,6 +1657,8 @@ int subset_rearrange_create(iosystem_desc_t *ios, int maplen, PIO_Offset *compma (iodesc->scount[0])++; } + /* Allocate an array for indicies on the computation tasks (the + * send side when writing). */ if (iodesc->scount[0] > 0) if (!(iodesc->sindex = calloc(iodesc->scount[0], sizeof(PIO_Offset)))) return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); @@ -1493,24 +1668,25 @@ int subset_rearrange_create(iosystem_desc_t *ios, int maplen, PIO_Offset *compma if (compmap[i] > 0) iodesc->sindex[j++] = i; - /* Pass the reduced maplen (without holes) from each compute task to its associated IO task - printf("%s %d %ld\n",__FILE__,__LINE__,iodesc->scount); */ - if ((mpierr = MPI_Gather(iodesc->scount, 1, MPI_INT, iodesc->rcount, rcnt, MPI_INT, - 0, iodesc->subset_comm))) + /* Pass the reduced maplen (without holes) from each compute task + * to its associated IO task. */ + if ((mpierr = MPI_Gather(iodesc->scount, 1, MPI_INT, iodesc->rcount, rcnt, + MPI_INT, 0, iodesc->subset_comm))) return check_mpi(NULL, mpierr, __FILE__, __LINE__); iodesc->llen = 0; int rdispls[ntasks]; - int recvlths[ntasks]; + int recvcounts[ntasks]; + /* On IO tasks determine llen. */ if (ios->ioproc) { - for (i = 0;i < ntasks; i++) + for (i = 0; i < ntasks; i++) { iodesc->llen += iodesc->rcount[i]; rdispls[i] = 0; - recvlths[i] = iodesc->rcount[i]; + recvcounts[i] = iodesc->rcount[i]; if (i > 0) rdispls[i] = rdispls[i - 1] + iodesc->rcount[i - 1]; } @@ -1528,20 +1704,24 @@ int subset_rearrange_create(iosystem_desc_t *ios, int maplen, PIO_Offset *compma { for (i = 0; i < ntasks; i++) { - recvlths[i] = 0; + recvcounts[i] = 0; rdispls[i] = 0; } } - if ((ret = determine_fill(ios, iodesc, gsize, compmap))) + + /* Determine whether fill values will be needed. */ + if ((ret = determine_fill(ios, iodesc, gdimlen, compmap))) return pio_err(ios, NULL, ret, __FILE__, __LINE__); /* Pass the sindex from each compute task to its associated IO task. */ if ((mpierr = MPI_Gatherv(iodesc->sindex, iodesc->scount[0], PIO_OFFSET, - srcindex, recvlths, rdispls, PIO_OFFSET, 0, + srcindex, recvcounts, rdispls, PIO_OFFSET, 0, iodesc->subset_comm))) return check_mpi(NULL, mpierr, __FILE__, __LINE__); - if (ios->ioproc && iodesc->llen>0) + /* On IO tasks which need it, allocate memory for the map and the + * iomap. */ + if (ios->ioproc && iodesc->llen > 0) { if (!(map = calloc(iodesc->llen, sizeof(mapsort)))) return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); @@ -1552,13 +1732,13 @@ int subset_rearrange_create(iosystem_desc_t *ios, int maplen, PIO_Offset *compma /* Now pass the compmap, skipping the holes. */ PIO_Offset *shrtmap; - if (maplen>iodesc->scount[0] && iodesc->scount[0] > 0) + if (maplen > iodesc->scount[0] && iodesc->scount[0] > 0) { if (!(shrtmap = calloc(iodesc->scount[0], sizeof(PIO_Offset)))) return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); j = 0; - for (i = 0; i < maplen; i++) + for (int i = 0; i < maplen; i++) if (compmap[i] > 0) shrtmap[j++] = compmap[i]; } @@ -1567,13 +1747,16 @@ int subset_rearrange_create(iosystem_desc_t *ios, int maplen, PIO_Offset *compma shrtmap = compmap; } - if ((mpierr = MPI_Gatherv(shrtmap, iodesc->scount[0], PIO_OFFSET, iomap, recvlths, rdispls, - PIO_OFFSET, 0, iodesc->subset_comm))) + /* Gather shrtmap from each task in the subset communicator, and + * put gathered results into iomap. */ + if ((mpierr = MPI_Gatherv(shrtmap, iodesc->scount[0], PIO_OFFSET, iomap, recvcounts, + rdispls, PIO_OFFSET, 0, iodesc->subset_comm))) return check_mpi(NULL, mpierr, __FILE__, __LINE__); if (shrtmap != compmap) free(shrtmap); + /* On IO tasks that have data in the local array ??? */ if (ios->ioproc && iodesc->llen > 0) { int pos = 0; @@ -1583,7 +1766,7 @@ int subset_rearrange_create(iosystem_desc_t *ios, int maplen, PIO_Offset *compma { for (j = 0; j < iodesc->rcount[i]; j++) { - mptr = map + k; + mptr = &map[k]; mptr->rfrom = i; mptr->soffset = srcindex[pos + j]; mptr->iomap = iomap[pos + j]; @@ -1591,6 +1774,7 @@ int subset_rearrange_create(iosystem_desc_t *ios, int maplen, PIO_Offset *compma } pos += iodesc->rcount[i]; } + /* sort the mapping, this will transpose the data into IO order */ qsort(map, iodesc->llen, sizeof(mapsort), compare_offsets); @@ -1605,21 +1789,20 @@ int subset_rearrange_create(iosystem_desc_t *ios, int maplen, PIO_Offset *compma for (i = 0; i < ntasks; i++) { cnt[i] = rdispls[i]; - - /* offsets to swapm are in bytes */ - /* rdispls[i]*=pio_offset_size; */ } - mapsort *mptr; + /* For IO tasks init rfrom and rindex arrays (compute tasks have + * llen of 0). */ for (i = 0; i < iodesc->llen; i++) { - mptr = map+i; + mapsort *mptr = &map[i]; iodesc->rfrom[i] = mptr->rfrom; iodesc->rindex[i] = i; iomap[i] = mptr->iomap; srcindex[(cnt[iodesc->rfrom[i]])++] = mptr->soffset; } + /* Handle fill values if needed. */ if (ios->ioproc && iodesc->needsfill) { /* we need the list of offsets which are not in the union of iomap */ @@ -1657,6 +1840,7 @@ int subset_rearrange_create(iosystem_desc_t *ios, int maplen, PIO_Offset *compma } } + /* Gather cnt from all tasks in the IO communicator into array gcnt. */ if ((mpierr = MPI_Gather(&cnt, 1, MPI_INT, gcnt, 1, MPI_INT, nio, ios->io_comm))) return check_mpi(NULL, mpierr, __FILE__, __LINE__); @@ -1675,11 +1859,12 @@ int subset_rearrange_create(iosystem_desc_t *ios, int maplen, PIO_Offset *compma myusegrid[i] = -1; } - if ((mpierr = MPI_Gatherv((iomap + imin), cnt, PIO_OFFSET, myusegrid, gcnt, + if ((mpierr = MPI_Gatherv(&iomap[imin], cnt, PIO_OFFSET, myusegrid, gcnt, displs, PIO_OFFSET, nio, ios->io_comm))) return check_mpi(NULL, mpierr, __FILE__, __LINE__); } + /* Allocate and initialize a grid to fill in missing values. ??? */ PIO_Offset grid[thisgridsize[ios->io_rank]]; for (i = 0; i < thisgridsize[ios->io_rank]; i++) grid[i] = 0; @@ -1688,7 +1873,8 @@ int subset_rearrange_create(iosystem_desc_t *ios, int maplen, PIO_Offset *compma for (i = 0; i < thisgridsize[ios->io_rank]; i++) { int j = myusegrid[i] - thisgridmin[ios->io_rank]; - pioassert(j < thisgridsize[ios->io_rank], "out of bounds array index", __FILE__, __LINE__); + pioassert(j < thisgridsize[ios->io_rank], "out of bounds array index", + __FILE__, __LINE__); if (j >= 0) { grid[j] = 1; @@ -1698,7 +1884,7 @@ int subset_rearrange_create(iosystem_desc_t *ios, int maplen, PIO_Offset *compma if (myusegrid) free(myusegrid); - iodesc->holegridsize=thisgridsize[ios->io_rank] - cnt; + iodesc->holegridsize = thisgridsize[ios->io_rank] - cnt; if (iodesc->holegridsize > 0) { /* Allocate space for the fillgrid. */ @@ -1716,7 +1902,7 @@ int subset_rearrange_create(iosystem_desc_t *ios, int maplen, PIO_Offset *compma if (grid[i] == 0) { if (myfillgrid[j] == -1) - myfillgrid[j++]=thisgridmin[ios->io_rank] + i; + myfillgrid[j++] = thisgridmin[ios->io_rank] + i; else return pio_err(ios, NULL, PIO_EINVAL, __FILE__, __LINE__); } @@ -1725,34 +1911,52 @@ int subset_rearrange_create(iosystem_desc_t *ios, int maplen, PIO_Offset *compma iodesc->maxfillregions = 0; if (myfillgrid) { - iodesc->fillregion = alloc_region(iodesc->ndims); - get_start_and_count_regions(iodesc->ndims, gsize, iodesc->holegridsize, myfillgrid, - &iodesc->maxfillregions, iodesc->fillregion); + /* Allocate a data region to hold fill values. */ + if ((ret = alloc_region2(ios, iodesc->ndims, &iodesc->fillregion))) + return pio_err(ios, NULL, ret, __FILE__, __LINE__); + if ((ret = get_regions(iodesc->ndims, gdimlen, iodesc->holegridsize, myfillgrid, + &iodesc->maxfillregions, iodesc->fillregion))) + return pio_err(ios, NULL, ret, __FILE__, __LINE__); free(myfillgrid); maxregions = iodesc->maxfillregions; } - if ((mpierr = MPI_Allreduce(MPI_IN_PLACE, &maxregions, 1, MPI_INT, MPI_MAX, ios->io_comm))) - return check_mpi(NULL, mpierr, __FILE__, __LINE__); + /* Get the max maxregions, and distribute it to all tasks in + * the IO communicator. */ + if ((mpierr = MPI_Allreduce(MPI_IN_PLACE, &maxregions, 1, MPI_INT, MPI_MAX, + ios->io_comm))) + return check_mpi(NULL, mpierr, __FILE__, __LINE__); iodesc->maxfillregions = maxregions; + /* Get the max maxholegridsize, and distribute it to all tasks + * in the IO communicator. */ iodesc->maxholegridsize = iodesc->holegridsize; - if ((mpierr = MPI_Allreduce(MPI_IN_PLACE, &(iodesc->maxholegridsize), 1, MPI_INT, MPI_MAX, ios->io_comm))) + if ((mpierr = MPI_Allreduce(MPI_IN_PLACE, &(iodesc->maxholegridsize), 1, MPI_INT, + MPI_MAX, ios->io_comm))) return check_mpi(NULL, mpierr, __FILE__, __LINE__); } - if ((mpierr = MPI_Scatterv((void *)srcindex, recvlths, rdispls, PIO_OFFSET, (void *)iodesc->sindex, - iodesc->scount[0], PIO_OFFSET, 0, iodesc->subset_comm))) + /* Scatter values of srcindex to subset communicator. ??? */ + if ((mpierr = MPI_Scatterv((void *)srcindex, recvcounts, rdispls, PIO_OFFSET, + (void *)iodesc->sindex, iodesc->scount[0], PIO_OFFSET, + 0, iodesc->subset_comm))) return check_mpi(NULL, mpierr, __FILE__, __LINE__); + if (ios->ioproc) { iodesc->maxregions = 0; - get_start_and_count_regions(iodesc->ndims,gsize,iodesc->llen, iomap,&(iodesc->maxregions), - iodesc->firstregion); + if ((ret = get_regions(iodesc->ndims, gdimlen, iodesc->llen, iomap, + &iodesc->maxregions, iodesc->firstregion))) + return pio_err(ios, NULL, ret, __FILE__, __LINE__); maxregions = iodesc->maxregions; + + /* Get the max maxregions, and distribute it to all tasks in + * the IO communicator. */ if ((mpierr = MPI_Allreduce(MPI_IN_PLACE, &maxregions, 1, MPI_INT, MPI_MAX, ios->io_comm))) return check_mpi(NULL, mpierr, __FILE__, __LINE__); iodesc->maxregions = maxregions; + + /* Free resources. */ if (iomap) free(iomap); @@ -1762,12 +1966,11 @@ int subset_rearrange_create(iosystem_desc_t *ios, int maplen, PIO_Offset *compma if (srcindex) free(srcindex); - compute_maxIObuffersize(ios->io_comm, iodesc); + /* Compute the max io buffer size needed for an iodesc. */ + if ((ret = compute_maxIObuffersize(ios->io_comm, iodesc))) + return pio_err(ios, NULL, ret, __FILE__, __LINE__); iodesc->nrecvs = ntasks; -#ifdef DEBUG - iodesc_dump(iodesc); -#endif } /* Using maxiobuflen compute the maximum number of vars of this type that the io @@ -1775,7 +1978,7 @@ int subset_rearrange_create(iosystem_desc_t *ios, int maplen, PIO_Offset *compma if ((ret = compute_maxaggregate_bytes(ios, iodesc))) return pio_err(ios, NULL, ret, __FILE__, __LINE__); - return ierr; + return PIO_NOERR; } /** @@ -1803,11 +2006,11 @@ void performance_tune_rearranger(iosystem_desc_t *ios, io_desc_t *iodesc) ibuf = NULL; if (iodesc->ndof > 0) if (!(cbuf = bget(iodesc->ndof * tsize))) - piomemerror(ios, iodesc->ndof * tsize, __FILE__, __LINE__); + return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__); if (iodesc->llen > 0) if (!(ibuf = bget(iodesc->llen * tsize))) - piomemerror(ios, iodesc->llen * tsize, __FILE__, __LINE__); + return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__); if (iodesc->rearranger == PIO_REARR_BOX) mycomm = ios->union_comm; @@ -1821,7 +2024,7 @@ void performance_tune_rearranger(iosystem_desc_t *ios, io_desc_t *iodesc) int log2 = log(nprocs) / log(2) + 1; if (!(wall = bget(2 * 4 * log2 * sizeof(double)))) - piomemerror(ios, 2 * 4 *log2 * sizeof(double), __FILE__, __LINE__); + return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__); double mintime; int k = 0; @@ -1835,16 +2038,16 @@ void performance_tune_rearranger(iosystem_desc_t *ios, io_desc_t *iodesc) if ((mpierr = MPI_Allreduce(MPI_IN_PLACE, &mintime, 1, MPI_DOUBLE, MPI_MAX, mycomm))) return check_mpi(NULL, mpierr, __FILE__, __LINE__); - handshake = iodesc->rearr_opts.comm_fc_opts_comp2io.enable_hs; + handshake = iodesc->rearr_opts.comp2io.hs; isend = iodesc->isend; maxreqs = iodesc->max_requests; for (int i = 0; i < 2; i++) { if (i == 0) - iodesc->rearr_opts.comm_fc_opts_comp2io.enable_hs = false; + iodesc->rearr_opts.comp2io.hs = false; else - iodesc->rearr_opts.comm_fc_opts_comp2io.enable_hs = true; + iodesc->rearr_opts.comp2io.hs = true; for (int j = 0; j < 2; j++) { @@ -1871,7 +2074,7 @@ void performance_tune_rearranger(iosystem_desc_t *ios, io_desc_t *iodesc) if (wall[1] < mintime * 0.95) { - handshake = iodesc->rearr_opts.comm_fc_opts_comp2io.enable_hs; + handshake = iodesc->rearr_opts.comp2io.hs; isend = iodesc->isend; maxreqs = nreqs; mintime = wall[1]; @@ -1884,7 +2087,7 @@ void performance_tune_rearranger(iosystem_desc_t *ios, io_desc_t *iodesc) } } - iodesc->rearr_opts.comm_fc_opts_comp2io.enable_hs = handshake; + iodesc->rearr_opts.comp2io.hs = handshake; iodesc->isend = isend; iodesc->max_requests = maxreqs; diff --git a/src/externals/pio2/src/clib/pio_spmd.c b/src/externals/pio2/src/clib/pio_spmd.c index decf3f7e3bb..c42fd60eff9 100644 --- a/src/externals/pio2/src/clib/pio_spmd.c +++ b/src/externals/pio2/src/clib/pio_spmd.c @@ -67,19 +67,12 @@ int pair(int np, int p, int k) * @param recvtypes array of datatypes (of length ntasks). Entry i * specifies the type of data received from process i. * @param comm MPI communicator for the MPI_Alltoallw call. - * @param handshake if true, use handshaking. - * @param isend the isend bool indicates whether sends should be - * posted using mpi_irsend which can be faster than blocking - * sends. When flow control is used max_requests > 0 and the number of - * irecvs posted from a given task will not exceed this value. On some - * networks too many outstanding irecvs will cause a communications - * bottleneck. - * @param max_requests If 0, no flow control is used. + * @param fc pointer to the struct that provided flow control options. * @returns 0 for success, error code otherwise. */ int pio_swapm(void *sendbuf, int *sendcounts, int *sdispls, MPI_Datatype *sendtypes, void *recvbuf, int *recvcounts, int *rdispls, MPI_Datatype *recvtypes, - MPI_Comm comm, bool handshake, bool isend, int max_requests) + MPI_Comm comm, rearr_comm_fc_opt_t *fc) { int ntasks; /* Number of tasks in communicator comm. */ int my_rank; /* Rank of this task in comm. */ @@ -96,8 +89,8 @@ int pio_swapm(void *sendbuf, int *sendcounts, int *sdispls, MPI_Datatype *sendty MPI_Status status; /* Not actually used - replace with MPI_STATUSES_IGNORE. */ int mpierr; /* Return code from MPI functions. */ - LOG((2, "pio_swapm handshake = %d isend = %d max_requests = %d", handshake, - isend, max_requests)); + LOG((2, "pio_swapm fc->hs = %d fc->isend = %d fc->max_pend_req = %d", fc->hs, + fc->isend, fc->max_pend_req)); /* Get my rank and size of communicator. */ if ((mpierr = MPI_Comm_size(comm, &ntasks))) @@ -117,23 +110,15 @@ int pio_swapm(void *sendbuf, int *sendcounts, int *sdispls, MPI_Datatype *sendty #if PIO_ENABLE_LOGGING { for (int p = 0; p < ntasks; p++) - LOG((3, "sendcounts[%d] = %d", p, sendcounts[p])); - for (int p = 0; p < ntasks; p++) - LOG((3, "sdispls[%d] = %d", p, sdispls[p])); - for (int p = 0; p < ntasks; p++) - LOG((3, "sendtypes[%d] = %d", p, sendtypes[p])); - for (int p = 0; p < ntasks; p++) - LOG((3, "recvcounts[%d] = %d", p, recvcounts[p])); - for (int p = 0; p < ntasks; p++) - LOG((3, "rdispls[%d] = %d", p, rdispls[p])); - for (int p = 0; p < ntasks; p++) - LOG((3, "recvtypes[%d] = %d", p, recvtypes[p])); + LOG((3, "sendcounts[%d] = %d sdispls[%d] = %d sendtypes[%d] = %d recvcounts[%d] = %d " + "rdispls[%d] = %d recvtypes[%d] = %d", p, sendcounts[p], p, sdispls[p], p, + sendtypes[p], p, recvcounts[p], p, rdispls[p], p, recvtypes[p])); } #endif /* PIO_ENABLE_LOGGING */ - /* If max_requests == 0 no throttling is requested and the default + /* If fc->max_pend_req == 0 no throttling is requested and the default * mpi_alltoallw function is used. */ - if (max_requests == 0) + if (fc->max_pend_req == 0) { /* Call the MPI alltoall without flow control. */ LOG((3, "Calling MPI_Alltoallw without flow control.")); @@ -194,11 +179,11 @@ int pio_swapm(void *sendbuf, int *sendcounts, int *sdispls, MPI_Datatype *sendty swapids[i] = 0; } - if (isend) + if (fc->isend) for (int i = 0; i < ntasks; i++) sndids[i] = MPI_REQUEST_NULL; - if (handshake) + if (fc->hs) for (int i = 0; i < ntasks; i++) hs_rcvids[i] = MPI_REQUEST_NULL; @@ -222,17 +207,17 @@ int pio_swapm(void *sendbuf, int *sendcounts, int *sdispls, MPI_Datatype *sendty } else { - if (max_requests == PIO_REARR_COMM_UNLIMITED_PEND_REQ) + if (fc->max_pend_req == PIO_REARR_COMM_UNLIMITED_PEND_REQ) { maxreq = steps; maxreqh = steps; } - else if (max_requests > 1 && max_requests < steps) + else if (fc->max_pend_req > 1 && fc->max_pend_req < steps) { - maxreq = max_requests; + maxreq = fc->max_pend_req; maxreqh = maxreq / 2; } - else if (max_requests == 1) + else if (fc->max_pend_req == 1) { /* Note that steps >= 2 here */ maxreq = 2; @@ -245,11 +230,11 @@ int pio_swapm(void *sendbuf, int *sendcounts, int *sdispls, MPI_Datatype *sendty } } - LOG((2, "max_requests=%d, maxreq=%d, maxreqh=%d", max_requests, maxreq, maxreqh)); + LOG((2, "fc->max_pend_req=%d, maxreq=%d, maxreqh=%d", fc->max_pend_req, maxreq, maxreqh)); /* If handshaking is in use, do a nonblocking recieve to listen * for it. */ - if (handshake) + if (fc->hs) { for (istep = 0; istep < maxreq; istep++) { @@ -276,7 +261,7 @@ int pio_swapm(void *sendbuf, int *sendcounts, int *sdispls, MPI_Datatype *sendty rcvids + istep))) return check_mpi(NULL, mpierr, __FILE__, __LINE__); - if (handshake) + if (fc->hs) if ((mpierr = MPI_Send(&hs, 1, MPI_INT, p, tag, comm))) return check_mpi(NULL, mpierr, __FILE__, __LINE__); } @@ -292,7 +277,7 @@ int pio_swapm(void *sendbuf, int *sendcounts, int *sdispls, MPI_Datatype *sendty tag = my_rank + offset_t; /* If handshake is enabled don't post sends until the * receiving task has posted recvs. */ - if (handshake) + if (fc->hs) { if ((mpierr = MPI_Wait(hs_rcvids + istep, &status))) return check_mpi(NULL, mpierr, __FILE__, __LINE__); @@ -308,7 +293,7 @@ int pio_swapm(void *sendbuf, int *sendcounts, int *sdispls, MPI_Datatype *sendty * used to choose between mpi_irsends and mpi_isends - the default * is still mpi_irsend */ - if (handshake && isend) + if (fc->hs && fc->isend) { #ifdef USE_MPI_ISEND_FOR_FC if ((mpierr = MPI_Isend(ptr, sendcounts[p], sendtypes[p], p, tag, comm, @@ -320,7 +305,7 @@ int pio_swapm(void *sendbuf, int *sendcounts, int *sdispls, MPI_Datatype *sendty return check_mpi(NULL, mpierr, __FILE__, __LINE__); #endif } - else if (isend) + else if (fc->isend) { if ((mpierr = MPI_Isend(ptr, sendcounts[p], sendtypes[p], p, tag, comm, sndids + istep))) @@ -347,7 +332,7 @@ int pio_swapm(void *sendbuf, int *sendcounts, int *sdispls, MPI_Datatype *sendty if (rstep < steps) { p = swapids[rstep]; - if (handshake && sendcounts[p] > 0) + if (fc->hs && sendcounts[p] > 0) { tag = my_rank + offset_t; if ((mpierr = MPI_Irecv(&hs, 1, MPI_INT, p, tag, comm, hs_rcvids+rstep))) @@ -360,7 +345,7 @@ int pio_swapm(void *sendbuf, int *sendcounts, int *sdispls, MPI_Datatype *sendty ptr = (char *)recvbuf + rdispls[p]; if ((mpierr = MPI_Irecv(ptr, recvcounts[p], recvtypes[p], p, tag, comm, rcvids + rstep))) return check_mpi(NULL, mpierr, __FILE__, __LINE__); - if (handshake) + if (fc->hs) if ((mpierr = MPI_Send(&hs, 1, MPI_INT, p, tag, comm))) return check_mpi(NULL, mpierr, __FILE__, __LINE__); } @@ -376,7 +361,7 @@ int pio_swapm(void *sendbuf, int *sendcounts, int *sdispls, MPI_Datatype *sendty LOG((2, "Waiting for outstanding msgs")); if ((mpierr = MPI_Waitall(steps, rcvids, MPI_STATUSES_IGNORE))) return check_mpi(NULL, mpierr, __FILE__, __LINE__); - if (isend) + if (fc->isend) if ((mpierr = MPI_Waitall(steps, sndids, MPI_STATUSES_IGNORE))) return check_mpi(NULL, mpierr, __FILE__, __LINE__); } diff --git a/src/externals/pio2/src/clib/pio_varm.c b/src/externals/pio2/src/clib/pio_varm.c index 18d55c5aea8..f02cb196c91 100644 --- a/src/externals/pio2/src/clib/pio_varm.c +++ b/src/externals/pio2/src/clib/pio_varm.c @@ -29,7 +29,7 @@ int PIOc_put_varm (int ncid, int varid, const PIO_Offset start[], const PIO_Offs ios = file->iosystem; /* Sorry, but varm functions are not supported by the async interface. */ - if (ios->async_interface) + if (ios->async) return PIO_EINVAL; if (ios->ioproc){ @@ -98,7 +98,7 @@ int PIOc_put_varm_uchar (int ncid, int varid, const PIO_Offset start[], const PI ios = file->iosystem; /* Sorry, but varm functions are not supported by the async interface. */ - if (ios->async_interface) + if (ios->async) return PIO_EINVAL; if (ios->ioproc){ @@ -167,7 +167,7 @@ int PIOc_put_varm_short (int ncid, int varid, const PIO_Offset start[], const PI ios = file->iosystem; /* Sorry, but varm functions are not supported by the async interface. */ - if (ios->async_interface) + if (ios->async) return PIO_EINVAL; if (ios->ioproc){ @@ -235,7 +235,7 @@ int PIOc_put_varm_text (int ncid, int varid, const PIO_Offset start[], const PIO ios = file->iosystem; /* Sorry, but varm functions are not supported by the async interface. */ - if (ios->async_interface) + if (ios->async) return PIO_EINVAL; @@ -306,7 +306,7 @@ int PIOc_put_varm_ushort (int ncid, int varid, const PIO_Offset start[], const P ios = file->iosystem; /* Sorry, but varm functions are not supported by the async interface. */ - if (ios->async_interface) + if (ios->async) return PIO_EINVAL; @@ -377,7 +377,7 @@ int PIOc_put_varm_ulonglong (int ncid, int varid, const PIO_Offset start[], cons ios = file->iosystem; /* Sorry, but varm functions are not supported by the async interface. */ - if (ios->async_interface) + if (ios->async) return PIO_EINVAL; @@ -447,7 +447,7 @@ int PIOc_put_varm_int (int ncid, int varid, const PIO_Offset start[], const PIO_ ios = file->iosystem; /* Sorry, but varm functions are not supported by the async interface. */ - if (ios->async_interface) + if (ios->async) return PIO_EINVAL; @@ -518,7 +518,7 @@ int PIOc_put_varm_float (int ncid, int varid, const PIO_Offset start[], const PI ios = file->iosystem; /* Sorry, but varm functions are not supported by the async interface. */ - if (ios->async_interface) + if (ios->async) return PIO_EINVAL; @@ -588,7 +588,7 @@ int PIOc_put_varm_long (int ncid, int varid, const PIO_Offset start[], const PIO ios = file->iosystem; /* Sorry, but varm functions are not supported by the async interface. */ - if (ios->async_interface) + if (ios->async) return PIO_EINVAL; @@ -660,7 +660,7 @@ int PIOc_put_varm_uint (int ncid, int varid, const PIO_Offset start[], const PIO /* Sorry, but varm functions are not supported by the async interface. */ - if (ios->async_interface) + if (ios->async) return PIO_EINVAL; @@ -731,7 +731,7 @@ int PIOc_put_varm_double (int ncid, int varid, const PIO_Offset start[], const P ios = file->iosystem; /* Sorry, but varm functions are not supported by the async interface. */ - if (ios->async_interface) + if (ios->async) return PIO_EINVAL; @@ -801,7 +801,7 @@ int PIOc_put_varm_schar (int ncid, int varid, const PIO_Offset start[], const PI ios = file->iosystem; /* Sorry, but varm functions are not supported by the async interface. */ - if (ios->async_interface) + if (ios->async) return PIO_EINVAL; @@ -872,7 +872,7 @@ int PIOc_put_varm_longlong (int ncid, int varid, const PIO_Offset start[], const ios = file->iosystem; /* Sorry, but varm functions are not supported by the async interface. */ - if (ios->async_interface) + if (ios->async) return PIO_EINVAL; if (ios->ioproc){ @@ -941,7 +941,7 @@ int PIOc_get_varm_uchar (int ncid, int varid, const PIO_Offset start[], const PI ierr = PIO_NOERR; /* Sorry, but varm functions are not supported by the async interface. */ - if (ios->async_interface) + if (ios->async) return PIO_EINVAL; if (ios->ioproc){ @@ -979,7 +979,7 @@ int PIOc_get_varm_uchar (int ncid, int varid, const PIO_Offset start[], const PI ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - if (ios->async_interface || bcast || + if (ios->async || bcast || (ios->num_iotasks < ios->num_comptasks)){ MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); } @@ -1010,7 +1010,7 @@ int PIOc_get_varm_schar (int ncid, int varid, const PIO_Offset start[], const PI ierr = PIO_NOERR; /* Sorry, but varm functions are not supported by the async interface. */ - if (ios->async_interface) + if (ios->async) return PIO_EINVAL; if (ios->ioproc){ @@ -1048,7 +1048,7 @@ int PIOc_get_varm_schar (int ncid, int varid, const PIO_Offset start[], const PI ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - if (ios->async_interface || bcast || + if (ios->async || bcast || (ios->num_iotasks < ios->num_comptasks)){ MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); } @@ -1079,7 +1079,7 @@ int PIOc_get_varm_double (int ncid, int varid, const PIO_Offset start[], const P ierr = PIO_NOERR; /* Sorry, but varm functions are not supported by the async interface. */ - if (ios->async_interface) + if (ios->async) return PIO_EINVAL; if (ios->ioproc){ @@ -1117,7 +1117,7 @@ int PIOc_get_varm_double (int ncid, int varid, const PIO_Offset start[], const P ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - if (ios->async_interface || bcast || + if (ios->async || bcast || (ios->num_iotasks < ios->num_comptasks)){ MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); } @@ -1148,7 +1148,7 @@ int PIOc_get_varm_text (int ncid, int varid, const PIO_Offset start[], const PIO ierr = PIO_NOERR; /* Sorry, but varm functions are not supported by the async interface. */ - if (ios->async_interface) + if (ios->async) return PIO_EINVAL; if (ios->ioproc){ @@ -1186,7 +1186,7 @@ int PIOc_get_varm_text (int ncid, int varid, const PIO_Offset start[], const PIO ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - if (ios->async_interface || bcast || + if (ios->async || bcast || (ios->num_iotasks < ios->num_comptasks)){ MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); } @@ -1217,7 +1217,7 @@ int PIOc_get_varm_int (int ncid, int varid, const PIO_Offset start[], const PIO_ ierr = PIO_NOERR; /* Sorry, but varm functions are not supported by the async interface. */ - if (ios->async_interface) + if (ios->async) return PIO_EINVAL; if (ios->ioproc){ @@ -1255,7 +1255,7 @@ int PIOc_get_varm_int (int ncid, int varid, const PIO_Offset start[], const PIO_ ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - if (ios->async_interface || bcast || + if (ios->async || bcast || (ios->num_iotasks < ios->num_comptasks)){ MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); } @@ -1286,7 +1286,7 @@ int PIOc_get_varm_uint (int ncid, int varid, const PIO_Offset start[], const PIO ierr = PIO_NOERR; /* Sorry, but varm functions are not supported by the async interface. */ - if (ios->async_interface) + if (ios->async) return PIO_EINVAL; if (ios->ioproc){ @@ -1324,7 +1324,7 @@ int PIOc_get_varm_uint (int ncid, int varid, const PIO_Offset start[], const PIO ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - if (ios->async_interface || bcast || + if (ios->async || bcast || (ios->num_iotasks < ios->num_comptasks)){ MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); } @@ -1350,7 +1350,7 @@ int PIOc_get_varm (int ncid, int varid, const PIO_Offset start[], const PIO_Offs ierr = PIO_NOERR; /* Sorry, but varm functions are not supported by the async interface. */ - if (ios->async_interface) + if (ios->async) return PIO_EINVAL; if (ios->ioproc){ @@ -1388,7 +1388,7 @@ int PIOc_get_varm (int ncid, int varid, const PIO_Offset start[], const PIO_Offs ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - if (ios->async_interface || bcast || + if (ios->async || bcast || (ios->num_iotasks < ios->num_comptasks)){ MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); } @@ -1419,7 +1419,7 @@ int PIOc_get_varm_float (int ncid, int varid, const PIO_Offset start[], const PI ierr = PIO_NOERR; /* Sorry, but varm functions are not supported by the async interface. */ - if (ios->async_interface) + if (ios->async) return PIO_EINVAL; if (ios->ioproc){ @@ -1457,7 +1457,7 @@ int PIOc_get_varm_float (int ncid, int varid, const PIO_Offset start[], const PI ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - if (ios->async_interface || bcast || + if (ios->async || bcast || (ios->num_iotasks < ios->num_comptasks)){ MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); } @@ -1488,7 +1488,7 @@ int PIOc_get_varm_long (int ncid, int varid, const PIO_Offset start[], const PIO ierr = PIO_NOERR; /* Sorry, but varm functions are not supported by the async interface. */ - if (ios->async_interface) + if (ios->async) return PIO_EINVAL; if (ios->ioproc){ @@ -1526,7 +1526,7 @@ int PIOc_get_varm_long (int ncid, int varid, const PIO_Offset start[], const PIO ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - if (ios->async_interface || bcast || + if (ios->async || bcast || (ios->num_iotasks < ios->num_comptasks)){ MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); } @@ -1557,7 +1557,7 @@ int PIOc_get_varm_ushort (int ncid, int varid, const PIO_Offset start[], const P ierr = PIO_NOERR; /* Sorry, but varm functions are not supported by the async interface. */ - if (ios->async_interface) + if (ios->async) return PIO_EINVAL; if (ios->ioproc){ @@ -1595,7 +1595,7 @@ int PIOc_get_varm_ushort (int ncid, int varid, const PIO_Offset start[], const P ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - if (ios->async_interface || bcast || + if (ios->async || bcast || (ios->num_iotasks < ios->num_comptasks)){ MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); } @@ -1626,7 +1626,7 @@ int PIOc_get_varm_longlong (int ncid, int varid, const PIO_Offset start[], const ierr = PIO_NOERR; /* Sorry, but varm functions are not supported by the async interface. */ - if (ios->async_interface) + if (ios->async) return PIO_EINVAL; if (ios->ioproc){ @@ -1664,7 +1664,7 @@ int PIOc_get_varm_longlong (int ncid, int varid, const PIO_Offset start[], const ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - if (ios->async_interface || bcast || + if (ios->async || bcast || (ios->num_iotasks < ios->num_comptasks)){ MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); } @@ -1695,7 +1695,7 @@ int PIOc_get_varm_short (int ncid, int varid, const PIO_Offset start[], const PI ierr = PIO_NOERR; /* Sorry, but varm functions are not supported by the async interface. */ - if (ios->async_interface) + if (ios->async) return PIO_EINVAL; if (ios->ioproc){ @@ -1733,7 +1733,7 @@ int PIOc_get_varm_short (int ncid, int varid, const PIO_Offset start[], const PI ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - if (ios->async_interface || bcast || + if (ios->async || bcast || (ios->num_iotasks < ios->num_comptasks)){ MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); } @@ -1764,7 +1764,7 @@ int PIOc_get_varm_ulonglong (int ncid, int varid, const PIO_Offset start[], cons ierr = PIO_NOERR; /* Sorry, but varm functions are not supported by the async interface. */ - if (ios->async_interface) + if (ios->async) return PIO_EINVAL; if (ios->ioproc){ @@ -1802,7 +1802,7 @@ int PIOc_get_varm_ulonglong (int ncid, int varid, const PIO_Offset start[], cons ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - if (ios->async_interface || bcast || + if (ios->async || bcast || (ios->num_iotasks < ios->num_comptasks)){ MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); } diff --git a/src/externals/pio2/src/clib/pioc.c b/src/externals/pio2/src/clib/pioc.c index 9612f65c8ee..0dc4fed89fd 100644 --- a/src/externals/pio2/src/clib/pioc.c +++ b/src/externals/pio2/src/clib/pioc.c @@ -208,7 +208,7 @@ int PIOc_Set_IOSystem_Error_Handling(int iosysid, int method) /* Get the iosystem info. */ if (iosysid != PIO_DEFAULT) if (!(ios = pio_get_iosystem_from_id(iosysid))) - return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__); + piodie("Could not find IO system.", __FILE__, __LINE__); /* Set the error handler. */ if (PIOc_set_iosystem_error_handling(iosysid, method, &oldmethod)) @@ -249,7 +249,7 @@ int PIOc_set_iosystem_error_handling(int iosysid, int method, int *old_method) /* If using async, and not an IO task, then send parameters. */ if (iosysid != PIO_DEFAULT) - if (ios->async_interface) + if (ios->async) { if (!ios->ioproc) { @@ -290,11 +290,25 @@ int PIOc_set_iosystem_error_handling(int iosysid, int method, int *old_method) * decomposition describes how the data will be distributed between * tasks. * + * Internally, this function will: + *
    + *
  • Allocate and initialize an iodesc struct for this + * decomposition. (This also allocates an io_region struct for the + * first region.) + *
  • (Box rearranger only) If iostart or iocount are NULL, call + * CalcStartandCount() to determine starts/counts. Then call + * compute_maxIObuffersize() to compute the max IO buffer size needed. + *
  • Create the rearranger. + *
  • Assign an ioid and add this decomposition to the list of open + * decompositions. + *
+ * * @param iosysid the IO system ID. - * @param basetype the basic PIO data type used. + * @param pio_type the basic PIO data type used. * @param ndims the number of dimensions in the variable, not * including the unlimited dimension. - * @param dims an array of global size of each dimension. + * @param gdimlen an array length ndims with the sizes of the global + * dimensions. * @param maplen the local length of the compmap array. * @param compmap a 1 based array of offsets into the array record on * file. A 0 in this array indicates a value which should not be @@ -303,13 +317,17 @@ int PIOc_set_iosystem_error_handling(int iosysid, int method, int *old_method) * @param rearranger pointer to the rearranger to be used for this * decomp or NULL to use the default. * @param iostart An array of start values for block cyclic - * decompositions. If NULL ??? + * decompositions for the SUBSET rearranger. Ignored if block + * rearranger is used. If NULL and SUBSET rearranger is used, the + * iostarts are generated. * @param iocount An array of count values for block cyclic - * decompositions. If NULL ??? + * decompositions for the SUBSET rearranger. Ignored if block + * rearranger is used. If NULL and SUBSET rearranger is used, the + * iostarts are generated. * @returns 0 on success, error code otherwise * @ingroup PIO_initdecomp */ -int PIOc_InitDecomp(int iosysid, int basetype, int ndims, const int *dims, int maplen, +int PIOc_InitDecomp(int iosysid, int pio_type, int ndims, const int *gdimlen, int maplen, const PIO_Offset *compmap, int *ioidp, const int *rearranger, const PIO_Offset *iostart, const PIO_Offset *iocount) { @@ -318,24 +336,24 @@ int PIOc_InitDecomp(int iosysid, int basetype, int ndims, const int *dims, int m int mpierr = MPI_SUCCESS, mpierr2; /* Return code from MPI function calls. */ int ierr; /* Return code. */ - LOG((1, "PIOc_InitDecomp iosysid = %d basetype = %d ndims = %d maplen = %d", - iosysid, basetype, ndims, maplen)); + LOG((1, "PIOc_InitDecomp iosysid = %d pio_type = %d ndims = %d maplen = %d", + iosysid, pio_type, ndims, maplen)); /* Get IO system info. */ if (!(ios = pio_get_iosystem_from_id(iosysid))) return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__); /* Caller must provide these. */ - if (!dims || !compmap || !ioidp) + if (!gdimlen || !compmap || !ioidp) return pio_err(ios, NULL, PIO_EINVAL, __FILE__, __LINE__); /* Check the dim lengths. */ for (int i = 0; i < ndims; i++) - if (dims[i] <= 0) + if (gdimlen[i] <= 0) return pio_err(ios, NULL, PIO_EINVAL, __FILE__, __LINE__); /* If async is in use, and this is not an IO task, bcast the parameters. */ - if (ios->async_interface) + if (ios->async) { if (!ios->ioproc) { @@ -350,11 +368,11 @@ int PIOc_InitDecomp(int iosysid, int basetype, int ndims, const int *dims, int m if (!mpierr) mpierr = MPI_Bcast(&iosysid, 1, MPI_INT, ios->compmaster, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&basetype, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&pio_type, 1, MPI_INT, ios->compmaster, ios->intercomm); if (!mpierr) mpierr = MPI_Bcast(&ndims, 1, MPI_INT, ios->compmaster, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast((int *)dims, ndims, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast((int *)gdimlen, ndims, MPI_INT, ios->compmaster, ios->intercomm); if (!mpierr) mpierr = MPI_Bcast(&maplen, 1, MPI_INT, ios->compmaster, ios->intercomm); if (!mpierr) @@ -374,8 +392,8 @@ int PIOc_InitDecomp(int iosysid, int basetype, int ndims, const int *dims, int m mpierr = MPI_Bcast(&iocount_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); if (iocount_present && !mpierr) mpierr = MPI_Bcast((PIO_Offset *)iocount, ndims, MPI_OFFSET, ios->compmaster, ios->intercomm); - LOG((2, "PIOc_InitDecomp iosysid = %d basetype = %d ndims = %d maplen = %d rearranger_present = %d iostart_present = %d " - "iocount_present = %d ", iosysid, basetype, ndims, maplen, rearranger_present, iostart_present, iocount_present)); + LOG((2, "PIOc_InitDecomp iosysid = %d pio_type = %d ndims = %d maplen = %d rearranger_present = %d iostart_present = %d " + "iocount_present = %d ", iosysid, pio_type, ndims, maplen, rearranger_present, iostart_present, iocount_present)); } /* Handle MPI errors. */ @@ -385,8 +403,10 @@ int PIOc_InitDecomp(int iosysid, int basetype, int ndims, const int *dims, int m return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); } - /* Allocate space for the iodesc info. */ - if ((ierr = malloc_iodesc(ios, basetype, ndims, &iodesc))) + /* Allocate space for the iodesc info. This also allocates the + * first region and copies the rearranger opts into this + * iodesc. */ + if ((ierr = malloc_iodesc(ios, pio_type, ndims, &iodesc))) return pio_err(ios, NULL, ierr, __FILE__, __LINE__); /* Remember the maplen. */ @@ -402,7 +422,7 @@ int PIOc_InitDecomp(int iosysid, int basetype, int ndims, const int *dims, int m if (!(iodesc->dimlen = malloc(sizeof(int) * ndims))) return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); for (int d = 0; d < ndims; d++) - iodesc->dimlen[d] = dims[d]; + iodesc->dimlen[d] = gdimlen[d]; /* Set the rearranger. */ if (!rearranger) @@ -414,71 +434,76 @@ int PIOc_InitDecomp(int iosysid, int basetype, int ndims, const int *dims, int m /* Is this the subset rearranger? */ if (iodesc->rearranger == PIO_REARR_SUBSET) { - LOG((2, "Handling subset rearranger.")); - if (iostart && iocount) - fprintf(stderr,"%s %s\n","Iostart and iocount arguments to PIOc_InitDecomp", - "are incompatable with subset rearrange method and will be ignored"); iodesc->num_aiotasks = ios->num_iotasks; - if ((ierr = subset_rearrange_create(ios, maplen, (PIO_Offset *)compmap, dims, + LOG((2, "creating subset rearranger iodesc->num_aiotasks = %d", + iodesc->num_aiotasks)); + if ((ierr = subset_rearrange_create(ios, maplen, (PIO_Offset *)compmap, gdimlen, ndims, iodesc))) - return pio_err(NULL, NULL, ierr, __FILE__, __LINE__); + return pio_err(ios, NULL, ierr, __FILE__, __LINE__); } - else + else /* box rearranger */ { - LOG((2, "Handling not the subset rearranger.")); if (ios->ioproc) { /* Unless the user specifies the start and count for each * IO task compute it. */ if (iostart && iocount) { - iodesc->maxiobuflen = 1; + LOG((3, "iostart and iocount provided")); for (int i = 0; i < ndims; i++) { iodesc->firstregion->start[i] = iostart[i]; iodesc->firstregion->count[i] = iocount[i]; - compute_maxIObuffersize(ios->io_comm, iodesc); - } iodesc->num_aiotasks = ios->num_iotasks; } else { - iodesc->num_aiotasks = CalcStartandCount(basetype, ndims, dims, - ios->num_iotasks, ios->io_rank, - iodesc->firstregion->start, iodesc->firstregion->count); + /* Compute start and count values for each io task. */ + LOG((2, "about to call CalcStartandCount pio_type = %d ndims = %d", pio_type, ndims)); + if ((ierr = CalcStartandCount(pio_type, ndims, gdimlen, ios->num_iotasks, + ios->io_rank, iodesc->firstregion->start, + iodesc->firstregion->count, &iodesc->num_aiotasks))) + return pio_err(ios, NULL, ierr, __FILE__, __LINE__); } - compute_maxIObuffersize(ios->io_comm, iodesc); + + /* Compute the max io buffer size needed for an iodesc. */ + if ((ierr = compute_maxIObuffersize(ios->io_comm, iodesc))) + return pio_err(ios, NULL, ierr, __FILE__, __LINE__); + LOG((3, "compute_maxIObuffersize called iodesc->maxiobuflen = %d", + iodesc->maxiobuflen)); } /* Depending on array size and io-blocksize the actual number * of io tasks used may vary. */ - if ((mpierr = MPI_Bcast(&(iodesc->num_aiotasks), 1, MPI_INT, ios->ioroot, ios->my_comm))) + if ((mpierr = MPI_Bcast(&(iodesc->num_aiotasks), 1, MPI_INT, ios->ioroot, + ios->my_comm))) return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); LOG((3, "iodesc->num_aiotasks = %d", iodesc->num_aiotasks)); /* Compute the communications pattern for this decomposition. */ if (iodesc->rearranger == PIO_REARR_BOX) - ierr = box_rearrange_create(ios, maplen, compmap, dims, ndims, iodesc); - - /* - if (ios->ioproc){ - io_region *ioregion = iodesc->firstregion; - while(ioregion != NULL){ - for (int i=0;istart[i],ioregion->count[i]); - ioregion = ioregion->next; - } - } - */ + if ((ierr = box_rearrange_create(ios, maplen, compmap, gdimlen, ndims, iodesc))) + return pio_err(ios, NULL, ierr, __FILE__, __LINE__); } /* Add this IO description to the list. */ *ioidp = pio_add_to_iodesc_list(iodesc); - LOG((3, "About to tune rearranger...")); +#if PIO_ENABLE_LOGGING + /* Log results. */ + LOG((2, "iodesc ioid = %d nrecvs = %d ndof = %d ndims = %d num_aiotasks = %d " + "rearranger = %d maxregions = %d needsfill = %d llen = %d maxiobuflen = %d", + iodesc->ioid, iodesc->nrecvs, iodesc->ndof, iodesc->ndims, iodesc->num_aiotasks, + iodesc->rearranger, iodesc->maxregions, iodesc->needsfill, iodesc->llen, + iodesc->maxiobuflen)); + for (int j = 0; j < iodesc->llen; j++) + LOG((3, "rindex[%d] = %lld", j, iodesc->rindex[j])); +#endif /* PIO_ENABLE_LOGGING */ + + /* This function only does something if pre-processor macro + * PERFTUNE is set. */ performance_tune_rearranger(ios, iodesc); - LOG((3, "Done with rearranger tune.")); return PIO_NOERR; } @@ -489,17 +514,19 @@ int PIOc_InitDecomp(int iosysid, int basetype, int ndims, const int *dims, int m * tasks. * * @param iosysid the IO system ID. - * @param basetype the basic PIO data type used. + * @param pio_type the basic PIO data type used. * @param ndims the number of dimensions in the variable, not * including the unlimited dimension. - * @param dims an array of global size of each dimension. + * @param gdimlen an array length ndims with the sizes of the global + * dimensions. * @param maplen the local length of the compmap array. * @param compmap a 0 based array of offsets into the array record on * file. A -1 in this array indicates a value which should not be * transfered. * @param ioidp pointer that will get the io description ID. - * @param rearranger pointer to the rearranger to be used for this - * decomp or NULL to use the default. + * @param rearranger the rearranger to be used for this decomp or 0 to + * use the default. Valid rearrangers are PIO_REARR_BOX and + * PIO_REARR_SUBSET. * @param iostart An array of start values for block cyclic * decompositions. If NULL ??? * @param iocount An array of count values for block cyclic @@ -507,21 +534,30 @@ int PIOc_InitDecomp(int iosysid, int basetype, int ndims, const int *dims, int m * @returns 0 on success, error code otherwise * @ingroup PIO_initdecomp */ -int PIOc_init_decomp(int iosysid, int basetype, int ndims, const int *dims, int maplen, - const PIO_Offset *compmap, int *ioidp, const int *rearranger, +int PIOc_init_decomp(int iosysid, int pio_type, int ndims, const int *gdimlen, int maplen, + const PIO_Offset *compmap, int *ioidp, int rearranger, const PIO_Offset *iostart, const PIO_Offset *iocount) { PIO_Offset compmap_1_based[maplen]; + int *rearrangerp = NULL; + + LOG((1, "PIOc_init_decomp iosysid = %d pio_type = %d ndims = %d maplen = %d", + iosysid, pio_type, ndims, maplen)); - LOG((1, "PIOc_init_decomp iosysid = %d basetype = %d ndims = %d maplen = %d", - iosysid, basetype, ndims, maplen)); + /* If the user specified a non-default rearranger, use it. */ + if (rearranger) + rearrangerp = &rearranger; /* Add 1 to all elements in compmap. */ for (int e = 0; e < maplen; e++) + { + LOG((3, "zero-based compmap[%d] = %d", e, compmap[e])); compmap_1_based[e] = compmap[e] + 1; + } - return PIOc_InitDecomp(iosysid, basetype, ndims, dims, maplen, compmap_1_based, - ioidp, rearranger, iostart, iocount); + /* Call the legacy version of the function. */ + return PIOc_InitDecomp(iosysid, pio_type, ndims, gdimlen, maplen, compmap_1_based, + ioidp, rearrangerp, iostart, iocount); } /** @@ -530,16 +566,17 @@ int PIOc_init_decomp(int iosysid, int basetype, int ndims, const int *dims, int * the file. In this case we compute the compdof. * * @param iosysid the IO system ID - * @param basetype + * @param pio_type * @param ndims the number of dimensions - * @param dims array of dimensions + * @param gdimlen an array length ndims with the sizes of the global + * dimensions. * @param start start array * @param count count array * @param pointer that gets the IO ID. * @returns 0 for success, error code otherwise * @ingroup PIO_initdecomp */ -int PIOc_InitDecomp_bc(int iosysid, int basetype, int ndims, const int *dims, +int PIOc_InitDecomp_bc(int iosysid, int pio_type, int ndims, const int *gdimlen, const long int *start, const long int *count, int *ioidp) { @@ -548,20 +585,20 @@ int PIOc_InitDecomp_bc(int iosysid, int basetype, int ndims, const int *dims, PIO_Offset prod[ndims], loc[ndims]; int rearr = PIO_REARR_SUBSET; - LOG((1, "PIOc_InitDecomp_bc iosysid = %d basetype = %d ndims = %d")); + LOG((1, "PIOc_InitDecomp_bc iosysid = %d pio_type = %d ndims = %d")); /* Get the info about the io system. */ if (!(ios = pio_get_iosystem_from_id(iosysid))) return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__); /* Check for required inputs. */ - if (!dims || !start || !count || !ioidp) + if (!gdimlen || !start || !count || !ioidp) return pio_err(ios, NULL, PIO_EINVAL, __FILE__, __LINE__); /* Check that dim, start, and count values are not obviously * incorrect. */ for (int i = 0; i < ndims; i++) - if (dims[i] <= 0 || start[i] < 0 || count[i] < 0 || (start[i] + count[i]) > dims[i]) + if (gdimlen[i] <= 0 || start[i] < 0 || count[i] < 0 || (start[i] + count[i]) > gdimlen[i]) return pio_err(ios, NULL, PIO_EINVAL, __FILE__, __LINE__); /* Find the maplen. */ @@ -576,7 +613,7 @@ int PIOc_InitDecomp_bc(int iosysid, int basetype, int ndims, const int *dims, loc[ndims - 1] = 0; for (n = ndims - 2; n >= 0; n--) { - prod[n] = prod[n + 1] * dims[n + 1]; + prod[n] = prod[n + 1] * gdimlen[n + 1]; loc[n] = 0; } for (i = 0; i < maplen; i++) @@ -594,7 +631,7 @@ int PIOc_InitDecomp_bc(int iosysid, int basetype, int ndims, const int *dims, } } - return PIOc_InitDecomp(iosysid, basetype, ndims, dims, maplen, compmap, ioidp, + return PIOc_InitDecomp(iosysid, pio_type, ndims, gdimlen, maplen, compmap, ioidp, &rearr, NULL, NULL); } @@ -608,13 +645,37 @@ int PIOc_InitDecomp_bc(int iosysid, int basetype, int ndims, const int *dims, * The caller must create all comp_comm and the io_comm MPI * communicators before calling this function. * - * @param comp_comm the MPI_Comm of the compute tasks - * @param num_iotasks the number of io tasks to use - * @param stride the offset between io tasks in the comp_comm - * @param base the comp_comm index of the first io task + * Internally, this function does the following: + * + *
    + *
  • Initialize logging system (if PIO_ENABLE_LOGGING is set). + *
  • Allocates and initializes the iosystem_desc_t struct (ios). + *
  • MPI duplicated user comp_comm to ios->comp_comm and + * ios->union_comm. + *
  • Set ios->my_comm to be ios->comp_comm. (Not an MPI + * duplication.) + *
  • Find MPI rank in comp_comm, determine ranks of IO tasks, + * determine whether this task is one of the IO tasks. + *
  • Identify the root IO tasks. + *
  • Create MPI groups for IO tasks, and for computation tasks. + *
  • On IO tasks, create an IO communicator (ios->io_comm). + *
  • Assign an iosystemid, and put this iosystem_desc_t into the + * list of open iosystems. + *
  • Initialize the bget buffer, unless PIO_USE_MALLOC was used. + *
+ * + * When complete, there are three MPI communicators (ios->comp_comm, + * ios->union_comm, and ios->io_comm), and two MPI groups + * (ios->compgroup and ios->iogroup) that must be freed by MPI. + * + * @param comp_comm the MPI_Comm of the compute tasks. + * @param num_iotasks the number of io tasks to use. + * @param stride the offset between io tasks in the comp_comm. + * @param base the comp_comm index of the first io task. * @param rearr the rearranger to use by default, this may be - * overriden in the @ref PIO_initdecomp - * @param iosysidp index of the defined system descriptor + * overriden in the PIO_init_decomp(). The rearranger is not used + * until the decomposition is initialized. + * @param iosysidp index of the defined system descriptor. * @return 0 on success, otherwise a PIO error code. * @ingroup PIO_init */ @@ -652,9 +713,13 @@ int PIOc_Init_Intracomm(MPI_Comm comp_comm, int num_iotasks, int stride, int bas ios->num_iotasks = num_iotasks; ios->num_comptasks = num_comptasks; - /* Initialize the rearranger options. */ - init_rearr_opts(ios); + /* For non-async, the IO tasks are a subset of the comptasks. */ + ios->num_uniontasks = num_comptasks; + /* Initialize the rearranger options. */ + ios->rearr_opts.comm_type = PIO_REARR_COMM_COLL; + ios->rearr_opts.fcd = PIO_REARR_COMM_FC_2D_DISABLE; + /* Copy the computation communicator into union_comm. */ if ((mpierr = MPI_Comm_dup(comp_comm, &ios->union_comm))) return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); @@ -667,35 +732,42 @@ int PIOc_Init_Intracomm(MPI_Comm comp_comm, int num_iotasks, int stride, int bas ios->my_comm = ios->comp_comm; ustride = stride; - /* Find MPI rank comp_comm communicator. */ + /* Find MPI rank in comp_comm communicator. */ if ((mpierr = MPI_Comm_rank(ios->comp_comm, &ios->comp_rank))) return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); + /* With non-async, all tasks are part of computation component. */ + ios->compproc = true; + + /* Create an array that holds the ranks of the tasks to be used + * for computation. */ + if (!(ios->compranks = calloc(ios->num_comptasks, sizeof(int)))) + return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); + for (int i = 0; i < ios->num_comptasks; i++) + ios->compranks[i] = i; + + /* Is this the comp master? */ if (ios->comp_rank == 0) ios->compmaster = MPI_ROOT; LOG((2, "comp_rank = %d num_comptasks = %d", ios->comp_rank, ios->num_comptasks)); /* Create an array that holds the ranks of the tasks to be used - * for IO. NOTE that sizeof(int) should probably be 1, not - * sizeof(int) ???*/ - if (!(ios->ioranks = calloc(sizeof(int), ios->num_iotasks))) + * for IO. */ + if (!(ios->ioranks = calloc(ios->num_iotasks, sizeof(int)))) return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); for (int i = 0; i < ios->num_iotasks; i++) { ios->ioranks[i] = (base + i * ustride) % ios->num_comptasks; if (ios->ioranks[i] == ios->comp_rank) ios->ioproc = true; + LOG((3, "ios->ioranks[%d] = %d", i, ios->ioranks[i])); } ios->ioroot = ios->ioranks[0]; - for (int i = 0; i < ios->num_iotasks; i++) - LOG((3, "ios->ioranks[%d] = %d", i, ios->ioranks[i])); - /* We are not providing an info object. */ ios->info = MPI_INFO_NULL; - /* The task that has an iomaster value of MPI_ROOT will be the - * root of the IO communicator. */ + /* Identify the task that will be the root of the IO communicator. */ if (ios->comp_rank == ios->ioranks[0]) ios->iomaster = MPI_ROOT; @@ -713,8 +785,8 @@ int PIOc_Init_Intracomm(MPI_Comm comp_comm, int num_iotasks, int stride, int bas return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); /* For the tasks that are doing IO, get their rank within the IO - * communicator. For some reason when I check the return value of - * this MPI call, all tests start to fail! */ + * communicator. If they are not doing IO, set their io_rank to + * -1. */ if (ios->ioproc) { if ((mpierr = MPI_Comm_rank(ios->io_comm, &ios->io_rank))) @@ -724,6 +796,7 @@ int PIOc_Init_Intracomm(MPI_Comm comp_comm, int num_iotasks, int stride, int bas ios->io_rank = -1; LOG((3, "ios->io_comm = %d ios->io_rank = %d", ios->io_comm, ios->io_rank)); + /* Rank in the union comm is the same as rank in the comp comm. */ ios->union_rank = ios->comp_rank; /* Add this ios struct to the list in the PIO library. */ @@ -770,12 +843,12 @@ int PIOc_Init_Intracomm_from_F90(int f90_comp_comm, LOG((1, "Setting rearranger options, iosys=%d", *iosysidp)); return PIOc_set_rearr_opts(*iosysidp, rearr_opts->comm_type, rearr_opts->fcd, - rearr_opts->comm_fc_opts_comp2io.enable_hs, - rearr_opts->comm_fc_opts_comp2io.enable_isend, - rearr_opts->comm_fc_opts_comp2io.max_pend_req, - rearr_opts->comm_fc_opts_io2comp.enable_hs, - rearr_opts->comm_fc_opts_io2comp.enable_isend, - rearr_opts->comm_fc_opts_io2comp.max_pend_req); + rearr_opts->comp2io.hs, + rearr_opts->comp2io.isend, + rearr_opts->comp2io.max_pend_req, + rearr_opts->io2comp.hs, + rearr_opts->io2comp.isend, + rearr_opts->io2comp.max_pend_req); } return ret; } @@ -841,7 +914,7 @@ int PIOc_finalize(int iosysid) * comp master to the IO processes. This may be called by * componets for other components iosysid. So don't send unless * there is a valid union_comm. */ - if (ios->async_interface && ios->union_comm != MPI_COMM_NULL) + if (ios->async && ios->union_comm != MPI_COMM_NULL) { int msg = PIO_MSG_EXIT; @@ -874,6 +947,9 @@ int PIOc_finalize(int iosysid) if (ios->ioranks) free(ios->ioranks); LOG((3, "Freed ioranks.")); + if (ios->compranks) + free(ios->compranks); + LOG((3, "Freed compranks.")); /* Free the buffer pool. */ int niosysid; @@ -1062,24 +1138,30 @@ int PIOc_iotype_available(int iotype) * duplicates and each must later be freed with MPI_Free() by the * caller.) * + * @param rearranger the default rearranger to use for decompositions + * in this IO system. Must be either PIO_REARR_BOX or + * PIO_REARR_SUBSET. + * * @param iosysidp pointer to array of length component_count that * gets the iosysid for each component. * * @return PIO_NOERR on success, error code otherwise. * @ingroup PIO_init */ -int PIOc_Init_Async(MPI_Comm world, int num_io_procs, int *io_proc_list, +int PIOc_init_async(MPI_Comm world, int num_io_procs, int *io_proc_list, int component_count, int *num_procs_per_comp, int **proc_list, - MPI_Comm *user_io_comm, MPI_Comm *user_comp_comm, int *iosysidp) + MPI_Comm *user_io_comm, MPI_Comm *user_comp_comm, int rearranger, + int *iosysidp) { - int my_rank; - int **my_proc_list; - int *my_io_proc_list; - int mpierr; - int ret; + int my_rank; /* Rank of this task. */ + int **my_proc_list; /* Array of arrays of procs for comp components. */ + int *my_io_proc_list; /* List of processors in IO component. */ + int mpierr; /* Return code from MPI functions. */ + int ret; /* Return code. */ /* Check input parameters. */ - if (num_io_procs < 1 || component_count < 1 || !num_procs_per_comp || !iosysidp) + if (num_io_procs < 1 || component_count < 1 || !num_procs_per_comp || !iosysidp || + (rearranger != PIO_REARR_BOX && rearranger != PIO_REARR_SUBSET)) return pio_err(NULL, NULL, PIO_EINVAL, __FILE__, __LINE__); /* Temporarily limit to one computational component. */ @@ -1088,16 +1170,21 @@ int PIOc_Init_Async(MPI_Comm world, int num_io_procs, int *io_proc_list, /* Turn on the logging system for PIO. */ pio_init_logging(); - LOG((1, "PIOc_Init_Async component_count = %d", component_count)); + LOG((1, "PIOc_Init_Async num_io_procs = %d component_count = %d", num_io_procs, + component_count)); /* If the user did not supply a list of process numbers to use for * IO, create it. */ if (!io_proc_list) { + LOG((3, "calculating processors for IO component")); if (!(my_io_proc_list = malloc(num_io_procs * sizeof(int)))) return pio_err(NULL, NULL, PIO_ENOMEM, __FILE__, __LINE__); for (int p = 0; p < num_io_procs; p++) + { my_io_proc_list[p] = p; + LOG((3, "my_io_proc_list[%d] = %d", p, my_io_proc_list[p])); + } } else my_io_proc_list = io_proc_list; @@ -1106,16 +1193,16 @@ int PIOc_Init_Async(MPI_Comm world, int num_io_procs, int *io_proc_list, * component, create one. */ if (!proc_list) { - int last_proc = 0; + int last_proc = num_io_procs; /* Allocate space for array of arrays. */ - if (!(my_proc_list = malloc((component_count + 1) * sizeof(int *)))) + if (!(my_proc_list = malloc((component_count) * sizeof(int *)))) return pio_err(NULL, NULL, PIO_ENOMEM, __FILE__, __LINE__); /* Fill the array of arrays. */ - for (int cmp = 0; cmp < component_count + 1; cmp++) + for (int cmp = 0; cmp < component_count; cmp++) { - LOG((3, "calculating processors for component %d", cmp)); + LOG((3, "calculating processors for component %d num_procs_per_comp[cmp] = %d", cmp, num_procs_per_comp[cmp])); /* Allocate space for each array. */ if (!(my_proc_list[cmp] = malloc(num_procs_per_comp[cmp] * sizeof(int)))) @@ -1133,20 +1220,19 @@ int PIOc_Init_Async(MPI_Comm world, int num_io_procs, int *io_proc_list, else my_proc_list = proc_list; - /* Get rank of this task. */ + /* Get rank of this task in world. */ if ((ret = MPI_Comm_rank(world, &my_rank))) return check_mpi(NULL, ret, __FILE__, __LINE__); /* Is this process in the IO component? */ int pidx; - for (pidx = 0; pidx < num_procs_per_comp[0]; pidx++) - if (my_rank == my_proc_list[0][pidx]) + for (pidx = 0; pidx < num_io_procs; pidx++) + if (my_rank == my_io_proc_list[pidx]) break; - int in_io = (pidx == num_procs_per_comp[0]) ? 0 : 1; + int in_io = (pidx == num_io_procs) ? 0 : 1; LOG((3, "in_io = %d", in_io)); /* Allocate struct to hold io system info for each computation component. */ - /* Allocate struct to hold io system info for each component. */ iosystem_desc_t *iosys[component_count], *my_iosys; for (int cmp1 = 0; cmp1 < component_count; cmp1++) if (!(iosys[cmp1] = (iosystem_desc_t *)calloc(1, sizeof(iosystem_desc_t)))) @@ -1174,9 +1260,7 @@ int PIOc_Init_Async(MPI_Comm world, int num_io_procs, int *io_proc_list, /* Create a group for the IO component. */ if ((ret = MPI_Group_incl(world_group, num_io_procs, my_io_proc_list, &io_group))) return check_mpi(NULL, ret, __FILE__, __LINE__); - LOG((3, "created IO group - io_group = %d group empty is %d", io_group, MPI_GROUP_EMPTY)); - for (int p = 0; p < num_io_procs; p++) - LOG((3, "my_io_proc_list[%d] = %d", p, my_io_proc_list[p])); + LOG((3, "created IO group - io_group = %d MPI_GROUP_EMPTY = %d", io_group, MPI_GROUP_EMPTY)); /* There is one shared IO comm. Create it. */ if ((ret = MPI_Comm_create(world, io_group, &io_comm))) @@ -1204,45 +1288,47 @@ int PIOc_Init_Async(MPI_Comm world, int num_io_procs, int *io_proc_list, io_comm, io_rank, iomaster == MPI_ROOT ? "MASTER" : "SERVANT")); } - /* We will create a group for each component. */ - MPI_Group group[component_count + 1]; + /* We will create a group for each computational component. */ + MPI_Group group[component_count]; /* We will also create a group for each component and the IO * component processes (i.e. a union of computation and IO * processes. */ MPI_Group union_group[component_count]; - /* For each component, starting with the IO component. */ - for (int cmp = 0; cmp < component_count + 1; cmp++) + /* For each computation component. */ + for (int cmp = 0; cmp < component_count; cmp++) { LOG((3, "processing component %d", cmp)); - /* Don't start initing iosys until after IO component. */ - if (cmp) - { - /* Get pointer to current iosys. */ - my_iosys = iosys[cmp - 1]; - - /* Initialize some values. */ - my_iosys->io_comm = MPI_COMM_NULL; - my_iosys->comp_comm = MPI_COMM_NULL; - my_iosys->union_comm = MPI_COMM_NULL; - my_iosys->intercomm = MPI_COMM_NULL; - my_iosys->my_comm = MPI_COMM_NULL; - my_iosys->async_interface = 1; - my_iosys->error_handler = default_error_handler; - my_iosys->num_comptasks = num_procs_per_comp[cmp]; - my_iosys->num_iotasks = num_procs_per_comp[0]; - my_iosys->compgroup = MPI_GROUP_NULL; - my_iosys->iogroup = MPI_GROUP_NULL; - - /* The rank of the computation leader in the union comm. */ - my_iosys->comproot = num_procs_per_comp[0]; - LOG((3, "my_iosys->comproot = %d", my_iosys->comproot)); - - /* We are not providing an info object. */ - my_iosys->info = MPI_INFO_NULL; - } + /* Get pointer to current iosys. */ + my_iosys = iosys[cmp]; + + /* Initialize some values. */ + my_iosys->io_comm = MPI_COMM_NULL; + my_iosys->comp_comm = MPI_COMM_NULL; + my_iosys->union_comm = MPI_COMM_NULL; + my_iosys->intercomm = MPI_COMM_NULL; + my_iosys->my_comm = MPI_COMM_NULL; + my_iosys->async = 1; + my_iosys->error_handler = default_error_handler; + my_iosys->num_comptasks = num_procs_per_comp[cmp]; + my_iosys->num_iotasks = num_io_procs; + my_iosys->num_uniontasks = my_iosys->num_comptasks + my_iosys->num_iotasks; + my_iosys->compgroup = MPI_GROUP_NULL; + my_iosys->iogroup = MPI_GROUP_NULL; + my_iosys->default_rearranger = rearranger; + + /* Initialize the rearranger options. */ + my_iosys->rearr_opts.comm_type = PIO_REARR_COMM_COLL; + my_iosys->rearr_opts.fcd = PIO_REARR_COMM_FC_2D_DISABLE; + + /* The rank of the computation leader in the union comm. */ + my_iosys->comproot = num_io_procs; + LOG((3, "my_iosys->comproot = %d", my_iosys->comproot)); + + /* We are not providing an info object. */ + my_iosys->info = MPI_INFO_NULL; /* Create a group for this component. */ if ((ret = MPI_Group_incl(world_group, num_procs_per_comp[cmp], my_proc_list[cmp], @@ -1250,39 +1336,47 @@ int PIOc_Init_Async(MPI_Comm world, int num_io_procs, int *io_proc_list, return check_mpi(NULL, ret, __FILE__, __LINE__); LOG((3, "created component MPI group - group[%d] = %d", cmp, group[cmp])); - /* For all the computation components (i.e. cmp != 0), create - * a union group with their processors and the processors of - * the (shared) IO component. */ - if (cmp) - { - /* How many processors in the union comm? */ - int nprocs_union = num_procs_per_comp[0] + num_procs_per_comp[cmp]; + /* For all the computation components create a union group + * with their processors and the processors of the (shared) IO + * component. */ - /* This will hold proc numbers from both computation and IO - * components. */ - int proc_list_union[nprocs_union]; + /* How many processors in the union comm? */ + int nprocs_union = num_io_procs + num_procs_per_comp[cmp]; - /* Add proc numbers from IO. */ - for (int p = 0; p < num_procs_per_comp[0]; p++) - proc_list_union[p] = my_proc_list[0][p]; + /* This will hold proc numbers from both computation and IO + * components. */ + int proc_list_union[nprocs_union]; - /* Add proc numbers from computation component. */ - for (int p = 0; p < num_procs_per_comp[cmp]; p++) - proc_list_union[p + num_procs_per_comp[0]] = my_proc_list[cmp][p]; + /* Add proc numbers from IO. */ + for (int p = 0; p < num_io_procs; p++) + proc_list_union[p] = my_io_proc_list[p]; - /* Create the union group. */ - if ((ret = MPI_Group_incl(world_group, nprocs_union, proc_list_union, - &union_group[cmp - 1]))) - return check_mpi(NULL, ret, __FILE__, __LINE__); - LOG((3, "created union MPI_group - union_group[%d] = %d with %d procs", cmp, union_group[cmp-1], nprocs_union)); - } + /* Add proc numbers from computation component. */ + for (int p = 0; p < num_procs_per_comp[cmp]; p++) + proc_list_union[p + num_io_procs] = my_proc_list[cmp][p]; + + /* Allocate space for computation task ranks. */ + if (!(my_iosys->compranks = calloc(my_iosys->num_comptasks, sizeof(int)))) + return pio_err(NULL, NULL, PIO_ENOMEM, __FILE__, __LINE__); + + /* Remember computation task ranks. */ + for (int p = 0; p < num_procs_per_comp[cmp]; p++) + my_iosys->compranks[p] = my_proc_list[cmp][p]; + + /* Create the union group. */ + if ((ret = MPI_Group_incl(world_group, nprocs_union, proc_list_union, &union_group[cmp]))) + return check_mpi(NULL, ret, __FILE__, __LINE__); + LOG((3, "created union MPI_group - union_group[%d] = %d with %d procs", cmp, + union_group[cmp], nprocs_union)); /* Remember whether this process is in the IO component. */ - if (cmp) - my_iosys->ioproc = in_io; + my_iosys->ioproc = in_io; + + /* With async, tasks are either in a computation component or + * the IO component. */ + my_iosys->compproc = !in_io; - /* Is this process in this computation component (which is the - * IO component if cmp == 0)? */ + /* Is this process in this computation component? */ int in_cmp = 0; for (pidx = 0; pidx < num_procs_per_comp[cmp]; pidx++) if (my_rank == my_proc_list[cmp][pidx]) @@ -1294,113 +1388,97 @@ int PIOc_Init_Async(MPI_Comm world, int num_io_procs, int *io_proc_list, /* Create an intracomm for this component. Only processes in * the component need to participate in the intracomm create * call. */ - /* Create the intracomm from the group. */ LOG((3, "creating intracomm cmp = %d from group[%d] = %d", cmp, cmp, group[cmp])); + if ((ret = MPI_Comm_create(world, group[cmp], &my_iosys->comp_comm))) + return check_mpi(NULL, ret, __FILE__, __LINE__); - /* We handle the IO comm differently (cmp == 0). */ - if (!cmp) + if (in_cmp) { - /* LOG((3, "about to create io comm")); */ - /* if ((ret = MPI_Comm_create_group(world, group[cmp], cmp, &io_comm))) */ - /* return check_mpi(NULL, ret, __FILE__, __LINE__); */ - /* LOG((3, "about to get io rank")); */ - /* if ((ret = MPI_Comm_rank(io_comm, &io_rank))) */ - /* return check_mpi(NULL, ret, __FILE__, __LINE__); */ - /* iomaster = !io_rank ? MPI_ROOT : MPI_PROC_NULL; */ - /* LOG((3, "intracomm created for cmp = %d io_comm = %d io_rank = %d IO %s", */ - /* cmp, io_comm, io_rank, iomaster == MPI_ROOT ? "MASTER" : "SERVANT")); */ - } - else - { - if ((ret = MPI_Comm_create(world, group[cmp], &my_iosys->comp_comm))) - return check_mpi(NULL, ret, __FILE__, __LINE__); + /* Does the user want a copy? */ + if (user_comp_comm) + if ((mpierr = MPI_Comm_dup(my_iosys->comp_comm, &user_comp_comm[cmp]))) + return check_mpi(NULL, mpierr, __FILE__, __LINE__); - if (in_cmp) - { - /* Does the user want a copy? */ - if (user_comp_comm) - if ((mpierr = MPI_Comm_dup(my_iosys->comp_comm, &user_comp_comm[cmp - 1]))) - return check_mpi(NULL, mpierr, __FILE__, __LINE__); - - /* Get the rank in this comp comm. */ - if ((ret = MPI_Comm_rank(my_iosys->comp_comm, &my_iosys->comp_rank))) - return check_mpi(NULL, ret, __FILE__, __LINE__); + /* Get the rank in this comp comm. */ + if ((ret = MPI_Comm_rank(my_iosys->comp_comm, &my_iosys->comp_rank))) + return check_mpi(NULL, ret, __FILE__, __LINE__); - /* Set comp_rank 0 to be the compmaster. It will have - * a setting of MPI_ROOT, all other tasks will have a - * setting of MPI_PROC_NULL. */ - my_iosys->compmaster = my_iosys->comp_rank ? MPI_PROC_NULL : MPI_ROOT; + /* Set comp_rank 0 to be the compmaster. It will have a + * setting of MPI_ROOT, all other tasks will have a + * setting of MPI_PROC_NULL. */ + my_iosys->compmaster = my_iosys->comp_rank ? MPI_PROC_NULL : MPI_ROOT; - LOG((3, "intracomm created for cmp = %d comp_comm = %d comp_rank = %d comp %s", - cmp, my_iosys->comp_comm, my_iosys->comp_rank, - my_iosys->compmaster == MPI_ROOT ? "MASTER" : "SERVANT")); - } + LOG((3, "intracomm created for cmp = %d comp_comm = %d comp_rank = %d comp %s", + cmp, my_iosys->comp_comm, my_iosys->comp_rank, + my_iosys->compmaster == MPI_ROOT ? "MASTER" : "SERVANT")); } - /* If this is the IO component, make a copy of the IO comm for * each computational component. */ if (in_io) - if (cmp) - { - LOG((3, "making a dup of io_comm = %d io_rank = %d", io_comm, io_rank)); - if ((ret = MPI_Comm_dup(io_comm, &my_iosys->io_comm))) - return check_mpi(NULL, ret, __FILE__, __LINE__); - LOG((3, "dup of io_comm = %d io_rank = %d", my_iosys->io_comm, io_rank)); - my_iosys->iomaster = iomaster; - my_iosys->io_rank = io_rank; - my_iosys->ioroot = 0; - my_iosys->comp_idx = cmp - 1; - } + { + LOG((3, "making a dup of io_comm = %d io_rank = %d", io_comm, io_rank)); + if ((ret = MPI_Comm_dup(io_comm, &my_iosys->io_comm))) + return check_mpi(NULL, ret, __FILE__, __LINE__); + LOG((3, "dup of io_comm = %d io_rank = %d", my_iosys->io_comm, io_rank)); + my_iosys->iomaster = iomaster; + my_iosys->io_rank = io_rank; + my_iosys->ioroot = 0; + my_iosys->comp_idx = cmp; + } + + /* Create an array that holds the ranks of the tasks to be used + * for IO. */ + if (!(my_iosys->ioranks = calloc(my_iosys->num_iotasks, sizeof(int)))) + return pio_err(NULL, NULL, PIO_ENOMEM, __FILE__, __LINE__); + for (int i = 0; i < my_iosys->num_iotasks; i++) + my_iosys->ioranks[i] = my_io_proc_list[i]; + my_iosys->ioroot = my_iosys->ioranks[0]; /* All the processes in this component, and the IO component, * are part of the union_comm. */ - if (cmp) + if (in_io || in_cmp) { - if (in_io || in_cmp) - { - LOG((3, "my_iosys->io_comm = %d group = %d", my_iosys->io_comm, union_group[cmp-1])); - /* Create a group for the union of the IO component - * and one of the computation components. */ - if ((ret = MPI_Comm_create(world, union_group[cmp - 1], - &my_iosys->union_comm))) - return check_mpi(NULL, ret, __FILE__, __LINE__); + LOG((3, "my_iosys->io_comm = %d group = %d", my_iosys->io_comm, union_group[cmp])); + /* Create a group for the union of the IO component + * and one of the computation components. */ + if ((ret = MPI_Comm_create(world, union_group[cmp], &my_iosys->union_comm))) + return check_mpi(NULL, ret, __FILE__, __LINE__); - if ((ret = MPI_Comm_rank(my_iosys->union_comm, &my_iosys->union_rank))) - return check_mpi(NULL, ret, __FILE__, __LINE__); + if ((ret = MPI_Comm_rank(my_iosys->union_comm, &my_iosys->union_rank))) + return check_mpi(NULL, ret, __FILE__, __LINE__); - /* Set my_comm to union_comm for async. */ - my_iosys->my_comm = my_iosys->union_comm; - LOG((3, "intracomm created for union cmp = %d union_rank = %d union_comm = %d", - cmp, my_iosys->union_rank, my_iosys->union_comm)); + /* Set my_comm to union_comm for async. */ + my_iosys->my_comm = my_iosys->union_comm; + LOG((3, "intracomm created for union cmp = %d union_rank = %d union_comm = %d", + cmp, my_iosys->union_rank, my_iosys->union_comm)); - if (in_io) - { - LOG((3, "my_iosys->io_comm = %d", my_iosys->io_comm)); - /* Create the intercomm from IO to computation component. */ - LOG((3, "about to create intercomm for IO component to cmp = %d " - "my_iosys->io_comm = %d", cmp, my_iosys->io_comm)); - if ((ret = MPI_Intercomm_create(my_iosys->io_comm, 0, my_iosys->union_comm, - my_proc_list[cmp][0], 0, &my_iosys->intercomm))) - return check_mpi(NULL, ret, __FILE__, __LINE__); - } - else - { - /* Create the intercomm from computation component to IO component. */ - LOG((3, "about to create intercomm for cmp = %d my_iosys->comp_comm = %d", cmp, - my_iosys->comp_comm)); - if ((ret = MPI_Intercomm_create(my_iosys->comp_comm, 0, my_iosys->union_comm, - my_proc_list[0][0], 0, &my_iosys->intercomm))) - return check_mpi(NULL, ret, __FILE__, __LINE__); - } - LOG((3, "intercomm created for cmp = %d", cmp)); + if (in_io) + { + LOG((3, "my_iosys->io_comm = %d", my_iosys->io_comm)); + /* Create the intercomm from IO to computation component. */ + LOG((3, "about to create intercomm for IO component to cmp = %d " + "my_iosys->io_comm = %d", cmp, my_iosys->io_comm)); + if ((ret = MPI_Intercomm_create(my_iosys->io_comm, 0, my_iosys->union_comm, + my_proc_list[cmp][0], 0, &my_iosys->intercomm))) + return check_mpi(NULL, ret, __FILE__, __LINE__); } - - /* Add this id to the list of PIO iosystem ids. */ - iosysidp[cmp - 1] = pio_add_to_iosystem_list(my_iosys); - LOG((2, "new iosys ID added to iosystem_list iosysid = %d\n", iosysidp[cmp - 1])); + else + { + /* Create the intercomm from computation component to IO component. */ + LOG((3, "about to create intercomm for cmp = %d my_iosys->comp_comm = %d", cmp, + my_iosys->comp_comm)); + if ((ret = MPI_Intercomm_create(my_iosys->comp_comm, 0, my_iosys->union_comm, + my_io_proc_list[0], 0, &my_iosys->intercomm))) + return check_mpi(NULL, ret, __FILE__, __LINE__); + } + LOG((3, "intercomm created for cmp = %d", cmp)); } - } + + /* Add this id to the list of PIO iosystem ids. */ + iosysidp[cmp] = pio_add_to_iosystem_list(my_iosys); + LOG((2, "new iosys ID added to iosystem_list iosysid = %d", iosysidp[cmp])); + } /* next computational component */ /* Now call the function from which the IO tasks will not return * until the PIO_MSG_EXIT message is sent. This will handle all @@ -1415,7 +1493,6 @@ int PIOc_Init_Async(MPI_Comm world, int num_io_procs, int *io_proc_list, } /* Free resources if needed. */ - LOG((2, "PIOc_Init_Async starting to free resources")); if (!io_proc_list) free(my_io_proc_list); @@ -1425,7 +1502,7 @@ int PIOc_Init_Async(MPI_Comm world, int num_io_procs, int *io_proc_list, if (!proc_list) { - for (int cmp = 0; cmp < component_count + 1; cmp++) + for (int cmp = 0; cmp < component_count; cmp++) free(my_proc_list[cmp]); free(my_proc_list); } @@ -1434,13 +1511,12 @@ int PIOc_Init_Async(MPI_Comm world, int num_io_procs, int *io_proc_list, if ((ret = MPI_Group_free(&io_group))) return check_mpi(NULL, ret, __FILE__, __LINE__); - for (int cmp = 0; cmp < component_count + 1; cmp++) + for (int cmp = 0; cmp < component_count; cmp++) { if ((ret = MPI_Group_free(&group[cmp]))) return check_mpi(NULL, ret, __FILE__, __LINE__); - if (cmp) - if ((ret = MPI_Group_free(&union_group[cmp - 1]))) - return check_mpi(NULL, ret, __FILE__, __LINE__); + if ((ret = MPI_Group_free(&union_group[cmp]))) + return check_mpi(NULL, ret, __FILE__, __LINE__); } if ((ret = MPI_Group_free(&world_group))) diff --git a/src/externals/pio2/src/clib/pioc_sc.c b/src/externals/pio2/src/clib/pioc_sc.c index 6b991b373db..98e3c6aa11a 100644 --- a/src/externals/pio2/src/clib/pioc_sc.c +++ b/src/externals/pio2/src/clib/pioc_sc.c @@ -74,11 +74,11 @@ int gcd_array(int nain, int *ain) } /** - * Return the gcd of nain and any value in ain as int_64. + * Return the greatest common devisor of array ain as int_64. * - * @param main - * @param ain - * @returns + * @param nain number of elements in ain. + * @param ain array of length nain. + * @returns GCD of elements in ain. */ long long lgcd_array(int nain, long long *ain) { @@ -101,35 +101,50 @@ long long lgcd_array(int nain, long long *ain) } /** - * Compute start and count arrays. + * Compute one element (dimension) of start and count arrays. This + * function is used by CalcStartandCount(). * - * @param gdim - * @param ioprocs - * @param rank - * @param start - * @param kount + * @param gdim global size of one dimension. + * @param ioprocs number of io tasks. + * @param rank IO rank of this task. + * @param start pointer to PIO_Offset that will get the start value. + * @param count pointer to PIO_Offset that will get the count value. */ -void computestartandcount(int gdim, int ioprocs, int rank, PIO_Offset *start, - PIO_Offset *kount) +void compute_one_dim(int gdim, int ioprocs, int rank, PIO_Offset *start, + PIO_Offset *count) { - int irank; + int irank; /* The IO rank for this task. */ int remainder; int adds; - PIO_Offset lstart, lkount; + PIO_Offset lstart, lcount; + /* Check inputs. */ + pioassert(gdim >= 0 && ioprocs > 0 && rank >= 0 && start && count, + "invalid input", __FILE__, __LINE__); + + /* Determin which IO task to use. */ irank = rank % ioprocs; - lkount = (long int)(gdim / ioprocs); - lstart = (long int)(lkount * irank); - remainder = gdim - lkount * ioprocs; - if (remainder >= ioprocs-irank) + /* Each IO task will have its share of the global dim. */ + lcount = (long int)(gdim / ioprocs); + + /* Find the start for this task. */ + lstart = (long int)(lcount * irank); + + /* Is there anything left over? */ + remainder = gdim - lcount * ioprocs; + + /* Distribute left over data to some IO tasks. */ + if (remainder >= ioprocs - irank) { - lkount++; + lcount++; if ((adds = irank + remainder - ioprocs) > 0) lstart += adds; } + + /* Return results to caller. */ *start = lstart; - *kount = lkount; + *count = lcount; } /** @@ -138,20 +153,29 @@ void computestartandcount(int gdim, int ioprocs, int rank, PIO_Offset *start, * * @param arrlen * @param arr_in - * @returns + * @returns the size of the block */ PIO_Offset GCDblocksize(int arrlen, const PIO_Offset *arr_in) { - int i, j, n, numblks, numtimes, ii, numgaps; - PIO_Offset bsize, bsizeg, blklensum; - PIO_Offset del_arr[arrlen - 1]; + int numblks = 0; /* Number of blocks. */ + int numtimes = 0; /* Number of times adjacent arr_in elements differ by != 1. */ + int numgaps = 0; /* Number of gaps. */ + int j; /* Loop counter. */ + int ii; /* Loop counter. */ + int n; + PIO_Offset bsize; /* Size of the block. */ + PIO_Offset bsizeg; /* Size of gap block. */ + PIO_Offset blklensum; /* Sum of all block lengths. */ + PIO_Offset del_arr[arrlen - 1]; /* Array of deltas between adjacent elements in arr_in. */ PIO_Offset loc_arr[arrlen - 1]; - numblks = 0; - numgaps = 0; - numtimes = 0; + /* Check inputs. */ + pioassert(arrlen > 0 && arr_in, "invalid input", __FILE__, __LINE__); - for (i = 0; i < arrlen - 1; i++) + /* Count the number of contiguous blocks in arr_in. If any if + these blocks is of size 1, we are done and can return. + Otherwise numtimes is the number of blocks. */ + for (int i = 0; i < arrlen - 1; i++) { del_arr[i] = arr_in[i + 1] - arr_in[i]; if (del_arr[i] != 1) @@ -162,76 +186,95 @@ PIO_Offset GCDblocksize(int arrlen, const PIO_Offset *arr_in) } } + /* If numtimes is 0 the all of the data in arr_in is contiguous + * and numblks=1. Not sure why I have three different variables + * here, seems like n,numblks and numtimes could be combined. */ numblks = numtimes + 1; if (numtimes == 0) n = numblks; else n = numtimes; + /* If numblks==1 then the result is arrlen and you can return. */ bsize = (PIO_Offset)arrlen; if (numblks > 1) { PIO_Offset blk_len[numblks]; PIO_Offset gaps[numtimes]; - if(numtimes > 0) + + /* If numblks > 1 then numtimes must be > 0 and this if block + * isn't needed. */ + if (numtimes > 0) { ii = 0; - for (i = 0; i < arrlen - 1; i++) + for (int i = 0; i < arrlen - 1; i++) if (del_arr[i] > 1) gaps[ii++] = del_arr[i] - 1; numgaps = ii; } j = 0; - for (i = 0; i < n; i++) + for (int i = 0; i < n; i++) loc_arr[i] = 1; - for (i = 0; i < arrlen - 1; i++) + for (int i = 0; i < arrlen - 1; i++) if(del_arr[i] != 1) loc_arr[j++] = i; blk_len[0] = loc_arr[0]; blklensum = blk_len[0]; - for(i = 1; i < numblks - 1; i++) + for(int i = 1; i < numblks - 1; i++) { blk_len[i] = loc_arr[i] - loc_arr[i - 1]; blklensum += blk_len[i]; } blk_len[numblks - 1] = arrlen - blklensum; + /* Get the GCD in blk_len array. */ bsize = lgcd_array(numblks, blk_len); + + /* I don't recall why i needed these next two blocks, I + * remember struggling to get this right in all cases and I'm + * afraid that the end result is that bsize is almost always + * 1. */ if (numgaps > 0) { bsizeg = lgcd_array(numgaps, gaps); bsize = lgcd(bsize, bsizeg); } - if(arr_in[0] > 0) + + /* ??? */ + if (arr_in[0] > 0) bsize = lgcd(bsize, arr_in[0]); } + return bsize; } /** - * Compute start and count values for each io task. + * Compute start and count values for each io task. This is used in + * PIOc_InitDecomp() for the box rearranger only. * - * @param basetype - * @param ndims - * @param gdims - * @param num_io_procs - * @param myiorank - * @param start - * @param kount + * @param pio_type the PIO data type used in this decompotion. + * @param ndims the number of dimensions in the variable, not + * including the unlimited dimension. + * @param gdims an array of global size of each dimension. + * @param num_io_procs the number of IO tasks. + * @param myiorank rank of this task in IO communicator. + * @param start array of length ndims with data start values. + * @param count array of length ndims with data count values. + * @param num_aiotasks the number of IO tasks used(?) + * @returns 0 for success, error code otherwise. */ -int CalcStartandCount(int basetype, int ndims, const int *gdims, int num_io_procs, - int myiorank, PIO_Offset *start, PIO_Offset *kount) +int CalcStartandCount(int pio_type, int ndims, const int *gdims, int num_io_procs, + int myiorank, PIO_Offset *start, PIO_Offset *count, int *num_aiotasks) { - int minbytes; + int minbytes; int maxbytes; - int minblocksize; - int basesize; + int minblocksize; /* Like minbytes, but in data elements. */ + int basesize; /* Size in bytes of base data type. */ int use_io_procs; int i; - long int p; long int pgdims; bool converged; int iorank; @@ -239,50 +282,61 @@ int CalcStartandCount(int basetype, int ndims, const int *gdims, int num_io_proc int tiorank; int ioprocs; int tioprocs; - int mystart[ndims], mykount[ndims]; + int mystart[ndims], mycount[ndims]; long int pknt; - long int tpsize=0; - + long int tpsize = 0; + int ret; + + /* Check inputs. */ + pioassert(pio_type > 0 && ndims > 0 && gdims && num_io_procs > 0 && start && count, + "invalid input", __FILE__, __LINE__); + LOG((1, "CalcStartandCount pio_type = %d ndims = %d num_io_procs = %d myiorank = %d", + pio_type, ndims, num_io_procs, myiorank)); + + /* We are trying to find start and count indices for each iotask + * such that each task has approximately blocksize data to write + * (read). The number of iotasks participating in the operation is + * blocksize/global_size. */ minbytes = blocksize - 256; - maxbytes = blocksize + 256; + maxbytes = blocksize + 256; - switch (basetype) - { - case PIO_INT: - basesize = sizeof(int); - break; - case PIO_REAL: - basesize = sizeof(float); - break; - case PIO_DOUBLE: - basesize = sizeof(double); - break; - default: - piodie("Invalid basetype ",__FILE__,__LINE__); - break; - } + /* Determine the size of the data type. */ + if ((ret = find_mpi_type(pio_type, NULL, &basesize))) + return ret; + + /* Determine the minimum block size. */ minblocksize = minbytes / basesize; + /* Find the total size of the data. */ pgdims = 1; for (i = 0; i < ndims; i++) pgdims *= (long int)gdims[i]; - p = pgdims; - use_io_procs = max(1, min((int)((float)p / (float)minblocksize + 0.5), num_io_procs)); + + /* Find the number of ioprocs that are needed so that we have + * blocksize data on each iotask*/ + use_io_procs = max(1, min((int)((float)pgdims / (float)minblocksize + 0.5), num_io_procs)); + + /* Initialize to 0. */ converged = 0; for (i = 0; i < ndims; i++) { mystart[i] = 0; - mykount[i] = 0; + mycount[i] = 0; } + /* Use_io_procs is the number of ioprocs that are needed so that + * we have blocksize data on each iotask, now find start and count + * values needed on each of these tasks. */ while (!converged) { + long int p; + for (iorank = 0; iorank < use_io_procs; iorank++) { for (i = 0; i < ndims; i++) { start[i] = 0; - kount[i] = gdims[i]; + count[i] = gdims[i]; } ldims = ndims - 1; p = basesize; @@ -298,7 +352,7 @@ int CalcStartandCount(int basetype, int ndims, const int *gdims, int num_io_proc if (gdims[ldims] < use_io_procs) { - if (ldims > 0 && gdims[ldims-1] > use_io_procs) + if (ldims > 0 && gdims[ldims - 1] > use_io_procs) ldims--; else use_io_procs -= (use_io_procs % gdims[ldims]); @@ -310,8 +364,8 @@ int CalcStartandCount(int basetype, int ndims, const int *gdims, int num_io_proc { if (gdims[i] >= ioprocs) { - computestartandcount(gdims[i], ioprocs, tiorank, start + i, kount + i); - if (start[i] + kount[i] > gdims[i] + 1) + compute_one_dim(gdims[i], ioprocs, tiorank, &start[i], &count[i]); + if (start[i] + count[i] > gdims[i] + 1) { piodie("Start plus count exceeds dimension bound",__FILE__,__LINE__); } @@ -320,7 +374,7 @@ int CalcStartandCount(int basetype, int ndims, const int *gdims, int num_io_proc { tioprocs = gdims[i]; tiorank = (iorank * tioprocs) / ioprocs; - computestartandcount(gdims[i], tioprocs, tiorank, start + i, kount + i); + compute_one_dim(gdims[i], tioprocs, tiorank, &start[i], &count[i]); ioprocs = ioprocs / tioprocs; tiorank = iorank % ioprocs; } @@ -332,13 +386,13 @@ int CalcStartandCount(int basetype, int ndims, const int *gdims, int num_io_proc for (i = 0; i < ndims; i++) { mystart[i] = start[i]; - mykount[i] = kount[i]; + mycount[i] = count[i]; } } pknt = 1; for(i = 0; i < ndims; i++) - pknt *= kount[i]; + pknt *= count[i]; tpsize += pknt; @@ -360,12 +414,14 @@ int CalcStartandCount(int basetype, int ndims, const int *gdims, int num_io_proc } } + /* On IO tasks, set the start/count arrays to computed values. On + * non-io tasks set start/count to zero. */ if (myiorank < use_io_procs) { for (i = 0; i < ndims; i++) { start[i] = mystart[i]; - kount[i] = mykount[i]; + count[i] = mycount[i]; } } else @@ -373,9 +429,12 @@ int CalcStartandCount(int basetype, int ndims, const int *gdims, int num_io_proc for (i = 0; i < ndims; i++) { start[i] = 0; - kount[i] = 0; + count[i] = 0; } } - return use_io_procs; + /* Return the number of IO procs used to the caller. */ + *num_aiotasks = use_io_procs; + + return PIO_NOERR; } diff --git a/src/externals/pio2/src/clib/pioc_support.c b/src/externals/pio2/src/clib/pioc_support.c index 8f45277a32f..fba1ad9070a 100644 --- a/src/externals/pio2/src/clib/pioc_support.c +++ b/src/externals/pio2/src/clib/pioc_support.c @@ -101,13 +101,14 @@ int PIOc_strerror(int pioerr, char *errmsg) */ int PIOc_set_log_level(int level) { - int ret; #if PIO_ENABLE_LOGGING /* Set the log level. */ pio_log_level = level; #if NETCDF_C_LOGGING_ENABLED + int ret; + /* If netcdf logging is available turn it on starting at level = 4. */ if (level > NC_LEVEL_DIFF) if ((ret = nc_set_log_level(level - NC_LEVEL_DIFF))) @@ -148,7 +149,7 @@ void pio_init_logging(void) /** * Finalize logging - close log files, if open. */ -void pio_finalize_logging(void ) +void pio_finalize_logging(void) { #if PIO_ENABLE_LOGGING pio_log_ref_cnt -= 1; @@ -288,22 +289,6 @@ void print_trace(FILE *fp) free(strings); } -/** - * Exit due to lack of memory. - * - * @param ios the iosystem description struct - * @param req amount of memory that was being requested - * @param fname name of code file where error occured - * @param line the line of code where the error occurred. - */ -void piomemerror(iosystem_desc_t *ios, size_t req, char *fname, int line) -{ - char msg[80]; - sprintf(msg, "out of memory requesting: %ld", req); - cn_buffer_report(ios, false); - piodie(msg, fname, line); -} - /** * Abort program and call MPI_Abort(). * @@ -532,145 +517,173 @@ int pio_err(iosystem_desc_t *ios, file_desc_t *file, int err_num, const char *fn } /** - * Allocate an region. + * Allocate a region struct, and initialize it. * - * ndims the number of dimensions for the data in this region. - * @returns a pointer to the newly allocated io_region struct. + * @param ios pointer to the IO system info, used for error + * handling. Ignored if NULL. + * @param ndims the number of dimensions for the data in this region. + * @param a pointer that gets a pointer to the newly allocated + * io_region struct. + * @returns 0 for success, error code otherwise. */ -io_region *alloc_region(int ndims) +int alloc_region2(iosystem_desc_t *ios, int ndims, io_region **regionp) { io_region *region; + /* Check inputs. */ + pioassert(ndims >= 0 && regionp, "invalid input", __FILE__, __LINE__); + LOG((1, "alloc_region2 ndims = %d sizeof(io_region) = %d", ndims, + sizeof(io_region))); + /* Allocate memory for the io_region struct. */ - if (!(region = bget(sizeof(io_region)))) - return NULL; + if (!(region = calloc(1, sizeof(io_region)))) + return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); /* Allocate memory for the array of start indicies. */ - if (!(region->start = bget(ndims * sizeof(PIO_Offset)))) - { - brel(region); - return NULL; - } + if (!(region->start = calloc(ndims, sizeof(PIO_Offset)))) + return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); /* Allocate memory for the array of counts. */ - if (!(region->count = bget(ndims * sizeof(PIO_Offset)))) - { - brel(region); - brel(region->start); - return NULL; - } - - region->loffset = 0; - region->next = NULL; - - /* Initialize start and count arrays to zero. */ - for (int i = 0; i < ndims; i++) - { - region->start[i] = 0; - region->count[i] = 0; - } + if (!(region->count = calloc(ndims, sizeof(PIO_Offset)))) + return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); - return region; + /* Return pointer to new region to caller. */ + *regionp = region; + + return PIO_NOERR; } /** - * Find the MPI type for a PIO type. + * Given a PIO type, find the MPI type and the type size. * * @param pio_type a PIO type, PIO_INT, PIO_FLOAT, etc. * @param mpi_type a pointer to MPI_Datatype that will get the MPI - * type that coresponds to the PIO type. + * type that coresponds to the PIO type. Ignored if NULL. + * @param type_size a pointer to int that will get the size of the + * type, in bytes. (For example, 4 for PIO_INT). Ignored if NULL. * @returns 0 for success, error code otherwise. */ -int find_mpi_type(int pio_type, MPI_Datatype *mpi_type) +int find_mpi_type(int pio_type, MPI_Datatype *mpi_type, int *type_size) { - /* Check input. */ - pioassert(mpi_type, "invalid input", __FILE__, __LINE__); + MPI_Datatype my_mpi_type; + int my_type_size; /* Decide on the base type. */ switch(pio_type) { case PIO_BYTE: - *mpi_type = MPI_BYTE; + my_mpi_type = MPI_BYTE; + my_type_size = NETCDF_CHAR_SIZE; break; case PIO_CHAR: - *mpi_type = MPI_CHAR; + my_mpi_type = MPI_CHAR; + my_type_size = NETCDF_CHAR_SIZE; break; case PIO_SHORT: - *mpi_type = MPI_SHORT; + my_mpi_type = MPI_SHORT; + my_type_size = NETCDF_SHORT_SIZE; break; case PIO_INT: - *mpi_type = MPI_INT; + my_mpi_type = MPI_INT; + my_type_size = NETCDF_INT_FLOAT_SIZE; break; case PIO_FLOAT: - *mpi_type = MPI_FLOAT; + my_mpi_type = MPI_FLOAT; + my_type_size = NETCDF_INT_FLOAT_SIZE; break; case PIO_DOUBLE: - *mpi_type = MPI_DOUBLE; + my_mpi_type = MPI_DOUBLE; + my_type_size = NETCDF_DOUBLE_INT64_SIZE; break; #ifdef _NETCDF4 case PIO_UBYTE: - *mpi_type = MPI_UNSIGNED_CHAR; + my_mpi_type = MPI_UNSIGNED_CHAR; + my_type_size = NETCDF_CHAR_SIZE; break; case PIO_USHORT: - *mpi_type = MPI_UNSIGNED_SHORT; + my_mpi_type = MPI_UNSIGNED_SHORT; + my_type_size = NETCDF_SHORT_SIZE; break; case PIO_UINT: - *mpi_type = MPI_UNSIGNED; + my_mpi_type = MPI_UNSIGNED; + my_type_size = NETCDF_INT_FLOAT_SIZE; break; case PIO_INT64: - *mpi_type = MPI_LONG_LONG; + my_mpi_type = MPI_LONG_LONG; + my_type_size = NETCDF_DOUBLE_INT64_SIZE; break; case PIO_UINT64: - *mpi_type = MPI_UNSIGNED_LONG_LONG; + my_mpi_type = MPI_UNSIGNED_LONG_LONG; + my_type_size = NETCDF_DOUBLE_INT64_SIZE; break; case PIO_STRING: - *mpi_type = MPI_CHAR; + my_mpi_type = MPI_CHAR; + my_type_size = NETCDF_CHAR_SIZE; break; #endif /* _NETCDF4 */ default: return PIO_EBADTYPE; } + /* If caller wants MPI type, set it. */ + if (mpi_type) + *mpi_type = my_mpi_type; + + /* If caller wants type size, set it. */ + if (type_size) + *type_size = my_type_size; + return PIO_NOERR; } /** - * Allocate space for an IO description struct. + * Allocate space for an IO description struct, and initialize it. * - * @param ios pointer to the IO system info. + * @param ios pointer to the IO system info, used for error + * handling. * @param piotype the PIO data type (ex. PIO_FLOAT, PIO_INT, etc.). * @param ndims the number of dimensions. - * @iodesc pointer that gets a pointer to the newly allocated - * io_desc_t or NULL if allocation failed. + * @param iodesc pointer that gets the newly allocated io_desc_t. * @returns 0 for success, error code otherwise. */ int malloc_iodesc(iosystem_desc_t *ios, int piotype, int ndims, io_desc_t **iodesc) { MPI_Datatype mpi_type; + int mpierr; int ret; /* Check input. */ - pioassert(ios && iodesc, "invalid input", __FILE__, __LINE__); + pioassert(ios && piotype > 0 && ndims >= 0 && iodesc, + "invalid input", __FILE__, __LINE__); - /* Allocate space for the io_desc_t struct. */ - if (!(*iodesc = calloc(1, sizeof(io_desc_t)))) - return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); + LOG((1, "malloc_iodesc piotype = %d ndims = %d", piotype, ndims)); /* Get the MPI type corresponding with the PIO type. */ - if ((ret = find_mpi_type(piotype, &mpi_type))) + if ((ret = find_mpi_type(piotype, &mpi_type, NULL))) return pio_err(ios, NULL, ret, __FILE__, __LINE__); - /* Decide on the base type. */ + /* Allocate space for the io_desc_t struct. */ + if (!(*iodesc = calloc(1, sizeof(io_desc_t)))) + return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); + + /* Remember the MPI type. */ (*iodesc)->basetype = mpi_type; + /* Get the size of the type. */ + if ((mpierr = MPI_Type_size((*iodesc)->basetype, &(*iodesc)->basetype_size))) + return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); + /* Initialize some values in the struct. */ (*iodesc)->maxregions = 1; (*iodesc)->ioid = -1; (*iodesc)->ndims = ndims; - (*iodesc)->firstregion = alloc_region(ndims); - /* Set the swap memory settings to defaults. */ + /* Allocate space for, and initialize, the first region. */ + if ((ret = alloc_region2(ios, ndims, &((*iodesc)->firstregion)))) + return pio_err(ios, NULL, ret, __FILE__, __LINE__); + + /* Set the swap memory settings to defaults for this IO system. */ (*iodesc)->rearr_opts = ios->rearr_opts; return PIO_NOERR; @@ -689,12 +702,12 @@ void free_region_list(io_region *top) while (ptr) { if (ptr->start) - brel(ptr->start); + free(ptr->start); if (ptr->count) - brel(ptr->count); + free(ptr->count); tptr = ptr; ptr = ptr->next; - brel(tptr); + free(tptr); } } @@ -718,7 +731,7 @@ int PIOc_freedecomp(int iosysid, int ioid) return pio_err(ios, NULL, PIO_EBADID, __FILE__, __LINE__); /* If async is in use, and this is not an IO task, bcast the parameters. */ - if (ios->async_interface) + if (ios->async) { if (!ios->ioproc) { @@ -754,7 +767,7 @@ int PIOc_freedecomp(int iosysid, int ioid) { for (int i = 0; i < iodesc->nrecvs; i++) if (iodesc->rtype[i] != PIO_DATATYPE_NULL) - if ((mpierr = MPI_Type_free(iodesc->rtype + i))) + if ((mpierr = MPI_Type_free(&iodesc->rtype[i]))) return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); free(iodesc->rtype); @@ -938,7 +951,6 @@ int PIOc_readmap_from_f90(const char *file, int *ndims, int **gdims, PIO_Offset * @param filename the filename to be used. * @param cmode for PIOc_create(). Will be bitwise or'd with NC_WRITE. * @param ioid the ID of the IO description. - * @param comm an MPI communicator. * @param title optial title attribute for the file. Must be less than * NC_MAX_NAME + 1 if provided. Ignored if NULL. * @param history optial history attribute for the file. Must be less @@ -948,12 +960,11 @@ int PIOc_readmap_from_f90(const char *file, int *ndims, int **gdims, PIO_Offset * @returns 0 for success, error code otherwise. */ int PIOc_write_nc_decomp(int iosysid, const char *filename, int cmode, int ioid, - MPI_Comm comm, char *title, char *history, int fortran_order) + char *title, char *history, int fortran_order) { - iosystem_desc_t *ios; - io_desc_t *iodesc; - int npes; /* Size of this communicator. */ - int myrank; /* Rank of this task. */ + iosystem_desc_t *ios; /* IO system info. */ + io_desc_t *iodesc; /* Decomposition info. */ + int max_maplen; /* The maximum maplen used for any task. */ int mpierr; int ret; @@ -963,42 +974,41 @@ int PIOc_write_nc_decomp(int iosysid, const char *filename, int cmode, int ioid, /* Check inputs. */ if (!filename) - return pio_err(NULL, NULL, PIO_EINVAL, __FILE__, __LINE__); + return pio_err(ios, NULL, PIO_EINVAL, __FILE__, __LINE__); + if (title) + if (strlen(title) > PIO_MAX_NAME) + return pio_err(ios, NULL, PIO_EINVAL, __FILE__, __LINE__); + if (history) + if (strlen(history) > PIO_MAX_NAME) + return pio_err(ios, NULL, PIO_EINVAL, __FILE__, __LINE__); - LOG((1, "PIOc_write_nc_decomp filename = %s iosysid = %d ioid = %d", filename, - iosysid, ioid)); + LOG((1, "PIOc_write_nc_decomp filename = %s iosysid = %d ioid = %d " + "ios->num_comptasks = %d", filename, iosysid, ioid, ios->num_comptasks)); /* Get the IO desc, which describes the decomposition. */ if (!(iodesc = pio_get_iodesc_from_id(ioid))) return pio_err(ios, NULL, PIO_EBADID, __FILE__, __LINE__); - /* Get the communicator size and task rank. */ - if ((mpierr = MPI_Comm_size(comm, &npes))) - return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); - if ((mpierr = MPI_Comm_rank(comm, &myrank))) - return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); - LOG((2, "npes = %d myrank = %d", npes, myrank)); + /* Allocate memory for array which will contain the length of the + * map on each task, for all computation tasks. */ + int task_maplen[ios->num_comptasks]; + LOG((3, "ios->num_comptasks = %d", ios->num_comptasks)); - /* Allocate memory for the nmaplen. On task 0, this will contain - * the length of the map on each task, for all tasks. */ - int task_maplen[npes]; - - /* Gather maplens from all tasks and fill the task_maplen array on - * all tasks. */ - if ((mpierr = MPI_Allgather(&iodesc->maplen, 1, MPI_INT, task_maplen, 1, MPI_INT, comm))) + /* Gather maplens from all computation tasks and fill the + * task_maplen array on all tasks. */ + if ((mpierr = MPI_Allgather(&iodesc->maplen, 1, MPI_INT, task_maplen, 1, MPI_INT, + ios->comp_comm))) return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); - /* We will need to know the maximum maplen used for any task. */ - int max_maplen; - /* Find the max maxplen. */ - if ((mpierr = MPI_Allreduce(&iodesc->maplen, &max_maplen, 1, MPI_INT, MPI_MAX, comm))) + if ((mpierr = MPI_Allreduce(&iodesc->maplen, &max_maplen, 1, MPI_INT, MPI_MAX, + ios->comp_comm))) return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); LOG((3, "max_maplen = %d", max_maplen)); - /* 2D array that, on task 0, will hold all the map information for - * all tasks. */ - int full_map[npes][max_maplen]; + /* 2D array that will hold all the map information for all + * tasks. */ + int full_map[ios->num_comptasks][max_maplen]; /* Fill local array with my map. Use the fill value for unused */ /* elements at the end if max_maplen is longer than maplen. Also @@ -1009,22 +1019,22 @@ int PIOc_write_nc_decomp(int iosysid, const char *filename, int cmode, int ioid, my_map[e] = e < iodesc->maplen ? iodesc->map[e] - 1 : NC_FILL_INT; LOG((3, "my_map[%d] = %d", e, my_map[e])); } - - /* Gather my_map from all tasks and fill the full_map array. */ + + /* Gather my_map from all computation tasks and fill the full_map array. */ if ((mpierr = MPI_Allgather(&my_map, max_maplen, MPI_INT, full_map, max_maplen, - MPI_INT, comm))) + MPI_INT, ios->comp_comm))) return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); - - for (int p = 0; p < npes; p++) + + for (int p = 0; p < ios->num_comptasks; p++) for (int e = 0; e < max_maplen; e++) LOG((3, "full_map[%d][%d] = %d", p, e, full_map[p][e])); /* Write the netCDF decomp file. */ - if ((ret = pioc_write_nc_decomp_int(iosysid, filename, cmode, iodesc->ndims, iodesc->dimlen, npes, - task_maplen, (int *)full_map, title, history, fortran_order))) + if ((ret = pioc_write_nc_decomp_int(ios, filename, cmode, iodesc->ndims, iodesc->dimlen, + ios->num_comptasks, task_maplen, (int *)full_map, title, + history, fortran_order))) return ret; - return PIO_NOERR; } @@ -1060,8 +1070,6 @@ int PIOc_read_nc_decomp(int iosysid, const char *filename, int *ioidp, MPI_Comm int num_tasks_decomp; /* The number of tasks for this decomp. */ int size; /* Size of comm. */ int my_rank; /* Task rank in comm. */ - MPI_Datatype mpi_type; /* Will be used as the basetype in iodesc. */ - int mpi_type_int; /* int version of mpi_type. */ char source_in[PIO_MAX_NAME + 1]; /* Text metadata in decomp file. */ char version_in[PIO_MAX_NAME + 1]; /* Text metadata in decomp file. */ int mpierr; @@ -1078,12 +1086,6 @@ int PIOc_read_nc_decomp(int iosysid, const char *filename, int *ioidp, MPI_Comm LOG((1, "PIOc_read_nc_decomp filename = %s iosysid = %d pio_type = %d", filename, iosysid, pio_type)); - /* Get the MPI type. We need it as an int. */ - if ((ret = find_mpi_type(pio_type, &mpi_type))) - return pio_err(ios, NULL, ret, __FILE__, __LINE__); - mpi_type_int = mpi_type; - LOG((2, "mpi_type = %d mpi_type_int = %d", mpi_type, mpi_type_int)); - /* Get the communicator size and task rank. */ if ((mpierr = MPI_Comm_size(comm, &size))) return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); @@ -1130,7 +1132,7 @@ int PIOc_read_nc_decomp(int iosysid, const char *filename, int *ioidp, MPI_Comm /* Write the decomp information in netCDF. This is an internal * function. * - * @param iosysid the IO system ID. + * @param ios pointer to io system info. * @param filename the name the decomp file will have. * @param cmode for PIOc_create(). Will be bitwise or'd with NC_WRITE. * @param ndims number of dims in the data being described. @@ -1151,31 +1153,22 @@ int PIOc_read_nc_decomp(int iosysid, const char *filename, int *ioidp, MPI_Comm * ordering, 0 for C array ordering. * @returns 0 for success, error code otherwise. */ -int pioc_write_nc_decomp_int(int iosysid, const char *filename, int cmode, int ndims, +int pioc_write_nc_decomp_int(iosystem_desc_t *ios, const char *filename, int cmode, int ndims, int *global_dimlen, int num_tasks, int *task_maplen, int *map, const char *title, const char *history, int fortran_order) { - iosystem_desc_t *ios; int max_maplen = 0; int ncid; int ret; - /* Get the IO system info. */ - if (!(ios = pio_get_iosystem_from_id(iosysid))) - return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__); - /* Check inputs. */ - if (!filename || !global_dimlen || !task_maplen) - return pio_err(ios, NULL, PIO_EINVAL, __FILE__, __LINE__); - if (title) - if (strlen(title) > PIO_MAX_NAME) - return pio_err(ios, NULL, PIO_EINVAL, __FILE__, __LINE__); - if (history) - if (strlen(history) > PIO_MAX_NAME) - return pio_err(ios, NULL, PIO_EINVAL, __FILE__, __LINE__); + pioassert(ios && filename && global_dimlen && task_maplen && + (!title || strlen(title) <= PIO_MAX_NAME) && + (!history || strlen(history) <= PIO_MAX_NAME), "invalid input", + __FILE__, __LINE__); - LOG((2, "pioc_write_nc_decomp_int iosysid = %d filename = %s ndims = %d num_tasks = %d", - iosysid, filename, ndims, num_tasks)); + LOG((2, "pioc_write_nc_decomp_int filename = %s ndims = %d num_tasks = %d", filename, + ndims, num_tasks)); /* Find the maximum maplen. */ for (int t = 0; t < num_tasks; t++) @@ -1184,7 +1177,7 @@ int pioc_write_nc_decomp_int(int iosysid, const char *filename, int cmode, int n LOG((3, "max_maplen = %d", max_maplen)); /* Create the netCDF decomp file. */ - if ((ret = PIOc_create(iosysid, filename, cmode | NC_WRITE, &ncid))) + if ((ret = PIOc_create(ios->iosysid, filename, cmode | NC_WRITE, &ncid))) return pio_err(ios, NULL, ret, __FILE__, __LINE__); /* Write an attribute with the version of this file. */ @@ -1407,7 +1400,7 @@ int pioc_read_nc_decomp_int(int iosysid, const char *filename, int *ndims, int * int max_maplen_in; if ((ret = PIOc_get_att_int(ncid, NC_GLOBAL, DECOMP_MAX_MAPLEN_ATT_NAME, &max_maplen_in))) return pio_err(ios, NULL, ret, __FILE__, __LINE__); - LOG((3, "max_maplen_in = %d", version_in)); + LOG((3, "max_maplen_in = %d", max_maplen_in)); if (max_maplen) *max_maplen = max_maplen_in; @@ -1737,11 +1730,11 @@ int PIOc_createfile_int(int iosysid, int *ncidp, int *iotype, const char *filena ios->io_rank == 0) file->do_io = 1; - LOG((2, "file->do_io = %d ios->async_interface = %d", file->do_io, ios->async_interface)); + LOG((2, "file->do_io = %d ios->async = %d", file->do_io, ios->async)); /* If async is in use, and this is not an IO task, bcast the * parameters. */ - if (ios->async_interface) + if (ios->async) { int msg = PIO_MSG_CREATE_FILE; size_t len = strlen(filename); @@ -1914,7 +1907,7 @@ int PIOc_openfile_retry(int iosysid, int *ncidp, int *iotype, const char *filena file->do_io = 1; /* If async is in use, and this is not an IO task, bcast the parameters. */ - if (ios->async_interface) + if (ios->async) { int msg = PIO_MSG_OPEN_FILE; size_t len = strlen(filename); @@ -2134,7 +2127,7 @@ int pioc_change_def(int ncid, int is_enddef) ios = file->iosystem; /* If async is in use, and this is not an IO task, bcast the parameters. */ - if (ios->async_interface) + if (ios->async) { if (!ios->ioproc) { @@ -2223,147 +2216,6 @@ int iotype_is_valid(int iotype) return ret; } -/** - * Internal function to compare rearranger flow control options. - * - * @param opt pointer to rearranger flow control options to compare. - * @param exp_opt pointer to rearranger flow control options with - * expected values. - * @return true if values in opt == values in exp_opt, false - * otherwise. - */ -bool cmp_rearr_comm_fc_opts(const rearr_comm_fc_opt_t *opt, - const rearr_comm_fc_opt_t *exp_opt) -{ - bool is_same = true; - - assert(opt && exp_opt); - - if (opt->enable_hs != exp_opt->enable_hs) - { - LOG((1, "Warning rearranger enable_hs = %s, expected = %s", - opt->enable_hs ? "TRUE" : "FALSE", exp_opt->enable_hs ? "TRUE" : "FALSE")); - is_same = false; - } - - if (opt->enable_isend != exp_opt->enable_isend) - { - LOG((1, "Warning rearranger enable_isend = %s, expected = %s", - opt->enable_isend ? "TRUE" : "FALSE", exp_opt->enable_isend ? "TRUE" : "FALSE")); - is_same = false; - } - - if (opt->max_pend_req != exp_opt->max_pend_req) - { - LOG((1, "Warning rearranger max_pend_req = %d, expected = %d", - opt->max_pend_req, exp_opt->max_pend_req)); - is_same = false; - } - - return is_same; -} - -/** - * Internal function to compare rearranger options. - * - * @param rearr_opts pointer to rearranger options to compare - * @param exp_rearr_opts pointer to rearranger options with the - * expected value - * @return true if values in rearr_opts == values in exp_rearr_opts - * false otherwise - */ -bool cmp_rearr_opts(const rearr_opt_t *rearr_opts, const rearr_opt_t *exp_rearr_opts) -{ - bool is_same = true; - - assert(rearr_opts && exp_rearr_opts); - - if (rearr_opts->comm_type != exp_rearr_opts->comm_type) - { - LOG((1, "Warning rearranger comm_type = %d, expected = %d. ", rearr_opts->comm_type, - exp_rearr_opts->comm_type)); - is_same = false; - } - - if (rearr_opts->fcd != exp_rearr_opts->fcd) - { - LOG((1, "Warning rearranger fcd = %d, expected = %d. ", rearr_opts->fcd, - exp_rearr_opts->fcd)); - is_same = false; - } - - is_same = is_same && cmp_rearr_comm_fc_opts(&(rearr_opts->comm_fc_opts_comp2io), - &(exp_rearr_opts->comm_fc_opts_comp2io)); - is_same = is_same && cmp_rearr_comm_fc_opts(&(rearr_opts->comm_fc_opts_io2comp), - &(exp_rearr_opts->comm_fc_opts_io2comp)); - - return is_same; -} - -/** - * Internal function to reset rearranger opts in iosystem to valid values. - * The old default for max pending requests was DEF_P2P_MAXREQ = 64. - * - * @param ios pointer to iosystem descriptor - */ -void check_and_reset_rearr_opts(iosystem_desc_t *ios) -{ - /* Disable handshake/isend and set max_pend_req to unlimited */ - const rearr_comm_fc_opt_t def_comm_nofc_opts = - { false, false, PIO_REARR_COMM_UNLIMITED_PEND_REQ }; - /* Disable handshake /isend and set max_pend_req = 0 to turn off throttling */ - const rearr_comm_fc_opt_t def_coll_comm_fc_opts = { false, false, 0 }; - const rearr_opt_t def_coll_rearr_opts = { - PIO_REARR_COMM_COLL, - PIO_REARR_COMM_FC_2D_DISABLE, - def_coll_comm_fc_opts, - def_coll_comm_fc_opts - }; - - assert(ios); - - /* Reset to defaults, if needed (user did not set it correctly) */ - if (ios->rearr_opts.comm_type == PIO_REARR_COMM_COLL) - { - /* Compare and log the user and default rearr opts for coll. */ - cmp_rearr_opts(&(ios->rearr_opts), &def_coll_rearr_opts); - /* Hard reset flow control options. */ - ios->rearr_opts = def_coll_rearr_opts; - } - else if (ios->rearr_opts.comm_type == PIO_REARR_COMM_P2P) - { - if (ios->rearr_opts.fcd == PIO_REARR_COMM_FC_2D_DISABLE) - { - /* Compare and log user and default opts. */ - cmp_rearr_comm_fc_opts(&(ios->rearr_opts.comm_fc_opts_comp2io), - &def_comm_nofc_opts); - cmp_rearr_comm_fc_opts(&(ios->rearr_opts.comm_fc_opts_io2comp), - &def_comm_nofc_opts); - /* Hard reset flow control opts to defaults. */ - ios->rearr_opts.comm_fc_opts_comp2io = def_comm_nofc_opts; - ios->rearr_opts.comm_fc_opts_io2comp = def_comm_nofc_opts; - } - else if (ios->rearr_opts.fcd == PIO_REARR_COMM_FC_1D_COMP2IO) - { - /* Compare and log user and default opts. */ - cmp_rearr_comm_fc_opts(&(ios->rearr_opts.comm_fc_opts_io2comp), - &def_comm_nofc_opts); - /* Hard reset io2comp dir to defaults. */ - ios->rearr_opts.comm_fc_opts_io2comp = def_comm_nofc_opts; - } - else if (ios->rearr_opts.fcd == PIO_REARR_COMM_FC_1D_IO2COMP) - { - /* Compare and log user and default opts. */ - cmp_rearr_comm_fc_opts(&(ios->rearr_opts.comm_fc_opts_comp2io), - &def_comm_nofc_opts); - /* Hard reset comp2io dir to defaults. */ - ios->rearr_opts.comm_fc_opts_comp2io = def_comm_nofc_opts; - } - /* Don't reset if flow control is enabled in both directions - * by user. */ - } -} - /** * Set the rearranger options associated with an iosystem * @@ -2404,21 +2256,25 @@ int PIOc_set_rearr_opts(int iosysid, int comm_type, int fcd, bool enable_hs_c2i, int max_pend_req_i2c) { iosystem_desc_t *ios; - int ret = PIO_NOERR; rearr_opt_t user_rearr_opts = { comm_type, fcd, {enable_hs_c2i,enable_isend_c2i, max_pend_req_c2i}, {enable_hs_i2c, enable_isend_i2c, max_pend_req_i2c} }; + /* Check inputs. */ + if ((comm_type != PIO_REARR_COMM_P2P && comm_type != PIO_REARR_COMM_FC_1D_COMP2IO) || + (fcd < 0 || fcd > PIO_REARR_COMM_FC_2D_DISABLE) || + (max_pend_req_c2i != PIO_REARR_COMM_UNLIMITED_PEND_REQ && max_pend_req_c2i < 0) || + (max_pend_req_i2c != PIO_REARR_COMM_UNLIMITED_PEND_REQ && max_pend_req_i2c < 0)) + return pio_err(NULL, NULL, PIO_EINVAL, __FILE__, __LINE__); + /* Get the IO system info. */ if (!(ios = pio_get_iosystem_from_id(iosysid))) return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__); + /* Set the options. */ ios->rearr_opts = user_rearr_opts; - /* Perform sanity checks on the user supplied values */ - check_and_reset_rearr_opts(ios); - - return ret; + return PIO_NOERR; } diff --git a/src/externals/pio2/tests/cunit/CMakeLists.txt b/src/externals/pio2/tests/cunit/CMakeLists.txt index 1ef8be7afc3..418340b7916 100644 --- a/src/externals/pio2/tests/cunit/CMakeLists.txt +++ b/src/externals/pio2/tests/cunit/CMakeLists.txt @@ -65,33 +65,45 @@ if (NOT PIO_USE_MPISERIAL) target_link_libraries (test_pioc_fill pioc) add_executable (test_darray EXCLUDE_FROM_ALL test_darray.c test_common.c) target_link_libraries (test_darray pioc) + add_executable (test_darray_multi EXCLUDE_FROM_ALL test_darray_multi.c test_common.c) + target_link_libraries (test_darray_multi pioc) add_executable (test_darray_multivar EXCLUDE_FROM_ALL test_darray_multivar.c test_common.c) target_link_libraries (test_darray_multivar pioc) + add_executable (test_darray_multivar2 EXCLUDE_FROM_ALL test_darray_multivar2.c test_common.c) + target_link_libraries (test_darray_multivar2 pioc) add_executable (test_darray_1d EXCLUDE_FROM_ALL test_darray_1d.c test_common.c) target_link_libraries (test_darray_1d pioc) add_executable (test_darray_3d EXCLUDE_FROM_ALL test_darray_3d.c test_common.c) target_link_libraries (test_darray_3d pioc) + add_executable (test_decomp_uneven EXCLUDE_FROM_ALL test_decomp_uneven.c test_common.c) + target_link_libraries (test_decomp_uneven pioc) add_executable (test_decomps EXCLUDE_FROM_ALL test_decomps.c test_common.c) target_link_libraries (test_decomps pioc) + add_executable (test_rearr EXCLUDE_FROM_ALL test_rearr.c test_common.c) + target_link_libraries (test_rearr pioc) endif () add_executable (test_spmd EXCLUDE_FROM_ALL test_spmd.c test_common.c) target_link_libraries (test_spmd pioc) add_dependencies (tests test_spmd) +add_dependencies (tests test_rearr) add_dependencies (tests test_pioc) add_dependencies (tests test_pioc_unlim) add_dependencies (tests test_pioc_putget) add_dependencies (tests test_pioc_fill) add_dependencies (tests test_darray) +add_dependencies (tests test_darray_multi) add_dependencies (tests test_darray_multivar) +add_dependencies (tests test_darray_multivar2) add_dependencies (tests test_darray_1d) add_dependencies (tests test_darray_3d) +add_dependencies (tests test_decomp_uneven) add_dependencies (tests test_decomps) # Test Timeout in seconds. if (PIO_VALGRIND_CHECK) - set (DEFAULT_TEST_TIMEOUT 120) + set (DEFAULT_TEST_TIMEOUT 240) else () - set (DEFAULT_TEST_TIMEOUT 60) + set (DEFAULT_TEST_TIMEOUT 120) endif () # All tests need a certain number of tasks, but they should be able to @@ -109,6 +121,10 @@ else () EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_spmd NUMPROCS ${AT_LEAST_FOUR_TASKS} TIMEOUT ${DEFAULT_TEST_TIMEOUT}) + add_mpi_test(test_rearr + EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_rearr + NUMPROCS ${AT_LEAST_FOUR_TASKS} + TIMEOUT ${DEFAULT_TEST_TIMEOUT}) add_mpi_test(test_intercomm2 EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_intercomm2 NUMPROCS ${AT_LEAST_FOUR_TASKS} @@ -169,10 +185,18 @@ else () EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_darray NUMPROCS ${AT_LEAST_FOUR_TASKS} TIMEOUT ${DEFAULT_TEST_TIMEOUT}) + add_mpi_test(test_darray_multi + EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_darray_multi + NUMPROCS ${AT_LEAST_FOUR_TASKS} + TIMEOUT ${DEFAULT_TEST_TIMEOUT}) add_mpi_test(test_darray_multivar EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_darray_multivar NUMPROCS ${AT_LEAST_FOUR_TASKS} TIMEOUT ${DEFAULT_TEST_TIMEOUT}) + add_mpi_test(test_darray_multivar2 + EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_darray_multivar2 + NUMPROCS ${AT_LEAST_FOUR_TASKS} + TIMEOUT ${DEFAULT_TEST_TIMEOUT}) add_mpi_test(test_darray_1d EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_darray_1d NUMPROCS ${AT_LEAST_FOUR_TASKS} @@ -181,6 +205,10 @@ else () EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_darray_3d NUMPROCS ${AT_LEAST_FOUR_TASKS} TIMEOUT ${DEFAULT_TEST_TIMEOUT}) + add_mpi_test(test_decomp_uneven + EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_decomp_uneven + NUMPROCS ${AT_LEAST_FOUR_TASKS} + TIMEOUT ${DEFAULT_TEST_TIMEOUT}) add_mpi_test(test_decomps EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_decomps NUMPROCS ${AT_LEAST_FOUR_TASKS} diff --git a/src/externals/pio2/tests/cunit/pio_tests.h b/src/externals/pio2/tests/cunit/pio_tests.h index 86ac35af547..5a601ea6df1 100644 --- a/src/externals/pio2/tests/cunit/pio_tests.h +++ b/src/externals/pio2/tests/cunit/pio_tests.h @@ -29,6 +29,9 @@ /* Number of NetCDF-4 types. */ #define NUM_NETCDF4_TYPES 12 +/* Number of PIO rearrangers. */ +#define NUM_REARRANGERS 2 + /* Number of sample files constructed for these tests. */ #define NUM_SAMPLES 3 diff --git a/src/externals/pio2/tests/cunit/test_async_2comp.c b/src/externals/pio2/tests/cunit/test_async_2comp.c deleted file mode 100644 index 3c79cb6101e..00000000000 --- a/src/externals/pio2/tests/cunit/test_async_2comp.c +++ /dev/null @@ -1,132 +0,0 @@ -/** - * @file Tests for PIOc_Intercomm. This tests basic asynch I/O capability. - * @author Ed Hartnett - * - * To run with valgrind, use this command: - * 
mpiexec -n 4 valgrind -v --leak-check=full --suppressions=../../../tests/unit/valsupp_test.supp
- * --error-exitcode=99 --track-origins=yes ./test_intercomm3
- * - */ -#include -#include - -/* Number of processors that will do IO. */ -#define NUM_IO_PROCS 2 - -/* Number of computational components to create. */ -#define COMPONENT_COUNT 2 - -/* The number of tasks this test should run on. */ -#define TARGET_NTASKS 4 - -/* The name of this test. */ -#define TEST_NAME "test_intercomm3" - -/** Run Tests for Init_Intercomm - * - */ -int -main(int argc, char **argv) -{ - int verbose = 1; - - /* Zero-based rank of processor. */ - int my_rank; - - /* Number of processors involved in current execution. */ - int ntasks; - - /* Different output flavors. */ - int flavor[NUM_FLAVORS]; - - int num_flavors; - - /* The ID for the parallel I/O system. */ - int iosysid[COMPONENT_COUNT]; - - /* The ncid of the netCDF file. */ - int ncid; - - /* The ID of the netCDF varable. */ - int varid; - - /* Return code. */ - int ret; - - /* Index for loops. */ - int fmt, d, d1, i; - - /* Initialize test. */ - if ((ret = pio_test_init(argc, argv, &my_rank, &ntasks, TARGET_NTASKS))) - ERR(ERR_INIT); - - /* Figure out iotypes. */ - if ((ret = get_iotypes(&num_flavors, flavor))) - ERR(ret); - - /* How many processors will be used for our IO and 2 computation components. */ - int num_procs[COMPONENT_COUNT + 1] = {2, 1, 1}; - - /* Is the current process a computation task? */ - int comp_task = my_rank < 2 ? 0 : 1; - - /* Index of computation task in iosysid array. Varies by rank and - * does not apply to IO component processes. */ - int my_comp_idx = comp_task ? my_rank - 2 : -1; - - /* Initialize the IO system. */ - if ((ret = PIOc_Init_Async(MPI_COMM_WORLD, NUM_IO_PROCS, NULL, COMPONENT_COUNT, - num_procs, NULL, iosysid))) - ERR(ERR_AWFUL); - - /* All the netCDF calls are only executed on the computation - * tasks. The IO tasks have not returned from PIOc_Init_Intercomm, - * and when the do, they should go straight to finalize. */ - if (comp_task) - { - for (int flv = 0; flv < num_flavors; flv++) - { - char filename[NC_MAX_NAME + 1]; - - /* Create a filename. */ - int sample = 1; - sprintf(filename, "%s_%s_%d_%d.nc", TEST_NAME, flavor_name(flv), sample, my_comp_idx); - - /* Create sample file 1. */ - printf("%d %s creating file %s\n", my_rank, TEST_NAME, filename); - if ((ret = create_nc_sample_1(iosysid[my_comp_idx], flavor[flv], filename, my_rank, NULL))) - ERR(ret); - - /* Check the file for correctness. */ - if ((ret = check_nc_sample_1(iosysid[my_comp_idx], flavor[flv], filename, my_rank, NULL))) - ERR(ret); - - } /* next netcdf format flavor */ - - /* If I don't sleep here for a second, there are problems. */ - sleep(2); - - /* Finalize the IO system. Only call this from the computation tasks. */ - if (verbose) - printf("%d test_intercomm3 Freeing PIO resources\n", my_rank); - for (int c = 0; c < COMPONENT_COUNT; c++) - { - if ((ret = PIOc_finalize(iosysid[c]))) - ERR(ret); - printf("%d test_intercomm3 PIOc_finalize completed for iosysid = %d\n", my_rank, iosysid[c]); - } - } /* endif comp_task */ - - /* Wait for everyone to catch up. */ - printf("%d %s waiting for all processes!\n", my_rank, TEST_NAME); - MPI_Barrier(MPI_COMM_WORLD); - - /* Finalize the MPI library. */ - printf("%d %s Finalizing...\n", my_rank, TEST_NAME); - if ((ret = pio_test_finalize(&test_comm))) - return ERR_AWFUL; - - printf("%d %s SUCCESS!!\n", my_rank, TEST_NAME); - - return 0; -} diff --git a/src/externals/pio2/tests/cunit/test_async_3proc.c b/src/externals/pio2/tests/cunit/test_async_3proc.c index 020ed78172c..4a2f8435fe2 100644 --- a/src/externals/pio2/tests/cunit/test_async_3proc.c +++ b/src/externals/pio2/tests/cunit/test_async_3proc.c @@ -39,7 +39,7 @@ int main(int argc, char **argv) MPI_Comm test_comm; /* comm for test */ /* Num procs for IO and computation. */ - int num_procs[NUM_COMBOS][COMPONENT_COUNT + 1] = {{2, 1}, {1, 2}}; + int num_procs[NUM_COMBOS][COMPONENT_COUNT] = {{1}, {2}}; /* Number of processors that will do IO. */ int num_io_procs[NUM_COMBOS] = {2, 1}; @@ -63,8 +63,8 @@ int main(int argc, char **argv) int comp_task = my_rank < num_io_procs[combo] ? 0 : 1; /* Initialize the IO system. */ - if ((ret = PIOc_Init_Async(test_comm, num_io_procs[combo], NULL, COMPONENT_COUNT, - num_procs[combo], NULL, NULL, NULL, iosysid))) + if ((ret = PIOc_init_async(test_comm, num_io_procs[combo], NULL, COMPONENT_COUNT, + num_procs[combo], NULL, NULL, NULL, PIO_REARR_BOX, iosysid))) ERR(ERR_INIT); for (int c = 0; c < COMPONENT_COUNT; c++) diff --git a/src/externals/pio2/tests/cunit/test_async_4proc.c b/src/externals/pio2/tests/cunit/test_async_4proc.c index 4f91b0c0eec..db0197ab9be 100644 --- a/src/externals/pio2/tests/cunit/test_async_4proc.c +++ b/src/externals/pio2/tests/cunit/test_async_4proc.c @@ -32,15 +32,14 @@ int main(int argc, char **argv) int ret; /* Return code. */ MPI_Comm test_comm; - /* Num procs for IO and computation. */ - int num_procs[NUM_COMBOS][COMPONENT_COUNT + 1] = {{3, 1}, {2, 2}, {1, 3}}; + /* Num procs for computation. */ + int num_procs2[NUM_COMBOS][COMPONENT_COUNT] = {{1}, {2}, {3}}; /* Number of processors that will do IO. */ int num_io_procs[NUM_COMBOS] = {3, 2, 1}; /* Initialize test. */ - if ((ret = pio_test_init(argc, argv, &my_rank, &ntasks, TARGET_NTASKS, - &test_comm))) + if ((ret = pio_test_init(argc, argv, &my_rank, &ntasks, TARGET_NTASKS, &test_comm))) ERR(ERR_INIT); /* Test code runs on TARGET_NTASKS tasks. The left over tasks do @@ -57,8 +56,8 @@ int main(int argc, char **argv) int comp_task = my_rank < num_io_procs[combo] ? 0 : 1; /* Initialize the IO system. */ - if ((ret = PIOc_Init_Async(test_comm, num_io_procs[combo], NULL, COMPONENT_COUNT, - num_procs[combo], NULL, NULL, NULL, iosysid))) + if ((ret = PIOc_init_async(test_comm, num_io_procs[combo], NULL, COMPONENT_COUNT, + num_procs2[combo], NULL, NULL, NULL, PIO_REARR_BOX, iosysid))) ERR(ERR_INIT); for (int c = 0; c < COMPONENT_COUNT; c++) diff --git a/src/externals/pio2/tests/cunit/test_async_simple.c b/src/externals/pio2/tests/cunit/test_async_simple.c index a0852b3e712..4b4836bcb02 100644 --- a/src/externals/pio2/tests/cunit/test_async_simple.c +++ b/src/externals/pio2/tests/cunit/test_async_simple.c @@ -29,13 +29,18 @@ /* Run simple async test. */ int main(int argc, char **argv) { +#define NUM_IO_PROCS 1 +#define NUM_COMP_PROCS 1 int my_rank; /* Zero-based rank of processor. */ int ntasks; /* Number of processors involved in current execution. */ int iosysid[COMPONENT_COUNT]; /* The ID for the parallel I/O system. */ int num_flavors; /* Number of PIO netCDF flavors in this build. */ int flavor[NUM_FLAVORS]; /* iotypes for the supported netCDF IO flavors. */ int ret; /* Return code. */ - int num_procs[COMPONENT_COUNT + 1] = {1, 1}; /* Num procs for IO and computation. */ + int num_procs[COMPONENT_COUNT] = {1}; /* Num procs for IO and computation. */ + int io_proc_list[NUM_IO_PROCS] = {0}; + int comp_proc_list[NUM_COMP_PROCS] = {1}; + int *proc_list[COMPONENT_COUNT] = {comp_proc_list}; MPI_Comm test_comm; /* Initialize test. */ @@ -54,19 +59,22 @@ int main(int argc, char **argv) int comp_task = my_rank < NUM_IO_PROCS ? 0 : 1; /* Check for invalid values. */ - if (PIOc_Init_Async(test_comm, NUM_IO_PROCS, NULL, COMPONENT_COUNT, - num_procs, NULL, NULL, NULL, NULL) != PIO_EINVAL) + if (PIOc_init_async(test_comm, NUM_IO_PROCS, NULL, COMPONENT_COUNT, + num_procs, NULL, NULL, NULL, PIO_REARR_BOX, NULL) != PIO_EINVAL) + ERR(ERR_WRONG); + if (PIOc_init_async(test_comm, NUM_IO_PROCS, NULL, COMPONENT_COUNT, + num_procs, NULL, NULL, NULL, TEST_VAL_42, iosysid) != PIO_EINVAL) ERR(ERR_WRONG); - if (PIOc_Init_Async(test_comm, NUM_IO_PROCS, NULL, -1, - num_procs, NULL, NULL, NULL, iosysid) != PIO_EINVAL) + if (PIOc_init_async(test_comm, NUM_IO_PROCS, NULL, -1, + num_procs, NULL, NULL, NULL, PIO_REARR_BOX, iosysid) != PIO_EINVAL) ERR(ERR_WRONG); - if (PIOc_Init_Async(test_comm, NUM_IO_PROCS, NULL, COMPONENT_COUNT, - NULL, NULL, NULL, NULL, iosysid) != PIO_EINVAL) + if (PIOc_init_async(test_comm, NUM_IO_PROCS, NULL, COMPONENT_COUNT, + NULL, NULL, NULL, NULL, PIO_REARR_BOX, iosysid) != PIO_EINVAL) ERR(ERR_WRONG); /* Initialize the IO system. */ - if ((ret = PIOc_Init_Async(test_comm, NUM_IO_PROCS, NULL, COMPONENT_COUNT, - num_procs, NULL, NULL, NULL, iosysid))) + if ((ret = PIOc_init_async(test_comm, NUM_IO_PROCS, io_proc_list, COMPONENT_COUNT, + num_procs, (int **)proc_list, NULL, NULL, PIO_REARR_BOX, iosysid))) ERR(ERR_INIT); /* All the netCDF calls are only executed on the computation diff --git a/src/externals/pio2/tests/cunit/test_darray_1d.c b/src/externals/pio2/tests/cunit/test_darray_1d.c index 01b894a8749..c5c6ac85012 100644 --- a/src/externals/pio2/tests/cunit/test_darray_1d.c +++ b/src/externals/pio2/tests/cunit/test_darray_1d.c @@ -107,18 +107,52 @@ int test_darray_fill(int iosysid, int ioid, int pio_type, int num_flavors, int * void *test_data_in; void *expected_in; PIO_Offset type_size; /* Size of the data type. */ - float my_float_rank = my_rank; /* my_rank in a float. */ - double my_double_rank = my_rank; /* my_rank in a double. */ + /* My rank as each type. */ + signed char my_byte_rank = my_rank; + char my_char_rank = my_rank; + short my_short_rank = my_rank; + float my_float_rank = my_rank; + double my_double_rank = my_rank; +#ifdef _NETCDF4 + unsigned char my_ubyte_rank = my_rank; + unsigned short my_ushort_rank = my_rank; + unsigned int my_uint_rank = my_rank; + long long my_int64_rank = my_rank; + unsigned long long my_uint64_rank = my_rank; +#endif /* _NETCDF4 */ + + /* Default fill value for each type. */ + signed char byte_fill = NC_FILL_BYTE; + char char_fill = NC_FILL_CHAR; + short short_fill = NC_FILL_SHORT; int int_fill = NC_FILL_INT; float float_fill = NC_FILL_FLOAT; double double_fill = NC_FILL_DOUBLE; +#ifdef _NETCDF4 + unsigned char ubyte_fill = NC_FILL_UBYTE; + unsigned short ushort_fill = NC_FILL_USHORT; + unsigned int uint_fill = NC_FILL_UINT; + long long int64_fill = NC_FILL_INT64; + unsigned long long uint64_fill = NC_FILL_UINT64; +#endif /* _NETCDF4 */ + void *bufr; - int ret; /* Return code. */ + int ret; /* Return code. */ /* Use PIO to create the example file in each of the four * available ways. */ for (int fmt = 0; fmt < num_flavors; fmt++) { + /* BYTE and CHAR don't work with pnetcdf. Don't know why yet. */ + if (flavor[fmt] == PIO_IOTYPE_PNETCDF && (pio_type == PIO_BYTE || pio_type == PIO_CHAR)) + continue; + + /* NetCDF-4 types only work with netCDF-4 formats. */ + printf("pio_type = %d flavor[fmt] = %d\n", pio_type, flavor[fmt]); + if (pio_type > PIO_DOUBLE && flavor[fmt] != PIO_IOTYPE_NETCDF4C && + flavor[fmt] != PIO_IOTYPE_NETCDF4P) + continue; + for (int with_fillvalue = 0; with_fillvalue < NUM_FILLVALUE_PRESENT_TESTS; with_fillvalue++) { /* Create the filename. */ @@ -152,11 +186,37 @@ int test_darray_fill(int iosysid, int ioid, int pio_type, int num_flavors, int * return ret; /* Initialize some data. */ + signed char byte_test_data[2] = {my_rank, my_rank}; + char char_test_data[2] = {my_rank, my_rank}; + short short_test_data[2] = {my_rank, my_rank}; int int_test_data[2] = {my_rank, my_rank}; float float_test_data[2] = {my_rank, my_rank}; double double_test_data[2] = {my_rank, my_rank}; +#ifdef _NETCDF4 + unsigned char ubyte_test_data[2] = {my_rank, my_rank}; + unsigned short ushort_test_data[2] = {my_rank, my_rank}; + unsigned int uint_test_data[2] = {my_rank, my_rank}; + long long int64_test_data[2] = {my_rank, my_rank}; + unsigned long long uint64_test_data[2] = {my_rank, my_rank}; +#endif /* _NETCDF4 */ + switch (pio_type) { + case PIO_BYTE: + test_data = byte_test_data; + fillvalue = with_fillvalue ? &byte_fill : NULL; + expected_in = &my_byte_rank; + break; + case PIO_CHAR: + test_data = char_test_data; + fillvalue = with_fillvalue ? &char_fill : NULL; + expected_in = &my_char_rank; + break; + case PIO_SHORT: + test_data = short_test_data; + fillvalue = with_fillvalue ? &short_fill : NULL; + expected_in = &my_short_rank; + break; case PIO_INT: test_data = int_test_data; fillvalue = with_fillvalue ? &int_fill : NULL; @@ -172,16 +232,42 @@ int test_darray_fill(int iosysid, int ioid, int pio_type, int num_flavors, int * fillvalue = with_fillvalue ? &double_fill : NULL; expected_in = &my_double_rank; break; +#ifdef _NETCDF4 + case PIO_UBYTE: + test_data = ubyte_test_data; + fillvalue = with_fillvalue ? &ubyte_fill : NULL; + expected_in = &my_ubyte_rank; + break; + case PIO_USHORT: + test_data = ushort_test_data; + fillvalue = with_fillvalue ? &ushort_fill : NULL; + expected_in = &my_ushort_rank; + break; + case PIO_UINT: + test_data = uint_test_data; + fillvalue = with_fillvalue ? &uint_fill : NULL; + expected_in = &my_uint_rank; + break; + case PIO_INT64: + test_data = int64_test_data; + fillvalue = with_fillvalue ? &int64_fill : NULL; + expected_in = &my_int64_rank; + break; + case PIO_UINT64: + test_data = uint64_test_data; + fillvalue = with_fillvalue ? &uint64_fill : NULL; + expected_in = &my_uint64_rank; + break; +#endif /* _NETCDF4 */ default: return ERR_WRONG; } - /* Write the data. Our test_data contains only one real value - * (instead of 2, as indicated by arraylen), but due to the - * decomposition, only the first value is used in the - * output. */ - if ((ret = PIOc_write_darray(ncid, varid, ioid, arraylen, test_data, - fillvalue))) + /* Write the data. Our test_data contains only one real + * value (instead of 2, as indicated by arraylen), but due + * to the decomposition, only the first value is used in + * the output. */ + if ((ret = PIOc_write_darray(ncid, varid, ioid, arraylen, test_data, fillvalue))) ERR(ret); /* Close the netCDF file. */ @@ -221,6 +307,18 @@ int test_darray_fill(int iosysid, int ioid, int pio_type, int num_flavors, int * { switch (pio_type) { + case PIO_BYTE: + if (((signed char *)bufr)[e] != (e < 4 ? e : NC_FILL_BYTE)) + return ERR_WRONG; + break; + case PIO_CHAR: + if (((char *)bufr)[e] != (e < 4 ? e : NC_FILL_CHAR)) + return ERR_WRONG; + break; + case PIO_SHORT: + if (((short *)bufr)[e] != (e < 4 ? e : NC_FILL_SHORT)) + return ERR_WRONG; + break; case PIO_INT: if (((int *)bufr)[e] != (e < 4 ? e : NC_FILL_INT)) return ERR_WRONG; @@ -233,6 +331,28 @@ int test_darray_fill(int iosysid, int ioid, int pio_type, int num_flavors, int * if (((double *)bufr)[e] != (e < 4 ? e : NC_FILL_DOUBLE)) return ERR_WRONG; break; +#ifdef _NETCDF4 + case PIO_UBYTE: + if (((unsigned char *)bufr)[e] != (e < 4 ? e : NC_FILL_UBYTE)) + return ERR_WRONG; + break; + case PIO_USHORT: + if (((unsigned short *)bufr)[e] != (e < 4 ? e : NC_FILL_USHORT)) + return ERR_WRONG; + break; + case PIO_UINT: + if (((unsigned int *)bufr)[e] != (e < 4 ? e : NC_FILL_UINT)) + return ERR_WRONG; + break; + case PIO_INT64: + if (((long long *)bufr)[e] != (e < 4 ? e : NC_FILL_INT64)) + return ERR_WRONG; + break; + case PIO_UINT64: + if (((unsigned long long *)bufr)[e] != (e < 4 ? e : NC_FILL_UINT64)) + return ERR_WRONG; + break; +#endif /* _NETCDF4 */ default: return ERR_WRONG; } @@ -277,11 +397,35 @@ int test_darray_fill_unlim(int iosysid, int ioid, int pio_type, int num_flavors, void *test_data_in; void *expected_in; PIO_Offset type_size; /* Size of the data type. */ - float my_float_rank = my_rank; /* my_rank in a float. */ - double my_double_rank = my_rank; /* my_rank in a double. */ + + /* My rank as each type. */ + signed char my_byte_rank = my_rank; + char my_char_rank = my_rank; + short my_short_rank = my_rank; + float my_float_rank = my_rank; + double my_double_rank = my_rank; +#ifdef _NETCDF4 + unsigned char my_ubyte_rank = my_rank; + unsigned short my_ushort_rank = my_rank; + unsigned int my_uint_rank = my_rank; + long long my_int64_rank = my_rank; + unsigned long long my_uint64_rank = my_rank; +#endif /* _NETCDF4 */ + + /* Default fill value for each type. */ + signed char byte_fill = NC_FILL_BYTE; + char char_fill = NC_FILL_CHAR; + short short_fill = NC_FILL_SHORT; int int_fill = NC_FILL_INT; float float_fill = NC_FILL_FLOAT; double double_fill = NC_FILL_DOUBLE; +#ifdef _NETCDF4 + unsigned char ubyte_fill = NC_FILL_UBYTE; + unsigned short ushort_fill = NC_FILL_USHORT; + unsigned int uint_fill = NC_FILL_UINT; + long long int64_fill = NC_FILL_INT64; + unsigned long long uint64_fill = NC_FILL_UINT64; +#endif /* _NETCDF4 */ void *bufr; int ret; /* Return code. */ @@ -289,6 +433,16 @@ int test_darray_fill_unlim(int iosysid, int ioid, int pio_type, int num_flavors, * available ways. */ for (int fmt = 0; fmt < num_flavors; fmt++) { + /* BYTE and CHAR don't work with pnetcdf. Don't know why yet. */ + if (flavor[fmt] == PIO_IOTYPE_PNETCDF && (pio_type == PIO_BYTE || pio_type == PIO_CHAR)) + continue; + + /* NetCDF-4 types only work with netCDF-4 formats. */ + printf("pio_type = %d flavor[fmt] = %d\n", pio_type, flavor[fmt]); + if (pio_type > PIO_DOUBLE && flavor[fmt] != PIO_IOTYPE_NETCDF4C && + flavor[fmt] != PIO_IOTYPE_NETCDF4P) + continue; + /* Create the filename. */ sprintf(filename, "data_%s_iotype_%d_pio_type_%d_unlim.nc", TEST_NAME, flavor[fmt], pio_type); @@ -322,11 +476,36 @@ int test_darray_fill_unlim(int iosysid, int ioid, int pio_type, int num_flavors, return ret; /* Initialize some data. */ + signed char byte_test_data[2] = {my_rank, my_rank}; + char char_test_data[2] = {my_rank, my_rank}; + short short_test_data[2] = {my_rank, my_rank}; int int_test_data[2] = {my_rank, my_rank}; float float_test_data[2] = {my_rank, my_rank}; double double_test_data[2] = {my_rank, my_rank}; +#ifdef _NETCDF4 + unsigned char ubyte_test_data[2] = {my_rank, my_rank}; + unsigned short ushort_test_data[2] = {my_rank, my_rank}; + unsigned int uint_test_data[2] = {my_rank, my_rank}; + long long int64_test_data[2] = {my_rank, my_rank}; + unsigned long long uint64_test_data[2] = {my_rank, my_rank}; +#endif /* _NETCDF4 */ switch (pio_type) { + case PIO_BYTE: + test_data = byte_test_data; + fillvalue = &byte_fill; + expected_in = &my_byte_rank; + break; + case PIO_CHAR: + test_data = char_test_data; + fillvalue = &char_fill; + expected_in = &my_char_rank; + break; + case PIO_SHORT: + test_data = short_test_data; + fillvalue = &short_fill; + expected_in = &my_short_rank; + break; case PIO_INT: test_data = int_test_data; fillvalue = &int_fill; @@ -342,6 +521,33 @@ int test_darray_fill_unlim(int iosysid, int ioid, int pio_type, int num_flavors, fillvalue = &double_fill; expected_in = &my_double_rank; break; +#ifdef _NETCDF4 + case PIO_UBYTE: + test_data = ubyte_test_data; + fillvalue = &ubyte_fill; + expected_in = &my_ubyte_rank; + break; + case PIO_USHORT: + test_data = ushort_test_data; + fillvalue = &ushort_fill; + expected_in = &my_ushort_rank; + break; + case PIO_UINT: + test_data = uint_test_data; + fillvalue = &uint_fill; + expected_in = &my_uint_rank; + break; + case PIO_INT64: + test_data = int64_test_data; + fillvalue = &int64_fill; + expected_in = &my_int64_rank; + break; + case PIO_UINT64: + test_data = uint64_test_data; + fillvalue = &uint64_fill; + expected_in = &my_uint64_rank; + break; +#endif /* _NETCDF4 */ default: return ERR_WRONG; } @@ -406,6 +612,18 @@ int test_darray_fill_unlim(int iosysid, int ioid, int pio_type, int num_flavors, { switch (pio_type) { + case PIO_BYTE: + if (((signed char *)bufr)[e] != (e % 8 < 4 ? e % 8 : NC_FILL_BYTE)) + return ERR_WRONG; + break; + case PIO_CHAR: + if (((char *)bufr)[e] != (e % 8 < 4 ? e % 8 : NC_FILL_CHAR)) + return ERR_WRONG; + break; + case PIO_SHORT: + if (((short *)bufr)[e] != (e % 8 < 4 ? e % 8 : NC_FILL_SHORT)) + return ERR_WRONG; + break; case PIO_INT: if (((int *)bufr)[e] != (e % 8 < 4 ? e % 8 : NC_FILL_INT)) return ERR_WRONG; @@ -418,6 +636,28 @@ int test_darray_fill_unlim(int iosysid, int ioid, int pio_type, int num_flavors, if (((double *)bufr)[e] != (e % 8 < 4 ? e % 8 : NC_FILL_DOUBLE)) return ERR_WRONG; break; +#ifdef _NETCDF4 + case PIO_UBYTE: + if (((unsigned char *)bufr)[e] != (e % 8 < 4 ? e % 8 : NC_FILL_UBYTE)) + return ERR_WRONG; + break; + case PIO_USHORT: + if (((unsigned short *)bufr)[e] != (e % 8 < 4 ? e % 8 : NC_FILL_USHORT)) + return ERR_WRONG; + break; + case PIO_UINT: + if (((unsigned int *)bufr)[e] != (e % 8 < 4 ? e % 8 : NC_FILL_UINT)) + return ERR_WRONG; + break; + case PIO_INT64: + if (((long long *)bufr)[e] != (e % 8 < 4 ? e % 8 : NC_FILL_INT64)) + return ERR_WRONG; + break; + case PIO_UINT64: + if (((unsigned long long *)bufr)[e] != (e % 8 < 4 ? e % 8 : NC_FILL_UINT64)) + return ERR_WRONG; + break; +#endif /* _NETCDF4 */ default: return ERR_WRONG; } @@ -466,8 +706,7 @@ int test_decomp_read_write(int iosysid, int ioid, int num_flavors, int *flavor, sprintf(filename, "decomp_%s_iotype_%d.nc", TEST_NAME, flavor[fmt]); printf("writing decomp file %s\n", filename); - if ((ret = PIOc_write_nc_decomp(iosysid, filename, 0, ioid, test_comm, NULL, - NULL, 0))) + if ((ret = PIOc_write_nc_decomp(iosysid, filename, 0, ioid, NULL, NULL, 0))) return ret; /* Read the data. */ @@ -480,10 +719,19 @@ int test_decomp_read_write(int iosysid, int ioid, int num_flavors, int *flavor, { iosystem_desc_t *ios; io_desc_t *iodesc; - int expected_basetype; + MPI_Datatype expected_basetype; switch (pio_type) { + case PIO_BYTE: + expected_basetype = MPI_BYTE; + break; + case PIO_CHAR: + expected_basetype = MPI_CHAR; + break; + case PIO_SHORT: + expected_basetype = MPI_SHORT; + break; case PIO_INT: expected_basetype = MPI_INT; break; @@ -493,6 +741,23 @@ int test_decomp_read_write(int iosysid, int ioid, int num_flavors, int *flavor, case PIO_DOUBLE: expected_basetype = MPI_DOUBLE; break; +#ifdef _NETCDF4 + case PIO_UBYTE: + expected_basetype = MPI_UNSIGNED_CHAR; + break; + case PIO_USHORT: + expected_basetype = MPI_UNSIGNED_SHORT; + break; + case PIO_UINT: + expected_basetype = MPI_UNSIGNED; + break; + case PIO_INT64: + expected_basetype = MPI_LONG_LONG; + break; + case PIO_UINT64: + expected_basetype = MPI_UNSIGNED_LONG_LONG; + break; +#endif /* _NETCDF4 */ default: return ERR_WRONG; } @@ -536,8 +801,14 @@ int main(int argc, char **argv) { #define NUM_REARRANGERS_TO_TEST 2 int rearranger[NUM_REARRANGERS_TO_TEST] = {PIO_REARR_BOX, PIO_REARR_SUBSET}; -#define NUM_TYPES_TO_TEST 3 - int test_type[NUM_TYPES_TO_TEST] = {PIO_INT, PIO_FLOAT, PIO_DOUBLE}; +#ifdef _NETCDF4 +#define NUM_TYPES_TO_TEST 11 + int test_type[NUM_TYPES_TO_TEST] = {PIO_BYTE, PIO_CHAR, PIO_SHORT, PIO_INT, PIO_FLOAT, PIO_DOUBLE, + PIO_UBYTE, PIO_USHORT, PIO_UINT, PIO_INT64, PIO_UINT64}; +#else +#define NUM_TYPES_TO_TEST 6 + int test_type[NUM_TYPES_TO_TEST] = {PIO_BYTE, PIO_CHAR, PIO_SHORT, PIO_INT, PIO_FLOAT, PIO_DOUBLE}; +#endif /* _NETCDF4 */ int my_rank; int ntasks; int num_flavors; /* Number of PIO netCDF flavors in this build. */ diff --git a/src/externals/pio2/tests/cunit/test_darray_3d.c b/src/externals/pio2/tests/cunit/test_darray_3d.c index 07f2cd4e303..e261c0cec43 100644 --- a/src/externals/pio2/tests/cunit/test_darray_3d.c +++ b/src/externals/pio2/tests/cunit/test_darray_3d.c @@ -92,7 +92,7 @@ int create_decomposition_3d(int ntasks, int my_rank, int iosysid, int *ioid) /* Create the PIO decomposition for this test. */ printf("%d Creating decomposition elements_per_pe = %lld\n", my_rank, elements_per_pe); if ((ret = PIOc_init_decomp(iosysid, PIO_INT, NDIM3, dim_len_3d, elements_per_pe, - compdof, ioid, NULL, NULL, NULL))) + compdof, ioid, 0, NULL, NULL))) ERR(ret); printf("%d decomposition initialized.\n", my_rank); @@ -259,8 +259,7 @@ int test_decomp_read_write(int iosysid, int ioid, int num_flavors, int *flavor, sprintf(filename, "decomp_%s_iotype_%d.nc", TEST_NAME, flavor[fmt]); printf("writing decomp file %s\n", filename); - if ((ret = PIOc_write_nc_decomp(iosysid, filename, 0, ioid, test_comm, NULL, - NULL, 0))) + if ((ret = PIOc_write_nc_decomp(iosysid, filename, 0, ioid, NULL, NULL, 0))) return ret; /* Read the data. */ diff --git a/src/externals/pio2/tests/cunit/test_darray_multi.c b/src/externals/pio2/tests/cunit/test_darray_multi.c new file mode 100644 index 00000000000..206b783e121 --- /dev/null +++ b/src/externals/pio2/tests/cunit/test_darray_multi.c @@ -0,0 +1,473 @@ +/* + * Tests for PIO distributed arrays. This program tests the + * PIOc_write_darray_multi() function with more than one variable. + * + * Ed Hartnett, 3/7/17 + */ +#include +#include +#include + +/* The number of tasks this test should run on. */ +#define TARGET_NTASKS 4 + +/* The minimum number of tasks this test should run on. */ +#define MIN_NTASKS 4 + +/* The name of this test. */ +#define TEST_NAME "test_darray_multi" + +/* Number of processors that will do IO. */ +#define NUM_IO_PROCS 1 + +/* Number of computational components to create. */ +#define COMPONENT_COUNT 1 + +/* The number of dimensions in the example data. In this test, we + * are using three-dimensional data. */ +#define NDIM 3 + +/* But sometimes we need arrays of the non-record dimensions. */ +#define NDIM2 2 + +/* The length of our sample data along each dimension. */ +#define X_DIM_LEN 4 +#define Y_DIM_LEN 4 + +/* The number of timesteps of data to write. */ +#define NUM_TIMESTEPS 2 + +/* Number of variables. */ +#define NVAR 3 + +/* For attributes. */ +#define NOTE_NAME "note" +#define NOTE "This is a test file for the PIO library, and may be deleted." + +/* Who would have thought? */ +#define TOTAL_NUMBER_OF_STOOGES_NAME "Total_Number_of_Stooges" +#define TOTAL_NUMBER_OF_STOOGES 6 + +/* The dimension names. */ +char dim_name[NDIM][PIO_MAX_NAME + 1] = {"year", "Stooge_popularity", "face_smacks"}; + +/* The variable names. */ +char var_name[NVAR][PIO_MAX_NAME + 1] = {"Larry", "Curly", "Moe"}; + +/* Length of the dimensions in the sample data. */ +int dim_len[NDIM] = {NC_UNLIMITED, X_DIM_LEN, Y_DIM_LEN}; + +/** + * Test the darray functionality. Create a netCDF file with 3 + * dimensions and 3 variable, and use PIOc_write_darray_multi() to + * write one record of data to all three vars at once. + * + * @param iosysid the IO system ID. + * @param ioid the ID of the decomposition. + * @param num_flavors the number of IOTYPES available in this build. + * @param flavor array of available iotypes. + * @param my_rank rank of this task. + * @param pio_type the type of the data. + * @returns 0 for success, error code otherwise. +*/ +int test_darray(int iosysid, int ioid, int num_flavors, int *flavor, int my_rank, + int pio_type) +{ +#define NUM_TEST_CASES_WRT_MULTI 2 +#define NUM_TEST_CASES_FILLVALUE 2 + + char filename[PIO_MAX_NAME + 1]; /* Name for the output files. */ + int dimids[NDIM]; /* The dimension IDs. */ + int ncid; /* The ncid of the netCDF file. */ + int ncid2; /* The ncid of the re-opened netCDF file. */ + int varid[NVAR]; /* The IDs of the netCDF varables. */ + int ret; /* Return code. */ + PIO_Offset arraylen = 4; /* Amount of data from each task. */ + void *fillvalue; /* Pointer to fill value. */ + void *test_data; /* Pointer to test data we will write. */ + void *test_data_in; /* Pointer to buffer we will read into. */ + + /* Default fill value array for each type. */ + signed char byte_fill[NVAR] = {NC_FILL_BYTE, NC_FILL_BYTE, NC_FILL_BYTE}; + char char_fill[NVAR] = {NC_FILL_CHAR, NC_FILL_CHAR, NC_FILL_CHAR}; + short short_fill[NVAR] = {NC_FILL_SHORT, NC_FILL_SHORT, NC_FILL_SHORT}; + int int_fill[NVAR] = {NC_FILL_INT, NC_FILL_INT, NC_FILL_INT}; + float float_fill[NVAR] = {NC_FILL_FLOAT, NC_FILL_FLOAT, NC_FILL_FLOAT}; + double double_fill[NVAR] = {NC_FILL_DOUBLE, NC_FILL_DOUBLE, NC_FILL_DOUBLE}; +#ifdef _NETCDF4 + unsigned char ubyte_fill[NVAR] = {NC_FILL_UBYTE, NC_FILL_UBYTE, NC_FILL_UBYTE}; + unsigned short ushort_fill[NVAR] = {NC_FILL_USHORT, NC_FILL_USHORT, NC_FILL_USHORT}; + unsigned int uint_fill[NVAR] = {NC_FILL_UINT, NC_FILL_UINT, NC_FILL_UINT}; + long long int64_fill[NVAR] = {NC_FILL_INT64, NC_FILL_INT64, NC_FILL_INT64}; + unsigned long long uint64_fill[NVAR] = {NC_FILL_UINT64, NC_FILL_UINT64, NC_FILL_UINT64}; +#endif /* _NETCDF4 */ + + /* Test data we will write. */ + signed char test_data_byte[arraylen * NVAR]; + char test_data_char[arraylen * NVAR]; + short test_data_short[arraylen * NVAR]; + int test_data_int[arraylen * NVAR]; + float test_data_float[arraylen * NVAR]; + double test_data_double[arraylen * NVAR]; +#ifdef _NETCDF4 + unsigned char test_data_ubyte[arraylen * NVAR]; + unsigned short test_data_ushort[arraylen * NVAR]; + unsigned int test_data_uint[arraylen * NVAR]; + long long test_data_int64[arraylen * NVAR]; + unsigned long long test_data_uint64[arraylen * NVAR]; +#endif /* _NETCDF4 */ + + /* We will read test data into these buffers. */ + signed char test_data_byte_in[arraylen]; + char test_data_char_in[arraylen]; + short test_data_short_in[arraylen]; + int test_data_int_in[arraylen]; + float test_data_float_in[arraylen]; + double test_data_double_in[arraylen]; +#ifdef _NETCDF4 + unsigned char test_data_ubyte_in[arraylen]; + unsigned short test_data_ushort_in[arraylen]; + unsigned int test_data_uint_in[arraylen]; + long long test_data_int64_in[arraylen]; + unsigned long long test_data_uint64_in[arraylen]; +#endif /* _NETCDF4 */ + + /* Initialize a big blob of test data for NVAR vars. */ + for (int f = 0; f < arraylen * NVAR; f++) + { + test_data_byte[f] = my_rank * 1 + f; + test_data_char[f] = my_rank * 2 + f; + test_data_short[f] = my_rank * 5 + f; + test_data_int[f] = my_rank * 10 + f; + test_data_float[f] = my_rank * 10 + f + 0.5; + test_data_double[f] = my_rank * 100000 + f + 0.5; +#ifdef _NETCDF4 + test_data_ubyte[f] = my_rank * 3 + f; + test_data_ushort[f] = my_rank * 9 + f; + test_data_uint[f] = my_rank * 100 + f; + test_data_int64[f] = my_rank * 10000 + f; + test_data_uint64[f] = my_rank * 100000 + f; +#endif /* _NETCDF4 */ + } + + /* Use PIO to create the example file in each of the four + * available ways. */ + for (int fmt = 0; fmt < num_flavors; fmt++) + { + /* 1-byte types not working with pnetcdf. */ + if (flavor[fmt] == PIO_IOTYPE_PNETCDF && (pio_type == PIO_BYTE || pio_type == PIO_CHAR)) + continue; + + /* NetCDF-4 types only work with netCDF-4. */ + if (pio_type > PIO_DOUBLE && (flavor[fmt] != PIO_IOTYPE_NETCDF4C && + flavor[fmt] != PIO_IOTYPE_NETCDF4P)) + continue; + + /* Add a couple of extra tests for the + * PIOc_write_darray_multi() function. */ + for (int test_multi = 0; test_multi < NUM_TEST_CASES_WRT_MULTI; test_multi++) + { + /* Test with/without providing a fill value to PIOc_write_darray(). */ + for (int provide_fill = 0; provide_fill < NUM_TEST_CASES_FILLVALUE; provide_fill++) + { + /* Create the filename. */ + sprintf(filename, "data_%s_iotype_%d_pio_type_%d_test_multi_%d_provide_fill_%d.nc", TEST_NAME, + flavor[fmt], pio_type, test_multi, provide_fill); + + /* Select the fill value and data. */ + switch (pio_type) + { + case PIO_BYTE: + fillvalue = provide_fill ? byte_fill : NULL; + test_data = test_data_byte; + test_data_in = test_data_byte_in; + break; + case PIO_CHAR: + fillvalue = provide_fill ? char_fill : NULL; + test_data = test_data_char; + test_data_in = test_data_char_in; + break; + case PIO_SHORT: + fillvalue = provide_fill ? short_fill : NULL; + test_data = test_data_short; + test_data_in = test_data_short_in; + break; + case PIO_INT: + fillvalue = provide_fill ? int_fill : NULL; + test_data = test_data_int; + test_data_in = test_data_int_in; + break; + case PIO_FLOAT: + fillvalue = provide_fill ? float_fill : NULL; + test_data = test_data_float; + test_data_in = test_data_float_in; + break; + case PIO_DOUBLE: + fillvalue = provide_fill ? double_fill : NULL; + test_data = test_data_double; + test_data_in = test_data_double_in; + break; +#ifdef _NETCDF4 + case PIO_UBYTE: + fillvalue = provide_fill ? ubyte_fill : NULL; + test_data = test_data_ubyte; + test_data_in = test_data_ubyte_in; + break; + case PIO_USHORT: + fillvalue = provide_fill ? ushort_fill : NULL; + test_data = test_data_ushort; + test_data_in = test_data_ushort_in; + break; + case PIO_UINT: + fillvalue = provide_fill ? uint_fill : NULL; + test_data = test_data_uint; + test_data_in = test_data_uint_in; + break; + case PIO_INT64: + fillvalue = provide_fill ? int64_fill : NULL; + test_data = test_data_int64; + test_data_in = test_data_int64_in; + break; + case PIO_UINT64: + fillvalue = provide_fill ? uint64_fill : NULL; + test_data = test_data_uint64; + test_data_in = test_data_uint64_in; + break; +#endif /* _NETCDF4 */ + default: + ERR(ERR_WRONG); + } + + /* Create the netCDF output file. */ + printf("rank: %d Creating sample file %s with format %d type %d\n", my_rank, filename, + flavor[fmt], pio_type); + if ((ret = PIOc_createfile(iosysid, &ncid, &flavor[fmt], filename, PIO_CLOBBER))) + ERR(ret); + + /* Define netCDF dimensions and variable. */ + printf("%d Defining netCDF metadata...\n", my_rank); + for (int d = 0; d < NDIM; d++) + if ((ret = PIOc_def_dim(ncid, dim_name[d], (PIO_Offset)dim_len[d], &dimids[d]))) + ERR(ret); + + /* Define a variable. */ + for (int v = 0; v < NVAR; v++) + if ((ret = PIOc_def_var(ncid, var_name[v], pio_type, NDIM, dimids, &varid[v]))) + ERR(ret); + + /* Leave a note. */ + if ((ret = PIOc_put_att_text(ncid, NC_GLOBAL, NOTE_NAME, strlen(NOTE), NOTE))) + ERR(ret); + int num_stooges = TOTAL_NUMBER_OF_STOOGES; + if ((ret = PIOc_put_att_int(ncid, NC_GLOBAL, TOTAL_NUMBER_OF_STOOGES_NAME, PIO_INT, 1, &num_stooges))) + ERR(ret); + + /* End define mode. */ + if ((ret = PIOc_enddef(ncid))) + ERR(ret); + + /* Set the value of the record dimension. */ + if ((ret = PIOc_setframe(ncid, varid[0], 0))) + ERR(ret); + + int frame[NVAR] = {0, 0, 0}; + int flushtodisk = test_multi; + + /* Write the data with the _multi function. */ + if ((ret = PIOc_write_darray_multi(ncid, varid, ioid, NVAR, arraylen, test_data, frame, + fillvalue, flushtodisk))) + ERR(ret); + + /* Close the netCDF file. */ + if ((ret = PIOc_closefile(ncid))) + ERR(ret); + + /* Reopen the file. */ + if ((ret = PIOc_openfile(iosysid, &ncid2, &flavor[fmt], filename, PIO_NOWRITE))) + ERR(ret); + + /* Now use read_darray on each var in turn and make + * sure we get correct data. */ + for (int v = 0; v < NVAR; v++) + { + /* Read the data. */ + if ((ret = PIOc_read_darray(ncid2, varid[v], ioid, arraylen, test_data_in))) + ERR(ret); + + /* Check the results. */ + for (int f = 0; f < arraylen; f++) + { + switch (pio_type) + { + case PIO_BYTE: + if (test_data_byte_in[f] != test_data_byte[f + arraylen * v]) + return ERR_WRONG; + break; + case PIO_CHAR: + if (test_data_char_in[f] != test_data_char[f + arraylen * v]) + return ERR_WRONG; + break; + case PIO_SHORT: + if (test_data_short_in[f] != test_data_short[f + arraylen * v]) + return ERR_WRONG; + break; + case PIO_INT: + if (test_data_int_in[f] != test_data_int[f + arraylen * v]) + return ERR_WRONG; + break; + case PIO_FLOAT: + if (test_data_float_in[f] != test_data_float[f + arraylen * v]) + return ERR_WRONG; + break; + case PIO_DOUBLE: + if (test_data_double_in[f] != test_data_double[f + arraylen * v]) + return ERR_WRONG; + break; +#ifdef _NETCDF4 + case PIO_UBYTE: + if (test_data_ubyte_in[f] != test_data_ubyte[f + arraylen * v]) + return ERR_WRONG; + break; + case PIO_USHORT: + if (test_data_ushort_in[f] != test_data_ushort[f + arraylen * v]) + return ERR_WRONG; + break; + case PIO_UINT: + if (test_data_uint_in[f] != test_data_uint[f + arraylen * v]) + return ERR_WRONG; + break; + case PIO_INT64: + if (test_data_int64_in[f] != test_data_int64[f + arraylen * v]) + return ERR_WRONG; + break; + case PIO_UINT64: + if (test_data_uint64_in[f] != test_data_uint64[f + arraylen * v]) + return ERR_WRONG; + break; +#endif /* _NETCDF4 */ + default: + ERR(ERR_WRONG); + } + } + } + + /* Close the netCDF file. */ + printf("%d Closing the sample data file...\n", my_rank); + if ((ret = PIOc_closefile(ncid2))) + ERR(ret); + } /* next fillvalue test case */ + } /* next test multi */ + } /* next iotype */ + + return PIO_NOERR; +} + +/** + * Run all the tests. + * + * @param iosysid the IO system ID. + * @param num_flavors number of available iotypes in the build. + * @param flavor pointer to array of the available iotypes. + * @param my_rank rank of this task. + * @param test_comm the communicator the test is running on. + * @returns 0 for success, error code otherwise. + */ +int test_all_darray(int iosysid, int num_flavors, int *flavor, int my_rank, + MPI_Comm test_comm) +{ +#ifdef _NETCDF4 +#define NUM_TYPES_TO_TEST 11 + int pio_type[NUM_TYPES_TO_TEST] = {PIO_BYTE, PIO_CHAR, PIO_SHORT, PIO_INT, PIO_FLOAT, PIO_DOUBLE, + PIO_UBYTE, PIO_USHORT, PIO_UINT, PIO_INT64, PIO_UINT64}; +#else +#define NUM_TYPES_TO_TEST 6 + int pio_type[NUM_TYPES_TO_TEST] = {PIO_BYTE, PIO_CHAR, PIO_SHORT, PIO_INT, PIO_FLOAT, PIO_DOUBLE}; +#endif /* _NETCDF4 */ + int ioid; + char filename[NC_MAX_NAME + 1]; + int dim_len_2d[NDIM2] = {X_DIM_LEN, Y_DIM_LEN}; + int ret; /* Return code. */ + + for (int t = 0; t < NUM_TYPES_TO_TEST; t++) + { + /* This will be our file name for writing out decompositions. */ + sprintf(filename, "%s_decomp_rank_%d_flavor_%d_type_%d.nc", TEST_NAME, my_rank, + *flavor, pio_type[t]); + + /* Decompose the data over the tasks. */ + if ((ret = create_decomposition_2d(TARGET_NTASKS, my_rank, iosysid, dim_len_2d, + &ioid, pio_type[t]))) + return ret; + + /* Run a simple darray test. */ + if ((ret = test_darray(iosysid, ioid, num_flavors, flavor, my_rank, pio_type[t]))) + return ret; + + /* Free the PIO decomposition. */ + if ((ret = PIOc_freedecomp(iosysid, ioid))) + ERR(ret); + } + + return PIO_NOERR; +} + +/* Run tests for darray functions. */ +int main(int argc, char **argv) +{ +#define NUM_REARRANGERS_TO_TEST 2 + int rearranger[NUM_REARRANGERS_TO_TEST] = {PIO_REARR_BOX, PIO_REARR_SUBSET}; + int my_rank; + int ntasks; + int num_flavors; /* Number of PIO netCDF flavors in this build. */ + int flavor[NUM_FLAVORS]; /* iotypes for the supported netCDF IO flavors. */ + MPI_Comm test_comm; /* A communicator for this test. */ + int ret; /* Return code. */ + + /* Initialize test. */ + if ((ret = pio_test_init2(argc, argv, &my_rank, &ntasks, MIN_NTASKS, + MIN_NTASKS, 3, &test_comm))) + ERR(ERR_INIT); + + if ((ret = PIOc_set_iosystem_error_handling(PIO_DEFAULT, PIO_RETURN_ERROR, NULL))) + return ret; + + /* Only do something on max_ntasks tasks. */ + if (my_rank < TARGET_NTASKS) + { + int iosysid; /* The ID for the parallel I/O system. */ + int ioproc_stride = 1; /* Stride in the mpi rank between io tasks. */ + int ioproc_start = 0; /* Zero based rank of first processor to be used for I/O. */ + int ret; /* Return code. */ + + /* Figure out iotypes. */ + if ((ret = get_iotypes(&num_flavors, flavor))) + ERR(ret); + printf("Runnings tests for %d flavors\n", num_flavors); + + for (int r = 0; r < NUM_REARRANGERS_TO_TEST; r++) + { + /* Initialize the PIO IO system. This specifies how + * many and which processors are involved in I/O. */ + if ((ret = PIOc_Init_Intracomm(test_comm, TARGET_NTASKS, ioproc_stride, + ioproc_start, rearranger[r], &iosysid))) + return ret; + + /* Run tests. */ + printf("%d Running tests...\n", my_rank); + if ((ret = test_all_darray(iosysid, num_flavors, flavor, my_rank, test_comm))) + return ret; + + /* Finalize PIO system. */ + if ((ret = PIOc_finalize(iosysid))) + return ret; + } /* next rearranger */ + } /* endif my_rank < TARGET_NTASKS */ + + /* Finalize the MPI library. */ + printf("%d %s Finalizing...\n", my_rank, TEST_NAME); + if ((ret = pio_test_finalize(&test_comm))) + return ret; + + printf("%d %s SUCCESS!!\n", my_rank, TEST_NAME); + return 0; +} diff --git a/src/externals/pio2/tests/cunit/test_darray_multivar.c b/src/externals/pio2/tests/cunit/test_darray_multivar.c index c4b91c32ae2..773de9b5f60 100644 --- a/src/externals/pio2/tests/cunit/test_darray_multivar.c +++ b/src/externals/pio2/tests/cunit/test_darray_multivar.c @@ -71,10 +71,11 @@ int dim_len[NDIM] = {NC_UNLIMITED, X_DIM_LEN, Y_DIM_LEN}; * (ignored if use_fill is 0). * @returns 0 for success, error code otherwise. */ -int test_3_empty(int iosysid, int ioid, int num_flavors, int *flavor, int my_rank, - int pio_type, MPI_Comm test_comm, int rearranger, int use_fill, - int use_default) +int test_multivar_darray(int iosysid, int ioid, int num_flavors, int *flavor, + int my_rank, int pio_type, MPI_Comm test_comm, + int rearranger, int use_fill, int use_default) { +#define NUM_MULTIVAR_TEST_CASES 2 char filename[PIO_MAX_NAME + 1]; /* Name for the output files. */ int dimids[NDIM]; /* The dimension IDs. */ int ncid; /* The ncid of the netCDF file. */ @@ -84,6 +85,18 @@ int test_3_empty(int iosysid, int ioid, int num_flavors, int *flavor, int my_ran void *fillvalue; void *test_data; void *test_data_in; + signed char fillvalue_byte = NC_FILL_BYTE; + signed char custom_fillvalue_byte = -TEST_VAL_42; + signed char test_data_byte[arraylen]; + signed char test_data_byte_in[arraylen]; + char fillvalue_char = NC_FILL_CHAR; + char custom_fillvalue_char = -TEST_VAL_42; + char test_data_char[arraylen]; + char test_data_char_in[arraylen]; + short fillvalue_short = NC_FILL_SHORT; + short custom_fillvalue_short = -TEST_VAL_42; + short test_data_short[arraylen]; + short test_data_short_in[arraylen]; int fillvalue_int = NC_FILL_INT; int custom_fillvalue_int = -TEST_VAL_42; int test_data_int[arraylen]; @@ -96,19 +109,66 @@ int test_3_empty(int iosysid, int ioid, int num_flavors, int *flavor, int my_ran double custom_fillvalue_double = (-TEST_VAL_42 * 100); double test_data_double[arraylen]; double test_data_double_in[arraylen]; +#ifdef _NETCDF4 + unsigned char fillvalue_ubyte = NC_FILL_UBYTE; + unsigned char custom_fillvalue_ubyte = TEST_VAL_42; + unsigned char test_data_ubyte[arraylen]; + unsigned char test_data_ubyte_in[arraylen]; + unsigned short fillvalue_ushort = NC_FILL_USHORT; + unsigned short custom_fillvalue_ushort = (TEST_VAL_42 * 100); + unsigned short test_data_ushort[arraylen]; + unsigned short test_data_ushort_in[arraylen]; + unsigned int fillvalue_uint = NC_FILL_UINT; + unsigned int custom_fillvalue_uint = (TEST_VAL_42 * 100); + unsigned int test_data_uint[arraylen]; + unsigned int test_data_uint_in[arraylen]; + long long fillvalue_int64 = NC_FILL_INT64; + long long custom_fillvalue_int64 = (TEST_VAL_42 * 100); + long long test_data_int64[arraylen]; + long long test_data_int64_in[arraylen]; + unsigned long long fillvalue_uint64 = NC_FILL_UINT64; + unsigned long long custom_fillvalue_uint64 = (TEST_VAL_42 * 100); + unsigned long long test_data_uint64[arraylen]; + unsigned long long test_data_uint64_in[arraylen]; +#endif /* _NETCDF4 */ int ret; /* Return code. */ /* Initialize some data. */ for (int f = 0; f < arraylen; f++) { + test_data_byte[f] = my_rank * 10 + f; + test_data_char[f] = my_rank * 10 + f; + test_data_short[f] = my_rank * 10 + f; test_data_int[f] = my_rank * 10 + f; test_data_float[f] = my_rank * 10 + f + 0.5; test_data_double[f] = my_rank * 100000 + f + 0.5; +#ifdef _NETCDF4 + test_data_ubyte[f] = my_rank * 10 + f; + test_data_ushort[f] = my_rank * 10 + f; + test_data_uint[f] = my_rank * 10 + f; + test_data_int64[f] = my_rank * 10 + f; + test_data_uint64[f] = my_rank * 10 + f; +#endif /* _NETCDF4 */ } /* Select the fill value and data. */ switch (pio_type) { + case PIO_BYTE: + fillvalue = use_default ? &fillvalue_byte : &custom_fillvalue_byte; + test_data = test_data_byte; + test_data_in = test_data_byte_in; + break; + case PIO_CHAR: + fillvalue = use_default ? &fillvalue_char : &custom_fillvalue_char; + test_data = test_data_char; + test_data_in = test_data_char_in; + break; + case PIO_SHORT: + fillvalue = use_default ? &fillvalue_short : &custom_fillvalue_short; + test_data = test_data_short; + test_data_in = test_data_short_in; + break; case PIO_INT: fillvalue = use_default ? &fillvalue_int : &custom_fillvalue_int; test_data = test_data_int; @@ -124,6 +184,33 @@ int test_3_empty(int iosysid, int ioid, int num_flavors, int *flavor, int my_ran test_data = test_data_double; test_data_in = test_data_double_in; break; +#ifdef _NETCDF4 + case PIO_UBYTE: + fillvalue = use_default ? &fillvalue_ubyte : &custom_fillvalue_ubyte; + test_data = test_data_ubyte; + test_data_in = test_data_ubyte_in; + break; + case PIO_USHORT: + fillvalue = use_default ? &fillvalue_ushort : &custom_fillvalue_ushort; + test_data = test_data_ushort; + test_data_in = test_data_ushort_in; + break; + case PIO_UINT: + fillvalue = use_default ? &fillvalue_uint : &custom_fillvalue_uint; + test_data = test_data_uint; + test_data_in = test_data_uint_in; + break; + case PIO_INT64: + fillvalue = use_default ? &fillvalue_int64 : &custom_fillvalue_int64; + test_data = test_data_int64; + test_data_in = test_data_int64_in; + break; + case PIO_UINT64: + fillvalue = use_default ? &fillvalue_uint64 : &custom_fillvalue_uint64; + test_data = test_data_uint64; + test_data_in = test_data_uint64_in; + break; +#endif /* _NETCDF4 */ default: ERR(ERR_WRONG); } @@ -168,122 +255,226 @@ int test_3_empty(int iosysid, int ioid, int num_flavors, int *flavor, int my_ran #endif /* _PNETCDF */ } - /* Use PIO to create the example file in each of the four - * available ways. */ - for (int fmt = 0; fmt < num_flavors; fmt++) + for (int tc = 0; tc < NUM_MULTIVAR_TEST_CASES; tc++) { - /* Create the filename. */ - sprintf(filename, "data_%s_iotype_%d_pio_type_%d_use_fill_%d_default_fill_%d.nc", - TEST_NAME, flavor[fmt], pio_type, use_fill, use_default); - - /* Create the netCDF output file. */ - printf("rank: %d Creating sample file %s with format %d type %d\n", my_rank, filename, - flavor[fmt], pio_type); - if ((ret = PIOc_createfile(iosysid, &ncid, &flavor[fmt], filename, PIO_CLOBBER))) - ERR(ret); - - /* Turn on fill mode if desired. */ - if (use_fill) - if ((ret = PIOc_set_fill(ncid, NC_FILL, NULL))) - ERR(ret); - - /* Define netCDF dimensions and variable. */ - printf("%d Defining netCDF metadata...\n", my_rank); - for (int d = 0; d < NDIM; d++) - if ((ret = PIOc_def_dim(ncid, dim_name[d], (PIO_Offset)dim_len[d], &dimids[d]))) - ERR(ret); - - /* Define the variables. */ - for (int v = 0; v < NUM_VAR; v++) + /* Use PIO to create the example file in each of the four + * available ways. */ + for (int fmt = 0; fmt < num_flavors; fmt++) { - if ((ret = PIOc_def_var(ncid, var_name[v], pio_type, NDIM, dimids, &varid[v]))) + /* BYTE and CHAR don't work with pnetcdf. Don't know why yet. */ + if (flavor[fmt] == PIO_IOTYPE_PNETCDF && (pio_type == PIO_BYTE || pio_type == PIO_CHAR)) + continue; + + /* NetCDF-4 types only work with netCDF-4 formats. */ + printf("pio_type = %d flavor[fmt] = %d\n", pio_type, flavor[fmt]); + if (pio_type > PIO_DOUBLE && flavor[fmt] != PIO_IOTYPE_NETCDF4C && + flavor[fmt] != PIO_IOTYPE_NETCDF4P) + continue; + + /* Create the filename. */ + sprintf(filename, "data_%s_iotype_%d_tc_%d_pio_type_%d_use_fill_%d_default_fill_%d.nc", + TEST_NAME, flavor[fmt], tc, pio_type, use_fill, use_default); + + /* Create the netCDF output file. */ + printf("rank: %d Creating sample file %s with format %d type %d\n", my_rank, filename, + flavor[fmt], pio_type); + if ((ret = PIOc_createfile(iosysid, &ncid, &flavor[fmt], filename, PIO_CLOBBER))) ERR(ret); - if (use_fill && !use_default) - if ((ret = PIOc_def_var_fill(ncid, varid[v], NC_FILL, fillvalue))) - ERR(ret); - } - /* End define mode. */ - if ((ret = PIOc_enddef(ncid))) - ERR(ret); + /* Turn on fill mode if desired. */ + if (use_fill) + if ((ret = PIOc_set_fill(ncid, NC_FILL, NULL))) + ERR(ret); - /* Set the value of the record dimension. */ - if ((ret = PIOc_setframe(ncid, varid[0], 0))) - ERR(ret); + /* Define netCDF dimensions and variable. */ + printf("%d Defining netCDF metadata...\n", my_rank); + for (int d = 0; d < NDIM; d++) + if ((ret = PIOc_def_dim(ncid, dim_name[d], (PIO_Offset)dim_len[d], &dimids[d]))) + ERR(ret); - /* Write the data. */ - if ((ret = PIOc_write_darray(ncid, varid[0], ioid, arraylen, test_data, fillvalue))) - ERR(ret); + /* Define the variables. */ + for (int v = 0; v < NUM_VAR; v++) + { + if ((ret = PIOc_def_var(ncid, var_name[v], pio_type, NDIM, dimids, &varid[v]))) + ERR(ret); + if (use_fill && !use_default) + if ((ret = PIOc_def_var_fill(ncid, varid[v], NC_FILL, fillvalue))) + ERR(ret); + } - /* Close the netCDF file. */ - if ((ret = PIOc_closefile(ncid))) - ERR(ret); + /* End define mode. */ + if ((ret = PIOc_enddef(ncid))) + ERR(ret); - /* Reopen the file. */ - if ((ret = PIOc_openfile(iosysid, &ncid2, &flavor[fmt], filename, PIO_NOWRITE))) - ERR(ret); + /* Set the value of the record dimension. */ + for (int v = 0; v < NUM_VAR; v++) + { + if ((ret = PIOc_setframe(ncid, varid[v], 0))) + ERR(ret); - /* Read the data. */ - if ((ret = PIOc_read_darray(ncid2, varid[0], ioid, arraylen, test_data_in))) - ERR(ret); + /* For the first test case we just write the first variable. */ + if (tc == 0) + break; + } - /* Check the results. */ - for (int f = 0; f < arraylen; f++) - { - switch (pio_type) + /* Write the data. */ + for (int v = 0; v < NUM_VAR; v++) { - case PIO_INT: - if (test_data_int_in[f] != test_data_int[f]) - return ERR_WRONG; - break; - case PIO_FLOAT: - if (test_data_float_in[f] != test_data_float[f]) - return ERR_WRONG; - break; - case PIO_DOUBLE: - if (test_data_double_in[f] != test_data_double[f]) - return ERR_WRONG; - break; - default: - ERR(ERR_WRONG); + if ((ret = PIOc_write_darray(ncid, varid[v], ioid, arraylen, test_data, fillvalue))) + ERR(ret); + + /* For the first test case we just write the first variable. */ + if (tc == 0) + break; } - } - /* If fill mode is in use the other vars should have fill values. */ - if (use_fill && flavor[fmt] != PIO_IOTYPE_PNETCDF) - { - /* Read the data. */ - if ((ret = PIOc_read_darray(ncid2, varid[1], ioid, arraylen, test_data_in))) + /* Close the netCDF file. */ + if ((ret = PIOc_closefile(ncid))) ERR(ret); - - /* Check the results. */ - for (int f = 0; f < arraylen; f++) + + /* Reopen the file. */ + if ((ret = PIOc_openfile(iosysid, &ncid2, &flavor[fmt], filename, PIO_NOWRITE))) + ERR(ret); + + for (int v = 0; v < NUM_VAR; v++) { - switch (pio_type) + /* Read the data. */ + if ((ret = PIOc_read_darray(ncid2, varid[0], ioid, arraylen, test_data_in))) + ERR(ret); + + /* Check the results. */ + for (int f = 0; f < arraylen; f++) { - case PIO_INT: - if (test_data_int_in[f] != (use_default ? NC_FILL_INT : custom_fillvalue_int)) - return ERR_WRONG; - break; - case PIO_FLOAT: - if (test_data_float_in[f] != (use_default ? NC_FILL_FLOAT : custom_fillvalue_float)) - return ERR_WRONG; - break; - case PIO_DOUBLE: - if (test_data_double_in[f] != (use_default ? NC_FILL_DOUBLE : custom_fillvalue_double)) - return ERR_WRONG; + switch (pio_type) + { + case PIO_BYTE: + if (test_data_byte_in[f] != test_data_byte[f]) + return ERR_WRONG; + break; + case PIO_CHAR: + if (test_data_char_in[f] != test_data_char[f]) + return ERR_WRONG; + break; + case PIO_SHORT: + if (test_data_short_in[f] != test_data_short[f]) + return ERR_WRONG; + break; + case PIO_INT: + if (test_data_int_in[f] != test_data_int[f]) + return ERR_WRONG; + break; + case PIO_FLOAT: + if (test_data_float_in[f] != test_data_float[f]) + return ERR_WRONG; + break; + case PIO_DOUBLE: + if (test_data_double_in[f] != test_data_double[f]) + return ERR_WRONG; + break; +#ifdef _NETCDF4 + case PIO_UBYTE: + if (test_data_ubyte_in[f] != test_data_ubyte[f]) + return ERR_WRONG; + break; + case PIO_USHORT: + if (test_data_ushort_in[f] != test_data_ushort[f]) + return ERR_WRONG; + break; + case PIO_UINT: + if (test_data_uint_in[f] != test_data_uint[f]) + return ERR_WRONG; + break; + case PIO_INT64: + if (test_data_int64_in[f] != test_data_int64[f]) + return ERR_WRONG; + break; + case PIO_UINT64: + if (test_data_uint64_in[f] != test_data_uint64[f]) + return ERR_WRONG; + break; +#endif /* _NETCDF4 */ + default: + ERR(ERR_WRONG); + } + } + + /* For the first test case we just write the first variable. */ + if (tc == 0) break; - default: - ERR(ERR_WRONG); + } /* next var */ + + /* For the first test case, if fill mode is in use the other vars + * should have fill values. */ + if (tc == 0 && use_fill && flavor[fmt] != PIO_IOTYPE_PNETCDF) + { + /* Read the data. */ + if ((ret = PIOc_read_darray(ncid2, varid[1], ioid, arraylen, test_data_in))) + ERR(ret); + + /* Check the results. */ + for (int f = 0; f < arraylen; f++) + { + switch (pio_type) + { + case PIO_BYTE: + if (test_data_byte_in[f] != (use_default ? NC_FILL_BYTE : custom_fillvalue_byte)) + return ERR_WRONG; + break; + case PIO_CHAR: + if (test_data_char_in[f] != (use_default ? NC_FILL_CHAR : custom_fillvalue_char)) + return ERR_WRONG; + break; + case PIO_SHORT: + if (test_data_short_in[f] != (use_default ? NC_FILL_SHORT : custom_fillvalue_short)) + return ERR_WRONG; + break; + case PIO_INT: + if (test_data_int_in[f] != (use_default ? NC_FILL_INT : custom_fillvalue_int)) + return ERR_WRONG; + break; + case PIO_FLOAT: + if (test_data_float_in[f] != (use_default ? NC_FILL_FLOAT : custom_fillvalue_float)) + return ERR_WRONG; + break; + case PIO_DOUBLE: + if (test_data_double_in[f] != (use_default ? NC_FILL_DOUBLE : custom_fillvalue_double)) + return ERR_WRONG; + break; +#ifdef _NETCDF4 + case PIO_UBYTE: + if (test_data_ubyte_in[f] != (use_default ? NC_FILL_UBYTE : custom_fillvalue_ubyte)) + return ERR_WRONG; + break; + case PIO_USHORT: + if (test_data_ushort_in[f] != (use_default ? NC_FILL_USHORT : custom_fillvalue_ushort)) + return ERR_WRONG; + break; + case PIO_UINT: + if (test_data_uint_in[f] != (use_default ? NC_FILL_UINT : custom_fillvalue_uint)) + return ERR_WRONG; + break; + case PIO_INT64: + if (test_data_int64_in[f] != (use_default ? NC_FILL_INT64 : custom_fillvalue_int64)) + return ERR_WRONG; + break; + case PIO_UINT64: + if (test_data_uint64_in[f] != (use_default ? NC_FILL_UINT64 : custom_fillvalue_uint64)) + return ERR_WRONG; + break; +#endif /* _NETCDF4 */ + default: + ERR(ERR_WRONG); + } } } - } - /* Close the netCDF file. */ - printf("%d Closing the sample data file...\n", my_rank); - if ((ret = PIOc_closefile(ncid2))) - ERR(ret); - } + /* Close the netCDF file. */ + printf("%d Closing the sample data file...\n", my_rank); + if ((ret = PIOc_closefile(ncid2))) + ERR(ret); + } + } /* next test case */ + return PIO_NOERR; } @@ -302,8 +493,14 @@ int test_all_darray(int iosysid, int num_flavors, int *flavor, int my_rank, MPI_Comm test_comm, int rearranger) { #define NUM_FILL_TESTS 3 -#define NUM_TYPES_TO_TEST 3 - int pio_type[NUM_TYPES_TO_TEST] = {PIO_INT, PIO_FLOAT, PIO_DOUBLE}; +#ifdef _NETCDF4 +#define NUM_TYPES_TO_TEST 11 + int test_type[NUM_TYPES_TO_TEST] = {PIO_BYTE, PIO_CHAR, PIO_SHORT, PIO_INT, PIO_FLOAT, PIO_DOUBLE, + PIO_UBYTE, PIO_USHORT, PIO_UINT, PIO_INT64, PIO_UINT64}; +#else +#define NUM_TYPES_TO_TEST 6 + int test_type[NUM_TYPES_TO_TEST] = {PIO_BYTE, PIO_CHAR, PIO_SHORT, PIO_INT, PIO_FLOAT, PIO_DOUBLE}; +#endif /* _NETCDF4 */ int ioid; int dim_len_2d[NDIM2] = {X_DIM_LEN, Y_DIM_LEN}; int ret; /* Return code. */ @@ -315,7 +512,7 @@ int test_all_darray(int iosysid, int num_flavors, int *flavor, int my_rank, /* Decompose the data over the tasks. */ if ((ret = create_decomposition_2d(TARGET_NTASKS, my_rank, iosysid, dim_len_2d, - &ioid, pio_type[t]))) + &ioid, test_type[t]))) return ret; /* Run the different combinations of use_fill and use_default. */ @@ -327,9 +524,9 @@ int test_all_darray(int iosysid, int num_flavors, int *flavor, int my_rank, if (f == 2) use_default++; - /* Run a simple darray test. */ - if ((ret = test_3_empty(iosysid, ioid, num_flavors, flavor, my_rank, pio_type[t], - test_comm, rearranger, use_fill, use_default))) + /* Run the multivar darray tests. */ + if ((ret = test_multivar_darray(iosysid, ioid, num_flavors, flavor, my_rank, test_type[t], + test_comm, rearranger, use_fill, use_default))) return ret; } diff --git a/src/externals/pio2/tests/cunit/test_darray_multivar2.c b/src/externals/pio2/tests/cunit/test_darray_multivar2.c new file mode 100644 index 00000000000..b25ba890ab7 --- /dev/null +++ b/src/externals/pio2/tests/cunit/test_darray_multivar2.c @@ -0,0 +1,287 @@ +/* + * Tests for PIO distributed arrays. + * + * Ed Hartnett, Jim Edwards, 4/20/17 + */ +#include +#include +#include + +/* The number of tasks this test should run on. */ +#define TARGET_NTASKS 4 + +/* The minimum number of tasks this test should run on. */ +#define MIN_NTASKS 4 + +/* The name of this test. */ +#define TEST_NAME "test_darray_multivar2" + +/* Number of processors that will do IO. */ +#define NUM_IO_PROCS 1 + +/* Number of computational components to create. */ +#define COMPONENT_COUNT 1 + +/* The number of dimensions in the example data. In this test, we + * are using three-dimensional data. */ +#define NDIM 3 + +/* But sometimes we need arrays of the non-record dimensions. */ +#define NDIM2 2 + +/* The length of our sample data along each dimension. */ +#define X_DIM_LEN 4 +#define Y_DIM_LEN 4 + +/* The number of timesteps of data to write. */ +#define NUM_TIMESTEPS 2 + +/* Number of variables in the test file. */ +#define NUM_VAR 2 + +/* The dimension names. */ +char dim_name[NDIM][PIO_MAX_NAME + 1] = {"timestep", "x", "y"}; + +/* The var names. */ +char var_name[NUM_VAR][PIO_MAX_NAME + 1] = {"Aubery", "Martin"}; + +/* Length of the dimensions in the sample data. */ +int dim_len[NDIM] = {NC_UNLIMITED, X_DIM_LEN, Y_DIM_LEN}; + +/** + * Test the darray functionality. Create a netCDF file with 3 + * dimensions and 2 variables. One of the vars uses the record + * dimension, the other does not. Then use darray to write to them. + * + * @param iosysid the IO system ID. + * @param ioid the ID of the decomposition. + * @param num_flavors the number of IOTYPES available in this build. + * @param flavor array of available iotypes. + * @param my_rank rank of this task. + * @param pio_type the type of the data. + * @param test_comm the communicator that is running this test. + * @returns 0 for success, error code otherwise. +*/ +int test_multivar_darray(int iosysid, int ioid, int num_flavors, int *flavor, + int my_rank, int pio_type, MPI_Comm test_comm) +{ + char filename[PIO_MAX_NAME + 1]; /* Name for the output files. */ + int dimids[NDIM]; /* The dimension IDs. */ + int ncid; /* The ncid of the netCDF file. */ + int varid[NUM_VAR]; /* The IDs of the netCDF varables. */ + PIO_Offset arraylen = 4; + int custom_fillvalue_int = -TEST_VAL_42; + int test_data_int[arraylen]; + int ret; /* Return code. */ + + /* Initialize some data. */ + for (int f = 0; f < arraylen; f++) + test_data_int[f] = my_rank * 10 + f; + + /* Use PIO to create the example file in each of the four + * available ways. */ + for (int fmt = 0; fmt < num_flavors; fmt++) + { + /* Create the filename. */ + sprintf(filename, "data_%s_iotype_%d_pio_type_%d.nc", TEST_NAME, flavor[fmt], pio_type); + + /* Create the netCDF output file. */ + printf("rank: %d Creating sample file %s with format %d type %d\n", my_rank, filename, + flavor[fmt], pio_type); + if ((ret = PIOc_createfile(iosysid, &ncid, &flavor[fmt], filename, PIO_CLOBBER))) + ERR(ret); + + /* Define netCDF dimensions and variable. */ + printf("%d Defining netCDF metadata...\n", my_rank); + for (int d = 0; d < NDIM; d++) + if ((ret = PIOc_def_dim(ncid, dim_name[d], (PIO_Offset)dim_len[d], &dimids[d]))) + ERR(ret); + + /* Var 0 does not have a record dim, varid 1 is a record var. */ + if ((ret = PIOc_def_var(ncid, var_name[0], pio_type, NDIM - 1, &dimids[1], &varid[0]))) + ERR(ret); + if ((ret = PIOc_def_var(ncid, var_name[1], pio_type, NDIM, dimids, &varid[1]))) + ERR(ret); + + /* End define mode. */ + if ((ret = PIOc_enddef(ncid))) + ERR(ret); + + /* Set the value of the record dimension for varid 1. */ + if ((ret = PIOc_setframe(ncid, varid[1], 0))) + ERR(ret); + + /* Write the data. */ + for (int v = 0; v < NUM_VAR; v++) + if ((ret = PIOc_write_darray(ncid, varid[v], ioid, arraylen, test_data_int, &custom_fillvalue_int))) + ERR(ret); + + /* Close the netCDF file. */ + if ((ret = PIOc_closefile(ncid))) + ERR(ret); + + /* Check the file contents. */ + { + int ncid2; /* The ncid of the re-opened netCDF file. */ + int test_data_int_in[arraylen]; + + /* Reopen the file. */ + if ((ret = PIOc_openfile(iosysid, &ncid2, &flavor[fmt], filename, PIO_NOWRITE))) + ERR(ret); + + for (int v = 0; v < NUM_VAR; v++) + { + /* Read the data. */ + if ((ret = PIOc_read_darray(ncid2, varid[v], ioid, arraylen, test_data_int_in))) + ERR(ret); + + /* Check the results. */ + for (int f = 0; f < arraylen; f++) + if (test_data_int_in[f] != test_data_int[f]) + return ERR_WRONG; + } /* next var */ + + /* Close the netCDF file. */ + if ((ret = PIOc_closefile(ncid2))) + ERR(ret); + } + } + + return PIO_NOERR; +} + +/* Create the decomposition to divide the 3-dimensional sample data + * between the 4 tasks. For the purposes of decomposition we are only + * concerned with 2 dimensions - we ignore the unlimited dimension. + * + * @param ntasks the number of available tasks + * @param my_rank rank of this task. + * @param iosysid the IO system ID. + * @param dim_len_2d an array of length 2 with the dim lengths. + * @param ioid a pointer that gets the ID of this decomposition. + * @param pio_type the data type to use for the decomposition. + * @returns 0 for success, error code otherwise. + **/ +int create_decomposition_2d_2(int ntasks, int my_rank, int iosysid, int *dim_len_2d, + int *ioid, int pio_type) +{ + PIO_Offset elements_per_pe; /* Array elements per processing unit. */ + PIO_Offset *compdof; /* The decomposition mapping. */ + int ret; + + /* How many data elements per task? In this example we will end up + * with 4. */ + elements_per_pe = dim_len_2d[0] * dim_len_2d[1] / ntasks; + + /* Allocate space for the decomposition array. */ + if (!(compdof = malloc(elements_per_pe * sizeof(PIO_Offset)))) + return PIO_ENOMEM; + + /* Describe the decomposition. This is a 1-based array, so add 1! */ + for (int i = 0; i < elements_per_pe; i++) + compdof[i] = my_rank * elements_per_pe + i + 1; + + /* Create the PIO decomposition for this test. */ + printf("%d Creating decomposition elements_per_pe = %lld\n", my_rank, elements_per_pe); + if ((ret = PIOc_InitDecomp(iosysid, pio_type, NDIM2, dim_len_2d, elements_per_pe, + compdof, ioid, NULL, NULL, NULL))) + ERR(ret); + + printf("%d decomposition initialized.\n", my_rank); + + /* Free the mapping. */ + free(compdof); + + return 0; +} + +/** + * Run all the tests. + * + * @param iosysid the IO system ID. + * @param num_flavors number of available iotypes in the build. + * @param flavor pointer to array of the available iotypes. + * @param my_rank rank of this task. + * @param test_comm the communicator the test is running on. + * @returns 0 for success, error code otherwise. + */ +int test_all_darray(int iosysid, int num_flavors, int *flavor, int my_rank, + MPI_Comm test_comm) +{ + int ioid; + int dim_len_2d[NDIM2] = {X_DIM_LEN, Y_DIM_LEN}; + int ret; /* Return code. */ + + /* Decompose the data over the tasks. */ + if ((ret = create_decomposition_2d_2(TARGET_NTASKS, my_rank, iosysid, dim_len_2d, + &ioid, PIO_INT))) + return ret; + + /* Run the multivar darray tests. */ + if ((ret = test_multivar_darray(iosysid, ioid, num_flavors, flavor, my_rank, PIO_INT, + test_comm))) + return ret; + + /* Free the PIO decomposition. */ + if ((ret = PIOc_freedecomp(iosysid, ioid))) + ERR(ret); + + return PIO_NOERR; +} + +/* Run tests for darray functions. */ +int main(int argc, char **argv) +{ + int my_rank; + int ntasks; + int num_flavors; /* Number of PIO netCDF flavors in this build. */ + int flavor[NUM_FLAVORS]; /* iotypes for the supported netCDF IO flavors. */ + MPI_Comm test_comm; /* A communicator for this test. */ + int ret; /* Return code. */ + + /* Initialize test. */ + if ((ret = pio_test_init2(argc, argv, &my_rank, &ntasks, MIN_NTASKS, MIN_NTASKS, + 3, &test_comm))) + ERR(ERR_INIT); + + if ((ret = PIOc_set_iosystem_error_handling(PIO_DEFAULT, PIO_RETURN_ERROR, NULL))) + return ret; + + /* Only do something on max_ntasks tasks. */ + if (my_rank < TARGET_NTASKS) + { + int iosysid; /* The ID for the parallel I/O system. */ + int ioproc_stride = 1; /* Stride in the mpi rank between io tasks. */ + int ioproc_start = 0; /* Zero based rank of first processor to be used for I/O. */ + int ret; /* Return code. */ + + /* Figure out iotypes. */ + if ((ret = get_iotypes(&num_flavors, flavor))) + ERR(ret); + printf("Runnings tests for %d flavors\n", num_flavors); + + /* Initialize the PIO IO system. This specifies how + * many and which processors are involved in I/O. */ + if ((ret = PIOc_Init_Intracomm(test_comm, TARGET_NTASKS, ioproc_stride, + ioproc_start, PIO_REARR_BOX, &iosysid))) + return ret; + + /* Run tests. */ + printf("%d Running tests...\n", my_rank); + if ((ret = test_all_darray(iosysid, num_flavors, flavor, my_rank, test_comm))) + return ret; + + /* Finalize PIO system. */ + if ((ret = PIOc_finalize(iosysid))) + return ret; + + } /* endif my_rank < TARGET_NTASKS */ + + /* Finalize the MPI library. */ + printf("%d %s Finalizing...\n", my_rank, TEST_NAME); + if ((ret = pio_test_finalize(&test_comm))) + return ret; + + printf("%d %s SUCCESS!!\n", my_rank, TEST_NAME); + return 0; +} diff --git a/src/externals/pio2/tests/cunit/test_decomp_uneven.c b/src/externals/pio2/tests/cunit/test_decomp_uneven.c new file mode 100644 index 00000000000..b6d47004eff --- /dev/null +++ b/src/externals/pio2/tests/cunit/test_decomp_uneven.c @@ -0,0 +1,380 @@ +/* + * Tests for PIO distributed arrays. This tests cases when arrays do + * not distribute evenly over the processors. + * + * Ed Hartnett, 3/6/17 + */ +#include +#include +#include + +/* The number of tasks this test should run on. */ +#define TARGET_NTASKS 4 + +/* The minimum number of tasks this test should run on. */ +#define MIN_NTASKS 4 + +/* The name of this test. */ +#define TEST_NAME "test_darray_uneven" + +/* Number of processors that will do IO. */ +#define NUM_IO_PROCS 1 + +/* Number of computational components to create. */ +#define COMPONENT_COUNT 1 + +/* This is for 3D data decompositions. */ +#define NDIM3 3 + +/* Create the decomposition to divide the 4-dimensional sample data + * between the 4 tasks. For the purposes of decomposition we are only + * concerned with 3 dimensions - we ignore the unlimited dimension. + * + * @param ntasks the number of available tasks + * @param my_rank rank of this task. + * @param iosysid the IO system ID. + * @param dim_len an array of length 3 with the dimension sizes. + * @param pio_type the type for this decomposition. + * @param ioid a pointer that gets the ID of this decomposition. + * lengths. + * @returns 0 for success, error code otherwise. + **/ +int create_decomposition_3d(int ntasks, int my_rank, int iosysid, int *dim_len, + int pio_type, int *ioid) +{ + PIO_Offset elements_per_pe; /* Array elements per processing unit. */ + PIO_Offset remainder; /* Left over array elements. */ + PIO_Offset *compdof; /* The decomposition mapping. */ + PIO_Offset data_size = 1; + int ret; + + /* How many data elements per task? In this example we will end up + * with 4. */ + for (int d = 0; d < NDIM3; d++) + data_size *= dim_len[d]; + elements_per_pe = data_size / ntasks; + remainder = data_size % ntasks; + + /* Distribute the remaining elements. */ + if (my_rank < remainder) + elements_per_pe++; + printf("%d elements_per_pe = %lld remainder = %lld\n", my_rank, elements_per_pe, remainder); + + /* Allocate space for the decomposition array. */ + if (!(compdof = malloc(elements_per_pe * sizeof(PIO_Offset)))) + return PIO_ENOMEM; + + /* Describe the decomposition. */ + for (int i = 0; i < elements_per_pe; i++) + { + int my_remainder = 0; + if (my_rank >= remainder) + my_remainder = remainder; + compdof[i] = my_rank * elements_per_pe + i + my_remainder; + printf("%d my_remainder = %d compdof[%d] = %lld\n", my_rank, i, my_remainder, compdof[i]); + } + + /* Create the PIO decomposition for this test. */ + printf("%d Creating decomposition elements_per_pe = %lld\n", my_rank, elements_per_pe); + if ((ret = PIOc_init_decomp(iosysid, pio_type, NDIM3, dim_len, elements_per_pe, + compdof, ioid, 0, NULL, NULL))) + ERR(ret); + + printf("%d decomposition initialized.\n", my_rank); + + /* Free the mapping. */ + free(compdof); + + return 0; +} + +/** + * Test the decomp read/write functionality. Given an ioid for a 3D + * decomposition, this function will write a decomp file, then read it + * in to ensure the correct values are read. + * + * @param iosysid the IO system ID. + * @param ioid the ID of the decomposition. + * @param num_flavors the number of IOTYPES available in this build. + * @param flavor array of available iotypes. + * @param my_rank rank of this task. + * @param rearranger the rearranger to use (PIO_REARR_BOX or + * PIO_REARR_SUBSET). + * @param test_comm the MPI communicator for this test. + * @param dim_len array of length 3 with dim lengths. + * @param expected_maplen pointer to array of length TARGET_NTASKS + * with the maplen we expect to get for each of the tasks running this + * test. + * @param pio_type the type we expect to be associated with + * this decomposition. + * @param full_maplen the length of the full map. + * @param pointer to expected map, an array of TARGET_NTASKS * + * max_maplen. + * @returns 0 for success, error code otherwise. +*/ +int test_decomp_read_write(int iosysid, int ioid, int num_flavors, int *flavor, int my_rank, + int rearranger, MPI_Comm test_comm, int *dim_len, int *expected_maplen, + int pio_type, int fill_maplen, int *expected_map) +{ +#define TEST_DECOMP_TITLE "Decomposition data for test_darray_uneven.c in PIO library." +#define TEST_DECOMP_HISTORY "This file may be deleted; it is for test purposes only: " + int ioid2; /* ID for decomp we read into. */ + char filename[PIO_MAX_NAME + 1]; /* Name for the output files. */ + char title[] = TEST_DECOMP_TITLE; + char history[PIO_MAX_NAME + 1] = TEST_DECOMP_HISTORY; + char title_in[PIO_MAX_NAME + 1]; + char history_in[PIO_MAX_NAME + 1]; + int fortran_order_in; /* Indicates fortran vs. c order. */ + int ret; /* Return code. */ + + /* Use PIO to create the decomp file in one of the four + * available ways. */ + for (int fmt = 0; fmt < 1; fmt++) + { + /* Create the filename. */ + sprintf(filename, "decomp_%s_pio_type_%d_dims_%d_x_%d_x_%d.nc", TEST_NAME, pio_type, + dim_len[0], dim_len[1], dim_len[2]); + + /* Create history string. */ + strncat(history, filename, NC_MAX_NAME - strlen(TEST_DECOMP_HISTORY)); + + printf("writing decomp file %s\n", filename); + if ((ret = PIOc_write_nc_decomp(iosysid, filename, 0, ioid, title, history, 0))) + return ret; + printf("about to check map with netCDF\n"); + + /* Open the decomposition file with netCDF. */ + int ncid_in; + int iotype = PIO_IOTYPE_NETCDF; + if ((ret = PIOc_openfile(iosysid, &ncid_in, &iotype, filename, NC_NOWRITE))) + return ret; + + /* Get the max maplen. */ + int max_maplen; + if ((ret = PIOc_get_att_int(ncid_in, NC_GLOBAL, DECOMP_MAX_MAPLEN_ATT_NAME, &max_maplen))) + return ret; + printf("max_maplen = %d\n", max_maplen); + + /* Check dims. */ + PIO_Offset ndims_in; + if ((ret = PIOc_inq_dim(ncid_in, 0, NULL, &ndims_in))) + return ret; + if (ndims_in != NDIM3) + return ERR_WRONG; + PIO_Offset ntasks_in; + if ((ret = PIOc_inq_dim(ncid_in, 1, NULL, &ntasks_in))) + return ret; + if (ntasks_in != TARGET_NTASKS) + return ERR_WRONG; + + /* Check the maplen. */ + int maplen_varid; + int maplen_in[TARGET_NTASKS]; + if ((ret = PIOc_inq_varid(ncid_in, DECOMP_MAPLEN_VAR_NAME, &maplen_varid))) + return ret; + if ((ret = PIOc_get_var(ncid_in, maplen_varid, &maplen_in))) + return ret; + for (int t = 0; t < TARGET_NTASKS; t++) + { + printf("%d maplen_in[%d] = %d expected_maplen[%d] = %d\n", my_rank, t, maplen_in[t], t, expected_maplen[t]); + if (maplen_in[t] != expected_maplen[t]) + return ERR_WRONG; + } + + /* Check the map. */ + int map_varid; + int map_in[TARGET_NTASKS][max_maplen]; + if ((ret = PIOc_inq_varid(ncid_in, DECOMP_MAP_VAR_NAME, &map_varid))) + return ret; + if ((ret = PIOc_get_var(ncid_in, map_varid, (int *)&map_in))) + return ret; + printf("about to check map\n"); + for (int t = 0; t < TARGET_NTASKS; t++) + { + for (int e = 0; e < max_maplen; e++) + { + printf("%d t = %d e = %d map_in[t][e] = %d expected_map[t * max_maplen + e] = %d\n", + my_rank, t, e, map_in[t][e], expected_map[t * max_maplen + e]); + if (map_in[t][e] != expected_map[t * max_maplen + e]) + return ERR_WRONG; + } + } + + /* Close the decomposition file. */ + if ((ret = PIOc_closefile(ncid_in))) + return ret; + + /* Read the decomposition file into PIO. */ + printf("reading decomp file %s\n", filename); + if ((ret = PIOc_read_nc_decomp(iosysid, filename, &ioid2, test_comm, pio_type, + title_in, history_in, &fortran_order_in))) + return ret; + + /* Check the results. */ + { + iosystem_desc_t *ios; + io_desc_t *iodesc; + + /* Get the IO system info. */ + if (!(ios = pio_get_iosystem_from_id(iosysid))) + return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__); + + /* Get the IO desc, which describes the decomposition. */ + if (!(iodesc = pio_get_iodesc_from_id(ioid2))) + return pio_err(ios, NULL, PIO_EBADID, __FILE__, __LINE__); + + /* We need to find the MPI type we will expect to see in + * iodesc. */ + MPI_Datatype expected_mpi_type; + if ((ret = find_mpi_type(pio_type, &expected_mpi_type, NULL))) + return ret; + + /* Check values in iodesc. */ + printf("ioid2 = %d iodesc->ioid = %d iodesc->maplen = %d iodesc->ndims = %d " + "iodesc->ndof = %d iodesc->rearranger = %d iodesc->maxregions = %d " + "iodesc->needsfill = %d iodesc->basetype = %d expected_mpi_type = %d\n", + ioid2, iodesc->ioid, iodesc->maplen, iodesc->ndims, iodesc->ndof, + iodesc->rearranger, iodesc->maxregions, iodesc->needsfill, iodesc->basetype, + expected_mpi_type); + if (strcmp(title, title_in) || strcmp(history, history_in)) + return ERR_WRONG; + if (iodesc->ioid != ioid2 || iodesc->rearranger != rearranger || + iodesc->basetype != expected_mpi_type) + return ERR_WRONG; + if (iodesc->ndims != NDIM3) + return ERR_WRONG; + if (iodesc->maplen != expected_maplen[my_rank]) + return ERR_WRONG; + if (iodesc->ndims != NDIM3 || iodesc->ndof != expected_maplen[my_rank]) + return ERR_WRONG; + if (iodesc->needsfill) + return ERR_WRONG; + /* Don't forget to add 1! */ + for (int e = 0; e < iodesc->maplen; e++) + { + printf("%d e = %d max_maplen = %d iodesc->map[e] = %lld expected_map[my_rank * max_maplen + e] = %d\n", + my_rank, e, max_maplen, iodesc->map[e], expected_map[my_rank * max_maplen + e]); + if (iodesc->map[e] != expected_map[my_rank * max_maplen + e] + 1) + return ERR_WRONG; + } + for (int d = 0; d < NDIM3; d++) + if (iodesc->dimlen[d] != dim_len[d]) + return ERR_WRONG; + } + + /* Free the PIO decomposition. */ + if ((ret = PIOc_freedecomp(iosysid, ioid2))) + ERR(ret); + } + return PIO_NOERR; +} + +/* Run tests for darray functions. */ +int main(int argc, char **argv) +{ +/* #define NUM_TYPES_TO_TEST 3 */ +/* int test_type[NUM_TYPES_TO_TEST] = {PIO_INT, PIO_FLOAT, PIO_DOUBLE}; */ +#define NUM_TYPES_TO_TEST 1 + int test_type[NUM_TYPES_TO_TEST] = {PIO_INT}; + int my_rank; + int ntasks; + int num_flavors; /* Number of PIO netCDF flavors in this build. */ + int flavor[NUM_FLAVORS]; /* iotypes for the supported netCDF IO flavors. */ + MPI_Comm test_comm; /* A communicator for this test. */ + int ret; /* Return code. */ + + /* Initialize test. */ + if ((ret = pio_test_init2(argc, argv, &my_rank, &ntasks, MIN_NTASKS, + MIN_NTASKS, 3, &test_comm))) + ERR(ERR_INIT); + + if ((ret = PIOc_set_iosystem_error_handling(PIO_DEFAULT, PIO_RETURN_ERROR, NULL))) + return ret; + + /* Only do something on max_ntasks tasks. */ + if (my_rank < TARGET_NTASKS) + { +#define NUM_REARRANGERS_TO_TEST 2 + int rearranger[NUM_REARRANGERS_TO_TEST] = {PIO_REARR_BOX, PIO_REARR_SUBSET}; + int iosysid; /* The ID for the parallel I/O system. */ + int ioproc_stride = 1; /* Stride in the mpi rank between io tasks. */ + int ioproc_start = 0; /* Zero based rank of first processor to be used for I/O. */ + int map_1x4x4[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}; + int map_2x4x4[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31}; + int map_3x4x4[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, + 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47}; + int map_1x3x3[] = {0, 1, 2, 3, 4, PIO_FILL_INT, 5, 6, PIO_FILL_INT, 7, 8, PIO_FILL_INT}; + int map_1x2x3[] = {0, 1, 2, 3, 4, PIO_FILL_INT, 5, PIO_FILL_INT}; +#define NUM_DIM_COMBOS_TO_TEST 5 + int dim_len[NUM_DIM_COMBOS_TO_TEST][NDIM3] = {{1, 4, 4}, + {2, 4, 4}, + {3, 4, 4}, + {1, 3, 3}, + {1, 2, 3}}; + int expected_maplen[NUM_DIM_COMBOS_TO_TEST][TARGET_NTASKS] = {{4, 4, 4, 4}, + {8, 8, 8, 8}, + {12, 12, 12, 12}, + {3, 2, 2, 2}, + {2, 2, 1, 1}}; + int *expected_map[NUM_DIM_COMBOS_TO_TEST] = {map_1x4x4, map_2x4x4, map_3x4x4, map_1x3x3, map_1x2x3}; + int ret; /* Return code. */ + + /* Figure out iotypes. */ + if ((ret = get_iotypes(&num_flavors, flavor))) + ERR(ret); + printf("Runnings tests for %d flavors\n", num_flavors); + + for (int r = 0; r < NUM_REARRANGERS_TO_TEST; r++) + { + int ioid; /* Decomposition ID. */ + + /* Initialize the PIO IO system. This specifies how + * many and which processors are involved in I/O. */ + if ((ret = PIOc_Init_Intracomm(test_comm, TARGET_NTASKS, ioproc_stride, + ioproc_start, rearranger[r], &iosysid))) + return ret; + + /* Run tests for each data type. */ + for (int t = 0; t < NUM_TYPES_TO_TEST; t++) + { + for (int dc = 0; dc < NUM_DIM_COMBOS_TO_TEST; dc++) + { + /* What is length of map for this combo? */ + int full_maplen = 1; + for (int d = 0; d < NDIM3; d++) + full_maplen *= dim_len[dc][d]; + + /* Decompose the data over the tasks. */ + if ((ret = create_decomposition_3d(TARGET_NTASKS, my_rank, iosysid, dim_len[dc], + test_type[t], &ioid))) + return ret; + + /* Test decomposition read/write. */ + if ((ret = test_decomp_read_write(iosysid, ioid, num_flavors, flavor, my_rank, + rearranger[r], test_comm, dim_len[dc], + expected_maplen[dc], test_type[t], full_maplen, + expected_map[dc]))) + return ret; + + /* Free the PIO decomposition. */ + if ((ret = PIOc_freedecomp(iosysid, ioid))) + ERR(ret); + } + } + + /* Finalize PIO system. */ + if ((ret = PIOc_finalize(iosysid))) + return ret; + + } /* next rearranger */ + } /* endif my_rank < TARGET_NTASKS */ + + /* Finalize the MPI library. */ + printf("%d %s Finalizing...\n", my_rank, TEST_NAME); + if ((ret = pio_test_finalize(&test_comm))) + return ret; + + printf("%d %s SUCCESS!!\n", my_rank, TEST_NAME); + return 0; +} diff --git a/src/externals/pio2/tests/cunit/test_decomps.c b/src/externals/pio2/tests/cunit/test_decomps.c index 032d10de7b0..96dc1a386e2 100644 --- a/src/externals/pio2/tests/cunit/test_decomps.c +++ b/src/externals/pio2/tests/cunit/test_decomps.c @@ -290,8 +290,7 @@ int test_decomp_read_write(int iosysid, int ioid, int num_flavors, int *flavor, decomp_file_type); printf("writing decomp file %s\n", filename); - if ((ret = PIOc_write_nc_decomp(iosysid, filename, cmode, ioid, test_comm, NULL, - NULL, 0))) + if ((ret = PIOc_write_nc_decomp(iosysid, filename, cmode, ioid, NULL, NULL, 0))) return ret; /* Read the data. */ diff --git a/src/externals/pio2/tests/cunit/test_intercomm2.c b/src/externals/pio2/tests/cunit/test_intercomm2.c index 0a6f4a5b2ff..e55c6e1ff2e 100644 --- a/src/externals/pio2/tests/cunit/test_intercomm2.c +++ b/src/externals/pio2/tests/cunit/test_intercomm2.c @@ -253,7 +253,7 @@ int check_file(int iosysid, int format, char *filename, int my_rank) ERR(ERR_WRONG); if (PIOc_inq_att(ncid + TEST_VAL_42, NC_GLOBAL, too_long_name, &atttype, &attlen) != PIO_EBADID) ERR(ERR_WRONG); - if (PIOc_get_att(ncid + TEST_VAL_42, NC_GLOBAL, ATT_NAME, &att_data) != PIO_EBADID) + if (PIOc_get_att(ncid, NC_GLOBAL, TEST_NAME, &att_data) != PIO_ENOTATT) ERR(ERR_WRONG); if (PIOc_get_att(ncid, NC_GLOBAL, NULL, &att_data) != PIO_EINVAL) ERR(ERR_WRONG); @@ -317,7 +317,7 @@ int main(int argc, char **argv) ERR(ret); /* How many processors will be used for our IO and 2 computation components. */ - int num_procs[COMPONENT_COUNT + 1] = {2, 2}; + int num_procs[COMPONENT_COUNT] = {2}; /* Is the current process a computation task? */ int comp_task = my_rank < 2 ? 0 : 1; @@ -327,8 +327,8 @@ int main(int argc, char **argv) int my_comp_idx = comp_task ? 0 : -1; /* Initialize the IO system. */ - if ((ret = PIOc_Init_Async(test_comm, NUM_IO_PROCS, NULL, COMPONENT_COUNT, - num_procs, NULL, NULL, NULL, iosysid))) + if ((ret = PIOc_init_async(test_comm, NUM_IO_PROCS, NULL, COMPONENT_COUNT, + num_procs, NULL, NULL, NULL, PIO_REARR_BOX, iosysid))) ERR(ERR_AWFUL); printf("%d: test_intercomm2 ParallelIO Library test_intercomm2 comp task returned.\n", @@ -517,7 +517,6 @@ int main(int argc, char **argv) for (int i = 0; i < DIM_LEN; i++) data[i] = i; printf("%d test_intercomm2 writing data\n", my_rank); - printf("%d test_intercomm2 writing data\n", my_rank); start[0] = 0; count[0] = DIM_LEN; if ((ret = PIOc_put_vars_tc(ncid, varid, start, count, NULL, NC_INT, data))) diff --git a/src/externals/pio2/tests/cunit/test_iosystem2_simple.c b/src/externals/pio2/tests/cunit/test_iosystem2_simple.c index 4cf217e72e5..66ae617372b 100644 --- a/src/externals/pio2/tests/cunit/test_iosystem2_simple.c +++ b/src/externals/pio2/tests/cunit/test_iosystem2_simple.c @@ -97,6 +97,8 @@ int main(int argc, char **argv) ERR(ret); if (active) ERR(ERR_WRONG); + if ((ret = PIOc_iosystem_is_active(iosysid, NULL))) + ERR(ret); int numiotasks; if (PIOc_get_numiotasks(iosysid + TEST_VAL_42, &numiotasks) != PIO_EBADID) diff --git a/src/externals/pio2/tests/cunit/test_pioc.c b/src/externals/pio2/tests/cunit/test_pioc.c index 3cffd645463..466ed99f706 100644 --- a/src/externals/pio2/tests/cunit/test_pioc.c +++ b/src/externals/pio2/tests/cunit/test_pioc.c @@ -100,22 +100,23 @@ int create_decomposition(int ntasks, int my_rank, int iosysid, int dim1_len, int if (!(compdof = malloc(elements_per_pe * sizeof(PIO_Offset)))) return PIO_ENOMEM; - /* Describe the decomposition. This is a 1-based array, so add 1! */ + /* Describe the decomposition. The new init_decomp uses a 0-based + * array, so don't add 1! */ for (int i = 0; i < elements_per_pe; i++) - compdof[i] = my_rank * elements_per_pe + i + 1; + compdof[i] = my_rank * elements_per_pe + i; /* These should fail. */ - if (PIOc_InitDecomp(iosysid + TEST_VAL_42, PIO_FLOAT, NDIM1, dim_len, elements_per_pe, - compdof, ioid, NULL, NULL, NULL) != PIO_EBADID) + if (PIOc_init_decomp(iosysid + TEST_VAL_42, PIO_FLOAT, NDIM1, dim_len, elements_per_pe, + compdof, ioid, 0, NULL, NULL) != PIO_EBADID) ERR(ERR_WRONG); - if (PIOc_InitDecomp(iosysid, PIO_FLOAT, NDIM1, bad_dim_len, elements_per_pe, - compdof, ioid, NULL, NULL, NULL) != PIO_EINVAL) + if (PIOc_init_decomp(iosysid, PIO_FLOAT, NDIM1, bad_dim_len, elements_per_pe, + compdof, ioid, 0, NULL, NULL) != PIO_EINVAL) ERR(ERR_WRONG); /* Create the PIO decomposition for this test. */ printf("%d Creating decomposition elements_per_pe = %lld\n", my_rank, elements_per_pe); - if ((ret = PIOc_InitDecomp(iosysid, PIO_FLOAT, NDIM1, dim_len, elements_per_pe, - compdof, ioid, NULL, NULL, NULL))) + if ((ret = PIOc_init_decomp(iosysid, PIO_FLOAT, NDIM1, dim_len, elements_per_pe, + compdof, ioid, 0, NULL, NULL))) ERR(ret); printf("%d decomposition initialized.\n", my_rank); @@ -315,7 +316,7 @@ int check_var_name(int my_rank, int ncid, MPI_Comm test_comm) * @param flavor the iotype * @param test_comm the MPI communicator of the test. * @param async 1 if we are testing async, 0 otherwise. - * @returns 0 for success, error code otherwise. + * @returns 0 for success, error code otherwise. */ int check_atts(int my_rank, int ncid, int flavor, MPI_Comm test_comm, int async) { @@ -391,7 +392,7 @@ int check_atts(int my_rank, int ncid, int flavor, MPI_Comm test_comm, int async) ERR(ret); if (att_int_value2 != ATT_VAL) return ERR_WRONG; - + /* Check second att. */ if ((ret = PIOc_inq_att(ncid, NC_GLOBAL, ATT_NAME2, &att_type, &att_len))) return ret; @@ -494,7 +495,7 @@ int test_iotypes(int my_rank) /* This is never present. */ if (PIOc_iotype_available(1000)) return ERR_WRONG; - + /* NetCDF is always present. */ if (!PIOc_iotype_available(PIO_IOTYPE_NETCDF)) return ERR_WRONG; @@ -740,26 +741,26 @@ int define_metadata(int ncid, int my_rank, int flavor) return ERR_WRONG; if (PIOc_def_var_fill(ncid, varid, NC_FILL, NULL) != PIO_EINVAL) return ERR_WRONG; - + /* Set the fill value. */ if ((ret = PIOc_def_var_fill(ncid, varid, NC_FILL, &int_fill))) return ret; - + /* These should not work. */ if (PIOc_inq_var_fill(ncid + TEST_VAL_42, varid, &fill_mode, &int_fill_in) != PIO_EBADID) return ERR_WRONG; if (PIOc_inq_var_fill(ncid, varid + TEST_VAL_42, &fill_mode, &int_fill_in) != PIO_ENOTVAR) return ERR_WRONG; - + /* Check the fill value. */ if ((ret = PIOc_inq_var_fill(ncid, varid, &fill_mode, &int_fill_in))) return ret; if (fill_mode != NC_FILL || int_fill_in != int_fill) ERR(ERR_WRONG); - + /* These should also work. */ int_fill_in = 0; - + /* This does not work for pnetcdf, but probably should. */ if (flavor != PIO_IOTYPE_PNETCDF) { @@ -886,10 +887,14 @@ int test_names(int iosysid, int num_flavors, int *flavor, int my_rank, return ERR_WRONG; if (PIOc_setframe(ncid, -1, 0) != PIO_EINVAL) return ERR_WRONG; + if (PIOc_setframe(ncid, NC_MAX_VARS + 1, 0) != PIO_EINVAL) + return ERR_WRONG; if (PIOc_advanceframe(ncid + TEST_VAL_42, 0) != PIO_EBADID) return ERR_WRONG; if (PIOc_advanceframe(ncid, -1) != PIO_EINVAL) return ERR_WRONG; + if (PIOc_advanceframe(ncid, NC_MAX_VARS + 1) != PIO_EINVAL) + return ERR_WRONG; /* Check the dimension names. */ if ((ret = check_dim_names(my_rank, ncid, test_comm))) @@ -1398,6 +1403,171 @@ int test_nc4(int iosysid, int num_flavors, int *flavor, int my_rank) return PIO_NOERR; } +/* This function is part of test_scalar(). It tests the contents of + * the scalar var. */ +int check_scalar_var(int ncid, int varid, int flavor) +{ + char var_name_in[PIO_MAX_NAME + 1]; + int var_type_in; + int ndims_in; + int natts_in; + int val_in; + int ret; + + /* Learn the var metadata. */ + if ((ret = PIOc_inq_var(ncid, varid, var_name_in, &var_type_in, &ndims_in, NULL, + &natts_in))) + return ret; + + /* Is the metadata correct? */ + if (strcmp(var_name_in, VAR_NAME) || var_type_in != PIO_INT || ndims_in != 0 || natts_in != 0) + return ERR_WRONG; + + /* Get the value. */ + if ((ret = PIOc_get_var_int(ncid, varid, &val_in))) + return ret; + printf("val_in = %d\n", val_in); + + /* Is the value correct? */ + if (val_in != TEST_VAL_42) + return ERR_WRONG; + + return 0; +} + +/* Test scalar vars. */ +int test_scalar(int iosysid, int num_flavors, int *flavor, int my_rank, int async, + MPI_Comm test_comm) +{ + int ncid; /* The ncid of the netCDF file. */ + int varid; /* The ID of the netCDF varable. */ + int ret; /* Return code. */ + + /* Use netCDF classic to create a file with a scalar var, then set + * and read the value. */ + if (my_rank == 0) + { + char test_file[] = "netcdf_test.nc"; + int test_val = TEST_VAL_42; + int test_val_in; + + if ((ret = nc_create(test_file, NC_CLOBBER, &ncid))) + return ret; + if ((ret = nc_def_var(ncid, VAR_NAME, NC_INT, 0, NULL, &varid))) + return ret; + if ((ret = nc_enddef(ncid))) + return ret; + if ((ret = nc_put_var(ncid, varid, &test_val))) + return ret; + if ((ret = nc_close(ncid))) + return ret; + if ((ret = nc_open(test_file, NC_NOWRITE, &ncid))) + return ret; + /* if ((ret = nc_get_var(ncid, varid, &test_val_in))) */ + /* return ret; */ + /* if (test_val_in != test_val) */ + /* return ERR_WRONG; */ + if ((ret = nc_get_vars(ncid, varid, NULL, NULL, NULL, &test_val_in))) + return ret; + if (test_val_in != test_val) + return ERR_WRONG; + if ((ret = nc_close(ncid))) + return ret; + } + + /* Use pnetCDF to create a file with a scalar var, then set and + * read the value. */ +#ifdef _PNETCDF + { + char test_file[] = "pnetcdf_test.nc"; + int test_val = TEST_VAL_42; + int test_val_in; + + if ((ret = ncmpi_create(test_comm, test_file, NC_CLOBBER, MPI_INFO_NULL, &ncid))) + return ret; + if ((ret = ncmpi_def_var(ncid, VAR_NAME, NC_INT, 0, NULL, &varid))) + return ret; + if ((ret = ncmpi_enddef(ncid))) + return ret; + if ((ret = ncmpi_put_var_int_all(ncid, varid, &test_val))) + return ret; + if ((ret = ncmpi_close(ncid))) + return ret; + if ((ret = ncmpi_open(test_comm, test_file, NC_NOWRITE, MPI_INFO_NULL, &ncid))) + return ret; + /* Turn on independent access for pnetcdf file. */ + if ((ret = ncmpi_begin_indep_data(ncid))) + return ret; + /* if ((ret = ncmpi_get_var_int(ncid, varid, &test_val_in))) */ + /* return ret; */ + if ((ret = ncmpi_get_vars_int(ncid, varid, NULL, NULL, NULL, &test_val_in))) + return ret; + if ((ret = ncmpi_end_indep_data(ncid))) + return ret; + if (test_val_in != test_val) + return ERR_WRONG; + printf("ret = %d test_val_in = %d\n", ret, test_val_in); + if (test_val_in != test_val) + return ERR_WRONG; + if ((ret = ncmpi_close(ncid))) + return ret; + } +#endif /* _PNETCDF */ + + /* Use PIO to create the example file in each of the four + * available ways. */ + for (int fmt = 0; fmt < num_flavors; fmt++) + { + char filename[PIO_MAX_NAME + 1]; /* Test filename. */ + char iotype_name[PIO_MAX_NAME + 1]; + + /* Create a filename. */ + if ((ret = get_iotype_name(flavor[fmt], iotype_name))) + return ret; + sprintf(filename, "%s_%s_scalar_async_%d.nc", TEST_NAME, iotype_name, async); + + /* Create the netCDF output file. */ + printf("%d Creating test file %s.\n", my_rank, filename); + if ((ret = PIOc_createfile(iosysid, &ncid, &(flavor[fmt]), filename, PIO_CLOBBER))) + ERR(ret); + + /* Define a scalar variable. */ + if ((ret = PIOc_def_var(ncid, VAR_NAME, PIO_INT, 0, NULL, &varid))) + ERR(ret); + + /* End define mode. */ + if ((ret = PIOc_enddef(ncid))) + ERR(ret); + + /* Write a scalar value. */ + int test_val = TEST_VAL_42; + if ((ret = PIOc_put_var_int(ncid, varid, &test_val))) + ERR(ret); + + /* Check the scalar var. */ + if ((ret = check_scalar_var(ncid, varid, flavor[fmt]))) + ERR(ret); + + /* Close the netCDF file. */ + printf("%d Closing the sample data file...\n", my_rank); + if ((ret = PIOc_closefile(ncid))) + ERR(ret); + + /* Reopen the file. */ + if ((ret = PIOc_openfile(iosysid, &ncid, &(flavor[fmt]), filename, PIO_NOWRITE))) + ERR(ret); + + /* Check the scalar var again. */ + if ((ret = check_scalar_var(ncid, varid, flavor[fmt]))) + ERR(ret); + + /* Close the netCDF file. */ + if ((ret = PIOc_closefile(ncid))) + ERR(ret); + } + return PIO_NOERR; +} + /** Test the malloc_iodesc() function. * * @param my_rank rank of this task. @@ -1411,11 +1581,13 @@ int test_malloc_iodesc2(int iosysid, int my_rank) #else int num_types = NUM_CLASSIC_TYPES; #endif /* _NETCDF4 */ - int test_type[NUM_NETCDF_TYPES] = {PIO_BYTE, PIO_CHAR, PIO_SHORT, PIO_INT, PIO_FLOAT, PIO_DOUBLE, - PIO_UBYTE, PIO_USHORT, PIO_UINT, PIO_INT64, PIO_UINT64, PIO_STRING}; - int mpi_type[NUM_NETCDF_TYPES] = {MPI_BYTE, MPI_CHAR, MPI_SHORT, MPI_INT, MPI_FLOAT, MPI_DOUBLE, - MPI_UNSIGNED_CHAR, MPI_UNSIGNED_SHORT, MPI_UNSIGNED, MPI_LONG_LONG, - MPI_UNSIGNED_LONG_LONG, MPI_CHAR}; + int test_type[NUM_NETCDF_TYPES] = {PIO_BYTE, PIO_CHAR, PIO_SHORT, PIO_INT, + PIO_FLOAT, PIO_DOUBLE, PIO_UBYTE, PIO_USHORT, + PIO_UINT, PIO_INT64, PIO_UINT64, PIO_STRING}; + MPI_Datatype mpi_type[NUM_NETCDF_TYPES] = {MPI_BYTE, MPI_CHAR, MPI_SHORT, MPI_INT, + MPI_FLOAT, MPI_DOUBLE, MPI_UNSIGNED_CHAR, + MPI_UNSIGNED_SHORT, MPI_UNSIGNED, MPI_LONG_LONG, + MPI_UNSIGNED_LONG_LONG, MPI_CHAR}; int ioid; iosystem_desc_t *ios; io_desc_t *iodesc; @@ -1423,10 +1595,11 @@ int test_malloc_iodesc2(int iosysid, int my_rank) if (!(ios = pio_get_iosystem_from_id(iosysid))) return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__); - + printf("test_malloc_iodesc2 num_types %d\n",num_types); /* Test with each type. */ for (int t = 0; t < num_types; t++) { + if ((ret = malloc_iodesc(ios, test_type[t], 1, &iodesc))) return ret; if (iodesc->basetype != mpi_type[t]) @@ -1439,7 +1612,6 @@ int test_malloc_iodesc2(int iosysid, int my_rank) if ((ret = pio_delete_iodesc_from_list(ioid))) return ret; } - return 0; } @@ -1450,7 +1622,7 @@ int test_decomp_internal(int my_test_size, int my_rank, int iosysid, int dim_len int ioid; char filename[NC_MAX_NAME + 1]; /* Test decomp filename. */ char nc_filename[NC_MAX_NAME + 1]; /* Test decomp filename (netcdf version). */ - char too_long_name[PIO_MAX_NAME * 5 + 1]; + iosystem_desc_t *ios; /* IO system info. */ int ret; /* This will be our file name for writing out decompositions. */ @@ -1472,37 +1644,12 @@ int test_decomp_internal(int my_test_size, int my_rank, int iosysid, int dim_len int task_maplen[TARGET_NTASKS] = {1, 1, 1, 1}; int map[TARGET_NTASKS][1] = {{0},{1},{2},{3}}; - /* These should not work. */ - memset(too_long_name, 74, PIO_MAX_NAME * 5); - too_long_name[PIO_MAX_NAME * 5] = 0; - if (pioc_write_nc_decomp_int(iosysid + TEST_VAL_42, nc_filename, 0, NDIM1, global_dimlen, - TARGET_NTASKS, task_maplen, (int *)map, title, - history, 0) != PIO_EBADID) - return ERR_WRONG; - if (pioc_write_nc_decomp_int(iosysid, NULL, 0, NDIM1, global_dimlen, - TARGET_NTASKS, task_maplen, (int *)map, title, - history, 0) != PIO_EINVAL) - return ERR_WRONG; - if (pioc_write_nc_decomp_int(iosysid, nc_filename, 0, NDIM1, NULL, - TARGET_NTASKS, task_maplen, (int *)map, title, - history, 0) != PIO_EINVAL) - return ERR_WRONG; - if (pioc_write_nc_decomp_int(iosysid, nc_filename, 0, NDIM1, global_dimlen, - TARGET_NTASKS, NULL, (int *)map, title, - history, 0) != PIO_EINVAL) - return ERR_WRONG; - if (pioc_write_nc_decomp_int(iosysid, nc_filename, 0, NDIM1, global_dimlen, - TARGET_NTASKS, task_maplen, (int *)map, too_long_name, - history, 0) != PIO_EINVAL) - return ERR_WRONG; - if (pioc_write_nc_decomp_int(iosysid, nc_filename, 0, NDIM1, global_dimlen, - TARGET_NTASKS, task_maplen, (int *)map, title, - too_long_name, 0) != PIO_EINVAL) - return ERR_WRONG; - + /* Get the IO system info. */ + if (!(ios = pio_get_iosystem_from_id(iosysid))) + return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__); /* Write the decomposition file. */ - if ((ret = pioc_write_nc_decomp_int(iosysid, nc_filename, 0, NDIM1, global_dimlen, + if ((ret = pioc_write_nc_decomp_int(ios, nc_filename, 0, NDIM1, global_dimlen, TARGET_NTASKS, task_maplen, (int *)map, title, history, 0))) return ret; @@ -1515,7 +1662,7 @@ int test_decomp_internal(int my_test_size, int my_rank, int iosysid, int dim_len char source_in[PIO_MAX_NAME + 1]; char version_in[PIO_MAX_NAME + 1]; char expected_source[] = "Decomposition file produced by PIO library."; - int *global_dimlen_in; + int *global_dimlen_in; int *task_maplen_in; int *map_in; int fortran_order_in; @@ -1540,6 +1687,7 @@ int test_decomp_internal(int my_test_size, int my_rank, int iosysid, int dim_len history_in, source_in, version_in, &fortran_order_in))) return ret; + /* Did we get the correct answers? */ printf("source_in = %s\n", source_in); if (strcmp(title, title_in) || strcmp(history, history_in) || @@ -1650,7 +1798,6 @@ int test_decomp_internal(int my_test_size, int my_rank, int iosysid, int dim_len free(task_maplen_in); free(map_in); - /* Free the PIO decomposition. */ if ((ret = PIOc_freedecomp(iosysid, ioid))) ERR(ret); @@ -1665,7 +1812,7 @@ int test_decomp_public(int my_test_size, int my_rank, int iosysid, int dim_len, int ioid; char nc_filename[NC_MAX_NAME + 1]; /* Test decomp filename (netcdf version). */ int ret; - + /* This will be our file name for writing out decompositions. */ sprintf(nc_filename, "nc_decomp_%s_rank_%d_async_%d.nc", TEST_NAME, my_rank, async); @@ -1679,15 +1826,29 @@ int test_decomp_public(int my_test_size, int my_rank, int iosysid, int dim_len, char *history = "Added to PIO automatic testing by Ed in February 2017."; /* These should not work. */ - if (PIOc_write_nc_decomp(iosysid + TEST_VAL_42, nc_filename, 0, ioid, test_comm, title, history, 0) != PIO_EBADID) + char too_long_name[PIO_MAX_NAME * 5 + 1]; + memset(too_long_name, 74, PIO_MAX_NAME * 5); + too_long_name[PIO_MAX_NAME * 5] = 0; + + if (PIOc_write_nc_decomp(iosysid + TEST_VAL_42, nc_filename, 0, ioid, + title, history, 0) != PIO_EBADID) return ERR_WRONG; - if (PIOc_write_nc_decomp(iosysid, NULL, 0, ioid, test_comm, title, history, 0) != PIO_EINVAL) + if (PIOc_write_nc_decomp(iosysid, NULL, 0, ioid, title, history, 0) != PIO_EINVAL) return ERR_WRONG; - if (PIOc_write_nc_decomp(iosysid, nc_filename, 0, ioid + TEST_VAL_42, test_comm, title, history, 0) != PIO_EBADID) + if (PIOc_write_nc_decomp(iosysid, nc_filename, 0, ioid + TEST_VAL_42, + title, history, 0) != PIO_EBADID) + return ERR_WRONG; + + if (PIOc_write_nc_decomp(iosysid, nc_filename, 0, ioid, + too_long_name, history, 0) != PIO_EINVAL) + return ERR_WRONG; + if (PIOc_write_nc_decomp(iosysid, nc_filename, 0, ioid, + title, too_long_name, 0) != PIO_EINVAL) return ERR_WRONG; /* Write a netCDF decomp file for this iosystem. */ - if ((ret = PIOc_write_nc_decomp(iosysid, nc_filename, 0, ioid, test_comm, title, history, 0))) + if ((ret = PIOc_write_nc_decomp(iosysid, nc_filename, 0, ioid, title, + history, 0))) return ret; int ioid_in; @@ -1696,8 +1857,8 @@ int test_decomp_public(int my_test_size, int my_rank, int iosysid, int dim_len, int fortran_order_in; /* These should not work. */ - if (PIOc_read_nc_decomp(iosysid + TEST_VAL_42, nc_filename, &ioid_in, test_comm, PIO_INT, - title_in, history_in, &fortran_order_in) != PIO_EBADID) + if (PIOc_read_nc_decomp(iosysid + TEST_VAL_42, nc_filename, &ioid_in, test_comm, + PIO_INT, title_in, history_in, &fortran_order_in) != PIO_EBADID) return ret; if (PIOc_read_nc_decomp(iosysid, NULL, &ioid_in, test_comm, PIO_INT, title_in, history_in, &fortran_order_in) != PIO_EINVAL) @@ -1705,7 +1866,7 @@ int test_decomp_public(int my_test_size, int my_rank, int iosysid, int dim_len, if (PIOc_read_nc_decomp(iosysid, nc_filename, NULL, test_comm, PIO_INT, title_in, history_in, &fortran_order_in) != PIO_EINVAL) return ret; - + /* Read it using the public read function. */ if ((ret = PIOc_read_nc_decomp(iosysid, nc_filename, &ioid_in, test_comm, PIO_INT, title_in, history_in, &fortran_order_in))) @@ -1780,9 +1941,88 @@ int test_decomp_public(int my_test_size, int my_rank, int iosysid, int dim_len, free(map_in); /* /\* These should also work. *\/ */ - /* if ((ret = PIOc_write_nc_decomp(iosysid, nc_filename, 0, ioid, test_comm, title, history, 0))) */ + /* if ((ret = PIOc_write_nc_decomp(iosysid, nc_filename, 0, ioid, title, history, 0))) */ /* return ret; */ + + /* Free the PIO decomposition. */ + if ((ret = PIOc_freedecomp(iosysid, ioid))) + ERR(ret); + + return 0; +} + +/* Test some decomp public API functions. */ +int test_decomp_public_2(int my_test_size, int my_rank, int iosysid, int dim_len, + MPI_Comm test_comm, int async) +{ + int ioid; + char nc_filename[NC_MAX_NAME + 1]; /* Test decomp filename (netcdf version). */ + int ret; + + /* This will be our file name for writing out decompositions. */ + sprintf(nc_filename, "nc_decomp_%s_rank_%d_async_%d.nc", TEST_NAME, my_rank, async); + + /* Decompose the data over the tasks. */ + if ((ret = create_decomposition(my_test_size, my_rank, iosysid, dim_len, &ioid))) + return ret; + + /* Write a netCDF decomp file for this iosystem. */ + if ((ret = PIOc_write_nc_decomp(iosysid, nc_filename, 0, ioid, NULL, NULL, 0))) + return ret; + + /* Free the PIO decomposition. */ + if ((ret = PIOc_freedecomp(iosysid, ioid))) + ERR(ret); + + return 0; +} + +/* Test some decomp public API functions. */ +int test_decomp_2(int my_test_size, int my_rank, int iosysid, int dim_len, + MPI_Comm test_comm, int async) +{ + int ioid; + char nc_filename[NC_MAX_NAME + 1]; /* Test decomp filename (netcdf version). */ + int ret; + + /* This will be our file name for writing out decompositions. */ + sprintf(nc_filename, "nc_decomp_%s_rank_%d_async_%d.nc", TEST_NAME, my_rank, async); + + /* Decompose the data over the tasks. */ + if ((ret = create_decomposition(my_test_size, my_rank, iosysid, dim_len, &ioid))) + return ret; + + /* Free the PIO decomposition. */ + if ((ret = PIOc_freedecomp(iosysid, ioid))) + ERR(ret); + + return 0; +} + +/* Test some decomp public API functions with async. */ +int test_decomp_public_async(int my_test_size, int my_rank, int iosysid, MPI_Comm test_comm, + int async) +{ +#define ELEM1 1 +#define LEN3 3 + int ioid; + int dim_len = LEN3; + PIO_Offset elements_per_pe = ELEM1; + PIO_Offset compdof[ELEM1] = {my_rank + 1}; + char filename[PIO_MAX_NAME + 1]; + int ret; + + sprintf(filename, "async_decomp_%s_rank_%d_async_%d.nc", TEST_NAME, my_rank, async); + /* Create the PIO decomposition for this test. */ + if ((ret = PIOc_init_decomp(iosysid, PIO_FLOAT, NDIM1, &dim_len, elements_per_pe, + compdof, &ioid, PIO_REARR_BOX, NULL, NULL))) + ERR(ret); + + /* Write the decomp file (on appropriate tasks). */ + if ((ret = PIOc_write_nc_decomp(iosysid, filename, 0, ioid, NULL, NULL, 0))) + return ret; + /* Free the PIO decomposition. */ if ((ret = PIOc_freedecomp(iosysid, ioid))) ERR(ret); @@ -1797,6 +2037,7 @@ int test_all(int iosysid, int num_flavors, int *flavor, int my_rank, MPI_Comm te int ioid; int my_test_size; char filename[NC_MAX_NAME + 1]; + char nc_filename[NC_MAX_NAME + 1]; int ret; /* Return code. */ if ((ret = MPI_Comm_size(test_comm, &my_test_size))) @@ -1804,7 +2045,14 @@ int test_all(int iosysid, int num_flavors, int *flavor, int my_rank, MPI_Comm te /* This will be our file name for writing out decompositions. */ sprintf(filename, "decomp_%d.txt", my_rank); + sprintf(nc_filename, "decomp_%d.nc", my_rank); + /* This is a simple test that just creates the decomp with + * async. */ + if (async) + if ((ret = test_decomp_public_async(my_test_size, my_rank, iosysid, test_comm, async))) + return ret; + /* Check iotypes. */ printf("%d Testing iotypes. async = %d\n", my_rank, async); if ((ret = test_iotypes(my_rank))) @@ -1828,25 +2076,25 @@ int test_all(int iosysid, int num_flavors, int *flavor, int my_rank, MPI_Comm te if (!async) if ((ret = test_decomp_internal(my_test_size, my_rank, iosysid, DIM_LEN, test_comm, async))) return ret; - /* Test decomposition public API functions. */ if (!async) if ((ret = test_decomp_public(my_test_size, my_rank, iosysid, DIM_LEN, test_comm, async))) return ret; + /* This is a simple test that just creates a decomp. */ + /* if ((ret = test_decomp_2(my_test_size, my_rank, iosysid, DIM_LEN, test_comm, async))) */ + /* return ret; */ + + /* This is a simple test that just writes the decomp. */ if (!async) - { - printf("%d Testing darray. async = %d\n", my_rank, async); - - /* Decompose the data over the tasks. */ - if ((ret = create_decomposition(my_test_size, my_rank, iosysid, DIM_LEN, &ioid))) + if ((ret = test_decomp_public_2(my_test_size, my_rank, iosysid, DIM_LEN, test_comm, async))) return ret; - /* Write out an ASCII version of the decomp file. */ - printf("%d Calling write_decomp. async = %d\n", my_rank, async); - if ((ret = PIOc_write_decomp(filename, iosysid, ioid, test_comm))) + /* Decompose the data over the tasks. */ + if (!async) + { + if ((ret = create_decomposition(my_test_size, my_rank, iosysid, DIM_LEN, &ioid))) return ret; - printf("%d Called write_decomp. async = %d\n", my_rank, async); /* Run the darray tests. */ for (int fv = 0; fv < 2; fv++) @@ -1872,6 +2120,11 @@ int test_all(int iosysid, int num_flavors, int *flavor, int my_rank, MPI_Comm te printf("%d Testing nc4 functions. async = %d\n", my_rank, async); if ((ret = test_nc4(iosysid, num_flavors, flavor, my_rank))) return ret; + + /* Test scalar var. */ + printf("%d Testing scalar var. async = %d\n", my_rank, async); + if ((ret = test_scalar(iosysid, num_flavors, flavor, my_rank, async, test_comm))) + return ret; return PIO_NOERR; } @@ -1880,6 +2133,6 @@ int test_all(int iosysid, int num_flavors, int *flavor, int my_rank, MPI_Comm te int main(int argc, char **argv) { /* Change the 5th arg to 3 to turn on logging. */ - return run_test_main(argc, argv, MIN_NTASKS, TARGET_NTASKS, 0, + return run_test_main(argc, argv, MIN_NTASKS, TARGET_NTASKS, 3, TEST_NAME, dim_len, COMPONENT_COUNT, NUM_IO_PROCS); } diff --git a/src/externals/pio2/tests/cunit/test_pioc_fill.c b/src/externals/pio2/tests/cunit/test_pioc_fill.c index e2015dee7f0..c699a566be5 100644 --- a/src/externals/pio2/tests/cunit/test_pioc_fill.c +++ b/src/externals/pio2/tests/cunit/test_pioc_fill.c @@ -440,14 +440,18 @@ int create_putget_file(int iosysid, int flavor, int *dim_len, int *varid, const int xtype[NUM_NETCDF_TYPES] = {PIO_BYTE, PIO_CHAR, PIO_SHORT, PIO_INT, PIO_FLOAT, PIO_DOUBLE, PIO_UBYTE, PIO_USHORT, PIO_UINT, PIO_INT64, PIO_UINT64, PIO_STRING}; int ncid; + int old_mode; int ret; /* Create the netCDF output file. */ if ((ret = PIOc_createfile(iosysid, &ncid, &flavor, filename, PIO_CLOBBER))) return ret; + /* This should not work. */ + if (PIOc_set_fill(ncid + TEST_VAL_42, NC_FILL, &old_mode) != PIO_EBADID) + return ret; + /* Turn on fill mode. */ - int old_mode; if ((ret = PIOc_set_fill(ncid, NC_FILL, &old_mode))) return ret; printf("old_mode = %d\n", old_mode); diff --git a/src/externals/pio2/tests/cunit/test_rearr.c b/src/externals/pio2/tests/cunit/test_rearr.c new file mode 100644 index 00000000000..89e278bfc14 --- /dev/null +++ b/src/externals/pio2/tests/cunit/test_rearr.c @@ -0,0 +1,1542 @@ +/* + * This program tests some internal functions in the library related + * to the box and subset rearranger, and the transfer of data betweeen + * IO and computation tasks. + * + * Ed Hartnett, 3/9/17 + */ +#include +#include +#include + +/* The number of tasks this test should run on. */ +#define TARGET_NTASKS 4 + +/* The minimum number of tasks this test should run on. */ +#define MIN_NTASKS 1 + +/* The name of this test. */ +#define TEST_NAME "test_rearr" + +/* For 1-D use. */ +#define NDIM1 1 + +/* For maplens of 2. */ +#define MAPLEN2 2 + +/* Name of test var. (Name of a Welsh town.)*/ +#define VAR_NAME "Llanfairpwllgwyngyllgogerychwyrndrobwllllantysiliogogogoch" + +/* Test some of the rearranger utility functions. */ +int test_rearranger_opts1(int iosysid) +{ + iosystem_desc_t *ios; + int ret; + + /* This should not work. */ + if (PIOc_set_rearr_opts(TEST_VAL_42, 0, 0, false, false, 0, false, + false, 0) != PIO_EBADID) + return ERR_WRONG; + if (PIOc_set_rearr_opts(iosysid, TEST_VAL_42, 0, false, false, 0, false, + false, 0) != PIO_EINVAL) + return ERR_WRONG; + if (PIOc_set_rearr_opts(iosysid, 0, TEST_VAL_42, false, false, 0, false, + false, 0) != PIO_EINVAL) + return ERR_WRONG; + if (PIOc_set_rearr_opts(iosysid, 0, 0, false, false, + PIO_REARR_COMM_UNLIMITED_PEND_REQ - 1, false, + false, 0) != PIO_EINVAL) + return ERR_WRONG; + if (PIOc_set_rearr_opts(iosysid, 0, 0, false, false, 0, false, + false, PIO_REARR_COMM_UNLIMITED_PEND_REQ - 1) != + PIO_EINVAL) + return ERR_WRONG; + + /* This should work. */ + if ((ret = PIOc_set_rearr_opts(iosysid, PIO_REARR_COMM_P2P, + PIO_REARR_COMM_FC_1D_COMP2IO, true, + true, TEST_VAL_42, true, true, TEST_VAL_42 + 1))) + return ret; + + /* Get the IO system info from the id. */ + if (!(ios = pio_get_iosystem_from_id(iosysid))) + return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__); + + /* Check the rearranger settings. */ + if (ios->rearr_opts.comm_type != PIO_REARR_COMM_P2P || + ios->rearr_opts.fcd != PIO_REARR_COMM_FC_1D_COMP2IO || + !ios->rearr_opts.comp2io.hs || !ios->rearr_opts.comp2io.isend || + !ios->rearr_opts.io2comp.hs || !ios->rearr_opts.io2comp.isend || + ios->rearr_opts.comp2io.max_pend_req != TEST_VAL_42 || + ios->rearr_opts.io2comp.max_pend_req != TEST_VAL_42 + 1) + return ERR_WRONG; + + return 0; +} + +/* Test the compare_offsets() function. */ +int test_compare_offsets() +{ + mapsort m1, m2, m3; + + m1.rfrom = 0; + m1.soffset = 0; + m1.iomap = 0; + m2.rfrom = 0; + m2.soffset = 0; + m2.iomap = 0; + m3.rfrom = 0; + m3.soffset = 0; + m3.iomap = 1; + + /* Return 0 if either or both parameters are null. */ + if (compare_offsets(NULL, &m2)) + return ERR_WRONG; + if (compare_offsets(&m1, NULL)) + return ERR_WRONG; + if (compare_offsets(NULL, NULL)) + return ERR_WRONG; + + /* m1 and m2 are the same. */ + if (compare_offsets(&m1, &m2)) + return ERR_WRONG; + + /* m1 and m3 are different. */ + if (compare_offsets(&m1, &m3) != -1) + return ERR_WRONG; + return 0; +} + +/* Test the ceil2() and pair() functions. */ +int test_ceil2_pair() +{ + /* Test the ceil2() function. */ + if (ceil2(1) != 1) + return ERR_WRONG; + if (ceil2(-100) != 1) + return ERR_WRONG; + if (ceil2(2) != 2) + return ERR_WRONG; + if (ceil2(3) != 4) + return ERR_WRONG; + if (ceil2(16) != 16) + return ERR_WRONG; + if (ceil2(17) != 32) + return ERR_WRONG; + + /* Test the pair() function. */ + if (pair(4, 0, 0) != 1) + return ERR_WRONG; + if (pair(4, 2, 2) != 1) + return ERR_WRONG; + + return 0; +} + +/* Test the create_mpi_datatypes() function. + * @returns 0 for success, error code otherwise.*/ +int test_create_mpi_datatypes() +{ + MPI_Datatype basetype = MPI_INT; + int *mfrom = NULL; + int mpierr; + int ret; + + { + int msgcnt = 1; + PIO_Offset mindex[1] = {0}; + int mcount[1] = {1}; + MPI_Datatype mtype; + + /* Create an MPI data type. */ + if ((ret = create_mpi_datatypes(basetype, msgcnt, mindex, mcount, mfrom, &mtype))) + return ret; + + /* Free the type. */ + if ((mpierr = MPI_Type_free(&mtype))) + MPIERR(mpierr); + } + + { + int msgcnt = 4; + PIO_Offset mindex[4] = {0, 0, 0, 0}; + int mcount[4] = {1, 1, 1, 1}; + MPI_Datatype mtype2[4]; + + /* Create 4 MPI data types. */ + if ((ret = create_mpi_datatypes(basetype, msgcnt, mindex, mcount, mfrom, mtype2))) + return ret; + + /* Check the size of the data types. It should be 4. */ + MPI_Aint lb, extent; + for (int t = 0; t < 4; t++) + { + if ((mpierr = MPI_Type_get_extent(mtype2[t], &lb, &extent))) + MPIERR(mpierr); + printf("t = %d lb = %ld extent = %ld\n", t, lb, extent); + if (lb != 0 || extent != 4) + return ERR_WRONG; + } + + /* Free them. */ + for (int t = 0; t < 4; t++) + if ((mpierr = MPI_Type_free(&mtype2[t]))) + return ERR_WRONG; + } + + return 0; +} + +/* Test the idx_to_dim_list() function. */ +int test_idx_to_dim_list() +{ + int ndims = 1; + int gdims[1] = {1}; + PIO_Offset idx = 0; + PIO_Offset dim_list[1]; + + /* This simplest case. */ + idx_to_dim_list(ndims, gdims, idx, dim_list); + + if (dim_list[0] != 0) + return ERR_WRONG; + + /* The case given in the function docs. */ + int ndims2 = 2; + int gdims2[2] = {3, 2}; + PIO_Offset idx2 = 4; + PIO_Offset dim_list2[2]; + + /* According to function docs, we should get 2,0 */ + idx_to_dim_list(ndims2, gdims2, idx2, dim_list2); + printf("dim_list2[0] = %lld\n", dim_list2[0]); + printf("dim_list2[1] = %lld\n", dim_list2[1]); + + /* This is the correct result! */ + if (dim_list2[0] != 2 || dim_list2[1] != 0) + return ERR_WRONG; + + return 0; +} + +/* Test the coord_to_lindex() function. */ +int test_coord_to_lindex() +{ + int ndims = 1; + PIO_Offset lcoord[1] = {0}; + PIO_Offset count[1] = {1}; + PIO_Offset lindex; + + /* Not sure what this function is really doing. */ + lindex = coord_to_lindex(ndims, lcoord, count); + if (lindex != 0) + return ERR_WRONG; + + int ndims2 = 2; + PIO_Offset lcoord2[2] = {0, 0}; + PIO_Offset count2[2] = {1, 1}; + PIO_Offset lindex2; + + lindex2 = coord_to_lindex(ndims2, lcoord2, count2); + if (lindex2 != 0) + return ERR_WRONG; + + int ndims3 = 2; + PIO_Offset lcoord3[2] = {1, 2}; + PIO_Offset count3[2] = {1, 1}; + PIO_Offset lindex3; + + lindex3 = coord_to_lindex(ndims3, lcoord3, count3); + printf("lindex = %lld\n", lindex3); + if (lindex3 != 3) + return ERR_WRONG; + + return 0; +} + +/* Test compute_maxIObuffersize() function. */ +int test_compute_maxIObuffersize(MPI_Comm test_comm, int my_rank) +{ + int ret; + + { + /* This is a simple test with one region containing 1 data + * element. */ + io_desc_t iodesc; + io_region *ior1; + int ndims = 1; + + /* This is how we allocate a region. */ + if ((ret = alloc_region2(NULL, ndims, &ior1))) + return ret; + ior1->next = NULL; + ior1->count[0] = 1; + + iodesc.firstregion = ior1; + iodesc.ndims = 1; + + /* Run the function. Simplest possible case. */ + if ((ret = compute_maxIObuffersize(test_comm, &iodesc))) + return ret; + if (iodesc.maxiobuflen != 1) + return ERR_WRONG; + + /* Free resources for the region. */ + free(ior1->start); + free(ior1->count); + free(ior1); + + } + + { + /* This also has a single region, but with 2 dims and count + * values > 1. */ + io_desc_t iodesc; + io_region *ior2; + int ndims = 2; + + /* This is how we allocate a region. */ + if ((ret = alloc_region2(NULL, ndims, &ior2))) + return ret; + + /* These should be 0. */ + for (int i = 0; i < ndims; i++) + if (ior2->start[i] != 0 || ior2->count[i] != 0) + return ERR_WRONG; + + ior2->next = NULL; + ior2->count[0] = 10; + ior2->count[1] = 2; + + iodesc.firstregion = ior2; + iodesc.ndims = 2; + + /* Run the function. */ + if ((ret = compute_maxIObuffersize(test_comm, &iodesc))) + return ret; + if (iodesc.maxiobuflen != 20) + return ERR_WRONG; + + /* Free resources for the region. */ + free(ior2->start); + free(ior2->count); + free(ior2); + } + + { + /* This test has two regions of different sizes. */ + io_desc_t iodesc; + io_region *ior3; + io_region *ior4; + int ndims = 2; + + /* This is how we allocate a region. */ + if ((ret = alloc_region2(NULL, ndims, &ior4))) + return ret; + ior4->next = NULL; + ior4->count[0] = 10; + ior4->count[1] = 2; + + if ((ret = alloc_region2(NULL, ndims, &ior3))) + return ret; + ior3->next = ior4; + ior3->count[0] = 100; + ior3->count[1] = 5; + + iodesc.firstregion = ior3; + iodesc.ndims = 2; + + /* Run the function. */ + if ((ret = compute_maxIObuffersize(test_comm, &iodesc))) + return ret; + printf("iodesc.maxiobuflen = %d\n", iodesc.maxiobuflen); + if (iodesc.maxiobuflen != 520) + return ERR_WRONG; + + /* Free resources for the region. */ + free(ior4->start); + free(ior4->count); + free(ior4); + free(ior3->start); + free(ior3->count); + free(ior3); + } + + return 0; +} + +/* Tests for determine_fill() function. */ +int test_determine_fill(MPI_Comm test_comm) +{ + iosystem_desc_t *ios; + io_desc_t *iodesc; + int gsize[1] = {4}; + PIO_Offset compmap[1] = {1}; + int ret; + + /* Initialize ios. */ + if (!(ios = calloc(1, sizeof(iosystem_desc_t)))) + return PIO_ENOMEM; + ios->union_comm = test_comm; + + /* Set up iodesc for test. */ + if (!(iodesc = calloc(1, sizeof(io_desc_t)))) + return PIO_ENOMEM; + iodesc->ndims = 1; + iodesc->rearranger = PIO_REARR_SUBSET; + iodesc->llen = 1; + + /* We don't need fill. */ + if ((ret = determine_fill(ios, iodesc, gsize, compmap))) + return ret; + if (iodesc->needsfill) + return ERR_WRONG; + + /* Change settings, so now we do need fill. */ + iodesc->llen = 0; + if ((ret = determine_fill(ios, iodesc, gsize, compmap))) + return ret; + if (!iodesc->needsfill) + return ERR_WRONG; + + /* Free test resources. */ + free(ios); + free(iodesc); + + return 0; +} + +/* Run tests for get_start_and_count_regions() funciton. */ +int test_get_regions(int my_rank) +{ +#define MAPLEN 2 + int ndims = NDIM1; + const int gdimlen[NDIM1] = {8}; + /* Don't forget map is 1-based!! */ + PIO_Offset map[MAPLEN] = {(my_rank * 2) + 1, ((my_rank + 1) * 2) + 1}; + int maxregions; + io_region *ior1; + int ret; + + /* This is how we allocate a region. */ + if ((ret = alloc_region2(NULL, NDIM1, &ior1))) + return ret; + ior1->next = NULL; + ior1->count[0] = 1; + + /* Call the function we are testing. */ + if ((ret = get_regions(ndims, gdimlen, MAPLEN, map, &maxregions, ior1))) + return ret; + if (maxregions != 2) + return ERR_WRONG; + + /* Free resources for the region. */ + free(ior1->next->start); + free(ior1->next->count); + free(ior1->next); + free(ior1->start); + free(ior1->count); + free(ior1); + + return 0; +} + +/* Run tests for find_region() function. */ +int test_find_region() +{ + int ndims = NDIM1; + int gdimlen[NDIM1] = {4}; + int maplen = 1; + PIO_Offset map[1] = {1}; + PIO_Offset start[NDIM1]; + PIO_Offset count[NDIM1]; + PIO_Offset regionlen; + + /* Call the function we are testing. */ + regionlen = find_region(ndims, gdimlen, maplen, map, start, count); + + /* Check results. */ + printf("regionlen = %lld start[0] = %lld count[0] = %lld\n", regionlen, start[0], count[0]); + if (regionlen != 1 || start[0] != 0 || count[0] != 1) + return ERR_WRONG; + + return 0; +} + +/* Run tests for expand_region() function. */ +int test_expand_region() +{ + int dim = 0; + int gdims[NDIM1] = {1}; + int maplen = 1; + PIO_Offset map[1] = {5}; + int region_size = 1; + int region_stride = 1; + int max_size[NDIM1] = {10}; + PIO_Offset count[NDIM1]; + + expand_region(dim, gdims, maplen, map, region_size, region_stride, max_size, count); + if (count[0] != 1) + return ERR_WRONG; + printf("max_size[0] = %d count[0] = %lld\n", max_size[0], count[0]); + + return 0; +} + +/* Test define_iodesc_datatypes() function. */ +int test_define_iodesc_datatypes() +{ +#define NUM_REARRANGERS 2 + int rearranger[NUM_REARRANGERS] = {PIO_REARR_BOX, PIO_REARR_SUBSET}; + int mpierr; + int ret; + + /* Run the functon. */ + for (int r = 0; r < NUM_REARRANGERS; r++) + { + iosystem_desc_t ios; + io_desc_t iodesc; + + /* Set up test for IO task with BOX rearranger to create one type. */ + ios.ioproc = 1; /* this is IO proc. */ + ios.num_iotasks = 4; /* The number of IO tasks. */ + iodesc.rtype = NULL; /* Array of MPI types will be created here. */ + iodesc.nrecvs = 1; /* Number of types created. */ + iodesc.basetype = MPI_INT; + iodesc.stype = NULL; /* Array of MPI types will be created here. */ + + /* Allocate space for arrays in iodesc that will be filled in + * define_iodesc_datatypes(). */ + if (!(iodesc.rcount = malloc(iodesc.nrecvs * sizeof(int)))) + return PIO_ENOMEM; + if (!(iodesc.rfrom = malloc(iodesc.nrecvs * sizeof(int)))) + return PIO_ENOMEM; + if (!(iodesc.rindex = malloc(1 * sizeof(PIO_Offset)))) + return PIO_ENOMEM; + iodesc.rindex[0] = 0; + iodesc.rcount[0] = 1; + + iodesc.rearranger = rearranger[r]; + + /* The two rearrangers create a different number of send types. */ + int num_send_types = iodesc.rearranger == PIO_REARR_BOX ? ios.num_iotasks : 1; + + if (!(iodesc.sindex = malloc(num_send_types * sizeof(PIO_Offset)))) + return PIO_ENOMEM; + if (!(iodesc.scount = malloc(num_send_types * sizeof(int)))) + return PIO_ENOMEM; + for (int st = 0; st < num_send_types; st++) + { + iodesc.sindex[st] = 0; + iodesc.scount[st] = 1; + } + + /* Run the test function. */ + if ((ret = define_iodesc_datatypes(&ios, &iodesc))) + return ret; + + /* We created send types, so free them. */ + for (int st = 0; st < num_send_types; st++) + if ((mpierr = MPI_Type_free(&iodesc.stype[st]))) + MPIERR(mpierr); + + /* We created one receive type, so free it. */ + if ((mpierr = MPI_Type_free(&iodesc.rtype[0]))) + MPIERR(mpierr); + + /* Free resources. */ + free(iodesc.rtype); + free(iodesc.sindex); + free(iodesc.scount); + free(iodesc.stype); + free(iodesc.rcount); + free(iodesc.rfrom); + free(iodesc.rindex); + } + + return 0; +} + +/* Test the compute_counts() function with the box rearranger. */ +int test_compute_counts(MPI_Comm test_comm, int my_rank) +{ + iosystem_desc_t *ios; + io_desc_t *iodesc; + int dest_ioproc[TARGET_NTASKS] = {0, 1, 2, 3}; + PIO_Offset dest_ioindex[TARGET_NTASKS] = {0, 1, 2, 3}; + int ret; + + /* Initialize ios. */ + if (!(ios = calloc(1, sizeof(iosystem_desc_t)))) + return PIO_ENOMEM; + + ios->num_iotasks = TARGET_NTASKS; + ios->num_comptasks = TARGET_NTASKS; + ios->num_uniontasks = TARGET_NTASKS; + ios->ioproc = 1; + ios->compproc = 1; + ios->union_comm = test_comm; + if (!(ios->ioranks = malloc(TARGET_NTASKS * sizeof(int)))) + return PIO_ENOMEM; + for (int t = 0; t < TARGET_NTASKS; t++) + ios->ioranks[t] = t; + if (!(ios->compranks = calloc(ios->num_comptasks, sizeof(int)))) + return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); + for (int i = 0; i < TARGET_NTASKS; i++) + ios->compranks[i] = i; + + /* Initialize iodesc. */ + if (!(iodesc = calloc(1, sizeof(io_desc_t)))) + return PIO_ENOMEM; + iodesc->rearranger = PIO_REARR_BOX; + iodesc->ndof = TARGET_NTASKS; + iodesc->llen = TARGET_NTASKS; + iodesc->rearr_opts.comm_type = PIO_REARR_COMM_COLL; + iodesc->rearr_opts.fcd = PIO_REARR_COMM_FC_2D_DISABLE; + + /* Test the function. */ + if ((ret = compute_counts(ios, iodesc, dest_ioproc, dest_ioindex))) + return ret; + + /* Check results. */ + for (int i = 0; i < ios->num_iotasks; i++) + if (iodesc->scount[i] != 1 || iodesc->sindex[i] != i) + return ERR_WRONG; + + for (int i = 0; i < iodesc->ndof; i++) + if (iodesc->rcount[i] != 1 || iodesc->rfrom[i] != i || + iodesc->rindex[i] != my_rank) + return ERR_WRONG; + + /* Free resources allocated in compute_counts(). */ + free(iodesc->scount); + free(iodesc->sindex); + free(iodesc->rcount); + free(iodesc->rfrom); + free(iodesc->rindex); + + /* Free test resources. */ + free(ios->ioranks); + free(ios->compranks); + free(iodesc); + free(ios); + + return 0; +} + +/* Call PIOc_InitDecomp() with parameters such that it calls + * box_rearrange_create() just like test_box_rearrange_create() will + * (see below). */ +int test_init_decomp(int iosysid, MPI_Comm test_comm, int my_rank) +{ + int ioid; + PIO_Offset compmap[MAPLEN2] = {my_rank * 2, (my_rank + 1) * 2}; + const int gdimlen[NDIM1] = {8}; + int ret; + + /* Initialize a decomposition. */ + if ((ret = PIOc_init_decomp(iosysid, PIO_INT, NDIM1, gdimlen, MAPLEN2, + compmap, &ioid, PIO_REARR_BOX, NULL, NULL))) + return ret; + + /* Free it. */ + if ((ret = PIOc_freedecomp(iosysid, ioid))) + return ret; + + return 0; +} + +/* Test for the box_rearrange_create() function. */ +int test_box_rearrange_create(MPI_Comm test_comm, int my_rank) +{ + iosystem_desc_t *ios; + io_desc_t *iodesc; + io_region *ior1; + int maplen = MAPLEN2; + PIO_Offset compmap[MAPLEN2] = {(my_rank * 2) + 1, ((my_rank + 1) * 2) + 1}; + const int gdimlen[NDIM1] = {8}; + int ndims = NDIM1; + int ret; + + /* Allocate IO system info struct for this test. */ + if (!(ios = calloc(1, sizeof(iosystem_desc_t)))) + return PIO_ENOMEM; + + /* Allocate IO desc struct for this test. */ + if (!(iodesc = calloc(1, sizeof(io_desc_t)))) + return PIO_ENOMEM; + + /* Default rearranger options. */ + iodesc->rearr_opts.comm_type = PIO_REARR_COMM_COLL; + iodesc->rearr_opts.fcd = PIO_REARR_COMM_FC_2D_DISABLE; + + /* Set up for determine_fill(). */ + ios->union_comm = test_comm; + ios->io_comm = test_comm; + iodesc->ndims = NDIM1; + iodesc->rearranger = PIO_REARR_BOX; + + /* Set up the IO task info for the test. */ + ios->ioproc = 1; + ios->compproc = 1; + ios->union_rank = my_rank; + ios->num_iotasks = 4; + ios->num_comptasks = 4; + ios->num_uniontasks = 4; + if (!(ios->ioranks = calloc(ios->num_iotasks, sizeof(int)))) + return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); + for (int i = 0; i < TARGET_NTASKS; i++) + ios->ioranks[i] = i; + if (!(ios->compranks = calloc(ios->num_comptasks, sizeof(int)))) + return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); + for (int i = 0; i < TARGET_NTASKS; i++) + ios->compranks[i] = i; + + /* This is how we allocate a region. */ + if ((ret = alloc_region2(NULL, NDIM1, &ior1))) + return ret; + if (my_rank == 0) + ior1->count[0] = 8; + + iodesc->firstregion = ior1; + + /* We are finally ready to run the code under test. */ + if ((ret = box_rearrange_create(ios, maplen, compmap, gdimlen, ndims, iodesc))) + return ret; + + /* Check some results. */ + if (iodesc->rearranger != PIO_REARR_BOX || iodesc->ndof != maplen || + iodesc->llen != my_rank ? 0 : 8 || !iodesc->needsfill) + return ERR_WRONG; + + /* for (int i = 0; i < ios->num_iotasks; i++) */ + /* { */ + /* /\* sindex is only allocated if scount[i] > 0. *\/ */ + /* if (iodesc->scount[i] != i ? 0 : 1 || */ + /* (iodesc->scount[i] && iodesc->sindex[i] != 0)) */ + /* return ERR_WRONG; */ + /* } */ + + /* for (int i = 0; i < iodesc->ndof; i++) */ + /* { */ + /* /\* rcount is 1 for rank 0, 0 on other tasks. *\/ */ + /* if (iodesc->rcount[i] != my_rank ? 0 : 1) */ + /* return ERR_WRONG; */ + + /* /\* rfrom is 0 everywhere, except task 0, array elemnt 1. *\/ */ + /* if (my_rank == 0 && i == 1) */ + /* { */ + /* if (iodesc->rfrom[i] != 1) */ + /* return ERR_WRONG; */ + /* } */ + /* else */ + /* { */ + /* if (iodesc->rfrom[i] != 0) */ + /* return ERR_WRONG; */ + /* } */ + + /* /\* rindex is only allocated where there is a non-zero count. *\/ */ + /* if (iodesc->rcount[i]) */ + /* if (iodesc->rindex[i] != 0) */ + /* return ERR_WRONG; */ + /* } */ + + /* Free resources allocated in compute_counts(). */ + free(iodesc->scount); + free(iodesc->sindex); + free(iodesc->rcount); + free(iodesc->rfrom); + free(iodesc->rindex); + + /* Free resources from test. */ + free(ior1->start); + free(ior1->count); + free(ior1); + free(ios->ioranks); + free(ios->compranks); + free(iodesc); + free(ios); + + return 0; +} + +/* Test for the box_rearrange_create() function. */ +int test_box_rearrange_create_2(MPI_Comm test_comm, int my_rank) +{ +#define MAPLEN2 2 + iosystem_desc_t *ios; + io_desc_t *iodesc; + io_region *ior1; + int maplen = MAPLEN2; + PIO_Offset compmap[MAPLEN2] = {1, 0}; + const int gdimlen[NDIM1] = {8}; + int ndims = NDIM1; + int ret; + + /* Allocate IO system info struct for this test. */ + if (!(ios = calloc(1, sizeof(iosystem_desc_t)))) + return PIO_ENOMEM; + + /* Allocate IO desc struct for this test. */ + if (!(iodesc = calloc(1, sizeof(io_desc_t)))) + return PIO_ENOMEM; + + /* Default rearranger options. */ + iodesc->rearr_opts.comm_type = PIO_REARR_COMM_COLL; + iodesc->rearr_opts.fcd = PIO_REARR_COMM_FC_2D_DISABLE; + + /* Set up for determine_fill(). */ + ios->union_comm = test_comm; + ios->io_comm = test_comm; + iodesc->ndims = NDIM1; + iodesc->rearranger = PIO_REARR_BOX; + + /* This is the size of the map in computation tasks. */ + iodesc->ndof = 2; + + /* Set up the IO task info for the test. */ + ios->ioproc = 1; + ios->compproc = 1; + ios->union_rank = my_rank; + ios->num_iotasks = 4; + ios->num_comptasks = 4; + ios->num_uniontasks = 4; + if (!(ios->ioranks = calloc(ios->num_iotasks, sizeof(int)))) + return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); + for (int i = 0; i < TARGET_NTASKS; i++) + ios->ioranks[i] = i; + if (!(ios->compranks = calloc(ios->num_comptasks, sizeof(int)))) + return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); + for (int i = 0; i < TARGET_NTASKS; i++) + ios->compranks[i] = i; + + /* This is how we allocate a region. */ + if ((ret = alloc_region2(NULL, NDIM1, &ior1))) + return ret; + ior1->next = NULL; + if (my_rank == 0) + ior1->count[0] = 8; + + iodesc->firstregion = ior1; + + /* We are finally ready to run the code under test. */ + if ((ret = box_rearrange_create(ios, maplen, compmap, gdimlen, ndims, iodesc))) + return ret; + + /* Check some results. */ + if (iodesc->rearranger != PIO_REARR_BOX || iodesc->ndof != maplen || + iodesc->llen != my_rank ? 0 : 8 || !iodesc->needsfill) + return ERR_WRONG; + + for (int i = 0; i < ios->num_iotasks; i++) + { + /* sindex is only allocated if scount[i] > 0. */ + if (iodesc->scount[i] != i ? 0 : 1 || + (iodesc->scount[i] && iodesc->sindex[i] != 0)) + return ERR_WRONG; + } + + if (my_rank == 0) + { + for (int i = 0; i < iodesc->ndof; i++) + { + /* rcount is 1 for rank 0, 0 on other tasks. */ + if (iodesc->rcount[i] != 1) + return ERR_WRONG; + + /* rfrom only matters if there is a non-zero count. */ + if (iodesc->rcount[i]) + if (iodesc->rfrom[i] != i ? 1 : 0) + return ERR_WRONG; + + /* rindex is only allocated where there is a non-zero count. */ + if (iodesc->rcount[i]) + if (iodesc->rindex[i] != 0) + return ERR_WRONG; + } + } + + /* Free resources allocated in compute_counts(). */ + free(iodesc->scount); + free(iodesc->sindex); + free(iodesc->rcount); + free(iodesc->rfrom); + free(iodesc->rindex); + + /* Free resources from test. */ + free(ior1->start); + free(ior1->count); + free(ior1); + free(ios->ioranks); + free(ios->compranks); + free(iodesc); + free(ios); + + return 0; +} + +/* Test function default_subset_partition. */ +int test_default_subset_partition(MPI_Comm test_comm, int my_rank) +{ + iosystem_desc_t *ios; + io_desc_t *iodesc; + int mpierr; + int ret; + + /* Allocate IO system info struct for this test. */ + if (!(ios = calloc(1, sizeof(iosystem_desc_t)))) + return PIO_ENOMEM; + + /* Allocate IO desc struct for this test. */ + if (!(iodesc = calloc(1, sizeof(io_desc_t)))) + return PIO_ENOMEM; + + ios->ioproc = 1; + ios->io_rank = my_rank; + ios->comp_comm = test_comm; + + /* Run the function to test. */ + if ((ret = default_subset_partition(ios, iodesc))) + return ret; + + /* Free the created communicator. */ + if ((mpierr = MPI_Comm_free(&iodesc->subset_comm))) + MPIERR(mpierr); + + /* Free resources from test. */ + free(iodesc); + free(ios); + + return 0; +} + +/* Test function rearrange_comp2io. */ +int test_rearrange_comp2io(MPI_Comm test_comm, int my_rank) +{ + iosystem_desc_t *ios; + io_desc_t *iodesc; + void *sbuf = NULL; + void *rbuf = NULL; + int nvars = 1; + io_region *ior1; + int maplen = 2; + PIO_Offset compmap[2] = {1, 0}; + const int gdimlen[NDIM1] = {8}; + int ndims = NDIM1; + int mpierr; + int ret; + + /* Allocate some space for data. */ + if (!(sbuf = calloc(4, sizeof(int)))) + return PIO_ENOMEM; + if (!(rbuf = calloc(4, sizeof(int)))) + return PIO_ENOMEM; + + /* Allocate IO system info struct for this test. */ + if (!(ios = calloc(1, sizeof(iosystem_desc_t)))) + return PIO_ENOMEM; + + /* Allocate IO desc struct for this test. */ + if (!(iodesc = calloc(1, sizeof(io_desc_t)))) + return PIO_ENOMEM; + + ios->ioproc = 1; + ios->compproc = 1; + ios->io_rank = my_rank; + ios->union_comm = test_comm; + ios->num_iotasks = TARGET_NTASKS; + ios->num_uniontasks = TARGET_NTASKS; + iodesc->rearranger = PIO_REARR_BOX; + iodesc->basetype = MPI_INT; + + /* Set up test for IO task with BOX rearranger to create one type. */ + iodesc->rtype = NULL; /* Array of MPI types will be created here. */ + iodesc->nrecvs = 1; /* Number of types created. */ + iodesc->basetype = MPI_INT; + iodesc->stype = NULL; /* Array of MPI types will be created here. */ + + /* The two rearrangers create a different number of send types. */ + int num_send_types = iodesc->rearranger == PIO_REARR_BOX ? ios->num_iotasks : 1; + + /* Default rearranger options. */ + iodesc->rearr_opts.comm_type = PIO_REARR_COMM_COLL; + iodesc->rearr_opts.fcd = PIO_REARR_COMM_FC_2D_DISABLE; + + /* Set up for determine_fill(). */ + ios->union_comm = test_comm; + ios->io_comm = test_comm; + iodesc->ndims = NDIM1; + iodesc->rearranger = PIO_REARR_BOX; + + iodesc->ndof = 4; + + /* Set up the IO task info for the test. */ + ios->union_rank = my_rank; + ios->num_comptasks = 4; + if (!(ios->ioranks = calloc(ios->num_iotasks, sizeof(int)))) + return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); + for (int i = 0; i < TARGET_NTASKS; i++) + ios->ioranks[i] = i; + if (!(ios->compranks = calloc(ios->num_comptasks, sizeof(int)))) + return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); + for (int i = 0; i < TARGET_NTASKS; i++) + ios->compranks[i] = i; + + /* This is how we allocate a region. */ + if ((ret = alloc_region2(NULL, NDIM1, &ior1))) + return ret; + ior1->next = NULL; + if (my_rank == 0) + ior1->count[0] = 8; + + iodesc->firstregion = ior1; + + /* Create the box rearranger. */ + if ((ret = box_rearrange_create(ios, maplen, compmap, gdimlen, ndims, iodesc))) + return ret; + + /* Run the function to test. */ + if ((ret = rearrange_comp2io(ios, iodesc, sbuf, rbuf, nvars))) + return ret; + printf("returned from rearrange_comp2io\n"); + + /* We created send types, so free them. */ + for (int st = 0; st < num_send_types; st++) + if (iodesc->stype[st] != PIO_DATATYPE_NULL) + if ((mpierr = MPI_Type_free(&iodesc->stype[st]))) + MPIERR(mpierr); + + /* We created one receive type, so free it. */ + if (iodesc->rtype) + for (int r = 0; r < iodesc->nrecvs; r++) + if (iodesc->rtype[r] != PIO_DATATYPE_NULL) + if ((mpierr = MPI_Type_free(&iodesc->rtype[r]))) + MPIERR(mpierr); + + /* Free resources allocated in library code. */ + free(iodesc->rtype); + free(iodesc->sindex); + free(iodesc->scount); + free(iodesc->stype); + free(iodesc->rcount); + free(iodesc->rfrom); + free(iodesc->rindex); + + /* Free resources from test. */ + free(ior1->start); + free(ior1->count); + free(ior1); + free(ios->ioranks); + free(ios->compranks); + free(iodesc); + free(ios); + free(sbuf); + free(rbuf); + + return 0; +} + +/* Test function rearrange_io2comp. */ +int test_rearrange_io2comp(MPI_Comm test_comm, int my_rank) +{ + iosystem_desc_t *ios; + io_desc_t *iodesc; + void *sbuf = NULL; + void *rbuf = NULL; + io_region *ior1; + int maplen = 2; + PIO_Offset compmap[2] = {1, 0}; + const int gdimlen[NDIM1] = {8}; + int ndims = NDIM1; + int mpierr; + int ret; + + /* Allocate some space for data. */ + if (!(sbuf = calloc(4, sizeof(int)))) + return PIO_ENOMEM; + if (!(rbuf = calloc(4, sizeof(int)))) + return PIO_ENOMEM; + + /* Allocate IO system info struct for this test. */ + if (!(ios = calloc(1, sizeof(iosystem_desc_t)))) + return PIO_ENOMEM; + + /* Allocate IO desc struct for this test. */ + if (!(iodesc = calloc(1, sizeof(io_desc_t)))) + return PIO_ENOMEM; + + ios->ioproc = 1; + ios->io_rank = my_rank; + ios->union_comm = test_comm; + ios->num_iotasks = TARGET_NTASKS; + iodesc->rearranger = PIO_REARR_BOX; + iodesc->basetype = MPI_INT; + + /* Set up test for IO task with BOX rearranger to create one type. */ + ios->ioproc = 1; /* this is IO proc. */ + ios->num_iotasks = 4; /* The number of IO tasks. */ + iodesc->rtype = NULL; /* Array of MPI types will be created here. */ + iodesc->nrecvs = 1; /* Number of types created. */ + iodesc->basetype = MPI_INT; + iodesc->stype = NULL; /* Array of MPI types will be created here. */ + + /* The two rearrangers create a different number of send types. */ + int num_send_types = iodesc->rearranger == PIO_REARR_BOX ? ios->num_iotasks : 1; + + /* Default rearranger options. */ + iodesc->rearr_opts.comm_type = PIO_REARR_COMM_COLL; + iodesc->rearr_opts.fcd = PIO_REARR_COMM_FC_2D_DISABLE; + + /* Set up for determine_fill(). */ + ios->union_comm = test_comm; + ios->io_comm = test_comm; + iodesc->ndims = NDIM1; + iodesc->rearranger = PIO_REARR_BOX; + + iodesc->ndof = 4; + + /* Set up the IO task info for the test. */ + ios->ioproc = 1; + ios->compproc = 1; + ios->union_rank = my_rank; + ios->num_iotasks = 4; + ios->num_comptasks = 4; + ios->num_uniontasks = 4; + if (!(ios->ioranks = calloc(ios->num_iotasks, sizeof(int)))) + return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); + for (int i = 0; i < TARGET_NTASKS; i++) + ios->ioranks[i] = i; + if (!(ios->compranks = calloc(ios->num_comptasks, sizeof(int)))) + return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); + for (int i = 0; i < TARGET_NTASKS; i++) + ios->compranks[i] = i; + + /* This is how we allocate a region. */ + if ((ret = alloc_region2(NULL, NDIM1, &ior1))) + return ret; + ior1->next = NULL; + if (my_rank == 0) + ior1->count[0] = 8; + + iodesc->firstregion = ior1; + + /* Create the box rearranger. */ + if ((ret = box_rearrange_create(ios, maplen, compmap, gdimlen, ndims, iodesc))) + return ret; + + /* Run the function to test. */ + if ((ret = rearrange_io2comp(ios, iodesc, sbuf, rbuf))) + return ret; + printf("returned from rearrange_comp2io\n"); + + /* We created send types, so free them. */ + for (int st = 0; st < num_send_types; st++) + if (iodesc->stype[st] != PIO_DATATYPE_NULL) + if ((mpierr = MPI_Type_free(&iodesc->stype[st]))) + MPIERR(mpierr); + + /* We created one receive type, so free it. */ + if (iodesc->rtype) + for (int r = 0; r < iodesc->nrecvs; r++) + if (iodesc->rtype[r] != PIO_DATATYPE_NULL) + if ((mpierr = MPI_Type_free(&iodesc->rtype[r]))) + MPIERR(mpierr); + + /* Free resources allocated in library code. */ + free(iodesc->rtype); + free(iodesc->sindex); + free(iodesc->scount); + free(iodesc->stype); + free(iodesc->rcount); + free(iodesc->rfrom); + free(iodesc->rindex); + + /* Free resources from test. */ + free(ior1->start); + free(ior1->count); + free(ior1); + free(ios->ioranks); + free(ios->compranks); + free(iodesc); + free(ios); + free(sbuf); + free(rbuf); + + return 0; +} + +/* These tests do not need an iosysid. */ +int run_no_iosys_tests(int my_rank, MPI_Comm test_comm) +{ + int ret; + + printf("%d running idx_to_dim_list tests\n", my_rank); + if ((ret = test_idx_to_dim_list())) + return ret; + + printf("%d running coord_to_lindex tests\n", my_rank); + if ((ret = test_coord_to_lindex())) + return ret; + + printf("%d running compute_maxIObuffersize tests\n", my_rank); + if ((ret = test_compute_maxIObuffersize(test_comm, my_rank))) + return ret; + + printf("%d running determine_fill\n", my_rank); + if ((ret = test_determine_fill(test_comm))) + return ret; + + printf("%d running tests for expand_region()\n", my_rank); + if ((ret = test_expand_region())) + return ret; + + printf("%d running tests for find_region()\n", my_rank); + if ((ret = test_find_region())) + return ret; + + printf("%d running tests for get_regions()\n", my_rank); + if ((ret = test_get_regions(my_rank))) + return ret; + + printf("%d running create_mpi_datatypes tests\n", my_rank); + if ((ret = test_create_mpi_datatypes())) + return ret; + + printf("%d running define_iodesc_datatypes tests\n", my_rank); + if ((ret = test_define_iodesc_datatypes())) + return ret; + + printf("%d running compare_offsets tests\n", my_rank); + if ((ret = test_compare_offsets())) + return ret; + + printf("%d running compute_counts tests for box rearranger\n", my_rank); + if ((ret = test_compute_counts(test_comm, my_rank))) + return ret; + + printf("%d running tests for box_rearrange_create\n", my_rank); + if ((ret = test_box_rearrange_create(test_comm, my_rank))) + return ret; + + printf("%d running more tests for box_rearrange_create\n", my_rank); + if ((ret = test_box_rearrange_create_2(test_comm, my_rank))) + return ret; + + printf("%d running tests for default_subset_partition\n", my_rank); + if ((ret = test_default_subset_partition(test_comm, my_rank))) + return ret; + + printf("%d running tests for rearrange_comp2io\n", my_rank); + if ((ret = test_rearrange_comp2io(test_comm, my_rank))) + return ret; + + printf("%d running tests for rearrange_io2comp\n", my_rank); + if ((ret = test_rearrange_io2comp(test_comm, my_rank))) + return ret; + + return 0; +} + +/* Test scalar vars. */ +int test_scalar(int numio, int iosysid, MPI_Comm test_comm, int my_rank, + int num_flavors, int *flavor) +{ + + int var_type[NUM_NETCDF4_TYPES - 1] = {PIO_BYTE, PIO_CHAR, PIO_SHORT, PIO_INT, PIO_FLOAT, + PIO_DOUBLE, PIO_UBYTE, PIO_USHORT, PIO_UINT, PIO_INT64, + PIO_UINT64}; + char char_data = 2; + signed char byte_data = -42; + short short_data = -300; + int int_data = -10000; + float float_data = -42.42; + double double_data = -420000000000.5; + unsigned char ubyte_data = 43; + unsigned short ushort_data = 666; + unsigned int uint_data = 666666; + long long int64_data = -99999999999; + unsigned long long uint64_data = 99999999999; + char char_data_in; + signed char byte_data_in; + short short_data_in; + unsigned char ubyte_data_in; + int int_data_in; + float float_data_in; + double double_data_in; + unsigned short ushort_data_in; + unsigned int uint_data_in; + long long int64_data_in; + unsigned long long uint64_data_in; + + int ret; + + /* Run tests with all available iotypes. */ + for (int fmt = 0; fmt < num_flavors; fmt++) + { + /* For netcdf-4, there are extra types. */ + int num_types = (flavor[fmt] == PIO_IOTYPE_NETCDF4C || flavor[fmt] == PIO_IOTYPE_NETCDF4P) ? + NUM_NETCDF4_TYPES - 1 : NUM_CLASSIC_TYPES; + + /* For each available type, create a file with a scalar var of + * that type. */ + for (int t = 0; t < num_types; t++) + { + int ncid; + int varid; + char filename[PIO_MAX_NAME + 1]; + + printf("test with t = %d\n", t); + + /* These iotypes only handle netCDF classic types. */ + if (t >= NUM_CLASSIC_TYPES && + (flavor[fmt] == PIO_IOTYPE_PNETCDF || flavor[fmt] == PIO_IOTYPE_NETCDF)) + continue; + + /* Create filename. */ + sprintf(filename, "%s_scalar_numio_%d_iotype_%d_var_type_%d.nc", TEST_NAME, + numio, flavor[fmt], var_type[t]); + + /* Create the file. */ + if ((ret = PIOc_createfile(iosysid, &ncid, &flavor[fmt], filename, NC_CLOBBER))) + return ret; + + /* Define scalar var. */ + if ((ret = PIOc_def_var(ncid, VAR_NAME, var_type[t], 0, NULL, &varid))) + return ret; + + /* End define mode. */ + if ((ret = PIOc_enddef(ncid))) + return ret; + + /* Write a value. */ + switch (var_type[t]) + { + case PIO_BYTE: + if ((ret = PIOc_put_var_schar(ncid, varid, &byte_data))) + return ret; + break; + case PIO_CHAR: + if ((ret = PIOc_put_var_text(ncid, varid, &char_data))) + return ret; + break; + case PIO_SHORT: + if ((ret = PIOc_put_var_short(ncid, varid, &short_data))) + return ret; + break; + case PIO_INT: + if ((ret = PIOc_put_var_int(ncid, varid, &int_data))) + return ret; + break; + case PIO_FLOAT: + if ((ret = PIOc_put_var_float(ncid, varid, &float_data))) + return ret; + break; + case PIO_DOUBLE: + if ((ret = PIOc_put_var_double(ncid, varid, &double_data))) + return ret; + break; +#ifdef _NETCDF4 + case PIO_UBYTE: + if ((ret = PIOc_put_var_uchar(ncid, varid, &ubyte_data))) + return ret; + break; + case PIO_USHORT: + if ((ret = PIOc_put_var_ushort(ncid, varid, &ushort_data))) + return ret; + break; + case PIO_UINT: + if ((ret = PIOc_put_var_uint(ncid, varid, &uint_data))) + return ret; + break; + case PIO_INT64: + if ((ret = PIOc_put_var_longlong(ncid, varid, &int64_data))) + return ret; + break; + case PIO_UINT64: + if ((ret = PIOc_put_var_ulonglong(ncid, varid, &uint64_data))) + return ret; + break; +#endif /* _NETCDF4 */ + default: + return ERR_WRONG; + } + + /* Close the file. */ + if ((ret = PIOc_closefile(ncid))) + return ret; + + /* Reopen the file. */ + if ((ret = PIOc_openfile(iosysid, &ncid, &flavor[fmt], filename, NC_NOWRITE))) + return ret; + + /* Read the value. Is the value correct? */ + switch (var_type[t]) + { + case PIO_BYTE: + if ((ret = PIOc_get_var_schar(ncid, varid, &byte_data_in))) + return ret; + if (byte_data_in != byte_data) + return ERR_WRONG; + break; + case PIO_CHAR: + if ((ret = PIOc_get_var_text(ncid, varid, &char_data_in))) + return ret; + if (char_data_in != char_data) + return ERR_WRONG; + break; + case PIO_SHORT: + if ((ret = PIOc_get_var_short(ncid, varid, &short_data_in))) + return ret; + if (short_data_in != short_data) + return ERR_WRONG; + break; + case PIO_INT: + if ((ret = PIOc_get_var_int(ncid, varid, &int_data_in))) + return ret; + if (int_data_in != int_data) + return ERR_WRONG; + break; + case PIO_FLOAT: + if ((ret = PIOc_get_var_float(ncid, varid, &float_data_in))) + return ret; + if (float_data_in != float_data) + return ERR_WRONG; + break; + case PIO_DOUBLE: + if ((ret = PIOc_get_var_double(ncid, varid, &double_data_in))) + return ret; + if (double_data_in != double_data) + return ERR_WRONG; + break; +#ifdef _NETCDF4 + case PIO_UBYTE: + if ((ret = PIOc_get_var_uchar(ncid, varid, &ubyte_data_in))) + return ret; + if (ubyte_data_in != ubyte_data) + return ERR_WRONG; + break; + case PIO_USHORT: + if ((ret = PIOc_get_var_ushort(ncid, varid, &ushort_data_in))) + return ret; + if (ushort_data_in != ushort_data) + return ERR_WRONG; + break; + case PIO_UINT: + if ((ret = PIOc_get_var_uint(ncid, varid, &uint_data_in))) + return ret; + if (uint_data_in != uint_data) + return ERR_WRONG; + break; + case PIO_INT64: + if ((ret = PIOc_get_var_longlong(ncid, varid, &int64_data_in))) + return ret; + if (int64_data_in != int64_data) + return ERR_WRONG; + break; + case PIO_UINT64: + if ((ret = PIOc_get_var_ulonglong(ncid, varid, &uint64_data_in))) + return ret; + if (uint64_data_in != uint64_data) + return ERR_WRONG; + break; +#endif /* _NETCDF4 */ + default: + return ERR_WRONG; + } + + /* Close the file. */ + if ((ret = PIOc_closefile(ncid))) + return ret; + + } /* next iotype */ + } /* next type */ + + return 0; +} + +/* These tests are run with different rearrangers and numbers of IO + * tasks. */ +int run_iosys_tests(int numio, int iosysid, int my_rank, MPI_Comm test_comm, + int num_flavors, int *flavor) +{ + int ret; + + printf("%d running rearranger opts tests 1\n", my_rank); + if ((ret = test_rearranger_opts1(iosysid))) + return ret; + + printf("%d running test for init_decomp\n", my_rank); + if ((ret = test_init_decomp(iosysid, test_comm, my_rank))) + return ret; + + printf("%d running test for init_decomp\n", my_rank); + if ((ret = test_scalar(numio, iosysid, test_comm, my_rank, num_flavors, flavor))) + return ret; + + return 0; +} + +/* Run Tests for pio_spmd.c functions. */ +int main(int argc, char **argv) +{ + int my_rank; /* Zero-based rank of processor. */ + int ntasks; /* Number of processors involved in current execution. */ + int num_flavors; /* Number of PIO netCDF flavors in this build. */ + int flavor[NUM_FLAVORS]; /* iotypes for the supported netCDF IO flavors. */ + MPI_Comm test_comm; /* A communicator for this test. */ + int ret; /* Return code. */ + + /* Initialize test. */ + if ((ret = pio_test_init2(argc, argv, &my_rank, &ntasks, MIN_NTASKS, + TARGET_NTASKS, 3, &test_comm))) + ERR(ERR_INIT); + if ((ret = PIOc_set_iosystem_error_handling(PIO_DEFAULT, PIO_RETURN_ERROR, NULL))) + return ret; + + /* Figure out iotypes. */ + if ((ret = get_iotypes(&num_flavors, flavor))) + ERR(ret); + printf("Runnings tests for %d flavors\n", num_flavors); + + /* Test code runs on TARGET_NTASKS tasks. The left over tasks do + * nothing. */ + if (my_rank < TARGET_NTASKS) + { + /* Run the tests that don't need an iosysid. */ + if ((ret = run_no_iosys_tests(my_rank, test_comm))) + return ret; + + /* Test code with both rearrangers. */ + for (int r = 0; r < NUM_REARRANGERS; r++) + { + /* Test code with 1, 2, 3, and 4 io tasks. */ + for (int numio = 1; numio <= TARGET_NTASKS; numio++) + { + int iosysid; + int rearranger = r ? PIO_REARR_SUBSET : PIO_REARR_BOX; + + if ((ret = PIOc_Init_Intracomm(test_comm, numio, 1, 0, rearranger, + &iosysid))) + return ret; + + /* Run the tests that need an iosysid. */ + if ((ret = run_iosys_tests(numio, iosysid, my_rank, test_comm, + num_flavors, flavor))) + return ret; + + /* Finalize PIO system. */ + if ((ret = PIOc_finalize(iosysid))) + return ret; + } /* next numio */ + } /* next rearranger */ + } /* endif my_rank < TARGET_NTASKS */ + + /* Finalize the MPI library. */ + printf("%d %s Finalizing...\n", my_rank, TEST_NAME); + if ((ret = pio_test_finalize(&test_comm))) + return ret; + + printf("%d %s SUCCESS!!\n", my_rank, TEST_NAME); + + return 0; +} diff --git a/src/externals/pio2/tests/cunit/test_shared.c b/src/externals/pio2/tests/cunit/test_shared.c index 3d6eb9cd1ff..0926e0be9fd 100644 --- a/src/externals/pio2/tests/cunit/test_shared.c +++ b/src/externals/pio2/tests/cunit/test_shared.c @@ -20,22 +20,21 @@ int test_async2(int my_rank, int num_flavors, int *flavor, MPI_Comm test_comm, int component_count, int num_io_procs, int target_ntasks, char *test_name) { int iosysid[component_count]; /* The ID for the parallel I/O system. */ - int num_procs[component_count + 1]; /* Num procs in each component. */ + int num_procs[component_count]; /* Num procs in each component. */ MPI_Comm io_comm; /* Will get a duplicate of IO communicator. */ MPI_Comm comp_comm[component_count]; /* Will get duplicates of computation communicators. */ int mpierr; /* Return code from MPI functions. */ int ret; /* Return code. */ - num_procs[0] = 1; - num_procs[1] = target_ntasks - 1; + num_procs[0] = target_ntasks - 1; /* Is the current process a computation task? */ int comp_task = my_rank < num_io_procs ? 0 : 1; printf("%d comp_task = %d\n", my_rank, comp_task); /* Initialize the IO system. */ - if ((ret = PIOc_Init_Async(test_comm, num_io_procs, NULL, component_count, - num_procs, NULL, &io_comm, comp_comm, iosysid))) + if ((ret = PIOc_init_async(test_comm, num_io_procs, NULL, component_count, + num_procs, NULL, &io_comm, comp_comm, PIO_REARR_BOX, iosysid))) ERR(ERR_INIT); for (int c = 0; c < component_count; c++) printf("%d iosysid[%d] = %d\n", my_rank, c, iosysid[c]); @@ -103,17 +102,17 @@ int test_no_async2(int my_rank, int num_flavors, int *flavor, MPI_Comm test_comm ioproc_start, PIO_REARR_SUBSET, &iosysid))) return ret; - /* Describe the decomposition. This is a 1-based array, so add 1! */ + /* Describe the decomposition. This is a 0-based array, so don't add 1! */ elements_per_pe = x_dim_len * y_dim_len / target_ntasks; if (!(compdof = malloc(elements_per_pe * sizeof(PIO_Offset)))) return PIO_ENOMEM; for (int i = 0; i < elements_per_pe; i++) - compdof[i] = my_rank * elements_per_pe + i + 1; + compdof[i] = my_rank * elements_per_pe + i; /* Create the PIO decomposition for this test. */ printf("%d Creating decomposition...\n", my_rank); - if ((ret = PIOc_InitDecomp(iosysid, PIO_FLOAT, 2, slice_dimlen, (PIO_Offset)elements_per_pe, - compdof, &ioid, NULL, NULL, NULL))) + if ((ret = PIOc_init_decomp(iosysid, PIO_FLOAT, 2, slice_dimlen, (PIO_Offset)elements_per_pe, + compdof, &ioid, 0, NULL, NULL))) return ret; free(compdof); diff --git a/src/externals/pio2/tests/cunit/test_spmd.c b/src/externals/pio2/tests/cunit/test_spmd.c index b4947137004..2c51ce2ccab 100644 --- a/src/externals/pio2/tests/cunit/test_spmd.c +++ b/src/externals/pio2/tests/cunit/test_spmd.c @@ -1,5 +1,5 @@ /* - * This program tests some internal functions in the library. + * This program tests some internal functions in the PIO library. * * Jim Edwards * Ed Hartnett, 11/23/16 @@ -20,8 +20,6 @@ /* Number of test cases in inner loop of test. */ #define NUM_TEST_CASES 5 -#define TEST_MAX_GATHER_BLOCK_SIZE 32 - /* Test MPI_Alltoallw by having processor i send different amounts of * data to each processor. The first test sends i items to processor * i from all processors. */ @@ -85,8 +83,9 @@ int run_spmd_tests(MPI_Comm test_comm) for (int itest = 0; itest < NUM_TEST_CASES; itest++) { - bool hs = false; - bool isend = false; + rearr_comm_fc_opt_t fc; + fc.hs = false; + fc.isend = false; /* Wait for all tasks. */ MPI_Barrier(test_comm); @@ -99,28 +98,28 @@ int run_spmd_tests(MPI_Comm test_comm) /* Set the parameters different for each test case. */ if (itest == 1) { - hs = true; - isend = true; + fc.hs = true; + fc.isend = true; } else if (itest == 2) { - hs = false; - isend = true; + fc.hs = false; + fc.isend = true; } else if (itest == 3) { - hs = false; - isend = false; + fc.hs = false; + fc.isend = false; } else if (itest == 4) { - hs = true; - isend = false; + fc.hs = true; + fc.isend = false; } /* Run the swapm function. */ if ((ret = pio_swapm(sbuf, sendcounts, sdispls, sendtypes, rbuf, recvcounts, - rdispls, recvtypes, test_comm, hs, isend, msg_cnt))) + rdispls, recvtypes, test_comm, &fc))) return ret; /* Print results. */ @@ -189,59 +188,19 @@ int run_sc_tests(MPI_Comm test_comm) if (gcd_array(SC_ARRAY_LEN, array4) != 1) return ERR_WRONG; - return 0; -} - -/* This test code was recovered from main() in pioc_sc.c. */ -int test_CalcStartandCount() -{ - int ndims = 2; - int gdims[2] = {31, 777602}; - int num_io_procs = 24; - bool converged = false; - PIO_Offset start[ndims], kount[ndims]; - int iorank, numaiotasks = 0; - long int tpsize = 0; - long int psize; - long int pgdims = 1; - int scnt; - - for (int i = 0; i < ndims; i++) - pgdims *= gdims[i]; - - while (!converged) - { - for (iorank = 0; iorank < num_io_procs; iorank++) - { - numaiotasks = CalcStartandCount(PIO_DOUBLE, ndims, gdims, num_io_procs, iorank, - start, kount); - if (iorank < numaiotasks) - printf("iorank %d start %lld %lld count %lld %lld\n", iorank, start[0], - start[1], kount[0], kount[1]); - - if (numaiotasks < 0) - return numaiotasks; - - psize = 1; - scnt = 0; - for (int i = 0; i < ndims; i++) - { - psize *= kount[i]; - scnt += kount[i]; - } - tpsize += psize; - } - - if (tpsize == pgdims) - converged = true; - else - { - printf("Failed to converge %ld %ld %d\n", tpsize, pgdims, num_io_procs); - tpsize = 0; - num_io_procs--; - } - } - + /* Test compute_one_dim. */ + PIO_Offset start, count; + compute_one_dim(4, 4, my_rank, &start, &count); + if (start != my_rank || count != 1) + return ERR_WRONG; + compute_one_dim(400, 4, my_rank, &start, &count); + if (start != my_rank * 100 || count != 100) + return ERR_WRONG; + /* Left over data will go to task 3. */ + compute_one_dim(5, 4, my_rank, &start, &count); + if (start != my_rank || count != (my_rank == 3 ? 2 : 1)) + return ERR_WRONG; + printf("my_rank = %d start = %lld count = %lld\n", my_rank, start, count); return 0; } @@ -264,94 +223,6 @@ int test_lists() return 0; } -/* Test some of the rearranger utility functions. */ -int test_rearranger_opts1() -{ - rearr_comm_fc_opt_t *ro1; - rearr_comm_fc_opt_t *ro2; - rearr_comm_fc_opt_t *ro3; - - if (!(ro1 = calloc(1, sizeof(rearr_comm_fc_opt_t)))) - return ERR_AWFUL; - if (!(ro2 = calloc(1, sizeof(rearr_comm_fc_opt_t)))) - return ERR_AWFUL; - if (!(ro3 = calloc(1, sizeof(rearr_comm_fc_opt_t)))) - return ERR_AWFUL; - - /* This should not work. */ - if (PIOc_set_rearr_opts(42, 1, 1, 0, 0, 0, 0, 0, 0) != PIO_EBADID) - return ERR_WRONG; - - /* ro1 and ro2 are the same. */ - if (!cmp_rearr_comm_fc_opts(ro1, ro2)) - return ERR_WRONG; - - /* Make ro3 different. */ - ro3->enable_hs = 1; - if (cmp_rearr_comm_fc_opts(ro1, ro3)) - return ERR_WRONG; - ro3->enable_hs = 0; - ro3->enable_isend = 1; - if (cmp_rearr_comm_fc_opts(ro1, ro3)) - return ERR_WRONG; - ro3->enable_isend = 0; - ro3->max_pend_req = 1; - if (cmp_rearr_comm_fc_opts(ro1, ro3)) - return ERR_WRONG; - - /* Free resourses. */ - free(ro1); - free(ro2); - free(ro3); - - return 0; -} - -/* Test some of the rearranger utility functions. */ -int test_rearranger_opts2() -{ - iosystem_desc_t my_ios; - iosystem_desc_t *ios = &my_ios; - - /* I'm not sure what the point of this function is... */ - check_and_reset_rearr_opts(ios); - - return 0; -} - -/* Test the compare_offsets() function. */ -int test_compare_offsets() -{ - mapsort m1, m2, m3; - - m1.rfrom = 0; - m1.soffset = 0; - m1.iomap = 0; - m2.rfrom = 0; - m2.soffset = 0; - m2.iomap = 0; - m3.rfrom = 0; - m3.soffset = 0; - m3.iomap = 1; - - /* Return 0 if either or both parameters are null. */ - if (compare_offsets(NULL, &m2)) - return ERR_WRONG; - if (compare_offsets(&m1, NULL)) - return ERR_WRONG; - if (compare_offsets(NULL, NULL)) - return ERR_WRONG; - - /* m1 and m2 are the same. */ - if (compare_offsets(&m1, &m2)) - return ERR_WRONG; - - /* m1 and m3 are different. */ - if (compare_offsets(&m1, &m3) != -1) - return ERR_WRONG; - return 0; -} - /* Test the ceil2() and pair() functions. */ int test_ceil2_pair() { @@ -382,72 +253,85 @@ int test_ceil2_pair() int test_find_mpi_type() { MPI_Datatype mpi_type; + int type_size; int ret; /* This should not work. */ - if (find_mpi_type(PIO_BYTE + 42, &mpi_type) != PIO_EBADTYPE) + if (find_mpi_type(PIO_BYTE + 42, &mpi_type, &type_size) != PIO_EBADTYPE) return ERR_WRONG; /* Try every atomic type. */ - if ((ret = find_mpi_type(PIO_BYTE, &mpi_type))) + if ((ret = find_mpi_type(PIO_BYTE, &mpi_type, &type_size))) return ret; - if (mpi_type != MPI_BYTE) + if (mpi_type != MPI_BYTE || type_size != 1) return ERR_WRONG; - if ((ret = find_mpi_type(PIO_CHAR, &mpi_type))) + if ((ret = find_mpi_type(PIO_CHAR, &mpi_type, &type_size))) return ret; - if (mpi_type != MPI_CHAR) + if (mpi_type != MPI_CHAR || type_size != 1) return ERR_WRONG; - if ((ret = find_mpi_type(PIO_SHORT, &mpi_type))) + if ((ret = find_mpi_type(PIO_SHORT, &mpi_type, &type_size))) return ret; - if (mpi_type != MPI_SHORT) + if (mpi_type != MPI_SHORT || type_size != 2) return ERR_WRONG; - if ((ret = find_mpi_type(PIO_INT, &mpi_type))) + if ((ret = find_mpi_type(PIO_INT, &mpi_type, &type_size))) return ret; - if (mpi_type != MPI_INT) + if (mpi_type != MPI_INT || type_size != 4) return ERR_WRONG; - if ((ret = find_mpi_type(PIO_FLOAT, &mpi_type))) + if ((ret = find_mpi_type(PIO_FLOAT, &mpi_type, &type_size))) return ret; - if (mpi_type != MPI_FLOAT) + if (mpi_type != MPI_FLOAT || type_size != 4) return ERR_WRONG; - if ((ret = find_mpi_type(PIO_DOUBLE, &mpi_type))) + if ((ret = find_mpi_type(PIO_DOUBLE, &mpi_type, &type_size))) return ret; - if (mpi_type != MPI_DOUBLE) + if (mpi_type != MPI_DOUBLE || type_size != 8) return ERR_WRONG; + /* These should also work. */ + if ((ret = find_mpi_type(PIO_INT, &mpi_type, NULL))) + return ret; + if (mpi_type != MPI_INT) + return ERR_WRONG; + if ((ret = find_mpi_type(PIO_INT, NULL, &type_size))) + return ret; + if (type_size != 4) + return ERR_WRONG; + if ((ret = find_mpi_type(PIO_INT, NULL, NULL))) + return ret; + #ifdef _NETCDF4 - if ((ret = find_mpi_type(PIO_UBYTE, &mpi_type))) + if ((ret = find_mpi_type(PIO_UBYTE, &mpi_type, &type_size))) return ret; - if (mpi_type != MPI_UNSIGNED_CHAR) + if (mpi_type != MPI_UNSIGNED_CHAR || type_size != 1) return ERR_WRONG; - if ((ret = find_mpi_type(PIO_USHORT, &mpi_type))) + if ((ret = find_mpi_type(PIO_USHORT, &mpi_type, &type_size))) return ret; - if (mpi_type != MPI_UNSIGNED_SHORT) + if (mpi_type != MPI_UNSIGNED_SHORT || type_size != 2) return ERR_WRONG; - if ((ret = find_mpi_type(PIO_UINT, &mpi_type))) + if ((ret = find_mpi_type(PIO_UINT, &mpi_type, &type_size))) return ret; - if (mpi_type != MPI_UNSIGNED) + if (mpi_type != MPI_UNSIGNED || type_size != 4) return ERR_WRONG; - if ((ret = find_mpi_type(PIO_INT64, &mpi_type))) + if ((ret = find_mpi_type(PIO_INT64, &mpi_type, &type_size))) return ret; - if (mpi_type != MPI_LONG_LONG) + if (mpi_type != MPI_LONG_LONG || type_size != 8) return ERR_WRONG; - if ((ret = find_mpi_type(PIO_UINT64, &mpi_type))) + if ((ret = find_mpi_type(PIO_UINT64, &mpi_type, &type_size))) return ret; - if (mpi_type != MPI_UNSIGNED_LONG_LONG) + if (mpi_type != MPI_UNSIGNED_LONG_LONG || type_size != 8) return ERR_WRONG; - if ((ret = find_mpi_type(PIO_STRING, &mpi_type))) + if ((ret = find_mpi_type(PIO_STRING, &mpi_type, &type_size))) return ret; - if (mpi_type != MPI_CHAR) + if (mpi_type != MPI_CHAR || type_size != 1) return ERR_WRONG; #endif /* _NETCDF4 */ @@ -461,6 +345,117 @@ int test_misc() /* This should not work. */ if (flush_buffer(TEST_VAL_42, &wmb, 0) != PIO_EBADID) return ERR_WRONG; + + return 0; +} + +/* This test code was recovered from main() in pioc_sc.c. */ +int test_CalcStartandCount() +{ + int ndims = 2; + int gdims[2] = {31, 777602}; + int num_io_procs = 24; + bool converged = false; + PIO_Offset start[ndims], kount[ndims]; + int iorank, numaiotasks = 0; + long int tpsize = 0; + long int psize; + long int pgdims = 1; + int scnt; + int ret; + + for (int i = 0; i < ndims; i++) + pgdims *= gdims[i]; + + while (!converged) + { + for (iorank = 0; iorank < num_io_procs; iorank++) + { + if ((ret = CalcStartandCount(PIO_DOUBLE, ndims, gdims, num_io_procs, iorank, + start, kount, &numaiotasks))) + return ret; + if (iorank < numaiotasks) + printf("iorank %d start %lld %lld count %lld %lld\n", iorank, start[0], + start[1], kount[0], kount[1]); + + if (numaiotasks < 0) + return numaiotasks; + + psize = 1; + scnt = 0; + for (int i = 0; i < ndims; i++) + { + psize *= kount[i]; + scnt += kount[i]; + } + tpsize += psize; + } + + if (tpsize == pgdims) + converged = true; + else + { + printf("Failed to converge %ld %ld %d\n", tpsize, pgdims, num_io_procs); + tpsize = 0; + num_io_procs--; + } + } + + return 0; +} + +/* Test the GDCblocksize() function. */ +int run_GDCblocksize_tests(MPI_Comm test_comm) +{ + { + int arrlen = 1; + PIO_Offset arr_in[1] = {0}; + PIO_Offset blocksize; + + blocksize = GCDblocksize(arrlen, arr_in); + if (blocksize != 1) + return ERR_WRONG; + } + + { + int arrlen = 4; + PIO_Offset arr_in[4] = {0, 1, 2, 3}; + PIO_Offset blocksize; + + blocksize = GCDblocksize(arrlen, arr_in); + if (blocksize != 4) + return ERR_WRONG; + } + + { + int arrlen = 4; + PIO_Offset arr_in[4] = {0, 2, 3, 4}; + PIO_Offset blocksize; + + blocksize = GCDblocksize(arrlen, arr_in); + if (blocksize != 1) + return ERR_WRONG; + } + + { + int arrlen = 4; + PIO_Offset arr_in[4] = {0, 1, 3, 4}; + PIO_Offset blocksize; + + blocksize = GCDblocksize(arrlen, arr_in); + if (blocksize != 1) + return ERR_WRONG; + } + + { + int arrlen = 4; + PIO_Offset arr_in[4] = {0, 1, 2, 4}; + PIO_Offset blocksize; + + blocksize = GCDblocksize(arrlen, arr_in); + if (blocksize != 1) + return ERR_WRONG; + } return 0; } @@ -482,10 +477,20 @@ int main(int argc, char **argv) * nothing. */ if (my_rank < TARGET_NTASKS) { + /* I don't need this iosystem, but it's the only way to get + * the logs to write. */ + int iosysid; + if ((ret = PIOc_Init_Intracomm(test_comm, TARGET_NTASKS, 1, 0, PIO_REARR_BOX, &iosysid))) + return ret; + printf("%d running tests for functions in pioc_sc.c\n", my_rank); if ((ret = run_sc_tests(test_comm))) return ret; + printf("%d running tests for GCDblocksize()\n", my_rank); + if ((ret = run_GDCblocksize_tests(test_comm))) + return ret; + printf("%d running spmd test code\n", my_rank); if ((ret = run_spmd_tests(test_comm))) return ret; @@ -498,18 +503,6 @@ int main(int argc, char **argv) if ((ret = test_lists())) return ret; - printf("%d running rearranger opts tests 1\n", my_rank); - if ((ret = test_rearranger_opts1())) - return ret; - - printf("%d running rearranger opts tests 2\n", my_rank); - if ((ret = test_rearranger_opts2())) - return ret; - - printf("%d running compare_offsets tests\n", my_rank); - if ((ret = test_compare_offsets())) - return ret; - printf("%d running ceil2/pair tests\n", my_rank); if ((ret = test_ceil2_pair())) return ret; @@ -522,6 +515,10 @@ int main(int argc, char **argv) if ((ret = test_misc())) return ret; + /* Finalize PIO system. */ + if ((ret = PIOc_finalize(iosysid))) + return ret; + } /* endif my_rank < TARGET_NTASKS */ /* Finalize the MPI library. */ diff --git a/src/externals/pio2/tests/general/ncdf_get_put.F90.in b/src/externals/pio2/tests/general/ncdf_get_put.F90.in index 75ccc75f3e9..ea8e5933b71 100644 --- a/src/externals/pio2/tests/general/ncdf_get_put.F90.in +++ b/src/externals/pio2/tests/general/ncdf_get_put.F90.in @@ -118,6 +118,67 @@ PIO_TF_AUTO_TEST_SUB_BEGIN test_put_get_1datt PIO_TF_AUTO_TEST_SUB_END test_put_get_1datt +PIO_TF_TEMPLATE +PIO_TF_AUTO_TEST_SUB_BEGIN test_put_get_0dvar + Implicit none + type(file_desc_t) :: pio_file + character(len=PIO_TF_MAX_STR_LEN) :: filename + type(var_desc_t) :: pio_var, pio_cvar + PIO_TF_FC_DATA_TYPE, dimension(1) :: pval, gval + CHARACTER(len=1) :: pcval, gcval + integer, dimension(:), allocatable :: iotypes + character(len=PIO_TF_MAX_STR_LEN), dimension(:), allocatable :: iotype_descs + integer :: num_iotypes + integer :: i, ret + + pval = pio_tf_world_sz_ + pcval = "D" + num_iotypes = 0 + call PIO_TF_Get_nc_iotypes(iotypes, iotype_descs, num_iotypes) + filename = "test_pio_ncdf_get_put.testfile" + do i=1,num_iotypes + PIO_TF_LOG(0,*) "Testing type :", iotype_descs(i) + ret = PIO_createfile(pio_tf_iosystem_, pio_file, iotypes(i), filename, PIO_CLOBBER) + PIO_TF_CHECK_ERR(ret, "Failed to open:" // trim(filename)) + + ! Since file is just created no need to enter redef + ret = PIO_def_var(pio_file, 'dummy_scalar_var_put_val', PIO_TF_DATA_TYPE, pio_var) + PIO_TF_CHECK_ERR(ret, "Failed to define scalar var:" // trim(filename)) + + ret = PIO_def_var(pio_file, 'dummy_scalar_var_put_cval', PIO_char, pio_cvar) + PIO_TF_CHECK_ERR(ret, "Failed to define scalar char var:" // trim(filename)) + + ret = PIO_enddef(pio_file) + PIO_TF_CHECK_ERR(ret, "Failed to enddef:" // trim(filename)) + + ret = PIO_put_var(pio_file, pio_var, pval); + PIO_TF_CHECK_ERR(ret, "Failed to put scalar var:" // trim(filename)) + + ret = PIO_put_var(pio_file, pio_cvar, pcval); + PIO_TF_CHECK_ERR(ret, "Failed to put scalar char var:" // trim(filename)) + + call PIO_syncfile(pio_file) + + ret = PIO_get_var(pio_file, pio_var, gval); + PIO_TF_CHECK_ERR(ret, "Failed to get scalar var:" // trim(filename)) + + PIO_TF_CHECK_VAL((gval, pval), "Got wrong value") + + ret = PIO_get_var(pio_file, pio_cvar, gcval); + PIO_TF_CHECK_ERR(ret, "Failed to get scalar char var:" // trim(filename)) + + PIO_TF_CHECK_VAL((gcval, pcval), "Got wrong value") + + call PIO_closefile(pio_file) + call PIO_deletefile(pio_tf_iosystem_, filename); + end do + if(allocated(iotypes)) then + deallocate(iotypes) + deallocate(iotype_descs) + end if + +PIO_TF_AUTO_TEST_SUB_END test_put_get_0dvar + PIO_TF_TEMPLATE PIO_TF_AUTO_TEST_SUB_BEGIN test_put_get_1dvar Implicit none @@ -184,6 +245,71 @@ PIO_TF_AUTO_TEST_SUB_BEGIN test_put_get_1dvar PIO_TF_AUTO_TEST_SUB_END test_put_get_1dvar +! Write out a 1d var slice from a 2d var +PIO_TF_TEMPLATE +PIO_TF_AUTO_TEST_SUB_BEGIN test_put_get_1dvar_slice + Implicit none + type(file_desc_t) :: pio_file + character(len=PIO_TF_MAX_STR_LEN) :: filename + type(var_desc_t) :: pio_var, pio_cvar + integer :: pio_dim + integer, parameter :: MAX_ROW_DIM_LEN = 100 + PIO_TF_FC_DATA_TYPE, dimension(MAX_ROW_DIM_LEN) :: gval, exp_val + integer, parameter :: MAX_COL_DIM_LEN = 4 + ! Only COL_WRITE_DIM of MAX_COL_DIM_LEN columns in pval is written out + integer, parameter :: COL_WRITE_DIM = 2 + PIO_TF_FC_DATA_TYPE, dimension(MAX_ROW_DIM_LEN, MAX_COL_DIM_LEN) :: pval + integer, dimension(:) :: start(4), count(4) + integer, dimension(:), allocatable :: iotypes + character(len=PIO_TF_MAX_STR_LEN), dimension(:), allocatable :: iotype_descs + integer :: num_iotypes + integer :: i, ret + + pval = -1 + pval(:,COL_WRITE_DIM) = pio_tf_world_sz_ + exp_val = pio_tf_world_sz_ + start = 0 + count = 0 + start(1) = 1 + count(1) = MAX_ROW_DIM_LEN + num_iotypes = 0 + call PIO_TF_Get_nc_iotypes(iotypes, iotype_descs, num_iotypes) + filename = "test_pio_ncdf_get_put_slice.testfile" + do i=1,num_iotypes + PIO_TF_LOG(0,*) "Testing type :", iotype_descs(i) + ret = PIO_createfile(pio_tf_iosystem_, pio_file, iotypes(i), filename, PIO_CLOBBER) + PIO_TF_CHECK_ERR(ret, "Failed to open:" // trim(filename)) + + ! Since file is just created no need to enter redef + ret = PIO_def_dim(pio_file, 'dummy_dim_put_val', MAX_ROW_DIM_LEN, pio_dim) + PIO_TF_CHECK_ERR(ret, "Failed to define dim:" // trim(filename)) + + ret = PIO_def_var(pio_file, 'dummy_var_put_val', PIO_TF_DATA_TYPE, (/pio_dim/), pio_var) + PIO_TF_CHECK_ERR(ret, "Failed to define var:" // trim(filename)) + + ret = PIO_enddef(pio_file) + PIO_TF_CHECK_ERR(ret, "Failed to enddef:" // trim(filename)) + + ret = PIO_put_var(pio_file, pio_var, start, count, pval(:,COL_WRITE_DIM)); + PIO_TF_CHECK_ERR(ret, "Failed to put var:" // trim(filename)) + + call PIO_syncfile(pio_file) + + ret = PIO_get_var(pio_file, pio_var, gval); + PIO_TF_CHECK_ERR(ret, "Failed to get var:" // trim(filename)) + + PIO_TF_CHECK_VAL((gval, exp_val), "Got wrong value") + + call PIO_closefile(pio_file) + call PIO_deletefile(pio_tf_iosystem_, filename); + end do + if(allocated(iotypes)) then + deallocate(iotypes) + deallocate(iotype_descs) + end if + +PIO_TF_AUTO_TEST_SUB_END test_put_get_1dvar_slice + PIO_TF_TEMPLATE PIO_TF_AUTO_TEST_SUB_BEGIN test_put_get_1dvar_4parts Implicit none @@ -267,3 +393,177 @@ PIO_TF_AUTO_TEST_SUB_BEGIN test_put_get_1dvar_4parts PIO_TF_AUTO_TEST_SUB_END test_put_get_1dvar_4parts +! Write out 2d/3d/4d vars, one time slice at a time +PIO_TF_TEMPLATE +PIO_TF_AUTO_TEST_SUB_BEGIN test_put_get_md2mdplus1_var + Implicit none + type(file_desc_t) :: pio_file + character(len=PIO_TF_MAX_STR_LEN) :: filename + integer, parameter :: MAX_DIMS = 4 + integer, parameter :: MAX_ROWS = 10 + integer, parameter :: MAX_COLS = 10 + integer, parameter :: MAX_LEVS = 3 + integer, parameter :: MAX_TIMES = 3 + integer, dimension(MAX_DIMS) :: pio_dims + type(var_desc_t) :: pio_2dvar, pio_3dvar, pio_4dvar + PIO_TF_FC_DATA_TYPE, dimension(MAX_ROWS,MAX_TIMES) :: gval_2d, exp_val_2d + PIO_TF_FC_DATA_TYPE, dimension(MAX_ROWS,MAX_COLS,MAX_TIMES) :: gval_3d, exp_val_3d + PIO_TF_FC_DATA_TYPE, dimension(MAX_ROWS,MAX_COLS,MAX_LEVS,MAX_TIMES) ::& + gval_4d, exp_val_4d + ! Only one slice is written out at a time + ! pval_1d is a 1d slice of gval_2d ... + PIO_TF_FC_DATA_TYPE, dimension(MAX_ROWS) :: pval_1d + PIO_TF_FC_DATA_TYPE, dimension(MAX_ROWS, MAX_COLS) :: pval_2d + PIO_TF_FC_DATA_TYPE, dimension(MAX_ROWS, MAX_COLS, MAX_LEVS) :: pval_3d + integer, dimension(:) :: start(MAX_DIMS), count(MAX_DIMS) + integer :: pval_start + integer, dimension(:), allocatable :: iotypes + character(len=PIO_TF_MAX_STR_LEN), dimension(:), allocatable :: iotype_descs + integer :: num_iotypes + integer :: i, k, l, m, n, tstep, ret + + num_iotypes = 0 + call PIO_TF_Get_nc_iotypes(iotypes, iotype_descs, num_iotypes) + filename = "test_pio_ncdf_get_put_md_slice.testfile" + do i=1,num_iotypes + PIO_TF_LOG(0,*) "Testing type :", iotype_descs(i) + ret = PIO_createfile(pio_tf_iosystem_, pio_file, iotypes(i), filename, PIO_CLOBBER) + PIO_TF_CHECK_ERR(ret, "Failed to open:" // trim(filename)) + + ! Since file is just created no need to enter redef + ret = PIO_def_dim(pio_file, 'nrows', MAX_ROWS, pio_dims(1)) + PIO_TF_CHECK_ERR(ret, "Failed to define dim:" // trim(filename)) + + ret = PIO_def_dim(pio_file, 'ncols', MAX_COLS, pio_dims(2)) + PIO_TF_CHECK_ERR(ret, "Failed to define dim:" // trim(filename)) + + ret = PIO_def_dim(pio_file, 'nlevs', MAX_LEVS, pio_dims(3)) + PIO_TF_CHECK_ERR(ret, "Failed to define dim:" // trim(filename)) + + ret = PIO_def_dim(pio_file, 'timesteps', MAX_TIMES, pio_dims(4)) + PIO_TF_CHECK_ERR(ret, "Failed to define dim:" // trim(filename)) + + ret = PIO_def_var(pio_file, '2d_val', PIO_TF_DATA_TYPE,& + (/pio_dims(1),pio_dims(4)/), pio_2dvar) + PIO_TF_CHECK_ERR(ret, "Failed to define var:" // trim(filename)) + + ret = PIO_def_var(pio_file, '3d_val', PIO_TF_DATA_TYPE,& + (/pio_dims(1),pio_dims(2),pio_dims(4)/), pio_3dvar) + PIO_TF_CHECK_ERR(ret, "Failed to define var:" // trim(filename)) + + ret = PIO_def_var(pio_file, '4d_val', PIO_TF_DATA_TYPE,& + pio_dims, pio_4dvar) + PIO_TF_CHECK_ERR(ret, "Failed to define var:" // trim(filename)) + + ret = PIO_enddef(pio_file) + PIO_TF_CHECK_ERR(ret, "Failed to enddef:" // trim(filename)) + + ! Put vals are for each timestep & + ! expected vals are combined for all timesteps + do k=1,MAX_ROWS + pval_1d(k) = k + end do + do tstep=1,MAX_TIMES + pval_start = (tstep - 1) * MAX_ROWS + exp_val_2d(:,tstep) = pval_1d + pval_start + end do + do l=1,MAX_COLS + do k=1,MAX_ROWS + pval_2d(k,l) = (l - 1)*MAX_ROWS + k + end do + end do + do tstep=1,MAX_TIMES + do l=1,MAX_COLS + do k=1,MAX_ROWS + pval_start = (tstep - 1) * (MAX_ROWS * MAX_COLS) + exp_val_3d(:,:,tstep) = pval_2d + pval_start + end do + end do + end do + do m=1,MAX_LEVS + do l=1,MAX_COLS + do k=1,MAX_ROWS + pval_3d(k,l,m) = ((m-1)*(MAX_COLS*MAX_ROWS)+(l - 1)*MAX_ROWS + k) + end do + end do + end do + do tstep=1,MAX_TIMES + do m=1,MAX_LEVS + do l=1,MAX_COLS + do k=1,MAX_ROWS + pval_start = (tstep - 1) * (MAX_ROWS * MAX_COLS * MAX_LEVS) + exp_val_4d(:,:,:,tstep) = pval_3d + pval_start + end do + end do + end do + end do + ! Put 2d/3d/4d vals, one timestep at a time + do tstep=1,MAX_TIMES + start = 0 + count = 0 + + start(1) = 1 + count(1) = MAX_ROWS + start(2) = tstep + count(2) = 1 + pval_start = (tstep - 1) * MAX_ROWS + ret = PIO_put_var(pio_file, pio_2dvar, start, count,& + pval_1d(:)+pval_start) + PIO_TF_CHECK_ERR(ret, "Failed to put 2d var:" // trim(filename)) + + start(1) = 1 + count(1) = MAX_ROWS + start(2) = 1 + count(2) = MAX_COLS + start(3) = tstep + count(3) = 1 + pval_start = (tstep - 1) * (MAX_ROWS * MAX_COLS) + ret = PIO_put_var(pio_file, pio_3dvar, start, count,& + pval_2d(:,:)+pval_start) + PIO_TF_CHECK_ERR(ret, "Failed to put 3d var:" // trim(filename)) + + start(1) = 1 + count(1) = MAX_ROWS + start(2) = 1 + count(2) = MAX_COLS + start(3) = 1 + count(3) = MAX_LEVS + start(4) = tstep + count(4) = 1 + pval_start = (tstep - 1) * (MAX_ROWS * MAX_COLS * MAX_LEVS) + ret = PIO_put_var(pio_file, pio_4dvar, start, count,& + pval_3d(:,:,:)+pval_start) + PIO_TF_CHECK_ERR(ret, "Failed to put 4d var:" // trim(filename)) + end do + + call PIO_syncfile(pio_file) + + ret = PIO_get_var(pio_file, pio_2dvar, gval_2d) + PIO_TF_CHECK_ERR(ret, "Failed to get 2d var:" // trim(filename)) + + PIO_TF_CHECK_VAL((gval_2d, exp_val_2d), "Got wrong value (2d var)") + + ret = PIO_get_var(pio_file, pio_3dvar, gval_3d) + PIO_TF_CHECK_ERR(ret, "Failed to get 3d var:" // trim(filename)) + + PIO_TF_CHECK_VAL((gval_3d, exp_val_3d), "Got wrong value (3d var)") + + ret = PIO_get_var(pio_file, pio_4dvar, gval_4d) + PIO_TF_CHECK_ERR(ret, "Failed to get 4d var:" // trim(filename)) + + ! Special code to handle 4d vals is required since the framework + ! currently does not support comparing 4d arrays + do tstep=1,MAX_TIMES + PIO_TF_CHECK_VAL((gval_4d(:,:,:,tstep), exp_val_4d(:,:,:,tstep)), "Got wrong value (4d var)") + end do + + call PIO_closefile(pio_file) + call PIO_deletefile(pio_tf_iosystem_, filename); + end do + if(allocated(iotypes)) then + deallocate(iotypes) + deallocate(iotype_descs) + end if + +PIO_TF_AUTO_TEST_SUB_END test_put_get_md2mdplus1_var + diff --git a/src/externals/pio2/tests/general/pio_decomp_frame_tests.F90.in b/src/externals/pio2/tests/general/pio_decomp_frame_tests.F90.in index e92b483c3c0..e75fdce61de 100644 --- a/src/externals/pio2/tests/general/pio_decomp_frame_tests.F90.in +++ b/src/externals/pio2/tests/general/pio_decomp_frame_tests.F90.in @@ -225,3 +225,196 @@ PIO_TF_AUTO_TEST_SUB_BEGIN nc_write_read_4d_col_decomp deallocate(rbuf) deallocate(wbuf) PIO_TF_AUTO_TEST_SUB_END nc_write_read_4d_col_decomp + +! Using a 3d decomp for writing out a 3d and a 4d var +! Write with one decomp (to force rearrangement) and read with another (no +! rearrangement) +PIO_TF_TEMPLATE +PIO_TF_AUTO_TEST_SUB_BEGIN nc_reuse_3d_decomp + implicit none + integer, parameter :: NDIMS = 4 + integer, parameter :: NFRAMES = 3 + type(var_desc_t) :: pio_var3d, pio_var4d + type(file_desc_t) :: pio_file + character(len=PIO_TF_MAX_STR_LEN) :: filename + type(io_desc_t) :: wr_iodesc, rd_iodesc + integer, dimension(:), allocatable :: compdof + integer, dimension(NDIMS) :: start, count + PIO_TF_FC_DATA_TYPE, dimension(:,:,:,:), allocatable :: rbuf4d, wbuf4d, exp_val4d + PIO_TF_FC_DATA_TYPE, dimension(:,:,:), allocatable :: rbuf3d, wbuf3d, exp_val3d + integer, dimension(NDIMS-1) :: dims + integer, dimension(NDIMS) :: pio_dims + integer :: i, j, k, tmp_idx, ierr, lsz, nrows, ncols, nhgts + integer(kind=pio_offset_kind) :: f + ! iotypes = valid io types + integer, dimension(:), allocatable :: iotypes + character(len=PIO_TF_MAX_STR_LEN), dimension(:), allocatable :: iotype_descs + integer :: num_iotypes + + ! Set the decomposition for writing data - forcing rearrangement + call get_3d_col_decomp_info(pio_tf_world_rank_, pio_tf_world_sz_, dims, start, count, .true.) + nrows = count(1) + ncols = count(2) + nhgts = count(3) + + ! Initialize the 4d var + allocate(wbuf4d(nrows, ncols, nhgts, NFRAMES)) + do f=1,NFRAMES + do k=1,nhgts + do j=1,ncols + do i=1,nrows + wbuf4d(i,j,k,f) = (start(3) - 1 + k - 1) * (dims(1) * dims(2)) +& + (start(2) - 1 + j - 1) * dims(1) + i + wbuf4d(i,j,k,f) = wbuf4d(i,j,k,f) + (f - 1) * (dims(1) * dims(2) * dims(3)) + end do + end do + end do + end do + allocate(compdof(nrows * ncols * nhgts)) + do k=1,nhgts + do j=1,ncols + do i=1,nrows + tmp_idx = (k - 1) * (ncols * nrows) + (j - 1) * nrows + i + compdof(tmp_idx) = wbuf4d(i,j,k,1) + end do + end do + end do + ! Initialize the 3d var + allocate(wbuf3d(nrows, ncols, nhgts)) + do k=1,nhgts + do j=1,ncols + do i=1,nrows + wbuf3d(i,j,k) = (start(3) - 1 + k - 1) * (dims(1) * dims(2)) +& + (start(2) - 1 + j - 1) * dims(1) + i + end do + end do + end do + + call PIO_initdecomp(pio_tf_iosystem_, PIO_TF_DATA_TYPE, dims, compdof, wr_iodesc) + deallocate(compdof) + + ! Set the decomposition for reading data - different from the write decomp + call get_3d_col_decomp_info(pio_tf_world_rank_, pio_tf_world_sz_, dims, start, count, .false.) + nrows = count(1) + ncols = count(2) + nhgts = count(3) + + allocate(rbuf4d(nrows, ncols, nhgts, NFRAMES)) + rbuf4d = 0 + ! Expected val for 4d var + allocate(exp_val4d(nrows, ncols, nhgts, NFRAMES)) + do f=1,NFRAMES + do k=1,nhgts + do j=1,ncols + do i=1,nrows + exp_val4d(i,j,k,f) = (start(3) - 1 + k - 1) * (dims(1) * dims(2)) +& + (start(2) - 1 + j - 1) * dims(1) + i + exp_val4d(i,j,k,f) = exp_val4d(i,j,k,f)+(f - 1) * (dims(1) * dims(2) * dims(3)) + end do + end do + end do + end do + allocate(compdof(nrows * ncols * nhgts)) + do k=1,nhgts + do j=1,ncols + do i=1,nrows + tmp_idx = (k - 1) * (ncols * nrows) + (j - 1) * nrows + i + compdof(tmp_idx) = exp_val4d(i,j,k,1) + end do + end do + end do + + allocate(rbuf3d(nrows, ncols, nhgts)) + rbuf3d = 0 + ! Expected val for 3d var + allocate(exp_val3d(nrows, ncols, nhgts)) + do k=1,nhgts + do j=1,ncols + do i=1,nrows + exp_val3d(i,j,k) = (start(3) - 1 + k - 1) * (dims(1) * dims(2)) +& + (start(2) - 1 + j - 1) * dims(1) + i + end do + end do + end do + + call PIO_initdecomp(pio_tf_iosystem_, PIO_TF_DATA_TYPE, dims, compdof, rd_iodesc) + deallocate(compdof) + + num_iotypes = 0 + call PIO_TF_Get_nc_iotypes(iotypes, iotype_descs, num_iotypes) + filename = "test_pio_decomp_simple_tests.testfile" + do i=1,num_iotypes + PIO_TF_LOG(0,*) "Testing : PIO_TF_DATA_TYPE : ", iotype_descs(i) + ierr = PIO_createfile(pio_tf_iosystem_, pio_file, iotypes(i), filename, PIO_CLOBBER) + PIO_TF_CHECK_ERR(ierr, "Could not create file " // trim(filename)) + + ierr = PIO_def_dim(pio_file, 'PIO_TF_test_dim_row', dims(1), pio_dims(1)) + PIO_TF_CHECK_ERR(ierr, "Failed to define a dim : " // trim(filename)) + + ierr = PIO_def_dim(pio_file, 'PIO_TF_test_dim_col', dims(2), pio_dims(2)) + PIO_TF_CHECK_ERR(ierr, "Failed to define a dim : " // trim(filename)) + + ierr = PIO_def_dim(pio_file, 'PIO_TF_test_dim_hgt', dims(3), pio_dims(3)) + PIO_TF_CHECK_ERR(ierr, "Failed to define a dim : " // trim(filename)) + + ierr = PIO_def_dim(pio_file, 'PIO_TF_test_dim_time', pio_unlimited, pio_dims(4)) + PIO_TF_CHECK_ERR(ierr, "Failed to define a dim : " // trim(filename)) + + ierr = PIO_def_var(pio_file, 'PIO_TF_test_3d_var', PIO_TF_DATA_TYPE, pio_dims(1:3), pio_var3d) + PIO_TF_CHECK_ERR(ierr, "Failed to define a 3d var : " // trim(filename)) + + ierr = PIO_def_var(pio_file, 'PIO_TF_test_4d_var', PIO_TF_DATA_TYPE, pio_dims, pio_var4d) + PIO_TF_CHECK_ERR(ierr, "Failed to define a 4d var : " // trim(filename)) + + ierr = PIO_enddef(pio_file) + PIO_TF_CHECK_ERR(ierr, "Failed to end redef mode : " // trim(filename)) + + call PIO_write_darray(pio_file, pio_var3d, wr_iodesc, wbuf3d, ierr) + PIO_TF_CHECK_ERR(ierr, "Failed to write 3d darray : " // trim(filename)) + + do f=1,NFRAMES + call PIO_setframe(pio_file, pio_var4d, f) + ! Write the current frame + call PIO_write_darray(pio_file, pio_var4d, wr_iodesc, wbuf4d(:,:,:,f), ierr) + PIO_TF_CHECK_ERR(ierr, "Failed to write 4d darray : " // trim(filename)) + end do + call PIO_syncfile(pio_file) + + rbuf4d = 0 + rbuf3d = 0 + + call PIO_read_darray(pio_file, pio_var3d, rd_iodesc, rbuf3d, ierr) + PIO_TF_CHECK_ERR(ierr, "Failed to read 3d darray : " // trim(filename)) + + do f=1,NFRAMES + call PIO_setframe(pio_file, pio_var4d, f) + call PIO_read_darray(pio_file, pio_var4d, rd_iodesc, rbuf4d(:,:,:,f), ierr) + PIO_TF_CHECK_ERR(ierr, "Failed to read 4d darray : " // trim(filename)) + end do + + do f=1,NFRAMES + PIO_TF_CHECK_VAL((rbuf4d(:,:,:,f), exp_val4d(:,:,:,f)), "Got wrong 4d val, frame=", f) + end do + PIO_TF_CHECK_VAL((rbuf3d, exp_val3d), "Got wrong 3dd val") + + call PIO_closefile(pio_file) + + call PIO_deletefile(pio_tf_iosystem_, filename); + end do + + if(allocated(iotypes)) then + deallocate(iotypes) + deallocate(iotype_descs) + end if + + call PIO_freedecomp(pio_tf_iosystem_, rd_iodesc) + call PIO_freedecomp(pio_tf_iosystem_, wr_iodesc) + + deallocate(exp_val3d) + deallocate(rbuf3d) + deallocate(wbuf3d) + + deallocate(exp_val4d) + deallocate(rbuf4d) + deallocate(wbuf4d) +PIO_TF_AUTO_TEST_SUB_END nc_reuse_3d_decomp diff --git a/src/externals/pio2/tests/general/util/pio_tutil.F90 b/src/externals/pio2/tests/general/util/pio_tutil.F90 index 43c0b634b4a..e4a076f3a10 100644 --- a/src/externals/pio2/tests/general/util/pio_tutil.F90 +++ b/src/externals/pio2/tests/general/util/pio_tutil.F90 @@ -129,7 +129,7 @@ SUBROUTINE PIO_TF_Init_(rearr) - pio_tf_log_level_ = 0 + pio_tf_log_level_ = 3 pio_tf_num_aggregators_ = 0 pio_tf_num_io_tasks_ = 0 pio_tf_stride_ = 1