From aac1aa0d7e4238d92ccef2e8de826d0614e41e67 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Tue, 6 Mar 2018 16:37:46 -0700 Subject: [PATCH] add for get_vard --- src/clib/CMakeLists.txt | 6 +- src/clib/pio_darray_int.c | 308 +++++++++++++++++++++++--------------- src/clib/pio_file.c | 4 +- src/clib/pio_lists.c | 2 +- src/clib/pioc_support.c | 4 + src/flib/CMakeLists.txt | 2 + 6 files changed, 197 insertions(+), 129 deletions(-) diff --git a/src/clib/CMakeLists.txt b/src/clib/CMakeLists.txt index 53ef0daa37e..baf4dcd3c75 100644 --- a/src/clib/CMakeLists.txt +++ b/src/clib/CMakeLists.txt @@ -8,12 +8,12 @@ project (PIOC C) add_library (pioc topology.c pio_file.c pioc_support.c pio_lists.c pioc.c pioc_sc.c pio_spmd.c pio_rearrange.c pio_nc4.c bget.c - pio_nc.c pio_put_nc.c pio_get_nc.c pio_getput_int.c pio_msg.c + pio_nc.c pio_put_nc.c pio_get_nc.c pio_getput_int.c pio_msg.c pio_darray.c pio_darray_int.c) # set up include-directories include_directories( - "${CMAKE_BINARY_DIR}" + "${CMAKE_BINARY_DIR}" "${PROJECT_SOURCE_DIR}" # to find foo/foo.h "${PROJECT_BINARY_DIR}") # to find foo/config.h @@ -36,7 +36,7 @@ elseif ("${CMAKE_C_COMPILER_ID}" STREQUAL "PGI") PRIVATE -c99) elseif ("${CMAKE_C_COMPILER_ID}" STREQUAL "Intel") target_compile_options (pioc - PRIVATE -std=c99) + PRIVATE -std=c99 -debug minimal) elseif ("${CMAKE_C_COMPILER_ID}" STREQUAL "Clang") target_compile_options (pioc PRIVATE -std=c99) diff --git a/src/clib/pio_darray_int.c b/src/clib/pio_darray_int.c index 4387db172f0..0c04abe73cc 100644 --- a/src/clib/pio_darray_int.c +++ b/src/clib/pio_darray_int.c @@ -66,6 +66,136 @@ int compute_buffer_init(iosystem_desc_t *ios) return PIO_NOERR; } +#if USE_VARD +int get_gdim0(file_desc_t *file,io_desc_t *iodesc, int varid, int fndims, MPI_Offset *gdim0) +{ + int ierr=PIO_NOERR; + if(file->iotype == PIO_IOTYPE_PNETCDF && iodesc->ndims < fndims) + { + int numunlimdims; + *gdim0 = 0; + /* We need to confirm the file has an unlimited dimension and if it doesn't we need to find + the extent of the first variable dimension */ + LOG((3,"look for numunlimdims")); + if ((ierr = PIOc_inq_unlimdims(file->pio_ncid, &numunlimdims, NULL))) + return check_netcdf(file, ierr, __FILE__, __LINE__); + LOG((3,"numunlimdims = %d", numunlimdims)); + if (numunlimdims <= 0) + { + int dimids[fndims]; + if ((ierr = PIOc_inq_vardimid(file->pio_ncid, varid, dimids))) + return check_netcdf(file, ierr, __FILE__, __LINE__); + if ((ierr = PIOc_inq_dimlen(file->pio_ncid, dimids[0], gdim0))) + return check_netcdf(file, ierr, __FILE__, __LINE__); + } + } + LOG((3,"gdim0 = %d",gdim0)); + return ierr; +} + +int get_vard_mpidatatype(io_desc_t *iodesc, MPI_Offset gdim0, PIO_Offset unlimdimoffset, + int rrcnt, int ndims, int fndims, + int frame, PIO_Offset **startlist, PIO_Offset **countlist, + MPI_Datatype *subarray, MPI_Datatype *filetype) +{ + + int sa_ndims; + int sacount[fndims]; + int sastart[fndims]; + int gdims[fndims]; + int dim_offset; + int mpierr; + MPI_Aint displacements[rrcnt]; + int blocklengths[rrcnt]; + + if(gdim0) + sacount[0] = 1; + + for ( int rc=0; rc ndims) + { + if ( gdim0 > 0) + { + gdims[0] = gdim0; + sa_ndims = fndims; + dim_offset = 0; + for (int i=1; i < fndims; i++) + gdims[i] = iodesc->dimlen[i-1]; + } + else + { + sa_ndims = ndims; + dim_offset = 1; + for (int i=0; i < ndims; i++) + gdims[i] = iodesc->dimlen[i]; + } + } + else + { + sa_ndims = fndims; + dim_offset = 0; + for (int i=0; i < fndims; i++) + gdims[i] = iodesc->dimlen[i]; + } + + for( int rc=0; rc 0) + { + sastart[0] = max(0, frame); + displacements[rc]=0; + } + else + displacements[rc] = unlimdimoffset * max(0, frame); +#if PIO_ENABLE_LOGGING + for (int i=0; i< sa_ndims; i++) + LOG((3, "vard: sastart[%d]=%d sacount[%d]=%d gdims[%d]=%d %ld %ld", + i,sastart[i], i,sacount[i], i, gdims[i], startlist[rc][i], countlist[rc][i])); +#endif + if((mpierr = MPI_Type_create_subarray(sa_ndims, gdims, + sacount, sastart,MPI_ORDER_C + ,iodesc->mpitype, subarray + rc))) + return check_mpi(NULL, mpierr, __FILE__, __LINE__); + + if((mpierr = MPI_Type_commit(subarray + rc))) + return check_mpi(NULL, mpierr, __FILE__, __LINE__); + +#if PIO_ENABLE_LOGGING + LOG((3,"vard: blocklengths[%d]=%d displacement[%d]=%ld unlimdimoffset=%ld",rc,blocklengths[rc], rc, displacements[rc], unlimdimoffset)); +#endif + + + } + + if((mpierr = MPI_Type_create_struct(rrcnt, blocklengths, displacements, subarray, filetype))) + return check_mpi(NULL, mpierr, __FILE__, __LINE__); + + if((mpierr = MPI_Type_commit(filetype))) + return check_mpi(NULL, mpierr, __FILE__, __LINE__); + +} + +#endif + + /** * Fill start/count arrays for write_darray_multi_par(). This is an @@ -196,26 +326,7 @@ int write_darray_multi_par(file_desc_t *file, int nvars, int fndims, const int * void *iobuf = fill ? vdesc->fillbuf : file->iobuf; #if USE_VARD - if(file->iotype == PIO_IOTYPE_PNETCDF && iodesc->ndims < fndims) - { - int numunlimdims; - gdim0 = 0; - /* We need to confirm the file has an unlimited dimension and if it doesn't we need to find - the extent of the first variable dimension */ - LOG((3,"look for numunlimdims")); - if ((ierr = PIOc_inq_unlimdims(file->pio_ncid, &numunlimdims, NULL))) - return check_netcdf(file, ierr, __FILE__, __LINE__); - LOG((3,"numunlimdims = %d", numunlimdims)); - if (numunlimdims <= 0) - { - int dimids[fndims]; - if ((ierr = PIOc_inq_vardimid(file->pio_ncid, varids[0], dimids))) - return check_netcdf(file, ierr, __FILE__, __LINE__); - if ((ierr = PIOc_inq_dimlen(file->pio_ncid, dimids[0], &gdim0))) - return check_netcdf(file, ierr, __FILE__, __LINE__); - } - } - LOG((3,"gdim0 = %d",gdim0)); + ierr = get_gdim0(file, iodesc, varids[0], fndims, &gdim0); #endif /* If this is an IO task write the data. */ @@ -320,104 +431,15 @@ int write_darray_multi_par(file_desc_t *file, int nvars, int fndims, const int * #if USE_VARD /* If this is the first variable or the frame has changed between variables (this should be rare) */ if(nv==0 || (nv > 0 && frame != NULL && frame[nv] != frame[nv-1])){ - int sa_ndims; - int sacount[fndims]; - int sastart[fndims]; - int gdims[fndims]; - int dim_offset; - int mpierr; - MPI_Aint displacements[rrcnt]; - int blocklengths[rrcnt]; - - if(gdim0) - sacount[0] = 1; - - for ( int rc=0; rc ndims) - { - if ( gdim0 > 0) - { - gdims[0] = gdim0; - sa_ndims = fndims; - dim_offset = 0; - for (int i=1; i < fndims; i++) - gdims[i] = iodesc->dimlen[i-1]; - } - else - { - sa_ndims = ndims; - dim_offset = 1; - for (int i=0; i < ndims; i++) - gdims[i] = iodesc->dimlen[i]; - } - } + int thisframe; + if (frame) + thisframe = frame[nv]; else - { - sa_ndims = fndims; - dim_offset = 0; - for (int i=0; i < fndims; i++) - gdims[i] = iodesc->dimlen[i]; - } - - for( int rc=0; rc 0) - { - if(frame != NULL) - { - sastart[0] = frame[nv]; - displacements[rc]=0; - } - } - else - if (frame != NULL) - displacements[rc] = unlimdimoffset * frame[nv]; - else - displacements[rc] = 0; - - for (int i=0; i< sa_ndims; i++) - LOG((3, "vard: sastart[%d]=%d sacount[%d]=%d gdims[%d]=%d %ld %ld", - i,sastart[i], i,sacount[i], i, gdims[i], startlist[rc][i], countlist[rc][i])); - - if((mpierr = MPI_Type_create_subarray(sa_ndims, gdims, - sacount, sastart,MPI_ORDER_C - ,iodesc->mpitype, subarray + rc))) - return check_mpi(NULL, mpierr, __FILE__, __LINE__); - - if((mpierr = MPI_Type_commit(subarray + rc))) - return check_mpi(NULL, mpierr, __FILE__, __LINE__); - - - LOG((3,"vard: blocklengths[%d]=%d displacement[%d]=%ld unlimdimoffset=%ld",rc,blocklengths[rc], rc, displacements[rc], unlimdimoffset)); - - - - } - - if((mpierr = MPI_Type_create_struct(rrcnt, blocklengths, displacements, subarray, &filetype))) - return check_mpi(NULL, mpierr, __FILE__, __LINE__); - - if((mpierr = MPI_Type_commit(&filetype))) - return check_mpi(NULL, mpierr, __FILE__, __LINE__); - + thisframe = 0; + ierr = get_vard_mpidatatype(iodesc, gdim0, unlimdimoffset, + rrcnt, ndims, fndims, + thisframe, startlist, countlist, + subarray, &filetype); } #else if (vdesc->record >= 0 && ndims < fndims) @@ -920,6 +942,9 @@ int pio_read_darray_nc(file_desc_t *file, io_desc_t *iodesc, int vid, void *iobu int ndims; /* Number of dims in decomposition. */ int fndims; /* Number of dims for this var in file. */ int ierr; /* Return code from netCDF functions. */ +#ifdef USE_VARD + MPI_Offset gdim0; +#endif /* Check inputs. */ pioassert(file && file->iosystem && iodesc && vid <= PIO_MAX_VARS, "invalid input", @@ -943,8 +968,11 @@ int pio_read_darray_nc(file_desc_t *file, io_desc_t *iodesc, int vid, void *iobu /* Get the number of dims for this var in the file. */ if ((ierr = PIOc_inq_varndims(file->pio_ncid, vid, &fndims))) return pio_err(ios, file, ierr, __FILE__, __LINE__); +#if USE_VARD + ierr = get_gdim0(file, iodesc, vid, fndims, &gdim0); +#endif - /* IO procs will actially read the data. */ + /* IO procs will read the data. */ if (ios->ioproc) { io_region *region; @@ -956,6 +984,20 @@ int pio_read_darray_nc(file_desc_t *file, io_desc_t *iodesc, int vid, void *iobu PIO_Offset *startlist[iodesc->maxregions]; PIO_Offset *countlist[iodesc->maxregions]; +#if USE_VARD + PIO_Offset unlimdimoffset; + int mpierr; + MPI_Datatype filetype; + MPI_Datatype subarray[iodesc->maxregions]; + filetype = MPI_DATATYPE_NULL; + + if (gdim0 == 0) /* if there is an unlimited dimension get the offset between records of a variable */ + { + if((ierr = ncmpi_inq_recsize(file->fh, &unlimdimoffset))) + return pio_err(NULL, file, ierr, __FILE__, __LINE__); + } +#endif + /* buffer is incremented by byte and loffset is in terms of the iodessc->mpitype so we need to multiply by the size of the mpitype. */ @@ -965,8 +1007,6 @@ int pio_read_darray_nc(file_desc_t *file, io_desc_t *iodesc, int vid, void *iobu * and the file. */ if (fndims > ndims) { - ndims++; - /* If the user did not call setframe, use a default frame * of 0. This is required for backward compatibility. */ if (vdesc->record < 0) @@ -1004,7 +1044,7 @@ int pio_read_darray_nc(file_desc_t *file, io_desc_t *iodesc, int vid, void *iobu /* This is a record var. The unlimited dimension * (0) is handled specially. */ start[0] = vdesc->record; - for (int i = 1; i < ndims; i++) + for (int i = 1; i < fndims; i++) { start[i] = region->start[i-1]; count[i] = region->count[i-1]; @@ -1017,7 +1057,7 @@ int pio_read_darray_nc(file_desc_t *file, io_desc_t *iodesc, int vid, void *iobu else { /* Non-time dependent array */ - for (int i = 0; i < ndims; i++) + for (int i = 0; i < fndims; i++) { start[i] = region->start[i]; count[i] = region->count[i]; @@ -1102,10 +1142,31 @@ int pio_read_darray_nc(file_desc_t *file, io_desc_t *iodesc, int vid, void *iobu /* Is this is the last region to process? */ if (regioncnt == iodesc->maxregions - 1) { +#if USE_VARD + MPI_Datatype filetype, subarray[rrlen]; + filetype = MPI_DATATYPE_NULL; + for(int i=0; irecord, startlist, countlist, + subarray, &filetype); + ierr = ncmpi_get_vard_all(file->fh, vid, filetype, iobuf, iodesc->llen, iodesc->mpitype); + for(int i=0; ifh, vid, rrlen, startlist, countlist, iobuf, iodesc->llen, iodesc->mpitype); - +#endif /* Release the start and count arrays. */ for (int i = 0; i < rrlen; i++) { @@ -1655,7 +1716,6 @@ int flush_buffer(int ncid, wmulti_buffer *wmb, bool flushtodisk) /* Check input. */ pioassert(wmb, "invalid input", __FILE__, __LINE__); - /* Get the file info (to get error handler). */ if ((ret = pio_get_file(ncid, &file))) return pio_err(NULL, NULL, ret, __FILE__, __LINE__); diff --git a/src/clib/pio_file.c b/src/clib/pio_file.c index 104dd6aa999..b6f894d930c 100644 --- a/src/clib/pio_file.c +++ b/src/clib/pio_file.c @@ -205,7 +205,6 @@ int PIOc_closefile(int ncid) int mpierr = MPI_SUCCESS, mpierr2; /* Return code from MPI function codes. */ LOG((1, "PIOc_closefile ncid = %d", ncid)); - /* Find the info about this file. */ if ((ierr = pio_get_file(ncid, &file))) return pio_err(NULL, NULL, ierr, __FILE__, __LINE__); @@ -387,6 +386,7 @@ int PIOc_sync(int ncid) wmulti_buffer *wmb, *twmb; LOG((3, "PIOc_sync checking buffers")); + wmb = &file->buffer; while (wmb) { @@ -394,6 +394,7 @@ int PIOc_sync(int ncid) * multibuffer, flush it. */ if (wmb->num_arrays > 0) flush_buffer(ncid, wmb, true); + twmb = wmb; wmb = wmb->next; if (twmb == &file->buffer) @@ -464,6 +465,7 @@ int PIOc_sync(int ncid) return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); if (ierr) return check_netcdf2(ios, NULL, ierr, __FILE__, __LINE__); + PIOc_set_log_level(0); return ierr; } diff --git a/src/clib/pio_lists.c b/src/clib/pio_lists.c index c4edb5d5042..4f5af7858ac 100644 --- a/src/clib/pio_lists.c +++ b/src/clib/pio_lists.c @@ -388,7 +388,7 @@ int get_var_desc(int varid, var_desc_t **varlist, var_desc_t **var_desc) var_desc_t *my_var; /* Check inputs. */ - pioassert(varlist, "invalid input", __FILE__, __LINE__); + pioassert(varlist, "invalid input", __FILE__, __LINE__); /* Empty varlist. */ if (!*varlist) diff --git a/src/clib/pioc_support.c b/src/clib/pioc_support.c index e47c36dcf7e..3c8832ee7e9 100644 --- a/src/clib/pioc_support.c +++ b/src/clib/pioc_support.c @@ -25,6 +25,8 @@ int my_rank; FILE *LOG_FILE = NULL; #endif /* PIO_ENABLE_LOGGING */ +int mycnt=0; + /** * The PIO library maintains its own set of ncids. This is the next * ncid number that will be assigned. @@ -213,6 +215,7 @@ void pio_log(int severity, const char *fmt, ...) ptr += strlen(ERROR_PREFIX); rem_len -= strlen(ERROR_PREFIX); } + for (t = 0; t < severity; t++) { strncpy(ptr++, "\t", (rem_len > 0) ? rem_len : 0); @@ -230,6 +233,7 @@ void pio_log(int severity, const char *fmt, ...) vsnprintf(ptr, ((rem_len > 0) ? rem_len : 0), fmt, argp); va_end(argp); + /* Put on a final linefeed. */ ptr = msg + strlen(msg); rem_len = MAX_LOG_MSG - strlen(msg); diff --git a/src/flib/CMakeLists.txt b/src/flib/CMakeLists.txt index 8d510961d4f..8942ec8871a 100644 --- a/src/flib/CMakeLists.txt +++ b/src/flib/CMakeLists.txt @@ -57,6 +57,8 @@ elseif (CMAKE_Fortran_COMPILER_ID STREQUAL "NAG") # PRIVATE -mismatch_all) elseif (CMAKE_Fortran_COMPILER_ID STREQUAL "Cray") set (CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -ef") +elseif (CMAKE_Fortran_COMPILER_ID STREQUAL "Intel") + set (CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -debug minimal") endif() # Look for c_sizeof capability