Skip to content

Commit

Permalink
Merge pull request ESMCI#1335 from jedwards4b/support_type_conversion
Browse files Browse the repository at this point in the history 
remove restrictions to data conversion
  • Loading branch information
@jedwards4b
jedwards4b authored Feb 20, 2019
2 parents 6e3be58 + ff6c885 commit 0c390bf
Show file tree
Hide file tree
Showing 9 changed files with 114 additions and 66 deletions.
7 changes: 1 addition & 6 deletions cmake/mpiexec.nwscla
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,4 @@

NP=$1
shift
if [[ "$LMOD_FAMILY_MPI" == "openmpi" ]]
then
mpirun -np $NP $@
else
mpiexec_mpt -n $NP $@
fi
mpirun -np $NP $@
2 changes: 2 additions & 0 deletions src/clib/pio.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -760,6 +760,8 @@ enum PIO_ERROR_HANDLERS
/** Define error codes for PIO. */
#define PIO_FIRST_ERROR_CODE (-500)
#define PIO_EBADIOTYPE (-500)
/** variable dimensions do not match in a multivar call */
#define PIO_EVARDIMMISMATCH (-501)

/** ??? */
#define PIO_REQ_NULL (NC_REQ_NULL-1)
Expand Down
70 changes: 39 additions & 31 deletions src/clib/pio_darray.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,7 @@ int PIOc_write_darray_multi(int ncid, const int *varids, int ioid, int nvars,
io_desc_t *iodesc; /* Pointer to IO description information. */
int rlen; /* Total data buffer size. */
var_desc_t *vdesc0; /* First entry in array of var_desc structure for each var. */
int fndims; /* Number of dims in the var in the file. */
int fndims, fndims2; /* Number of dims in the var in the file. */
int mpierr = MPI_SUCCESS, mpierr2; /* Return code from MPI function calls. */
int ierr; /* Return code. */
void *tmparray;
Expand Down Expand Up @@ -145,8 +145,8 @@ int PIOc_write_darray_multi(int ncid, const int *varids, int ioid, int nvars,
var_desc_t *vdesc;
if ((ierr = get_var_desc(varids[v], &file->varlist, &vdesc)))
return pio_err(ios, file, ierr, __FILE__, __LINE__);
if (vdesc->pio_type != iodesc->piotype)
return pio_err(ios, file, PIO_EINVAL, __FILE__, __LINE__);
// if (vdesc->pio_type != iodesc->piotype)
// return pio_err(ios, file, PIO_EINVAL, __FILE__, __LINE__);
}

/* Get a pointer to the variable info for the first variable. */
Expand All @@ -162,6 +162,13 @@ int PIOc_write_darray_multi(int ncid, const int *varids, int ioid, int nvars,
if ((ierr = PIOc_inq_varndims(file->pio_ncid, varids[0], &fndims)))
return check_netcdf(file, ierr, __FILE__, __LINE__);
LOG((3, "called PIOc_inq_varndims varids[0] = %d fndims = %d", varids[0], fndims));
for (int v=1; v < nvars; v++){
if ((ierr = PIOc_inq_varndims(file->pio_ncid, varids[v], &fndims2)))
return check_netcdf(file, ierr, __FILE__, __LINE__);
if(fndims != fndims2)
return pio_err(ios, file, PIO_EVARDIMMISMATCH, __FILE__, __LINE__);
}

}

/* If async is in use, and this is not an IO task, bcast the parameters. */
Expand Down Expand Up @@ -247,7 +254,7 @@ int PIOc_write_darray_multi(int ncid, const int *varids, int ioid, int nvars,

/* If fill values are desired, and we're using the BOX
* rearranger, insert fill values. */
if (iodesc->needsfill && iodesc->rearranger == PIO_REARR_BOX)
if (iodesc->needsfill && iodesc->rearranger == PIO_REARR_BOX && fillvalue)
{
LOG((3, "inerting fill values iodesc->maxiobuflen = %d", iodesc->maxiobuflen));
for (int nv = 0; nv < nvars; nv++)
Expand Down Expand Up @@ -339,10 +346,11 @@ int PIOc_write_darray_multi(int ncid, const int *varids, int ioid, int nvars,
/* copying the fill value into the data buffer for the box
* rearranger. This will be overwritten with data where
* provided. */
for (int nv = 0; nv < nvars; nv++)
for (int i = 0; i < iodesc->holegridsize; i++)
memcpy(&((char *)vdesc0->fillbuf)[iodesc->mpitype_size * (i + nv * iodesc->holegridsize)],
&((char *)fillvalue)[iodesc->mpitype_size * nv], iodesc->mpitype_size);
if(fillvalue)
for (int nv = 0; nv < nvars; nv++)
for (int i = 0; i < iodesc->holegridsize; i++)
memcpy(&((char *)vdesc0->fillbuf)[iodesc->mpitype_size * (i + nv * iodesc->holegridsize)],
&((char *)fillvalue)[iodesc->mpitype_size * nv], iodesc->mpitype_size);

/* Write the darray based on the iotype. */
switch (file->iotype)
Expand Down Expand Up @@ -647,10 +655,10 @@ int PIOc_write_darray(int ncid, int varid, int ioid, PIO_Offset arraylen, void *

/* If the type of the var doesn't match the type of the
* decomposition, return an error. */
if (iodesc->piotype != vdesc->pio_type)
return pio_err(ios, file, PIO_EINVAL, __FILE__, __LINE__);
pioassert(iodesc->mpitype_size == vdesc->mpi_type_size, "wrong mpi info",
__FILE__, __LINE__);
// if (iodesc->piotype != vdesc->pio_type)
// return pio_err(ios, file, PIO_EINVAL, __FILE__, __LINE__);
// pioassert(iodesc->mpitype_size == vdesc->mpi_type_size, "wrong mpi info",
// __FILE__, __LINE__);

/* If we don't know the fill value for this var, get it. */
if (!vdesc->fillvalue)
Expand Down Expand Up @@ -689,13 +697,13 @@ int PIOc_write_darray(int ncid, int varid, int ioid, PIO_Offset arraylen, void *
wmb->frame = NULL;
wmb->fillvalue = NULL;
}
LOG((2, "wmb->num_arrays = %d arraylen = %d vdesc->mpi_type_size = %d\n",
wmb->num_arrays, arraylen, vdesc->mpi_type_size));
LOG((2, "wmb->num_arrays = %d arraylen = %d iodesc->mpitype_size = %d\n",
wmb->num_arrays, arraylen, iodesc->mpitype_size));
#if PIO_USE_MALLOC
/* Try realloc first and call flush if realloc fails. */
if (arraylen > 0)
{
size_t data_size = (1 + wmb->num_arrays) * arraylen * vdesc->mpi_type_size;
size_t data_size = (1 + wmb->num_arrays) * arraylen * iodesc->mpitype_size;

if ((realloc_data = realloc(wmb->data, data_size)))
{
Expand All @@ -716,12 +724,12 @@ int PIOc_write_darray(int ncid, int varid, int ioid, PIO_Offset arraylen, void *
/* maxfree is the available memory. If that is < 10% greater than
* the size of the current request needsflush is true. */
if (needsflush == 0)
needsflush = (maxfree <= 1.1 * (1 + wmb->num_arrays) * arraylen * vdesc->mpi_type_size);
needsflush = (maxfree <= 1.1 * (1 + wmb->num_arrays) * arraylen * iodesc->mpitype_size);
#endif
/* the limit of data_size < INT_MAX is due to a bug in ROMIO which limits
the size of contiguous data to INT_MAX, a fix has been proposed in
https://github.com/pmodels/mpich/pull/2888 */
io_data_size = (1 + wmb->num_arrays) * iodesc->maxiobuflen * vdesc->mpi_type_size;
io_data_size = (1 + wmb->num_arrays) * iodesc->maxiobuflen * iodesc->mpitype_size;
if(io_data_size > INT_MAX)
needsflush = 2;

Expand All @@ -740,8 +748,8 @@ int PIOc_write_darray(int ncid, int varid, int ioid, PIO_Offset arraylen, void *
/* Collect a debug report about buffer. */
cn_buffer_report(ios, true);
LOG((2, "maxfree = %ld wmb->num_arrays = %d (1 + wmb->num_arrays) *"
" arraylen * vdesc->mpi_type_size = %ld totfree = %ld\n", maxfree, wmb->num_arrays,
(1 + wmb->num_arrays) * arraylen * vdesc->mpi_type_size, totfree));
" arraylen * iodesc->mpitype_size = %ld totfree = %ld\n", maxfree, wmb->num_arrays,
(1 + wmb->num_arrays) * arraylen * iodesc->mpitype_size, totfree));
#endif /* PIO_ENABLE_LOGGING */
#endif /* !PIO_USE_MALLOC */

Expand All @@ -756,17 +764,17 @@ int PIOc_write_darray(int ncid, int varid, int ioid, PIO_Offset arraylen, void *
/* Try realloc again if there is a flush. */
if (arraylen > 0 && needsflush > 0)
{
if (!(wmb->data = realloc(wmb->data, (1 + wmb->num_arrays) * arraylen * vdesc->mpi_type_size)))
if (!(wmb->data = realloc(wmb->data, (1 + wmb->num_arrays) * arraylen * iodesc->mpitype_size)))
return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__);
LOG((2, "after a flush, realloc got %ld bytes for data", (1 + wmb->num_arrays) * arraylen * vdesc->mpi_type_size));
LOG((2, "after a flush, realloc got %ld bytes for data", (1 + wmb->num_arrays) * arraylen * iodesc->mpitype_size));
}
#else
/* Get memory for data. */
if (arraylen > 0)
{
if (!(wmb->data = bgetr(wmb->data, (1 + wmb->num_arrays) * arraylen * vdesc->mpi_type_size)))
if (!(wmb->data = bgetr(wmb->data, (1 + wmb->num_arrays) * arraylen * iodesc->mpitype_size)))
return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__);
LOG((2, "got %ld bytes for data", (1 + wmb->num_arrays) * arraylen * vdesc->mpi_type_size));
LOG((2, "got %ld bytes for data", (1 + wmb->num_arrays) * arraylen * iodesc->mpitype_size));
}
#endif

Expand All @@ -788,11 +796,11 @@ int PIOc_write_darray(int ncid, int varid, int ioid, PIO_Offset arraylen, void *
if (iodesc->needsfill)
{
/* Get memory to hold fill value. */
if (!(wmb->fillvalue = bgetr(wmb->fillvalue, vdesc->mpi_type_size * (1 + wmb->num_arrays))))
if (!(wmb->fillvalue = bgetr(wmb->fillvalue, iodesc->mpitype_size * (1 + wmb->num_arrays))))
return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__);

memcpy((char *)wmb->fillvalue + vdesc->mpi_type_size * wmb->num_arrays,
vdesc->fillvalue, vdesc->mpi_type_size);
memcpy((char *)wmb->fillvalue + iodesc->mpitype_size * wmb->num_arrays,
vdesc->fillvalue, iodesc->mpitype_size);
}

/* Tell the buffer about the data it is getting. */
Expand All @@ -802,11 +810,11 @@ int PIOc_write_darray(int ncid, int varid, int ioid, PIO_Offset arraylen, void *
wmb->vid[wmb->num_arrays]));

/* Copy the user-provided data to the buffer. */
bufptr = (void *)((char *)wmb->data + arraylen * vdesc->mpi_type_size * wmb->num_arrays);
bufptr = (void *)((char *)wmb->data + arraylen * iodesc->mpitype_size * wmb->num_arrays);
if (arraylen > 0)
{
memcpy(bufptr, array, arraylen * vdesc->mpi_type_size);
LOG((3, "copied %ld bytes of user data", arraylen * vdesc->mpi_type_size));
memcpy(bufptr, array, arraylen * iodesc->mpitype_size);
LOG((3, "copied %ld bytes of user data", arraylen * iodesc->mpitype_size));
}

/* Add the unlimited dimension value of this variable to the frame
Expand All @@ -815,9 +823,9 @@ int PIOc_write_darray(int ncid, int varid, int ioid, PIO_Offset arraylen, void *
wmb->frame[wmb->num_arrays] = vdesc->record;
wmb->num_arrays++;

LOG((2, "wmb->num_arrays = %d iodesc->maxbytes / vdesc->mpi_type_size = %d "
LOG((2, "wmb->num_arrays = %d iodesc->maxbytes / iodesc->mpitype_size = %d "
"iodesc->ndof = %d iodesc->llen = %d", wmb->num_arrays,
iodesc->maxbytes / vdesc->mpi_type_size, iodesc->ndof, iodesc->llen));
iodesc->maxbytes / iodesc->mpitype_size, iodesc->ndof, iodesc->llen));

return PIO_NOERR;
}
Expand Down
6 changes: 4 additions & 2 deletions src/clib/pio_darray_int.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -638,7 +638,9 @@ int write_darray_multi_par(file_desc_t *file, int nvars, int fndims, const int *
/* Write, in non-blocking fashion, a list of subarrays. */
LOG((3, "about to call ncmpi_iput_varn() varids[%d] = %d rrcnt = %d, llen = %d",
nv, varids[nv], rrcnt, llen));
ierr = ncmpi_iput_varn(file->fh, varids[nv], rrcnt, startlist, countlist,
for (int i=0; i< rrcnt; i++)
LOG((3, "i %d start %ld count %ld start %ld count %ld\n",i,startlist[i][0], countlist[i][0],startlist[i][1], countlist[i][1]));
ierr = ncmpi_iput_varn(file->fh, varids[nv], rrcnt, startlist, countlist,
bufptr, llen, iodesc->mpitype, &vdesc->request[vdesc->nreqs]);

/* keeps wait calls in sync */
Expand Down Expand Up @@ -1213,7 +1215,7 @@ int pio_read_darray_nc(file_desc_t *file, io_desc_t *iodesc, int vid, void *iobu
}
}

#ifdef LOGGING
#ifdef PIO_ENABLE_LOGGING
for (int i = 1; i < ndims; i++)
LOG((3, "start[%d] %d count[%d] %d", i, start[i], i, count[i]));
#endif /* LOGGING */
Expand Down
3 changes: 3 additions & 0 deletions src/clib/pio_internal.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -362,6 +362,9 @@ extern "C" {
int **proc_list, int **my_proc_list);

int pio_sorted_copy(const void *array, void *tmparray, io_desc_t *iodesc, int nvars, int direction);

int PIOc_inq_att_eh(int ncid, int varid, const char *name, int eh,
nc_type *xtypep, PIO_Offset *lenp);
#if defined(__cplusplus)
}
#endif
Expand Down
3 changes: 3 additions & 0 deletions src/clib/pioc_support.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,9 @@ int PIOc_strerror(int pioerr, char *errmsg)
case PIO_EBADIOTYPE:
strcpy(errmsg, "Bad IO type");
break;
case PIO_EVARDIMMISMATCH:
strcpy(errmsg, "Variable dim mismatch in multivar call");
break;
default:
strcpy(errmsg, "Unknown Error: Unrecognized error code");
}
Expand Down
57 changes: 48 additions & 9 deletions tests/cunit/test_darray.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,11 +76,14 @@ int test_darray(int iosysid, int ioid, int num_flavors, int *flavor, int my_rank
int ncid; /* The ncid of the netCDF file. */
int ncid2; /* The ncid of the re-opened netCDF file. */
int varid; /* The ID of the netCDF varable. */
int varid2; /* The ID of a varable of different type. */
int varid2; /* The ID of a netCDF varable of different type. */
int wrong_varid = TEST_VAL_42; /* A wrong ID. */
int ret; /* Return code. */
MPI_Datatype mpi_type;
int type_size; /* size of a variable of type pio_type */
int other_type; /* another variable of the same size but different type */
PIO_Offset arraylen = 4;
void *fillvalue;
void *fillvalue, *ofillvalue;
void *test_data;
void *test_data_in;
int fillvalue_int = NC_FILL_INT;
Expand Down Expand Up @@ -116,7 +119,6 @@ int test_darray(int iosysid, int ioid, int num_flavors, int *flavor, int my_rank
/* Create the filename. */
sprintf(filename, "data_%s_iotype_%d_pio_type_%d_test_multi_%d_provide_fill_%d.nc", TEST_NAME,
flavor[fmt], pio_type, test_multi, provide_fill);

/* Select the fill value and data. */
switch (pio_type)
{
Expand Down Expand Up @@ -152,9 +154,27 @@ int test_darray(int iosysid, int ioid, int num_flavors, int *flavor, int my_rank
if ((ret = PIOc_def_var(ncid, VAR_NAME, pio_type, NDIM, dimids, &varid)))
ERR(ret);

/* Define a variable with a different type. */
int other_type = pio_type == PIO_INT ? PIO_FLOAT : PIO_INT;
if ((ret = PIOc_def_var(ncid, VAR_NAME2, other_type, NDIM, dimids, &varid2)))
/* Define a variable with a different type but same size. */
if ((ret = find_mpi_type(pio_type, &mpi_type, &type_size)))
ERR(ret);
if (type_size == NETCDF_INT_FLOAT_SIZE)
other_type = pio_type == PIO_INT ? PIO_FLOAT : PIO_INT;
// else if(type_size == NETCDF_DOUBLE_INT64_SIZE)
// other_type = pio_type == PIO_INT64 ? PIO_DOUBLE : PIO_INT64;
else
other_type = 0; /* skip the test */
switch (other_type)
{
case PIO_INT:
ofillvalue = provide_fill ? &fillvalue_int : NULL;
break;
case PIO_FLOAT:
ofillvalue = provide_fill ? &fillvalue_float : NULL;
break;
default:
break;
}
if (other_type && (ret = PIOc_def_var(ncid, VAR_NAME2, other_type, NDIM, dimids, &varid2)))
ERR(ret);

/* End define mode. */
Expand All @@ -164,6 +184,8 @@ int test_darray(int iosysid, int ioid, int num_flavors, int *flavor, int my_rank
/* Set the value of the record dimension. */
if ((ret = PIOc_setframe(ncid, varid, 0)))
ERR(ret);
if (other_type && (ret = PIOc_setframe(ncid, varid2, 0)))
ERR(ret);

int frame = 0;
int flushtodisk = test_multi - 1;
Expand All @@ -178,12 +200,15 @@ int test_darray(int iosysid, int ioid, int num_flavors, int *flavor, int my_rank
ERR(ERR_WRONG);
if (PIOc_write_darray(ncid, TEST_VAL_42, ioid, arraylen, test_data, fillvalue) != PIO_ENOTVAR)
ERR(ERR_WRONG);
if (PIOc_write_darray(ncid, varid2, ioid, arraylen, test_data, fillvalue) != PIO_EINVAL)
ERR(ERR_WRONG);

/* This should work - library type conversion */
if (other_type && (ret = PIOc_write_darray(ncid, varid2, ioid, arraylen, test_data, ofillvalue)))
ERR(ret);

/* Write the data. */
if ((ret = PIOc_write_darray(ncid, varid, ioid, arraylen, test_data, fillvalue)))
ERR(ret);

}
else
{
Expand All @@ -205,9 +230,16 @@ int test_darray(int iosysid, int ioid, int num_flavors, int *flavor, int my_rank
if (PIOc_write_darray_multi(ncid, &varid_big, ioid, 1, arraylen, test_data, &frame,
fillvalue, flushtodisk) != PIO_ENOTVAR)
ERR(ERR_WRONG);
// pio_setloglevel(3);
if (PIOc_write_darray_multi(ncid, &wrong_varid, ioid, 1, arraylen, test_data, &frame,
fillvalue, flushtodisk) != PIO_ENOTVAR)
ERR(ERR_WRONG);
// pio_setloglevel(0);

/* This should work - library type conversion */
if (other_type && (ret = PIOc_write_darray_multi(ncid, &varid2, ioid, 1, arraylen, test_data, &frame,
fillvalue, flushtodisk)))
ERR(ret);

/* Write the data with the _multi function. */
if ((ret = PIOc_write_darray_multi(ncid, &varid, ioid, 1, arraylen, test_data, &frame,
Expand All @@ -219,10 +251,18 @@ int test_darray(int iosysid, int ioid, int num_flavors, int *flavor, int my_rank
if ((ret = PIOc_closefile(ncid)))
ERR(ret);


/* Reopen the file. */
if ((ret = PIOc_openfile(iosysid, &ncid2, &flavor[fmt], filename, PIO_NOWRITE)))
ERR(ret);

PIO_Offset dimlen;
/* check the unlimited dim size - it should be 1 */
if ((ret = PIOc_inq_dimlen(ncid2, dimids[0], &dimlen)))
ERR(ret);
if (dimlen != 1)
ERR(ERR_WRONG);

/* These should not work. */
if (PIOc_read_darray(ncid2 + TEST_VAL_42, varid, ioid, arraylen,
test_data_in) != PIO_EBADID)
Expand Down Expand Up @@ -273,7 +313,6 @@ int test_darray(int iosysid, int ioid, int num_flavors, int *flavor, int my_rank
fillvalue, flushtodisk) != PIO_EPERM)
ERR(ERR_WRONG);
}

/* Close the netCDF file. */
if ((ret = PIOc_closefile(ncid2)))
ERR(ret);
Expand Down
6 changes: 3 additions & 3 deletions tests/cunit/test_darray_multi.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -283,9 +283,9 @@ int test_darray(int iosysid, int ioid, int num_flavors, int *flavor, int my_rank
wrong_varid[0] = varid[0];
wrong_varid[1] = varid[1];
wrong_varid[0] = other_varid;
if (PIOc_write_darray_multi(ncid, wrong_varid, ioid, NVAR, arraylen, test_data, frame,
fillvalue, flushtodisk) != PIO_EINVAL)
ERR(ERR_WRONG);
// if (PIOc_write_darray_multi(ncid, wrong_varid, ioid, NVAR, arraylen, test_data, frame,
// fillvalue, flushtodisk) != PIO_EINVAL)
// ERR(ERR_WRONG);

/* Write the data with the _multi function. */
if ((ret = PIOc_write_darray_multi(ncid, varid, ioid, NVAR, arraylen, test_data, frame,
Expand Down
Loading

0 comments on commit 0c390bf

Please sign in to comment.