Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix Subfile Read Issue #62

Open
wants to merge 22 commits into
base: master
Choose a base branch
from
Open

Fix Subfile Read Issue #62

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
ba718a1
fix: correct subfile name at open
yzanhua Mar 5, 2023
f7049e4
temporarily diable subfile read
yzanhua Mar 5, 2023
d905666
check return error
wkliao Mar 7, 2023
493ea35
check env H5VL_LOG_NSUBFILES
wkliao Mar 7, 2023
63963d0
add test program subfile_dwrite.cpp
wkliao Mar 7, 2023
5d1fed5
test subfiling by setting env H5VL_LOG_NSUBFILES
wkliao Mar 7, 2023
ebd19c2
clean test output subfile folders
wkliao Mar 7, 2023
0e11f4f
change single_subfile_read to fapl
yzanhua Apr 24, 2023
d21c041
subfile create/open: use fp->info
yzanhua Apr 24, 2023
eeca154
fix: correct nldset and nmdset for subfiles
yzanhua Apr 24, 2023
9436082
subfile read: support read after open
yzanhua Apr 24, 2023
3601bbc
add subfile read testcases
yzanhua Apr 24, 2023
b21a80c
init fp->subfilerecords to NULL
yzanhua Apr 24, 2023
c32690c
fix: subfile read: mismatch of nsubfiles and nproc
yzanhua Apr 26, 2023
6c0a053
fix: possible hangs during dataset read.
yzanhua Apr 26, 2023
817fd11
testcase: subfile_dread tests multiple scenarios
yzanhua Apr 26, 2023
c15c360
subfile_read test: fewer processes cases
yzanhua May 30, 2023
3b3a0a2
DEBUG PRINT
yzanhua Jun 5, 2023
4e61d18
add print internally
yzanhua Jun 5, 2023
2f85ab9
add more print internally
yzanhua Jun 5, 2023
7f368c6
test smaller mem
yzanhua Jun 5, 2023
815918f
possible fix 1
yzanhua Jun 5, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions src/H5VL_log.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -657,9 +657,9 @@ herr_t H5Pset_single_subfile_read (hid_t plist, hbool_t single_subfile_read) {
htri_t pexist;

try {
isfapl = H5Pisa_class (plist, H5P_FILE_CREATE);
isfapl = H5Pisa_class (plist, H5P_FILE_ACCESS);
CHECK_ID (isfapl)
if (isfapl == 0) ERR_OUT ("Not fcplid")
if (isfapl == 0) ERR_OUT ("Not faplid")

pexist = H5Pexist (plist, SINGLE_SUBFILE_READ_PROPERTY_NAME);
CHECK_ID (pexist)
Expand All @@ -684,7 +684,7 @@ herr_t H5Pget_single_subfile_read (hid_t plist, hbool_t *single_subfile_read) {
htri_t isfapl, pexist;

try {
isfapl = H5Pisa_class (plist, H5P_FILE_CREATE);
isfapl = H5Pisa_class (plist, H5P_FILE_ACCESS);
CHECK_ID (isfapl)
if (isfapl == 0)
*single_subfile_read = false; // Default property will not pass class check
Expand Down
50 changes: 39 additions & 11 deletions src/H5VL_log_file.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@
#include "H5VL_logi.hpp"
#include "H5VL_logi_util.hpp"

#define DEBUG_PRINT {printf("\tDEBUG: %s:%d\n", __FILE__, __LINE__);}

/********************* */
/* Function prototypes */
/********************* */
Expand Down Expand Up @@ -69,8 +71,9 @@ void *H5VL_log_file_create (
req);
}
#endif

DEBUG_PRINT
fp = H5VL_log_filei_search (name);
DEBUG_PRINT
if (fp) {
fp = NULL;
RET_ERR (
Expand All @@ -80,6 +83,7 @@ void *H5VL_log_file_create (
H5VL_LOGI_PROFILING_TIMER_START;
// Try get info about under VOL
H5Pget_vol_info (fapl_id, (void **)&info);
DEBUG_PRINT

if (info) {
uvlid = info->uvlid;
Expand All @@ -92,6 +96,7 @@ void *H5VL_log_file_create (
CHECK_ID (uvlid)
under_vol_info = NULL;
}
DEBUG_PRINT

// Make sure we have mpi enabled
fdid = H5Pget_driver (fapl_id);
Expand All @@ -104,6 +109,7 @@ void *H5VL_log_file_create (
comm = MPI_COMM_SELF;
mpiinfo = MPI_INFO_NULL;
}
DEBUG_PRINT

// Init file obj
fp = new H5VL_log_file_t (uvlid);
Expand All @@ -117,6 +123,8 @@ void *H5VL_log_file_create (
fp->zbuf = NULL;
fp->is_log_based_file = true;
fp->is_new = true;
fp->subfile_records = NULL;
fp->nsubfiles = 0;
mpierr = MPI_Comm_dup (comm, &(fp->comm));
CHECK_MPIERR
if (mpiinfo != MPI_INFO_NULL) {
Expand All @@ -125,6 +133,7 @@ void *H5VL_log_file_create (
} else {
fp->info = MPI_INFO_NULL;
}
DEBUG_PRINT
mpierr = MPI_Comm_rank (comm, &(fp->rank));
CHECK_MPIERR
mpierr = MPI_Comm_size (comm, &(fp->np));
Expand All @@ -133,42 +142,57 @@ void *H5VL_log_file_create (
fp->name = std::string (name);
err = H5Pget_nb_buffer_size (fapl_id, &(fp->bsize));
CHECK_ERR
DEBUG_PRINT
H5VL_log_filei_parse_fapl (fp, fapl_id);

DEBUG_PRINT
H5VL_log_filei_parse_fcpl (fp, fcpl_id);

DEBUG_PRINT
H5VL_log_filei_init_idx (fp);
DEBUG_PRINT

H5VL_LOGI_PROFILING_TIMER_STOP (fp, TIMER_H5VL_LOG_FILE_CREATE_INIT);

// Create the file with underlying VOL
H5VL_LOGI_PROFILING_TIMER_START;
DEBUG_PRINT
fp->ufaplid = H5VL_log_filei_get_under_plist (fapl_id);
DEBUG_PRINT
err = H5Pset_vol (fp->ufaplid, uvlid, under_vol_info);
DEBUG_PRINT
CHECK_ERR
err = H5Pset_all_coll_metadata_ops (fp->ufaplid, (hbool_t) false);
DEBUG_PRINT
CHECK_ERR
err = H5Pset_coll_metadata_write (fp->ufaplid, (hbool_t) true);
DEBUG_PRINT
CHECK_ERR
// err = H5Pset_alignment (fp->ufaplid, 4096, 4096);
// CHECK_ERR
ufcplid = H5VL_log_filei_get_under_plist (fcpl_id);
CHECK_ID (ufcplid)
DEBUG_PRINT

if (fp->config & H5VL_FILEI_CONFIG_SUBFILING) {
ufaplid = H5Pcreate (H5P_FILE_ACCESS);
CHECK_ID (ufaplid)
err = H5Pset_vol (ufaplid, uvlid, under_vol_info);
CHECK_ERR
if (fp->rank) {
err = H5Pset_fapl_core (ufaplid, 16 * 1048576, false);
CHECK_ERR
}
// DEBUG_PRINT
// ufaplid = H5Pcreate (H5P_FILE_ACCESS);
// CHECK_ID (ufaplid)
// DEBUG_PRINT
// err = H5Pset_vol (ufaplid, uvlid, under_vol_info);
// CHECK_ERR
// DEBUG_PRINT
// if (fp->rank) {
// err = H5Pset_fapl_core (ufaplid, ((size_t)2) * ((size_t)1024), false);
// CHECK_ERR
// }
// DEBUG_PRINT
ufaplid = fp->ufaplid;
} else {
ufaplid = fp->ufaplid;
}
DEBUG_PRINT
H5VL_LOGI_PROFILING_TIMER_START;
fp->uo = H5VLfile_create (name, flags, ufcplid, ufaplid, dxpl_id, NULL);
DEBUG_PRINT
CHECK_PTR (fp->uo)
H5VL_LOGI_PROFILING_TIMER_STOP (fp, TIMER_H5VLFILE_CREATE);
H5VL_LOGI_PROFILING_TIMER_STOP (fp, TIMER_H5VL_LOG_FILE_CREATE_FILE);
Expand Down Expand Up @@ -266,6 +290,8 @@ void *H5VL_log_file_open (
fp->zbuf = NULL;
fp->is_log_based_file = true;
fp->is_new = false;
fp->subfile_records = NULL;
fp->nsubfiles = 0;
mpierr = MPI_Comm_dup (comm, &(fp->comm));
CHECK_MPIERR
if (mpiinfo != MPI_INFO_NULL) {
Expand Down Expand Up @@ -450,6 +476,7 @@ herr_t H5VL_log_file_specific (void *file,
H5VL_log_filei_flush(fp, dxpl_id);
} else {
err = H5VLfile_specific (fp->uo, fp->uvlid, args, dxpl_id, req);
CHECK_ERR
}
H5VL_LOGI_PROFILING_TIMER_STOP (fp, TIMER_H5VL_LOG_FILE_SPECIFIC);
} break;
Expand All @@ -458,6 +485,7 @@ herr_t H5VL_log_file_specific (void *file,
ERR_OUT ("Unsupported args->op_type")
} else {
err = H5VLfile_specific (fp->uo, fp->uvlid, args, dxpl_id, req);
CHECK_ERR
}
}

Expand Down
14 changes: 14 additions & 0 deletions src/H5VL_log_file.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,14 @@ typedef struct H5VL_log_cord_t {
MPI_Offset cord[H5S_MAX_RANK];
} H5VL_log_cord_t;

typedef struct H5VL_log_subfile_record_t {
MPI_File fh; // MPI file handle to the subfile
void* uo; // Under VOL object of the subfile
int nldset; // the number of log data datasets, of the subfile
int nmdset; // the number of log metadata datasets , of the subfile
void* lgp; // log group of the subfile
} H5VL_log_subfile_record_t;

using stcrtstat = struct stat;

/* The log VOL file object */
Expand All @@ -50,6 +58,10 @@ typedef struct H5VL_log_file_t : H5VL_log_obj_t {
int group_np; // Number of processes in the group
int ngroup; // Number of groups. NOTE: This value is only valid when
// H5VL_FILEI_CONFIG_SUBFILING is set in config
int nsubfiles; // Number of subfiles. NOTE: This value is only valid when
// H5VL_FILEI_CONFIG_SUBFILING is set in config. This is
// used in case of H5Fopen, where ngroup might be different
// from nsubfiles.
int prev_rank; // We only start writing after prev_rank finishes writing
int next_rank; // We have to notify next_rank to start writing after we finish
int target_ost; // What OST should we write to in aligned data layout
Expand Down Expand Up @@ -118,6 +130,8 @@ typedef struct H5VL_log_file_t : H5VL_log_obj_t {
bool is_log_based_file; // indicate if a file is a regular file (false) or a log-based file
// (false)

H5VL_log_subfile_record_t *subfile_records; // records of all subfiles, only used for reading

#ifdef ENABLE_PROFILING
#ifndef REPLAY_BUILD
//#pragma message ( "C Preprocessor got here!" )
Expand Down
Loading