From d82f3848fd8ae894614a9503fa82d65e36a33409 Mon Sep 17 00:00:00 2001 From: Matthew Larson Date: Wed, 7 Jun 2023 11:31:33 -0500 Subject: [PATCH] Add benchmarks, allow sec2 VFD write with ros3 --- benchmarks/C/Makefile | 2 +- benchmarks/C/icesat2_selection.c | 31 +++++++++++++++++-------------- benchmarks/select_time.csv | 4 ++++ 3 files changed, 22 insertions(+), 15 deletions(-) diff --git a/benchmarks/C/Makefile b/benchmarks/C/Makefile index 5afa6a9..6691878 100644 --- a/benchmarks/C/Makefile +++ b/benchmarks/C/Makefile @@ -1,6 +1,6 @@ CC=gcc CFLAGS=-I$(HDF5_PATH)/include -I$(REST_VOL_PATH)/src -g -O0 -LIBS=-L$(HDF5_PATH)/lib/ -lhdf5 -L$(REST_VOL_PATH)/build/bin -lhdf5_vol_rest -lyaml +LIBS=-L$(HDF5_PATH)/lib/ -lm -lhdf5_debug -L$(REST_VOL_PATH)/build/bin -lhdf5_vol_rest -lyaml benchmark: icesat2_selection.c $(CC) -o icesat2_selection $(CFLAGS) icesat2_selection.c $(LIBS) diff --git a/benchmarks/C/icesat2_selection.c b/benchmarks/C/icesat2_selection.c index 669f97e..1f18abf 100644 --- a/benchmarks/C/icesat2_selection.c +++ b/benchmarks/C/icesat2_selection.c @@ -911,6 +911,7 @@ int main(int argc, char **argv) { hid_t fapl_id = H5I_INVALID_HID; + hid_t fapl_id2 = H5I_INVALID_HID; hid_t fcpl_id = H5I_INVALID_HID; hid_t fin = H5I_INVALID_HID; hid_t fout = H5I_INVALID_HID; @@ -936,16 +937,15 @@ int main(int argc, char **argv) } } - if (use_ros3 && !readonly) { - FUNC_GOTO_ERROR("ROS3 VFD requires -readonly") - } - fcpl_id = H5Pcreate(H5P_FILE_CREATE); if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) == H5I_INVALID_HID) { FUNC_GOTO_ERROR("Failed to create FAPL") } + if ((fapl_id2 = H5Pcreate(H5P_FILE_ACCESS)) == H5I_INVALID_HID) { + FUNC_GOTO_ERROR("Failed to create FAPL2") + } #ifdef USE_REST_VOL H5rest_init(); @@ -988,29 +988,31 @@ int main(int argc, char **argv) FUNC_GOTO_ERROR("Failed to set page buffer size") } + if (!readonly) { + if (H5Pset_page_buffer_size(fapl_id2, page_buf_size, 0, 0) < 0) { + FUNC_GOTO_ERROR("Failed to set page buffer size") + } + + if (H5Pset_file_space_strategy(fcpl_id, H5F_FSPACE_STRATEGY_PAGE, 0, 0) < 0) { + FUNC_GOTO_ERROR("Failed to set page strategy for output file") + } + } } input_path = malloc(strlen(config->input_filename) + strlen(config->input_foldername) + 1); strcpy(input_path, config->input_foldername); strcat(input_path, config->input_filename); - if (readonly) { - if ((fin = H5Fopen(input_path, H5F_ACC_RDONLY, fapl_id)) == H5I_INVALID_HID) { - FUNC_GOTO_ERROR("Failed to open input file") - } - } else { - if ((fin = H5Fopen(input_path, H5F_ACC_RDWR, fapl_id)) == H5I_INVALID_HID) { - FUNC_GOTO_ERROR("Failed to open input file") - } + if ((fin = H5Fopen(input_path, H5F_ACC_RDONLY, fapl_id)) == H5I_INVALID_HID) { + FUNC_GOTO_ERROR("Failed to open input file") } - output_path = malloc(strlen(config->output_filename) + strlen(config->output_foldername) + 1); strcpy(output_path, config->output_foldername); strcat(output_path, config->output_filename); if (!readonly) { - if ((fout = H5Fcreate(output_path, H5F_ACC_TRUNC, fcpl_id, H5P_DEFAULT)) == H5I_INVALID_HID) { + if ((fout = H5Fcreate(output_path, H5F_ACC_TRUNC, fcpl_id, fapl_id2)) == H5I_INVALID_HID) { FUNC_GOTO_ERROR("Failed to create output file") } @@ -1186,6 +1188,7 @@ int main(int argc, char **argv) #endif H5Pclose(fapl_id); + H5Pclose(fapl_id2); H5Pclose(fcpl_id); H5Fclose(fin); diff --git a/benchmarks/select_time.csv b/benchmarks/select_time.csv index 4db5106..ac58d88 100644 --- a/benchmarks/select_time.csv +++ b/benchmarks/select_time.csv @@ -36,6 +36,10 @@ run_number, start, finish, benchmark, elapsed (sec), machine, input_folder, 2, , , C (ros3 read-only), 50.4, m5.4xlarge, http://s3.us-west-2.amazonaws.com/hdf5.sample/data/NASA/ICESat2/, N/A, ATL03_20181017222812_02950102_005_01.h5, , , , , , , 1, , , C (ros3 read-only), 88.7, m5.4xlarge, http://s3.us-west-2.amazonaws.com/hdf5.sample/data/NASA/ICESat2/, N/A, PAGE10MiB_ATL03_20181017222812_02950102_005_01.h5, , , , , , , 2, , , C (ros3 read-only), 75.6, m5.4xlarge, http://s3.us-west-2.amazonaws.com/hdf5.sample/data/NASA/ICESat2/, N/A, PAGE10MiB_ATL03_20181017222812_02950102_005_01.h5, , , , , , , +1, , , C (ros3), 56.2, m5.4xlarge, http://s3.us-west-2.amazonaws.com/hdf5.sample/data/NASA/ICESat2/, ./, ATL03_20181017222812_02950102_005_01.h5, , , , , , , writing done with standard VFD +2, , , C (ros3), 51.6, m5.4xlarge, http://s3.us-west-2.amazonaws.com/hdf5.sample/data/NASA/ICESat2/, ./, ATL03_20181017222812_02950102_005_01.h5, , , , , , , writing done with standard VFD +1, , , C (ros3), 98.6, m5.4xlarge, http://s3.us-west-2.amazonaws.com/hdf5.sample/data/NASA/ICESat2/, ./, PAGE10MiB_ATL03_20181017222812_02950102_005_01.h5, , , , , , , writing done with standard VFD, writing also paged +2, , , C (ros3), 78.8, m5.4xlarge, http://s3.us-west-2.amazonaws.com/hdf5.sample/data/NASA/ICESat2/, ./, PAGE10MiB_ATL03_20181017222812_02950102_005_01.h5, , , , , , , writing done with standard VFD, writing also paged 1, , , C (REST VOL read-only), 24.7, m5.4xlarge, /home/test_user1/icesat2/, N/A, ATL03_20181017222812_02950102_005_01.h5, , , , , , , 2, , , C (REST VOL read-only), 1.9, m5.4xlarge, /home/test_user1/icesat2/, N/A, ATL03_20181017222812_02950102_005_01.h5, , , , , , , 1, , , C (REST VOL read-only), 54.9, m5.4xlarge, /home/test_user1/icesat2/linked/, N/A, ATL03_20181017222812_02950102_005_01.h5, , , , , , ,