added changes to support mmapped file.

argonne-lcf · Nov 28, 2023 · faee51e · faee51e
1 parent 52011b4
commit faee51e
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 2 deletions.
diff --git a/dlio_benchmark/configs/workload/megatron_deepspeed.yaml b/dlio_benchmark/configs/workload/megatron_deepspeed.yaml
@@ -12,7 +12,7 @@ dataset:
   format: mmap_indexed_binary
   num_files_train: 1
   num_samples_per_file: 277203535
-  record_length: 4096
+  record_length: 1024
 
 reader: 
   data_loader: pytorch

diff --git a/dlio_benchmark/data_generator/indexed_binary_generator.py b/dlio_benchmark/data_generator/indexed_binary_generator.py
@@ -55,7 +55,7 @@ def generate(self):
             sample_size = dim1 * dim2
             total_size = sample_size * self.num_samples
             write_size = total_size
-            MEMORY_SIZE = 2*GB
+            MEMORY_SIZE = 16*GB
             if total_size > MEMORY_SIZE:
                 write_size = MEMORY_SIZE - (MEMORY_SIZE % sample_size)
             out_path_spec = self.storage.get_uri(self._file_list[i])