-
Notifications
You must be signed in to change notification settings - Fork 67
use dfs command to create file parallel #1505
use dfs command to create file parallel #1505
Conversation
@@ -73,6 +73,27 @@ def test_create_10M_DFSIO(self): | |||
subprocess.call("hdfs dfs -mv /benchmarks/TestDFSIO/io_data " + | |||
TEST_DIR + str(i) + "_data") | |||
|
|||
|
|||
def test_create_10K_0KB_DFSIO_parallel(self): | |||
dir_name = TEST_DIR + random_string() |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Move this line below your comments.
|
||
file_index = 0; | ||
for i in range(10000 / process_group_size): | ||
process_group = [] |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This array will become larger and larger without deleting any items.
file_index = file_index + 1 | ||
|
||
# wait | ||
for k in range(process_group_size): |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
You can refine this method based on
SIMPLE WAY TO EXECUTE MULTIPLE PROCESS IN PARALLEL (PYTHON RECIPE)
@@ -73,6 +73,27 @@ def test_create_10M_DFSIO(self): | |||
subprocess.call("hdfs dfs -mv /benchmarks/TestDFSIO/io_data " + | |||
TEST_DIR + str(i) + "_data") | |||
|
|||
|
|||
def test_create_10K_0KB_DFSIO_parallel(self): |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I think each client should create several dirs, each dir has 10,000~100,000 files. So, maybe you need another loop.
@@ -16,6 +18,62 @@ def test_create_100M_0KB_thread(max_number): | |||
wait_for_cmdlets(cids) | |||
|
|||
|
|||
def cpu_count(): |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Move this method to util.py
No description provided.