From d7f68b807ca9f4c792ffb56ba1d51a0241c3e90a Mon Sep 17 00:00:00 2001 From: Qiyuan Gong Date: Wed, 13 Dec 2017 13:07:03 +0800 Subject: [PATCH] Set "/ssmtest/" as HDFS test directory, add scalability test script (#1477) * Replace "/test/" with "/ssmtest/" for HDFS test directory. * Add methods for creating 100M/500K 0KB files in `reset_env.py`. --- supports/integration-test/README.md | 10 ++--- supports/integration-test/reset_env.py | 39 +++++++++++++++++-- supports/integration-test/test_rule.py | 14 +++---- supports/integration-test/test_stress_rule.py | 8 ++-- supports/integration-test/util.py | 7 ++-- 5 files changed, 55 insertions(+), 23 deletions(-) diff --git a/supports/integration-test/README.md b/supports/integration-test/README.md index 8a63913887b..3cee8ab58da 100644 --- a/supports/integration-test/README.md +++ b/supports/integration-test/README.md @@ -18,14 +18,14 @@ BASE_URL = "http://{SSM_Server}:7045" `{SSM_Server}` is the IP address of SSM. ## Init/Rest Test Environment -1. Remove all files in hdfs:/test/ +1. Remove all files in hdfs:/ssmtest/ Run this command in HDFS enviroment. ``` -HDFS dfs -ls -rm -r /test/ -HDFS dfs -mkdir /test/ +HDFS dfs -ls -rm -r /ssmtest/ +HDFS dfs -mkdir /ssmtest/ ``` -2. Create 10000 files (1MB) in hdfs:/test/ +2. Create 10000 files (1MB) in hdfs:/ssmtest/ ``` python reset_env.py -v @@ -72,7 +72,7 @@ Run all stress/performance test cases with the following command: python test_stress.py -v ``` -If you want to increase the number of files in `hdfs:/test/`, please remove all delete file actions in `test_stress.py`. +If you want to increase the number of files in `hdfs:/ssmtest/`, please remove all delete file actions in `test_stress.py`. ``` for i in range(max_number): cids.append(delete_file(file_paths[i])) diff --git a/supports/integration-test/reset_env.py b/supports/integration-test/reset_env.py index 956cd2ebb0e..3e868f5bde9 100644 --- a/supports/integration-test/reset_env.py +++ b/supports/integration-test/reset_env.py @@ -29,6 +29,39 @@ def test_delete_all_files(self): except OSError: print "HDFS Envs is not configured!" + def test_create_100M_0KB(self): + """ + Create 100M=500K * 200 files in /ssmtest/. + Files will be kept in dir named from 1 to 200. + Files are named from 0-499999. + """ + max_number = 500000 + dir_number = 200 + for i in range(dir_number): + cids = [] + dir_name = TEST_DIR + str(dir_number) + # 200 dirs + for j in range(max_number): + # each has 500K files + cid = create_file(dir_name + "/" + str(j), 0) + cids.append(cid) + wait_for_cmdlets(cids) + + def test_create_500K_0KB(self): + """ + Create 500K files in /ssmtest/. + All files will be kept in one dir with random name. + Files are named from 0-499999. + """ + max_number = 500000 + cids = [] + dir_name = TEST_DIR + random_string() + for i in range(max_number): + # each has 500K files + cid = create_file(dir_name + "/" + str(i), 0) + cids.append(cid) + wait_for_cmdlets(cids) + def test_create_10000_1MB(self): """ Create 10000 * 1 MB files in /1MB/ @@ -73,7 +106,7 @@ def test_create_1000_100MB(self): def test_create_files_10000(self): """ - Create 10000 * 1 MB files in /test/ + Create 10000 * 1 MB files in /ssmtest/ """ max_number = 10000 file_paths = [] @@ -87,7 +120,7 @@ def test_create_files_10000(self): def test_create_files_1000(self): """ - Create 1000 * 1 MB files in /test/ + Create 1000 * 1 MB files in /ssmtest/ """ max_number = 10000 file_paths = [] @@ -101,7 +134,7 @@ def test_create_files_1000(self): def test_create_files_100(self): """ - Create 100 * 1 MB files in /test/ + Create 100 * 1 MB files in /ssmtest/ """ max_number = 10000 file_paths = [] diff --git a/supports/integration-test/test_rule.py b/supports/integration-test/test_rule.py index c410b312dc3..782dc9eb95e 100644 --- a/supports/integration-test/test_rule.py +++ b/supports/integration-test/test_rule.py @@ -6,11 +6,11 @@ class TestRule(unittest.TestCase): def test_rule_access_count(self): # rule: - # file : path matches "/test/*" and accessCount(1m) > 1 | allssd + # file : path matches "/ssmtest/*" and accessCount(1m) > 1 | allssd file_path = create_random_file(10 * 1024 * 1024) # submit rule rule_str = "file : path matches " + \ - "\"/test/*\" and accessCount(1m) > 1 | allssd " + "\"/ssmtest/*\" and accessCount(1m) > 1 | allssd " rid = submit_rule(rule_str) # Activate rule start_rule(rid) @@ -29,10 +29,10 @@ def test_rule_access_count(self): def test_rule_age(self): # rule: - # file : path matches "/test/*" and age > 4s | archive + # file : path matches "/ssmtest/*" and age > 4s | archive file_path = create_random_file(10 * 1024 * 1024) # submit rule - rule_str = "file : path matches \"/test/*\" and age > 4s | archive " + rule_str = "file : path matches \"/ssmtest/*\" and age > 4s | archive " rid = submit_rule(rule_str) # Activate rule start_rule(rid) @@ -46,18 +46,18 @@ def test_rule_age(self): def test_rule_scheduled(self): # rule: - # file: every 4s from now to now + 15s | path matches "/test/data*.dat" | onessd + # file: every 4s from now to now + 15s | path matches "/ssmtest/data*.dat" | onessd # From now to now + 15s # Create 3 random files for _ in range(3): - file_path = "/test/data" + \ + file_path = "/ssmtest/data" + \ random_string() + ".dat" wait_for_cmdlet(create_file(file_path, 10 * 1024 * 1024)) # submit rule rule_str = "file: " + \ "every 4s from now to now + 15s |" + \ " path matches " + \ - "\"/test/data*.dat\"" + \ + "\"/ssmtest/data*.dat\"" + \ " | onessd " rid = submit_rule(rule_str) # Activate rule diff --git a/supports/integration-test/test_stress_rule.py b/supports/integration-test/test_stress_rule.py index 4f1856c0f20..fa574d29d0a 100644 --- a/supports/integration-test/test_stress_rule.py +++ b/supports/integration-test/test_stress_rule.py @@ -12,7 +12,7 @@ def test_rule_200(self): rule_str = "file: " + \ "every 4s from now to now + 1d |" + \ " path matches " + \ - "\"/test/" + random_string()[:5] + " *\"" + \ + "\"/ssmtest/" + random_string()[:5] + " *\"" + \ " | onessd " rids.append(submit_rule(rule_str)) # activate all rules @@ -30,7 +30,7 @@ def test_rule_500(self): rule_str = "file: " + \ "every 4s from now to now + 1d |" + \ " path matches " + \ - "\"/test/" + random_string()[:5] + " *\"" + \ + "\"/ssmtest/" + random_string()[:5] + " *\"" + \ " | onessd " rids.append(submit_rule(rule_str)) # activate all rules @@ -48,7 +48,7 @@ def test_rule_1000(self): rule_str = "file: " + \ "every 4s from now to now + 1d |" + \ " path matches " + \ - "\"/test/" + random_string()[:5] + " *\"" + \ + "\"/ssmtest/" + random_string()[:5] + " *\"" + \ " | onessd " rids.append(submit_rule(rule_str)) # activate all rules @@ -66,7 +66,7 @@ def test_rule_10000(self): rule_str = "file: " + \ "every 4s from now to now + 1d |" + \ " path matches " + \ - "\"/test/" + random_string()[:5] + " *\"" + \ + "\"/ssmtest/" + random_string()[:5] + " *\"" + \ " | onessd " rids.append(submit_rule(rule_str)) # activate all rules diff --git a/supports/integration-test/util.py b/supports/integration-test/util.py index 4a1cfc11a07..2b82c136cfb 100644 --- a/supports/integration-test/util.py +++ b/supports/integration-test/util.py @@ -20,7 +20,7 @@ "allssd", "archive"] -TEST_DIR = "/test/" +TEST_DIR = "/ssmtest/" def random_file_path(): @@ -146,7 +146,7 @@ def create_file(file_path, length=1024): def create_random_file(length=1024): """ - create a random file in /test/ + create a random file in /ssmtest/ """ file_path = TEST_DIR + random_string() cmdlet_str = "write -file " + \ @@ -157,7 +157,7 @@ def create_random_file(length=1024): def create_random_file_parallel(length=1024): """ - create a random file in /test/ + create a random file in /ssmtest/ """ return create_random_file_parallel(TEST_DIR, length) @@ -193,7 +193,6 @@ def copy_file_to_S3(file_path, dest_path): return submit_cmdlet(cmdlet_str) - def delete_file(file_path, recursivly=True): cmdlet_str = "delete -file " + file_path return submit_cmdlet(cmdlet_str)