Set "/ssmtest/" as HDFS test directory, add scalability test script …

…(#1477)

* Replace "/test/" with "/ssmtest/" for HDFS test directory.
* Add methods for creating 100M/500K 0KB files in `reset_env.py`.

supports/integration-test/README.md

@@ -18,14 +18,14 @@ BASE_URL = "http://{SSM_Server}:7045"
 `{SSM_Server}` is the IP address of SSM.
 
 ## Init/Rest Test Environment
-1. Remove all files in hdfs:/test/
+1. Remove all files in hdfs:/ssmtest/
 Run this command in HDFS enviroment.
 ```
-HDFS dfs -ls -rm -r /test/
-HDFS dfs -mkdir /test/
+HDFS dfs -ls -rm -r /ssmtest/
+HDFS dfs -mkdir /ssmtest/
 ```
 
-2. Create 10000 files (1MB) in hdfs:/test/
+2. Create 10000 files (1MB) in hdfs:/ssmtest/
 
 ```
 python reset_env.py -v
@@ -72,7 +72,7 @@ Run all stress/performance test cases with the following command:
 python test_stress.py -v
 ```
 
-If you want to increase the number of files in `hdfs:/test/`, please remove all delete file actions in `test_stress.py`.
+If you want to increase the number of files in `hdfs:/ssmtest/`, please remove all delete file actions in `test_stress.py`.
 ```
 for i in range(max_number):
     cids.append(delete_file(file_paths[i]))

supports/integration-test/reset_env.py

@@ -29,6 +29,39 @@ def test_delete_all_files(self):
         except OSError:
             print "HDFS Envs is not configured!"
 
+    def test_create_100M_0KB(self):
+        """
+        Create 100M=500K * 200 files in /ssmtest/.
+        Files will be kept in dir named from 1 to 200.
+        Files are named from 0-499999.
+        """
+        max_number = 500000
+        dir_number = 200
+        for i in range(dir_number):
+            cids = []
+            dir_name = TEST_DIR + str(dir_number)
+            # 200 dirs
+            for j in range(max_number):
+                # each has 500K files
+                cid = create_file(dir_name + "/" + str(j), 0)
+            cids.append(cid)
+            wait_for_cmdlets(cids)
+
+    def test_create_500K_0KB(self):
+        """
+        Create 500K files in /ssmtest/.
+        All files will be kept in one dir with random name.
+        Files are named from 0-499999.
+        """
+        max_number = 500000
+        cids = []
+        dir_name = TEST_DIR + random_string()
+        for i in range(max_number):
+            # each has 500K files
+            cid = create_file(dir_name + "/" + str(i), 0)
+        cids.append(cid)
+        wait_for_cmdlets(cids)
+
     def test_create_10000_1MB(self):
         """
         Create 10000 * 1 MB files in /1MB/
@@ -73,7 +106,7 @@ def test_create_1000_100MB(self):
 
     def test_create_files_10000(self):
         """
-        Create 10000 * 1 MB files in /test/
+        Create 10000 * 1 MB files in /ssmtest/
         """
         max_number = 10000
         file_paths = []
@@ -87,7 +120,7 @@ def test_create_files_10000(self):
 
     def test_create_files_1000(self):
         """
-        Create 1000 * 1 MB files  in /test/
+        Create 1000 * 1 MB files  in /ssmtest/
         """
         max_number = 10000
         file_paths = []
@@ -101,7 +134,7 @@ def test_create_files_1000(self):
 
     def test_create_files_100(self):
         """
-        Create 100 * 1 MB files  in /test/
+        Create 100 * 1 MB files  in /ssmtest/
         """
         max_number = 10000
         file_paths = []

supports/integration-test/test_rule.py

@@ -6,11 +6,11 @@
 class TestRule(unittest.TestCase):
     def test_rule_access_count(self):
         # rule:
-        # file : path matches "/test/*" and accessCount(1m) > 1 | allssd
+        # file : path matches "/ssmtest/*" and accessCount(1m) > 1 | allssd
         file_path = create_random_file(10 * 1024 * 1024)
         # submit rule
         rule_str = "file : path matches " + \
-            "\"/test/*\" and accessCount(1m) > 1 | allssd "
+            "\"/ssmtest/*\" and accessCount(1m) > 1 | allssd "
         rid = submit_rule(rule_str)
         # Activate rule
         start_rule(rid)
@@ -29,10 +29,10 @@ def test_rule_access_count(self):
 
     def test_rule_age(self):
         # rule:
-        # file : path matches "/test/*" and age > 4s | archive
+        # file : path matches "/ssmtest/*" and age > 4s | archive
         file_path = create_random_file(10 * 1024 * 1024)
         # submit rule
-        rule_str = "file : path matches \"/test/*\" and age > 4s | archive "
+        rule_str = "file : path matches \"/ssmtest/*\" and age > 4s | archive "
         rid = submit_rule(rule_str)
         # Activate rule
         start_rule(rid)
@@ -46,18 +46,18 @@ def test_rule_age(self):
 
     def test_rule_scheduled(self):
         # rule:
-        # file: every 4s from now to now + 15s | path matches "/test/data*.dat" | onessd
+        # file: every 4s from now to now + 15s | path matches "/ssmtest/data*.dat" | onessd
         # From now to now + 15s
         # Create 3 random files
         for _ in range(3):
-            file_path = "/test/data" + \
+            file_path = "/ssmtest/data" + \
                 random_string() + ".dat"
             wait_for_cmdlet(create_file(file_path, 10 * 1024 * 1024))
         # submit rule
         rule_str = "file: " + \
             "every 4s from now to now + 15s |" + \
             " path matches " + \
-            "\"/test/data*.dat\"" + \
+            "\"/ssmtest/data*.dat\"" + \
             " | onessd "
         rid = submit_rule(rule_str)
         # Activate rule

supports/integration-test/test_stress_rule.py

@@ -12,7 +12,7 @@ def test_rule_200(self):
             rule_str = "file: " + \
                 "every 4s from now to now + 1d |" + \
                 " path matches " + \
-                "\"/test/" + random_string()[:5] + " *\"" + \
+                "\"/ssmtest/" + random_string()[:5] + " *\"" + \
                 " | onessd "
             rids.append(submit_rule(rule_str))
         # activate all rules
@@ -30,7 +30,7 @@ def test_rule_500(self):
             rule_str = "file: " + \
                 "every 4s from now to now + 1d |" + \
                 " path matches " + \
-                "\"/test/" + random_string()[:5] + " *\"" + \
+                "\"/ssmtest/" + random_string()[:5] + " *\"" + \
                 " | onessd "
             rids.append(submit_rule(rule_str))
         # activate all rules
@@ -48,7 +48,7 @@ def test_rule_1000(self):
             rule_str = "file: " + \
                 "every 4s from now to now + 1d |" + \
                 " path matches " + \
-                "\"/test/" + random_string()[:5] + " *\"" + \
+                "\"/ssmtest/" + random_string()[:5] + " *\"" + \
                 " | onessd "
             rids.append(submit_rule(rule_str))
         # activate all rules
@@ -66,7 +66,7 @@ def test_rule_10000(self):
             rule_str = "file: " + \
                 "every 4s from now to now + 1d |" + \
                 " path matches " + \
-                "\"/test/" + random_string()[:5] + " *\"" + \
+                "\"/ssmtest/" + random_string()[:5] + " *\"" + \
                 " | onessd "
             rids.append(submit_rule(rule_str))
         # activate all rules

supports/integration-test/util.py

@@ -20,7 +20,7 @@
              "allssd",
              "archive"]
 
-TEST_DIR = "/test/"
+TEST_DIR = "/ssmtest/"
 
 
 def random_file_path():
@@ -146,7 +146,7 @@ def create_file(file_path, length=1024):
 
 def create_random_file(length=1024):
     """
-    create a random file in /test/
+    create a random file in /ssmtest/
     """
     file_path = TEST_DIR + random_string()
     cmdlet_str = "write -file " + \
@@ -157,7 +157,7 @@ def create_random_file(length=1024):
 
 def create_random_file_parallel(length=1024):
     """
-    create a random file in /test/
+    create a random file in /ssmtest/
     """
     return create_random_file_parallel(TEST_DIR, length)
 
@@ -193,7 +193,6 @@ def copy_file_to_S3(file_path, dest_path):
     return submit_cmdlet(cmdlet_str)
 
 
-
 def delete_file(file_path, recursivly=True):
     cmdlet_str = "delete -file " + file_path
     return submit_cmdlet(cmdlet_str)