feat(rla_scripts): add a migrate script

RLA/easy_log/log_tools.py

@@ -161,7 +161,7 @@ def delete_related_log(self, skip_ask=False, delete_log_types=None):
         else:
             s = input("delete these files? (y/n)")
         if s == 'y':
-            print("do delete ...")
+            print("deleting ...")
             return self._delete_related_log(show=False, regex=self.regex, delete_log_types=delete_log_types)
         else:
             return 0
@@ -189,10 +189,11 @@ def delete_small_timestep_log(self, skip_ask=False):
                 s = input("delete these files? (y/n)")
             if s == 'y' or skip_ask:
                 for res in self.small_timestep_regs:
-                    print("do delete: ", res[1])
+                    print("deleting: ", res[1])
                     log_found += self._delete_related_log(show=False, regex=res[0] + '*')
         return log_found
 
+
 class ArchiveLogTool(BasicLogTool):
     def __init__(self, proj_root, task_table_name, regex, archive_table_name=ARCHIVED_TABLE, *args, **kwargs):
         self.proj_root = proj_root
@@ -241,10 +242,64 @@ def archive_log(self, skip_ask=False):
         else:
             s = input("archive these files? (y/n) \n ")
         if s == 'y':
-            print("do archive ...")
+            print("archiving ...")
+            self._archive_log(show=False)
+
+
+
+class MoveLogTool(BasicLogTool):
+    def __init__(self, proj_root, task_table_name, regex, target_task_table_name, *args, **kwargs):
+        self.proj_root = proj_root
+        self.task_table_name = task_table_name
+        self.regex = regex
+        self.target_task_table_name = target_task_table_name
+        super(MoveLogTool, self).__init__(*args, **kwargs)
+
+    def _archive_log(self, show=False):
+        for log_type in self.log_types:
+            root_dir_regex = osp.join(self.proj_root, log_type, self.task_table_name, self.regex)
+            target_root_dir = osp.join(self.proj_root, log_type, self.target_task_table_name)
+            prefix_dir = osp.join(self.proj_root, log_type)
+            prefix_len = len(prefix_dir)
+            empty = True
+            # os.system("chmod +x -R \"{}\"".format(prefix_dir))
+            for root_dir in glob.glob(root_dir_regex):
+                empty = False
+                if os.path.exists(root_dir):
+                    # remove the overlapped path.
+                    archiving_target = osp.join(target_root_dir, root_dir[prefix_len+1:])
+                    archiving_target_dir = '/'.join(archiving_target.split('/')[:-1])
+                    os.makedirs(archiving_target_dir, exist_ok=True)
+                    if os.path.isdir(root_dir):
+                        if not show:
+                            # os.makedirs(archiving_target, exist_ok=True)
+                            try:
+                                shutil.copytree(root_dir, archiving_target)
+                            except FileExistsError as e:
+                                print(e)
+
+                        print("copy dir {}, to {}".format(root_dir, archiving_target))
+                    else:
+                        if not show:
+                            shutil.copy(root_dir, archiving_target)
+                        print("copy file {}, to {}".format(root_dir, archiving_target))
+                else:
+                    print("no dir {}".format(root_dir))
+            if empty: print("empty regex {}".format(root_dir_regex))
+        pass
+
+    def archive_log(self, skip_ask=False):
+        self._archive_log(show=True)
+        if skip_ask:
+            s = 'y'
+        else:
+            s = input("archive these files? (y/n) \n ")
+        if s == 'y':
+            print("moving ...")
             self._archive_log(show=False)
 
 
+
 class ViewLogTool(BasicLogTool):
     def __init__(self, proj_root, task_table_name, regex, *args, **kwargs):
         self.proj_root = proj_root

RLA/easy_plot/plot_util.py

@@ -222,10 +222,10 @@ def load_results(root_dir_or_dirs, names, x_bound, enable_progress=True, use_buf
                                 print("read buf: {}".format(buf_csv))
                                 raw_df = read_csv(buf_csv)
                             else:
-                                reader = pd.read_csv(progcsv, chunksize=100000,  quoting=csv.QUOTE_NONE,
-                                                     encoding='utf-8', index_col=False, comment='#')
-                                raw_df = pd.DataFrame()
-
+                                reader = pd.read_csv(progcsv, chunksize=5000,  quoting=csv.QUOTE_NONE,
+                                                     encoding='utf-8', index_col=False, comment='#', memory_map=True)
+                                # raw_df = pd.DataFrame()
+                                slim_chunk_list = []
                                 for chunk in reader:
                                     slim_chunk = chunk
                                     # if set(names).issubset(slim_chunk.columns):
@@ -244,7 +244,8 @@ def load_results(root_dir_or_dirs, names, x_bound, enable_progress=True, use_buf
                                                                                    slim_chunk[x_bound[0]] > x_bound[1][0])]
                                         else:
                                             slim_chunk = slim_chunk[slim_chunk[x_bound[0]] < x_bound[1]]
-                                    raw_df = pd.concat([raw_df, slim_chunk], ignore_index=True)
+                                    slim_chunk_list.append(slim_chunk)
+                                raw_df = pd.concat(slim_chunk_list, ignore_index=True)
                                 import csv
                                 raw_df.to_csv(buf_csv, index=False)
                             result['progress'] = raw_df

rla_scripts/archive_expt.py

@@ -9,7 +9,7 @@
 
 from RLA.easy_log.log_tools import ArchiveLogTool
 import argparse
-from config import *
+from rla_script_config import *
 
 def argsparser():
     parser = argparse.ArgumentParser("Archive Log")

rla_scripts/delete_expt.py

@@ -4,7 +4,7 @@
 """
 from RLA.easy_log.log_tools import DeleteLogTool, Filter
 import argparse
-from config import *
+from rla_script_config import *
 
 def argsparser():
     parser = argparse.ArgumentParser("Delete Log")

rla_scripts/migrate_expt.py

@@ -0,0 +1,26 @@
+# Created by xionghuichen at 2022/12/6
+# Email: chenxh@lamda.nju.edu.cn
+"""
+A script to move some important experiments to another task_table
+
+
+"""
+from rla_script_config import *
+from RLA.easy_log.log_tools import MoveLogTool
+import argparse
+
+def argsparser():
+    parser = argparse.ArgumentParser("Archive Log")
+    # reduce setting
+    parser.add_argument('--task_table', type=str)
+    parser.add_argument('--target_task_table', type=str)
+    parser.add_argument('--regex', type=str)
+
+    args = parser.parse_args()
+    return args
+
+if __name__=='__main__':
+    args = argsparser()
+    dlt = MoveLogTool(proj_root=DATA_ROOT, task_table_name=args.task_table, regex=args.regex,
+                      target_task_table_name=args.target_task_table)
+    dlt.archive_log()

rla_scripts/rla_script_config.py

@@ -0,0 +1 @@
+DATA_ROOT = '../../out_debug'

rla_scripts/start_pretty_plotter.py

@@ -7,7 +7,7 @@
 from RLA.easy_log.log_tools import PrettyPlotterTool, Filter
 import argparse
 from RLA.rla_argparser import boolean_flag
-from config import *
+from rla_script_config import *
 
 from smart_logger.front_page.page import start_page_server
 import smart_logger.common.plot_config as plot_config

rla_scripts/view_expt.py

@@ -4,7 +4,7 @@
 
 from RLA.easy_log.log_tools import ViewLogTool
 import argparse
-from config import *
+from rla_script_config import *
 
 def argsparser():
     parser = argparse.ArgumentParser("View Log")

setup.py

@@ -5,7 +5,7 @@
 
 setup(
         name='RLA',
-        version="0.6.0",
+        version="0.6.1",
         description=(
             'RL assistant'
         ),