Merge branch 'main' into refactor/zong-zhe/add_eval_to_kclvm_runner

kclvm/runner/Cargo.toml

@@ -16,6 +16,7 @@ indexmap = "1.0"
 fslock = "0.2.1"
 libloading = "0.7.3"
 threadpool = "1.0"
+chrono = "0.4.19"
 
 kclvm-ast = {path = "../ast", version = "0.1.0"}
 kclvm-parser = {path = "../parser", version = "0.1.0"}
@@ -24,6 +25,7 @@ kclvm-config = {path = "../config", version = "0.1.0"}
 kclvm-runtime = {path = "../runtime", version = "0.1.0"}
 kclvm-sema = {path = "../sema", version = "0.1.0"}
 kclvm-version = {path = "../version", version = "0.1.0"}
+kclvm-error = {path = "../error", version="0.1.0"}
 
 [dev-dependencies]
 kclvm-parser = {path = "../parser", version = "0.1.0"}

kclvm/runner/src/assembler.rs

@@ -1,4 +1,5 @@
 use indexmap::IndexMap;
+use kclvm_error::bug;
 use std::{
     collections::HashMap,
     path::{Path, PathBuf},
@@ -7,29 +8,45 @@ use std::{
 use threadpool::ThreadPool;
 
 use crate::command::Command;
-use kclvm_ast::ast;
+use kclvm_ast::ast::{self, Program};
 use kclvm_compiler::codegen::{llvm::emit_code, EmitOptions};
 use kclvm_config::cache::{load_pkg_cache, save_pkg_cache, CacheOption};
 use kclvm_sema::resolver::scope::ProgramScope;
 
 /// LLVM IR file suffix.
 const LL_FILE: &str = "_a.out";
 
-/// KclvmAssembler is mainly responsible for generating bytecode,
-/// LLVM IR or dylib, and take the result of kclvm-parser and kclvm-sema
-/// as input.
-pub struct KclvmAssembler;
+/// KclvmAssembler is mainly responsible for assembling the generated bytecode,
+/// LLVM IR or other IR code into dynamic link libraries, and take the result of
+/// kclvm-parser, kclvm-sema and kclvm-compiler as input.
+pub struct KclvmAssembler {
+    thread_count: usize,
+}
 impl KclvmAssembler {
-    /// Generate the dylibs and return file paths from .
+    pub fn new() -> Self {
+        Self { thread_count: 4 }
+    }
+
+    pub fn new_with_thread_count(thread_count: usize) -> Self {
+        if thread_count <= 0 {
+            bug!("Illegal Thread Count");
+        }
+        Self { thread_count }
+    }
+
+    /// Generate the dylibs and return file paths.
+    ///
+    /// In the method, multiple threads will be created to concurrently generate dylibs
+    /// under different package paths.
     ///
-    /// In the method, 4 threads will be created to concurrently
-    /// generate dylibs under different package paths.
-    /// This method will generate “.out” and ".ll" files,
-    /// and return the file paths of the generated files in Vec<String>.
+    /// This method will generate “.out” and ".ll" files, and return the file paths of
+    /// the generated files in Vec<String>.
     pub fn gen_dylibs(
+        &self,
         program: ast::Program,
         scope: ProgramScope,
         plugin_agent: u64,
+        entry_file: &String,
     ) -> Vec<String> {
         // gen bc or ll_file
         let path = std::path::Path::new(LL_FILE);
@@ -77,43 +94,28 @@ impl KclvmAssembler {
                 (compile_prog, scope.import_names.clone(), cache_dir.clone()),
             );
         }
-        let pool = ThreadPool::new(4);
+        let pool = ThreadPool::new(self.thread_count);
         let (tx, rx) = channel();
         let prog_count = compile_progs.len();
+        // let temp_entry_file = temp_file();
         for (pkgpath, (compile_prog, import_names, cache_dir)) in compile_progs {
             let tx = tx.clone();
+            let temp_entry_file = entry_file.clone();
             pool.execute(move || {
                 let root = &compile_prog.root;
                 let is_main_pkg = pkgpath == kclvm_ast::MAIN_PKG;
-                let file = if is_main_pkg {
-                    PathBuf::from(&pkgpath)
-                } else {
-                    cache_dir.join(&pkgpath)
-                };
-                let ll_file = file.to_str().unwrap();
-                let ll_path = format!("{}.ll", ll_file);
-                let dylib_path = format!("{}{}", ll_file, Command::get_lib_suffix());
-                let mut ll_path_lock =
-                    fslock::LockFile::open(&format!("{}.lock", ll_path)).unwrap();
-                ll_path_lock.lock().unwrap();
-                if Path::new(&ll_path).exists() {
-                    std::fs::remove_file(&ll_path).unwrap();
-                }
+                // The main package does not perform cache reading and writing,
+                // and other packages perform read and write caching. Because
+                // KCL supports multi-file compilation, it is impossible to
+                // specify a standard entry for these multi-files and cannot
+                // be shared, so the cache of the main package is not read and
+                // written.
                 let dylib_path = if is_main_pkg {
-                    emit_code(
-                        &compile_prog,
-                        import_names,
-                        &EmitOptions {
-                            from_path: None,
-                            emit_path: Some(&ll_file),
-                            no_link: true,
-                        },
-                    )
-                    .expect("Compile KCL to LLVM error");
-                    let mut cmd = Command::new(plugin_agent);
-                    cmd.run_clang_single(&ll_path, &dylib_path)
+                    let file = PathBuf::from(&temp_entry_file);
+                    lock_ll_file_and_gen_dylib(&compile_prog, import_names, &file, &plugin_agent)
                 } else {
-                    // If AST module has been modified, ignore the dylib cache
+                    let file = cache_dir.join(&pkgpath);
+                    // Read the dylib cache
                     let dylib_relative_path: Option<String> =
                         load_pkg_cache(root, &pkgpath, CacheOption::default());
                     match dylib_relative_path {
@@ -125,20 +127,13 @@ impl KclvmAssembler {
                             }
                         }
                         None => {
-                            emit_code(
+                            let dylib_path = lock_ll_file_and_gen_dylib(
                                 &compile_prog,
                                 import_names,
-                                &EmitOptions {
-                                    from_path: None,
-                                    emit_path: Some(&ll_file),
-                                    no_link: true,
-                                },
-                            )
-                            .expect("Compile KCL to LLVM error");
-                            let mut cmd = Command::new(plugin_agent);
-                            let dylib_path = cmd.run_clang_single(&ll_path, &dylib_path);
+                                &file,
+                                &plugin_agent,
+                            );
                             let dylib_relative_path = dylib_path.replacen(root, ".", 1);
-
                             save_pkg_cache(
                                 root,
                                 &pkgpath,
@@ -149,14 +144,62 @@ impl KclvmAssembler {
                         }
                     }
                 };
-                if Path::new(&ll_path).exists() {
-                    std::fs::remove_file(&ll_path).unwrap();
-                }
-                ll_path_lock.unlock().unwrap();
                 tx.send(dylib_path)
                     .expect("channel will be there waiting for the pool");
             });
         }
         rx.iter().take(prog_count).collect::<Vec<String>>()
     }
 }
+
+fn lock_ll_file_and_gen_dylib(
+    compile_prog: &Program,
+    import_names: IndexMap<String, IndexMap<String, String>>,
+    file: &PathBuf,
+    plugin_agent: &u64,
+) -> String {
+    // ll_file: file_name
+    // ll_path: file_name.ll
+    // dylib_path: file_name.dll.lib or file_name.dylib or file_name.so
+    let ll_file = file.to_str().unwrap();
+    let ll_path = format!("{}.ll", ll_file);
+    let dylib_path = format!("{}{}", ll_file, Command::get_lib_suffix());
+
+    // Locking "*.ll" file for parallel code generation
+    let mut ll_path_lock = fslock::LockFile::open(&format!("{}.lock", ll_path)).unwrap();
+    ll_path_lock.lock().unwrap();
+
+    // Clean "*.ll" file path
+    clean_ll_path(&ll_path);
+
+    // gen code
+    emit_code(
+        compile_prog,
+        import_names,
+        &EmitOptions {
+            from_path: None,
+            emit_path: Some(ll_file),
+            no_link: true,
+        },
+    )
+    .expect("Compile KCL to LLVM error");
+
+    // assemble dylib
+    let mut cmd = Command::new(*plugin_agent);
+    let gen_dylib_path = cmd.run_clang_single(&ll_path, &dylib_path);
+
+    // Clean "*.ll" file path
+    clean_ll_path(&ll_path);
+
+    // unlock "*.ll" file
+    ll_path_lock.unlock().unwrap();
+
+    return gen_dylib_path;
+}
+
+// Clean "*.ll" file path
+fn clean_ll_path(ll_path: &String) {
+    if Path::new(ll_path).exists() {
+        std::fs::remove_file(&ll_path).unwrap();
+    }
+}

kclvm/runner/src/command.rs

@@ -202,11 +202,14 @@ impl Command {
     }
 
     pub fn link_dylibs(&mut self, dylibs: &[String], dylib_path: &str) -> String {
-        let mut dylib_path = dylib_path.to_string();
-
-        if dylib_path.is_empty() {
-            dylib_path = format!("{}{}", "_a.out", Self::get_lib_suffix());
-        }
+        let dylib_suffix = Self::get_lib_suffix();
+        let dylib_path = if dylib_path.is_empty() {
+            format!("{}{}", "_a.out", dylib_suffix)
+        } else if !dylib_path.ends_with(&dylib_suffix) {
+            format!("{}{}", dylib_path, dylib_suffix)
+        } else {
+            dylib_path.to_string()
+        };
 
         let mut args: Vec<String> = vec![
             "-Wno-override-module".to_string(),

kclvm/runner/src/lib.rs

@@ -1,3 +1,6 @@
+use std::path::Path;
+
+use command::Command;
 use kclvm_ast::ast::Program;
 use kclvm_sema::resolver::resolve_program;
 use runner::{ExecProgramArgs, KclvmRunner, KclvmRunnerOptions};
@@ -67,10 +70,15 @@ pub fn execute(
     scope.check_scope_diagnostics();
 
     // generate dylibs
-    let dylib_paths = assembler::KclvmAssembler::gen_dylibs(program, scope, plugin_agent);
+    let temp_entry_file = temp_file();
+    let dylib_paths =
+        assembler::KclvmAssembler::new().gen_dylibs(program, scope, plugin_agent, &temp_entry_file);
 
     // link dylibsKclvmRunner
-    let dylib_path = linker::KclvmLinker::link_all_dylibs(dylib_paths, plugin_agent);
+    let dylib_suffix = Command::get_lib_suffix();
+    let temp_out_dylib_file = format!("{}.out{}", temp_entry_file, dylib_suffix);
+    let dylib_path =
+        linker::KclvmLinker::link_all_dylibs(dylib_paths, temp_out_dylib_file, plugin_agent);
 
     // run
     let runner = KclvmRunner::new(
@@ -79,5 +87,35 @@ pub fn execute(
             plugin_agent_ptr: plugin_agent,
         }),
     );
-    runner.run(&args)
+    let result = runner.run(&args);
+
+    // clean files
+    remove_file(&dylib_path);
+    clean_tmp_files(&temp_entry_file, &dylib_suffix);
+    result
+}
+
+#[inline]
+/// Clean all the tmp files generated during dylib generating and linking.
+fn clean_tmp_files(temp_entry_file: &String, dylib_suffix: &String) {
+    let ll_lock_suffix = ".ll.lock";
+    let temp_entry_dylib_file = format!("{}{}", temp_entry_file, dylib_suffix);
+    let temp_entry_ll_lock_file = format!("{}{}", temp_entry_file, ll_lock_suffix);
+    remove_file(&temp_entry_dylib_file);
+    remove_file(&temp_entry_ll_lock_file);
+}
+
+#[inline]
+fn remove_file(file: &str) {
+    if Path::new(&file).exists() {
+        std::fs::remove_file(&file).unwrap();
+    }
+}
+
+#[inline]
+/// Returns a temporary file name consisting of timestamp and process id.
+fn temp_file() -> String {
+    let timestamp = chrono::Local::now().timestamp_nanos();
+    let id = std::process::id();
+    format!("{}_{}", id, timestamp)
 }

kclvm/runner/src/linker.rs

@@ -4,8 +4,12 @@ use crate::command::Command;
 pub struct KclvmLinker;
 impl KclvmLinker {
     /// Link the dylibs generated by method "gen_bc_or_ll_file".
-    pub fn link_all_dylibs(dylib_paths: Vec<String>, plugin_agent: u64) -> String {
+    pub fn link_all_dylibs(
+        dylib_paths: Vec<String>,
+        dylib_path: String,
+        plugin_agent: u64,
+    ) -> String {
         let mut cmd = Command::new(plugin_agent);
-        cmd.link_dylibs(&dylib_paths, "")
+        cmd.link_dylibs(&dylib_paths, &dylib_path)
     }
 }