refactor(kclvm-runner): encapsulate dylib generating, linking and exe…

…cuting in kclvm/lib.rs into kclvm-runner

Encapsulate dylibs generating and executing in kclvm/lib.rs into kclvm-runner.
Add struct "KclvmAssembler" in kclvm-runner/runner.rs to provide method "gen_dylibs" for dylibs generating.
Add struct "KclvmLinker" in kclvm-runner/runner.rs to provide method "link_all_dylibs" for dylib linking.
Add method "execute" in kclvm-runner/lib.rs to encapsulate dylibs generating(gen_dylib), dylib linking(link_all_dylib) and running(runner.run) together
The main purpose of separating the three parts from kclvm/lib.rs and encapsulating them into kclvm-runner is to support reuse in kcl-vet.

fix #67

kclvm/runner/Cargo.toml

@@ -15,6 +15,7 @@ libc = "0.2.112"
 indexmap = "1.0"
 fslock = "0.2.1"
 libloading = "0.7.3"
+threadpool = "1.0"
 
 kclvm-ast = {path = "../ast", version = "0.1.0"}
 kclvm-parser = {path = "../parser", version = "0.1.0"}

kclvm/runner/src/command.rs

@@ -423,7 +423,6 @@ impl Command {
         } else {
             "clang"
         };
-
 
         if let Some(s) = Self::find_it(clang_exe) {
             return s.to_str().unwrap().to_string();

kclvm/runner/src/eval.rs

@@ -0,0 +1,66 @@
+use indexmap::IndexMap;
+use std::{collections::HashMap, path::Path, path::PathBuf, sync::mpsc::channel};
+use threadpool::ThreadPool;
+
+use kclvm_ast::ast::{self, Program};
+use kclvm_compiler::codegen::{llvm::emit_code, EmitOptions};
+use kclvm_config::cache::{load_pkg_cache, save_pkg_cache, CacheOption};
+use kclvm_sema::resolver::resolve_program;
+use kclvm_sema::resolver::scope::ProgramScope;
+
+use crate::{
+    command::Command,
+    runner::{ExecProgramArgs, KclvmRunner, KclvmRunnerOptions},
+};
+
+const LL_FILE: &str = "_a.out";
+
+/// Evaluator is used to resolve kcl ast, generate dylibs and execute.
+pub struct Evaluator {}
+
+impl Evaluator {
+    pub fn new() -> Self {
+        Self {}
+    }
+
+    /// Take ast.program as input，and resolve ast, generate and link dylib, execute.
+    pub fn eval(
+        &self,
+        mut program: Program,
+        plugin_agent: u64,
+        args: &ExecProgramArgs,
+    ) -> Result<String, String> {
+        // resolve ast
+        let scope = resolve_program(&mut program);
+        scope.check_scope_diagnostics();
+
+        // generate dylibs
+        let dylib_paths = self.gen_dylibs(program, scope, plugin_agent);
+
+        // link dylibs
+        let dylib_path = self.link_all_dylibs(dylib_paths, plugin_agent);
+
+        // execute
+        self.run_dylib(dylib_path, plugin_agent, args)
+    }
+
+
+
+
+
+    /// Execute the dylibs linked by method "link_all_dylibs".
+    fn run_dylib(
+        &self,
+        dylib_path: String,
+        plugin_agent: u64,
+        args: &ExecProgramArgs,
+    ) -> Result<String, String> {
+        let runner = KclvmRunner::new(
+            dylib_path.as_str(),
+            Some(KclvmRunnerOptions {
+                plugin_agent_ptr: plugin_agent,
+            }),
+        );
+        runner.run(&args)
+    }
+}

kclvm/runner/src/lib.rs

@@ -1,2 +1,28 @@
+use kclvm_ast::ast::Program;
+use kclvm_sema::resolver::scope::ProgramScope;
+use runner::{ExecProgramArgs, KclvmAssembler, KclvmLinker, KclvmRunner, KclvmRunnerOptions};
+
 pub mod command;
 pub mod runner;
+
+pub fn execute(
+    program: Program,
+    scope: ProgramScope,
+    plugin_agent: u64,
+    args: &ExecProgramArgs,
+) -> Result<String, String> {
+    // generate dylibs
+    let dylib_paths = KclvmAssembler::gen_dylibs(program, scope, plugin_agent);
+
+    // link dylibs
+    let dylib_path = KclvmLinker::link_all_dylibs(dylib_paths, plugin_agent);
+
+    // run
+    let runner = KclvmRunner::new(
+        dylib_path.as_str(),
+        Some(KclvmRunnerOptions {
+            plugin_agent_ptr: plugin_agent,
+        }),
+    );
+    runner.run(&args)
+}

kclvm/runner/src/runner.rs

@@ -1,6 +1,19 @@
+use std::{
+    collections::HashMap,
+    path::{Path, PathBuf},
+    sync::mpsc::channel,
+};
+
+use indexmap::IndexMap;
+use kclvm_compiler::codegen::{llvm::emit_code, EmitOptions};
+use kclvm_config::cache::{load_pkg_cache, save_pkg_cache, CacheOption};
+use kclvm_sema::resolver::scope::ProgramScope;
 use serde::{Deserialize, Serialize};
 
 use kclvm_ast::ast;
+use threadpool::ThreadPool;
+
+use crate::command::Command;
 
 #[allow(non_camel_case_types)]
 pub type kclvm_char_t = i8;
@@ -79,6 +92,154 @@ pub struct KclvmRunnerOptions {
     pub plugin_agent_ptr: u64,
 }
 
+pub struct KclvmAssembler {}
+const LL_FILE: &str = "_a.out";
+impl KclvmAssembler {
+    /// Generate the dylibs and return file paths.
+    pub fn gen_dylibs(
+        program: ast::Program,
+        scope: ProgramScope,
+        plugin_agent: u64,
+    ) -> Vec<String> {
+        // gen bc or ll_file
+        let path = std::path::Path::new(LL_FILE);
+        if path.exists() {
+            std::fs::remove_file(path).unwrap();
+        }
+        for entry in glob::glob(&format!("{}*.ll", LL_FILE)).unwrap() {
+            match entry {
+                Ok(path) => {
+                    if path.exists() {
+                        std::fs::remove_file(path).unwrap();
+                    }
+                }
+                Err(e) => println!("{:?}", e),
+            };
+        }
+
+        let cache_dir = Path::new(&program.root)
+            .join(".kclvm")
+            .join("cache")
+            .join(kclvm_version::get_full_version());
+        if !cache_dir.exists() {
+            std::fs::create_dir_all(&cache_dir).unwrap();
+        }
+        let mut compile_progs: IndexMap<
+            String,
+            (
+                ast::Program,
+                IndexMap<String, IndexMap<String, String>>,
+                PathBuf,
+            ),
+        > = IndexMap::default();
+        for (pkgpath, modules) in program.pkgs {
+            let mut pkgs = HashMap::new();
+            pkgs.insert(pkgpath.clone(), modules);
+            let compile_prog = ast::Program {
+                root: program.root.clone(),
+                main: program.main.clone(),
+                pkgs,
+                cmd_args: vec![],
+                cmd_overrides: vec![],
+            };
+            compile_progs.insert(
+                pkgpath,
+                (compile_prog, scope.import_names.clone(), cache_dir.clone()),
+            );
+        }
+        let pool = ThreadPool::new(4);
+        let (tx, rx) = channel();
+        let prog_count = compile_progs.len();
+        for (pkgpath, (compile_prog, import_names, cache_dir)) in compile_progs {
+            let tx = tx.clone();
+            pool.execute(move || {
+                let root = &compile_prog.root;
+                let is_main_pkg = pkgpath == kclvm_ast::MAIN_PKG;
+                let file = if is_main_pkg {
+                    PathBuf::from(&pkgpath)
+                } else {
+                    cache_dir.join(&pkgpath)
+                };
+                let ll_file = file.to_str().unwrap();
+                let ll_path = format!("{}.ll", ll_file);
+                let dylib_path = format!("{}{}", ll_file, Command::get_lib_suffix());
+                let mut ll_path_lock =
+                    fslock::LockFile::open(&format!("{}.lock", ll_path)).unwrap();
+                ll_path_lock.lock().unwrap();
+                if Path::new(&ll_path).exists() {
+                    std::fs::remove_file(&ll_path).unwrap();
+                }
+                let dylib_path = if is_main_pkg {
+                    emit_code(
+                        &compile_prog,
+                        import_names,
+                        &EmitOptions {
+                            from_path: None,
+                            emit_path: Some(&ll_file),
+                            no_link: true,
+                        },
+                    )
+                    .expect("Compile KCL to LLVM error");
+                    let mut cmd = Command::new(plugin_agent);
+                    cmd.run_clang_single(&ll_path, &dylib_path)
+                } else {
+                    // If AST module has been modified, ignore the dylib cache
+                    let dylib_relative_path: Option<String> =
+                        load_pkg_cache(root, &pkgpath, CacheOption::default());
+                    match dylib_relative_path {
+                        Some(dylib_relative_path) => {
+                            if dylib_relative_path.starts_with('.') {
+                                dylib_relative_path.replacen(".", root, 1)
+                            } else {
+                                dylib_relative_path
+                            }
+                        }
+                        None => {
+                            emit_code(
+                                &compile_prog,
+                                import_names,
+                                &EmitOptions {
+                                    from_path: None,
+                                    emit_path: Some(&ll_file),
+                                    no_link: true,
+                                },
+                            )
+                            .expect("Compile KCL to LLVM error");
+                            let mut cmd = Command::new(plugin_agent);
+                            let dylib_path = cmd.run_clang_single(&ll_path, &dylib_path);
+                            let dylib_relative_path = dylib_path.replacen(root, ".", 1);
+
+                            save_pkg_cache(
+                                root,
+                                &pkgpath,
+                                dylib_relative_path,
+                                CacheOption::default(),
+                            );
+                            dylib_path
+                        }
+                    }
+                };
+                if Path::new(&ll_path).exists() {
+                    std::fs::remove_file(&ll_path).unwrap();
+                }
+                ll_path_lock.unlock().unwrap();
+                tx.send(dylib_path)
+                    .expect("channel will be there waiting for the pool");
+            });
+        }
+        rx.iter().take(prog_count).collect::<Vec<String>>()
+    }
+}
+
+pub struct KclvmLinker {}
+impl KclvmLinker {
+    /// Link the dylibs generated by method "gen_bc_or_ll_file".
+    pub fn link_all_dylibs(dylib_paths: Vec<String>, plugin_agent: u64) -> String {
+        let mut cmd = Command::new(plugin_agent);
+        cmd.link_dylibs(&dylib_paths, "")
+    }
+}
+
 pub struct KclvmRunner {
     opts: KclvmRunnerOptions,
     lib: libloading::Library,