diff --git a/kclvm/Cargo.lock b/kclvm/Cargo.lock index 401fbf1ee..e1037be0d 100644 --- a/kclvm/Cargo.lock +++ b/kclvm/Cargo.lock @@ -568,6 +568,7 @@ dependencies = [ "kclvm-parser", "kclvm-runtime", "kclvm-sema", + "kclvm-tools", "kclvm-version", "libc", "libloading", diff --git a/kclvm/runner/Cargo.lock b/kclvm/runner/Cargo.lock index 86e49eac8..1479f00e1 100644 --- a/kclvm/runner/Cargo.lock +++ b/kclvm/runner/Cargo.lock @@ -647,6 +647,7 @@ dependencies = [ "kclvm-parser", "kclvm-runtime", "kclvm-sema", + "kclvm-tools", "kclvm-version", "libc", "libloading", @@ -712,6 +713,17 @@ dependencies = [ "scoped-tls", ] +[[package]] +name = "kclvm-tools" +version = "0.1.0" +dependencies = [ + "fancy-regex", + "indexmap", + "kclvm-ast", + "kclvm-error", + "kclvm-parser", +] + [[package]] name = "kclvm-version" version = "0.1.0" diff --git a/kclvm/runner/Cargo.toml b/kclvm/runner/Cargo.toml index e8d7eba6f..8ea375737 100644 --- a/kclvm/runner/Cargo.toml +++ b/kclvm/runner/Cargo.toml @@ -26,11 +26,16 @@ kclvm-runtime = {path = "../runtime", version = "0.1.0"} kclvm-sema = {path = "../sema", version = "0.1.0"} kclvm-version = {path = "../version", version = "0.1.0"} kclvm-error = {path = "../error", version="0.1.0"} +kclvm-tools = {path = "../tools", version = "0.1.0"} [dev-dependencies] kclvm-parser = {path = "../parser", version = "0.1.0"} criterion = "0.3" +# [[bench]] +# name = "bench_runner" +# harness = false + [[bench]] name = "bench_runner" harness = false \ No newline at end of file diff --git a/kclvm/runner/benches/bench_runner.rs b/kclvm/runner/benches/bench_runner.rs index df44433d7..f560bd64f 100644 --- a/kclvm/runner/benches/bench_runner.rs +++ b/kclvm/runner/benches/bench_runner.rs @@ -1,6 +1,7 @@ use criterion::{criterion_group, criterion_main, Criterion}; use kclvm_parser::load_program; use kclvm_runner::{execute, runner::ExecProgramArgs}; +use kclvm_tools::query::apply_overrides; const TEST_CASE_PATH: &str = "/src/test_datas/init_check_order_0/main.k"; @@ -10,17 +11,30 @@ pub fn criterion_benchmark(c: &mut Criterion) { std::env::current_dir().unwrap().to_str().unwrap(), TEST_CASE_PATH ); - let plugin_agent = 0; - c.bench_function("load_program -> execute", |b| { + c.bench_function("refactor kclvm-runner", |b| { b.iter(|| { - let args = ExecProgramArgs::default(); - let opts = args.get_load_program_options(); - let program = load_program(&[kcl_path], Some(opts)).unwrap(); - execute(program, plugin_agent, &args).unwrap() + after_refactor(kcl_path.to_string()); }) }); } criterion_group!(benches, criterion_benchmark); criterion_main!(benches); + +fn after_refactor(k_path: String) { + let mut args = ExecProgramArgs::default(); + args.k_filename_list.push(k_path); + + let plugin_agent = 0; + + let files = args.get_files(); + let opts = args.get_load_program_options(); + + // load ast + let mut program = load_program(&files, Some(opts)).unwrap(); + apply_overrides(&mut program, &args.overrides, &[]); + + // resolve ast, generate libs, link libs and execute. + execute(program, plugin_agent, &args); +} diff --git a/kclvm/runner/src/assembler.rs b/kclvm/runner/src/assembler.rs index 9bb5ebef8..4a8793236 100644 --- a/kclvm/runner/src/assembler.rs +++ b/kclvm/runner/src/assembler.rs @@ -1,5 +1,10 @@ +use crate::command::Command; use indexmap::IndexMap; +use kclvm_ast::ast::{self, Program}; +use kclvm_compiler::codegen::{llvm::emit_code, EmitOptions}; +use kclvm_config::cache::{load_pkg_cache, save_pkg_cache, CacheOption}; use kclvm_error::bug; +use kclvm_sema::resolver::scope::ProgramScope; use std::{ collections::HashMap, path::{Path, PathBuf}, @@ -7,53 +12,388 @@ use std::{ }; use threadpool::ThreadPool; -use crate::command::Command; -use kclvm_ast::ast::{self, Program}; -use kclvm_compiler::codegen::{llvm::emit_code, EmitOptions}; -use kclvm_config::cache::{load_pkg_cache, save_pkg_cache, CacheOption}; -use kclvm_sema::resolver::scope::ProgramScope; +/// IR code file suffix. +const IR_FILE: &str = "_a.out"; + +/// LibAssembler trait is used to indicate the general interface +/// that must be implemented when different intermediate codes are assembled +/// into dynamic link libraries. +/// +/// Note: LibAssembler is only for single file kcl program. For multi-file kcl programs, +/// KclvmAssembler is provided to support for multi-file parallel compilation to improve +/// the performance of the compiler. +pub trait LibAssembler { + /// Add a suffix to the file name according to the file suffix of different intermediate codes. + /// e.g. LLVM IR + /// code_file : "/test_dir/test_code_file" + /// return : "/test_dir/test_code_file.ll" + fn add_code_file_suffix(&self, code_file: &str) -> String; + + /// Return the file suffix of different intermediate codes. + /// e.g. LLVM IR + /// return : ".ll" + fn get_code_file_suffix(&self) -> &str; + + /// Assemble different intermediate codes into dynamic link libraries for single file kcl program. + /// + /// Inputs: + /// compile_prog: Reference of kcl program ast. + /// + /// "import_names" is import pkgpath and name of kcl program. + /// Type of import_names is "IndexMap>". + /// "kcl_file_name" is the kcl file name string. + /// "import_name" is the name string of import stmt. + /// e.g. "import test/main_pkg as main", "main" is an import_name. + /// "import_path" is the path string of import stmt. + /// e.g. "import test/main_pkg as main", "test/main_pkg" is an import_path. + /// import_names is from "ProgramScope.import_names" returned by "resolve_program" after resolving kcl ast by kclvm-sema. + /// + /// "code_file" is the filename of the generated intermediate code file. + /// e.g. code_file : "/test_dir/test_code_file" + /// + /// "code_file_path" is the full filename of the generated intermediate code file with suffix. + /// e.g. code_file_path : "/test_dir/test_code_file.ll" + /// + /// "lib_path" is the file path of the dynamic link library. + /// e.g. lib_path : "/test_dir/test_code_file.ll.dylib" (mac) + /// e.g. lib_path : "/test_dir/test_code_file.ll.dll.lib" (windows) + /// e.g. lib_path : "/test_dir/test_code_file.ll.so" (ubuntu) + /// + /// "plugin_agent" is a pointer to the plugin address. + /// + /// Returns the path of the dynamic link library. + fn assemble_lib( + &self, + compile_prog: &Program, + import_names: IndexMap>, + code_file: &str, + code_file_path: &str, + lib_path: &str, + plugin_agent: &u64, + ) -> String; + + /// This method is prepared for concurrent compilation in KclvmAssembler. + /// It is an atomic method executed by each thread in concurrent compilation. + /// + /// This method will take the above method “assemble_lib” as a hook method to + /// generate the dynamic link library, and lock the file before calling “assemble_lib”, + /// unlocked after the call ends, + #[inline] + fn lock_file_and_gen_lib( + &self, + compile_prog: &Program, + import_names: IndexMap>, + file: &PathBuf, + plugin_agent: &u64, + ) -> String { + // e.g. LLVM IR + // code_file: file_name + // code_file_path: file_name.ll + // lock_file_path: file_name.dll.lib or file_name.lib or file_name.so + let code_file = file.to_str().unwrap(); + let code_file_path = &self.add_code_file_suffix(code_file); + let lock_file_path = &format!("{}.lock", code_file_path); + let lib_path = format!("{}{}", code_file, Command::get_lib_suffix()); + + // locking file for parallel code generation. + let mut ll_path_lock = fslock::LockFile::open(lock_file_path).unwrap(); + ll_path_lock.lock().unwrap(); + + // Calling the hook method will generate the corresponding intermediate code + // according to the implementation of method "assemble_lib". + let gen_lib_path = self.assemble_lib( + compile_prog, + import_names, + &code_file, + code_file_path, + &lib_path, + plugin_agent, + ); + + // unlock file + ll_path_lock.unlock().unwrap(); + + return gen_lib_path; + } + + // Clean file path + // Delete the file in "path". + #[inline] + fn clean_path(&self, path: &String) { + if Path::new(path).exists() { + std::fs::remove_file(&path).unwrap(); + } + } + + // Clean lock file + // Clear the lock files generated during concurrent compilation. + #[inline] + fn clean_lock_file(&self, path: &String) { + let lock_path = &format!("{}.lock", self.add_code_file_suffix(path)); + self.clean_path(lock_path); + } +} + +/// This enum lists all the intermediate code assemblers currently supported by kclvm. +/// Currently only supports assemble llvm intermediate code into dynamic link library. +#[derive(Clone)] +pub enum KclvmLibAssembler { + LLVM(LlvmLibAssembler), +} + +/// KclvmLibAssembler is a dispatcher, responsible for calling corresponding methods +/// according to different types of intermediate codes. +/// +/// KclvmLibAssembler implements the LibAssembler trait, +/// and calls the corresponding method according to different assembler. +impl LibAssembler for KclvmLibAssembler { + #[inline] + fn assemble_lib( + &self, + compile_prog: &Program, + import_names: IndexMap>, + code_file: &str, + code_file_path: &str, + lib_path: &str, + plugin_agent: &u64, + ) -> String { + match &self { + KclvmLibAssembler::LLVM(llvm_a) => llvm_a.assemble_lib( + compile_prog, + import_names, + code_file, + code_file_path, + lib_path, + plugin_agent, + ), + } + } + + #[inline] + fn add_code_file_suffix(&self, code_file: &str) -> String { + match &self { + KclvmLibAssembler::LLVM(llvm_a) => llvm_a.add_code_file_suffix(code_file), + } + } + + #[inline] + fn get_code_file_suffix(&self) -> &str { + match &self { + KclvmLibAssembler::LLVM(llvm_a) => llvm_a.get_code_file_suffix(), + } + } + + #[inline] + fn lock_file_and_gen_lib( + &self, + compile_prog: &Program, + import_names: IndexMap>, + file: &PathBuf, + plugin_agent: &u64, + ) -> String { + match &self { + KclvmLibAssembler::LLVM(llvm_a) => { + llvm_a.lock_file_and_gen_lib(compile_prog, import_names, file, plugin_agent) + } + } + } +} + +/// LlvmLibAssembler is mainly responsible for assembling the generated LLVM IR into a dynamic link library. +#[derive(Clone)] +pub struct LlvmLibAssembler; + +/// KclvmLibAssembler implements the LibAssembler trait, +impl LibAssembler for LlvmLibAssembler { + /// "assemble_lib" will call the "kclvm_compiler/codegen/llvm/emit_code" + /// to generate LLVM IR into ".ll" file. + /// + /// And then assemble the dynamic link library based on the LLVM IR, + /// + /// At last remove the "*.ll" file and return the dynamic link library path. + /// # Examples + /// + /// ``` + /// use kclvm_runner::runner::ExecProgramArgs; + /// use kclvm_parser::load_program; + /// use kclvm_sema::resolver::resolve_program; + /// use kclvm_runner::assembler::LlvmLibAssembler; + /// use crate::kclvm_runner::assembler::LibAssembler; + /// + /// // default args and configuration + /// let mut args = ExecProgramArgs::default(); + /// let k_path = "./src/test_datas/init_check_order_0/main.k"; + /// args.k_filename_list.push(k_path.to_string()); + /// let plugin_agent = 0; + /// let files = args.get_files(); + /// let opts = args.get_load_program_options(); + /// + /// // parse and resolve kcl + /// let mut program = load_program(&files, Some(opts)).unwrap(); + /// let scope = resolve_program(&mut program); + /// + /// // tmp file + /// let temp_entry_file = "test_enrty_file"; + /// let temp_entry_file_path = &format!("{}.ll", temp_entry_file); + /// let temp_entry_file_lib = &format!("{}.dylib", temp_entry_file); + /// + /// // assemble libs + /// let llvm_assembler = LlvmLibAssembler{}; + /// let lib_file = llvm_assembler.assemble_lib( + /// &program, + /// scope.import_names.clone(), + /// temp_entry_file, + /// temp_entry_file_path, + /// temp_entry_file_lib, + /// &plugin_agent + /// ); + /// let lib_path = std::path::Path::new(&lib_file); + /// assert_eq!(lib_path.exists(), true); + /// llvm_assembler.clean_path(&lib_file); + /// assert_eq!(lib_path.exists(), false); + /// ``` + #[inline] + fn assemble_lib( + &self, + compile_prog: &Program, + import_names: IndexMap>, + code_file: &str, + code_file_path: &str, + lib_path: &str, + plugin_agent: &u64, + ) -> String { + // clean "*.ll" file path. + self.clean_path(&code_file_path.to_string()); + + // gen LLVM IR code into ".ll" file. + emit_code( + compile_prog, + import_names, + &EmitOptions { + from_path: None, + emit_path: Some(code_file), + no_link: true, + }, + ) + .expect("Compile KCL to LLVM error"); + + // assemble lib + let mut cmd = Command::new(*plugin_agent); + let gen_lib_path = cmd.run_clang_single(&code_file_path, &lib_path); + + // clean "*.ll" file path + self.clean_path(&code_file_path.to_string()); + gen_lib_path + } + + /// Add ".ll" suffix to a file path. + #[inline] + fn add_code_file_suffix(&self, code_file: &str) -> String { + format!("{}.ll", code_file) + } -/// LLVM IR file suffix. -const LL_FILE: &str = "_a.out"; + /// Get String ".ll" + #[inline] + fn get_code_file_suffix(&self) -> &str { + ".ll" + } +} -/// KclvmAssembler is mainly responsible for assembling the generated bytecode, -/// LLVM IR or other IR code into dynamic link libraries, and take the result of -/// kclvm-parser, kclvm-sema and kclvm-compiler as input. +/// KclvmAssembler is mainly responsible for assembling the generated bytecode +/// LLVM IR or other IR code into dynamic link libraries, for multi-file kcl programs, +/// and take the result of kclvm-parser, kclvm-sema and kclvm-compiler as input. +/// +/// KclvmAssembler improves the performance of kclvm by concurrently compiling kcl multi-file programs. +/// The member "thread_count" of KclvmAssembler is the number of threads in multi-file compilation. +/// +/// KclvmAssembler provides an atomic operation for generating a dynamic link library for a single file +/// through KclvmLibAssembler for each thread. pub struct KclvmAssembler { thread_count: usize, } + impl KclvmAssembler { + /// Constructs an KclvmAssembler instance with a default value 4 + /// for the number of threads in multi-file compilation. + /// + /// # Examples + /// + /// ``` + /// use kclvm_runner::assembler::KclvmAssembler; + /// + /// let kclvm_assem = KclvmAssembler::new(); + /// ``` + #[inline] pub fn new() -> Self { Self { thread_count: 4 } } + /// Constructs an KclvmAssembler instance with a value + /// for the number of threads in multi-file compilation. + /// + /// # Examples + /// + /// ``` + /// use kclvm_runner::assembler::KclvmAssembler; + /// + /// let kclvm_assem = KclvmAssembler::new_with_thread_count(5); + /// ``` + #[inline] pub fn new_with_thread_count(thread_count: usize) -> Self { if thread_count <= 0 { - bug!("Illegal Thread Count"); + bug!("Illegal thread count in multi-file compilation"); } Self { thread_count } } - /// Generate the dylibs and return file paths. + /// Clean up the path of the dynamic link libraries generated. + /// It will remove the file in "file_path" and all the files in file_path end with ir code file suffix. /// - /// In the method, multiple threads will be created to concurrently generate dylibs - /// under different package paths. + /// # Examples /// - /// This method will generate “.out” and ".ll" files, and return the file paths of - /// the generated files in Vec. - pub fn gen_dylibs( - &self, - program: ast::Program, - scope: ProgramScope, - plugin_agent: u64, - entry_file: &String, - ) -> Vec { - // gen bc or ll_file - let path = std::path::Path::new(LL_FILE); + /// ``` + /// use std::fs; + /// use std::fs::File; + /// use kclvm_runner::assembler::KclvmAssembler; + /// + /// // create test dir + /// std::fs::create_dir_all("./src/test_datas/test_clean").unwrap(); + /// + /// //file name and suffix for test + /// let test_path = "./src/test_datas/test_clean/test.out"; + /// let file_suffix = ".ll"; + /// + /// // creat file "./src/test_datas/test_clean/test.out" + /// File::create(test_path); + /// let path = std::path::Path::new(test_path); + /// assert_eq!(path.exists(), true); + /// + /// // delete file "./src/test_datas/test_clean/test.out" and "./src/test_datas/test_clean/test.out*.ll" + /// KclvmAssembler::new().clean_path_for_genlibs(test_path, file_suffix); + /// assert_eq!(path.exists(), false); + /// + /// // create files whose filename end with "*.ll" + /// let ll_test1 = &format!("{}{}", test_path, ".test1.ll"); + /// let ll_test2 = &format!("{}{}", test_path, ".test2.ll"); + /// File::create(ll_test1); + /// File::create(ll_test2); + /// let path1 = std::path::Path::new(ll_test1); + /// let path2 = std::path::Path::new(ll_test2); + /// assert_eq!(path1.exists(), true); + /// assert_eq!(path2.exists(), true); + /// + /// // delete file "./src/test_datas/test_clean/test.out" and "./src/test_datas/test_clean/test.out*.ll" + /// KclvmAssembler::new().clean_path_for_genlibs(test_path, file_suffix); + /// assert_eq!(path1.exists(), false); + /// assert_eq!(path2.exists(), false); + /// ``` + #[inline] + pub fn clean_path_for_genlibs(&self, file_path: &str, suffix: &str) { + let path = std::path::Path::new(file_path); if path.exists() { std::fs::remove_file(path).unwrap(); } - for entry in glob::glob(&format!("{}*.ll", LL_FILE)).unwrap() { + for entry in glob::glob(&format!("{}*{}", file_path, suffix)).unwrap() { match entry { Ok(path) => { if path.exists() { @@ -63,14 +403,103 @@ impl KclvmAssembler { Err(e) => println!("{:?}", e), }; } + } - let cache_dir = Path::new(&program.root) + /// Generate cache dir from ast.Program.root. + /// Create cache dir if it doesn't exist. + /// + /// # Examples + /// + /// ``` + /// use std::fs; + /// use kclvm_runner::assembler::KclvmAssembler; + /// + /// let expected_dir = "test_prog_name/.kclvm/cache/0.4.2-e07ed7af0d9bd1e86a3131714e4bd20c"; + /// let path = std::path::Path::new(expected_dir); + /// assert_eq!(path.exists(), false); + /// + /// let cache_dir = KclvmAssembler::new().load_cache_dir("test_prog_name"); + /// assert_eq!(cache_dir.display().to_string(), expected_dir); + /// + /// let path = std::path::Path::new(expected_dir); + /// assert_eq!(path.exists(), true); + /// + /// fs::remove_dir(expected_dir); + /// assert_eq!(path.exists(), false); + /// ``` + #[inline] + pub fn load_cache_dir(&self, prog_root_name: &str) -> PathBuf { + let cache_dir = Path::new(prog_root_name) .join(".kclvm") .join("cache") .join(kclvm_version::get_full_version()); if !cache_dir.exists() { std::fs::create_dir_all(&cache_dir).unwrap(); } + cache_dir + } + + /// Generate the dynamic link libraries by llvm IR and return file paths. + /// + /// In the method, multiple threads will be created to concurrently generate dynamic link libraries + /// under different package paths. + /// + /// This method will generate dynamic link library files (“*.dylib”, "*.dll.lib", "*.so") + /// and ir code files ("*.ll"), and return the file paths of the dynamic link library files in Vec. + /// + /// "gen_libs" will create multiple threads and call the method provided by "KclvmLibAssembler" in each thread + /// to generate the dynamic link library in parallel. + /// + /// # Examples + /// + ///``` + /// use std::fs; + /// use kclvm_parser::load_program; + /// use kclvm_runner::runner::ExecProgramArgs; + /// use kclvm_runner::assembler::KclvmAssembler; + /// use kclvm_runner::assembler::KclvmLibAssembler; + /// use kclvm_runner::assembler::LlvmLibAssembler; + /// use kclvm_sema::resolver::resolve_program; + /// + /// let plugin_agent = 0; + /// + /// let args = ExecProgramArgs::default(); + /// let opts = args.get_load_program_options(); + /// + /// let kcl_path = "./src/test_datas/init_check_order_0/main.k"; + /// let mut prog = load_program(&[kcl_path], Some(opts)).unwrap(); + /// let scope = resolve_program(&mut prog); + /// + /// let lib_paths = KclvmAssembler::new().gen_libs( + /// prog, + /// scope, + /// plugin_agent, + /// &("test_entry_file_name".to_string()), + /// KclvmLibAssembler::LLVM(LlvmLibAssembler {})); + /// assert_eq!(lib_paths.len(), 1); + /// + /// let expected_lib_path = fs::canonicalize("./test_entry_file_name.dylib").unwrap().display().to_string(); + /// assert_eq!(*lib_paths.get(0).unwrap(), expected_lib_path); + /// + /// let path = std::path::Path::new(&expected_lib_path); + /// assert_eq!(path.exists(), true); + /// + /// KclvmAssembler::new().clean_path_for_genlibs(&expected_lib_path, ".dylib"); + /// assert_eq!(path.exists(), false); + ///``` + pub fn gen_libs( + &self, + program: ast::Program, + scope: ProgramScope, + plugin_agent: u64, + entry_file: &String, + single_file_assembler: KclvmLibAssembler, + ) -> Vec { + // clean the code generated path. + self.clean_path_for_genlibs(IR_FILE, single_file_assembler.get_code_file_suffix()); + // load cache + let cache_dir = self.load_cache_dir(&program.root); + let mut compile_progs: IndexMap< String, ( @@ -97,10 +526,11 @@ impl KclvmAssembler { let pool = ThreadPool::new(self.thread_count); let (tx, rx) = channel(); let prog_count = compile_progs.len(); - // let temp_entry_file = temp_file(); for (pkgpath, (compile_prog, import_names, cache_dir)) in compile_progs { let tx = tx.clone(); let temp_entry_file = entry_file.clone(); + // clone a single file assembler for one thread. + let assembler = single_file_assembler.clone(); pool.execute(move || { let root = &compile_prog.root; let is_main_pkg = pkgpath == kclvm_ast::MAIN_PKG; @@ -110,96 +540,54 @@ impl KclvmAssembler { // specify a standard entry for these multi-files and cannot // be shared, so the cache of the main package is not read and // written. - let dylib_path = if is_main_pkg { + let lib_path = if is_main_pkg { let file = PathBuf::from(&temp_entry_file); - lock_ll_file_and_gen_dylib(&compile_prog, import_names, &file, &plugin_agent) + // generate dynamic link library for single file kcl program + assembler.lock_file_and_gen_lib( + &compile_prog, + import_names, + &file, + &plugin_agent, + ) } else { let file = cache_dir.join(&pkgpath); - // Read the dylib cache - let dylib_relative_path: Option = + // Read the lib cache + let lib_relative_path: Option = load_pkg_cache(root, &pkgpath, CacheOption::default()); - match dylib_relative_path { - Some(dylib_relative_path) => { - if dylib_relative_path.starts_with('.') { - dylib_relative_path.replacen(".", root, 1) + match lib_relative_path { + Some(lib_relative_path) => { + if lib_relative_path.starts_with('.') { + lib_relative_path.replacen(".", root, 1) } else { - dylib_relative_path + lib_relative_path } } None => { - let dylib_path = lock_ll_file_and_gen_dylib( + // generate dynamic link library for single file kcl program + let lib_path = assembler.lock_file_and_gen_lib( &compile_prog, import_names, &file, &plugin_agent, ); - let dylib_relative_path = dylib_path.replacen(root, ".", 1); + let lib_relative_path = lib_path.replacen(root, ".", 1); save_pkg_cache( root, &pkgpath, - dylib_relative_path, + lib_relative_path, CacheOption::default(), ); - dylib_path + lib_path } } }; - tx.send(dylib_path) + tx.send(lib_path) .expect("channel will be there waiting for the pool"); }); } - rx.iter().take(prog_count).collect::>() - } -} - -fn lock_ll_file_and_gen_dylib( - compile_prog: &Program, - import_names: IndexMap>, - file: &PathBuf, - plugin_agent: &u64, -) -> String { - // ll_file: file_name - // ll_path: file_name.ll - // dylib_path: file_name.dll.lib or file_name.dylib or file_name.so - let ll_file = file.to_str().unwrap(); - let ll_path = format!("{}.ll", ll_file); - let dylib_path = format!("{}{}", ll_file, Command::get_lib_suffix()); - - // Locking "*.ll" file for parallel code generation - let mut ll_path_lock = fslock::LockFile::open(&format!("{}.lock", ll_path)).unwrap(); - ll_path_lock.lock().unwrap(); - - // Clean "*.ll" file path - clean_ll_path(&ll_path); - - // gen code - emit_code( - compile_prog, - import_names, - &EmitOptions { - from_path: None, - emit_path: Some(ll_file), - no_link: true, - }, - ) - .expect("Compile KCL to LLVM error"); - - // assemble dylib - let mut cmd = Command::new(*plugin_agent); - let gen_dylib_path = cmd.run_clang_single(&ll_path, &dylib_path); - - // Clean "*.ll" file path - clean_ll_path(&ll_path); - - // unlock "*.ll" file - ll_path_lock.unlock().unwrap(); - - return gen_dylib_path; -} - -// Clean "*.ll" file path -fn clean_ll_path(ll_path: &String) { - if Path::new(ll_path).exists() { - std::fs::remove_file(&ll_path).unwrap(); + let lib_paths = rx.iter().take(prog_count).collect::>(); + // clean the lock file. + single_file_assembler.clean_lock_file(entry_file); + lib_paths } } diff --git a/kclvm/runner/src/command.rs b/kclvm/runner/src/command.rs index e00a9af59..2ebafcdb2 100644 --- a/kclvm/runner/src/command.rs +++ b/kclvm/runner/src/command.rs @@ -7,7 +7,7 @@ use kclvm_config::settings::SettingsFile; #[derive(Debug)] pub struct Command { clang_path: String, - rust_libstd_dylib: String, + rust_libstd_lib: String, executable_root: String, plugin_method_ptr: u64, } @@ -15,20 +15,20 @@ pub struct Command { impl Command { pub fn new(plugin_method_ptr: u64) -> Self { let executable_root = Self::get_executable_root(); - let rust_libstd_dylib = Self::get_rust_libstd_dylib(executable_root.as_str()); + let rust_libstd_lib = Self::get_rust_libstd_lib(executable_root.as_str()); let clang_path = Self::get_clang_path(); Self { clang_path, - rust_libstd_dylib, + rust_libstd_lib, executable_root, plugin_method_ptr, } } - pub fn run_dylib(&self, dylib_path: &str) -> Result { + pub fn run_lib(&self, lib_path: &str) -> Result { unsafe { - let lib = libloading::Library::new(dylib_path).unwrap(); + let lib = libloading::Library::new(lib_path).unwrap(); // get kclvm_plugin_init let kclvm_plugin_init: libloading::Symbol< @@ -119,13 +119,13 @@ impl Command { } } - pub fn run_dylib_with_settings( + pub fn run_lib_with_settings( &self, - dylib_path: &str, + lib_path: &str, settings: SettingsFile, ) -> Result { unsafe { - let lib = libloading::Library::new(dylib_path).unwrap(); + let lib = libloading::Library::new(lib_path).unwrap(); let kcl_run: libloading::Symbol< unsafe extern "C" fn( @@ -201,14 +201,14 @@ impl Command { Ok("".to_string()) } - pub fn link_dylibs(&mut self, dylibs: &[String], dylib_path: &str) -> String { - let dylib_suffix = Self::get_lib_suffix(); - let dylib_path = if dylib_path.is_empty() { - format!("{}{}", "_a.out", dylib_suffix) - } else if !dylib_path.ends_with(&dylib_suffix) { - format!("{}{}", dylib_path, dylib_suffix) + pub fn link_libs(&mut self, libs: &[String], lib_path: &str) -> String { + let lib_suffix = Self::get_lib_suffix(); + let lib_path = if lib_path.is_empty() { + format!("{}{}", "_a.out", lib_suffix) + } else if !lib_path.ends_with(&lib_suffix) { + format!("{}{}", lib_path, lib_suffix) } else { - dylib_path.to_string() + lib_path.to_string() }; let mut args: Vec = vec![ @@ -223,13 +223,13 @@ impl Command { "-lkclvm_native_shared".to_string(), format!("-I{}/include", self.executable_root), ]; - let mut bc_files = dylibs.to_owned(); + let mut bc_files = libs.to_owned(); args.append(&mut bc_files); let mut more_args = vec![ - self.rust_libstd_dylib.clone(), + self.rust_libstd_lib.clone(), "-fPIC".to_string(), "-o".to_string(), - dylib_path.to_string(), + lib_path.to_string(), ]; args.append(&mut more_args); @@ -240,12 +240,12 @@ impl Command { .output() .expect("clang failed"); - dylib_path + lib_path } - pub fn run_clang(&mut self, bc_path: &str, dylib_path: &str) -> String { + pub fn run_clang(&mut self, bc_path: &str, lib_path: &str) -> String { let mut bc_path = bc_path.to_string(); - let mut dylib_path = dylib_path.to_string(); + let mut lib_path = lib_path.to_string(); let mut bc_files = vec![]; @@ -276,8 +276,8 @@ impl Command { } } - if dylib_path.is_empty() { - dylib_path = format!("{}{}", bc_path, Self::get_lib_suffix()); + if lib_path.is_empty() { + lib_path = format!("{}{}", bc_path, Self::get_lib_suffix()); } let mut args: Vec = vec![ @@ -294,10 +294,10 @@ impl Command { ]; args.append(&mut bc_files); let mut more_args = vec![ - self.rust_libstd_dylib.clone(), + self.rust_libstd_lib.clone(), "-fPIC".to_string(), "-o".to_string(), - dylib_path.to_string(), + lib_path.to_string(), ]; args.append(&mut more_args); @@ -308,12 +308,12 @@ impl Command { .output() .expect("clang failed"); - dylib_path + lib_path } - pub fn run_clang_single(&mut self, bc_path: &str, dylib_path: &str) -> String { + pub fn run_clang_single(&mut self, bc_path: &str, lib_path: &str) -> String { let mut bc_path = bc_path.to_string(); - let mut dylib_path = dylib_path.to_string(); + let mut lib_path = lib_path.to_string(); if !Self::path_exist(bc_path.as_str()) { let s = format!("{}.ll", bc_path); @@ -327,8 +327,8 @@ impl Command { } } - if dylib_path.is_empty() { - dylib_path = format!("{}{}", bc_path, Self::get_lib_suffix()); + if lib_path.is_empty() { + lib_path = format!("{}{}", bc_path, Self::get_lib_suffix()); } let mut args: Vec = vec![ @@ -346,10 +346,10 @@ impl Command { let mut bc_files = vec![bc_path]; args.append(&mut bc_files); let mut more_args = vec![ - self.rust_libstd_dylib.clone(), + self.rust_libstd_lib.clone(), "-fPIC".to_string(), "-o".to_string(), - dylib_path.to_string(), + lib_path.to_string(), ]; args.append(&mut more_args); @@ -360,7 +360,7 @@ impl Command { .output() .expect("clang failed"); // Use absolute path. - let path = PathBuf::from(&dylib_path).canonicalize().unwrap(); + let path = PathBuf::from(&lib_path).canonicalize().unwrap(); path.to_str().unwrap().to_string() } @@ -384,7 +384,7 @@ impl Command { p.to_str().unwrap().to_string() } - fn get_rust_libstd_dylib(executable_root: &str) -> String { + fn get_rust_libstd_lib(executable_root: &str) -> String { let txt_path = std::path::Path::new(&executable_root) .join(if Self::is_windows() { "libs" } else { "lib" }) .join("rust-libstd-name.txt"); diff --git a/kclvm/runner/src/lib.rs b/kclvm/runner/src/lib.rs index c93014e57..fe0a5e0e3 100644 --- a/kclvm/runner/src/lib.rs +++ b/kclvm/runner/src/lib.rs @@ -1,5 +1,6 @@ use std::path::Path; +use assembler::{KclvmLibAssembler, LlvmLibAssembler}; use command::Command; use kclvm_ast::ast::Program; use kclvm_sema::resolver::resolve_program; @@ -14,7 +15,7 @@ pub mod runner; pub mod tests; /// After the kcl program passed through kclvm-parser in the compiler frontend, -/// KCLVM needs to resolve ast, generate corresponding LLVM IR, dylibs or +/// KCLVM needs to resolve ast, generate corresponding LLVM IR, dynamic link library or /// executable file for kcl program in the compiler backend. /// /// Method “execute” is the entry point for the compiler backend. @@ -27,11 +28,16 @@ pub mod tests; /// /// This method will first resolve “program” (ast.Program) and save the result to the "scope" (ProgramScope). /// -/// Then, dylibs is generated by KclvmAssembler, and method "KclvmAssembler::gen_dylibs" -/// will return dylibs path in a "Vec"; +/// Then, dynamic link libraries is generated by KclvmAssembler, and method "KclvmAssembler::gen_libs" +/// will return dynamic link library paths in a "Vec"; /// -/// After linking all dylibs by KclvmLinker, method "KclvmLinker::link_all_dylibs" will return a path -/// for dylib. +/// KclvmAssembler is mainly responsible for concurrent compilation of multiple files. +/// Single-file compilation in each thread in concurrent compilation is the responsibility of KclvmLibAssembler. +/// In the future, it may support the dynamic link library generation of multiple intermediate language. +/// KclvmLibAssembler currently only supports LLVM IR. +/// +/// After linking all dynamic link libraries by KclvmLinker, method "KclvmLinker::link_all_libs" will return a path +/// for dynamic link library after linking. /// /// At last, KclvmRunner will be constructed and call method "run" to execute the kcl program. /// @@ -54,12 +60,11 @@ pub mod tests; /// let kcl_path = "./src/test_datas/init_check_order_0/main.k"; /// let prog = load_program(&[kcl_path], Some(opts)).unwrap(); /// -/// // resolve ast, generate dylibs, link dylibs and execute. +/// // resolve ast, generate libs, link libs and execute. /// // result is the kcl in json format. /// let result = execute(prog, plugin_agent, &args).unwrap(); /// } /// ``` -/// pub fn execute( mut program: Program, plugin_agent: u64, @@ -69,20 +74,24 @@ pub fn execute( let scope = resolve_program(&mut program); scope.check_scope_diagnostics(); - // generate dylibs + // generate libs let temp_entry_file = temp_file(); - let dylib_paths = - assembler::KclvmAssembler::new().gen_dylibs(program, scope, plugin_agent, &temp_entry_file); + let lib_paths = assembler::KclvmAssembler::new().gen_libs( + program, + scope, + plugin_agent, + &temp_entry_file, + KclvmLibAssembler::LLVM(LlvmLibAssembler {}), + ); - // link dylibsKclvmRunner - let dylib_suffix = Command::get_lib_suffix(); - let temp_out_dylib_file = format!("{}.out{}", temp_entry_file, dylib_suffix); - let dylib_path = - linker::KclvmLinker::link_all_dylibs(dylib_paths, temp_out_dylib_file, plugin_agent); + // link libs + let lib_suffix = Command::get_lib_suffix(); + let temp_out_lib_file = format!("{}.out{}", temp_entry_file, lib_suffix); + let lib_path = linker::KclvmLinker::link_all_libs(lib_paths, temp_out_lib_file, plugin_agent); // run let runner = KclvmRunner::new( - dylib_path.as_str(), + lib_path.as_str(), Some(KclvmRunnerOptions { plugin_agent_ptr: plugin_agent, }), @@ -90,19 +99,16 @@ pub fn execute( let result = runner.run(&args); // clean files - remove_file(&dylib_path); - clean_tmp_files(&temp_entry_file, &dylib_suffix); + remove_file(&lib_path); + clean_tmp_files(&temp_entry_file, &lib_suffix); result } +/// Clean all the tmp files generated during lib generating and linking. #[inline] -/// Clean all the tmp files generated during dylib generating and linking. -fn clean_tmp_files(temp_entry_file: &String, dylib_suffix: &String) { - let ll_lock_suffix = ".ll.lock"; - let temp_entry_dylib_file = format!("{}{}", temp_entry_file, dylib_suffix); - let temp_entry_ll_lock_file = format!("{}{}", temp_entry_file, ll_lock_suffix); - remove_file(&temp_entry_dylib_file); - remove_file(&temp_entry_ll_lock_file); +fn clean_tmp_files(temp_entry_file: &String, lib_suffix: &String) { + let temp_entry_lib_file = format!("{}{}", temp_entry_file, lib_suffix); + remove_file(&temp_entry_lib_file); } #[inline] @@ -112,8 +118,8 @@ fn remove_file(file: &str) { } } -#[inline] /// Returns a temporary file name consisting of timestamp and process id. +#[inline] fn temp_file() -> String { let timestamp = chrono::Local::now().timestamp_nanos(); let id = std::process::id(); diff --git a/kclvm/runner/src/linker.rs b/kclvm/runner/src/linker.rs index d449a7b9f..4be7a8492 100644 --- a/kclvm/runner/src/linker.rs +++ b/kclvm/runner/src/linker.rs @@ -1,15 +1,11 @@ use crate::command::Command; -/// KclvmLinker is mainly responsible for linking the dylibs generated by KclvmAssembler. +/// KclvmLinker is mainly responsible for linking the libs generated by KclvmAssembler. pub struct KclvmLinker; impl KclvmLinker { - /// Link the dylibs generated by method "gen_bc_or_ll_file". - pub fn link_all_dylibs( - dylib_paths: Vec, - dylib_path: String, - plugin_agent: u64, - ) -> String { + /// Link the libs generated by method "gen_bc_or_ll_file". + pub fn link_all_libs(lib_paths: Vec, lib_path: String, plugin_agent: u64) -> String { let mut cmd = Command::new(plugin_agent); - cmd.link_dylibs(&dylib_paths, &dylib_path) + cmd.link_libs(&lib_paths, &lib_path) } } diff --git a/kclvm/runner/src/runner.rs b/kclvm/runner/src/runner.rs index 2e5b6cdf8..2694c48cf 100644 --- a/kclvm/runner/src/runner.rs +++ b/kclvm/runner/src/runner.rs @@ -84,9 +84,9 @@ pub struct KclvmRunner { } impl KclvmRunner { - pub fn new(dylib_path: &str, opts: Option) -> Self { + pub fn new(lib_path: &str, opts: Option) -> Self { let lib = unsafe { - libloading::Library::new(std::path::PathBuf::from(dylib_path).canonicalize().unwrap()) + libloading::Library::new(std::path::PathBuf::from(lib_path).canonicalize().unwrap()) .unwrap() }; Self { @@ -97,14 +97,14 @@ impl KclvmRunner { pub fn run(&self, args: &ExecProgramArgs) -> Result { unsafe { - Self::dylib_kclvm_plugin_init(&self.lib, self.opts.plugin_agent_ptr); - Self::dylib_kcl_run(&self.lib, &args) + Self::lib_kclvm_plugin_init(&self.lib, self.opts.plugin_agent_ptr); + Self::lib_kcl_run(&self.lib, &args) } } } impl KclvmRunner { - unsafe fn dylib_kclvm_plugin_init(lib: &libloading::Library, plugin_method_ptr: u64) { + unsafe fn lib_kclvm_plugin_init(lib: &libloading::Library, plugin_method_ptr: u64) { // get kclvm_plugin_init let kclvm_plugin_init: libloading::Symbol< unsafe extern "C" fn( @@ -134,7 +134,7 @@ impl KclvmRunner { kclvm_plugin_init(plugin_method); } - unsafe fn dylib_kcl_run( + unsafe fn lib_kcl_run( lib: &libloading::Library, args: &ExecProgramArgs, ) -> Result { diff --git a/kclvm/runner/src/tests.rs b/kclvm/runner/src/tests.rs index db303d82f..07c540eb3 100644 --- a/kclvm/runner/src/tests.rs +++ b/kclvm/runner/src/tests.rs @@ -15,7 +15,7 @@ const KCL_FILE_NAME: &str = "main.k"; const MAIN_PKG_NAME: &str = "__main__"; /// Load test kcl file to ast.Program -pub fn load_program(filename: String) -> Program { +pub fn load_program1(filename: String) -> Program { let module = load_module(filename); construct_program(module) } @@ -62,8 +62,8 @@ pub fn execute_for_test(kcl_path: &String) -> String { let plugin_agent = 0; let args = ExecProgramArgs::default(); // parse kcl file - let program = load_program(kcl_path.to_string()); - // generate dylibs, link dylibs and execute. + let program = load_program1(kcl_path.to_string()); + // generate libs, link libs and execute. execute(program, plugin_agent, &args).unwrap() } diff --git a/kclvm/src/main.rs b/kclvm/src/main.rs index 934dce543..3dcd65543 100644 --- a/kclvm/src/main.rs +++ b/kclvm/src/main.rs @@ -110,13 +110,13 @@ fn main() { format!("{}.lock", cache_dir.join(&pkgpath).to_str().unwrap()); let ll_file = file.to_str().unwrap(); let ll_path = format!("{}.ll", ll_file); - let dylib_path = format!("{}{}", ll_file, Command::get_lib_suffix()); + let lib_path = format!("{}{}", ll_file, Command::get_lib_suffix()); let mut ll_path_lock = fslock::LockFile::open(&lock_file).unwrap(); ll_path_lock.lock().unwrap(); if Path::new(&ll_path).exists() { std::fs::remove_file(&ll_path).unwrap(); } - let dylib_path = if is_main_pkg { + let lib_path = if is_main_pkg { emit_code( &compile_prog, import_names, @@ -128,17 +128,17 @@ fn main() { ) .expect("Compile KCL to LLVM error"); let mut cmd = Command::new(0); - cmd.run_clang_single(&ll_path, &dylib_path) + cmd.run_clang_single(&ll_path, &lib_path) } else { - // If AST module has been modified, ignore the dylib cache - let dylib_relative_path: Option = + // If AST module has been modified, ignore the lib cache + let lib_relative_path: Option = load_pkg_cache(root, &pkgpath, CacheOption::default()); - match dylib_relative_path { - Some(dylib_relative_path) => { - if dylib_relative_path.starts_with('.') { - dylib_relative_path.replacen(".", root, 1) + match lib_relative_path { + Some(lib_relative_path) => { + if lib_relative_path.starts_with('.') { + lib_relative_path.replacen(".", root, 1) } else { - dylib_relative_path + lib_relative_path } } None => { @@ -153,16 +153,16 @@ fn main() { ) .expect("Compile KCL to LLVM error"); let mut cmd = Command::new(0); - let dylib_path = cmd.run_clang_single(&ll_path, &dylib_path); - let dylib_relative_path = dylib_path.replacen(root, ".", 1); + let lib_path = cmd.run_clang_single(&ll_path, &lib_path); + let lib_relative_path = lib_path.replacen(root, ".", 1); save_pkg_cache( root, &pkgpath, - dylib_relative_path, + lib_relative_path, CacheOption::default(), ); - dylib_path + lib_path } } }; @@ -170,24 +170,24 @@ fn main() { std::fs::remove_file(&ll_path).unwrap(); } ll_path_lock.unlock().unwrap(); - dylib_path + lib_path }); theads.push(t); } - let mut dylib_paths = vec![]; + let mut lib_paths = vec![]; for t in theads { - let dylib_path = t.join().unwrap(); - dylib_paths.push(dylib_path); + let lib_path = t.join().unwrap(); + lib_paths.push(lib_path); } let mut cmd = Command::new(0); - // link all dylibs - let dylib_path = cmd.link_dylibs(&dylib_paths, ""); + // link all libs + let lib_path = cmd.link_libs(&lib_paths, ""); // Config build let settings = build_settings(&matches); - cmd.run_dylib_with_settings(&dylib_path, settings).unwrap(); - for dylib_path in dylib_paths { - if dylib_path.contains(kclvm_ast::MAIN_PKG) && Path::new(&dylib_path).exists() { - std::fs::remove_file(&dylib_path).unwrap(); + cmd.run_lib_with_settings(&lib_path, settings).unwrap(); + for lib_path in lib_paths { + if lib_path.contains(kclvm_ast::MAIN_PKG) && Path::new(&lib_path).exists() { + std::fs::remove_file(&lib_path).unwrap(); } } }