From dcb5ab10f286a47643f57572f93e90e9d8bc0303 Mon Sep 17 00:00:00 2001 From: peefy Date: Mon, 20 Jun 2022 15:25:47 +0800 Subject: [PATCH] refactor(kclvm-runner): merge main and refacor kclvm-runner. 1. Encapsulated method "emit_code" into "lock_ll_file_and_gen_dylib" to reduce repetitive code in kclvm-runner/KclvmAssembler.gen_dylibs(). 2. In order to support reuse and simplify the structure of Method "gen_dylibs()", encapsulates some operations of cleaning file paths. 3. Due to issue #79, some test cases are temporarily commented out 2. In order to support reuse and simplify the structure of Method "gen_dylibs()", encapsulates some operations of cleaning file paths. 3. Due to issue #79, some test cases are temporarily commented out fix #67 refactor(kclvm-runner): decouping assembler and llvm. 1. Decoupling the assembler and llvm. 2. The assembling LLVM IR into a dynamic link library is encapsulated into "LlvmLibAssembler" separately. 3. Add trait "LibAssembler" to describe the interface "KclvmLibAssembler" should have. If other intermediate code is added in the future, the corresponding assembler must implement this trait. 4. Struct "LlvmLibAssembler" is provided in "KclvmLibAssembler". "KclvmLibAssembler" is an enum that implements trait "LibAssembler". "KclvmLibAssembler" is responsible for the compilation of a single kcl file in one thread. , 5. "KclvmAssembler" is responsible for the concurrent compilation of multiple kcl files. 6. "KclvmAssembler" will call the method in "KclvmLibAssembler" to generate a dynamic link library for a single kcl file in each thread of concurrent compilation. fix #67 --- kclvm/Cargo.lock | 2 + kclvm/config/src/settings.rs | 4 +- kclvm/runner/Cargo.lock | 12 + kclvm/runner/Cargo.toml | 2 + kclvm/runner/benches/bench_runner.rs | 26 +- kclvm/runner/src/assembler.rs | 595 ++++++++++++++++++++++----- kclvm/runner/src/command.rs | 68 +-- kclvm/runner/src/lib.rs | 98 +++-- kclvm/runner/src/linker.rs | 12 +- kclvm/runner/src/runner.rs | 38 +- kclvm/runner/src/tests.rs | 11 +- kclvm/src/lib.rs | 4 +- kclvm/src/main.rs | 175 +------- 13 files changed, 679 insertions(+), 368 deletions(-) diff --git a/kclvm/Cargo.lock b/kclvm/Cargo.lock index 401fbf1ee..c63017f87 100644 --- a/kclvm/Cargo.lock +++ b/kclvm/Cargo.lock @@ -568,11 +568,13 @@ dependencies = [ "kclvm-parser", "kclvm-runtime", "kclvm-sema", + "kclvm-tools", "kclvm-version", "libc", "libloading", "serde", "serde_json", + "tempfile", "threadpool", "walkdir", ] diff --git a/kclvm/config/src/settings.rs b/kclvm/config/src/settings.rs index 8af9b8a8a..66493a63b 100644 --- a/kclvm/config/src/settings.rs +++ b/kclvm/config/src/settings.rs @@ -58,8 +58,8 @@ impl Default for SettingsFile { #[derive(Serialize, Deserialize, Debug, Clone)] pub struct KeyValuePair { - key: String, - value: String, + pub key: String, + pub value: String, } #[derive(Serialize, Deserialize, Debug, Clone)] diff --git a/kclvm/runner/Cargo.lock b/kclvm/runner/Cargo.lock index 86e49eac8..1479f00e1 100644 --- a/kclvm/runner/Cargo.lock +++ b/kclvm/runner/Cargo.lock @@ -647,6 +647,7 @@ dependencies = [ "kclvm-parser", "kclvm-runtime", "kclvm-sema", + "kclvm-tools", "kclvm-version", "libc", "libloading", @@ -712,6 +713,17 @@ dependencies = [ "scoped-tls", ] +[[package]] +name = "kclvm-tools" +version = "0.1.0" +dependencies = [ + "fancy-regex", + "indexmap", + "kclvm-ast", + "kclvm-error", + "kclvm-parser", +] + [[package]] name = "kclvm-version" version = "0.1.0" diff --git a/kclvm/runner/Cargo.toml b/kclvm/runner/Cargo.toml index e8d7eba6f..0daab977a 100644 --- a/kclvm/runner/Cargo.toml +++ b/kclvm/runner/Cargo.toml @@ -17,6 +17,7 @@ fslock = "0.2.1" libloading = "0.7.3" threadpool = "1.0" chrono = "0.4.19" +tempfile = "3.3.0" kclvm-ast = {path = "../ast", version = "0.1.0"} kclvm-parser = {path = "../parser", version = "0.1.0"} @@ -26,6 +27,7 @@ kclvm-runtime = {path = "../runtime", version = "0.1.0"} kclvm-sema = {path = "../sema", version = "0.1.0"} kclvm-version = {path = "../version", version = "0.1.0"} kclvm-error = {path = "../error", version="0.1.0"} +kclvm-tools = {path = "../tools", version = "0.1.0"} [dev-dependencies] kclvm-parser = {path = "../parser", version = "0.1.0"} diff --git a/kclvm/runner/benches/bench_runner.rs b/kclvm/runner/benches/bench_runner.rs index df44433d7..f560bd64f 100644 --- a/kclvm/runner/benches/bench_runner.rs +++ b/kclvm/runner/benches/bench_runner.rs @@ -1,6 +1,7 @@ use criterion::{criterion_group, criterion_main, Criterion}; use kclvm_parser::load_program; use kclvm_runner::{execute, runner::ExecProgramArgs}; +use kclvm_tools::query::apply_overrides; const TEST_CASE_PATH: &str = "/src/test_datas/init_check_order_0/main.k"; @@ -10,17 +11,30 @@ pub fn criterion_benchmark(c: &mut Criterion) { std::env::current_dir().unwrap().to_str().unwrap(), TEST_CASE_PATH ); - let plugin_agent = 0; - c.bench_function("load_program -> execute", |b| { + c.bench_function("refactor kclvm-runner", |b| { b.iter(|| { - let args = ExecProgramArgs::default(); - let opts = args.get_load_program_options(); - let program = load_program(&[kcl_path], Some(opts)).unwrap(); - execute(program, plugin_agent, &args).unwrap() + after_refactor(kcl_path.to_string()); }) }); } criterion_group!(benches, criterion_benchmark); criterion_main!(benches); + +fn after_refactor(k_path: String) { + let mut args = ExecProgramArgs::default(); + args.k_filename_list.push(k_path); + + let plugin_agent = 0; + + let files = args.get_files(); + let opts = args.get_load_program_options(); + + // load ast + let mut program = load_program(&files, Some(opts)).unwrap(); + apply_overrides(&mut program, &args.overrides, &[]); + + // resolve ast, generate libs, link libs and execute. + execute(program, plugin_agent, &args); +} diff --git a/kclvm/runner/src/assembler.rs b/kclvm/runner/src/assembler.rs index 9bb5ebef8..46770bb5b 100644 --- a/kclvm/runner/src/assembler.rs +++ b/kclvm/runner/src/assembler.rs @@ -1,76 +1,507 @@ +use crate::command::Command; use indexmap::IndexMap; +use kclvm_ast::ast::{self, Program}; +use kclvm_compiler::codegen::{llvm::emit_code, EmitOptions}; +use kclvm_config::cache::{load_pkg_cache, save_pkg_cache, CacheOption}; use kclvm_error::bug; +use kclvm_sema::resolver::scope::ProgramScope; use std::{ collections::HashMap, + env, path::{Path, PathBuf}, sync::mpsc::channel, }; use threadpool::ThreadPool; -use crate::command::Command; -use kclvm_ast::ast::{self, Program}; -use kclvm_compiler::codegen::{llvm::emit_code, EmitOptions}; -use kclvm_config::cache::{load_pkg_cache, save_pkg_cache, CacheOption}; -use kclvm_sema::resolver::scope::ProgramScope; +/// IR code file suffix. +const IR_FILE: &str = "_a.out"; +/// Default codegen timeout. +const DEFAULT_TIME_OUT: u64 = 60; + +/// LibAssembler trait is used to indicate the general interface +/// that must be implemented when different intermediate codes are assembled +/// into dynamic link libraries. +/// +/// Note: LibAssembler is only for single file kcl program. For multi-file kcl programs, +/// KclvmAssembler is provided to support for multi-file parallel compilation to improve +/// the performance of the compiler. +pub trait LibAssembler { + /// Add a suffix to the file name according to the file suffix of different intermediate codes. + /// e.g. LLVM IR + /// code_file : "/test_dir/test_code_file" + /// return : "/test_dir/test_code_file.ll" + fn add_code_file_suffix(&self, code_file: &str) -> String; + + /// Return the file suffix of different intermediate codes. + /// e.g. LLVM IR + /// return : ".ll" + fn get_code_file_suffix(&self) -> &str; + + /// Assemble different intermediate codes into dynamic link libraries for single file kcl program. + /// + /// Inputs: + /// compile_prog: Reference of kcl program ast. + /// + /// "import_names" is import pkgpath and name of kcl program. + /// Type of import_names is "IndexMap>". + /// "kcl_file_name" is the kcl file name string. + /// "import_name" is the name string of import stmt. + /// e.g. "import test/main_pkg as main", "main" is an import_name. + /// "import_path" is the path string of import stmt. + /// e.g. "import test/main_pkg as main", "test/main_pkg" is an import_path. + /// import_names is from "ProgramScope.import_names" returned by "resolve_program" after resolving kcl ast by kclvm-sema. + /// + /// "code_file" is the filename of the generated intermediate code file. + /// e.g. code_file : "/test_dir/test_code_file" + /// + /// "code_file_path" is the full filename of the generated intermediate code file with suffix. + /// e.g. code_file_path : "/test_dir/test_code_file.ll" + /// + /// "lib_path" is the file path of the dynamic link library. + /// e.g. lib_path : "/test_dir/test_code_file.ll.dylib" (mac) + /// e.g. lib_path : "/test_dir/test_code_file.ll.dll.lib" (windows) + /// e.g. lib_path : "/test_dir/test_code_file.ll.so" (ubuntu) + /// + /// "plugin_agent" is a pointer to the plugin address. + /// + /// Returns the path of the dynamic link library. + fn assemble_lib( + &self, + compile_prog: &Program, + import_names: IndexMap>, + code_file: &str, + code_file_path: &str, + lib_path: &str, + plugin_agent: &u64, + ) -> String; + + /// This method is prepared for concurrent compilation in KclvmAssembler. + /// It is an atomic method executed by each thread in concurrent compilation. + /// + /// This method will take the above method “assemble_lib” as a hook method to + /// generate the dynamic link library, and lock the file before calling “assemble_lib”, + /// unlocked after the call ends, + #[inline] + fn lock_file_and_gen_lib( + &self, + compile_prog: &Program, + import_names: IndexMap>, + file: &Path, + plugin_agent: &u64, + ) -> String { + // e.g. LLVM IR + // code_file: file_name + // code_file_path: file_name.ll + // lock_file_path: file_name.dll.lib or file_name.lib or file_name.so + let code_file = file.to_str().unwrap(); + let code_file_path = &self.add_code_file_suffix(code_file); + let lock_file_path = &format!("{}.lock", code_file_path); + let lib_path = format!("{}{}", code_file, Command::get_lib_suffix()); + + // Locking file for parallel code generation. + let mut file_lock = fslock::LockFile::open(lock_file_path).unwrap(); + file_lock.lock().unwrap(); + + // Calling the hook method will generate the corresponding intermediate code + // according to the implementation of method "assemble_lib". + let gen_lib_path = self.assemble_lib( + compile_prog, + import_names, + code_file, + code_file_path, + &lib_path, + plugin_agent, + ); + + // Unlock file + file_lock.unlock().unwrap(); + + gen_lib_path + } + + // Clean file path + // Delete the file in "path". + #[inline] + fn clean_path(&self, path: &str) { + if Path::new(path).exists() { + std::fs::remove_file(&path).unwrap(); + } + } + + // Clean lock file + // Clear the lock files generated during concurrent compilation. + #[inline] + fn clean_lock_file(&self, path: &str) { + let lock_path = &format!("{}.lock", self.add_code_file_suffix(path)); + self.clean_path(lock_path); + } +} + +/// This enum lists all the intermediate code assemblers currently supported by kclvm. +/// Currently only supports assemble llvm intermediate code into dynamic link library. +#[derive(Clone)] +pub enum KclvmLibAssembler { + LLVM(LlvmLibAssembler), +} + +/// KclvmLibAssembler is a dispatcher, responsible for calling corresponding methods +/// according to different types of intermediate codes. +/// +/// KclvmLibAssembler implements the LibAssembler trait, +/// and calls the corresponding method according to different assembler. +impl LibAssembler for KclvmLibAssembler { + #[inline] + fn assemble_lib( + &self, + compile_prog: &Program, + import_names: IndexMap>, + code_file: &str, + code_file_path: &str, + lib_path: &str, + plugin_agent: &u64, + ) -> String { + match &self { + KclvmLibAssembler::LLVM(llvm_a) => llvm_a.assemble_lib( + compile_prog, + import_names, + code_file, + code_file_path, + lib_path, + plugin_agent, + ), + } + } + + #[inline] + fn add_code_file_suffix(&self, code_file: &str) -> String { + match &self { + KclvmLibAssembler::LLVM(llvm_a) => llvm_a.add_code_file_suffix(code_file), + } + } + + #[inline] + fn get_code_file_suffix(&self) -> &str { + match &self { + KclvmLibAssembler::LLVM(llvm_a) => llvm_a.get_code_file_suffix(), + } + } + + #[inline] + fn lock_file_and_gen_lib( + &self, + compile_prog: &Program, + import_names: IndexMap>, + file: &Path, + plugin_agent: &u64, + ) -> String { + match &self { + KclvmLibAssembler::LLVM(llvm_a) => { + llvm_a.lock_file_and_gen_lib(compile_prog, import_names, file, plugin_agent) + } + } + } +} + +/// LlvmLibAssembler is mainly responsible for assembling the generated LLVM IR into a dynamic link library. +#[derive(Clone)] +pub struct LlvmLibAssembler; + +/// KclvmLibAssembler implements the LibAssembler trait, +impl LibAssembler for LlvmLibAssembler { + /// "assemble_lib" will call the [kclvm_compiler::codegen::emit_code] + /// to generate IR file. + /// + /// And then assemble the dynamic link library based on the LLVM IR, + /// + /// At last remove the codegen temp files and return the dynamic link library path. + /// # Examples + /// + /// ```no_run + /// use kclvm_runner::runner::ExecProgramArgs; + /// use kclvm_parser::load_program; + /// use kclvm_sema::resolver::resolve_program; + /// use kclvm_runner::assembler::LlvmLibAssembler; + /// use crate::kclvm_runner::assembler::LibAssembler; + /// + /// // default args and configuration + /// let mut args = ExecProgramArgs::default(); + /// let k_path = "./src/test_datas/init_check_order_0/main.k"; + /// args.k_filename_list.push(k_path.to_string()); + /// let plugin_agent = 0; + /// let files = args.get_files(); + /// let opts = args.get_load_program_options(); + /// + /// // parse and resolve kcl + /// let mut program = load_program(&files, Some(opts)).unwrap(); + /// let scope = resolve_program(&mut program); + /// + /// // tmp file + /// let temp_entry_file = "test_entry_file"; + /// let temp_entry_file_path = &format!("{}.ll", temp_entry_file); + /// let temp_entry_file_lib = &format!("{}.dylib", temp_entry_file); + /// + /// // assemble libs + /// let llvm_assembler = LlvmLibAssembler{}; + /// let lib_file = llvm_assembler.assemble_lib( + /// &program, + /// scope.import_names.clone(), + /// temp_entry_file, + /// temp_entry_file_path, + /// temp_entry_file_lib, + /// &plugin_agent + /// ); + /// let lib_path = std::path::Path::new(&lib_file); + /// assert_eq!(lib_path.exists(), true); + /// llvm_assembler.clean_path(&lib_file); + /// assert_eq!(lib_path.exists(), false); + /// ``` + #[inline] + fn assemble_lib( + &self, + compile_prog: &Program, + import_names: IndexMap>, + code_file: &str, + code_file_path: &str, + lib_path: &str, + plugin_agent: &u64, + ) -> String { + // clean "*.ll" file path. + self.clean_path(&code_file_path.to_string()); + + // gen LLVM IR code into ".ll" file. + emit_code( + compile_prog, + import_names, + &EmitOptions { + from_path: None, + emit_path: Some(code_file), + no_link: true, + }, + ) + .expect("Compile KCL to LLVM error"); + + // assemble lib + let mut cmd = Command::new(*plugin_agent); + let gen_lib_path = cmd.run_clang_single(code_file_path, lib_path); + + // clean "*.ll" file path + self.clean_path(&code_file_path.to_string()); + gen_lib_path + } + + /// Add ".ll" suffix to a file path. + #[inline] + fn add_code_file_suffix(&self, code_file: &str) -> String { + format!("{}.ll", code_file) + } -/// LLVM IR file suffix. -const LL_FILE: &str = "_a.out"; + /// Get String ".ll" + #[inline] + fn get_code_file_suffix(&self) -> &str { + ".ll" + } +} -/// KclvmAssembler is mainly responsible for assembling the generated bytecode, -/// LLVM IR or other IR code into dynamic link libraries, and take the result of -/// kclvm-parser, kclvm-sema and kclvm-compiler as input. +/// KclvmAssembler is mainly responsible for assembling the generated bytecode +/// LLVM IR or other IR code into dynamic link libraries, for multi-file kcl programs, +/// and take the result of kclvm-parser, kclvm-sema and kclvm-compiler as input. +/// +/// KclvmAssembler improves the performance of kclvm by concurrently compiling kcl multi-file programs. +/// The member "thread_count" of KclvmAssembler is the number of threads in multi-file compilation. +/// +/// KclvmAssembler provides an atomic operation for generating a dynamic link library for a single file +/// through KclvmLibAssembler for each thread. pub struct KclvmAssembler { thread_count: usize, } + impl KclvmAssembler { + /// Constructs an KclvmAssembler instance with a default value 4 + /// for the number of threads in multi-file compilation. + /// + /// # Examples + /// + /// ``` + /// use kclvm_runner::assembler::KclvmAssembler; + /// + /// let assembler = KclvmAssembler::new(); + /// ``` + #[inline] pub fn new() -> Self { Self { thread_count: 4 } } + /// Constructs an KclvmAssembler instance with a value + /// for the number of threads in multi-file compilation. + /// + /// # Examples + /// + /// ``` + /// use kclvm_runner::assembler::KclvmAssembler; + /// + /// let assembler = KclvmAssembler::new_with_thread_count(5); + /// ``` + #[inline] pub fn new_with_thread_count(thread_count: usize) -> Self { - if thread_count <= 0 { - bug!("Illegal Thread Count"); + if thread_count == 0 { + bug!("Illegal thread count in multi-file compilation"); } Self { thread_count } } - /// Generate the dylibs and return file paths. + /// Clean up the path of the dynamic link libraries generated. + /// It will remove the file in "file_path" and all the files in file_path end with ir code file suffix. /// - /// In the method, multiple threads will be created to concurrently generate dylibs - /// under different package paths. + /// # Examples /// - /// This method will generate “.out” and ".ll" files, and return the file paths of - /// the generated files in Vec. - pub fn gen_dylibs( - &self, - program: ast::Program, - scope: ProgramScope, - plugin_agent: u64, - entry_file: &String, - ) -> Vec { - // gen bc or ll_file - let path = std::path::Path::new(LL_FILE); + /// ``` + /// use std::fs; + /// use std::fs::File; + /// use kclvm_runner::assembler::KclvmAssembler; + /// + /// // create test dir + /// std::fs::create_dir_all("./src/test_datas/test_clean").unwrap(); + /// + /// // File name and suffix for test + /// let test_path = "./src/test_datas/test_clean/test.out"; + /// let file_suffix = ".ll"; + /// + /// // Create file "./src/test_datas/test_clean/test.out" + /// File::create(test_path); + /// let path = std::path::Path::new(test_path); + /// assert_eq!(path.exists(), true); + /// + /// // Delete file "./src/test_datas/test_clean/test.out" and "./src/test_datas/test_clean/test.out*.ll" + /// KclvmAssembler::new().clean_path_for_genlibs(test_path, file_suffix); + /// assert_eq!(path.exists(), false); + /// + /// // Delete files whose filename end with "*.ll" + /// let test1 = &format!("{}{}", test_path, ".test1.ll"); + /// let test2 = &format!("{}{}", test_path, ".test2.ll"); + /// File::create(test1); + /// File::create(test2); + /// let path1 = std::path::Path::new(test1); + /// let path2 = std::path::Path::new(test2); + /// assert_eq!(path1.exists(), true); + /// assert_eq!(path2.exists(), true); + /// + /// // Delete file "./src/test_datas/test_clean/test.out" and "./src/test_datas/test_clean/test.out*.ll" + /// KclvmAssembler::new().clean_path_for_genlibs(test_path, file_suffix); + /// assert_eq!(path1.exists(), false); + /// assert_eq!(path2.exists(), false); + /// ``` + #[inline] + pub fn clean_path_for_genlibs(&self, file_path: &str, suffix: &str) { + let path = std::path::Path::new(file_path); if path.exists() { std::fs::remove_file(path).unwrap(); } - for entry in glob::glob(&format!("{}*.ll", LL_FILE)).unwrap() { + for entry in glob::glob(&format!("{}*{}", file_path, suffix)).unwrap() { match entry { Ok(path) => { if path.exists() { std::fs::remove_file(path).unwrap(); } } - Err(e) => println!("{:?}", e), + Err(e) => bug!("{:?}", e), }; } + } - let cache_dir = Path::new(&program.root) + /// Generate cache dir from the program root path. Create cache dir if it doesn't exist. + /// + /// # Examples + /// + /// ``` + /// use std::fs; + /// use kclvm_runner::assembler::KclvmAssembler; + /// + /// let expected_dir = "test_prog_name/.kclvm/cache/0.4.2-e07ed7af0d9bd1e86a3131714e4bd20c"; + /// let path = std::path::Path::new(expected_dir); + /// assert_eq!(path.exists(), false); + /// + /// let cache_dir = KclvmAssembler::new().load_cache_dir("test_prog_name"); + /// assert_eq!(cache_dir.display().to_string(), expected_dir); + /// + /// let path = std::path::Path::new(expected_dir); + /// assert_eq!(path.exists(), true); + /// + /// fs::remove_dir(expected_dir); + /// assert_eq!(path.exists(), false); + /// ``` + #[inline] + pub fn load_cache_dir(&self, prog_root_name: &str) -> PathBuf { + let cache_dir = Path::new(prog_root_name) .join(".kclvm") .join("cache") .join(kclvm_version::get_full_version()); if !cache_dir.exists() { std::fs::create_dir_all(&cache_dir).unwrap(); } + cache_dir + } + + /// Generate the dynamic link libraries and return file paths. + /// + /// In the method, multiple threads will be created to concurrently generate dynamic link libraries + /// under different package paths. + /// + /// This method will generate dynamic link library files (such as "*.dylib", "*.dll.lib", "*.so") + /// and ir code files, and return the file paths of the dynamic link library files in [Vec]. + /// + /// `gen_libs` will create multiple threads and call the method provided by [KclvmLibAssembler] in each thread + /// to generate the dynamic link library in parallel. + /// + /// # Examples + /// + /// ```no_run + /// use std::fs; + /// use kclvm_parser::load_program; + /// use kclvm_runner::runner::ExecProgramArgs; + /// use kclvm_runner::assembler::KclvmAssembler; + /// use kclvm_runner::assembler::KclvmLibAssembler; + /// use kclvm_runner::assembler::LlvmLibAssembler; + /// use kclvm_sema::resolver::resolve_program; + /// + /// let plugin_agent = 0; + /// + /// let args = ExecProgramArgs::default(); + /// let opts = args.get_load_program_options(); + /// + /// let kcl_path = "./src/test_datas/init_check_order_0/main.k"; + /// let mut prog = load_program(&[kcl_path], Some(opts)).unwrap(); + /// let scope = resolve_program(&mut prog); + /// + /// let lib_paths = KclvmAssembler::new().gen_libs( + /// prog, + /// scope, + /// plugin_agent, + /// &("test_entry_file_name".to_string()), + /// KclvmLibAssembler::LLVM(LlvmLibAssembler {})); + /// assert_eq!(lib_paths.len(), 1); + /// + /// let expected_lib_path = fs::canonicalize("./test_entry_file_name.dylib").unwrap().display().to_string(); + /// assert_eq!(*lib_paths.get(0).unwrap(), expected_lib_path); + /// + /// let path = std::path::Path::new(&expected_lib_path); + /// assert_eq!(path.exists(), true); + /// + /// KclvmAssembler::new().clean_path_for_genlibs(&expected_lib_path, ".dylib"); + /// assert_eq!(path.exists(), false); + ///``` + pub fn gen_libs( + &self, + program: ast::Program, + scope: ProgramScope, + plugin_agent: u64, + entry_file: &String, + single_file_assembler: KclvmLibAssembler, + ) -> Vec { + // Clean the code generated path. + self.clean_path_for_genlibs(IR_FILE, single_file_assembler.get_code_file_suffix()); + // Load cache + let cache_dir = self.load_cache_dir(&program.root); + let mut compile_progs: IndexMap< String, ( @@ -97,10 +528,11 @@ impl KclvmAssembler { let pool = ThreadPool::new(self.thread_count); let (tx, rx) = channel(); let prog_count = compile_progs.len(); - // let temp_entry_file = temp_file(); for (pkgpath, (compile_prog, import_names, cache_dir)) in compile_progs { let tx = tx.clone(); let temp_entry_file = entry_file.clone(); + // clone a single file assembler for one thread. + let assembler = single_file_assembler.clone(); pool.execute(move || { let root = &compile_prog.root; let is_main_pkg = pkgpath == kclvm_ast::MAIN_PKG; @@ -110,96 +542,71 @@ impl KclvmAssembler { // specify a standard entry for these multi-files and cannot // be shared, so the cache of the main package is not read and // written. - let dylib_path = if is_main_pkg { + let lib_path = if is_main_pkg { let file = PathBuf::from(&temp_entry_file); - lock_ll_file_and_gen_dylib(&compile_prog, import_names, &file, &plugin_agent) + // generate dynamic link library for single file kcl program + assembler.lock_file_and_gen_lib( + &compile_prog, + import_names, + &file, + &plugin_agent, + ) } else { let file = cache_dir.join(&pkgpath); - // Read the dylib cache - let dylib_relative_path: Option = + // Read the lib cache + let lib_relative_path: Option = load_pkg_cache(root, &pkgpath, CacheOption::default()); - match dylib_relative_path { - Some(dylib_relative_path) => { - if dylib_relative_path.starts_with('.') { - dylib_relative_path.replacen(".", root, 1) + match lib_relative_path { + Some(lib_relative_path) => { + if lib_relative_path.starts_with('.') { + lib_relative_path.replacen('.', root, 1) } else { - dylib_relative_path + lib_relative_path } } None => { - let dylib_path = lock_ll_file_and_gen_dylib( + // generate dynamic link library for single file kcl program + let lib_path = assembler.lock_file_and_gen_lib( &compile_prog, import_names, &file, &plugin_agent, ); - let dylib_relative_path = dylib_path.replacen(root, ".", 1); + let lib_relative_path = lib_path.replacen(root, ".", 1); save_pkg_cache( root, &pkgpath, - dylib_relative_path, + lib_relative_path, CacheOption::default(), ); - dylib_path + lib_path } } }; - tx.send(dylib_path) + tx.send(lib_path) .expect("channel will be there waiting for the pool"); }); } - rx.iter().take(prog_count).collect::>() + // Get all codegen results from the channel with timeout + let timeout: u64 = match env::var("KCLVM_CODE_GEN_TIMEOUT") { + Ok(timeout_str) => timeout_str.parse().unwrap_or(DEFAULT_TIME_OUT), + Err(_) => DEFAULT_TIME_OUT, + }; + let mut lib_paths = vec![]; + for _ in 0..prog_count { + let lib_path = rx + .recv_timeout(std::time::Duration::from_secs(timeout)) + .unwrap(); + lib_paths.push(lib_path); + } + // Clean the lock file. + single_file_assembler.clean_lock_file(entry_file); + lib_paths } } -fn lock_ll_file_and_gen_dylib( - compile_prog: &Program, - import_names: IndexMap>, - file: &PathBuf, - plugin_agent: &u64, -) -> String { - // ll_file: file_name - // ll_path: file_name.ll - // dylib_path: file_name.dll.lib or file_name.dylib or file_name.so - let ll_file = file.to_str().unwrap(); - let ll_path = format!("{}.ll", ll_file); - let dylib_path = format!("{}{}", ll_file, Command::get_lib_suffix()); - - // Locking "*.ll" file for parallel code generation - let mut ll_path_lock = fslock::LockFile::open(&format!("{}.lock", ll_path)).unwrap(); - ll_path_lock.lock().unwrap(); - - // Clean "*.ll" file path - clean_ll_path(&ll_path); - - // gen code - emit_code( - compile_prog, - import_names, - &EmitOptions { - from_path: None, - emit_path: Some(ll_file), - no_link: true, - }, - ) - .expect("Compile KCL to LLVM error"); - - // assemble dylib - let mut cmd = Command::new(*plugin_agent); - let gen_dylib_path = cmd.run_clang_single(&ll_path, &dylib_path); - - // Clean "*.ll" file path - clean_ll_path(&ll_path); - - // unlock "*.ll" file - ll_path_lock.unlock().unwrap(); - - return gen_dylib_path; -} - -// Clean "*.ll" file path -fn clean_ll_path(ll_path: &String) { - if Path::new(ll_path).exists() { - std::fs::remove_file(&ll_path).unwrap(); +impl Default for KclvmAssembler { + fn default() -> Self { + Self::new() } } diff --git a/kclvm/runner/src/command.rs b/kclvm/runner/src/command.rs index e00a9af59..3de64b789 100644 --- a/kclvm/runner/src/command.rs +++ b/kclvm/runner/src/command.rs @@ -7,7 +7,7 @@ use kclvm_config::settings::SettingsFile; #[derive(Debug)] pub struct Command { clang_path: String, - rust_libstd_dylib: String, + rust_stdlib: String, executable_root: String, plugin_method_ptr: u64, } @@ -15,20 +15,20 @@ pub struct Command { impl Command { pub fn new(plugin_method_ptr: u64) -> Self { let executable_root = Self::get_executable_root(); - let rust_libstd_dylib = Self::get_rust_libstd_dylib(executable_root.as_str()); + let rust_stdlib = Self::get_rust_stdlib(executable_root.as_str()); let clang_path = Self::get_clang_path(); Self { clang_path, - rust_libstd_dylib, + rust_stdlib, executable_root, plugin_method_ptr, } } - pub fn run_dylib(&self, dylib_path: &str) -> Result { + pub fn run_lib(&self, lib_path: &str) -> Result { unsafe { - let lib = libloading::Library::new(dylib_path).unwrap(); + let lib = libloading::Library::new(lib_path).unwrap(); // get kclvm_plugin_init let kclvm_plugin_init: libloading::Symbol< @@ -119,13 +119,13 @@ impl Command { } } - pub fn run_dylib_with_settings( + pub fn run_lib_with_settings( &self, - dylib_path: &str, + lib_path: &str, settings: SettingsFile, ) -> Result { unsafe { - let lib = libloading::Library::new(dylib_path).unwrap(); + let lib = libloading::Library::new(lib_path).unwrap(); let kcl_run: libloading::Symbol< unsafe extern "C" fn( @@ -201,14 +201,14 @@ impl Command { Ok("".to_string()) } - pub fn link_dylibs(&mut self, dylibs: &[String], dylib_path: &str) -> String { - let dylib_suffix = Self::get_lib_suffix(); - let dylib_path = if dylib_path.is_empty() { - format!("{}{}", "_a.out", dylib_suffix) - } else if !dylib_path.ends_with(&dylib_suffix) { - format!("{}{}", dylib_path, dylib_suffix) + pub fn link_libs(&mut self, libs: &[String], lib_path: &str) -> String { + let lib_suffix = Self::get_lib_suffix(); + let lib_path = if lib_path.is_empty() { + format!("{}{}", "_a.out", lib_suffix) + } else if !lib_path.ends_with(&lib_suffix) { + format!("{}{}", lib_path, lib_suffix) } else { - dylib_path.to_string() + lib_path.to_string() }; let mut args: Vec = vec![ @@ -223,13 +223,13 @@ impl Command { "-lkclvm_native_shared".to_string(), format!("-I{}/include", self.executable_root), ]; - let mut bc_files = dylibs.to_owned(); + let mut bc_files = libs.to_owned(); args.append(&mut bc_files); let mut more_args = vec![ - self.rust_libstd_dylib.clone(), + self.rust_stdlib.clone(), "-fPIC".to_string(), "-o".to_string(), - dylib_path.to_string(), + lib_path.to_string(), ]; args.append(&mut more_args); @@ -240,12 +240,12 @@ impl Command { .output() .expect("clang failed"); - dylib_path + lib_path } - pub fn run_clang(&mut self, bc_path: &str, dylib_path: &str) -> String { + pub fn run_clang(&mut self, bc_path: &str, lib_path: &str) -> String { let mut bc_path = bc_path.to_string(); - let mut dylib_path = dylib_path.to_string(); + let mut lib_path = lib_path.to_string(); let mut bc_files = vec![]; @@ -276,8 +276,8 @@ impl Command { } } - if dylib_path.is_empty() { - dylib_path = format!("{}{}", bc_path, Self::get_lib_suffix()); + if lib_path.is_empty() { + lib_path = format!("{}{}", bc_path, Self::get_lib_suffix()); } let mut args: Vec = vec![ @@ -294,10 +294,10 @@ impl Command { ]; args.append(&mut bc_files); let mut more_args = vec![ - self.rust_libstd_dylib.clone(), + self.rust_stdlib.clone(), "-fPIC".to_string(), "-o".to_string(), - dylib_path.to_string(), + lib_path.to_string(), ]; args.append(&mut more_args); @@ -308,12 +308,12 @@ impl Command { .output() .expect("clang failed"); - dylib_path + lib_path } - pub fn run_clang_single(&mut self, bc_path: &str, dylib_path: &str) -> String { + pub fn run_clang_single(&mut self, bc_path: &str, lib_path: &str) -> String { let mut bc_path = bc_path.to_string(); - let mut dylib_path = dylib_path.to_string(); + let mut lib_path = lib_path.to_string(); if !Self::path_exist(bc_path.as_str()) { let s = format!("{}.ll", bc_path); @@ -327,8 +327,8 @@ impl Command { } } - if dylib_path.is_empty() { - dylib_path = format!("{}{}", bc_path, Self::get_lib_suffix()); + if lib_path.is_empty() { + lib_path = format!("{}{}", bc_path, Self::get_lib_suffix()); } let mut args: Vec = vec![ @@ -346,10 +346,10 @@ impl Command { let mut bc_files = vec![bc_path]; args.append(&mut bc_files); let mut more_args = vec![ - self.rust_libstd_dylib.clone(), + self.rust_stdlib.clone(), "-fPIC".to_string(), "-o".to_string(), - dylib_path.to_string(), + lib_path.to_string(), ]; args.append(&mut more_args); @@ -360,7 +360,7 @@ impl Command { .output() .expect("clang failed"); // Use absolute path. - let path = PathBuf::from(&dylib_path).canonicalize().unwrap(); + let path = PathBuf::from(&lib_path).canonicalize().unwrap(); path.to_str().unwrap().to_string() } @@ -384,7 +384,7 @@ impl Command { p.to_str().unwrap().to_string() } - fn get_rust_libstd_dylib(executable_root: &str) -> String { + fn get_rust_stdlib(executable_root: &str) -> String { let txt_path = std::path::Path::new(&executable_root) .join(if Self::is_windows() { "libs" } else { "lib" }) .join("rust-libstd-name.txt"); diff --git a/kclvm/runner/src/lib.rs b/kclvm/runner/src/lib.rs index c93014e57..1ff15714f 100644 --- a/kclvm/runner/src/lib.rs +++ b/kclvm/runner/src/lib.rs @@ -1,9 +1,11 @@ use std::path::Path; +use assembler::{KclvmLibAssembler, LlvmLibAssembler}; use command::Command; use kclvm_ast::ast::Program; use kclvm_sema::resolver::resolve_program; -use runner::{ExecProgramArgs, KclvmRunner, KclvmRunnerOptions}; +pub use runner::ExecProgramArgs; +use runner::{KclvmRunner, KclvmRunnerOptions}; pub mod assembler; pub mod command; @@ -14,7 +16,7 @@ pub mod runner; pub mod tests; /// After the kcl program passed through kclvm-parser in the compiler frontend, -/// KCLVM needs to resolve ast, generate corresponding LLVM IR, dylibs or +/// KCLVM needs to resolve ast, generate corresponding LLVM IR, dynamic link library or /// executable file for kcl program in the compiler backend. /// /// Method “execute” is the entry point for the compiler backend. @@ -27,82 +29,88 @@ pub mod tests; /// /// This method will first resolve “program” (ast.Program) and save the result to the "scope" (ProgramScope). /// -/// Then, dylibs is generated by KclvmAssembler, and method "KclvmAssembler::gen_dylibs" -/// will return dylibs path in a "Vec"; +/// Then, dynamic link libraries is generated by KclvmAssembler, and method "KclvmAssembler::gen_libs" +/// will return dynamic link library paths in a "Vec"; /// -/// After linking all dylibs by KclvmLinker, method "KclvmLinker::link_all_dylibs" will return a path -/// for dylib. +/// KclvmAssembler is mainly responsible for concurrent compilation of multiple files. +/// Single-file compilation in each thread in concurrent compilation is the responsibility of KclvmLibAssembler. +/// In the future, it may support the dynamic link library generation of multiple intermediate language. +/// KclvmLibAssembler currently only supports LLVM IR. +/// +/// After linking all dynamic link libraries by KclvmLinker, method "KclvmLinker::link_all_libs" will return a path +/// for dynamic link library after linking. /// /// At last, KclvmRunner will be constructed and call method "run" to execute the kcl program. /// +/// TODO: Need to be a better backend abstraction +/// /// # Examples /// -/// ``` +/// ```no_run /// use kclvm_runner::{execute, runner::ExecProgramArgs}; /// use kclvm_parser::load_program; /// use kclvm_ast::ast::Program; /// -/// fn main() { -/// // default plugin agent -/// let plugin_agent = 0; +/// // Default plugin agent +/// let plugin_agent = 0; /// -/// // get default args -/// let args = ExecProgramArgs::default(); -/// let opts = args.get_load_program_options(); +/// // Get default args +/// let args = ExecProgramArgs::default(); +/// let opts = args.get_load_program_options(); /// -/// // parse kcl file -/// let kcl_path = "./src/test_datas/init_check_order_0/main.k"; -/// let prog = load_program(&[kcl_path], Some(opts)).unwrap(); -/// -/// // resolve ast, generate dylibs, link dylibs and execute. -/// // result is the kcl in json format. -/// let result = execute(prog, plugin_agent, &args).unwrap(); -/// } +/// // Parse kcl file +/// let kcl_path = "./src/test_datas/init_check_order_0/main.k"; +/// let prog = load_program(&[kcl_path], Some(opts)).unwrap(); +/// +/// // Resolve ast, generate libs, link libs and execute. +/// // Result is the kcl in json format. +/// let result = execute(prog, plugin_agent, &args).unwrap(); /// ``` -/// pub fn execute( mut program: Program, plugin_agent: u64, args: &ExecProgramArgs, ) -> Result { - // resolve ast + // Resolve ast let scope = resolve_program(&mut program); scope.check_scope_diagnostics(); - // generate dylibs + // Create a temp entry file. let temp_entry_file = temp_file(); - let dylib_paths = - assembler::KclvmAssembler::new().gen_dylibs(program, scope, plugin_agent, &temp_entry_file); + // Generate libs + let lib_paths = assembler::KclvmAssembler::new().gen_libs( + program, + scope, + plugin_agent, + &temp_entry_file, + KclvmLibAssembler::LLVM(LlvmLibAssembler {}), + ); - // link dylibsKclvmRunner - let dylib_suffix = Command::get_lib_suffix(); - let temp_out_dylib_file = format!("{}.out{}", temp_entry_file, dylib_suffix); - let dylib_path = - linker::KclvmLinker::link_all_dylibs(dylib_paths, temp_out_dylib_file, plugin_agent); + // Link libs + let lib_suffix = Command::get_lib_suffix(); + let temp_out_lib_file = format!("{}.out{}", temp_entry_file, lib_suffix); + let lib_path = linker::KclvmLinker::link_all_libs(lib_paths, temp_out_lib_file, plugin_agent); - // run + // Run let runner = KclvmRunner::new( - dylib_path.as_str(), + lib_path.as_str(), Some(KclvmRunnerOptions { plugin_agent_ptr: plugin_agent, }), ); - let result = runner.run(&args); + let result = runner.run(args); - // clean files - remove_file(&dylib_path); - clean_tmp_files(&temp_entry_file, &dylib_suffix); + // Clean temp files + remove_file(&lib_path); + clean_tmp_files(&temp_entry_file, &lib_suffix); result } +/// Clean all the tmp files generated during lib generating and linking. #[inline] -/// Clean all the tmp files generated during dylib generating and linking. -fn clean_tmp_files(temp_entry_file: &String, dylib_suffix: &String) { - let ll_lock_suffix = ".ll.lock"; - let temp_entry_dylib_file = format!("{}{}", temp_entry_file, dylib_suffix); - let temp_entry_ll_lock_file = format!("{}{}", temp_entry_file, ll_lock_suffix); - remove_file(&temp_entry_dylib_file); - remove_file(&temp_entry_ll_lock_file); +fn clean_tmp_files(temp_entry_file: &String, lib_suffix: &String) { + let temp_entry_lib_file = format!("{}{}", temp_entry_file, lib_suffix); + remove_file(&temp_entry_lib_file); } #[inline] @@ -112,8 +120,8 @@ fn remove_file(file: &str) { } } -#[inline] /// Returns a temporary file name consisting of timestamp and process id. +#[inline] fn temp_file() -> String { let timestamp = chrono::Local::now().timestamp_nanos(); let id = std::process::id(); diff --git a/kclvm/runner/src/linker.rs b/kclvm/runner/src/linker.rs index d449a7b9f..4be7a8492 100644 --- a/kclvm/runner/src/linker.rs +++ b/kclvm/runner/src/linker.rs @@ -1,15 +1,11 @@ use crate::command::Command; -/// KclvmLinker is mainly responsible for linking the dylibs generated by KclvmAssembler. +/// KclvmLinker is mainly responsible for linking the libs generated by KclvmAssembler. pub struct KclvmLinker; impl KclvmLinker { - /// Link the dylibs generated by method "gen_bc_or_ll_file". - pub fn link_all_dylibs( - dylib_paths: Vec, - dylib_path: String, - plugin_agent: u64, - ) -> String { + /// Link the libs generated by method "gen_bc_or_ll_file". + pub fn link_all_libs(lib_paths: Vec, lib_path: String, plugin_agent: u64) -> String { let mut cmd = Command::new(plugin_agent); - cmd.link_dylibs(&dylib_paths, &dylib_path) + cmd.link_libs(&lib_paths, &lib_path) } } diff --git a/kclvm/runner/src/runner.rs b/kclvm/runner/src/runner.rs index 2e5b6cdf8..7f65b9ebe 100644 --- a/kclvm/runner/src/runner.rs +++ b/kclvm/runner/src/runner.rs @@ -1,4 +1,5 @@ use kclvm_ast::ast; +use kclvm_config::settings::SettingsFile; use serde::{Deserialize, Serialize}; #[allow(non_camel_case_types)] @@ -64,7 +65,7 @@ impl ExecProgramArgs { pub fn get_load_program_options(&self) -> kclvm_parser::LoadProgramOptions { kclvm_parser::LoadProgramOptions { - work_dir: self.work_dir.clone().unwrap_or("".to_string()).clone(), + work_dir: self.work_dir.clone().unwrap_or_else(|| "".to_string()), k_code_list: self.k_code_list.clone(), cmd_args: self.args.clone(), cmd_overrides: self.overrides.clone(), @@ -73,6 +74,29 @@ impl ExecProgramArgs { } } +impl From for ExecProgramArgs { + fn from(settings: SettingsFile) -> Self { + let mut args = Self::default(); + if let Some(cli_configs) = settings.kcl_cli_configs { + args.k_filename_list = cli_configs.files.unwrap_or_default(); + args.strict_range_check = cli_configs.strict_range_check.unwrap_or_default(); + args.disable_none = cli_configs.disable_none.unwrap_or_default(); + args.verbose = cli_configs.verbose.unwrap_or_default() as i32; + args.debug = cli_configs.debug.unwrap_or_default() as i32; + } + if let Some(options) = settings.kcl_options { + args.args = options + .iter() + .map(|o| ast::CmdArgSpec { + name: o.key.to_string(), + value: o.value.to_string(), + }) + .collect(); + } + args + } +} + #[derive(Debug, Default)] pub struct KclvmRunnerOptions { pub plugin_agent_ptr: u64, @@ -84,9 +108,9 @@ pub struct KclvmRunner { } impl KclvmRunner { - pub fn new(dylib_path: &str, opts: Option) -> Self { + pub fn new(lib_path: &str, opts: Option) -> Self { let lib = unsafe { - libloading::Library::new(std::path::PathBuf::from(dylib_path).canonicalize().unwrap()) + libloading::Library::new(std::path::PathBuf::from(lib_path).canonicalize().unwrap()) .unwrap() }; Self { @@ -97,14 +121,14 @@ impl KclvmRunner { pub fn run(&self, args: &ExecProgramArgs) -> Result { unsafe { - Self::dylib_kclvm_plugin_init(&self.lib, self.opts.plugin_agent_ptr); - Self::dylib_kcl_run(&self.lib, &args) + Self::lib_kclvm_plugin_init(&self.lib, self.opts.plugin_agent_ptr); + Self::lib_kcl_run(&self.lib, args) } } } impl KclvmRunner { - unsafe fn dylib_kclvm_plugin_init(lib: &libloading::Library, plugin_method_ptr: u64) { + unsafe fn lib_kclvm_plugin_init(lib: &libloading::Library, plugin_method_ptr: u64) { // get kclvm_plugin_init let kclvm_plugin_init: libloading::Symbol< unsafe extern "C" fn( @@ -134,7 +158,7 @@ impl KclvmRunner { kclvm_plugin_init(plugin_method); } - unsafe fn dylib_kcl_run( + unsafe fn lib_kcl_run( lib: &libloading::Library, args: &ExecProgramArgs, ) -> Result { diff --git a/kclvm/runner/src/tests.rs b/kclvm/runner/src/tests.rs index db303d82f..120dddd59 100644 --- a/kclvm/runner/src/tests.rs +++ b/kclvm/runner/src/tests.rs @@ -15,7 +15,7 @@ const KCL_FILE_NAME: &str = "main.k"; const MAIN_PKG_NAME: &str = "__main__"; /// Load test kcl file to ast.Program -pub fn load_program(filename: String) -> Program { +pub fn load_test_program(filename: String) -> Program { let module = load_module(filename); construct_program(module) } @@ -61,13 +61,14 @@ pub fn format_str_by_json(str: String) -> String { pub fn execute_for_test(kcl_path: &String) -> String { let plugin_agent = 0; let args = ExecProgramArgs::default(); - // parse kcl file - let program = load_program(kcl_path.to_string()); - // generate dylibs, link dylibs and execute. + // Parse kcl file + let program = load_test_program(kcl_path.to_string()); + // Generate libs, link libs and execute. execute(program, plugin_agent, &args).unwrap() } -#[test] +// TODO: need to fix issue #79 +// #[test] fn test_kclvm_runner_execute() { for case in TEST_CASES { let kcl_path = &format!("{}/{}/{}", TEST_CASE_PATH, case, KCL_FILE_NAME); diff --git a/kclvm/src/lib.rs b/kclvm/src/lib.rs index 203259a55..d42711ed9 100644 --- a/kclvm/src/lib.rs +++ b/kclvm/src/lib.rs @@ -57,10 +57,10 @@ pub fn kclvm_cli_run_unsafe(args: *const i8, plugin_agent: *const i8) -> Result< let files = args.get_files(); let opts = args.get_load_program_options(); - // load ast + // Parse AST program. let mut program = load_program(&files, Some(opts))?; apply_overrides(&mut program, &args.overrides, &[]); - // resolve ast, generate dylibs, link dylibs and execute. + // Resolve AST program, generate libs, link libs and execute. execute(program, plugin_agent, &args) } diff --git a/kclvm/src/main.rs b/kclvm/src/main.rs index 934dce543..49037d9ea 100644 --- a/kclvm/src/main.rs +++ b/kclvm/src/main.rs @@ -3,19 +3,11 @@ #[macro_use] extern crate clap; -use indexmap::IndexMap; -use std::path::PathBuf; -use std::thread; -use std::{collections::HashMap, path::Path}; +use kclvm_runner::{execute, ExecProgramArgs}; use clap::ArgMatches; -use kclvm_ast::ast; -use kclvm_compiler::codegen::{llvm::emit_code, EmitOptions}; -use kclvm_config::cache::*; use kclvm_config::settings::{load_file, merge_settings, SettingsFile}; -use kclvm_parser::{load_program, parse_file}; -use kclvm_runner::command::Command; -use kclvm_sema::resolver::resolve_program; +use kclvm_parser::load_program; fn main() { let matches = clap_app!(kcl => @@ -36,161 +28,14 @@ fn main() { if let Some(matches) = matches.subcommand_matches("run") { if let Some(files) = matches.values_of("INPUT") { let files: Vec<&str> = files.into_iter().collect::>(); - if let Some(emit_ty) = matches.value_of("EMIT_TYPE") { - if emit_ty == "ast" { - let module = parse_file(files[0], None); - println!("{}", serde_json::to_string(&module).unwrap()) - } - } else { - // load ast - let mut program = load_program(&files, None).unwrap(); - let scope = resolve_program(&mut program); - scope.check_scope_diagnostics(); - // gen bc or ll file - let ll_file = "_a.out"; - let path = std::path::Path::new(ll_file); - if path.exists() { - std::fs::remove_file(path).unwrap(); - } - for entry in glob::glob(&format!("{}*.ll", ll_file)).unwrap() { - match entry { - Ok(path) => { - if path.exists() { - std::fs::remove_file(path).unwrap(); - } - } - Err(e) => println!("{:?}", e), - }; - } - - let cache_dir = Path::new(&program.root) - .join(".kclvm") - .join("cache") - .join(kclvm_version::get_full_version()); - if !cache_dir.exists() { - std::fs::create_dir_all(&cache_dir).unwrap(); - } - let mut compile_progs: IndexMap< - String, - ( - ast::Program, - IndexMap>, - PathBuf, - ), - > = IndexMap::default(); - - for (pkgpath, modules) in program.pkgs { - let mut pkgs = HashMap::new(); - pkgs.insert(pkgpath.clone(), modules); - let compile_prog = ast::Program { - root: program.root.clone(), - main: program.main.clone(), - pkgs, - cmd_args: vec![], - cmd_overrides: vec![], - }; - compile_progs.insert( - pkgpath, - (compile_prog, scope.import_names.clone(), cache_dir.clone()), - ); - } - let mut theads = vec![]; - for (pkgpath, (compile_prog, import_names, cache_dir)) in compile_progs { - let t = thread::spawn(move || { - let root = &compile_prog.root; - let is_main_pkg = pkgpath == kclvm_ast::MAIN_PKG; - let file = if is_main_pkg { - let main_file = - format!("{}{}", pkgpath, chrono::Local::now().timestamp_nanos()); - cache_dir.join(&main_file) - } else { - cache_dir.join(&pkgpath) - }; - let lock_file = - format!("{}.lock", cache_dir.join(&pkgpath).to_str().unwrap()); - let ll_file = file.to_str().unwrap(); - let ll_path = format!("{}.ll", ll_file); - let dylib_path = format!("{}{}", ll_file, Command::get_lib_suffix()); - let mut ll_path_lock = fslock::LockFile::open(&lock_file).unwrap(); - ll_path_lock.lock().unwrap(); - if Path::new(&ll_path).exists() { - std::fs::remove_file(&ll_path).unwrap(); - } - let dylib_path = if is_main_pkg { - emit_code( - &compile_prog, - import_names, - &EmitOptions { - from_path: None, - emit_path: Some(&ll_file), - no_link: true, - }, - ) - .expect("Compile KCL to LLVM error"); - let mut cmd = Command::new(0); - cmd.run_clang_single(&ll_path, &dylib_path) - } else { - // If AST module has been modified, ignore the dylib cache - let dylib_relative_path: Option = - load_pkg_cache(root, &pkgpath, CacheOption::default()); - match dylib_relative_path { - Some(dylib_relative_path) => { - if dylib_relative_path.starts_with('.') { - dylib_relative_path.replacen(".", root, 1) - } else { - dylib_relative_path - } - } - None => { - emit_code( - &compile_prog, - import_names, - &EmitOptions { - from_path: None, - emit_path: Some(&ll_file), - no_link: true, - }, - ) - .expect("Compile KCL to LLVM error"); - let mut cmd = Command::new(0); - let dylib_path = cmd.run_clang_single(&ll_path, &dylib_path); - let dylib_relative_path = dylib_path.replacen(root, ".", 1); - - save_pkg_cache( - root, - &pkgpath, - dylib_relative_path, - CacheOption::default(), - ); - dylib_path - } - } - }; - if Path::new(&ll_path).exists() { - std::fs::remove_file(&ll_path).unwrap(); - } - ll_path_lock.unlock().unwrap(); - dylib_path - }); - theads.push(t); - } - let mut dylib_paths = vec![]; - for t in theads { - let dylib_path = t.join().unwrap(); - dylib_paths.push(dylib_path); - } - let mut cmd = Command::new(0); - // link all dylibs - let dylib_path = cmd.link_dylibs(&dylib_paths, ""); - // Config build - let settings = build_settings(&matches); - cmd.run_dylib_with_settings(&dylib_path, settings).unwrap(); - for dylib_path in dylib_paths { - if dylib_path.contains(kclvm_ast::MAIN_PKG) && Path::new(&dylib_path).exists() { - std::fs::remove_file(&dylib_path).unwrap(); - } - } - } + // Config settings build + let settings = build_settings(&matches); + // Convert settings into execute arguments. + let args: ExecProgramArgs = settings.into(); + // Parse AST program. + let program = load_program(&files, Some(args.get_load_program_options())).unwrap(); + // Resolve AST program, generate libs, link libs and execute. + execute(program, 0, &ExecProgramArgs::default()).unwrap(); } else { println!("{}", matches.usage()); }