From fb33832a02e519775126d7d70423b65589f8588a Mon Sep 17 00:00:00 2001 From: afinch7 Date: Tue, 16 Jul 2019 13:11:35 -0400 Subject: [PATCH] Feature: Native plugins --- .travis.yml | 5 +- Cargo.lock | 33 ++++++ Cargo.toml | 1 + build_extra/rust/BUILD.gn | 58 +++++++++++ build_extra/rust/rust.gni | 17 ++++ cli/BUILD.gn | 2 + cli/Cargo.toml | 1 + cli/deno_error.rs | 13 +++ cli/dispatch_minimal.rs | 4 +- cli/flags.rs | 13 ++- cli/msg.fbs | 36 +++++++ cli/ops.rs | 113 +++++++++++++++++++++ cli/permissions.rs | 32 ++++++ cli/resources.rs | 51 ++++++++++ cli/state.rs | 5 + core/lib.rs | 1 + core/plugins.rs | 16 +++ js/deno.ts | 7 ++ js/dispatch.ts | 51 +++++++--- js/native_plugins.ts | 171 ++++++++++++++++++++++++++++++++ tests/034_plugin.out | 4 + tests/034_plugin.test | 2 + tests/034_plugin.ts | 67 +++++++++++++ tests/plugin/Cargo.toml | 15 +++ tests/plugin/lib.rs | 43 ++++++++ third_party | 2 +- tools/build.py | 9 +- tools/format.py | 6 +- tools/permission_prompt_test.py | 4 + tools/permission_prompt_test.ts | 57 +++++++++-- tools/test_util.py | 2 + 31 files changed, 804 insertions(+), 37 deletions(-) create mode 100644 core/plugins.rs create mode 100644 js/native_plugins.ts create mode 100644 tests/034_plugin.out create mode 100644 tests/034_plugin.test create mode 100644 tests/034_plugin.ts create mode 100644 tests/plugin/Cargo.toml create mode 100644 tests/plugin/lib.rs diff --git a/.travis.yml b/.travis.yml index 8157e153e16f87..da57c140144c11 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,7 +9,6 @@ env: - CARGO_HOME=$TRAVIS_BUILD_DIR/third_party/rust_crates/ - RUSTUP_HOME=$HOME/.rustup/ - RUST_BACKTRACE=full - - CARGO_TARGET_DIR=$HOME/target - PATH=$TRAVIS_BUILD_DIR/third_party/llvm-build/Release+Asserts/bin:$CARGO_HOME/bin:$PATH - PYTHONPATH=third_party/python_packages - RUSTC_WRAPPER=sccache @@ -72,8 +71,8 @@ before_script: script: - ./tools/lint.py - ./tools/test_format.py -- ./tools/build.py -C target/release -- DENO_BUILD_MODE=release ./tools/test.py +- ./tools/build.py --release -C target/release +- DENO_BUILD_MODE=release ./tools/test.py -v jobs: fast_finish: true diff --git a/Cargo.lock b/Cargo.lock index 6a6c57939bc8d6..a0130b410d4d5e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -217,6 +217,7 @@ dependencies = [ "clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)", "deno 0.11.0", "dirs 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)", + "dlopen 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", "flatbuffers 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", "futures 0.1.27 (registry+https://github.com/rust-lang/crates.io-index)", "fwdansi 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -282,6 +283,28 @@ dependencies = [ "winapi 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "dlopen" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "dlopen_derive 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", + "kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.58 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "dlopen_derive" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.58 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 0.15.36 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "either" version = "1.5.2" @@ -1101,6 +1124,14 @@ dependencies = [ "redox_termios 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "test_plugin" +version = "0.1.0" +dependencies = [ + "deno 0.11.0", + "futures 0.1.27 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "textwrap" version = "0.11.0" @@ -1559,6 +1590,8 @@ dependencies = [ "checksum dirs 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)" = "3fd78930633bd1c6e35c4b42b1df7b0cbc6bc191146e512bb3bedf243fcc3901" "checksum dirs 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "1c4ef5a8b902d393339e2a2c7fe573af92ce7e0ee5a3ff827b4c9ad7e07e4fa1" "checksum dirs-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "937756392ec77d1f2dd9dc3ac9d69867d109a2121479d72c364e42f4cab21e2d" +"checksum dlopen 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "e0b1f4b2b1d3bc2f8edb3398c7b7ca5ee6d255ca2e8a5c0cf9d730b7a678de74" +"checksum dlopen_derive 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "f236d9e1b1fbd81cea0f9cbdc8dcc7e8ebcd80e6659cd7cb2ad5f6c05946c581" "checksum either 1.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "5527cfe0d098f36e3f8839852688e63c8fff1c90b2b405aef730615f9a7bcf7b" "checksum failure 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "795bd83d3abeb9220f257e597aa0080a508b27533824adf336529648f6abf7e2" "checksum failure_derive 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "ea1063915fd7ef4309e222a5a07cf9c319fb9c7836b1f89b85458672dbb127e1" diff --git a/Cargo.toml b/Cargo.toml index 1aa1732857beb1..61611489c56e05 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,4 +3,5 @@ members = [ "cli", "core", "tools/hyper_hello", + "tests/plugin", ] diff --git a/build_extra/rust/BUILD.gn b/build_extra/rust/BUILD.gn index 8151eb89fa2009..ea7ecba246fbd9 100644 --- a/build_extra/rust/BUILD.gn +++ b/build_extra/rust/BUILD.gn @@ -256,6 +256,51 @@ rust_rlib("dirs_sys") { } } +rust_proc_macro("dlopen_derive") { + edition = "2015" + source_root = "$cargo_home/registry/src/github.com-1ecc6299db9ec823/dlopen_derive-0.1.4/src/lib.rs" + args = [ + "--cap-lints", + "allow", + ] + extern_rlib = [ + "syn", + "quote", + "libc", + ] +} + +rust_rlib("dlopen") { + edition = "2015" + source_root = "$cargo_home/registry/src/github.com-1ecc6299db9ec823/dlopen-0.1.7/src/lib.rs" + args = [ + "--cap-lints", + "allow", + ] + extern = [ + { + label = ":dlopen_derive" + crate_name = "dlopen_derive" + crate_type = "proc_macro" + }, + ] + extern_rlib = [ "lazy_static" ] + if (is_posix) { + extern_rlib += [ "libc" ] + } + if (is_win) { + extern += [ + { + label = ":winapi-0.2.8" + crate_name = "winapi" + crate_type = "rlib" + crate_version = "0.2.8" + }, + ] + extern_rlib += [ "kernel32" ] + } +} + rust_rlib("either") { edition = "2015" source_root = "$cargo_home/registry/src/github.com-1ecc6299db9ec823/either-1.5.2/src/lib.rs" @@ -579,6 +624,10 @@ rust_rlib("log") { edition = "2015" source_root = "$cargo_home/registry/src/github.com-1ecc6299db9ec823/log-0.4.6/src/lib.rs" extern_rlib = [ "cfg_if" ] + features = [ + "default", + "std", + ] args = [ "--cap-lints", "allow", @@ -1267,11 +1316,20 @@ rust_rlib("serde") { features = [ "default", "std", + "derive", + "serde_derive", ] args = [ "--cap-lints", "allow", ] + extern = [ + { + label = ":serde_derive" + crate_name = "serde_derive" + crate_type = "proc_macro" + }, + ] # Added by custom-build script. cfg = [ diff --git a/build_extra/rust/rust.gni b/build_extra/rust/rust.gni index b4c128e710ee39..9b84648bbabe75 100644 --- a/build_extra/rust/rust.gni +++ b/build_extra/rust/rust.gni @@ -122,6 +122,9 @@ template("_rust_crate") { } else if (crate_type == "rlib") { out_file = "lib$crate_name$crate_suffix.rlib" emit_type = "link" + } else if (crate_type == "dylib" || crate_type == "cdylib") { + out_file = "$shared_lib_prefix$crate_name$crate_suffix$shared_lib_suffix" + emit_type = "link" } out_path = "$out_dir/$out_file" @@ -308,6 +311,20 @@ template("rust_rlib") { } } +template("rust_dylib") { + _rust_crate(target_name) { + forward_variables_from(invoker, "*") + crate_type = "dylib" + } +} + +template("rust_cdylib") { + _rust_crate(target_name) { + forward_variables_from(invoker, "*") + crate_type = "cdylib" + } +} + template("rust_proc_macro") { _rust_crate(target_name) { forward_variables_from(invoker, "*") diff --git a/cli/BUILD.gn b/cli/BUILD.gn index 7f19f8b952662a..c9121bc5c7f88b 100644 --- a/cli/BUILD.gn +++ b/cli/BUILD.gn @@ -24,6 +24,7 @@ main_extern_rlib = [ "atty", "clap", "dirs", + "dlopen", "flatbuffers", "futures", "fwdansi", @@ -105,6 +106,7 @@ ts_sources = [ "../js/metrics.ts", "../js/mkdir.ts", "../js/mock_builtin.js", + "../js/native_plugins.ts", "../js/net.ts", "../js/os.ts", "../js/permissions.ts", diff --git a/cli/Cargo.toml b/cli/Cargo.toml index dfcafc21583749..b05a929b16570d 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -22,6 +22,7 @@ ansi_term = "0.11.0" atty = "0.2.11" clap = "2.33.0" dirs = "2.0.1" +dlopen = "0.1.7" flatbuffers = "0.6.0" futures = "0.1.27" http = "0.1.17" diff --git a/cli/deno_error.rs b/cli/deno_error.rs index 544a411bac9dbc..94e6f2f2206c05 100644 --- a/cli/deno_error.rs +++ b/cli/deno_error.rs @@ -6,6 +6,7 @@ pub use crate::msg::ErrorKind; use deno::AnyError; use deno::ErrBox; use deno::ModuleResolutionError; +use dlopen::Error as DlopenError; use http::uri; use hyper; use rustyline::error::ReadlineError; @@ -199,6 +200,17 @@ impl GetErrorKind for ReadlineError { } } +impl GetErrorKind for DlopenError { + fn kind(&self) -> ErrorKind { + match self { + DlopenError::NullCharacter(_) => ErrorKind::PluginNullCharacter, + DlopenError::OpeningLibraryError(io_err) + | DlopenError::SymbolGettingError(io_err) => GetErrorKind::kind(io_err), + DlopenError::NullSymbol => ErrorKind::PluginNullSymbol, + } + } +} + #[cfg(unix)] mod unix { use super::{ErrorKind, GetErrorKind}; @@ -245,6 +257,7 @@ impl GetErrorKind for dyn AnyError { .or_else(|| self.downcast_ref::().map(Get::kind)) .or_else(|| self.downcast_ref::().map(Get::kind)) .or_else(|| self.downcast_ref::().map(Get::kind)) + .or_else(|| self.downcast_ref::().map(Get::kind)) .or_else(|| unix_error_kind(self)) .unwrap_or_else(|| { panic!("Can't get ErrorKind for {:?}", self); diff --git a/cli/dispatch_minimal.rs b/cli/dispatch_minimal.rs index 9d82595d15f652..567ed09f7de137 100644 --- a/cli/dispatch_minimal.rs +++ b/cli/dispatch_minimal.rs @@ -117,9 +117,9 @@ pub fn dispatch_minimal( Ok(buf) })); if is_sync { - Op::Sync(fut.wait().unwrap()) + Op::Sync(fut.wait().unwrap()) as CoreOp } else { - Op::Async(fut) + Op::Async(fut) as CoreOp } } diff --git a/cli/flags.rs b/cli/flags.rs index 704ef1f566e071..6f8db4755563d1 100644 --- a/cli/flags.rs +++ b/cli/flags.rs @@ -37,6 +37,7 @@ pub struct DenoFlags { pub allow_env: bool, pub allow_run: bool, pub allow_hrtime: bool, + pub allow_plugins: bool, pub no_prompts: bool, pub no_fetch: bool, pub seed: Option, @@ -87,6 +88,10 @@ fn add_run_args<'a, 'b>(app: App<'a, 'b>) -> App<'a, 'b> { Arg::with_name("allow-hrtime") .long("allow-hrtime") .help("Allow high resolution time measurement"), + ).arg( + Arg::with_name("allow-plugins") + .long("allow-plugins") + .help("Allow loading native plugins via dlopen"), ).arg( Arg::with_name("allow-all") .short("A") @@ -540,14 +545,17 @@ fn parse_run_args(mut flags: DenoFlags, matches: &ArgMatches) -> DenoFlags { if matches.is_present("allow-hrtime") { flags.allow_hrtime = true; } + if matches.is_present("allow-plugins") { + flags.allow_plugins = true; + } if matches.is_present("allow-all") { flags.allow_read = true; + flags.allow_write = true; flags.allow_env = true; flags.allow_net = true; flags.allow_run = true; - flags.allow_read = true; - flags.allow_write = true; flags.allow_hrtime = true; + flags.allow_plugins = true; } if matches.is_present("no-prompt") { flags.no_prompts = true; @@ -960,6 +968,7 @@ mod tests { allow_read: true, allow_write: true, allow_hrtime: true, + allow_plugins: true, ..DenoFlags::default() } ); diff --git a/cli/msg.fbs b/cli/msg.fbs index 9b531147dfd3fc..abfe774d4f174d 100644 --- a/cli/msg.fbs +++ b/cli/msg.fbs @@ -11,6 +11,12 @@ union Any { Cwd, CwdRes, Dial, + PluginOpen, + PluginOpenRes, + PluginSym, + PluginSymRes, + PluginCall, + PluginCallRes, Environ, EnvironRes, Exit, @@ -148,6 +154,10 @@ enum ErrorKind: byte { // other kinds Diagnostic, JSError, + + // dlopen errors + PluginNullSymbol, + PluginNullCharacter, } table Cwd {} @@ -598,4 +608,30 @@ table Seek { table GetRandomValues {} +table PluginOpen { + filename: string; +} + +table PluginOpenRes { + rid: uint32; +} + +table PluginSym { + rid: uint32; + name: string; +} + +table PluginSymRes { + rid: uint32; +} + +table PluginCall { + rid: uint32; + data: [ubyte]; +} + +table PluginCallRes { + data: [ubyte]; +} + root_type Base; diff --git a/cli/ops.rs b/cli/ops.rs index 018d2ea09082d5..ed934785264ca3 100644 --- a/cli/ops.rs +++ b/cli/ops.rs @@ -203,6 +203,9 @@ pub fn op_selector_std(inner_type: msg::Any) -> Option { msg::Any::CreateWorker => Some(op_create_worker), msg::Any::Cwd => Some(op_cwd), msg::Any::Dial => Some(op_dial), + msg::Any::PluginOpen => Some(op_plugin_open), + msg::Any::PluginSym => Some(op_plugin_sym), + msg::Any::PluginCall => Some(op_plugin_call), msg::Any::Environ => Some(op_env), msg::Any::Exit => Some(op_exit), msg::Any::Fetch => Some(op_fetch), @@ -2187,3 +2190,113 @@ fn op_get_random_values( ok_buf(empty_buf()) } + +fn op_plugin_open( + state: &ThreadSafeState, + base: &msg::Base<'_>, + _data: Option, +) -> CliOpResult { + let cmd_id = base.cmd_id(); + let inner = base.inner_as_plugin_open().unwrap(); + let (filename, filename_) = resolve_from_cwd(inner.filename().unwrap())?; + + state.check_plugins(&filename_)?; + + let lib = resources::add_plugin(filename)?; + + let builder = &mut FlatBufferBuilder::new(); + let msg_inner = msg::PluginOpenRes::create( + builder, + &msg::PluginOpenResArgs { rid: lib.rid }, + ); + + ok_buf(serialize_response( + cmd_id, + builder, + msg::BaseArgs { + inner: Some(msg_inner.as_union_value()), + inner_type: msg::Any::PluginOpenRes, + ..Default::default() + }, + )) +} + +fn op_plugin_sym( + _state: &ThreadSafeState, + base: &msg::Base<'_>, + _data: Option, +) -> CliOpResult { + let cmd_id = base.cmd_id(); + let inner = base.inner_as_plugin_sym().unwrap(); + let rid = inner.rid(); + let name = inner.name().unwrap(); + + let fun = resources::add_plugin_op(rid, name)?; + + let builder = &mut FlatBufferBuilder::new(); + let msg_inner = + msg::PluginSymRes::create(builder, &msg::PluginSymResArgs { rid: fun.rid }); + + ok_buf(serialize_response( + cmd_id, + builder, + msg::BaseArgs { + inner: Some(msg_inner.as_union_value()), + inner_type: msg::Any::PluginSymRes, + ..Default::default() + }, + )) +} + +fn op_plugin_call( + _state: &ThreadSafeState, + base: &msg::Base<'_>, + data: Option, +) -> CliOpResult { + let cmd_id = base.cmd_id(); + let inner = base.inner_as_plugin_call().unwrap(); + let rid = inner.rid(); + + let result = resources::call_plugin_op(rid, inner.data().unwrap(), data)?; + + match result { + Op::Sync(buf) => { + let builder = &mut FlatBufferBuilder::new(); + let data = Some(builder.create_vector(&buf)); + let msg_inner = + msg::PluginCallRes::create(builder, &msg::PluginCallResArgs { data }); + + ok_buf(serialize_response( + cmd_id, + builder, + msg::BaseArgs { + inner: Some(msg_inner.as_union_value()), + inner_type: msg::Any::PluginCallRes, + ..Default::default() + }, + )) + } + Op::Async(result) => Ok(Op::Async(Box::new( + result + .map_err(|_| panic!("Plugin op returned error future.")) + .and_then(move |buf| { + let builder = &mut FlatBufferBuilder::new(); + let data = Some(builder.create_vector(&buf)); + let msg_inner = msg::PluginCallRes::create( + builder, + &msg::PluginCallResArgs { data }, + ); + + Ok(serialize_response( + cmd_id, + builder, + msg::BaseArgs { + inner: Some(msg_inner.as_union_value()), + inner_type: msg::Any::PluginCallRes, + ..Default::default() + }, + )) + }), + ))), + } +} diff --git a/cli/permissions.rs b/cli/permissions.rs index 8549e97795bdeb..5380845cf329ca 100644 --- a/cli/permissions.rs +++ b/cli/permissions.rs @@ -137,6 +137,8 @@ pub struct DenoPermissions { pub allow_env: PermissionAccessor, pub allow_run: PermissionAccessor, pub allow_hrtime: PermissionAccessor, + // TODO(afinch7) maybe add permissions whitelist for this? + pub allow_plugins: PermissionAccessor, pub no_prompts: AtomicBool, } @@ -154,6 +156,7 @@ impl DenoPermissions { allow_env: PermissionAccessor::from(flags.allow_env), allow_run: PermissionAccessor::from(flags.allow_run), allow_hrtime: PermissionAccessor::from(flags.allow_hrtime), + allow_plugins: PermissionAccessor::from(flags.allow_plugins), no_prompts: AtomicBool::new(flags.no_prompts), } } @@ -347,6 +350,26 @@ impl DenoPermissions { } } + pub fn check_plugins(&self, filename: &str) -> Result<(), ErrBox> { + let msg = &format!("access to native plugin {}", filename); + match self.allow_plugins.get_state() { + PermissionAccessorState::Allow => { + self.log_perm_access(msg); + Ok(()) + } + PermissionAccessorState::Ask => match self.try_permissions_prompt(msg) { + Err(e) => Err(e), + Ok(v) => { + self.allow_plugins.update_with_prompt_result(&v); + v.check()?; + self.log_perm_access(msg); + Ok(()) + } + }, + PermissionAccessorState::Deny => Err(permission_denied()), + } + } + /// Try to present the user with a permission prompt /// will error with permission_denied if no_prompts is enabled fn try_permissions_prompt( @@ -397,6 +420,10 @@ impl DenoPermissions { self.allow_hrtime.is_allow() } + pub fn allows_dlopen(&self) -> bool { + self.allow_plugins.is_allow() + } + pub fn revoke_run(&self) -> Result<(), ErrBox> { self.allow_run.revoke(); Ok(()) @@ -425,6 +452,11 @@ impl DenoPermissions { self.allow_hrtime.revoke(); Ok(()) } + + pub fn revoke_plugins(&self) -> Result<(), ErrBox> { + self.allow_plugins.revoke(); + Ok(()) + } } /// Quad-state value for representing user input on permission prompt diff --git a/cli/resources.rs b/cli/resources.rs index 9ef12a4293de65..b8804f324d79f4 100644 --- a/cli/resources.rs +++ b/cli/resources.rs @@ -14,9 +14,13 @@ use crate::http_body::HttpBody; use crate::repl::Repl; use crate::state::WorkerChannels; +use deno::plugins::PluginDispatchFn; use deno::Buf; +use deno::CoreOp; use deno::ErrBox; +use deno::PinnedBuf; +use dlopen::symbor::Library; use futures; use futures::Future; use futures::Poll; @@ -25,6 +29,7 @@ use futures::Stream; use hyper; use std; use std::collections::HashMap; +use std::ffi::OsStr; use std::io::{Error, Read, Seek, SeekFrom, Write}; use std::net::{Shutdown, SocketAddr}; use std::process::ExitStatus; @@ -99,6 +104,8 @@ enum Repr { ChildStdout(tokio_process::ChildStdout), ChildStderr(tokio_process::ChildStderr), Worker(WorkerChannels), + Plugin(Library), + PluginOp(PluginDispatchFn), } /// If the given rid is open, this returns the type of resource, E.G. "worker". @@ -141,6 +148,8 @@ fn inspect_repr(repr: &Repr) -> String { Repr::ChildStdout(_) => "childStdout", Repr::ChildStderr(_) => "childStderr", Repr::Worker(_) => "worker", + Repr::Plugin(_) => "plugin", + Repr::PluginOp(_) => "pluginOp", }; String::from(h_repr) @@ -556,3 +565,45 @@ pub fn seek( Err(err) => Box::new(futures::future::err(err)), } } + +pub fn add_plugin>(lib_path: P) -> Result { + debug!("LOADING NATIVE BINDING LIB: {:#?}", lib_path.as_ref()); + + let lib = Library::open(lib_path)?; + + let rid = new_rid(); + let mut tg = RESOURCE_TABLE.lock().unwrap(); + let r = tg.insert(rid, Repr::Plugin(lib)); + assert!(r.is_none()); + Ok(Resource { rid }) +} + +pub fn add_plugin_op( + lib_resource: ResourceId, + name: &str, +) -> Result { + let mut tg = RESOURCE_TABLE.lock().unwrap(); + let lib = match tg.get(&lib_resource) { + Some(Repr::Plugin(lib)) => lib, + Some(_) | None => return Err(bad_resource()), + }; + let fun = *unsafe { lib.symbol::(name) }?; + let rid = new_rid(); + let r = tg.insert(rid, Repr::PluginOp(fun)); + assert!(r.is_none()); + Ok(Resource { rid }) +} + +pub fn call_plugin_op( + fn_resource: ResourceId, + data: &[u8], + zero_copy: Option, +) -> Result { + let tg = RESOURCE_TABLE.lock().unwrap(); + let fun = match tg.get(&fn_resource) { + Some(Repr::PluginOp(fun)) => fun, + Some(_) | None => return Err(bad_resource()), + }; + let result = fun(data, zero_copy); + Ok(result) +} diff --git a/cli/state.rs b/cli/state.rs index 8d3d116d95906d..832faaf3323088 100644 --- a/cli/state.rs +++ b/cli/state.rs @@ -349,6 +349,11 @@ impl ThreadSafeState { self.permissions.check_run() } + #[inline] + pub fn check_plugins(&self, filename: &str) -> Result<(), ErrBox> { + self.permissions.check_plugins(filename) + } + #[cfg(test)] pub fn mock(argv: Vec) -> ThreadSafeState { ThreadSafeState::new( diff --git a/core/lib.rs b/core/lib.rs index 61521aecb00022..96a01be3d76d6b 100644 --- a/core/lib.rs +++ b/core/lib.rs @@ -11,6 +11,7 @@ mod js_errors; mod libdeno; mod module_specifier; mod modules; +pub mod plugins; mod shared_queue; pub use crate::any_error::*; diff --git a/core/plugins.rs b/core/plugins.rs new file mode 100644 index 00000000000000..e3b94b0ed4c3c4 --- /dev/null +++ b/core/plugins.rs @@ -0,0 +1,16 @@ +use crate::isolate::CoreOp; +use crate::libdeno::PinnedBuf; + +/// Funciton type for plugin ops +pub type PluginDispatchFn = + fn(data: &[u8], zero_copy: Option) -> CoreOp; + +#[macro_export] +macro_rules! declare_plugin_op { + ($name:ident, $fn:path) => { + #[no_mangle] + pub fn $name(data: &[u8], zero_copy: Option) -> CoreOp { + $fn(data, zero_copy) + } + }; +} diff --git a/js/deno.ts b/js/deno.ts index f20b6eff1f04f6..bf7a3e59ba004e 100644 --- a/js/deno.ts +++ b/js/deno.ts @@ -88,6 +88,13 @@ export const args: string[] = []; /** @internal */ export { core } from "./core"; +export { + PluginOp, + PluginImpl as Plugin, + openPlugin, + pluginFilename +} from "./native_plugins"; + // TODO Don't expose Console nor stringifyArgs. /** @internal */ export { Console, stringifyArgs } from "./console"; diff --git a/js/dispatch.ts b/js/dispatch.ts index cd11c93f6c6ca9..3bb84b06e2ccd8 100644 --- a/js/dispatch.ts +++ b/js/dispatch.ts @@ -64,7 +64,7 @@ function sendInternal( inner: flatbuffers.Offset, zeroCopy: undefined | ArrayBufferView, isSync: false -): Promise; +): Uint8Array | Promise; function sendInternal( builder: flatbuffers.Builder, innerType: msg.Any, @@ -95,18 +95,6 @@ function sendInternal( promiseTable.set(cmdId, promise); return promise; } else { - if (!isSync) { - // We can easily and correctly allow for sync responses to async calls - // by creating and returning a promise from the sync response. - const bb = new flatbuffers.ByteBuffer(response); - const base = msg.Base.getRootAsBase(bb); - const err = errors.maybeError(base); - if (err != null) { - return Promise.reject(err); - } else { - return Promise.resolve(base); - } - } return response; } } @@ -118,7 +106,21 @@ export function sendAsync( inner: flatbuffers.Offset, data?: ArrayBufferView ): Promise { - return sendInternal(builder, innerType, inner, data, false); + const response = sendInternal(builder, innerType, inner, data, false); + if (response instanceof Promise) { + return response; + } else { + // We can easily and correctly allow for sync responses to async calls + // by creating and returning a promise from the sync response. + const bb = new flatbuffers.ByteBuffer(response); + const base = msg.Base.getRootAsBase(bb); + const err = errors.maybeError(base); + if (err != null) { + return Promise.reject(err); + } else { + return Promise.resolve(base); + } + } } // @internal @@ -138,3 +140,24 @@ export function sendSync( return baseRes; } } + +export function sendAnySync( + builder: flatbuffers.Builder, + innerType: msg.Any, + inner: flatbuffers.Offset, + data?: ArrayBufferView +): null | msg.Base | Promise { + const response = sendInternal(builder, innerType, inner, data, false); + if (response instanceof Promise) { + return response; + } else { + if (response!.length === 0) { + return null; + } else { + const bb = new flatbuffers.ByteBuffer(response!); + const baseRes = msg.Base.getRootAsBase(bb); + errors.maybeThrowError(baseRes); + return baseRes; + } + } +} diff --git a/js/native_plugins.ts b/js/native_plugins.ts new file mode 100644 index 00000000000000..968b00f20648dc --- /dev/null +++ b/js/native_plugins.ts @@ -0,0 +1,171 @@ +// Copyright 2018-2019 the Deno authors. All rights reserved. MIT license. +// TODO rename this file to js/plugins.ts +import { sendSync, sendAnySync } from "./dispatch"; +import * as msg from "gen/cli/msg_generated"; +import * as flatbuffers from "./flatbuffers"; +import { assert } from "./util"; +import { build } from "./build"; + +export type PluginCallReturn = Uint8Array | undefined; + +function pluginCallInner(baseRes: msg.Base): PluginCallReturn { + assert(baseRes != null); + assert( + msg.Any.PluginCallRes === baseRes!.innerType(), + `base.innerType() unexpectedly is ${baseRes!.innerType()}` + ); + const res = new msg.PluginCallRes(); + assert(baseRes!.inner(res) != null); + + const dataArray = res.dataArray(); + if (dataArray === null) { + return undefined; + } + return dataArray; +} + +function pluginCall( + rid: number, + data: Uint8Array, + zeroCopy?: ArrayBufferView +): Promise | PluginCallReturn { + const builder = flatbuffers.createBuilder(); + const data_ = builder.createString(data); + const inner = msg.PluginCall.createPluginCall(builder, rid, data_); + const response = sendAnySync(builder, msg.Any.PluginCall, inner, zeroCopy); + if (response instanceof Promise) { + return new Promise( + async (resolve): Promise => { + resolve(pluginCallInner(await response)); + } + ); + } else { + if (response != null) { + return pluginCallInner(response); + } else { + return undefined; + } + } +} + +function pluginSym(libId: number, name: string): number { + const builder = flatbuffers.createBuilder(); + const name_ = builder.createString(name); + const inner = msg.PluginSym.createPluginSym(builder, libId, name_); + const baseRes = sendSync(builder, msg.Any.PluginSym, inner); + assert(baseRes != null); + assert( + msg.Any.PluginSymRes === baseRes!.innerType(), + `base.innerType() unexpectedly is ${baseRes!.innerType()}` + ); + const res = new msg.PluginSymRes(); + assert(baseRes!.inner(res) != null); + return res.rid(); +} + +export interface PluginOp { + dispatch( + data: Uint8Array, + zeroCopy?: ArrayBufferView + ): Promise | PluginCallReturn; +} + +// A loaded dynamic lib function. +// Loaded functions will need to loaded and addressed by unique identifiers +// for performance, since loading a function from a library for every call +// would likely be the limiting factor for many use cases. +// @internal +class PluginOpImpl implements PluginOp { + private readonly rid: number; + + constructor(dlId: number, name: string) { + this.rid = pluginSym(dlId, name); + } + + dispatch( + data: Uint8Array, + zeroCopy?: ArrayBufferView + ): Promise | PluginCallReturn { + return pluginCall(this.rid, data, zeroCopy); + } +} + +// TODO Rename to pluginOpen +function dlOpen(filename: string): number { + const builder = flatbuffers.createBuilder(); + const filename_ = builder.createString(filename); + const inner = msg.PluginOpen.createPluginOpen(builder, filename_); + const baseRes = sendSync(builder, msg.Any.PluginOpen, inner); + assert(baseRes != null); + assert( + msg.Any.PluginOpenRes === baseRes!.innerType(), + `base.innerType() unexpectedly is ${baseRes!.innerType()}` + ); + const res = new msg.PluginOpenRes(); + assert(baseRes!.inner(res) != null); + return res.rid(); +} + +export interface Plugin { + loadOp(name: string): PluginOp; +} + +// A loaded dynamic lib. +// Dynamic libraries need to remain loaded into memory on the rust side +// ,and then be addressed by their unique identifier to avoid loading +// the same library multiple times. +export class PluginImpl implements Plugin { + // unique resource identifier for the loaded dynamic lib rust side + private readonly rid: number; + private readonly fnMap: Map = new Map(); + + // @internal + constructor(libraryPath: string) { + this.rid = dlOpen(libraryPath); + } + + loadOp(name: string): PluginOp { + const cachedFn = this.fnMap.get(name); + if (cachedFn) { + return cachedFn; + } else { + const dlFn = new PluginOpImpl(this.rid, name); + this.fnMap.set(name, dlFn); + return dlFn; + } + } +} + +export function openPlugin(filename: string): Plugin { + return new PluginImpl(filename); +} + +export type PluginFilePrefix = "lib" | ""; + +const pluginFilePrefix = ((): PluginFilePrefix => { + switch (build.os) { + case "linux": + case "mac": + return "lib"; + case "win": + default: + return ""; + } +})(); + +export type PluginFileExtension = "so" | "dylib" | "dll"; + +const pluginFileExtension = ((): PluginFileExtension => { + switch (build.os) { + case "linux": + return "so"; + case "mac": + return "dylib"; + case "win": + return "dll"; + } +})(); + +export function pluginFilename(filenameBase: string): string { + return pluginFilePrefix + filenameBase + "." + pluginFileExtension; +} diff --git a/tests/034_plugin.out b/tests/034_plugin.out new file mode 100644 index 00000000000000..dfc98d51a214f7 --- /dev/null +++ b/tests/034_plugin.out @@ -0,0 +1,4 @@ +Hello from native bindings. data: "test" | zero_copy: {"some":"data"} +test +Hello from native bindings. data: "test" | zero_copy: {"some":"data"} +test diff --git a/tests/034_plugin.test b/tests/034_plugin.test new file mode 100644 index 00000000000000..e43037a67c4c4b --- /dev/null +++ b/tests/034_plugin.test @@ -0,0 +1,2 @@ +args: run --reload --allow-plugins --allow-env tests/034_plugin.ts +output: tests/034_plugin.out \ No newline at end of file diff --git a/tests/034_plugin.ts b/tests/034_plugin.ts new file mode 100644 index 00000000000000..82e27dbd6b9fce --- /dev/null +++ b/tests/034_plugin.ts @@ -0,0 +1,67 @@ +const { openPlugin, pluginFilename, env } = Deno; + +const plugin = openPlugin( + env().DENO_BUILD_PATH + "/" + pluginFilename("test_plugin") +); +const testOp = plugin.loadOp("test_op"); +const asyncTestOp = plugin.loadOp("async_test_op"); + +interface TestOptions { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + data: any; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + zeroCopyData: any; +} + +interface TestResponse { + data: Uint8Array; +} + +const textEncoder = new TextEncoder(); + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +function encodeTestOp(args: any): Uint8Array { + return textEncoder.encode(JSON.stringify(args)); +} + +const textDecoder = new TextDecoder(); + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +function decodeTestOp(data: Uint8Array): string { + return textDecoder.decode(data); +} + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +const doTestOp = (args: TestOptions): any => { + const response = testOp.dispatch( + encodeTestOp(args.data), + encodeTestOp(args.zeroCopyData) + ); + if (response instanceof Uint8Array) { + return decodeTestOp(response); + } else { + throw new Error("Unexpected response type"); + } +}; + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +async function doAsyncTestOp(args: TestOptions): Promise { + const response = asyncTestOp.dispatch( + encodeTestOp(args.data), + encodeTestOp(args.zeroCopyData) + ); + if (response instanceof Promise) { + return decodeTestOp(await response); + } else { + throw new Error("Unexpected response type"); + } +} + +async function main(): Promise { + console.log(doTestOp({ data: "test", zeroCopyData: { some: "data" } })); + console.log( + await doAsyncTestOp({ data: "test", zeroCopyData: { some: "data" } }) + ); +} + +main(); diff --git a/tests/plugin/Cargo.toml b/tests/plugin/Cargo.toml new file mode 100644 index 00000000000000..af117d4bce1767 --- /dev/null +++ b/tests/plugin/Cargo.toml @@ -0,0 +1,15 @@ +# Copyright 2018-2019 the Deno authors. All rights reserved. MIT license. + +[package] +name = "test_plugin" +version = "0.1.0" +edition = "2018" + +[lib] +path = "lib.rs" +crate-type = ["cdylib"] + +[dependencies] +deno = { path = "../../core" } + +futures = "0.1.27" \ No newline at end of file diff --git a/tests/plugin/lib.rs b/tests/plugin/lib.rs new file mode 100644 index 00000000000000..8855b2d83e8628 --- /dev/null +++ b/tests/plugin/lib.rs @@ -0,0 +1,43 @@ +// Copyright 2018-2019 the Deno authors. All rights reserved. MIT license. +use deno::CoreOp; +use deno::Op; +use deno::{Buf, PinnedBuf}; +use futures::future::lazy; + +#[macro_use] +extern crate deno; + +pub fn op_test_op(data: &[u8], zero_copy: Option) -> CoreOp { + if let Some(buf) = zero_copy { + let data_str = std::str::from_utf8(&data[..]).unwrap(); + let buf_str = std::str::from_utf8(&buf[..]).unwrap(); + println!( + "Hello from native bindings. data: {} | zero_copy: {}", + data_str, buf_str + ); + } + let result = b"test"; + let result_box: Buf = Box::new(*result); + Op::Sync(result_box) +} + +declare_plugin_op!(test_op, op_test_op); + +pub fn op_async_test_op(data: &[u8], zero_copy: Option) -> CoreOp { + if let Some(buf) = zero_copy { + let data_str = std::str::from_utf8(&data[..]).unwrap(); + let buf_str = std::str::from_utf8(&buf[..]).unwrap(); + println!( + "Hello from native bindings. data: {} | zero_copy: {}", + data_str, buf_str + ); + } + let op = Box::new(lazy(move || { + let result = b"test"; + let result_box: Buf = Box::new(*result); + Ok(result_box) + })); + Op::Async(op) +} + +declare_plugin_op!(async_test_op, op_async_test_op); diff --git a/third_party b/third_party index e319136a174471..828e597fb278a0 160000 --- a/third_party +++ b/third_party @@ -1 +1 @@ -Subproject commit e319136a1744710ffd3c2ea372a7dfc1d462f120 +Subproject commit 828e597fb278a074e1d8f5fd203c7e6db411bbda diff --git a/tools/build.py b/tools/build.py index 82426f5b26e8db..c464ff25311705 100755 --- a/tools/build.py +++ b/tools/build.py @@ -5,7 +5,7 @@ import os import sys import third_party -from util import build_path, enable_ansi_colors, run +from util import build_mode, build_path, enable_ansi_colors, run parser = argparse.ArgumentParser() parser.add_argument( @@ -33,6 +33,13 @@ def main(argv): env=third_party.google_env(), quiet=True) + cargo_args = ["build", "-p", "test_plugin", "-vv", "--locked"] + + if build_mode() == "release": + cargo_args += ["--release"] + + run(["cargo"] + cargo_args, env=third_party.google_env(), quiet=True) + if __name__ == '__main__': sys.exit(main(sys.argv)) diff --git a/tools/format.py b/tools/format.py index aa7586c5bee019..2f8a1e276883d1 100755 --- a/tools/format.py +++ b/tools/format.py @@ -24,8 +24,8 @@ def qrun(cmd, env=None): find_exts(["core"], [".cc", ".h"])) print "gn format" -for fn in ["BUILD.gn", ".gn"] + find_exts(["build_extra", "cli", "core"], - [".gn", ".gni"]): +for fn in ["BUILD.gn", ".gn"] + find_exts( + ["build_extra", "cli", "core", "tests/plugin"], [".gn", ".gni"]): qrun(["third_party/depot_tools/gn", "format", fn], env=google_env()) print "yapf" @@ -46,4 +46,4 @@ def qrun(cmd, env=None): "third_party/rustfmt/" + platform() + "/rustfmt", "--config-path", rustfmt_config, -] + find_exts(["cli", "core", "tools"], [".rs"])) +] + find_exts(["cli", "core", "tools", "tests/plugin"], [".rs"])) diff --git a/tools/permission_prompt_test.py b/tools/permission_prompt_test.py index 68069cb0eee465..ebd1c1181e0966 100755 --- a/tools/permission_prompt_test.py +++ b/tools/permission_prompt_test.py @@ -136,6 +136,10 @@ class RunPromptTest(DenoTestCase, BasePromptTest): test_type = "run" +class PluginPromptTest(DenoTestCase, BasePromptTest): + test_type = "plugins" + + def permission_prompt_tests(): return BasePromptTest.__subclasses__() diff --git a/tools/permission_prompt_test.ts b/tools/permission_prompt_test.ts index a4c9e43626ad3f..fc3973c7810431 100644 --- a/tools/permission_prompt_test.ts +++ b/tools/permission_prompt_test.ts @@ -1,43 +1,60 @@ // Copyright 2018-2019 the Deno authors. All rights reserved. MIT license. -const { args, listen, env, exit, makeTempDirSync, readFileSync, run } = Deno; +const { + args, + listen, + env, + exit, + makeTempDirSync, + readFileSync, + run, + openPlugin +} = Deno; const firstCheckFailedMessage = "First check failed"; const name = args[1]; const test = { - needsRead: async () => { + needsRead: async (): Promise => { try { readFileSync("package.json"); } catch (e) { - console.log(firstCheckFailedMessage); + if (e.kind === Deno.ErrorKind.PermissionDenied) { + console.log(firstCheckFailedMessage); + } } readFileSync("package.json"); }, - needsWrite: () => { + needsWrite: (): void => { try { makeTempDirSync(); } catch (e) { - console.log(firstCheckFailedMessage); + if (e.kind === Deno.ErrorKind.PermissionDenied) { + console.log(firstCheckFailedMessage); + } } makeTempDirSync(); }, - needsEnv: () => { + needsEnv: (): void => { try { env().home; } catch (e) { - console.log(firstCheckFailedMessage); + if (e.kind === Deno.ErrorKind.PermissionDenied) { + console.log(firstCheckFailedMessage); + } } env().home; }, - needsNet: () => { + needsNet: (): void => { try { listen("tcp", "127.0.0.1:4540"); } catch (e) { - console.log(firstCheckFailedMessage); + if (e.kind === Deno.ErrorKind.PermissionDenied) { + console.log(firstCheckFailedMessage); + } } listen("tcp", "127.0.0.1:4541"); }, - needsRun: () => { + needsRun: (): void => { try { const process = run({ args: [ @@ -47,7 +64,9 @@ const test = { ] }); } catch (e) { - console.log(firstCheckFailedMessage); + if (e.kind === Deno.ErrorKind.PermissionDenied) { + console.log(firstCheckFailedMessage); + } } const process = run({ args: [ @@ -56,6 +75,22 @@ const test = { "import sys; sys.stdout.write('hello'); sys.stdout.flush()" ] }); + }, + needsPlugins: (): void => { + try { + const plugin = openPlugin("some/fake/path"); + } catch (e) { + if (e.kind === Deno.ErrorKind.PermissionDenied) { + console.log(firstCheckFailedMessage); + } + } + try { + const plugin = openPlugin("some/fake/path"); + } catch (e) { + if (e.kind === Deno.ErrorKind.PermissionDenied) { + throw e; + } + } } }[name]; diff --git a/tools/test_util.py b/tools/test_util.py index 6540f37aa46ff7..a0a0d7d6353d22 100644 --- a/tools/test_util.py +++ b/tools/test_util.py @@ -114,6 +114,8 @@ def filter_test_suite(suite, pattern): def run_tests(test_cases=None): + os.environ["DENO_BUILD_PATH"] = build_path() + args = parse_test_args() loader = unittest.TestLoader()