From 8a2d4e27261927c0b0ce0eac5657941c982b9746 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= <biwanczuk@gmail.com> Date: Thu, 30 Apr 2020 16:16:12 +0200 Subject: [PATCH 01/47] add permissions argument to file fetcher methods --- cli/global_state.rs | 4 ++++ cli/ops/compiler.rs | 1 + 2 files changed, 5 insertions(+) diff --git a/cli/global_state.rs b/cli/global_state.rs index 4e9bdbb99c9411..ecda7c588ebdf4 100644 --- a/cli/global_state.rs +++ b/cli/global_state.rs @@ -100,6 +100,10 @@ impl GlobalState { let state2 = self.clone(); let module_specifier = module_specifier.clone(); + // TODO(bartlomieju): currently unused, but file fetcher will + // require them in the near future + let permissions = DenoPermissions::default(); + let out = self .file_fetcher .fetch_source_file(&module_specifier, maybe_referrer, permissions.clone()) diff --git a/cli/ops/compiler.rs b/cli/ops/compiler.rs index b844011877c390..948b7c85e9303d 100644 --- a/cli/ops/compiler.rs +++ b/cli/ops/compiler.rs @@ -4,6 +4,7 @@ use super::dispatch_json::JsonOp; use super::dispatch_json::Value; use crate::futures::future::try_join_all; use crate::op_error::OpError; +use crate::permissions::DenoPermissions; use crate::state::State; use deno_core::CoreIsolate; use deno_core::ModuleLoader; From c7dc06593f2a910a3ff55999beb9b85dcc194f31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= <biwanczuk@gmail.com> Date: Sun, 10 May 2020 13:52:59 +0200 Subject: [PATCH 02/47] drop permissions prefix --- cli/file_fetcher.rs | 36 ++++++++++++++++++++++++++++++++++++ cli/global_state.rs | 2 +- cli/ops/compiler.rs | 2 +- 3 files changed, 38 insertions(+), 2 deletions(-) diff --git a/cli/file_fetcher.rs b/cli/file_fetcher.rs index 2e75517e632523..ac685fc67376c1 100644 --- a/cli/file_fetcher.rs +++ b/cli/file_fetcher.rs @@ -110,7 +110,11 @@ impl SourceFileFetcher { pub fn fetch_cached_source_file( &self, specifier: &ModuleSpecifier, +<<<<<<< HEAD permissions: Permissions, +======= + _permissions: Permissions, +>>>>>>> drop permissions prefix ) -> Option<SourceFile> { let maybe_source_file = self.source_file_cache.get(specifier.to_string()); @@ -150,7 +154,11 @@ impl SourceFileFetcher { &self, specifier: &ModuleSpecifier, maybe_referrer: Option<ModuleSpecifier>, +<<<<<<< HEAD permissions: Permissions, +======= + _permissions: Permissions, +>>>>>>> drop permissions prefix ) -> Result<SourceFile, ErrBox> { let module_url = specifier.as_url().to_owned(); debug!("fetch_source_file specifier: {} ", &module_url); @@ -1001,7 +1009,11 @@ mod tests { // first download let r = fetcher +<<<<<<< HEAD .fetch_source_file(&specifier, None, Permissions::allow_all()) +======= + .fetch_source_file(&specifier, None, Permissions::default()) +>>>>>>> drop permissions prefix .await; assert!(r.is_ok()); @@ -1019,7 +1031,11 @@ mod tests { // header file creation timestamp (should be the same as after first // download) let r = fetcher +<<<<<<< HEAD .fetch_source_file(&specifier, None, Permissions::allow_all()) +======= + .fetch_source_file(&specifier, None, Permissions::default()) +>>>>>>> drop permissions prefix .await; assert!(r.is_ok()); @@ -1493,7 +1509,11 @@ mod tests { let specifier = ModuleSpecifier::resolve_url(file_url!("/baddir/hello.ts")).unwrap(); let r = fetcher +<<<<<<< HEAD .fetch_source_file(&specifier, None, Permissions::allow_all()) +======= + .fetch_source_file(&specifier, None, Permissions::default()) +>>>>>>> drop permissions prefix .await; assert!(r.is_err()); @@ -1502,7 +1522,11 @@ mod tests { let specifier = ModuleSpecifier::resolve_url_or_path(p.to_str().unwrap()).unwrap(); let r = fetcher +<<<<<<< HEAD .fetch_source_file(&specifier, None, Permissions::allow_all()) +======= + .fetch_source_file(&specifier, None, Permissions::default()) +>>>>>>> drop permissions prefix .await; assert!(r.is_ok()); } @@ -1516,7 +1540,11 @@ mod tests { let specifier = ModuleSpecifier::resolve_url(file_url!("/baddir/hello.ts")).unwrap(); let r = fetcher +<<<<<<< HEAD .fetch_source_file(&specifier, None, Permissions::allow_all()) +======= + .fetch_source_file(&specifier, None, Permissions::default()) +>>>>>>> drop permissions prefix .await; assert!(r.is_err()); @@ -1525,7 +1553,11 @@ mod tests { let specifier = ModuleSpecifier::resolve_url_or_path(p.to_str().unwrap()).unwrap(); let r = fetcher +<<<<<<< HEAD .fetch_source_file(&specifier, None, Permissions::allow_all()) +======= + .fetch_source_file(&specifier, None, Permissions::default()) +>>>>>>> drop permissions prefix .await; assert!(r.is_ok()); } @@ -1540,7 +1572,11 @@ mod tests { let specifier = ModuleSpecifier::resolve_url_or_path(p.to_str().unwrap()).unwrap(); let r = fetcher +<<<<<<< HEAD .fetch_source_file(&specifier, None, Permissions::allow_all()) +======= + .fetch_source_file(&specifier, None, Permissions::default()) +>>>>>>> drop permissions prefix .await; assert!(r.is_ok()); } diff --git a/cli/global_state.rs b/cli/global_state.rs index ecda7c588ebdf4..94c4387e845895 100644 --- a/cli/global_state.rs +++ b/cli/global_state.rs @@ -102,7 +102,7 @@ impl GlobalState { // TODO(bartlomieju): currently unused, but file fetcher will // require them in the near future - let permissions = DenoPermissions::default(); + let permissions = Permissions::default(); let out = self .file_fetcher diff --git a/cli/ops/compiler.rs b/cli/ops/compiler.rs index 948b7c85e9303d..3db93ae69f3300 100644 --- a/cli/ops/compiler.rs +++ b/cli/ops/compiler.rs @@ -4,7 +4,7 @@ use super::dispatch_json::JsonOp; use super::dispatch_json::Value; use crate::futures::future::try_join_all; use crate::op_error::OpError; -use crate::permissions::DenoPermissions; +use crate::permissions::Permissions; use crate::state::State; use deno_core::CoreIsolate; use deno_core::ModuleLoader; From a8f431de3a64250e19f902ab7aa3a0b2571bd331 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= <biwanczuk@gmail.com> Date: Sun, 10 May 2020 14:17:19 +0200 Subject: [PATCH 03/47] add permission check to SourceFileFetcher --- cli/file_fetcher.rs | 36 ------------------------------------ cli/permissions.rs | 16 ++++++++++++++++ 2 files changed, 16 insertions(+), 36 deletions(-) diff --git a/cli/file_fetcher.rs b/cli/file_fetcher.rs index ac685fc67376c1..2e75517e632523 100644 --- a/cli/file_fetcher.rs +++ b/cli/file_fetcher.rs @@ -110,11 +110,7 @@ impl SourceFileFetcher { pub fn fetch_cached_source_file( &self, specifier: &ModuleSpecifier, -<<<<<<< HEAD permissions: Permissions, -======= - _permissions: Permissions, ->>>>>>> drop permissions prefix ) -> Option<SourceFile> { let maybe_source_file = self.source_file_cache.get(specifier.to_string()); @@ -154,11 +150,7 @@ impl SourceFileFetcher { &self, specifier: &ModuleSpecifier, maybe_referrer: Option<ModuleSpecifier>, -<<<<<<< HEAD permissions: Permissions, -======= - _permissions: Permissions, ->>>>>>> drop permissions prefix ) -> Result<SourceFile, ErrBox> { let module_url = specifier.as_url().to_owned(); debug!("fetch_source_file specifier: {} ", &module_url); @@ -1009,11 +1001,7 @@ mod tests { // first download let r = fetcher -<<<<<<< HEAD .fetch_source_file(&specifier, None, Permissions::allow_all()) -======= - .fetch_source_file(&specifier, None, Permissions::default()) ->>>>>>> drop permissions prefix .await; assert!(r.is_ok()); @@ -1031,11 +1019,7 @@ mod tests { // header file creation timestamp (should be the same as after first // download) let r = fetcher -<<<<<<< HEAD .fetch_source_file(&specifier, None, Permissions::allow_all()) -======= - .fetch_source_file(&specifier, None, Permissions::default()) ->>>>>>> drop permissions prefix .await; assert!(r.is_ok()); @@ -1509,11 +1493,7 @@ mod tests { let specifier = ModuleSpecifier::resolve_url(file_url!("/baddir/hello.ts")).unwrap(); let r = fetcher -<<<<<<< HEAD .fetch_source_file(&specifier, None, Permissions::allow_all()) -======= - .fetch_source_file(&specifier, None, Permissions::default()) ->>>>>>> drop permissions prefix .await; assert!(r.is_err()); @@ -1522,11 +1502,7 @@ mod tests { let specifier = ModuleSpecifier::resolve_url_or_path(p.to_str().unwrap()).unwrap(); let r = fetcher -<<<<<<< HEAD .fetch_source_file(&specifier, None, Permissions::allow_all()) -======= - .fetch_source_file(&specifier, None, Permissions::default()) ->>>>>>> drop permissions prefix .await; assert!(r.is_ok()); } @@ -1540,11 +1516,7 @@ mod tests { let specifier = ModuleSpecifier::resolve_url(file_url!("/baddir/hello.ts")).unwrap(); let r = fetcher -<<<<<<< HEAD .fetch_source_file(&specifier, None, Permissions::allow_all()) -======= - .fetch_source_file(&specifier, None, Permissions::default()) ->>>>>>> drop permissions prefix .await; assert!(r.is_err()); @@ -1553,11 +1525,7 @@ mod tests { let specifier = ModuleSpecifier::resolve_url_or_path(p.to_str().unwrap()).unwrap(); let r = fetcher -<<<<<<< HEAD .fetch_source_file(&specifier, None, Permissions::allow_all()) -======= - .fetch_source_file(&specifier, None, Permissions::default()) ->>>>>>> drop permissions prefix .await; assert!(r.is_ok()); } @@ -1572,11 +1540,7 @@ mod tests { let specifier = ModuleSpecifier::resolve_url_or_path(p.to_str().unwrap()).unwrap(); let r = fetcher -<<<<<<< HEAD .fetch_source_file(&specifier, None, Permissions::allow_all()) -======= - .fetch_source_file(&specifier, None, Permissions::default()) ->>>>>>> drop permissions prefix .await; assert!(r.is_ok()); } diff --git a/cli/permissions.rs b/cli/permissions.rs index bc4d0844e0ec63..e0880ccad83296 100644 --- a/cli/permissions.rs +++ b/cli/permissions.rs @@ -344,6 +344,22 @@ fn permission_prompt(message: &str) -> bool { } } +#[cfg(test)] +impl Permissions { + pub fn allow_all() -> Self { + Self { + allow_read: PermissionState::from(true), + allow_write: PermissionState::from(true), + allow_net: PermissionState::from(true), + allow_env: PermissionState::from(true), + allow_run: PermissionState::from(true), + allow_plugin: PermissionState::from(true), + allow_hrtime: PermissionState::from(true), + ..Default::default() + } + } +} + #[cfg(test)] lazy_static! { /// Lock this when you use `set_prompt_result` in a test case. From 17d06b3900cc608194979f88f5012f77e2dabdd7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= <biwanczuk@gmail.com> Date: Sun, 10 May 2020 14:24:05 +0200 Subject: [PATCH 04/47] use Permissions::allow_all --- cli/global_state.rs | 2 +- cli/permissions.rs | 16 ---------------- 2 files changed, 1 insertion(+), 17 deletions(-) diff --git a/cli/global_state.rs b/cli/global_state.rs index 94c4387e845895..8a0392e44d40bf 100644 --- a/cli/global_state.rs +++ b/cli/global_state.rs @@ -102,7 +102,7 @@ impl GlobalState { // TODO(bartlomieju): currently unused, but file fetcher will // require them in the near future - let permissions = Permissions::default(); + let permissions = Permissions::allow_all(); let out = self .file_fetcher diff --git a/cli/permissions.rs b/cli/permissions.rs index e0880ccad83296..bc4d0844e0ec63 100644 --- a/cli/permissions.rs +++ b/cli/permissions.rs @@ -344,22 +344,6 @@ fn permission_prompt(message: &str) -> bool { } } -#[cfg(test)] -impl Permissions { - pub fn allow_all() -> Self { - Self { - allow_read: PermissionState::from(true), - allow_write: PermissionState::from(true), - allow_net: PermissionState::from(true), - allow_env: PermissionState::from(true), - allow_run: PermissionState::from(true), - allow_plugin: PermissionState::from(true), - allow_hrtime: PermissionState::from(true), - ..Default::default() - } - } -} - #[cfg(test)] lazy_static! { /// Lock this when you use `set_prompt_result` in a test case. From 38ebecdee89dfc527d3543b88136dce89f5258f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= <biwanczuk@gmail.com> Date: Sun, 10 May 2020 14:36:36 +0200 Subject: [PATCH 05/47] propagate proper permissions --- cli/global_state.rs | 4 ---- cli/ops/compiler.rs | 1 - 2 files changed, 5 deletions(-) diff --git a/cli/global_state.rs b/cli/global_state.rs index 8a0392e44d40bf..4e9bdbb99c9411 100644 --- a/cli/global_state.rs +++ b/cli/global_state.rs @@ -100,10 +100,6 @@ impl GlobalState { let state2 = self.clone(); let module_specifier = module_specifier.clone(); - // TODO(bartlomieju): currently unused, but file fetcher will - // require them in the near future - let permissions = Permissions::allow_all(); - let out = self .file_fetcher .fetch_source_file(&module_specifier, maybe_referrer, permissions.clone()) diff --git a/cli/ops/compiler.rs b/cli/ops/compiler.rs index 3db93ae69f3300..b844011877c390 100644 --- a/cli/ops/compiler.rs +++ b/cli/ops/compiler.rs @@ -4,7 +4,6 @@ use super::dispatch_json::JsonOp; use super::dispatch_json::Value; use crate::futures::future::try_join_all; use crate::op_error::OpError; -use crate::permissions::Permissions; use crate::state::State; use deno_core::CoreIsolate; use deno_core::ModuleLoader; From 98984e47cf830c8c871ab9b27e1232b8f0f4c77e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= <biwanczuk@gmail.com> Date: Sun, 10 May 2020 18:07:58 +0200 Subject: [PATCH 06/47] add docs --- cli/state.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cli/state.rs b/cli/state.rs index 9501ed2866f109..b36331085c0de0 100644 --- a/cli/state.rs +++ b/cli/state.rs @@ -318,8 +318,8 @@ impl ModuleLoader for State { let permissions = if state.is_main { Permissions::allow_all() } else { - state.permissions.clone() - }; + state.permissions.clone() + }; let fut = async move { let compiled_module = global_state From 62e6db4cb1567f61c8ce8da069160a0b91c5abfb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= <biwanczuk@gmail.com> Date: Thu, 7 May 2020 18:18:24 +0200 Subject: [PATCH 07/47] remove cli/compilers/js.rs --- cli/compilers/mod.rs | 23 +++++++++++++++++++++++ cli/global_state.rs | 13 +++++++++---- 2 files changed, 32 insertions(+), 4 deletions(-) create mode 100644 cli/compilers/mod.rs diff --git a/cli/compilers/mod.rs b/cli/compilers/mod.rs new file mode 100644 index 00000000000000..d32ff40eaf5552 --- /dev/null +++ b/cli/compilers/mod.rs @@ -0,0 +1,23 @@ +// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. +use crate::ops::JsonResult; +use deno_core::ErrBox; +use futures::Future; + +mod compiler_worker; +mod ts; + +pub use ts::runtime_compile; +pub use ts::runtime_transpile; +pub use ts::TargetLib; +pub use ts::TsCompiler; + +pub type CompilationResultFuture = dyn Future<Output = JsonResult>; + +#[derive(Debug, Clone)] +pub struct CompiledModule { + pub code: String, + pub name: String, +} + +pub type CompiledModuleFuture = + dyn Future<Output = Result<CompiledModule, ErrBox>>; diff --git a/cli/global_state.rs b/cli/global_state.rs index 4e9bdbb99c9411..50a39092bbdfbb 100644 --- a/cli/global_state.rs +++ b/cli/global_state.rs @@ -1,4 +1,7 @@ // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. +use crate::compilers::CompiledModule; +use crate::compilers::TargetLib; +use crate::compilers::TsCompiler; use crate::deno_dir; use crate::file_fetcher::SourceFileFetcher; use crate::flags; @@ -144,10 +147,12 @@ impl GlobalState { }) } } - _ => Ok(CompiledModule { - code: String::from_utf8(out.source_code)?, - name: out.url.to_string(), - }), + _ => { + Ok(CompiledModule { + code: String::from_utf8(out.source_code)?, + name: out.url.to_string(), + }) + }, }?; drop(compile_lock); From 161a36150242dd87ad75db3fea2da1fe0265ae23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= <biwanczuk@gmail.com> Date: Thu, 7 May 2020 18:33:25 +0200 Subject: [PATCH 08/47] remove cli/compilers/; add cli/tsc.rs --- cli/compilers/mod.rs | 23 ----------------------- cli/global_state.rs | 13 ++++--------- 2 files changed, 4 insertions(+), 32 deletions(-) delete mode 100644 cli/compilers/mod.rs diff --git a/cli/compilers/mod.rs b/cli/compilers/mod.rs deleted file mode 100644 index d32ff40eaf5552..00000000000000 --- a/cli/compilers/mod.rs +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. -use crate::ops::JsonResult; -use deno_core::ErrBox; -use futures::Future; - -mod compiler_worker; -mod ts; - -pub use ts::runtime_compile; -pub use ts::runtime_transpile; -pub use ts::TargetLib; -pub use ts::TsCompiler; - -pub type CompilationResultFuture = dyn Future<Output = JsonResult>; - -#[derive(Debug, Clone)] -pub struct CompiledModule { - pub code: String, - pub name: String, -} - -pub type CompiledModuleFuture = - dyn Future<Output = Result<CompiledModule, ErrBox>>; diff --git a/cli/global_state.rs b/cli/global_state.rs index 50a39092bbdfbb..4e9bdbb99c9411 100644 --- a/cli/global_state.rs +++ b/cli/global_state.rs @@ -1,7 +1,4 @@ // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. -use crate::compilers::CompiledModule; -use crate::compilers::TargetLib; -use crate::compilers::TsCompiler; use crate::deno_dir; use crate::file_fetcher::SourceFileFetcher; use crate::flags; @@ -147,12 +144,10 @@ impl GlobalState { }) } } - _ => { - Ok(CompiledModule { - code: String::from_utf8(out.source_code)?, - name: out.url.to_string(), - }) - }, + _ => Ok(CompiledModule { + code: String::from_utf8(out.source_code)?, + name: out.url.to_string(), + }), }?; drop(compile_lock); From c1e55bc41b4505785238a1fcb9eba1e60ffeb0d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= <biwanczuk@gmail.com> Date: Fri, 1 May 2020 16:38:50 +0200 Subject: [PATCH 09/47] prototype module graph loader --- cli/lib.rs | 1 + cli/module_graph.rs | 131 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 132 insertions(+) create mode 100644 cli/module_graph.rs diff --git a/cli/lib.rs b/cli/lib.rs index 0b88d734609204..402d54199cc67f 100644 --- a/cli/lib.rs +++ b/cli/lib.rs @@ -42,6 +42,7 @@ pub mod installer; mod js; mod lockfile; mod metrics; +mod module_graph; pub mod msg; pub mod op_error; pub mod ops; diff --git a/cli/module_graph.rs b/cli/module_graph.rs new file mode 100644 index 00000000000000..21b5dea47498ee --- /dev/null +++ b/cli/module_graph.rs @@ -0,0 +1,131 @@ +// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. +#![allow(unused)] + +use crate::file_fetcher::SourceFileFetcher; +use crate::swc_util::analyze_dependencies; +use deno_core::ErrBox; +use deno_core::ModuleSpecifier; +use serde::Serialize; +use std::collections::HashMap; + +#[derive(Debug, Serialize)] +struct ModuleGraph(HashMap<String, ModuleGraphFile>); + +#[derive(Debug, Serialize)] +struct ModuleGraphFile { + pub specifier: String, + pub deps: Vec<String>, +} + +struct ModuleGraphLoader { + file_fetcher: SourceFileFetcher, + to_visit: Vec<ModuleSpecifier>, + pub graph: ModuleGraph, +} + +// struct ModuleGraphFuture { +// file_fetcher: SourceFileFetcher, +// to_visit: Vec<ModuleSpecifier>, +// pending_lods: +// FuturesUnordered<Pin<Box<dyn Future<Output = Result<SourceFile, ErrBox>>>>>, +// has_loaded: HashSet<ModuleSpecifier>, +// pending_analysis: HashSet<ModuleSpecifier>, +// } + +impl ModuleGraphLoader { + pub fn new(file_fetcher: SourceFileFetcher) -> Self { + Self { + file_fetcher, + to_visit: vec![], + graph: ModuleGraph(HashMap::new()), + } + } + + pub async fn build_graph( + mut self, + specifier: &ModuleSpecifier, + ) -> Result<HashMap<String, ModuleGraphFile>, ErrBox> { + self.to_visit.push(specifier.to_owned()); + while let Some(spec) = self.to_visit.pop() { + self.visit_module(&spec).await?; + let file = self.graph.0.get(&spec.to_string()).unwrap(); + for dep in &file.deps { + self + .to_visit + .push(ModuleSpecifier::resolve_url_or_path(dep).unwrap()); + } + } + Ok(self.graph.0) + } + + async fn visit_module( + &mut self, + specifier: &ModuleSpecifier, + ) -> Result<(), ErrBox> { + if self.graph.0.contains_key(&specifier.to_string()) { + return Ok(()); + } + + let source_file = + self.file_fetcher.fetch_source_file(specifier, None).await?; + + let raw_deps = + analyze_dependencies(&String::from_utf8(source_file.source_code)?, true)?; + + // TODO(bartlomieju): apply import map, using State + // or should it be passed explicitly + let mut deps = vec![]; + for raw_dep in raw_deps { + let specifier = + ModuleSpecifier::resolve_import(&raw_dep, &specifier.to_string())?; + deps.push(specifier.to_string()); + } + + self.graph.0.insert( + specifier.to_string(), + ModuleGraphFile { + specifier: specifier.to_string(), + deps, + }, + ); + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::GlobalState; + use std::path::PathBuf; + + fn rel_module_specifier(relpath: &str) -> ModuleSpecifier { + let p = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join(relpath) + .into_os_string(); + let ps = p.to_str().unwrap(); + // TODO(ry) Why doesn't ModuleSpecifier::resolve_path actually take a + // Path?! + ModuleSpecifier::resolve_url_or_path(ps).unwrap() + } + + #[tokio::test] + async fn source_graph_fetch() { + let http_server_guard = crate::test_util::http_server(); + + let global_state = GlobalState::new(Default::default()).unwrap(); + let module_specifier = rel_module_specifier("tests/019_media_types.ts"); + + let graph_loader = + ModuleGraphLoader::new(global_state.file_fetcher.clone()); + let graph = graph_loader.build_graph(&module_specifier).await.unwrap(); + + assert_eq!(graph.len(), 9); + let r = graph + .get("http://localhost:4545/cli/tests/subdir/mt_text_typescript.t1.ts"); + assert!(r.is_some()); + + println!("{}", serde_json::to_string_pretty(&graph).unwrap()); + + drop(http_server_guard); + } +} From ea6e891202e9b0f786c57ae3f1262df89d426d9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= <biwanczuk@gmail.com> Date: Fri, 1 May 2020 16:53:11 +0200 Subject: [PATCH 10/47] add test for circular deps --- cli/module_graph.rs | 31 ++++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/cli/module_graph.rs b/cli/module_graph.rs index 21b5dea47498ee..1705fbd511b6f1 100644 --- a/cli/module_graph.rs +++ b/cli/module_graph.rs @@ -50,9 +50,13 @@ impl ModuleGraphLoader { self.visit_module(&spec).await?; let file = self.graph.0.get(&spec.to_string()).unwrap(); for dep in &file.deps { - self - .to_visit - .push(ModuleSpecifier::resolve_url_or_path(dep).unwrap()); + if let Some(_exists) = self.graph.0.get(dep) { + continue; + } else { + self + .to_visit + .push(ModuleSpecifier::resolve_url_or_path(dep).unwrap()); + } } } Ok(self.graph.0) @@ -128,4 +132,25 @@ mod tests { drop(http_server_guard); } + + #[tokio::test] + async fn source_graph_fetch_circular() { + let http_server_guard = crate::test_util::http_server(); + + let global_state = GlobalState::new(Default::default()).unwrap(); + let module_specifier = rel_module_specifier("tests/circular1.js"); + + let graph_loader = + ModuleGraphLoader::new(global_state.file_fetcher.clone()); + let graph = graph_loader.build_graph(&module_specifier).await.unwrap(); + + // assert_eq!(graph.len(), 9); + // let r = graph + // .get("http://localhost:4545/cli/tests/subdir/mt_text_typescript.t1.ts"); + // assert!(r.is_some()); + + println!("{}", serde_json::to_string_pretty(&graph).unwrap()); + + drop(http_server_guard); + } } From 198166a036aad4773dd17bffd147279864febec9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= <biwanczuk@gmail.com> Date: Mon, 4 May 2020 14:10:34 +0200 Subject: [PATCH 11/47] update tests --- cli/module_graph.rs | 117 +++++++++++++++++++++++++++++++------------- 1 file changed, 82 insertions(+), 35 deletions(-) diff --git a/cli/module_graph.rs b/cli/module_graph.rs index 1705fbd511b6f1..0effa13129658b 100644 --- a/cli/module_graph.rs +++ b/cli/module_graph.rs @@ -23,15 +23,6 @@ struct ModuleGraphLoader { pub graph: ModuleGraph, } -// struct ModuleGraphFuture { -// file_fetcher: SourceFileFetcher, -// to_visit: Vec<ModuleSpecifier>, -// pending_lods: -// FuturesUnordered<Pin<Box<dyn Future<Output = Result<SourceFile, ErrBox>>>>>, -// has_loaded: HashSet<ModuleSpecifier>, -// pending_analysis: HashSet<ModuleSpecifier>, -// } - impl ModuleGraphLoader { pub fn new(file_fetcher: SourceFileFetcher) -> Self { Self { @@ -102,34 +93,77 @@ mod tests { use crate::GlobalState; use std::path::PathBuf; - fn rel_module_specifier(relpath: &str) -> ModuleSpecifier { - let p = PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join(relpath) - .into_os_string(); - let ps = p.to_str().unwrap(); - // TODO(ry) Why doesn't ModuleSpecifier::resolve_path actually take a - // Path?! - ModuleSpecifier::resolve_url_or_path(ps).unwrap() - } + // fn rel_module_specifier(relpath: &str) -> ModuleSpecifier { + // let p = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + // .join(relpath) + // .into_os_string(); + // let ps = p.to_str().unwrap(); + // ModuleSpecifier::resolve_url_or_path(ps).unwrap() + // } #[tokio::test] async fn source_graph_fetch() { let http_server_guard = crate::test_util::http_server(); let global_state = GlobalState::new(Default::default()).unwrap(); - let module_specifier = rel_module_specifier("tests/019_media_types.ts"); - + let module_specifier = ModuleSpecifier::resolve_url_or_path( + "http://localhost:4545/cli/tests/019_media_types.ts", + ) + .unwrap(); let graph_loader = ModuleGraphLoader::new(global_state.file_fetcher.clone()); let graph = graph_loader.build_graph(&module_specifier).await.unwrap(); - assert_eq!(graph.len(), 9); - let r = graph - .get("http://localhost:4545/cli/tests/subdir/mt_text_typescript.t1.ts"); - assert!(r.is_some()); - - println!("{}", serde_json::to_string_pretty(&graph).unwrap()); - + assert_eq!( + serde_json::to_value(&graph).unwrap(), + json!({ + "http://localhost:4545/cli/tests/subdir/mt_text_typescript.t1.ts": { + "specifier": "http://localhost:4545/cli/tests/subdir/mt_text_typescript.t1.ts", + "deps": [] + }, + "http://localhost:4545/cli/tests/019_media_types.ts": { + "specifier": "http://localhost:4545/cli/tests/019_media_types.ts", + "deps": [ + "http://localhost:4545/cli/tests/subdir/mt_text_typescript.t1.ts", + "http://localhost:4545/cli/tests/subdir/mt_video_vdn.t2.ts", + "http://localhost:4545/cli/tests/subdir/mt_video_mp2t.t3.ts", + "http://localhost:4545/cli/tests/subdir/mt_application_x_typescript.t4.ts", + "http://localhost:4545/cli/tests/subdir/mt_text_javascript.j1.js", + "http://localhost:4545/cli/tests/subdir/mt_application_ecmascript.j2.js", + "http://localhost:4545/cli/tests/subdir/mt_text_ecmascript.j3.js", + "http://localhost:4545/cli/tests/subdir/mt_application_x_javascript.j4.js" + ] + }, + "http://localhost:4545/cli/tests/subdir/mt_text_ecmascript.j3.js": { + "specifier": "http://localhost:4545/cli/tests/subdir/mt_text_ecmascript.j3.js", + "deps": [] + }, + "http://localhost:4545/cli/tests/subdir/mt_video_vdn.t2.ts": { + "specifier": "http://localhost:4545/cli/tests/subdir/mt_video_vdn.t2.ts", + "deps": [] + }, + "http://localhost:4545/cli/tests/subdir/mt_application_x_typescript.t4.ts": { + "specifier": "http://localhost:4545/cli/tests/subdir/mt_application_x_typescript.t4.ts", + "deps": [] + }, + "http://localhost:4545/cli/tests/subdir/mt_video_mp2t.t3.ts": { + "specifier": "http://localhost:4545/cli/tests/subdir/mt_video_mp2t.t3.ts", + "deps": [] + }, + "http://localhost:4545/cli/tests/subdir/mt_application_x_javascript.j4.js": { + "specifier": "http://localhost:4545/cli/tests/subdir/mt_application_x_javascript.j4.js", + "deps": [] + }, + "http://localhost:4545/cli/tests/subdir/mt_application_ecmascript.j2.js": { + "specifier": "http://localhost:4545/cli/tests/subdir/mt_application_ecmascript.j2.js", + "deps": [] + }, + "http://localhost:4545/cli/tests/subdir/mt_text_javascript.j1.js": { + "specifier": "http://localhost:4545/cli/tests/subdir/mt_text_javascript.j1.js", + "deps": [] + } + }) + ); drop(http_server_guard); } @@ -138,19 +172,32 @@ mod tests { let http_server_guard = crate::test_util::http_server(); let global_state = GlobalState::new(Default::default()).unwrap(); - let module_specifier = rel_module_specifier("tests/circular1.js"); + let module_specifier = ModuleSpecifier::resolve_url_or_path( + "http://localhost:4545/cli/tests/circular1.js", + ) + .unwrap(); let graph_loader = ModuleGraphLoader::new(global_state.file_fetcher.clone()); let graph = graph_loader.build_graph(&module_specifier).await.unwrap(); - // assert_eq!(graph.len(), 9); - // let r = graph - // .get("http://localhost:4545/cli/tests/subdir/mt_text_typescript.t1.ts"); - // assert!(r.is_some()); - - println!("{}", serde_json::to_string_pretty(&graph).unwrap()); - + assert_eq!( + serde_json::to_value(&graph).unwrap(), + json!({ + "http://localhost:4545/cli/tests/circular2.js": { + "specifier": "http://localhost:4545/cli/tests/circular2.js", + "deps": [ + "http://localhost:4545/cli/tests/circular1.js" + ] + }, + "http://localhost:4545/cli/tests/circular1.js": { + "specifier": "http://localhost:4545/cli/tests/circular1.js", + "deps": [ + "http://localhost:4545/cli/tests/circular2.js" + ] + } + }) + ); drop(http_server_guard); } } From cf840fb690f1222a6bb3afebeacf0725e9ec29eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= <biwanczuk@gmail.com> Date: Wed, 6 May 2020 15:42:01 +0200 Subject: [PATCH 12/47] prototype tests --- cli/module_graph.rs | 138 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 138 insertions(+) diff --git a/cli/module_graph.rs b/cli/module_graph.rs index 0effa13129658b..a418b4aebd020d 100644 --- a/cli/module_graph.rs +++ b/cli/module_graph.rs @@ -15,6 +15,11 @@ struct ModuleGraph(HashMap<String, ModuleGraphFile>); struct ModuleGraphFile { pub specifier: String, pub deps: Vec<String>, + + // pub imports: Vec<ImportDescriptor>, + // pub referenced_files: Vec<ReferenceDescriptor>, + // pub lib_directives: Vec<LibDirective>, + // pub types_directives: Vec<TypeDirective>, } struct ModuleGraphLoader { @@ -200,4 +205,137 @@ mod tests { ); drop(http_server_guard); } + + #[tokio::test] + async fn source_graph_type_references() { + let http_server_guard = crate::test_util::http_server(); + + let global_state = GlobalState::new(Default::default()).unwrap(); + let module_specifier = ModuleSpecifier::resolve_url_or_path( + "http://localhost:4545/cli/tests/type_definitions.ts", + ) + .unwrap(); + + let graph_loader = + ModuleGraphLoader::new(global_state.file_fetcher.clone()); + let graph = graph_loader.build_graph(&module_specifier).await.unwrap(); + + assert_eq!( + serde_json::to_value(&graph).unwrap(), + json!({ + "http://localhost:4545/cli/tests/type_definitions.ts": { + "imports": [ + { + "specifier": "./type_definitions/foo.js", + "resolvedUrl": "http://localhost:4545/cli/tests/type_definitions/foo.js" + "typeDirective": "./type_definitions/foo.d.ts", + "resolvedTypeDirective": "http://localhost:4545/cli/tests/type_definitions/foo.d.ts" + }, + { + "specifier": "./type_definitions/fizz.js", + "resolvedUrl": "http://localhost:4545/cli/tests/type_definitions/fizz.js" + "typeDirective": "./type_definitions/fizz.d.ts", + "resolvedTypeDirective": "http://localhost:4545/cli/tests/type_definitions/fizz.d.ts" + }, + { + "specifier": "./type_definitions/qat.js", + "resolvedUrl": "http://localhost:4545/cli/tests/type_definitions/qat.js" + }, + ] + }, + "http://localhost:4545/cli/tests/type_definitions/foo.js": { + "imports": [], + }, + "http://localhost:4545/cli/tests/type_definitions/foo.d.ts": { + "imports": [], + }, + "http://localhost:4545/cli/tests/type_definitions/fizz.js": { + "imports": [], + }, + "http://localhost:4545/cli/tests/type_definitions/fizz.d.ts": { + "imports": [], + }, + "http://localhost:4545/cli/tests/type_definitions/qat.js": { + "imports": [], + } + }) + ); + drop(http_server_guard); + } + + #[tokio::test] + async fn source_graph_type_references2() { + let http_server_guard = crate::test_util::http_server(); + + let global_state = GlobalState::new(Default::default()).unwrap(); + let module_specifier = ModuleSpecifier::resolve_url_or_path( + "http://localhost:4545/cli/tests/type_directives_02.ts", + ) + .unwrap(); + + let graph_loader = + ModuleGraphLoader::new(global_state.file_fetcher.clone()); + let graph = graph_loader.build_graph(&module_specifier).await.unwrap(); + + assert_eq!( + serde_json::to_value(&graph).unwrap(), + json!({ + "http://localhost:4545/cli/tests/type_directives_02.ts": { + "imports": [ + { + "specifier": "./subdir/type_reference.js", + "resolvedUrl": "http://localhost:4545/cli/tests/subdir/type_reference.js" + } + ] + }, + "http://localhost:4545/cli/tests/subdir/type_reference.js": { + "typeReferences": [ + { + "specifier": "./type_reference.d.ts", + "resolvedUrl": "http://localhost:4545/cli/tests/subdir/type_reference.d.ts" + } + ], + } + }) + ); + drop(http_server_guard); + } + + #[tokio::test] + async fn source_graph_type_references3() { + let http_server_guard = crate::test_util::http_server(); + + let global_state = GlobalState::new(Default::default()).unwrap(); + let module_specifier = ModuleSpecifier::resolve_url_or_path( + "http://localhost:4545/cli/tests/type_directives_01.ts", + ) + .unwrap(); + + let graph_loader = + ModuleGraphLoader::new(global_state.file_fetcher.clone()); + let graph = graph_loader.build_graph(&module_specifier).await.unwrap(); + + assert_eq!( + serde_json::to_value(&graph).unwrap(), + json!({ + "http://localhost:4545/cli/tests/type_directives_01.ts": { + "imports": [ + { + "specifier": "./xTypeScriptTypes.js", + "resolvedUrl": "http://localhost:4545/cli/tests/xTypeScriptTypes.js" + } + ] + }, + "http://localhost:4545/cli/tests/xTypeScriptTypes.js": { + "typeHeaders": [ + { + "specifier": "./xTypeScriptTypes.d.ts", + "resolvedUrl": "http://localhost:4545/cli/tests/xTypeScriptTypes.d.ts" + } + ], + } + }) + ); + drop(http_server_guard); + } } From 22e879f27dc0c18e59c3109de587a56ee0efc6e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= <biwanczuk@gmail.com> Date: Wed, 6 May 2020 19:59:51 +0200 Subject: [PATCH 13/47] parse imports and type directives --- cli/module_graph.rs | 18 ++- cli/swc_util.rs | 278 +++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 289 insertions(+), 7 deletions(-) diff --git a/cli/module_graph.rs b/cli/module_graph.rs index a418b4aebd020d..fddfe0353781da 100644 --- a/cli/module_graph.rs +++ b/cli/module_graph.rs @@ -11,11 +11,23 @@ use std::collections::HashMap; #[derive(Debug, Serialize)] struct ModuleGraph(HashMap<String, ModuleGraphFile>); +// #[derive(Debug, Serialize)] +// #[serde(rename_all = "camelCase")] +// struct ImportDescriptor { +// specifier: String, +// resolved_specifier: ModuleSpecifier, +// // These two fields are for support of @deno-types directive +// // directly prepending import statement +// #[serde(skip_serializing_if = "Option::is_none")] +// type_directive: Option<String>, +// #[serde(skip_serializing_if = "Option::is_none")] +// resolved_type_directive: Option<ModuleSpecifier>, +// } + #[derive(Debug, Serialize)] struct ModuleGraphFile { pub specifier: String, pub deps: Vec<String>, - // pub imports: Vec<ImportDescriptor>, // pub referenced_files: Vec<ReferenceDescriptor>, // pub lib_directives: Vec<LibDirective>, @@ -227,13 +239,13 @@ mod tests { "imports": [ { "specifier": "./type_definitions/foo.js", - "resolvedUrl": "http://localhost:4545/cli/tests/type_definitions/foo.js" + "resolvedUrl": "http://localhost:4545/cli/tests/type_definitions/foo.js", "typeDirective": "./type_definitions/foo.d.ts", "resolvedTypeDirective": "http://localhost:4545/cli/tests/type_definitions/foo.d.ts" }, { "specifier": "./type_definitions/fizz.js", - "resolvedUrl": "http://localhost:4545/cli/tests/type_definitions/fizz.js" + "resolvedUrl": "http://localhost:4545/cli/tests/type_definitions/fizz.js", "typeDirective": "./type_definitions/fizz.d.ts", "resolvedTypeDirective": "http://localhost:4545/cli/tests/type_definitions/fizz.d.ts" }, diff --git a/cli/swc_util.rs b/cli/swc_util.rs index a2e5d50fdc5fe2..643a9ffd747abd 100644 --- a/cli/swc_util.rs +++ b/cli/swc_util.rs @@ -1,11 +1,16 @@ // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. + +#![allow(unused)] + use crate::swc_common; +use crate::swc_common::comments::CommentKind; use crate::swc_common::comments::Comments; use crate::swc_common::errors::Diagnostic; use crate::swc_common::errors::DiagnosticBuilder; use crate::swc_common::errors::Emitter; use crate::swc_common::errors::Handler; use crate::swc_common::errors::HandlerFlags; +use crate::swc_common::BytePos; use crate::swc_common::FileName; use crate::swc_common::Globals; use crate::swc_common::SourceMap; @@ -158,10 +163,16 @@ impl AstParser { &self, span: Span, ) -> Vec<swc_common::comments::Comment> { - self - .comments - .take_leading_comments(span.lo()) - .unwrap_or_else(|| vec![]) + let maybe_comments = self.comments.take_leading_comments(span.lo()); + + if let Some(comments) = maybe_comments { + // clone the comments and put them back in map + let to_return = comments.clone(); + self.comments.add_leading(span.lo(), comments); + to_return + } else { + vec![] + } } } @@ -317,3 +328,262 @@ const a = await import("./" + "buzz.ts"); ] ); } + +#[derive(Clone, Debug, PartialEq)] +enum DependencyKind { + Import, + Export, +} + +#[derive(Clone, Debug, PartialEq)] +struct DependencyDescriptor { + span: Span, + specifier: String, + kind: DependencyKind, +} + +struct NewDependencyVisitor { + dependencies: Vec<DependencyDescriptor>, +} + +impl Visit for NewDependencyVisitor { + fn visit_import_decl( + &mut self, + import_decl: &swc_ecma_ast::ImportDecl, + _parent: &dyn Node, + ) { + let src_str = import_decl.src.value.to_string(); + self.dependencies.push(DependencyDescriptor { + specifier: src_str, + kind: DependencyKind::Import, + span: import_decl.span, + }); + } + + fn visit_named_export( + &mut self, + named_export: &swc_ecma_ast::NamedExport, + _parent: &dyn Node, + ) { + if let Some(src) = &named_export.src { + let src_str = src.value.to_string(); + self.dependencies.push(DependencyDescriptor { + specifier: src_str, + kind: DependencyKind::Export, + span: named_export.span, + }); + } + } + + fn visit_export_all( + &mut self, + export_all: &swc_ecma_ast::ExportAll, + _parent: &dyn Node, + ) { + let src_str = export_all.src.value.to_string(); + self.dependencies.push(DependencyDescriptor { + specifier: src_str, + kind: DependencyKind::Export, + span: export_all.span, + }); + } +} + +fn get_deno_types(parser: &AstParser, span: Span) -> Option<String> { + let comments = parser.get_span_comments(span); + + if comments.is_empty() { + return None; + } + + // @deno-types must directly prepend import statement - hence + // checking last comment for span + let last = comments.last().unwrap(); + let comment = last.text.trim_start(); + + if comment.starts_with("@deno-types") { + let split: Vec<&str> = comment.split("=").collect(); + assert_eq!(split.len(), 2); + let specifier_in_quotes = split.get(1).unwrap().to_string(); + let specifier = specifier_in_quotes + .trim_start_matches("\"") + .trim_start_matches("\'") + .trim_end_matches("\"") + .trim_end_matches("\'") + .to_string(); + return Some(specifier); + } + + None +} + +#[derive(Clone, Debug, PartialEq)] +struct ImportDescriptor { + specifier: String, + deno_types: Option<String>, +} + +#[derive(Clone, Debug, PartialEq)] +enum TsReferenceKind { + Lib, + Types, + Path, +} + +#[derive(Clone, Debug, PartialEq)] +struct TsReferenceDescriptor { + kind: TsReferenceKind, + specifier: String, +} + +#[allow(unused)] +fn analyze_dependencies_and_references( + source_code: &str, + analyze_dynamic_imports: bool, +) -> Result< + (Vec<ImportDescriptor>, Vec<TsReferenceDescriptor>), + SwcDiagnosticBuffer, +> { + let parser = AstParser::new(); + parser.parse_module("root.ts", source_code, |parse_result| { + let module = parse_result?; + let mut collector = NewDependencyVisitor { + dependencies: vec![], + }; + let module_span = module.span; + collector.visit_module(&module, &module); + + let dependency_descriptors = collector.dependencies; + + // for each import check if there's relevant @deno-types directive + let imports = dependency_descriptors + .iter() + .map(|mut desc| { + if desc.kind == DependencyKind::Import { + let deno_types = get_deno_types(&parser, desc.span); + ImportDescriptor { + specifier: desc.specifier.to_string(), + deno_types, + } + } else { + ImportDescriptor { + specifier: desc.specifier.to_string(), + deno_types: None, + } + } + }) + .collect(); + + // analyze comment from beginning of the file and find TS directives + eprintln!("module span {:?}", module_span); + let comments = parser + .comments + .take_leading_comments(module_span.lo()) + .unwrap_or_else(|| vec![]); + + let mut references = vec![]; + for comment in comments { + if comment.kind != CommentKind::Line { + continue; + } + + let text = comment.text.to_string(); + let (kind, specifier_in_quotes) = + if text.starts_with("/ <reference path=") { + ( + TsReferenceKind::Path, + text.trim_start_matches("/ <reference path="), + ) + } else if text.starts_with("/ <reference lib=") { + ( + TsReferenceKind::Lib, + text.trim_start_matches("/ <reference lib="), + ) + } else if text.starts_with("/ <reference types=") { + ( + TsReferenceKind::Types, + text.trim_start_matches("/ <reference types="), + ) + } else { + continue; + }; + let specifier = specifier_in_quotes + .trim_end_matches("/>") + .trim_end() + .trim_start_matches("\"") + .trim_start_matches("\'") + .trim_end_matches("\"") + .trim_end_matches("\'") + .to_string(); + + references.push(TsReferenceDescriptor { kind, specifier }); + } + Ok((imports, references)) + }) +} + +#[test] +fn test_analyze_dependencies_and_directives() { + let source = r#" +// This comment is placed to make sure that directives are parsed +// even when they start on non-first line + +/// <reference lib="dom" /> +/// <reference types="./type_reference.d.ts" /> +/// <reference path="./type_reference/dep.ts" /> +// @deno-types="./type_definitions/foo.d.ts" +import { foo } from "./type_definitions/foo.js"; +// @deno-types="./type_definitions/fizz.d.ts" +import "./type_definitions/fizz.js"; + +/// <reference path="./type_reference/dep2.ts" /> + +import * as qat from "./type_definitions/qat.ts"; + +console.log(foo); +console.log(fizz); +console.log(qat.qat); +"#; + + let (imports, references) = + analyze_dependencies_and_references(source, true).expect("Failed to parse"); + + assert_eq!( + imports, + vec![ + ImportDescriptor { + specifier: "./type_definitions/foo.js".to_string(), + deno_types: Some("./type_definitions/foo.d.ts".to_string()) + }, + ImportDescriptor { + specifier: "./type_definitions/fizz.js".to_string(), + deno_types: Some("./type_definitions/fizz.d.ts".to_string()) + }, + ImportDescriptor { + specifier: "./type_definitions/qat.ts".to_string(), + deno_types: None + }, + ] + ); + + // According to TS docs (https://www.typescriptlang.org/docs/handbook/triple-slash-directives.html) + // directives that are not at the top of the file are ignored, so only + // 3 references should be captured instead of 4. + assert_eq!( + references, + vec![ + TsReferenceDescriptor { + specifier: "dom".to_string(), + kind: TsReferenceKind::Lib, + }, + TsReferenceDescriptor { + specifier: "./type_reference.d.ts".to_string(), + kind: TsReferenceKind::Types, + }, + TsReferenceDescriptor { + specifier: "./type_reference/dep.ts".to_string(), + kind: TsReferenceKind::Path, + }, + ] + ); +} From ac3668c4376b94c7dc039049adfc3e09a134caf2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= <biwanczuk@gmail.com> Date: Wed, 6 May 2020 22:04:49 +0200 Subject: [PATCH 14/47] type directives analysis --- cli/file_fetcher.rs | 7 + cli/lib.rs | 2 + cli/module_graph.rs | 313 +++++++++++++++++++++++++++++++++++--------- cli/swc_util.rs | 17 +-- cli/tsc.rs | 3 + 5 files changed, 275 insertions(+), 67 deletions(-) diff --git a/cli/file_fetcher.rs b/cli/file_fetcher.rs index 2e75517e632523..cbfb340ddc6121 100644 --- a/cli/file_fetcher.rs +++ b/cli/file_fetcher.rs @@ -34,6 +34,7 @@ pub struct SourceFile { pub url: Url, pub filename: PathBuf, pub types_url: Option<Url>, + pub types_header: Option<String>, pub media_type: msg::MediaType, pub source_code: Vec<u8>, } @@ -323,6 +324,7 @@ impl SourceFileFetcher { media_type, source_code, types_url, + types_header: None, }) } @@ -380,6 +382,7 @@ impl SourceFileFetcher { &fake_filepath, headers.get("content-type").map(|e| e.as_str()), ); + let types_header = headers.get("x-typescript-types").map(|e| e.to_string()); let types_url = match media_type { msg::MediaType::JavaScript | msg::MediaType::JSX => get_types_url( &module_url, @@ -394,6 +397,7 @@ impl SourceFileFetcher { media_type, source_code, types_url, + types_header, })) } @@ -502,6 +506,8 @@ impl SourceFileFetcher { headers.get("content-type").map(String::as_str), ); + let types_header = + headers.get("x-typescript-types").map(String::to_string); let types_url = match media_type { msg::MediaType::JavaScript | msg::MediaType::JSX => get_types_url( &module_url, @@ -517,6 +523,7 @@ impl SourceFileFetcher { media_type, source_code: source, types_url, + types_header, }; Ok(source_file) diff --git a/cli/lib.rs b/cli/lib.rs index 402d54199cc67f..bea5d4d84a05f8 100644 --- a/cli/lib.rs +++ b/cli/lib.rs @@ -356,6 +356,7 @@ async fn eval_command( filename: main_module_url.to_file_path().unwrap(), url: main_module_url, types_url: None, + types_header: None, media_type: if as_typescript { MediaType::TypeScript } else { @@ -532,6 +533,7 @@ async fn test_command( filename: test_file_url.to_file_path().unwrap(), url: test_file_url, types_url: None, + types_header: None, media_type: MediaType::TypeScript, source_code: test_file.clone().into_bytes(), }; diff --git a/cli/module_graph.rs b/cli/module_graph.rs index fddfe0353781da..3c63829a190ab3 100644 --- a/cli/module_graph.rs +++ b/cli/module_graph.rs @@ -2,36 +2,72 @@ #![allow(unused)] use crate::file_fetcher::SourceFileFetcher; -use crate::swc_util::analyze_dependencies; +use crate::msg::MediaType; +use crate::swc_util::analyze_dependencies_and_references; +use crate::swc_util::TsReferenceKind; use deno_core::ErrBox; use deno_core::ModuleSpecifier; use serde::Serialize; +use serde::Serializer; use std::collections::HashMap; +fn serialize_module_specifier<S>( + spec: &ModuleSpecifier, + s: S, +) -> Result<S::Ok, S::Error> +where + S: Serializer, +{ + s.serialize_str(&spec.to_string()) +} + +fn serialize_option_module_specifier<S>( + maybe_spec: &Option<ModuleSpecifier>, + s: S, +) -> Result<S::Ok, S::Error> +where + S: Serializer, +{ + if let Some(spec) = maybe_spec { + serialize_module_specifier(spec, s) + } else { + s.serialize_none() + } +} + #[derive(Debug, Serialize)] struct ModuleGraph(HashMap<String, ModuleGraphFile>); -// #[derive(Debug, Serialize)] -// #[serde(rename_all = "camelCase")] -// struct ImportDescriptor { -// specifier: String, -// resolved_specifier: ModuleSpecifier, -// // These two fields are for support of @deno-types directive -// // directly prepending import statement -// #[serde(skip_serializing_if = "Option::is_none")] -// type_directive: Option<String>, -// #[serde(skip_serializing_if = "Option::is_none")] -// resolved_type_directive: Option<ModuleSpecifier>, -// } +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +struct ImportDescriptor { + specifier: String, + #[serde(serialize_with = "serialize_module_specifier")] + resolved_specifier: ModuleSpecifier, + // These two fields are for support of @deno-types directive + // directly prepending import statement + type_directive: Option<String>, + #[serde(serialize_with = "serialize_option_module_specifier")] + resolved_type_directive: Option<ModuleSpecifier>, +} #[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +struct ReferenceDescriptor { + specifier: String, + #[serde(serialize_with = "serialize_module_specifier")] + resolved_specifier: ModuleSpecifier, +} + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] struct ModuleGraphFile { pub specifier: String, - pub deps: Vec<String>, - // pub imports: Vec<ImportDescriptor>, - // pub referenced_files: Vec<ReferenceDescriptor>, - // pub lib_directives: Vec<LibDirective>, - // pub types_directives: Vec<TypeDirective>, + pub imports: Vec<ImportDescriptor>, + pub referenced_files: Vec<ReferenceDescriptor>, + pub lib_directives: Vec<ReferenceDescriptor>, + pub types_directives: Vec<ReferenceDescriptor>, + pub type_headers: Vec<ReferenceDescriptor>, } struct ModuleGraphLoader { @@ -56,48 +92,139 @@ impl ModuleGraphLoader { self.to_visit.push(specifier.to_owned()); while let Some(spec) = self.to_visit.pop() { self.visit_module(&spec).await?; - let file = self.graph.0.get(&spec.to_string()).unwrap(); - for dep in &file.deps { - if let Some(_exists) = self.graph.0.get(dep) { - continue; - } else { - self - .to_visit - .push(ModuleSpecifier::resolve_url_or_path(dep).unwrap()); - } - } } Ok(self.graph.0) } async fn visit_module( &mut self, - specifier: &ModuleSpecifier, + module_specifier: &ModuleSpecifier, ) -> Result<(), ErrBox> { - if self.graph.0.contains_key(&specifier.to_string()) { + if self.graph.0.contains_key(&module_specifier.to_string()) { return Ok(()); } - let source_file = - self.file_fetcher.fetch_source_file(specifier, None).await?; + let source_file = self + .file_fetcher + .fetch_source_file(module_specifier, None) + .await?; + + let mut imports = vec![]; + let mut referenced_files = vec![]; + let mut lib_directives = vec![]; + let mut types_directives = vec![]; + let mut type_headers = vec![]; + + if source_file.media_type == MediaType::JavaScript + || source_file.media_type == MediaType::TypeScript + { + if let Some(types_specifier) = source_file.types_header { + let type_header = ReferenceDescriptor { + specifier: types_specifier.to_string(), + resolved_specifier: ModuleSpecifier::resolve_import( + &types_specifier, + &module_specifier.to_string(), + )?, + }; + type_headers.push(type_header); + } + + let (import_descs, ref_descs) = analyze_dependencies_and_references( + &String::from_utf8(source_file.source_code)?, + true, + )?; + + // TODO(bartlomieju): apply import map, using State + // or should it be passed explicitly + for import_desc in import_descs { + let resolved_specifier = ModuleSpecifier::resolve_import( + &import_desc.specifier, + &module_specifier.to_string(), + )?; + + let resolved_type_directive = + if let Some(types_specifier) = import_desc.deno_types.as_ref() { + Some(ModuleSpecifier::resolve_import( + &types_specifier, + &module_specifier.to_string(), + )?) + } else { + None + }; + + let import_descriptor = ImportDescriptor { + specifier: import_desc.specifier.to_string(), + resolved_specifier, + type_directive: import_desc.deno_types, + resolved_type_directive, + }; + + if self + .graph + .0 + .get(&import_descriptor.resolved_specifier.to_string()) + .is_none() + { + self + .to_visit + .push(import_descriptor.resolved_specifier.clone()); + } - let raw_deps = - analyze_dependencies(&String::from_utf8(source_file.source_code)?, true)?; + if let Some(type_dir_url) = + import_descriptor.resolved_type_directive.as_ref() + { + if self.graph.0.get(&type_dir_url.to_string()).is_none() { + self.to_visit.push(type_dir_url.clone()); + } + } - // TODO(bartlomieju): apply import map, using State - // or should it be passed explicitly - let mut deps = vec![]; - for raw_dep in raw_deps { - let specifier = - ModuleSpecifier::resolve_import(&raw_dep, &specifier.to_string())?; - deps.push(specifier.to_string()); + imports.push(import_descriptor); + } + + for ref_desc in ref_descs { + let resolved_specifier = ModuleSpecifier::resolve_import( + &ref_desc.specifier, + &module_specifier.to_string(), + )?; + let reference_descriptor = ReferenceDescriptor { + specifier: ref_desc.specifier.to_string(), + resolved_specifier, + }; + + if self + .graph + .0 + .get(&reference_descriptor.resolved_specifier.to_string()) + .is_none() + { + self + .to_visit + .push(reference_descriptor.resolved_specifier.clone()); + } + + match ref_desc.kind { + TsReferenceKind::Lib => { + lib_directives.push(reference_descriptor); + } + TsReferenceKind::Types => { + types_directives.push(reference_descriptor); + } + TsReferenceKind::Path => { + referenced_files.push(reference_descriptor); + } + } + } } self.graph.0.insert( - specifier.to_string(), + module_specifier.to_string(), ModuleGraphFile { - specifier: specifier.to_string(), - deps, + specifier: module_specifier.to_string(), + imports, + referenced_files, + lib_directives, + types_directives, + type_headers, }, ); Ok(()) @@ -232,43 +359,77 @@ mod tests { ModuleGraphLoader::new(global_state.file_fetcher.clone()); let graph = graph_loader.build_graph(&module_specifier).await.unwrap(); + eprintln!("json {:#?}", serde_json::to_value(&graph).unwrap()); + assert_eq!( serde_json::to_value(&graph).unwrap(), json!({ "http://localhost:4545/cli/tests/type_definitions.ts": { + "specifier": "http://localhost:4545/cli/tests/type_definitions.ts", "imports": [ { "specifier": "./type_definitions/foo.js", - "resolvedUrl": "http://localhost:4545/cli/tests/type_definitions/foo.js", + "resolvedSpecifier": "http://localhost:4545/cli/tests/type_definitions/foo.js", "typeDirective": "./type_definitions/foo.d.ts", "resolvedTypeDirective": "http://localhost:4545/cli/tests/type_definitions/foo.d.ts" }, { "specifier": "./type_definitions/fizz.js", - "resolvedUrl": "http://localhost:4545/cli/tests/type_definitions/fizz.js", + "resolvedSpecifier": "http://localhost:4545/cli/tests/type_definitions/fizz.js", "typeDirective": "./type_definitions/fizz.d.ts", "resolvedTypeDirective": "http://localhost:4545/cli/tests/type_definitions/fizz.d.ts" }, { - "specifier": "./type_definitions/qat.js", - "resolvedUrl": "http://localhost:4545/cli/tests/type_definitions/qat.js" + "specifier": "./type_definitions/qat.ts", + "resolvedSpecifier": "http://localhost:4545/cli/tests/type_definitions/qat.ts", + "typeDirective": null, + "resolvedTypeDirective": null, }, - ] + ], + "typesDirectives": [], + "referencedFiles": [], + "libDirectives": [], + "typeHeaders": [], }, "http://localhost:4545/cli/tests/type_definitions/foo.js": { + "specifier": "http://localhost:4545/cli/tests/type_definitions/foo.js", "imports": [], + "referencedFiles": [], + "libDirectives": [], + "typesDirectives": [], + "typeHeaders": [], }, "http://localhost:4545/cli/tests/type_definitions/foo.d.ts": { + "specifier": "http://localhost:4545/cli/tests/type_definitions/foo.d.ts", "imports": [], + "referencedFiles": [], + "libDirectives": [], + "typesDirectives": [], + "typeHeaders": [], }, "http://localhost:4545/cli/tests/type_definitions/fizz.js": { + "specifier": "http://localhost:4545/cli/tests/type_definitions/fizz.js", "imports": [], + "referencedFiles": [], + "libDirectives": [], + "typesDirectives": [], + "typeHeaders": [], }, "http://localhost:4545/cli/tests/type_definitions/fizz.d.ts": { + "specifier": "http://localhost:4545/cli/tests/type_definitions/fizz.d.ts", "imports": [], + "referencedFiles": [], + "libDirectives": [], + "typesDirectives": [], + "typeHeaders": [], }, - "http://localhost:4545/cli/tests/type_definitions/qat.js": { + "http://localhost:4545/cli/tests/type_definitions/qat.ts": { + "specifier": "http://localhost:4545/cli/tests/type_definitions/qat.ts", "imports": [], + "referencedFiles": [], + "libDirectives": [], + "typesDirectives": [], + "typeHeaders": [], } }) ); @@ -289,24 +450,46 @@ mod tests { ModuleGraphLoader::new(global_state.file_fetcher.clone()); let graph = graph_loader.build_graph(&module_specifier).await.unwrap(); + eprintln!("{:#?}", serde_json::to_value(&graph).unwrap()); + assert_eq!( serde_json::to_value(&graph).unwrap(), json!({ "http://localhost:4545/cli/tests/type_directives_02.ts": { + "specifier": "http://localhost:4545/cli/tests/type_directives_02.ts", "imports": [ { "specifier": "./subdir/type_reference.js", - "resolvedUrl": "http://localhost:4545/cli/tests/subdir/type_reference.js" + "resolvedSpecifier": "http://localhost:4545/cli/tests/subdir/type_reference.js", + "typeDirective": null, + "resolvedTypeDirective": null, } - ] + ], + "typesDirectives": [], + "referencedFiles": [], + "libDirectives": [], + "typeHeaders": [], + }, + "http://localhost:4545/cli/tests/subdir/type_reference.d.ts": { + "specifier": "http://localhost:4545/cli/tests/subdir/type_reference.d.ts", + "imports": [], + "referencedFiles": [], + "libDirectives": [], + "typesDirectives": [], + "typeHeaders": [], }, "http://localhost:4545/cli/tests/subdir/type_reference.js": { - "typeReferences": [ + "specifier": "http://localhost:4545/cli/tests/subdir/type_reference.js", + "imports": [], + "referencedFiles": [], + "libDirectives": [], + "typesDirectives": [ { "specifier": "./type_reference.d.ts", - "resolvedUrl": "http://localhost:4545/cli/tests/subdir/type_reference.d.ts" + "resolvedSpecifier": "http://localhost:4545/cli/tests/subdir/type_reference.d.ts", } ], + "typeHeaders": [], } }) ); @@ -331,20 +514,32 @@ mod tests { serde_json::to_value(&graph).unwrap(), json!({ "http://localhost:4545/cli/tests/type_directives_01.ts": { + "specifier": "http://localhost:4545/cli/tests/type_directives_01.ts", "imports": [ { - "specifier": "./xTypeScriptTypes.js", - "resolvedUrl": "http://localhost:4545/cli/tests/xTypeScriptTypes.js" + "specifier": "http://127.0.0.1:4545/xTypeScriptTypes.js", + "resolvedSpecifier": "http://127.0.0.1:4545/xTypeScriptTypes.js", + "typeDirective": null, + "resolvedTypeDirective": null, } - ] + ], + "referencedFiles": [], + "libDirectives": [], + "typesDirectives": [], + "typeHeaders": [], }, - "http://localhost:4545/cli/tests/xTypeScriptTypes.js": { + "http://127.0.0.1:4545/xTypeScriptTypes.js": { + "specifier": "http://127.0.0.1:4545/xTypeScriptTypes.js", "typeHeaders": [ { "specifier": "./xTypeScriptTypes.d.ts", - "resolvedUrl": "http://localhost:4545/cli/tests/xTypeScriptTypes.d.ts" + "resolvedSpecifier": "http://127.0.0.1:4545/xTypeScriptTypes.d.ts" } ], + "imports": [], + "referencedFiles": [], + "libDirectives": [], + "typesDirectives": [], } }) ); diff --git a/cli/swc_util.rs b/cli/swc_util.rs index 643a9ffd747abd..fb177614837db9 100644 --- a/cli/swc_util.rs +++ b/cli/swc_util.rs @@ -418,26 +418,26 @@ fn get_deno_types(parser: &AstParser, span: Span) -> Option<String> { } #[derive(Clone, Debug, PartialEq)] -struct ImportDescriptor { - specifier: String, - deno_types: Option<String>, +pub struct ImportDescriptor { + pub specifier: String, + pub deno_types: Option<String>, } #[derive(Clone, Debug, PartialEq)] -enum TsReferenceKind { +pub enum TsReferenceKind { Lib, Types, Path, } #[derive(Clone, Debug, PartialEq)] -struct TsReferenceDescriptor { - kind: TsReferenceKind, - specifier: String, +pub struct TsReferenceDescriptor { + pub kind: TsReferenceKind, + pub specifier: String, } #[allow(unused)] -fn analyze_dependencies_and_references( +pub fn analyze_dependencies_and_references( source_code: &str, analyze_dynamic_imports: bool, ) -> Result< @@ -487,6 +487,7 @@ fn analyze_dependencies_and_references( continue; } + // TODO(bartlomieju): you can do better than that... let text = comment.text.to_string(); let (kind, specifier_in_quotes) = if text.starts_with("/ <reference path=") { diff --git a/cli/tsc.rs b/cli/tsc.rs index f207558a5c6f1b..e69361b10093bd 100644 --- a/cli/tsc.rs +++ b/cli/tsc.rs @@ -593,6 +593,7 @@ impl TsCompiler { media_type: msg::MediaType::JavaScript, source_code: compiled_code, types_url: None, + types_header: None, }; Ok(compiled_module) @@ -668,6 +669,7 @@ impl TsCompiler { media_type: msg::MediaType::JavaScript, source_code, types_url: None, + types_header: None, }; Ok(source_map_file) @@ -876,6 +878,7 @@ mod tests { media_type: msg::MediaType::TypeScript, source_code: include_bytes!("./tests/002_hello.ts").to_vec(), types_url: None, + types_header: None, }; let mock_state = GlobalState::mock(vec![String::from("deno"), String::from("hello.js")]); From d6aff9389b6950a5777acb70f449b22d5a247405 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= <biwanczuk@gmail.com> Date: Thu, 7 May 2020 15:43:45 +0200 Subject: [PATCH 15/47] lint --- cli/swc_util.rs | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/cli/swc_util.rs b/cli/swc_util.rs index fb177614837db9..d602c935bb8563 100644 --- a/cli/swc_util.rs +++ b/cli/swc_util.rs @@ -402,14 +402,15 @@ fn get_deno_types(parser: &AstParser, span: Span) -> Option<String> { let comment = last.text.trim_start(); if comment.starts_with("@deno-types") { - let split: Vec<&str> = comment.split("=").collect(); + let split: Vec<String> = + comment.split('=').map(|s| s.to_string()).collect(); assert_eq!(split.len(), 2); let specifier_in_quotes = split.get(1).unwrap().to_string(); let specifier = specifier_in_quotes - .trim_start_matches("\"") - .trim_start_matches("\'") - .trim_end_matches("\"") - .trim_end_matches("\'") + .trim_start_matches('\"') + .trim_start_matches('\'') + .trim_end_matches('\"') + .trim_end_matches('\'') .to_string(); return Some(specifier); } @@ -511,10 +512,10 @@ pub fn analyze_dependencies_and_references( let specifier = specifier_in_quotes .trim_end_matches("/>") .trim_end() - .trim_start_matches("\"") - .trim_start_matches("\'") - .trim_end_matches("\"") - .trim_end_matches("\'") + .trim_start_matches('\"') + .trim_start_matches('\'') + .trim_end_matches('\"') + .trim_end_matches('\'') .to_string(); references.push(TsReferenceDescriptor { kind, specifier }); From 319b94ab322397e8efaf5bfe30a61b86b32981ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= <biwanczuk@gmail.com> Date: Thu, 7 May 2020 16:43:13 +0200 Subject: [PATCH 16/47] basic compilation working --- cli/global_state.rs | 4 +- cli/js/compiler.ts | 200 ++++++++++++++++++++++++++++++++++++++++++++ cli/lib.rs | 2 +- cli/module_graph.rs | 22 +++-- cli/msg.rs | 4 +- cli/tsc.rs | 184 +++++++++++++++++++++++++++++++++++++++- 6 files changed, 399 insertions(+), 17 deletions(-) diff --git a/cli/global_state.rs b/cli/global_state.rs index 4e9bdbb99c9411..e8125af731b4b0 100644 --- a/cli/global_state.rs +++ b/cli/global_state.rs @@ -115,14 +115,14 @@ impl GlobalState { | msg::MediaType::JSX => { state1 .ts_compiler - .compile(state1.clone(), &out, target_lib, permissions.clone()) + .new_compile(state1.clone(), &out, target_lib) .await } msg::MediaType::JavaScript => { if state1.ts_compiler.compile_js { state2 .ts_compiler - .compile(state1.clone(), &out, target_lib, permissions.clone()) + .new_compile(state1.clone(), &out, target_lib) .await } else { if let Some(types_url) = out.types_url.clone() { diff --git a/cli/js/compiler.ts b/cli/js/compiler.ts index 146529edd1445c..8036be469c52f0 100644 --- a/cli/js/compiler.ts +++ b/cli/js/compiler.ts @@ -756,6 +756,59 @@ async function processImports( return resolvedSources; } +function buildSourceFileCache(sourceFileMap: Record<string, SourceFileMapEntry>): void { + for (const entry of Object.values(sourceFileMap)) { + assert(entry.sourceCode.length > 0); + SourceFile.addToCache({ + url: entry.url, + filename: entry.url, + mediaType: entry.mediaType, + sourceCode: entry.sourceCode, + }); + + for (const importDesc of entry.imports) { + let mappedUrl = importDesc.resolvedSpecifier; + const importedFile = sourceFileMap[importDesc.resolvedSpecifier]; + assert(importedFile); + const isJsOrJsx = + importedFile.mediaType === MediaType.JavaScript || + importedFile.mediaType === MediaType.JSX; + // If JS or JSX perform substitution for types if available + if (isJsOrJsx) { + if (importedFile.typeHeaders.length > 0) { + const typeHeaders = importedFile.typeHeaders[0]; + mappedUrl = typeHeaders.resolvedSpecifier; + } else if (importDesc.resolvedTypeDirective) { + mappedUrl = importDesc.resolvedTypeDirective; + } else if (importedFile.typesDirectives.length > 0) { + const typeDirective = importedFile.typesDirectives[0]; + mappedUrl = typeDirective.resolvedSpecifier; + } + } + + SourceFile.cacheResolvedUrl( + mappedUrl, + importDesc.specifier, + entry.url, + ); + } + for (const fileRef of entry.referencedFiles) { + SourceFile.cacheResolvedUrl( + fileRef.resolvedSpecifier, + fileRef.specifier, + entry.url, + ); + } + for (const fileRef of entry.libDirectives) { + SourceFile.cacheResolvedUrl( + fileRef.resolvedSpecifier, + fileRef.specifier, + entry.url, + ); + } + } +} + interface FileReference { fileName: string; pos: number; @@ -855,6 +908,7 @@ enum CompilerRequestType { Compile = 0, RuntimeCompile = 1, RuntimeTranspile = 2, + CompileNew = 3, } // TODO(bartlomieju): probably could be defined inline? @@ -1222,6 +1276,43 @@ interface CompilerRequestCompile { cwd: string; } +interface ImportDescriptor { + specifier: string; + resolvedSpecifier: string; + typeDirective?: string; + resolvedTypeDirective?: string; +} + +interface ReferenceDescriptor { + specifier: string; + resolvedSpecifier: string; +} + +interface SourceFileMapEntry { + // fully resolved URL + url: string; + sourceCode: string; + mediaType: MediaType; + imports: ImportDescriptor[], + referencedFiles: ReferenceDescriptor[], + libDirectives: ReferenceDescriptor[], + typesDirectives: ReferenceDescriptor[], + typeHeaders: ReferenceDescriptor[], +} + +interface CompilerRequestCompileNew { + type: CompilerRequestType.CompileNew; + target: CompilerHostTarget; + rootNames: string[]; + configPath?: string; + config?: string; + unstable: boolean; + bundle: boolean; + cwd: string; + // key value is fully resolved URL + sourceFileMap: Record<string, SourceFileMapEntry>; +} + interface CompilerRequestRuntimeCompile { type: CompilerRequestType.RuntimeCompile; target: CompilerHostTarget; @@ -1239,6 +1330,7 @@ interface CompilerRequestRuntimeTranspile { } type CompilerRequest = + | CompilerRequestCompileNew | CompilerRequestCompile | CompilerRequestRuntimeCompile | CompilerRequestRuntimeTranspile; @@ -1259,6 +1351,109 @@ interface RuntimeBundleResult { diagnostics: DiagnosticItem[]; } +async function compileNew( + request: CompilerRequestCompileNew +): Promise<CompileResult> { + const { + bundle, + config, + configPath, + rootNames, + target, + unstable, + cwd, + sourceFileMap, + } = request; + util.log(">>> compile start", { + rootNames, + type: CompilerRequestType[request.type], + }); + + // When a programme is emitted, TypeScript will call `writeFile` with + // each file that needs to be emitted. The Deno compiler host delegates + // this, to make it easier to perform the right actions, which vary + // based a lot on the request. + const state: WriteFileState = { + type: request.type, + emitMap: {}, + bundle, + host: undefined, + rootNames, + }; + let writeFile: WriteFileCallback; + if (bundle) { + writeFile = createBundleWriteFile(state); + } else { + writeFile = createCompileWriteFile(state); + } + const host = (state.host = new Host({ + bundle, + target, + writeFile, + unstable, + })); + let diagnostics: readonly ts.Diagnostic[] = []; + + // if there is a configuration supplied, we need to parse that + if (config && config.length && configPath) { + const configResult = host.configure(cwd, configPath, config); + diagnostics = processConfigureResponse(configResult, configPath) || []; + } + + buildSourceFileCache(sourceFileMap); + // if there was a configuration and no diagnostics with it, we will continue + // to generate the program and possibly emit it. + if (diagnostics.length === 0) { + const options = host.getCompilationSettings(); + const program = ts.createProgram({ + rootNames, + options, + host, + oldProgram: TS_SNAPSHOT_PROGRAM, + }); + + diagnostics = ts + .getPreEmitDiagnostics(program) + .filter(({ code }) => !ignoredDiagnostics.includes(code)); + + // We will only proceed with the emit if there are no diagnostics. + if (diagnostics && diagnostics.length === 0) { + if (bundle) { + // we only support a single root module when bundling + assert(rootNames.length === 1); + setRootExports(program, rootNames[0]); + } + const emitResult = program.emit(); + assert(emitResult.emitSkipped === false, "Unexpected skip of the emit."); + // emitResult.diagnostics is `readonly` in TS3.5+ and can't be assigned + // without casting. + diagnostics = emitResult.diagnostics; + } + } + + let bundleOutput = undefined; + + if (bundle) { + assert(state.bundleOutput); + bundleOutput = state.bundleOutput; + } + + assert(state.emitMap); + const result: CompileResult = { + emitMap: state.emitMap, + bundleOutput, + diagnostics: fromTypeScriptDiagnostic(diagnostics), + }; + + util.log("<<< compile end", { + rootNames, + type: CompilerRequestType[request.type], + }); + + return result; +} + + async function compile( request: CompilerRequestCompile ): Promise<CompileResult> { @@ -1549,6 +1744,11 @@ async function tsCompilerOnMessage({ globalThis.postMessage(result); break; } + case CompilerRequestType.CompileNew: { + const result = await compileNew(request as CompilerRequestCompileNew); + globalThis.postMessage(result); + break; + } case CompilerRequestType.RuntimeCompile: { const result = await runtimeCompile( request as CompilerRequestRuntimeCompile diff --git a/cli/lib.rs b/cli/lib.rs index bea5d4d84a05f8..db62e530a4f221 100644 --- a/cli/lib.rs +++ b/cli/lib.rs @@ -390,7 +390,7 @@ async fn bundle_command( debug!(">>>>> bundle START"); let bundle_result = global_state .ts_compiler - .bundle(global_state.clone(), module_name.to_string(), out_file) + .new_bundle(global_state.clone(), module_name, out_file) .await; debug!(">>>>> bundle END"); bundle_result diff --git a/cli/module_graph.rs b/cli/module_graph.rs index 3c63829a190ab3..6005374a59055f 100644 --- a/cli/module_graph.rs +++ b/cli/module_graph.rs @@ -36,11 +36,11 @@ where } #[derive(Debug, Serialize)] -struct ModuleGraph(HashMap<String, ModuleGraphFile>); +pub struct ModuleGraph(HashMap<String, ModuleGraphFile>); #[derive(Debug, Serialize)] #[serde(rename_all = "camelCase")] -struct ImportDescriptor { +pub struct ImportDescriptor { specifier: String, #[serde(serialize_with = "serialize_module_specifier")] resolved_specifier: ModuleSpecifier, @@ -53,7 +53,7 @@ struct ImportDescriptor { #[derive(Debug, Serialize)] #[serde(rename_all = "camelCase")] -struct ReferenceDescriptor { +pub struct ReferenceDescriptor { specifier: String, #[serde(serialize_with = "serialize_module_specifier")] resolved_specifier: ModuleSpecifier, @@ -61,16 +61,18 @@ struct ReferenceDescriptor { #[derive(Debug, Serialize)] #[serde(rename_all = "camelCase")] -struct ModuleGraphFile { +pub struct ModuleGraphFile { pub specifier: String, pub imports: Vec<ImportDescriptor>, pub referenced_files: Vec<ReferenceDescriptor>, pub lib_directives: Vec<ReferenceDescriptor>, pub types_directives: Vec<ReferenceDescriptor>, pub type_headers: Vec<ReferenceDescriptor>, + pub media_type: MediaType, + pub source_code: String, } -struct ModuleGraphLoader { +pub struct ModuleGraphLoader { file_fetcher: SourceFileFetcher, to_visit: Vec<ModuleSpecifier>, pub graph: ModuleGraph, @@ -115,6 +117,8 @@ impl ModuleGraphLoader { let mut types_directives = vec![]; let mut type_headers = vec![]; + let source_code = String::from_utf8(source_file.source_code)?; + if source_file.media_type == MediaType::JavaScript || source_file.media_type == MediaType::TypeScript { @@ -129,10 +133,8 @@ impl ModuleGraphLoader { type_headers.push(type_header); } - let (import_descs, ref_descs) = analyze_dependencies_and_references( - &String::from_utf8(source_file.source_code)?, - true, - )?; + let (import_descs, ref_descs) = + analyze_dependencies_and_references(&source_code, true)?; // TODO(bartlomieju): apply import map, using State // or should it be passed explicitly @@ -220,6 +222,8 @@ impl ModuleGraphLoader { module_specifier.to_string(), ModuleGraphFile { specifier: module_specifier.to_string(), + media_type: source_file.media_type, + source_code, imports, referenced_files, lib_directives, diff --git a/cli/msg.rs b/cli/msg.rs index 146f45ea5485ec..186fde42c644e8 100644 --- a/cli/msg.rs +++ b/cli/msg.rs @@ -2,9 +2,11 @@ // Warning! The values in this enum are duplicated in js/compiler.ts // Update carefully! +use serde::Serialize; + #[allow(non_camel_case_types)] #[repr(i8)] -#[derive(Clone, Copy, PartialEq, Debug)] +#[derive(Clone, Copy, PartialEq, Debug, Serialize)] pub enum MediaType { JavaScript = 0, JSX = 1, diff --git a/cli/tsc.rs b/cli/tsc.rs index e69361b10093bd..102abe0edeae43 100644 --- a/cli/tsc.rs +++ b/cli/tsc.rs @@ -8,6 +8,7 @@ use crate::file_fetcher::SourceFileFetcher; use crate::fmt; use crate::fs as deno_fs; use crate::global_state::GlobalState; +use crate::module_graph::ModuleGraphLoader; use crate::msg; use crate::op_error::OpError; use crate::ops; @@ -370,6 +371,83 @@ impl TsCompiler { worker } + pub async fn new_bundle( + &self, + global_state: GlobalState, + module_specifier: ModuleSpecifier, + out_file: Option<PathBuf>, + ) -> Result<(), ErrBox> { + debug!( + "Invoking the compiler to bundle. module_name: {}", + module_specifier.to_string() + ); + eprintln!("Bundling {}", module_specifier.to_string()); + + let module_graph_loader = + ModuleGraphLoader::new(global_state.file_fetcher.clone()); + let module_graph = + module_graph_loader.build_graph(&module_specifier).await?; + let module_graph_json = + serde_json::to_value(module_graph).expect("Failed to serialize data"); + + let root_names = vec![module_specifier.to_string()]; + let bundle = true; + let target = "main"; + let unstable = global_state.flags.unstable; + let compiler_config = self.config.clone(); + let cwd = std::env::current_dir().unwrap(); + let j = match (compiler_config.path, compiler_config.content) { + (Some(config_path), Some(config_data)) => json!({ + "type": msg::CompilerRequestType::Compile as i32, + "target": target, + "rootNames": root_names, + "bundle": bundle, + "unstable": unstable, + "configPath": config_path, + "config": str::from_utf8(&config_data).unwrap(), + "cwd": cwd, + "sourceFileMap": module_graph_json, + }), + _ => json!({ + "type": msg::CompilerRequestType::Compile as i32, + "target": target, + "rootNames": root_names, + "bundle": bundle, + "unstable": unstable, + "cwd": cwd, + "sourceFileMap": module_graph_json, + }), + }; + + let req_msg = j.to_string().into_boxed_str().into_boxed_bytes(); + + let msg = execute_in_thread(global_state.clone(), req_msg).await?; + let json_str = std::str::from_utf8(&msg).unwrap(); + debug!("Message: {}", json_str); + + let bundle_response: BundleResponse = serde_json::from_str(json_str)?; + + if !bundle_response.diagnostics.items.is_empty() { + return Err(ErrBox::from(bundle_response.diagnostics)); + } + + if let Some(out_file_) = out_file.as_ref() { + eprintln!("Emitting bundle to {:?}", out_file_); + + let output_bytes = bundle_response.bundle_output.as_bytes(); + let output_len = output_bytes.len(); + + deno_fs::write_file(out_file_, output_bytes, 0o666)?; + // TODO(bartlomieju): add "humanFileSize" method + eprintln!("{} bytes emmited.", output_len); + } else { + println!("{}", bundle_response.bundle_output); + } + + Ok(()) + } + + #[allow(unused)] pub async fn bundle( &self, global_state: GlobalState, @@ -537,6 +615,105 @@ impl TsCompiler { None } + #[allow(unused)] + pub async fn new_compile( + &self, + global_state: GlobalState, + source_file: &SourceFile, + target: TargetLib, + ) -> Result<CompiledModule, ErrBox> { + if self.has_compiled(&source_file.url) { + return self.get_compiled_module(&source_file.url); + } + + if self.use_disk_cache { + // Try to load cached version: + // 1. check if there's 'meta' file + if let Some(metadata) = self.get_metadata(&source_file.url) { + // 2. compare version hashes + // TODO: it would probably be good idea to make it method implemented on SourceFile + let version_hash_to_validate = source_code_version_hash( + &source_file.source_code, + version::DENO, + &self.config.hash, + ); + + if metadata.version_hash == version_hash_to_validate { + debug!("load_cache metadata version hash match"); + if let Ok(compiled_module) = + self.get_compiled_module(&source_file.url) + { + self.mark_compiled(&source_file.url); + return Ok(compiled_module); + } + } + } + } + let source_file_ = source_file.clone(); + let module_url = source_file.url.clone(); + let module_specifier = ModuleSpecifier::from(source_file.url.clone()); + let module_graph_loader = + ModuleGraphLoader::new(global_state.file_fetcher.clone()); + let module_graph = + module_graph_loader.build_graph(&module_specifier).await?; + let module_graph_json = + serde_json::to_value(module_graph).expect("Failed to serialize data"); + + let target = match target { + TargetLib::Main => "main", + TargetLib::Worker => "worker", + }; + let root_names = vec![module_url.to_string()]; + let bundle = false; + let unstable = global_state.flags.unstable; + let compiler_config = self.config.clone(); + let cwd = std::env::current_dir().unwrap(); + let j = match (compiler_config.path, compiler_config.content) { + (Some(config_path), Some(config_data)) => json!({ + "type": msg::CompilerRequestType::Compile as i32, + "target": target, + "rootNames": root_names, + "bundle": bundle, + "unstable": unstable, + "configPath": config_path, + "config": str::from_utf8(&config_data).unwrap(), + "cwd": cwd, + "sourceFileMap": module_graph_json, + }), + _ => json!({ + "type": msg::CompilerRequestType::Compile as i32, + "target": target, + "rootNames": root_names, + "bundle": bundle, + "unstable": unstable, + "cwd": cwd, + "sourceFileMap": module_graph_json, + }), + }; + + let req_msg = j.to_string().into_boxed_str().into_boxed_bytes(); + + let ts_compiler = self.clone(); + + info!( + "{} {}", + colors::green("Compile".to_string()), + module_url.to_string() + ); + + let msg = execute_in_thread(global_state.clone(), req_msg).await?; + let json_str = std::str::from_utf8(&msg).unwrap(); + + let compile_response: CompileResponse = serde_json::from_str(json_str)?; + + if !compile_response.diagnostics.items.is_empty() { + return Err(ErrBox::from(compile_response.diagnostics)); + } + + self.cache_emitted_files(compile_response.emit_map)?; + ts_compiler.get_compiled_module(&source_file_.url) + } + fn cache_emitted_files( &self, emit_map: HashMap<String, EmittedSource>, @@ -906,9 +1083,8 @@ mod tests { .unwrap() .join("cli/tests/002_hello.ts"); use deno_core::ModuleSpecifier; - let module_name = ModuleSpecifier::resolve_url_or_path(p.to_str().unwrap()) - .unwrap() - .to_string(); + let module_name = + ModuleSpecifier::resolve_url_or_path(p.to_str().unwrap()).unwrap(); let state = GlobalState::mock(vec![ String::from("deno"), @@ -918,7 +1094,7 @@ mod tests { let result = state .ts_compiler - .bundle(state.clone(), module_name, None) + .new_bundle(state.clone(), module_name, None) .await; assert!(result.is_ok()); } From db5405acac7dbf6f68f411f4d1ac18e072b2de35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= <biwanczuk@gmail.com> Date: Thu, 7 May 2020 21:18:18 +0200 Subject: [PATCH 17/47] fmt --- cli/js/compiler.ts | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/cli/js/compiler.ts b/cli/js/compiler.ts index 8036be469c52f0..ed1fa15814fa6a 100644 --- a/cli/js/compiler.ts +++ b/cli/js/compiler.ts @@ -756,7 +756,9 @@ async function processImports( return resolvedSources; } -function buildSourceFileCache(sourceFileMap: Record<string, SourceFileMapEntry>): void { +function buildSourceFileCache( + sourceFileMap: Record<string, SourceFileMapEntry> +): void { for (const entry of Object.values(sourceFileMap)) { assert(entry.sourceCode.length > 0); SourceFile.addToCache({ @@ -786,24 +788,20 @@ function buildSourceFileCache(sourceFileMap: Record<string, SourceFileMapEntry>) } } - SourceFile.cacheResolvedUrl( - mappedUrl, - importDesc.specifier, - entry.url, - ); + SourceFile.cacheResolvedUrl(mappedUrl, importDesc.specifier, entry.url); } for (const fileRef of entry.referencedFiles) { SourceFile.cacheResolvedUrl( fileRef.resolvedSpecifier, fileRef.specifier, - entry.url, + entry.url ); } for (const fileRef of entry.libDirectives) { SourceFile.cacheResolvedUrl( fileRef.resolvedSpecifier, fileRef.specifier, - entry.url, + entry.url ); } } @@ -1293,11 +1291,11 @@ interface SourceFileMapEntry { url: string; sourceCode: string; mediaType: MediaType; - imports: ImportDescriptor[], - referencedFiles: ReferenceDescriptor[], - libDirectives: ReferenceDescriptor[], - typesDirectives: ReferenceDescriptor[], - typeHeaders: ReferenceDescriptor[], + imports: ImportDescriptor[]; + referencedFiles: ReferenceDescriptor[]; + libDirectives: ReferenceDescriptor[]; + typesDirectives: ReferenceDescriptor[]; + typeHeaders: ReferenceDescriptor[]; } interface CompilerRequestCompileNew { @@ -1351,7 +1349,7 @@ interface RuntimeBundleResult { diagnostics: DiagnosticItem[]; } -async function compileNew( +function compileNew( request: CompilerRequestCompileNew ): Promise<CompileResult> { const { @@ -1453,7 +1451,6 @@ async function compileNew( return result; } - async function compile( request: CompilerRequestCompile ): Promise<CompileResult> { @@ -1745,7 +1742,7 @@ async function tsCompilerOnMessage({ break; } case CompilerRequestType.CompileNew: { - const result = await compileNew(request as CompilerRequestCompileNew); + const result = compileNew(request as CompilerRequestCompileNew); globalThis.postMessage(result); break; } From 9bbd7a5b0caf38edb0a828bf8a3e59641ed3424b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= <biwanczuk@gmail.com> Date: Thu, 7 May 2020 22:35:05 +0200 Subject: [PATCH 18/47] fix bundle tests --- cli/js/compiler.ts | 4 +--- cli/module_graph.rs | 45 +++++++++++++++++++++++++++++++++------------ cli/state.rs | 2 +- cli/swc_util.rs | 1 - cli/tsc.rs | 32 ++++++++++++++++++++++++++++---- 5 files changed, 63 insertions(+), 21 deletions(-) diff --git a/cli/js/compiler.ts b/cli/js/compiler.ts index ed1fa15814fa6a..d959a1e9a0d9e1 100644 --- a/cli/js/compiler.ts +++ b/cli/js/compiler.ts @@ -1349,9 +1349,7 @@ interface RuntimeBundleResult { diagnostics: DiagnosticItem[]; } -function compileNew( - request: CompilerRequestCompileNew -): Promise<CompileResult> { +function compileNew(request: CompilerRequestCompileNew): CompileResult { const { bundle, config, diff --git a/cli/module_graph.rs b/cli/module_graph.rs index 6005374a59055f..862a5fd22b3e2e 100644 --- a/cli/module_graph.rs +++ b/cli/module_graph.rs @@ -2,6 +2,7 @@ #![allow(unused)] use crate::file_fetcher::SourceFileFetcher; +use crate::import_map::ImportMap; use crate::msg::MediaType; use crate::swc_util::analyze_dependencies_and_references; use crate::swc_util::TsReferenceKind; @@ -74,14 +75,19 @@ pub struct ModuleGraphFile { pub struct ModuleGraphLoader { file_fetcher: SourceFileFetcher, + maybe_import_map: Option<ImportMap>, to_visit: Vec<ModuleSpecifier>, pub graph: ModuleGraph, } impl ModuleGraphLoader { - pub fn new(file_fetcher: SourceFileFetcher) -> Self { + pub fn new( + file_fetcher: SourceFileFetcher, + maybe_import_map: Option<ImportMap>, + ) -> Self { Self { file_fetcher, + maybe_import_map, to_visit: vec![], graph: ModuleGraph(HashMap::new()), } @@ -136,13 +142,23 @@ impl ModuleGraphLoader { let (import_descs, ref_descs) = analyze_dependencies_and_references(&source_code, true)?; - // TODO(bartlomieju): apply import map, using State - // or should it be passed explicitly for import_desc in import_descs { - let resolved_specifier = ModuleSpecifier::resolve_import( - &import_desc.specifier, - &module_specifier.to_string(), - )?; + let maybe_resolved = + if let Some(import_map) = self.maybe_import_map.as_ref() { + import_map + .resolve(&import_desc.specifier, &module_specifier.to_string())? + } else { + None + }; + + let resolved_specifier = if let Some(resolved) = maybe_resolved { + resolved + } else { + ModuleSpecifier::resolve_import( + &import_desc.specifier, + &module_specifier.to_string(), + )? + }; let resolved_type_directive = if let Some(types_specifier) = import_desc.deno_types.as_ref() { @@ -249,6 +265,7 @@ mod tests { // ModuleSpecifier::resolve_url_or_path(ps).unwrap() // } + #[ignore] #[tokio::test] async fn source_graph_fetch() { let http_server_guard = crate::test_util::http_server(); @@ -259,7 +276,7 @@ mod tests { ) .unwrap(); let graph_loader = - ModuleGraphLoader::new(global_state.file_fetcher.clone()); + ModuleGraphLoader::new(global_state.file_fetcher.clone(), None); let graph = graph_loader.build_graph(&module_specifier).await.unwrap(); assert_eq!( @@ -315,6 +332,7 @@ mod tests { drop(http_server_guard); } + #[ignore] #[tokio::test] async fn source_graph_fetch_circular() { let http_server_guard = crate::test_util::http_server(); @@ -326,7 +344,7 @@ mod tests { .unwrap(); let graph_loader = - ModuleGraphLoader::new(global_state.file_fetcher.clone()); + ModuleGraphLoader::new(global_state.file_fetcher.clone(), None); let graph = graph_loader.build_graph(&module_specifier).await.unwrap(); assert_eq!( @@ -349,6 +367,7 @@ mod tests { drop(http_server_guard); } + #[ignore] #[tokio::test] async fn source_graph_type_references() { let http_server_guard = crate::test_util::http_server(); @@ -360,7 +379,7 @@ mod tests { .unwrap(); let graph_loader = - ModuleGraphLoader::new(global_state.file_fetcher.clone()); + ModuleGraphLoader::new(global_state.file_fetcher.clone(), None); let graph = graph_loader.build_graph(&module_specifier).await.unwrap(); eprintln!("json {:#?}", serde_json::to_value(&graph).unwrap()); @@ -440,6 +459,7 @@ mod tests { drop(http_server_guard); } + #[ignore] #[tokio::test] async fn source_graph_type_references2() { let http_server_guard = crate::test_util::http_server(); @@ -451,7 +471,7 @@ mod tests { .unwrap(); let graph_loader = - ModuleGraphLoader::new(global_state.file_fetcher.clone()); + ModuleGraphLoader::new(global_state.file_fetcher.clone(), None); let graph = graph_loader.build_graph(&module_specifier).await.unwrap(); eprintln!("{:#?}", serde_json::to_value(&graph).unwrap()); @@ -500,6 +520,7 @@ mod tests { drop(http_server_guard); } + #[ignore] #[tokio::test] async fn source_graph_type_references3() { let http_server_guard = crate::test_util::http_server(); @@ -511,7 +532,7 @@ mod tests { .unwrap(); let graph_loader = - ModuleGraphLoader::new(global_state.file_fetcher.clone()); + ModuleGraphLoader::new(global_state.file_fetcher.clone(), None); let graph = graph_loader.build_graph(&module_specifier).await.unwrap(); assert_eq!( diff --git a/cli/state.rs b/cli/state.rs index b36331085c0de0..53fa1b418ad653 100644 --- a/cli/state.rs +++ b/cli/state.rs @@ -248,7 +248,7 @@ impl State { } } -fn exit_unstable(api_name: &str) { +pub fn exit_unstable(api_name: &str) { eprintln!( "Unstable API '{}'. The --unstable flag must be provided.", api_name diff --git a/cli/swc_util.rs b/cli/swc_util.rs index d602c935bb8563..91c155bf6b7f98 100644 --- a/cli/swc_util.rs +++ b/cli/swc_util.rs @@ -476,7 +476,6 @@ pub fn analyze_dependencies_and_references( .collect(); // analyze comment from beginning of the file and find TS directives - eprintln!("module span {:?}", module_span); let comments = parser .comments .take_leading_comments(module_span.lo()) diff --git a/cli/tsc.rs b/cli/tsc.rs index 102abe0edeae43..383259b33964f9 100644 --- a/cli/tsc.rs +++ b/cli/tsc.rs @@ -8,6 +8,7 @@ use crate::file_fetcher::SourceFileFetcher; use crate::fmt; use crate::fs as deno_fs; use crate::global_state::GlobalState; +use crate::import_map::ImportMap; use crate::module_graph::ModuleGraphLoader; use crate::msg; use crate::op_error::OpError; @@ -15,6 +16,7 @@ use crate::ops; use crate::permissions::Permissions; use crate::source_maps::SourceMapGetter; use crate::startup_data; +use crate::state::exit_unstable; use crate::state::State; use crate::state::*; use crate::tokio_util; @@ -383,8 +385,18 @@ impl TsCompiler { ); eprintln!("Bundling {}", module_specifier.to_string()); + let import_map: Option<ImportMap> = + match global_state.flags.import_map_path.as_ref() { + None => None, + Some(file_path) => { + if !global_state.flags.unstable { + exit_unstable("--importmap") + } + Some(ImportMap::load(file_path)?) + } + }; let module_graph_loader = - ModuleGraphLoader::new(global_state.file_fetcher.clone()); + ModuleGraphLoader::new(global_state.file_fetcher.clone(), import_map); let module_graph = module_graph_loader.build_graph(&module_specifier).await?; let module_graph_json = @@ -431,17 +443,19 @@ impl TsCompiler { return Err(ErrBox::from(bundle_response.diagnostics)); } + let output_string = fmt::format_text(&bundle_response.bundle_output)?; + if let Some(out_file_) = out_file.as_ref() { eprintln!("Emitting bundle to {:?}", out_file_); - let output_bytes = bundle_response.bundle_output.as_bytes(); + let output_bytes = output_string.as_bytes(); let output_len = output_bytes.len(); deno_fs::write_file(out_file_, output_bytes, 0o666)?; // TODO(bartlomieju): add "humanFileSize" method eprintln!("{} bytes emmited.", output_len); } else { - println!("{}", bundle_response.bundle_output); + println!("{}", output_string); } Ok(()) @@ -652,8 +666,18 @@ impl TsCompiler { let source_file_ = source_file.clone(); let module_url = source_file.url.clone(); let module_specifier = ModuleSpecifier::from(source_file.url.clone()); + let import_map: Option<ImportMap> = + match global_state.flags.import_map_path.as_ref() { + None => None, + Some(file_path) => { + if !global_state.flags.unstable { + exit_unstable("--importmap") + } + Some(ImportMap::load(file_path)?) + } + }; let module_graph_loader = - ModuleGraphLoader::new(global_state.file_fetcher.clone()); + ModuleGraphLoader::new(global_state.file_fetcher.clone(), import_map); let module_graph = module_graph_loader.build_graph(&module_specifier).await?; let module_graph_json = From cfdef1e3553f0627d53c2632be019d4639f671e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= <biwanczuk@gmail.com> Date: Mon, 11 May 2020 02:23:32 +0200 Subject: [PATCH 19/47] fixes --- cli/global_state.rs | 4 ++-- cli/module_graph.rs | 16 ++++++++++------ cli/tsc.rs | 22 ++++++++++++++++------ 3 files changed, 28 insertions(+), 14 deletions(-) diff --git a/cli/global_state.rs b/cli/global_state.rs index e8125af731b4b0..7be983d79dc220 100644 --- a/cli/global_state.rs +++ b/cli/global_state.rs @@ -115,14 +115,14 @@ impl GlobalState { | msg::MediaType::JSX => { state1 .ts_compiler - .new_compile(state1.clone(), &out, target_lib) + .new_compile(state1.clone(), &out, target_lib, permissions) .await } msg::MediaType::JavaScript => { if state1.ts_compiler.compile_js { state2 .ts_compiler - .new_compile(state1.clone(), &out, target_lib) + .new_compile(state1.clone(), &out, target_lib, permissions) .await } else { if let Some(types_url) = out.types_url.clone() { diff --git a/cli/module_graph.rs b/cli/module_graph.rs index 862a5fd22b3e2e..107777b60a5231 100644 --- a/cli/module_graph.rs +++ b/cli/module_graph.rs @@ -4,6 +4,7 @@ use crate::file_fetcher::SourceFileFetcher; use crate::import_map::ImportMap; use crate::msg::MediaType; +use crate::permissions::Permissions; use crate::swc_util::analyze_dependencies_and_references; use crate::swc_util::TsReferenceKind; use deno_core::ErrBox; @@ -74,6 +75,7 @@ pub struct ModuleGraphFile { } pub struct ModuleGraphLoader { + permissions: Permissions, file_fetcher: SourceFileFetcher, maybe_import_map: Option<ImportMap>, to_visit: Vec<ModuleSpecifier>, @@ -84,9 +86,11 @@ impl ModuleGraphLoader { pub fn new( file_fetcher: SourceFileFetcher, maybe_import_map: Option<ImportMap>, + permissions: Permissions, ) -> Self { Self { file_fetcher, + permissions, maybe_import_map, to_visit: vec![], graph: ModuleGraph(HashMap::new()), @@ -114,7 +118,7 @@ impl ModuleGraphLoader { let source_file = self .file_fetcher - .fetch_source_file(module_specifier, None) + .fetch_source_file(module_specifier, None, self.permissions.clone()) .await?; let mut imports = vec![]; @@ -276,7 +280,7 @@ mod tests { ) .unwrap(); let graph_loader = - ModuleGraphLoader::new(global_state.file_fetcher.clone(), None); + ModuleGraphLoader::new(global_state.file_fetcher.clone(), None, Permissions::allow_all()); let graph = graph_loader.build_graph(&module_specifier).await.unwrap(); assert_eq!( @@ -344,7 +348,7 @@ mod tests { .unwrap(); let graph_loader = - ModuleGraphLoader::new(global_state.file_fetcher.clone(), None); + ModuleGraphLoader::new(global_state.file_fetcher.clone(), None, Permissions::allow_all()); let graph = graph_loader.build_graph(&module_specifier).await.unwrap(); assert_eq!( @@ -379,7 +383,7 @@ mod tests { .unwrap(); let graph_loader = - ModuleGraphLoader::new(global_state.file_fetcher.clone(), None); + ModuleGraphLoader::new(global_state.file_fetcher.clone(), None, Permissions::allow_all()); let graph = graph_loader.build_graph(&module_specifier).await.unwrap(); eprintln!("json {:#?}", serde_json::to_value(&graph).unwrap()); @@ -471,7 +475,7 @@ mod tests { .unwrap(); let graph_loader = - ModuleGraphLoader::new(global_state.file_fetcher.clone(), None); + ModuleGraphLoader::new(global_state.file_fetcher.clone(), None, Permissions::allow_all()); let graph = graph_loader.build_graph(&module_specifier).await.unwrap(); eprintln!("{:#?}", serde_json::to_value(&graph).unwrap()); @@ -532,7 +536,7 @@ mod tests { .unwrap(); let graph_loader = - ModuleGraphLoader::new(global_state.file_fetcher.clone(), None); + ModuleGraphLoader::new(global_state.file_fetcher.clone(), None, Permissions::allow_all()); let graph = graph_loader.build_graph(&module_specifier).await.unwrap(); assert_eq!( diff --git a/cli/tsc.rs b/cli/tsc.rs index 383259b33964f9..c7a2bcfba7e617 100644 --- a/cli/tsc.rs +++ b/cli/tsc.rs @@ -395,8 +395,12 @@ impl TsCompiler { Some(ImportMap::load(file_path)?) } }; - let module_graph_loader = - ModuleGraphLoader::new(global_state.file_fetcher.clone(), import_map); + let permissions = Permissions::allow_all(); + let module_graph_loader = ModuleGraphLoader::new( + global_state.file_fetcher.clone(), + import_map, + permissions.clone(), + ); let module_graph = module_graph_loader.build_graph(&module_specifier).await?; let module_graph_json = @@ -433,7 +437,8 @@ impl TsCompiler { let req_msg = j.to_string().into_boxed_str().into_boxed_bytes(); - let msg = execute_in_thread(global_state.clone(), req_msg).await?; + let msg = + execute_in_thread(global_state.clone(), permissions, req_msg).await?; let json_str = std::str::from_utf8(&msg).unwrap(); debug!("Message: {}", json_str); @@ -635,6 +640,7 @@ impl TsCompiler { global_state: GlobalState, source_file: &SourceFile, target: TargetLib, + permissions: Permissions, ) -> Result<CompiledModule, ErrBox> { if self.has_compiled(&source_file.url) { return self.get_compiled_module(&source_file.url); @@ -676,8 +682,11 @@ impl TsCompiler { Some(ImportMap::load(file_path)?) } }; - let module_graph_loader = - ModuleGraphLoader::new(global_state.file_fetcher.clone(), import_map); + let module_graph_loader = ModuleGraphLoader::new( + global_state.file_fetcher.clone(), + import_map, + permissions.clone(), + ); let module_graph = module_graph_loader.build_graph(&module_specifier).await?; let module_graph_json = @@ -725,7 +734,8 @@ impl TsCompiler { module_url.to_string() ); - let msg = execute_in_thread(global_state.clone(), req_msg).await?; + let msg = + execute_in_thread(global_state.clone(), permissions, req_msg).await?; let json_str = std::str::from_utf8(&msg).unwrap(); let compile_response: CompileResponse = serde_json::from_str(json_str)?; From 7827a56e7d60c19686ff6c32ac4c4f81f17fa5b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= <biwanczuk@gmail.com> Date: Mon, 11 May 2020 02:29:10 +0200 Subject: [PATCH 20/47] fix integration tests --- cli/tests/error_011_bad_module_specifier.ts.out | 10 +--------- cli/tests/error_type_definitions.ts.out | 11 +---------- 2 files changed, 2 insertions(+), 19 deletions(-) diff --git a/cli/tests/error_011_bad_module_specifier.ts.out b/cli/tests/error_011_bad_module_specifier.ts.out index 39726d5c633406..e6f9b2321b0eed 100644 --- a/cli/tests/error_011_bad_module_specifier.ts.out +++ b/cli/tests/error_011_bad_module_specifier.ts.out @@ -1,9 +1 @@ -[WILDCARD]error: Uncaught URIError: relative import path "bad-module.ts" not prefixed with / or ./ or ../ Imported from "[WILDCARD]/error_011_bad_module_specifier.ts" - at unwrapResponse ($deno$/ops/dispatch_json.ts:[WILDCARD]) - at Object.sendSync ($deno$/ops/dispatch_json.ts:[WILDCARD]) - at resolveModules ($deno$/compiler.ts:[WILDCARD]) - at processImports ($deno$/compiler.ts:[WILDCARD]) - at processImports ($deno$/compiler.ts:[WILDCARD]) - at async compile ($deno$/compiler.ts:[WILDCARD]) - at async tsCompilerOnMessage ($deno$/compiler.ts:[WILDCARD]) - at async workerMessageRecvCallback ([WILDCARD]runtime_worker.ts:[WILDCARD]) +[WILDCARD]error: relative import path "bad-module.ts" not prefixed with / or ./ or ../ Imported from "[WILDCARD]/error_011_bad_module_specifier.ts" diff --git a/cli/tests/error_type_definitions.ts.out b/cli/tests/error_type_definitions.ts.out index bacca4f80e9061..32c3c9b5253b02 100644 --- a/cli/tests/error_type_definitions.ts.out +++ b/cli/tests/error_type_definitions.ts.out @@ -1,10 +1 @@ -[WILDCARD]error: Uncaught URIError: relative import path "baz" not prefixed with / or ./ or ../ Imported from "[WILDCARD]/type_definitions/bar.d.ts" - at unwrapResponse ($deno$/ops/dispatch_json.ts:[WILDCARD]) - at Object.sendSync ($deno$/ops/dispatch_json.ts:[WILDCARD]) - at resolveModules ($deno$/compiler.ts:[WILDCARD]) - at processImports ($deno$/compiler.ts:[WILDCARD]) - at processImports ($deno$/compiler.ts:[WILDCARD]) - at async processImports ($deno$/compiler.ts:[WILDCARD]) - at async compile ($deno$/compiler.ts:[WILDCARD]) - at async tsCompilerOnMessage ($deno$/compiler.ts:[WILDCARD]) - at async workerMessageRecvCallback ([WILDCARD]runtime_worker.ts:[WILDCARD]) +[WILDCARD]error: relative import path "baz" not prefixed with / or ./ or ../ Imported from "[WILDCARD]/type_definitions/bar.d.ts" From ae1d69321cb84148c0140b391c9a253977c44146 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= <biwanczuk@gmail.com> Date: Mon, 11 May 2020 02:29:55 +0200 Subject: [PATCH 21/47] fmt --- cli/module_graph.rs | 35 +++++++++++++++++++++++++---------- 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/cli/module_graph.rs b/cli/module_graph.rs index 107777b60a5231..15a0889af69c44 100644 --- a/cli/module_graph.rs +++ b/cli/module_graph.rs @@ -279,8 +279,11 @@ mod tests { "http://localhost:4545/cli/tests/019_media_types.ts", ) .unwrap(); - let graph_loader = - ModuleGraphLoader::new(global_state.file_fetcher.clone(), None, Permissions::allow_all()); + let graph_loader = ModuleGraphLoader::new( + global_state.file_fetcher.clone(), + None, + Permissions::allow_all(), + ); let graph = graph_loader.build_graph(&module_specifier).await.unwrap(); assert_eq!( @@ -347,8 +350,11 @@ mod tests { ) .unwrap(); - let graph_loader = - ModuleGraphLoader::new(global_state.file_fetcher.clone(), None, Permissions::allow_all()); + let graph_loader = ModuleGraphLoader::new( + global_state.file_fetcher.clone(), + None, + Permissions::allow_all(), + ); let graph = graph_loader.build_graph(&module_specifier).await.unwrap(); assert_eq!( @@ -382,8 +388,11 @@ mod tests { ) .unwrap(); - let graph_loader = - ModuleGraphLoader::new(global_state.file_fetcher.clone(), None, Permissions::allow_all()); + let graph_loader = ModuleGraphLoader::new( + global_state.file_fetcher.clone(), + None, + Permissions::allow_all(), + ); let graph = graph_loader.build_graph(&module_specifier).await.unwrap(); eprintln!("json {:#?}", serde_json::to_value(&graph).unwrap()); @@ -474,8 +483,11 @@ mod tests { ) .unwrap(); - let graph_loader = - ModuleGraphLoader::new(global_state.file_fetcher.clone(), None, Permissions::allow_all()); + let graph_loader = ModuleGraphLoader::new( + global_state.file_fetcher.clone(), + None, + Permissions::allow_all(), + ); let graph = graph_loader.build_graph(&module_specifier).await.unwrap(); eprintln!("{:#?}", serde_json::to_value(&graph).unwrap()); @@ -535,8 +547,11 @@ mod tests { ) .unwrap(); - let graph_loader = - ModuleGraphLoader::new(global_state.file_fetcher.clone(), None, Permissions::allow_all()); + let graph_loader = ModuleGraphLoader::new( + global_state.file_fetcher.clone(), + None, + Permissions::allow_all(), + ); let graph = graph_loader.build_graph(&module_specifier).await.unwrap(); assert_eq!( From ccf641307d65933a0eb0587817084d4a5460688f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= <biwanczuk@gmail.com> Date: Mon, 11 May 2020 03:04:31 +0200 Subject: [PATCH 22/47] tweak tests --- cli/global_state.rs | 17 ++++- cli/lib.rs | 1 + cli/module_graph.rs | 62 +++++++++++++++---- cli/state.rs | 1 + cli/tests/037_fetch_multiple.out | 4 +- cli/tests/error_004_missing_module.ts.out | 9 +-- cli/tests/error_006_import_ext_failure.ts.out | 9 +-- ...ror_local_static_import_from_remote.ts.out | 9 +-- cli/tsc.rs | 3 + 9 files changed, 75 insertions(+), 40 deletions(-) diff --git a/cli/global_state.rs b/cli/global_state.rs index 7be983d79dc220..e4ecf65d5df6d2 100644 --- a/cli/global_state.rs +++ b/cli/global_state.rs @@ -95,6 +95,7 @@ impl GlobalState { maybe_referrer: Option<ModuleSpecifier>, target_lib: TargetLib, permissions: Permissions, + is_dyn_import: bool, ) -> Result<CompiledModule, ErrBox> { let state1 = self.clone(); let state2 = self.clone(); @@ -115,14 +116,26 @@ impl GlobalState { | msg::MediaType::JSX => { state1 .ts_compiler - .new_compile(state1.clone(), &out, target_lib, permissions) + .new_compile( + state1.clone(), + &out, + target_lib, + permissions, + is_dyn_import, + ) .await } msg::MediaType::JavaScript => { if state1.ts_compiler.compile_js { state2 .ts_compiler - .new_compile(state1.clone(), &out, target_lib, permissions) + .new_compile( + state1.clone(), + &out, + target_lib, + permissions, + is_dyn_import, + ) .await } else { if let Some(types_url) = out.types_url.clone() { diff --git a/cli/lib.rs b/cli/lib.rs index db62e530a4f221..65ed6248f7e353 100644 --- a/cli/lib.rs +++ b/cli/lib.rs @@ -212,6 +212,7 @@ async fn print_file_info( None, TargetLib::Main, Permissions::allow_all(), + false, ) .await?; diff --git a/cli/module_graph.rs b/cli/module_graph.rs index 15a0889af69c44..ab653aaae79f8f 100644 --- a/cli/module_graph.rs +++ b/cli/module_graph.rs @@ -4,6 +4,7 @@ use crate::file_fetcher::SourceFileFetcher; use crate::import_map::ImportMap; use crate::msg::MediaType; +use crate::op_error::OpError; use crate::permissions::Permissions; use crate::swc_util::analyze_dependencies_and_references; use crate::swc_util::TsReferenceKind; @@ -78,8 +79,9 @@ pub struct ModuleGraphLoader { permissions: Permissions, file_fetcher: SourceFileFetcher, maybe_import_map: Option<ImportMap>, - to_visit: Vec<ModuleSpecifier>, + to_visit: Vec<(ModuleSpecifier, Option<ModuleSpecifier>)>, pub graph: ModuleGraph, + is_dyn_import: bool, } impl ModuleGraphLoader { @@ -87,6 +89,7 @@ impl ModuleGraphLoader { file_fetcher: SourceFileFetcher, maybe_import_map: Option<ImportMap>, permissions: Permissions, + is_dyn_import: bool, ) -> Self { Self { file_fetcher, @@ -94,6 +97,7 @@ impl ModuleGraphLoader { maybe_import_map, to_visit: vec![], graph: ModuleGraph(HashMap::new()), + is_dyn_import, } } @@ -101,9 +105,9 @@ impl ModuleGraphLoader { mut self, specifier: &ModuleSpecifier, ) -> Result<HashMap<String, ModuleGraphFile>, ErrBox> { - self.to_visit.push(specifier.to_owned()); - while let Some(spec) = self.to_visit.pop() { - self.visit_module(&spec).await?; + self.to_visit.push((specifier.to_owned(), None)); + while let Some((spec, maybe_referrer)) = self.to_visit.pop() { + self.visit_module(&spec, maybe_referrer).await?; } Ok(self.graph.0) } @@ -111,14 +115,39 @@ impl ModuleGraphLoader { async fn visit_module( &mut self, module_specifier: &ModuleSpecifier, + maybe_referrer: Option<ModuleSpecifier>, ) -> Result<(), ErrBox> { if self.graph.0.contains_key(&module_specifier.to_string()) { return Ok(()); } + if !self.is_dyn_import { + // Verify that remote file doesn't try to statically import local file. + if let Some(referrer) = maybe_referrer.as_ref() { + let referrer_url = referrer.as_url(); + match referrer_url.scheme() { + "http" | "https" => { + let specifier_url = module_specifier.as_url(); + match specifier_url.scheme() { + "http" | "https" => {} + _ => { + let e = OpError::permission_denied("Remote module are not allowed to statically import local modules. Use dynamic import instead.".to_string()); + return Err(e.into()); + } + } + } + _ => {} + } + } + } + let source_file = self .file_fetcher - .fetch_source_file(module_specifier, None, self.permissions.clone()) + .fetch_source_file( + module_specifier, + maybe_referrer, + self.permissions.clone(), + ) .await?; let mut imports = vec![]; @@ -187,16 +216,19 @@ impl ModuleGraphLoader { .get(&import_descriptor.resolved_specifier.to_string()) .is_none() { - self - .to_visit - .push(import_descriptor.resolved_specifier.clone()); + self.to_visit.push(( + import_descriptor.resolved_specifier.clone(), + Some(module_specifier.clone()), + )); } if let Some(type_dir_url) = import_descriptor.resolved_type_directive.as_ref() { if self.graph.0.get(&type_dir_url.to_string()).is_none() { - self.to_visit.push(type_dir_url.clone()); + self + .to_visit + .push((type_dir_url.clone(), Some(module_specifier.clone()))); } } @@ -219,9 +251,10 @@ impl ModuleGraphLoader { .get(&reference_descriptor.resolved_specifier.to_string()) .is_none() { - self - .to_visit - .push(reference_descriptor.resolved_specifier.clone()); + self.to_visit.push(( + reference_descriptor.resolved_specifier.clone(), + Some(module_specifier.clone()), + )); } match ref_desc.kind { @@ -283,6 +316,7 @@ mod tests { global_state.file_fetcher.clone(), None, Permissions::allow_all(), + false, ); let graph = graph_loader.build_graph(&module_specifier).await.unwrap(); @@ -354,6 +388,7 @@ mod tests { global_state.file_fetcher.clone(), None, Permissions::allow_all(), + false, ); let graph = graph_loader.build_graph(&module_specifier).await.unwrap(); @@ -392,6 +427,7 @@ mod tests { global_state.file_fetcher.clone(), None, Permissions::allow_all(), + false, ); let graph = graph_loader.build_graph(&module_specifier).await.unwrap(); @@ -487,6 +523,7 @@ mod tests { global_state.file_fetcher.clone(), None, Permissions::allow_all(), + false, ); let graph = graph_loader.build_graph(&module_specifier).await.unwrap(); @@ -551,6 +588,7 @@ mod tests { global_state.file_fetcher.clone(), None, Permissions::allow_all(), + false, ); let graph = graph_loader.build_graph(&module_specifier).await.unwrap(); diff --git a/cli/state.rs b/cli/state.rs index 53fa1b418ad653..3d2358c8bb1ea9 100644 --- a/cli/state.rs +++ b/cli/state.rs @@ -328,6 +328,7 @@ impl ModuleLoader for State { maybe_referrer, target_lib, permissions, + is_dyn_import, ) .await?; Ok(deno_core::ModuleSource { diff --git a/cli/tests/037_fetch_multiple.out b/cli/tests/037_fetch_multiple.out index cdb6fe2ba747d4..1a703a10c42009 100644 --- a/cli/tests/037_fetch_multiple.out +++ b/cli/tests/037_fetch_multiple.out @@ -1,5 +1,5 @@ -Compile [WILDCARD]/fetch/test.ts Download http://localhost:4545/cli/tests/subdir/mod2.ts Download http://localhost:4545/cli/tests/subdir/print_hello.ts -Compile [WILDCARD]/fetch/other.ts +Compile [WILDCARD]/fetch/test.ts Download http://localhost:4545/cli/tests/subdir/mt_text_typescript.t1.ts +Compile [WILDCARD]/fetch/other.ts diff --git a/cli/tests/error_004_missing_module.ts.out b/cli/tests/error_004_missing_module.ts.out index b93f2e61321b99..d851882ebc7a19 100644 --- a/cli/tests/error_004_missing_module.ts.out +++ b/cli/tests/error_004_missing_module.ts.out @@ -1,8 +1 @@ -[WILDCARD]error: Uncaught NotFound: Cannot resolve module "[WILDCARD]/bad-module.ts" from "[WILDCARD]/error_004_missing_module.ts" - at unwrapResponse ([WILDCARD]dispatch_json.ts:[WILDCARD]) - at Object.sendAsync ([WILDCARD]dispatch_json.ts:[WILDCARD]) - at async processImports ($deno$/compiler.ts:[WILDCARD]) - at async processImports ($deno$/compiler.ts:[WILDCARD]) - at async compile ($deno$/compiler.ts:[WILDCARD]) - at async tsCompilerOnMessage ($deno$/compiler.ts:[WILDCARD]) - at async workerMessageRecvCallback ([WILDCARD]runtime_worker.ts:[WILDCARD]) +[WILDCARD]error: Cannot resolve module "[WILDCARD]/bad-module.ts" from "[WILDCARD]/error_004_missing_module.ts" diff --git a/cli/tests/error_006_import_ext_failure.ts.out b/cli/tests/error_006_import_ext_failure.ts.out index 5c89adeff86dee..c44d5e746940ac 100644 --- a/cli/tests/error_006_import_ext_failure.ts.out +++ b/cli/tests/error_006_import_ext_failure.ts.out @@ -1,8 +1 @@ -[WILDCARD]error: Uncaught NotFound: Cannot resolve module "[WILDCARD]/non-existent" from "[WILDCARD]/error_006_import_ext_failure.ts" - at unwrapResponse ([WILDCARD]dispatch_json.ts:[WILDCARD]) - at Object.sendAsync ([WILDCARD]dispatch_json.ts:[WILDCARD]) - at async processImports ($deno$/compiler.ts:[WILDCARD]) - at async processImports ($deno$/compiler.ts:[WILDCARD]) - at async compile ($deno$/compiler.ts:[WILDCARD]) - at async tsCompilerOnMessage ($deno$/compiler.ts:[WILDCARD]) - at async workerMessageRecvCallback ([WILDCARD]runtime_worker.ts:[WILDCARD]) +[WILDCARD]error: Cannot resolve module "[WILDCARD]/non-existent" from "[WILDCARD]/error_006_import_ext_failure.ts" diff --git a/cli/tests/error_local_static_import_from_remote.ts.out b/cli/tests/error_local_static_import_from_remote.ts.out index af3c8852fe42f5..4e6a9d4e7046ab 100644 --- a/cli/tests/error_local_static_import_from_remote.ts.out +++ b/cli/tests/error_local_static_import_from_remote.ts.out @@ -1,9 +1,2 @@ [WILDCARD] -error: Uncaught PermissionDenied: Remote module are not allowed to statically import local modules. Use dynamic import instead. - at unwrapResponse ($deno$/ops/dispatch_json.ts:[WILDCARD]) - at Object.sendAsync ($deno$/ops/dispatch_json.ts:[WILDCARD]) - at async processImports ($deno$/compiler.ts:[WILDCARD]) - at async processImports ($deno$/compiler.ts:[WILDCARD]) - at async compile ($deno$/compiler.ts:[WILDCARD]) - at async tsCompilerOnMessage ($deno$/compiler.ts:[WILDCARD]) - at async workerMessageRecvCallback ($deno$/runtime_worker.ts:[WILDCARD]) +error: Remote module are not allowed to statically import local modules. Use dynamic import instead. diff --git a/cli/tsc.rs b/cli/tsc.rs index c7a2bcfba7e617..afc553fc9b3576 100644 --- a/cli/tsc.rs +++ b/cli/tsc.rs @@ -400,6 +400,7 @@ impl TsCompiler { global_state.file_fetcher.clone(), import_map, permissions.clone(), + false, ); let module_graph = module_graph_loader.build_graph(&module_specifier).await?; @@ -641,6 +642,7 @@ impl TsCompiler { source_file: &SourceFile, target: TargetLib, permissions: Permissions, + is_dyn_import: bool, ) -> Result<CompiledModule, ErrBox> { if self.has_compiled(&source_file.url) { return self.get_compiled_module(&source_file.url); @@ -686,6 +688,7 @@ impl TsCompiler { global_state.file_fetcher.clone(), import_map, permissions.clone(), + is_dyn_import, ); let module_graph = module_graph_loader.build_graph(&module_specifier).await?; From 4ae457c874ba53174b6994dc56090709c694c561 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= <biwanczuk@gmail.com> Date: Mon, 11 May 2020 13:26:02 +0200 Subject: [PATCH 23/47] fmt --- cli/state.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cli/state.rs b/cli/state.rs index 3d2358c8bb1ea9..d532906944d7bd 100644 --- a/cli/state.rs +++ b/cli/state.rs @@ -318,8 +318,8 @@ impl ModuleLoader for State { let permissions = if state.is_main { Permissions::allow_all() } else { - state.permissions.clone() - }; + state.permissions.clone() + }; let fut = async move { let compiled_module = global_state From d2eb3b3066bbbd9b2a56204d488010eb84fe97e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= <biwanczuk@gmail.com> Date: Thu, 14 May 2020 20:39:38 +0200 Subject: [PATCH 24/47] concurrent downloads --- cli/file_fetcher.rs | 12 +++ cli/module_graph.rs | 195 +++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 204 insertions(+), 3 deletions(-) diff --git a/cli/file_fetcher.rs b/cli/file_fetcher.rs index cbfb340ddc6121..32971e71f06b91 100644 --- a/cli/file_fetcher.rs +++ b/cli/file_fetcher.rs @@ -147,6 +147,18 @@ impl SourceFileFetcher { self.source_file_cache.set(specifier.to_string(), file); } + // TODO(bartlomieju): remove + pub async fn new_fetch_source_file( + &self, + specifier: ModuleSpecifier, + maybe_referrer: Option<ModuleSpecifier>, + permissions: Permissions, + ) -> Result<SourceFile, ErrBox> { + self + .fetch_source_file(&specifier, maybe_referrer, permissions) + .await + } + pub async fn fetch_source_file( &self, specifier: &ModuleSpecifier, diff --git a/cli/module_graph.rs b/cli/module_graph.rs index ab653aaae79f8f..9e46511c34f6a2 100644 --- a/cli/module_graph.rs +++ b/cli/module_graph.rs @@ -1,6 +1,7 @@ // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. #![allow(unused)] +use crate::file_fetcher::SourceFile; use crate::file_fetcher::SourceFileFetcher; use crate::import_map::ImportMap; use crate::msg::MediaType; @@ -10,9 +11,14 @@ use crate::swc_util::analyze_dependencies_and_references; use crate::swc_util::TsReferenceKind; use deno_core::ErrBox; use deno_core::ModuleSpecifier; +use futures::stream::FuturesUnordered; +use futures::stream::StreamExt; +use futures::Future; +use futures::FutureExt; use serde::Serialize; use serde::Serializer; use std::collections::HashMap; +use std::pin::Pin; fn serialize_module_specifier<S>( spec: &ModuleSpecifier, @@ -75,10 +81,13 @@ pub struct ModuleGraphFile { pub source_code: String, } +type SourceFileFuture = Pin<Box<dyn Future<Output = Result<SourceFile, ErrBox>>>>; + pub struct ModuleGraphLoader { permissions: Permissions, file_fetcher: SourceFileFetcher, maybe_import_map: Option<ImportMap>, + pending_downloads: FuturesUnordered<SourceFileFuture>, to_visit: Vec<(ModuleSpecifier, Option<ModuleSpecifier>)>, pub graph: ModuleGraph, is_dyn_import: bool, @@ -96,6 +105,7 @@ impl ModuleGraphLoader { permissions, maybe_import_map, to_visit: vec![], + pending_downloads: FuturesUnordered::new(), graph: ModuleGraph(HashMap::new()), is_dyn_import, } @@ -105,13 +115,192 @@ impl ModuleGraphLoader { mut self, specifier: &ModuleSpecifier, ) -> Result<HashMap<String, ModuleGraphFile>, ErrBox> { - self.to_visit.push((specifier.to_owned(), None)); - while let Some((spec, maybe_referrer)) = self.to_visit.pop() { - self.visit_module(&spec, maybe_referrer).await?; + self.download_module(specifier.clone(), None)?; + + loop { + let load_result = self.pending_downloads.next().await.unwrap(); + let source_file = load_result?; + let spec = ModuleSpecifier::from(source_file.url.clone()); + self.new_visit_module(&spec, source_file)?; + if self.pending_downloads.is_empty() { + break; + } } + Ok(self.graph.0) } + fn download_module( + &mut self, + module_specifier: ModuleSpecifier, + maybe_referrer: Option<ModuleSpecifier>, + ) -> Result<(), ErrBox> { + if self.graph.0.contains_key(&module_specifier.to_string()) { + return Ok(()); + } + + if !self.is_dyn_import { + // Verify that remote file doesn't try to statically import local file. + if let Some(referrer) = maybe_referrer.as_ref() { + let referrer_url = referrer.as_url(); + match referrer_url.scheme() { + "http" | "https" => { + let specifier_url = module_specifier.as_url(); + match specifier_url.scheme() { + "http" | "https" => {} + _ => { + let e = OpError::permission_denied("Remote module are not allowed to statically import local modules. Use dynamic import instead.".to_string()); + return Err(e.into()); + } + } + } + _ => {} + } + } + } + + let spec = module_specifier; + let file_fetcher = self.file_fetcher.clone(); + let perms = self.permissions.clone(); + + let load_future = async move { + file_fetcher + .new_fetch_source_file(spec, maybe_referrer, perms) + .await + } + .boxed_local(); + + self.pending_downloads.push(load_future); + Ok(()) + } + + fn new_visit_module( + &mut self, + module_specifier: &ModuleSpecifier, + source_file: SourceFile, + ) -> Result<(), ErrBox> { + let mut imports = vec![]; + let mut referenced_files = vec![]; + let mut lib_directives = vec![]; + let mut types_directives = vec![]; + let mut type_headers = vec![]; + + let source_code = String::from_utf8(source_file.source_code)?; + + if source_file.media_type == MediaType::JavaScript + || source_file.media_type == MediaType::TypeScript + { + if let Some(types_specifier) = source_file.types_header { + let type_header = ReferenceDescriptor { + specifier: types_specifier.to_string(), + resolved_specifier: ModuleSpecifier::resolve_import( + &types_specifier, + &module_specifier.to_string(), + )?, + }; + type_headers.push(type_header); + } + + let (import_descs, ref_descs) = + analyze_dependencies_and_references(&source_code, true)?; + + for import_desc in import_descs { + let maybe_resolved = + if let Some(import_map) = self.maybe_import_map.as_ref() { + import_map + .resolve(&import_desc.specifier, &module_specifier.to_string())? + } else { + None + }; + + let resolved_specifier = if let Some(resolved) = maybe_resolved { + resolved + } else { + ModuleSpecifier::resolve_import( + &import_desc.specifier, + &module_specifier.to_string(), + )? + }; + + let resolved_type_directive = + if let Some(types_specifier) = import_desc.deno_types.as_ref() { + Some(ModuleSpecifier::resolve_import( + &types_specifier, + &module_specifier.to_string(), + )?) + } else { + None + }; + + let import_descriptor = ImportDescriptor { + specifier: import_desc.specifier.to_string(), + resolved_specifier, + type_directive: import_desc.deno_types, + resolved_type_directive, + }; + + self.download_module( + import_descriptor.resolved_specifier.clone(), + Some(module_specifier.clone()), + )?; + + if let Some(type_dir_url) = + import_descriptor.resolved_type_directive.as_ref() + { + self.download_module( + type_dir_url.clone(), + Some(module_specifier.clone()), + )?; + } + + imports.push(import_descriptor); + } + + for ref_desc in ref_descs { + let resolved_specifier = ModuleSpecifier::resolve_import( + &ref_desc.specifier, + &module_specifier.to_string(), + )?; + let reference_descriptor = ReferenceDescriptor { + specifier: ref_desc.specifier.to_string(), + resolved_specifier, + }; + + self.download_module( + reference_descriptor.resolved_specifier.clone(), + Some(module_specifier.clone()), + )?; + + match ref_desc.kind { + TsReferenceKind::Lib => { + lib_directives.push(reference_descriptor); + } + TsReferenceKind::Types => { + types_directives.push(reference_descriptor); + } + TsReferenceKind::Path => { + referenced_files.push(reference_descriptor); + } + } + } + } + + self.graph.0.insert( + module_specifier.to_string(), + ModuleGraphFile { + specifier: module_specifier.to_string(), + media_type: source_file.media_type, + source_code, + imports, + referenced_files, + lib_directives, + types_directives, + type_headers, + }, + ); + Ok(()) + } + async fn visit_module( &mut self, module_specifier: &ModuleSpecifier, From 10f1ce8e7fc1076eec21e0ebe09f24118bf9dd71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= <biwanczuk@gmail.com> Date: Thu, 14 May 2020 20:43:59 +0200 Subject: [PATCH 25/47] fmt --- cli/module_graph.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cli/module_graph.rs b/cli/module_graph.rs index 9e46511c34f6a2..c04c05fd8a865c 100644 --- a/cli/module_graph.rs +++ b/cli/module_graph.rs @@ -81,7 +81,8 @@ pub struct ModuleGraphFile { pub source_code: String, } -type SourceFileFuture = Pin<Box<dyn Future<Output = Result<SourceFile, ErrBox>>>>; +type SourceFileFuture = + Pin<Box<dyn Future<Output = Result<SourceFile, ErrBox>>>>; pub struct ModuleGraphLoader { permissions: Permissions, From 28ce2020aa42714b49365d8ec9c0d62ce3706253 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= <biwanczuk@gmail.com> Date: Thu, 14 May 2020 21:06:02 +0200 Subject: [PATCH 26/47] cleanup --- cli/global_state.rs | 10 +- cli/main.rs | 2 +- cli/module_graph.rs | 180 +---------------------------- cli/tests/type_definitions/bar.js | 5 + cli/tsc.rs | 184 +++--------------------------- 5 files changed, 24 insertions(+), 357 deletions(-) create mode 100644 cli/tests/type_definitions/bar.js diff --git a/cli/global_state.rs b/cli/global_state.rs index 3bec4a500c7fc7..90961ed6271a8c 100644 --- a/cli/global_state.rs +++ b/cli/global_state.rs @@ -116,20 +116,14 @@ impl GlobalState { | msg::MediaType::JSX => { state1 .ts_compiler - .new_compile( - state1.clone(), - &out, - target_lib, - permissions, - is_dyn_import, - ) + .compile(state1.clone(), &out, target_lib, permissions, is_dyn_import) .await } msg::MediaType::JavaScript => { if state1.ts_compiler.compile_js { state2 .ts_compiler - .new_compile( + .compile( state1.clone(), &out, target_lib, diff --git a/cli/main.rs b/cli/main.rs index 32b25e4f632c3b..cadb76fb4b108b 100644 --- a/cli/main.rs +++ b/cli/main.rs @@ -390,7 +390,7 @@ async fn bundle_command( debug!(">>>>> bundle START"); let bundle_result = global_state .ts_compiler - .new_bundle(global_state.clone(), module_name, out_file) + .bundle(global_state.clone(), module_name, out_file) .await; debug!(">>>>> bundle END"); bundle_result diff --git a/cli/module_graph.rs b/cli/module_graph.rs index c04c05fd8a865c..aedad94f22edcc 100644 --- a/cli/module_graph.rs +++ b/cli/module_graph.rs @@ -122,7 +122,7 @@ impl ModuleGraphLoader { let load_result = self.pending_downloads.next().await.unwrap(); let source_file = load_result?; let spec = ModuleSpecifier::from(source_file.url.clone()); - self.new_visit_module(&spec, source_file)?; + self.visit_module(&spec, source_file)?; if self.pending_downloads.is_empty() { break; } @@ -175,7 +175,7 @@ impl ModuleGraphLoader { Ok(()) } - fn new_visit_module( + fn visit_module( &mut self, module_specifier: &ModuleSpecifier, source_file: SourceFile, @@ -233,6 +233,7 @@ impl ModuleGraphLoader { None }; + eprintln!("type directive {:#?}", import_desc.deno_types); let import_descriptor = ImportDescriptor { specifier: import_desc.specifier.to_string(), resolved_specifier, @@ -301,181 +302,6 @@ impl ModuleGraphLoader { ); Ok(()) } - - async fn visit_module( - &mut self, - module_specifier: &ModuleSpecifier, - maybe_referrer: Option<ModuleSpecifier>, - ) -> Result<(), ErrBox> { - if self.graph.0.contains_key(&module_specifier.to_string()) { - return Ok(()); - } - - if !self.is_dyn_import { - // Verify that remote file doesn't try to statically import local file. - if let Some(referrer) = maybe_referrer.as_ref() { - let referrer_url = referrer.as_url(); - match referrer_url.scheme() { - "http" | "https" => { - let specifier_url = module_specifier.as_url(); - match specifier_url.scheme() { - "http" | "https" => {} - _ => { - let e = OpError::permission_denied("Remote module are not allowed to statically import local modules. Use dynamic import instead.".to_string()); - return Err(e.into()); - } - } - } - _ => {} - } - } - } - - let source_file = self - .file_fetcher - .fetch_source_file( - module_specifier, - maybe_referrer, - self.permissions.clone(), - ) - .await?; - - let mut imports = vec![]; - let mut referenced_files = vec![]; - let mut lib_directives = vec![]; - let mut types_directives = vec![]; - let mut type_headers = vec![]; - - let source_code = String::from_utf8(source_file.source_code)?; - - if source_file.media_type == MediaType::JavaScript - || source_file.media_type == MediaType::TypeScript - { - if let Some(types_specifier) = source_file.types_header { - let type_header = ReferenceDescriptor { - specifier: types_specifier.to_string(), - resolved_specifier: ModuleSpecifier::resolve_import( - &types_specifier, - &module_specifier.to_string(), - )?, - }; - type_headers.push(type_header); - } - - let (import_descs, ref_descs) = - analyze_dependencies_and_references(&source_code, true)?; - - for import_desc in import_descs { - let maybe_resolved = - if let Some(import_map) = self.maybe_import_map.as_ref() { - import_map - .resolve(&import_desc.specifier, &module_specifier.to_string())? - } else { - None - }; - - let resolved_specifier = if let Some(resolved) = maybe_resolved { - resolved - } else { - ModuleSpecifier::resolve_import( - &import_desc.specifier, - &module_specifier.to_string(), - )? - }; - - let resolved_type_directive = - if let Some(types_specifier) = import_desc.deno_types.as_ref() { - Some(ModuleSpecifier::resolve_import( - &types_specifier, - &module_specifier.to_string(), - )?) - } else { - None - }; - - let import_descriptor = ImportDescriptor { - specifier: import_desc.specifier.to_string(), - resolved_specifier, - type_directive: import_desc.deno_types, - resolved_type_directive, - }; - - if self - .graph - .0 - .get(&import_descriptor.resolved_specifier.to_string()) - .is_none() - { - self.to_visit.push(( - import_descriptor.resolved_specifier.clone(), - Some(module_specifier.clone()), - )); - } - - if let Some(type_dir_url) = - import_descriptor.resolved_type_directive.as_ref() - { - if self.graph.0.get(&type_dir_url.to_string()).is_none() { - self - .to_visit - .push((type_dir_url.clone(), Some(module_specifier.clone()))); - } - } - - imports.push(import_descriptor); - } - - for ref_desc in ref_descs { - let resolved_specifier = ModuleSpecifier::resolve_import( - &ref_desc.specifier, - &module_specifier.to_string(), - )?; - let reference_descriptor = ReferenceDescriptor { - specifier: ref_desc.specifier.to_string(), - resolved_specifier, - }; - - if self - .graph - .0 - .get(&reference_descriptor.resolved_specifier.to_string()) - .is_none() - { - self.to_visit.push(( - reference_descriptor.resolved_specifier.clone(), - Some(module_specifier.clone()), - )); - } - - match ref_desc.kind { - TsReferenceKind::Lib => { - lib_directives.push(reference_descriptor); - } - TsReferenceKind::Types => { - types_directives.push(reference_descriptor); - } - TsReferenceKind::Path => { - referenced_files.push(reference_descriptor); - } - } - } - } - - self.graph.0.insert( - module_specifier.to_string(), - ModuleGraphFile { - specifier: module_specifier.to_string(), - media_type: source_file.media_type, - source_code, - imports, - referenced_files, - lib_directives, - types_directives, - type_headers, - }, - ); - Ok(()) - } } #[cfg(test)] diff --git a/cli/tests/type_definitions/bar.js b/cli/tests/type_definitions/bar.js new file mode 100644 index 00000000000000..616306027d438f --- /dev/null +++ b/cli/tests/type_definitions/bar.js @@ -0,0 +1,5 @@ +export class Bar { + constructor() { + this.baz = "baz"; + } +} \ No newline at end of file diff --git a/cli/tsc.rs b/cli/tsc.rs index c8763d315c8cde..a3ba7aab3177b2 100644 --- a/cli/tsc.rs +++ b/cli/tsc.rs @@ -222,39 +222,6 @@ impl CompiledFileMetadata { serde_json::to_string(&value_map) } } -/// Creates the JSON message send to compiler.ts's onmessage. -fn req( - request_type: msg::CompilerRequestType, - root_names: Vec<String>, - compiler_config: CompilerConfig, - target: &str, - bundle: bool, - unstable: bool, -) -> Buf { - let cwd = std::env::current_dir().unwrap(); - let j = match (compiler_config.path, compiler_config.content) { - (Some(config_path), Some(config_data)) => json!({ - "type": request_type as i32, - "target": target, - "rootNames": root_names, - "bundle": bundle, - "unstable": unstable, - "configPath": config_path, - "config": str::from_utf8(&config_data).unwrap(), - "cwd": cwd, - }), - _ => json!({ - "type": request_type as i32, - "target": target, - "rootNames": root_names, - "bundle": bundle, - "unstable": unstable, - "cwd": cwd, - }), - }; - - j.to_string().into_boxed_str().into_boxed_bytes() -} /// Emit a SHA256 hash based on source code, deno version and TS config. /// Used to check if a recompilation for source code is needed. @@ -369,7 +336,7 @@ impl TsCompiler { worker } - pub async fn new_bundle( + pub async fn bundle( &self, global_state: GlobalState, module_specifier: ModuleSpecifier, @@ -463,59 +430,6 @@ impl TsCompiler { Ok(()) } - #[allow(unused)] - pub async fn bundle( - &self, - global_state: GlobalState, - module_name: String, - out_file: Option<PathBuf>, - ) -> Result<(), ErrBox> { - debug!( - "Invoking the compiler to bundle. module_name: {}", - module_name - ); - eprintln!("Bundling {}", module_name); - - let root_names = vec![module_name]; - let req_msg = req( - msg::CompilerRequestType::Compile, - root_names, - self.config.clone(), - "main", - true, - global_state.flags.unstable, - ); - - let permissions = Permissions::allow_all(); - let msg = - execute_in_thread(global_state.clone(), permissions, req_msg).await?; - let json_str = std::str::from_utf8(&msg).unwrap(); - debug!("Message: {}", json_str); - - let bundle_response: BundleResponse = serde_json::from_str(json_str)?; - - if !bundle_response.diagnostics.items.is_empty() { - return Err(ErrBox::from(bundle_response.diagnostics)); - } - - let output_string = fmt::format_text(&bundle_response.bundle_output)?; - - if let Some(out_file_) = out_file.as_ref() { - eprintln!("Emitting bundle to {:?}", out_file_); - - let output_bytes = output_string.as_bytes(); - let output_len = output_bytes.len(); - - deno_fs::write_file(out_file_, output_bytes, 0o666)?; - // TODO(bartlomieju): add "humanFileSize" method - eprintln!("{} bytes emmited.", output_len); - } else { - println!("{}", output_string); - } - - Ok(()) - } - /// Mark given module URL as compiled to avoid multiple compilations of same /// module in single run. fn mark_compiled(&self, url: &Url) { @@ -530,87 +444,6 @@ impl TsCompiler { c.contains(url) } - /// Asynchronously compile module and all it's dependencies. - /// - /// This method compiled every module at most once. - /// - /// If `--reload` flag was provided then compiler will not on-disk cache and - /// force recompilation. - /// - /// If compilation is required then new V8 worker is spawned with fresh TS - /// compiler. - pub async fn compile( - &self, - global_state: GlobalState, - source_file: &SourceFile, - target: TargetLib, - permissions: Permissions, - ) -> Result<CompiledModule, ErrBox> { - if self.has_compiled(&source_file.url) { - return self.get_compiled_module(&source_file.url); - } - - if self.use_disk_cache { - // Try to load cached version: - // 1. check if there's 'meta' file - if let Some(metadata) = self.get_metadata(&source_file.url) { - // 2. compare version hashes - // TODO: it would probably be good idea to make it method implemented on SourceFile - let version_hash_to_validate = source_code_version_hash( - &source_file.source_code, - version::DENO, - &self.config.hash, - ); - - if metadata.version_hash == version_hash_to_validate { - debug!("load_cache metadata version hash match"); - if let Ok(compiled_module) = - self.get_compiled_module(&source_file.url) - { - self.mark_compiled(&source_file.url); - return Ok(compiled_module); - } - } - } - } - let source_file_ = source_file.clone(); - let module_url = source_file.url.clone(); - let target = match target { - TargetLib::Main => "main", - TargetLib::Worker => "worker", - }; - let root_names = vec![module_url.to_string()]; - let req_msg = req( - msg::CompilerRequestType::Compile, - root_names, - self.config.clone(), - target, - false, - global_state.flags.unstable, - ); - - let ts_compiler = self.clone(); - - info!( - "{} {}", - colors::green("Compile".to_string()), - module_url.to_string() - ); - - let msg = - execute_in_thread(global_state.clone(), permissions, req_msg).await?; - let json_str = std::str::from_utf8(&msg).unwrap(); - - let compile_response: CompileResponse = serde_json::from_str(json_str)?; - - if !compile_response.diagnostics.items.is_empty() { - return Err(ErrBox::from(compile_response.diagnostics)); - } - - self.cache_emitted_files(compile_response.emit_map)?; - ts_compiler.get_compiled_module(&source_file_.url) - } - /// Get associated `CompiledFileMetadata` for given module if it exists. pub fn get_metadata(&self, url: &Url) -> Option<CompiledFileMetadata> { // Try to load cached version: @@ -631,8 +464,16 @@ impl TsCompiler { None } - #[allow(unused)] - pub async fn new_compile( + /// Asynchronously compile module and all it's dependencies. + /// + /// This method compiled every module at most once. + /// + /// If `--reload` flag was provided then compiler will not on-disk cache and + /// force recompilation. + /// + /// If compilation is required then new V8 worker is spawned with fresh TS + /// compiler. + pub async fn compile( &self, global_state: GlobalState, source_file: &SourceFile, @@ -1143,6 +984,7 @@ mod tests { &out, TargetLib::Main, Permissions::allow_all(), + false, ) .await; assert!(result.is_ok()); @@ -1197,7 +1039,7 @@ mod tests { let result = state .ts_compiler - .new_bundle(state.clone(), module_name, None) + .bundle(state.clone(), module_name, None) .await; assert!(result.is_ok()); } From aa096a97ed89275c621c386f550791812ea5a444 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= <biwanczuk@gmail.com> Date: Thu, 14 May 2020 23:43:31 +0200 Subject: [PATCH 27/47] cleanup2 --- cli/js/compiler.ts | 10 ++++++++++ cli/module_graph.rs | 9 ++++++--- cli/msg.rs | 1 + cli/tests/type_definitions/bar.js | 2 +- cli/tsc.rs | 11 +++++++++-- 5 files changed, 27 insertions(+), 6 deletions(-) diff --git a/cli/js/compiler.ts b/cli/js/compiler.ts index d959a1e9a0d9e1..38641e438166d0 100644 --- a/cli/js/compiler.ts +++ b/cli/js/compiler.ts @@ -1397,6 +1397,7 @@ function compileNew(request: CompilerRequestCompileNew): CompileResult { } buildSourceFileCache(sourceFileMap); + const start = new Date(); // if there was a configuration and no diagnostics with it, we will continue // to generate the program and possibly emit it. if (diagnostics.length === 0) { @@ -1427,6 +1428,9 @@ function compileNew(request: CompilerRequestCompileNew): CompileResult { } } + const end = new Date(); + // @ts-ignore + util.log("time in internal TS", end - start); let bundleOutput = undefined; if (bundle) { @@ -1740,7 +1744,13 @@ async function tsCompilerOnMessage({ break; } case CompilerRequestType.CompileNew: { + const start = new Date(); const result = compileNew(request as CompilerRequestCompileNew); + const end = new Date(); + util.log( + // @ts-ignore + `Time spent in TS program "${end - start}"` + ); globalThis.postMessage(result); break; } diff --git a/cli/module_graph.rs b/cli/module_graph.rs index aedad94f22edcc..9561c151ed66e3 100644 --- a/cli/module_graph.rs +++ b/cli/module_graph.rs @@ -72,12 +72,14 @@ pub struct ReferenceDescriptor { #[serde(rename_all = "camelCase")] pub struct ModuleGraphFile { pub specifier: String, + pub url: String, + pub filename: String, pub imports: Vec<ImportDescriptor>, pub referenced_files: Vec<ReferenceDescriptor>, pub lib_directives: Vec<ReferenceDescriptor>, pub types_directives: Vec<ReferenceDescriptor>, pub type_headers: Vec<ReferenceDescriptor>, - pub media_type: MediaType, + pub media_type: i32, pub source_code: String, } @@ -233,7 +235,6 @@ impl ModuleGraphLoader { None }; - eprintln!("type directive {:#?}", import_desc.deno_types); let import_descriptor = ImportDescriptor { specifier: import_desc.specifier.to_string(), resolved_specifier, @@ -291,7 +292,9 @@ impl ModuleGraphLoader { module_specifier.to_string(), ModuleGraphFile { specifier: module_specifier.to_string(), - media_type: source_file.media_type, + url: source_file.url.to_string(), + filename: source_file.filename.to_str().unwrap().to_string(), + media_type: source_file.media_type as i32, source_code, imports, referenced_files, diff --git a/cli/msg.rs b/cli/msg.rs index 186fde42c644e8..dc6765776419df 100644 --- a/cli/msg.rs +++ b/cli/msg.rs @@ -38,4 +38,5 @@ pub enum CompilerRequestType { Compile = 0, RuntimeCompile = 1, RuntimeTranspile = 2, + CompileNew = 3, } diff --git a/cli/tests/type_definitions/bar.js b/cli/tests/type_definitions/bar.js index 616306027d438f..e9c2e5193cce21 100644 --- a/cli/tests/type_definitions/bar.js +++ b/cli/tests/type_definitions/bar.js @@ -2,4 +2,4 @@ export class Bar { constructor() { this.baz = "baz"; } -} \ No newline at end of file +} diff --git a/cli/tsc.rs b/cli/tsc.rs index a3ba7aab3177b2..81d8608fb0887b 100644 --- a/cli/tsc.rs +++ b/cli/tsc.rs @@ -50,6 +50,7 @@ use std::sync::atomic::Ordering; use std::sync::Arc; use std::sync::Mutex; use std::task::Poll; +use std::time::Instant; use url::Url; #[derive(Debug, Clone)] @@ -543,7 +544,7 @@ impl TsCompiler { let cwd = std::env::current_dir().unwrap(); let j = match (compiler_config.path, compiler_config.content) { (Some(config_path), Some(config_data)) => json!({ - "type": msg::CompilerRequestType::Compile as i32, + "type": msg::CompilerRequestType::CompileNew as i32, "target": target, "rootNames": root_names, "bundle": bundle, @@ -554,7 +555,7 @@ impl TsCompiler { "sourceFileMap": module_graph_json, }), _ => json!({ - "type": msg::CompilerRequestType::Compile as i32, + "type": msg::CompilerRequestType::CompileNew as i32, "target": target, "rootNames": root_names, "bundle": bundle, @@ -574,8 +575,14 @@ impl TsCompiler { module_url.to_string() ); + let start = Instant::now(); + let msg = execute_in_thread(global_state.clone(), permissions, req_msg).await?; + + let end = Instant::now(); + debug!("time spent in compiler thread {:#?}", end - start); + let json_str = std::str::from_utf8(&msg).unwrap(); let compile_response: CompileResponse = serde_json::from_str(json_str)?; From 3a1c842ac9771a08411b001748e7150d2dd6e0eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= <biwanczuk@gmail.com> Date: Sat, 16 May 2020 15:30:39 +0200 Subject: [PATCH 28/47] runtime compile remote files --- cli/js/compiler.ts | 120 ++++++++++++++++ cli/module_graph.rs | 55 +++++++ cli/msg.rs | 1 + cli/ops/runtime_compiler.rs | 4 +- cli/tests/compiler_api_test.ts | 254 ++++++++++++++++----------------- cli/tsc.rs | 81 +++++++++++ 6 files changed, 386 insertions(+), 129 deletions(-) diff --git a/cli/js/compiler.ts b/cli/js/compiler.ts index 38641e438166d0..7ade3d500f7e7b 100644 --- a/cli/js/compiler.ts +++ b/cli/js/compiler.ts @@ -907,6 +907,7 @@ enum CompilerRequestType { RuntimeCompile = 1, RuntimeTranspile = 2, CompileNew = 3, + RuntimeCompileNew = 4, } // TODO(bartlomieju): probably could be defined inline? @@ -1321,6 +1322,16 @@ interface CompilerRequestRuntimeCompile { options?: string; } +interface CompilerRequestRuntimeCompileNew { + type: CompilerRequestType.RuntimeCompileNew; + target: CompilerHostTarget; + rootNames: string[]; + sourceFileMap: Record<string, SourceFileMapEntry>, + unstable?: boolean; + bundle?: boolean; + options?: string; +} + interface CompilerRequestRuntimeTranspile { type: CompilerRequestType.RuntimeTranspile; sources: Record<string, string>; @@ -1331,6 +1342,7 @@ type CompilerRequest = | CompilerRequestCompileNew | CompilerRequestCompile | CompilerRequestRuntimeCompile + | CompilerRequestRuntimeCompileNew | CompilerRequestRuntimeTranspile; interface CompileResult { @@ -1565,6 +1577,107 @@ async function compile( return result; } +async function runtimeCompileNew( + request: CompilerRequestRuntimeCompileNew +): Promise<RuntimeCompileResult | RuntimeBundleResult> { + const { bundle, options, rootNames, target, unstable, sourceFileMap } = request; + + util.log(">>> runtime compile start", { + rootNames, + bundle, + }); + + // if there are options, convert them into TypeScript compiler options, + // and resolve any external file references + let convertedOptions: ts.CompilerOptions | undefined; + if (options) { + const result = convertCompilerOptions(options); + convertedOptions = result.options; + } + + buildSourceFileCache(sourceFileMap); + + const state: WriteFileState = { + type: request.type, + bundle, + host: undefined, + rootNames, + emitMap: {}, + bundleOutput: undefined, + }; + let writeFile: WriteFileCallback; + if (bundle) { + writeFile = createBundleWriteFile(state); + } else { + writeFile = createCompileWriteFile(state); + } + + const host = (state.host = new Host({ + bundle, + target, + writeFile, + })); + const compilerOptions = [DEFAULT_RUNTIME_COMPILE_OPTIONS]; + if (convertedOptions) { + compilerOptions.push(convertedOptions); + } + if (unstable) { + compilerOptions.push({ + lib: [ + "deno.unstable", + ...((convertedOptions && convertedOptions.lib) || ["deno.window"]), + ], + }); + } + if (bundle) { + compilerOptions.push(DEFAULT_BUNDLER_OPTIONS); + } + host.mergeOptions(...compilerOptions); + + const program = ts.createProgram({ + rootNames, + options: host.getCompilationSettings(), + host, + oldProgram: TS_SNAPSHOT_PROGRAM, + }); + + if (bundle) { + setRootExports(program, rootNames[0]); + } + + const diagnostics = ts + .getPreEmitDiagnostics(program) + .filter(({ code }) => !ignoredDiagnostics.includes(code)); + + const emitResult = program.emit(); + + assert(emitResult.emitSkipped === false, "Unexpected skip of the emit."); + + assert(state.emitMap); + util.log("<<< runtime compile finish", { + rootNames, + bundle, + emitMap: Object.keys(state.emitMap), + }); + + const maybeDiagnostics = diagnostics.length + ? fromTypeScriptDiagnostic(diagnostics).items + : []; + + if (bundle) { + return { + diagnostics: maybeDiagnostics, + output: state.bundleOutput, + } as RuntimeBundleResult; + } else { + return { + diagnostics: maybeDiagnostics, + emitMap: state.emitMap, + } as RuntimeCompileResult; + } +} + + async function runtimeCompile( request: CompilerRequestRuntimeCompile ): Promise<RuntimeCompileResult | RuntimeBundleResult> { @@ -1761,6 +1874,13 @@ async function tsCompilerOnMessage({ globalThis.postMessage(result); break; } + case CompilerRequestType.RuntimeCompileNew: { + const result = await runtimeCompileNew( + request as CompilerRequestRuntimeCompileNew + ); + globalThis.postMessage(result); + break; + } case CompilerRequestType.RuntimeTranspile: { const result = await runtimeTranspile( request as CompilerRequestRuntimeTranspile diff --git a/cli/module_graph.rs b/cli/module_graph.rs index 9561c151ed66e3..ea3bad98892fa6 100644 --- a/cli/module_graph.rs +++ b/cli/module_graph.rs @@ -114,6 +114,51 @@ impl ModuleGraphLoader { } } + pub async fn add_to_graph( + &mut self, + specifier: &ModuleSpecifier, + ) -> Result<(), ErrBox> { + self.download_module(specifier.clone(), None)?; + + loop { + let load_result = self.pending_downloads.next().await.unwrap(); + let source_file = load_result?; + let spec = ModuleSpecifier::from(source_file.url.clone()); + self.visit_module(&spec, source_file)?; + if self.pending_downloads.is_empty() { + break; + } + } + + Ok(()) + } + + pub fn build_local_graph( + &mut self, + root_name: &str, + source_map: HashMap<String, String>, + ) -> Result<(), ErrBox> { + for spec, source_code in source_map { + + } + + self.download_module(specifier.clone(), None)?; + + loop { + let load_result = self.pending_downloads.next().await.unwrap(); + let source_file = load_result?; + let spec = ModuleSpecifier::from(source_file.url.clone()); + self.visit_module(&spec, source_file)?; + if self.pending_downloads.is_empty() { + break; + } + } + + Ok(()) + } + + + // TODO: remove pub async fn build_graph( mut self, specifier: &ModuleSpecifier, @@ -133,6 +178,12 @@ impl ModuleGraphLoader { Ok(self.graph.0) } + pub fn get_graph( + self, + ) -> HashMap<String, ModuleGraphFile> { + self.graph.0 + } + fn download_module( &mut self, module_specifier: ModuleSpecifier, @@ -201,6 +252,10 @@ impl ModuleGraphLoader { &module_specifier.to_string(), )?, }; + self.download_module( + type_header.resolved_specifier.clone(), + Some(module_specifier.clone()), + )?; type_headers.push(type_header); } diff --git a/cli/msg.rs b/cli/msg.rs index dc6765776419df..6c8382268b87a6 100644 --- a/cli/msg.rs +++ b/cli/msg.rs @@ -39,4 +39,5 @@ pub enum CompilerRequestType { RuntimeCompile = 1, RuntimeTranspile = 2, CompileNew = 3, + RuntimeCompileNew = 4, } diff --git a/cli/ops/runtime_compiler.rs b/cli/ops/runtime_compiler.rs index f3b741861a8668..bb958838669ea5 100644 --- a/cli/ops/runtime_compiler.rs +++ b/cli/ops/runtime_compiler.rs @@ -3,7 +3,7 @@ use super::dispatch_json::{Deserialize, JsonOp, Value}; use crate::futures::FutureExt; use crate::op_error::OpError; use crate::state::State; -use crate::tsc::runtime_compile; +use crate::tsc::new_runtime_compile; use crate::tsc::runtime_transpile; use deno_core::CoreIsolate; use deno_core::ZeroCopyBuf; @@ -34,7 +34,7 @@ fn op_compile( let global_state = s.global_state.clone(); let permissions = s.permissions.clone(); let fut = async move { - runtime_compile( + new_runtime_compile( global_state, permissions, &args.root_name, diff --git a/cli/tests/compiler_api_test.ts b/cli/tests/compiler_api_test.ts index cdc2be6d28dbe7..66fe2e704cde76 100644 --- a/cli/tests/compiler_api_test.ts +++ b/cli/tests/compiler_api_test.ts @@ -3,20 +3,20 @@ import { assert, assertEquals } from "../../std/testing/asserts.ts"; const { compile, transpileOnly, bundle, test } = Deno; -test("compilerApiCompileSources", async function () { - const [diagnostics, actual] = await compile("/foo.ts", { - "/foo.ts": `import * as bar from "./bar.ts";\n\nconsole.log(bar);\n`, - "/bar.ts": `export const bar = "bar";\n`, - }); - assert(diagnostics == null); - assert(actual); - assertEquals(Object.keys(actual), [ - "/bar.js.map", - "/bar.js", - "/foo.js.map", - "/foo.js", - ]); -}); +// test("compilerApiCompileSources", async function () { +// const [diagnostics, actual] = await compile("/foo.ts", { +// "/foo.ts": `import * as bar from "./bar.ts";\n\nconsole.log(bar);\n`, +// "/bar.ts": `export const bar = "bar";\n`, +// }); +// assert(diagnostics == null); +// assert(actual); +// assertEquals(Object.keys(actual), [ +// "/bar.js.map", +// "/bar.js", +// "/foo.js.map", +// "/foo.js", +// ]); +// }); test("compilerApiCompileNoSources", async function () { const [diagnostics, actual] = await compile("./subdir/mod1.ts"); @@ -28,125 +28,125 @@ test("compilerApiCompileNoSources", async function () { assert(keys[1].endsWith("print_hello.js")); }); -test("compilerApiCompileOptions", async function () { - const [diagnostics, actual] = await compile( - "/foo.ts", - { - "/foo.ts": `export const foo = "foo";`, - }, - { - module: "amd", - sourceMap: false, - } - ); - assert(diagnostics == null); - assert(actual); - assertEquals(Object.keys(actual), ["/foo.js"]); - assert(actual["/foo.js"].startsWith("define(")); -}); +// test("compilerApiCompileOptions", async function () { +// const [diagnostics, actual] = await compile( +// "/foo.ts", +// { +// "/foo.ts": `export const foo = "foo";`, +// }, +// { +// module: "amd", +// sourceMap: false, +// } +// ); +// assert(diagnostics == null); +// assert(actual); +// assertEquals(Object.keys(actual), ["/foo.js"]); +// assert(actual["/foo.js"].startsWith("define(")); +// }); -test("compilerApiCompileLib", async function () { - const [diagnostics, actual] = await compile( - "/foo.ts", - { - "/foo.ts": `console.log(document.getElementById("foo")); - console.log(Deno.args);`, - }, - { - lib: ["dom", "es2018", "deno.ns"], - } - ); - assert(diagnostics == null); - assert(actual); - assertEquals(Object.keys(actual), ["/foo.js.map", "/foo.js"]); -}); +// test("compilerApiCompileLib", async function () { +// const [diagnostics, actual] = await compile( +// "/foo.ts", +// { +// "/foo.ts": `console.log(document.getElementById("foo")); +// console.log(Deno.args);`, +// }, +// { +// lib: ["dom", "es2018", "deno.ns"], +// } +// ); +// assert(diagnostics == null); +// assert(actual); +// assertEquals(Object.keys(actual), ["/foo.js.map", "/foo.js"]); +// }); -test("compilerApiCompileTypes", async function () { - const [diagnostics, actual] = await compile( - "/foo.ts", - { - "/foo.ts": `console.log(Foo.bar);`, - }, - { - types: ["./subdir/foo_types.d.ts"], - } - ); - assert(diagnostics == null); - assert(actual); - assertEquals(Object.keys(actual), ["/foo.js.map", "/foo.js"]); -}); +// test("compilerApiCompileTypes", async function () { +// const [diagnostics, actual] = await compile( +// "/foo.ts", +// { +// "/foo.ts": `console.log(Foo.bar);`, +// }, +// { +// types: ["./subdir/foo_types.d.ts"], +// } +// ); +// assert(diagnostics == null); +// assert(actual); +// assertEquals(Object.keys(actual), ["/foo.js.map", "/foo.js"]); +// }); -test("transpileOnlyApi", async function () { - const actual = await transpileOnly({ - "foo.ts": `export enum Foo { Foo, Bar, Baz };\n`, - }); - assert(actual); - assertEquals(Object.keys(actual), ["foo.ts"]); - assert(actual["foo.ts"].source.startsWith("export var Foo;")); - assert(actual["foo.ts"].map); -}); +// test("transpileOnlyApi", async function () { +// const actual = await transpileOnly({ +// "foo.ts": `export enum Foo { Foo, Bar, Baz };\n`, +// }); +// assert(actual); +// assertEquals(Object.keys(actual), ["foo.ts"]); +// assert(actual["foo.ts"].source.startsWith("export var Foo;")); +// assert(actual["foo.ts"].map); +// }); -test("transpileOnlyApiConfig", async function () { - const actual = await transpileOnly( - { - "foo.ts": `export enum Foo { Foo, Bar, Baz };\n`, - }, - { - sourceMap: false, - module: "amd", - } - ); - assert(actual); - assertEquals(Object.keys(actual), ["foo.ts"]); - assert(actual["foo.ts"].source.startsWith("define(")); - assert(actual["foo.ts"].map == null); -}); +// test("transpileOnlyApiConfig", async function () { +// const actual = await transpileOnly( +// { +// "foo.ts": `export enum Foo { Foo, Bar, Baz };\n`, +// }, +// { +// sourceMap: false, +// module: "amd", +// } +// ); +// assert(actual); +// assertEquals(Object.keys(actual), ["foo.ts"]); +// assert(actual["foo.ts"].source.startsWith("define(")); +// assert(actual["foo.ts"].map == null); +// }); -test("bundleApiSources", async function () { - const [diagnostics, actual] = await bundle("/foo.ts", { - "/foo.ts": `export * from "./bar.ts";\n`, - "/bar.ts": `export const bar = "bar";\n`, - }); - assert(diagnostics == null); - assert(actual.includes(`__instantiate("foo")`)); - assert(actual.includes(`__exp["bar"]`)); -}); +// test("bundleApiSources", async function () { +// const [diagnostics, actual] = await bundle("/foo.ts", { +// "/foo.ts": `export * from "./bar.ts";\n`, +// "/bar.ts": `export const bar = "bar";\n`, +// }); +// assert(diagnostics == null); +// assert(actual.includes(`__instantiate("foo")`)); +// assert(actual.includes(`__exp["bar"]`)); +// }); -test("bundleApiNoSources", async function () { - const [diagnostics, actual] = await bundle("./subdir/mod1.ts"); - assert(diagnostics == null); - assert(actual.includes(`__instantiate("mod1")`)); - assert(actual.includes(`__exp["printHello3"]`)); -}); +// test("bundleApiNoSources", async function () { +// const [diagnostics, actual] = await bundle("./subdir/mod1.ts"); +// assert(diagnostics == null); +// assert(actual.includes(`__instantiate("mod1")`)); +// assert(actual.includes(`__exp["printHello3"]`)); +// }); -test("bundleApiConfig", async function () { - const [diagnostics, actual] = await bundle( - "/foo.ts", - { - "/foo.ts": `// random comment\nexport * from "./bar.ts";\n`, - "/bar.ts": `export const bar = "bar";\n`, - }, - { - removeComments: true, - } - ); - assert(diagnostics == null); - assert(!actual.includes(`random`)); -}); +// test("bundleApiConfig", async function () { +// const [diagnostics, actual] = await bundle( +// "/foo.ts", +// { +// "/foo.ts": `// random comment\nexport * from "./bar.ts";\n`, +// "/bar.ts": `export const bar = "bar";\n`, +// }, +// { +// removeComments: true, +// } +// ); +// assert(diagnostics == null); +// assert(!actual.includes(`random`)); +// }); -test("bundleApiJsModules", async function () { - const [diagnostics, actual] = await bundle("/foo.js", { - "/foo.js": `export * from "./bar.js";\n`, - "/bar.js": `export const bar = "bar";\n`, - }); - assert(diagnostics == null); - assert(actual.includes(`System.register("bar",`)); -}); +// test("bundleApiJsModules", async function () { +// const [diagnostics, actual] = await bundle("/foo.js", { +// "/foo.js": `export * from "./bar.js";\n`, +// "/bar.js": `export const bar = "bar";\n`, +// }); +// assert(diagnostics == null); +// assert(actual.includes(`System.register("bar",`)); +// }); -test("diagnosticsTest", async function () { - const [diagnostics] = await compile("/foo.ts", { - "/foo.ts": `document.getElementById("foo");`, - }); - assert(Array.isArray(diagnostics)); - assert(diagnostics.length === 1); -}); +// test("diagnosticsTest", async function () { +// const [diagnostics] = await compile("/foo.ts", { +// "/foo.ts": `document.getElementById("foo");`, +// }); +// assert(Array.isArray(diagnostics)); +// assert(diagnostics.length === 1); +// }); diff --git a/cli/tsc.rs b/cli/tsc.rs index 81d8608fb0887b..d60fc4f4d52ba0 100644 --- a/cli/tsc.rs +++ b/cli/tsc.rs @@ -891,6 +891,87 @@ async fn execute_in_thread( } /// This function is used by `Deno.compile()` and `Deno.bundle()` APIs. +pub async fn new_runtime_compile<S: BuildHasher>( + global_state: GlobalState, + permissions: Permissions, + root_name: &str, + sources: &Option<HashMap<String, String, S>>, + bundle: bool, + maybe_options: &Option<String>, +) -> Result<Value, OpError> { + let mut root_names = vec![]; + let mut module_graph_loader = ModuleGraphLoader::new( + global_state.file_fetcher.clone(), + None, + permissions.clone(), + false, + ); + + if let Some(s_map) = sources { + module_graph_loader.build_local_graph(root_name, s_map)?; + } else { + let module_specifier = ModuleSpecifier::resolve_import(root_name, "<unknown>")?; + root_names.push(module_specifier.to_string()); + module_graph_loader.add_to_graph(&module_specifier).await?; + + + // TODO: download all additional files from TSconfig and add them to root_names + if let Some(options) = maybe_options { + let options_json: serde_json::Value = serde_json::from_str(options)?; + if let Some(types_option) = options_json.get("types") { + let types_arr = types_option.as_array().expect("types is not an array"); + + for type_value in types_arr { + let type_str = type_value.as_str().expect("type is not a string").to_string(); + let type_specifier = ModuleSpecifier::resolve_url(&type_str)?; + module_graph_loader.add_to_graph(&type_specifier).await?; + root_names.push(type_specifier.to_string()) + } + } + } + } + + let module_graph = module_graph_loader.get_graph(); + let module_graph_json = + serde_json::to_value(module_graph).expect("Failed to serialize data"); + + let req_msg = json!({ + "type": msg::CompilerRequestType::RuntimeCompileNew as i32, + "target": "runtime", + "rootNames": root_names, + "sourceFileMap": module_graph_json, + "options": maybe_options, + "bundle": bundle, + "unstable": global_state.flags.unstable, + }) + .to_string() + .into_boxed_str() + .into_boxed_bytes(); + + let compiler = global_state.ts_compiler.clone(); + + let msg = execute_in_thread(global_state, permissions, req_msg).await?; + let json_str = std::str::from_utf8(&msg).unwrap(); + + // TODO(bartlomieju): factor `bundle` path into separate function `runtime_bundle` + if bundle { + let _response: RuntimeBundleResponse = serde_json::from_str(json_str)?; + return Ok(serde_json::from_str::<Value>(json_str).unwrap()); + } + + let response: RuntimeCompileResponse = serde_json::from_str(json_str)?; + + if response.diagnostics.is_empty() && sources.is_none() { + compiler.cache_emitted_files(response.emit_map)?; + } + + // We're returning `Ok()` instead of `Err()` because it's not runtime + // error if there were diagnostics produces; we want to let user handle + // diagnostics in the runtime. + Ok(serde_json::from_str::<Value>(json_str).unwrap()) +} + +#[allow(unused)] pub async fn runtime_compile<S: BuildHasher>( global_state: GlobalState, permissions: Permissions, From 5fccde9b47eb3dde66e389d77244f18f2e66b016 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= <biwanczuk@gmail.com> Date: Sat, 16 May 2020 17:00:04 +0200 Subject: [PATCH 29/47] runtime compiler tests passing --- cli/js/compiler.ts | 77 +++++++++- cli/module_graph.rs | 120 +++++++++++++--- cli/tests/compiler_api_test.ts | 254 ++++++++++++++++----------------- cli/tsc.rs | 38 ++--- 4 files changed, 323 insertions(+), 166 deletions(-) diff --git a/cli/js/compiler.ts b/cli/js/compiler.ts index 7ade3d500f7e7b..147026a8b5f756 100644 --- a/cli/js/compiler.ts +++ b/cli/js/compiler.ts @@ -546,6 +546,11 @@ class Host implements ts.CompilerHost { return moduleNames.map((specifier) => { const maybeUrl = SourceFile.getResolvedUrl(specifier, containingFile); + util.log("compiler::host.resolveModulenames receiverd", { + maybeUrl, + sf: SourceFile.getCached(maybeUrl!), + }); + let sourceFile: SourceFile | undefined = undefined; if (specifier.startsWith(ASSETS)) { @@ -756,6 +761,58 @@ async function processImports( return resolvedSources; } +function buildLocalSourceFileCache( + sourceFileMap: Record<string, SourceFileMapEntry> +): void { + for (const entry of Object.values(sourceFileMap)) { + assert(entry.sourceCode.length > 0); + SourceFile.addToCache({ + url: entry.url, + filename: entry.url, + mediaType: getMediaType(entry.url), + sourceCode: entry.sourceCode, + }); + + for (const importDesc of entry.imports) { + let mappedUrl = importDesc.resolvedSpecifier; + const importedFile = sourceFileMap[importDesc.resolvedSpecifier]; + assert(importedFile); + const isJsOrJsx = + importedFile.mediaType === MediaType.JavaScript || + importedFile.mediaType === MediaType.JSX; + // If JS or JSX perform substitution for types if available + if (isJsOrJsx) { + if (importedFile.typeHeaders.length > 0) { + const typeHeaders = importedFile.typeHeaders[0]; + mappedUrl = typeHeaders.resolvedSpecifier; + } else if (importDesc.resolvedTypeDirective) { + mappedUrl = importDesc.resolvedTypeDirective; + } else if (importedFile.typesDirectives.length > 0) { + const typeDirective = importedFile.typesDirectives[0]; + mappedUrl = typeDirective.resolvedSpecifier; + } + } + + mappedUrl = mappedUrl.replace("memory://", ""); + SourceFile.cacheResolvedUrl(mappedUrl, importDesc.specifier, entry.url); + } + for (const fileRef of entry.referencedFiles) { + SourceFile.cacheResolvedUrl( + fileRef.resolvedSpecifier.replace("memory://", ""), + fileRef.specifier, + entry.url + ); + } + for (const fileRef of entry.libDirectives) { + SourceFile.cacheResolvedUrl( + fileRef.resolvedSpecifier.replace("memory://", ""), + fileRef.specifier, + entry.url + ); + } + } +} + function buildSourceFileCache( sourceFileMap: Record<string, SourceFileMapEntry> ): void { @@ -1326,7 +1383,7 @@ interface CompilerRequestRuntimeCompileNew { type: CompilerRequestType.RuntimeCompileNew; target: CompilerHostTarget; rootNames: string[]; - sourceFileMap: Record<string, SourceFileMapEntry>, + sourceFileMap: Record<string, SourceFileMapEntry>; unstable?: boolean; bundle?: boolean; options?: string; @@ -1577,10 +1634,17 @@ async function compile( return result; } -async function runtimeCompileNew( +function runtimeCompileNew( request: CompilerRequestRuntimeCompileNew -): Promise<RuntimeCompileResult | RuntimeBundleResult> { - const { bundle, options, rootNames, target, unstable, sourceFileMap } = request; +): RuntimeCompileResult | RuntimeBundleResult { + const { + bundle, + options, + rootNames, + target, + unstable, + sourceFileMap, + } = request; util.log(">>> runtime compile start", { rootNames, @@ -1595,7 +1659,7 @@ async function runtimeCompileNew( convertedOptions = result.options; } - buildSourceFileCache(sourceFileMap); + buildLocalSourceFileCache(sourceFileMap); const state: WriteFileState = { type: request.type, @@ -1677,7 +1741,6 @@ async function runtimeCompileNew( } } - async function runtimeCompile( request: CompilerRequestRuntimeCompile ): Promise<RuntimeCompileResult | RuntimeBundleResult> { @@ -1875,7 +1938,7 @@ async function tsCompilerOnMessage({ break; } case CompilerRequestType.RuntimeCompileNew: { - const result = await runtimeCompileNew( + const result = runtimeCompileNew( request as CompilerRequestRuntimeCompileNew ); globalThis.postMessage(result); diff --git a/cli/module_graph.rs b/cli/module_graph.rs index ea3bad98892fa6..35a996a5c66a2b 100644 --- a/cli/module_graph.rs +++ b/cli/module_graph.rs @@ -18,6 +18,7 @@ use futures::FutureExt; use serde::Serialize; use serde::Serializer; use std::collections::HashMap; +use std::hash::BuildHasher; use std::pin::Pin; fn serialize_module_specifier<S>( @@ -133,31 +134,120 @@ impl ModuleGraphLoader { Ok(()) } - pub fn build_local_graph( + pub fn build_local_graph<S: BuildHasher>( &mut self, root_name: &str, - source_map: HashMap<String, String>, + source_map: &HashMap<String, String, S>, ) -> Result<(), ErrBox> { - for spec, source_code in source_map { - + for (spec, source_code) in source_map.iter() { + self.visit_memory_module(spec.to_string(), source_code.to_string()); } - self.download_module(specifier.clone(), None)?; + Ok(()) + } - loop { - let load_result = self.pending_downloads.next().await.unwrap(); - let source_file = load_result?; - let spec = ModuleSpecifier::from(source_file.url.clone()); - self.visit_module(&spec, source_file)?; - if self.pending_downloads.is_empty() { - break; + fn visit_memory_module( + &mut self, + specifier: String, + source_code: String, + ) -> Result<(), ErrBox> { + let mut imports = vec![]; + let mut referenced_files = vec![]; + let mut lib_directives = vec![]; + let mut types_directives = vec![]; + + let mut dummy_prefix = false; + + let module_specifier = if specifier.starts_with('/') { + dummy_prefix = true; + ModuleSpecifier::resolve_url(&format!("memory://{}", specifier)) + } else { + ModuleSpecifier::resolve_url(&specifier) + }?; + + let (import_descs, ref_descs) = + analyze_dependencies_and_references(&source_code, true)?; + + for import_desc in import_descs { + let maybe_resolved = + if let Some(import_map) = self.maybe_import_map.as_ref() { + import_map + .resolve(&import_desc.specifier, &module_specifier.to_string())? + } else { + None + }; + + let resolved_specifier = if let Some(resolved) = maybe_resolved { + resolved + } else { + ModuleSpecifier::resolve_import( + &import_desc.specifier, + &module_specifier.to_string(), + )? + }; + + let resolved_type_directive = + if let Some(types_specifier) = import_desc.deno_types.as_ref() { + Some(ModuleSpecifier::resolve_import( + &types_specifier, + &module_specifier.to_string(), + )?) + } else { + None + }; + + let import_descriptor = ImportDescriptor { + specifier: import_desc.specifier.to_string(), + resolved_specifier, + type_directive: import_desc.deno_types, + resolved_type_directive, + }; + + imports.push(import_descriptor); + } + + for ref_desc in ref_descs { + let resolved_specifier = ModuleSpecifier::resolve_import( + &ref_desc.specifier, + &module_specifier.to_string(), + )?; + let reference_descriptor = ReferenceDescriptor { + specifier: ref_desc.specifier.to_string(), + resolved_specifier, + }; + + match ref_desc.kind { + TsReferenceKind::Lib => { + lib_directives.push(reference_descriptor); + } + TsReferenceKind::Types => { + types_directives.push(reference_descriptor); + } + TsReferenceKind::Path => { + referenced_files.push(reference_descriptor); + } } } + self.graph.0.insert( + module_specifier.to_string(), + ModuleGraphFile { + specifier: specifier.to_string(), + url: specifier.to_string(), + filename: specifier, + // ignored, it's set in TS worker + media_type: MediaType::JavaScript as i32, + source_code, + imports, + referenced_files, + lib_directives, + types_directives, + type_headers: vec![], + }, + ); Ok(()) } - // TODO: remove pub async fn build_graph( mut self, @@ -178,9 +268,7 @@ impl ModuleGraphLoader { Ok(self.graph.0) } - pub fn get_graph( - self, - ) -> HashMap<String, ModuleGraphFile> { + pub fn get_graph(self) -> HashMap<String, ModuleGraphFile> { self.graph.0 } diff --git a/cli/tests/compiler_api_test.ts b/cli/tests/compiler_api_test.ts index 66fe2e704cde76..cdc2be6d28dbe7 100644 --- a/cli/tests/compiler_api_test.ts +++ b/cli/tests/compiler_api_test.ts @@ -3,20 +3,20 @@ import { assert, assertEquals } from "../../std/testing/asserts.ts"; const { compile, transpileOnly, bundle, test } = Deno; -// test("compilerApiCompileSources", async function () { -// const [diagnostics, actual] = await compile("/foo.ts", { -// "/foo.ts": `import * as bar from "./bar.ts";\n\nconsole.log(bar);\n`, -// "/bar.ts": `export const bar = "bar";\n`, -// }); -// assert(diagnostics == null); -// assert(actual); -// assertEquals(Object.keys(actual), [ -// "/bar.js.map", -// "/bar.js", -// "/foo.js.map", -// "/foo.js", -// ]); -// }); +test("compilerApiCompileSources", async function () { + const [diagnostics, actual] = await compile("/foo.ts", { + "/foo.ts": `import * as bar from "./bar.ts";\n\nconsole.log(bar);\n`, + "/bar.ts": `export const bar = "bar";\n`, + }); + assert(diagnostics == null); + assert(actual); + assertEquals(Object.keys(actual), [ + "/bar.js.map", + "/bar.js", + "/foo.js.map", + "/foo.js", + ]); +}); test("compilerApiCompileNoSources", async function () { const [diagnostics, actual] = await compile("./subdir/mod1.ts"); @@ -28,125 +28,125 @@ test("compilerApiCompileNoSources", async function () { assert(keys[1].endsWith("print_hello.js")); }); -// test("compilerApiCompileOptions", async function () { -// const [diagnostics, actual] = await compile( -// "/foo.ts", -// { -// "/foo.ts": `export const foo = "foo";`, -// }, -// { -// module: "amd", -// sourceMap: false, -// } -// ); -// assert(diagnostics == null); -// assert(actual); -// assertEquals(Object.keys(actual), ["/foo.js"]); -// assert(actual["/foo.js"].startsWith("define(")); -// }); +test("compilerApiCompileOptions", async function () { + const [diagnostics, actual] = await compile( + "/foo.ts", + { + "/foo.ts": `export const foo = "foo";`, + }, + { + module: "amd", + sourceMap: false, + } + ); + assert(diagnostics == null); + assert(actual); + assertEquals(Object.keys(actual), ["/foo.js"]); + assert(actual["/foo.js"].startsWith("define(")); +}); -// test("compilerApiCompileLib", async function () { -// const [diagnostics, actual] = await compile( -// "/foo.ts", -// { -// "/foo.ts": `console.log(document.getElementById("foo")); -// console.log(Deno.args);`, -// }, -// { -// lib: ["dom", "es2018", "deno.ns"], -// } -// ); -// assert(diagnostics == null); -// assert(actual); -// assertEquals(Object.keys(actual), ["/foo.js.map", "/foo.js"]); -// }); +test("compilerApiCompileLib", async function () { + const [diagnostics, actual] = await compile( + "/foo.ts", + { + "/foo.ts": `console.log(document.getElementById("foo")); + console.log(Deno.args);`, + }, + { + lib: ["dom", "es2018", "deno.ns"], + } + ); + assert(diagnostics == null); + assert(actual); + assertEquals(Object.keys(actual), ["/foo.js.map", "/foo.js"]); +}); -// test("compilerApiCompileTypes", async function () { -// const [diagnostics, actual] = await compile( -// "/foo.ts", -// { -// "/foo.ts": `console.log(Foo.bar);`, -// }, -// { -// types: ["./subdir/foo_types.d.ts"], -// } -// ); -// assert(diagnostics == null); -// assert(actual); -// assertEquals(Object.keys(actual), ["/foo.js.map", "/foo.js"]); -// }); +test("compilerApiCompileTypes", async function () { + const [diagnostics, actual] = await compile( + "/foo.ts", + { + "/foo.ts": `console.log(Foo.bar);`, + }, + { + types: ["./subdir/foo_types.d.ts"], + } + ); + assert(diagnostics == null); + assert(actual); + assertEquals(Object.keys(actual), ["/foo.js.map", "/foo.js"]); +}); -// test("transpileOnlyApi", async function () { -// const actual = await transpileOnly({ -// "foo.ts": `export enum Foo { Foo, Bar, Baz };\n`, -// }); -// assert(actual); -// assertEquals(Object.keys(actual), ["foo.ts"]); -// assert(actual["foo.ts"].source.startsWith("export var Foo;")); -// assert(actual["foo.ts"].map); -// }); +test("transpileOnlyApi", async function () { + const actual = await transpileOnly({ + "foo.ts": `export enum Foo { Foo, Bar, Baz };\n`, + }); + assert(actual); + assertEquals(Object.keys(actual), ["foo.ts"]); + assert(actual["foo.ts"].source.startsWith("export var Foo;")); + assert(actual["foo.ts"].map); +}); -// test("transpileOnlyApiConfig", async function () { -// const actual = await transpileOnly( -// { -// "foo.ts": `export enum Foo { Foo, Bar, Baz };\n`, -// }, -// { -// sourceMap: false, -// module: "amd", -// } -// ); -// assert(actual); -// assertEquals(Object.keys(actual), ["foo.ts"]); -// assert(actual["foo.ts"].source.startsWith("define(")); -// assert(actual["foo.ts"].map == null); -// }); +test("transpileOnlyApiConfig", async function () { + const actual = await transpileOnly( + { + "foo.ts": `export enum Foo { Foo, Bar, Baz };\n`, + }, + { + sourceMap: false, + module: "amd", + } + ); + assert(actual); + assertEquals(Object.keys(actual), ["foo.ts"]); + assert(actual["foo.ts"].source.startsWith("define(")); + assert(actual["foo.ts"].map == null); +}); -// test("bundleApiSources", async function () { -// const [diagnostics, actual] = await bundle("/foo.ts", { -// "/foo.ts": `export * from "./bar.ts";\n`, -// "/bar.ts": `export const bar = "bar";\n`, -// }); -// assert(diagnostics == null); -// assert(actual.includes(`__instantiate("foo")`)); -// assert(actual.includes(`__exp["bar"]`)); -// }); +test("bundleApiSources", async function () { + const [diagnostics, actual] = await bundle("/foo.ts", { + "/foo.ts": `export * from "./bar.ts";\n`, + "/bar.ts": `export const bar = "bar";\n`, + }); + assert(diagnostics == null); + assert(actual.includes(`__instantiate("foo")`)); + assert(actual.includes(`__exp["bar"]`)); +}); -// test("bundleApiNoSources", async function () { -// const [diagnostics, actual] = await bundle("./subdir/mod1.ts"); -// assert(diagnostics == null); -// assert(actual.includes(`__instantiate("mod1")`)); -// assert(actual.includes(`__exp["printHello3"]`)); -// }); +test("bundleApiNoSources", async function () { + const [diagnostics, actual] = await bundle("./subdir/mod1.ts"); + assert(diagnostics == null); + assert(actual.includes(`__instantiate("mod1")`)); + assert(actual.includes(`__exp["printHello3"]`)); +}); -// test("bundleApiConfig", async function () { -// const [diagnostics, actual] = await bundle( -// "/foo.ts", -// { -// "/foo.ts": `// random comment\nexport * from "./bar.ts";\n`, -// "/bar.ts": `export const bar = "bar";\n`, -// }, -// { -// removeComments: true, -// } -// ); -// assert(diagnostics == null); -// assert(!actual.includes(`random`)); -// }); +test("bundleApiConfig", async function () { + const [diagnostics, actual] = await bundle( + "/foo.ts", + { + "/foo.ts": `// random comment\nexport * from "./bar.ts";\n`, + "/bar.ts": `export const bar = "bar";\n`, + }, + { + removeComments: true, + } + ); + assert(diagnostics == null); + assert(!actual.includes(`random`)); +}); -// test("bundleApiJsModules", async function () { -// const [diagnostics, actual] = await bundle("/foo.js", { -// "/foo.js": `export * from "./bar.js";\n`, -// "/bar.js": `export const bar = "bar";\n`, -// }); -// assert(diagnostics == null); -// assert(actual.includes(`System.register("bar",`)); -// }); +test("bundleApiJsModules", async function () { + const [diagnostics, actual] = await bundle("/foo.js", { + "/foo.js": `export * from "./bar.js";\n`, + "/bar.js": `export const bar = "bar";\n`, + }); + assert(diagnostics == null); + assert(actual.includes(`System.register("bar",`)); +}); -// test("diagnosticsTest", async function () { -// const [diagnostics] = await compile("/foo.ts", { -// "/foo.ts": `document.getElementById("foo");`, -// }); -// assert(Array.isArray(diagnostics)); -// assert(diagnostics.length === 1); -// }); +test("diagnosticsTest", async function () { + const [diagnostics] = await compile("/foo.ts", { + "/foo.ts": `document.getElementById("foo");`, + }); + assert(Array.isArray(diagnostics)); + assert(diagnostics.length === 1); +}); diff --git a/cli/tsc.rs b/cli/tsc.rs index d60fc4f4d52ba0..1bf38e855cbccc 100644 --- a/cli/tsc.rs +++ b/cli/tsc.rs @@ -908,33 +908,38 @@ pub async fn new_runtime_compile<S: BuildHasher>( ); if let Some(s_map) = sources { + root_names.push(root_name.to_string()); module_graph_loader.build_local_graph(root_name, s_map)?; } else { - let module_specifier = ModuleSpecifier::resolve_import(root_name, "<unknown>")?; + let module_specifier = + ModuleSpecifier::resolve_import(root_name, "<unknown>")?; root_names.push(module_specifier.to_string()); module_graph_loader.add_to_graph(&module_specifier).await?; - - - // TODO: download all additional files from TSconfig and add them to root_names - if let Some(options) = maybe_options { - let options_json: serde_json::Value = serde_json::from_str(options)?; - if let Some(types_option) = options_json.get("types") { - let types_arr = types_option.as_array().expect("types is not an array"); - - for type_value in types_arr { - let type_str = type_value.as_str().expect("type is not a string").to_string(); - let type_specifier = ModuleSpecifier::resolve_url(&type_str)?; - module_graph_loader.add_to_graph(&type_specifier).await?; - root_names.push(type_specifier.to_string()) - } + } + + // TODO: download all additional files from TSconfig and add them to root_names + if let Some(options) = maybe_options { + let options_json: serde_json::Value = serde_json::from_str(options)?; + if let Some(types_option) = options_json.get("types") { + let types_arr = types_option.as_array().expect("types is not an array"); + + for type_value in types_arr { + let type_str = type_value + .as_str() + .expect("type is not a string") + .to_string(); + let type_specifier = ModuleSpecifier::resolve_url_or_path(&type_str)?; + module_graph_loader.add_to_graph(&type_specifier).await?; + root_names.push(type_specifier.to_string()) } } } - + let module_graph = module_graph_loader.get_graph(); let module_graph_json = serde_json::to_value(module_graph).expect("Failed to serialize data"); + // eprintln!("source graph {:#?}", module_graph_json); let req_msg = json!({ "type": msg::CompilerRequestType::RuntimeCompileNew as i32, "target": "runtime", @@ -965,6 +970,7 @@ pub async fn new_runtime_compile<S: BuildHasher>( compiler.cache_emitted_files(response.emit_map)?; } + // eprintln!("returned {:#?}", json_str); // We're returning `Ok()` instead of `Err()` because it's not runtime // error if there were diagnostics produces; we want to let user handle // diagnostics in the runtime. From e26cc3e3fe80e66024831227e387b15e922dc422 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= <biwanczuk@gmail.com> Date: Sat, 16 May 2020 17:25:27 +0200 Subject: [PATCH 30/47] fix bundles --- cli/js/compiler.ts | 4 +-- cli/main.rs | 12 ++++++++- cli/ops/runtime_compiler.rs | 4 +-- cli/tsc.rs | 51 +++---------------------------------- 4 files changed, 18 insertions(+), 53 deletions(-) diff --git a/cli/js/compiler.ts b/cli/js/compiler.ts index 147026a8b5f756..65545ad207746d 100644 --- a/cli/js/compiler.ts +++ b/cli/js/compiler.ts @@ -1502,7 +1502,7 @@ function compileNew(request: CompilerRequestCompileNew): CompileResult { util.log("time in internal TS", end - start); let bundleOutput = undefined; - if (bundle) { + if (diagnostics && diagnostics.length === 0 && bundle) { assert(state.bundleOutput); bundleOutput = state.bundleOutput; } @@ -1614,7 +1614,7 @@ async function compile( let bundleOutput = undefined; - if (bundle) { + if (diagnostics && diagnostics.length === 0 && bundle) { assert(state.bundleOutput); bundleOutput = state.bundleOutput; } diff --git a/cli/main.rs b/cli/main.rs index cadb76fb4b108b..5985a1040aec72 100644 --- a/cli/main.rs +++ b/cli/main.rs @@ -70,6 +70,7 @@ pub use dprint_plugin_typescript::swc_ecma_parser; use crate::doc::parser::DocFileLoader; use crate::file_fetcher::SourceFile; use crate::file_fetcher::SourceFileFetcher; +use crate::fs as deno_fs; use crate::global_state::GlobalState; use crate::msg::MediaType; use crate::op_error::OpError; @@ -385,7 +386,16 @@ async fn bundle_command( source_file: String, out_file: Option<PathBuf>, ) -> Result<(), ErrBox> { - let module_name = ModuleSpecifier::resolve_url_or_path(&source_file)?; + let mut module_name = ModuleSpecifier::resolve_url_or_path(&source_file)?; + let url = module_name.as_url(); + + // TODO(bartlomieju): fix this hack in ModuleSpecifier + if url.scheme() == "file" { + let a = deno_fs::normalize_path(&url.to_file_path().unwrap()); + let u = Url::from_file_path(a).unwrap(); + module_name = ModuleSpecifier::from(u) + } + let global_state = GlobalState::new(flags)?; debug!(">>>>> bundle START"); let bundle_result = global_state diff --git a/cli/ops/runtime_compiler.rs b/cli/ops/runtime_compiler.rs index bb958838669ea5..f3b741861a8668 100644 --- a/cli/ops/runtime_compiler.rs +++ b/cli/ops/runtime_compiler.rs @@ -3,7 +3,7 @@ use super::dispatch_json::{Deserialize, JsonOp, Value}; use crate::futures::FutureExt; use crate::op_error::OpError; use crate::state::State; -use crate::tsc::new_runtime_compile; +use crate::tsc::runtime_compile; use crate::tsc::runtime_transpile; use deno_core::CoreIsolate; use deno_core::ZeroCopyBuf; @@ -34,7 +34,7 @@ fn op_compile( let global_state = s.global_state.clone(); let permissions = s.permissions.clone(); let fut = async move { - new_runtime_compile( + runtime_compile( global_state, permissions, &args.root_name, diff --git a/cli/tsc.rs b/cli/tsc.rs index 1bf38e855cbccc..1651651f4bfe50 100644 --- a/cli/tsc.rs +++ b/cli/tsc.rs @@ -379,7 +379,7 @@ impl TsCompiler { let cwd = std::env::current_dir().unwrap(); let j = match (compiler_config.path, compiler_config.content) { (Some(config_path), Some(config_data)) => json!({ - "type": msg::CompilerRequestType::Compile as i32, + "type": msg::CompilerRequestType::CompileNew as i32, "target": target, "rootNames": root_names, "bundle": bundle, @@ -390,7 +390,7 @@ impl TsCompiler { "sourceFileMap": module_graph_json, }), _ => json!({ - "type": msg::CompilerRequestType::Compile as i32, + "type": msg::CompilerRequestType::CompileNew as i32, "target": target, "rootNames": root_names, "bundle": bundle, @@ -891,7 +891,7 @@ async fn execute_in_thread( } /// This function is used by `Deno.compile()` and `Deno.bundle()` APIs. -pub async fn new_runtime_compile<S: BuildHasher>( +pub async fn runtime_compile<S: BuildHasher>( global_state: GlobalState, permissions: Permissions, root_name: &str, @@ -977,51 +977,6 @@ pub async fn new_runtime_compile<S: BuildHasher>( Ok(serde_json::from_str::<Value>(json_str).unwrap()) } -#[allow(unused)] -pub async fn runtime_compile<S: BuildHasher>( - global_state: GlobalState, - permissions: Permissions, - root_name: &str, - sources: &Option<HashMap<String, String, S>>, - bundle: bool, - options: &Option<String>, -) -> Result<Value, OpError> { - let req_msg = json!({ - "type": msg::CompilerRequestType::RuntimeCompile as i32, - "target": "runtime", - "rootName": root_name, - "sources": sources, - "options": options, - "bundle": bundle, - "unstable": global_state.flags.unstable, - }) - .to_string() - .into_boxed_str() - .into_boxed_bytes(); - - let compiler = global_state.ts_compiler.clone(); - - let msg = execute_in_thread(global_state, permissions, req_msg).await?; - let json_str = std::str::from_utf8(&msg).unwrap(); - - // TODO(bartlomieju): factor `bundle` path into separate function `runtime_bundle` - if bundle { - let _response: RuntimeBundleResponse = serde_json::from_str(json_str)?; - return Ok(serde_json::from_str::<Value>(json_str).unwrap()); - } - - let response: RuntimeCompileResponse = serde_json::from_str(json_str)?; - - if response.diagnostics.is_empty() && sources.is_none() { - compiler.cache_emitted_files(response.emit_map)?; - } - - // We're returning `Ok()` instead of `Err()` because it's not runtime - // error if there were diagnostics produces; we want to let user handle - // diagnostics in the runtime. - Ok(serde_json::from_str::<Value>(json_str).unwrap()) -} - /// This function is used by `Deno.transpileOnly()` API. pub async fn runtime_transpile<S: BuildHasher>( global_state: GlobalState, From a28794ebac98d97215dd0bc6baf6c985aa3c2f83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= <biwanczuk@gmail.com> Date: Sat, 16 May 2020 17:38:36 +0200 Subject: [PATCH 31/47] cleanup compiler --- cli/js/compiler.ts | 576 +------------------------------------------- cli/module_graph.rs | 3 + cli/msg.rs | 2 - cli/ops/compiler.rs | 144 +---------- cli/tsc.rs | 10 +- 5 files changed, 18 insertions(+), 717 deletions(-) diff --git a/cli/js/compiler.ts b/cli/js/compiler.ts index 65545ad207746d..8e77ada67f5a41 100644 --- a/cli/js/compiler.ts +++ b/cli/js/compiler.ts @@ -17,39 +17,12 @@ import { CompilerOptions } from "./compiler_options.ts"; import { Diagnostic, DiagnosticItem } from "./diagnostics.ts"; import { fromTypeScriptDiagnostic } from "./diagnostics_util.ts"; import { TranspileOnlyResult } from "./ops/runtime_compiler.ts"; -import { sendAsync, sendSync } from "./ops/dispatch_json.ts"; import { bootstrapWorkerRuntime } from "./runtime_worker.ts"; -import { assert, log } from "./util.ts"; +import { assert } from "./util.ts"; import * as util from "./util.ts"; import { TextDecoder, TextEncoder } from "./web/text_encoding.ts"; import { core } from "./core.ts"; -export function resolveModules( - specifiers: string[], - referrer?: string -): string[] { - util.log("compiler::resolveModules", { specifiers, referrer }); - return sendSync("op_resolve_modules", { specifiers, referrer }); -} - -export function fetchSourceFiles( - specifiers: string[], - referrer?: string -): Promise< - Array<{ - url: string; - filename: string; - mediaType: number; - sourceCode: string; - }> -> { - util.log("compiler::fetchSourceFiles", { specifiers, referrer }); - return sendAsync("op_fetch_source_files", { - specifiers, - referrer, - }); -} - const encoder = new TextEncoder(); const decoder = new TextDecoder(); @@ -263,75 +236,6 @@ class SourceFile { this.extension = getExtension(this.url, this.mediaType); } - imports(processJsImports: boolean): SourceFileSpecifierMap[] { - if (this.processed) { - throw new Error("SourceFile has already been processed."); - } - assert(this.sourceCode != null); - // we shouldn't process imports for files which contain the nocheck pragma - // (like bundles) - if (this.sourceCode.match(/\/{2}\s+@ts-nocheck/)) { - log(`Skipping imports for "${this.filename}"`); - return []; - } - - const readImportFiles = true; - const isJsOrJsx = - this.mediaType === MediaType.JavaScript || - this.mediaType === MediaType.JSX; - const detectJsImports = isJsOrJsx; - - const preProcessedFileInfo = ts.preProcessFile( - this.sourceCode, - readImportFiles, - detectJsImports - ); - this.processed = true; - const files: SourceFileSpecifierMap[] = []; - - function process(references: Array<{ fileName: string }>): void { - for (const { fileName } of references) { - files.push({ original: fileName, mapped: fileName }); - } - } - - const { - importedFiles, - referencedFiles, - libReferenceDirectives, - typeReferenceDirectives, - } = preProcessedFileInfo; - const typeDirectives = parseTypeDirectives(this.sourceCode); - - if (typeDirectives) { - for (const importedFile of importedFiles) { - // If there's a type directive for current processed file; then we provide - // different `mapped` specifier. - const mappedModuleName = getMappedModuleName( - importedFile, - typeDirectives - ); - files.push({ - original: importedFile.fileName, - mapped: mappedModuleName ?? importedFile.fileName, - }); - } - } else if (processJsImports || !isJsOrJsx) { - process(importedFiles); - } - process(referencedFiles); - // built in libs comes across as `"dom"` for example, and should be filtered - // out during pre-processing as they are either already cached or they will - // be lazily fetched by the compiler host. Ones that contain full files are - // not filtered out and will be fetched as normal. - const filteredLibs = libReferenceDirectives.filter( - ({ fileName }) => !ts.libMap.has(fileName.toLowerCase()) - ); - process(filteredLibs); - process(typeReferenceDirectives); - return files; - } - static addToCache(json: SourceFileJson): SourceFile { if (SOURCE_FILE_CACHE.has(json.url)) { throw new TypeError("SourceFile already exists"); @@ -637,25 +541,6 @@ const TS_SNAPSHOT_PROGRAM = ts.createProgram({ // This function is called only during snapshotting process const SYSTEM_LOADER = getAsset("system_loader.js"); -function resolveSpecifier(specifier: string, referrer: string): string { - // The resolveModules op only handles fully qualified URLs for referrer. - // However we will have cases where referrer is "/foo.ts". We add this dummy - // prefix "file://" in order to use the op. - // TODO(ry) Maybe we should perhaps ModuleSpecifier::resolve_import() to - // handle this situation. - let dummyPrefix = false; - const prefix = "file://"; - if (referrer.startsWith("/")) { - dummyPrefix = true; - referrer = prefix + referrer; - } - let r = resolveModules([specifier], referrer)[0]; - if (dummyPrefix) { - r = r.replace(prefix, ""); - } - return r; -} - function getMediaType(filename: string): MediaType { const maybeExtension = /\.([a-zA-Z]+)$/.exec(filename); if (!maybeExtension) { @@ -680,87 +565,6 @@ function getMediaType(filename: string): MediaType { } } -function processLocalImports( - sources: Record<string, string>, - specifiers: SourceFileSpecifierMap[], - referrer?: string, - processJsImports = false -): string[] { - if (!specifiers.length) { - return []; - } - const moduleNames = specifiers.map((specifierMap) => { - if (referrer) { - return resolveSpecifier(specifierMap.mapped, referrer); - } else { - return specifierMap.mapped; - } - }); - - for (let i = 0; i < moduleNames.length; i++) { - const moduleName = moduleNames[i]; - const specifierMap = specifiers[i]; - assert(moduleName in sources, `Missing module in sources: "${moduleName}"`); - let sourceFile = SourceFile.getCached(moduleName); - if (typeof sourceFile === "undefined") { - sourceFile = SourceFile.addToCache({ - url: moduleName, - filename: moduleName, - sourceCode: sources[moduleName], - mediaType: getMediaType(moduleName), - }); - } - assert(sourceFile); - SourceFile.cacheResolvedUrl( - sourceFile.url, - specifierMap.original, - referrer - ); - if (!sourceFile.processed) { - processLocalImports( - sources, - sourceFile.imports(processJsImports), - sourceFile.url, - processJsImports - ); - } - } - return moduleNames; -} - -async function processImports( - specifiers: SourceFileSpecifierMap[], - referrer?: string, - processJsImports = false -): Promise<string[]> { - if (!specifiers.length) { - return []; - } - const sources = specifiers.map(({ mapped }) => mapped); - const resolvedSources = resolveModules(sources, referrer); - const sourceFiles = await fetchSourceFiles(resolvedSources, referrer); - assert(sourceFiles.length === specifiers.length); - for (let i = 0; i < sourceFiles.length; i++) { - const specifierMap = specifiers[i]; - const sourceFileJson = sourceFiles[i]; - let sourceFile = SourceFile.getCached(sourceFileJson.url); - if (typeof sourceFile === "undefined") { - sourceFile = SourceFile.addToCache(sourceFileJson); - } - assert(sourceFile); - SourceFile.cacheResolvedUrl( - sourceFile.url, - specifierMap.original, - referrer - ); - if (!sourceFile.processed) { - const sourceFileImports = sourceFile.imports(processJsImports); - await processImports(sourceFileImports, sourceFile.url, processJsImports); - } - } - return resolvedSources; -} - function buildLocalSourceFileCache( sourceFileMap: Record<string, SourceFileMapEntry> ): void { @@ -864,76 +668,6 @@ function buildSourceFileCache( } } -interface FileReference { - fileName: string; - pos: number; - end: number; -} - -function getMappedModuleName( - source: FileReference, - typeDirectives: Map<FileReference, string> -): string | undefined { - const { fileName: sourceFileName, pos: sourcePos } = source; - for (const [{ fileName, pos }, value] of typeDirectives.entries()) { - if (sourceFileName === fileName && sourcePos === pos) { - return value; - } - } - return undefined; -} - -const typeDirectiveRegEx = /@deno-types\s*=\s*(["'])((?:(?=(\\?))\3.)*?)\1/gi; - -const importExportRegEx = /(?:import|export)(?:\s+|\s+[\s\S]*?from\s+)?(["'])((?:(?=(\\?))\3.)*?)\1/; - -function parseTypeDirectives( - sourceCode: string | undefined -): Map<FileReference, string> | undefined { - if (!sourceCode) { - return; - } - - // collect all the directives in the file and their start and end positions - const directives: FileReference[] = []; - let maybeMatch: RegExpExecArray | null = null; - while ((maybeMatch = typeDirectiveRegEx.exec(sourceCode))) { - const [matchString, , fileName] = maybeMatch; - const { index: pos } = maybeMatch; - directives.push({ - fileName, - pos, - end: pos + matchString.length, - }); - } - if (!directives.length) { - return; - } - - // work from the last directive backwards for the next `import`/`export` - // statement - directives.reverse(); - const results = new Map<FileReference, string>(); - for (const { end, fileName, pos } of directives) { - const searchString = sourceCode.substring(end); - const maybeMatch = importExportRegEx.exec(searchString); - if (maybeMatch) { - const [matchString, , targetFileName] = maybeMatch; - const targetPos = - end + maybeMatch.index + matchString.indexOf(targetFileName) - 1; - const target: FileReference = { - fileName: targetFileName, - pos: targetPos, - end: targetPos + targetFileName.length, - }; - results.set(target, fileName); - } - sourceCode = sourceCode.substring(0, pos); - } - - return results; -} - interface EmmitedSource { // original filename filename: string; @@ -963,8 +697,6 @@ enum CompilerRequestType { Compile = 0, RuntimeCompile = 1, RuntimeTranspile = 2, - CompileNew = 3, - RuntimeCompileNew = 4, } // TODO(bartlomieju): probably could be defined inline? @@ -1319,19 +1051,6 @@ function setRootExports(program: ts.Program, rootModule: string): void { .map((sym) => sym.getName()); } -interface CompilerRequestCompile { - type: CompilerRequestType.Compile; - target: CompilerHostTarget; - rootNames: string[]; - // TODO(ry) add compiler config to this interface. - // options: ts.CompilerOptions; - configPath?: string; - config?: string; - unstable: boolean; - bundle: boolean; - cwd: string; -} - interface ImportDescriptor { specifier: string; resolvedSpecifier: string; @@ -1356,8 +1075,8 @@ interface SourceFileMapEntry { typeHeaders: ReferenceDescriptor[]; } -interface CompilerRequestCompileNew { - type: CompilerRequestType.CompileNew; +interface CompilerRequestCompile { + type: CompilerRequestType.Compile; target: CompilerHostTarget; rootNames: string[]; configPath?: string; @@ -1372,16 +1091,6 @@ interface CompilerRequestCompileNew { interface CompilerRequestRuntimeCompile { type: CompilerRequestType.RuntimeCompile; target: CompilerHostTarget; - rootName: string; - sources?: Record<string, string>; - unstable?: boolean; - bundle?: boolean; - options?: string; -} - -interface CompilerRequestRuntimeCompileNew { - type: CompilerRequestType.RuntimeCompileNew; - target: CompilerHostTarget; rootNames: string[]; sourceFileMap: Record<string, SourceFileMapEntry>; unstable?: boolean; @@ -1396,10 +1105,8 @@ interface CompilerRequestRuntimeTranspile { } type CompilerRequest = - | CompilerRequestCompileNew | CompilerRequestCompile | CompilerRequestRuntimeCompile - | CompilerRequestRuntimeCompileNew | CompilerRequestRuntimeTranspile; interface CompileResult { @@ -1418,7 +1125,7 @@ interface RuntimeBundleResult { diagnostics: DiagnosticItem[]; } -function compileNew(request: CompilerRequestCompileNew): CompileResult { +function compile(request: CompilerRequestCompile): CompileResult { const { bundle, config, @@ -1522,120 +1229,8 @@ function compileNew(request: CompilerRequestCompileNew): CompileResult { return result; } -async function compile( - request: CompilerRequestCompile -): Promise<CompileResult> { - const { - bundle, - config, - configPath, - rootNames, - target, - unstable, - cwd, - } = request; - util.log(">>> compile start", { - rootNames, - type: CompilerRequestType[request.type], - }); - - // When a programme is emitted, TypeScript will call `writeFile` with - // each file that needs to be emitted. The Deno compiler host delegates - // this, to make it easier to perform the right actions, which vary - // based a lot on the request. - const state: WriteFileState = { - type: request.type, - emitMap: {}, - bundle, - host: undefined, - rootNames, - }; - let writeFile: WriteFileCallback; - if (bundle) { - writeFile = createBundleWriteFile(state); - } else { - writeFile = createCompileWriteFile(state); - } - const host = (state.host = new Host({ - bundle, - target, - writeFile, - unstable, - })); - let diagnostics: readonly ts.Diagnostic[] = []; - - // if there is a configuration supplied, we need to parse that - if (config && config.length && configPath) { - const configResult = host.configure(cwd, configPath, config); - diagnostics = processConfigureResponse(configResult, configPath) || []; - } - - // This will recursively analyse all the code for other imports, - // requesting those from the privileged side, populating the in memory - // cache which will be used by the host, before resolving. - const specifiers = rootNames.map((rootName) => { - return { original: rootName, mapped: rootName }; - }); - const resolvedRootModules = await processImports( - specifiers, - undefined, - bundle || host.getCompilationSettings().checkJs - ); - - // if there was a configuration and no diagnostics with it, we will continue - // to generate the program and possibly emit it. - if (diagnostics.length === 0) { - const options = host.getCompilationSettings(); - const program = ts.createProgram({ - rootNames, - options, - host, - oldProgram: TS_SNAPSHOT_PROGRAM, - }); - - diagnostics = ts - .getPreEmitDiagnostics(program) - .filter(({ code }) => !ignoredDiagnostics.includes(code)); - - // We will only proceed with the emit if there are no diagnostics. - if (diagnostics && diagnostics.length === 0) { - if (bundle) { - // we only support a single root module when bundling - assert(resolvedRootModules.length === 1); - setRootExports(program, resolvedRootModules[0]); - } - const emitResult = program.emit(); - assert(emitResult.emitSkipped === false, "Unexpected skip of the emit."); - // emitResult.diagnostics is `readonly` in TS3.5+ and can't be assigned - // without casting. - diagnostics = emitResult.diagnostics; - } - } - - let bundleOutput = undefined; - - if (diagnostics && diagnostics.length === 0 && bundle) { - assert(state.bundleOutput); - bundleOutput = state.bundleOutput; - } - - assert(state.emitMap); - const result: CompileResult = { - emitMap: state.emitMap, - bundleOutput, - diagnostics: fromTypeScriptDiagnostic(diagnostics), - }; - - util.log("<<< compile end", { - rootNames, - type: CompilerRequestType[request.type], - }); - - return result; -} - -function runtimeCompileNew( - request: CompilerRequestRuntimeCompileNew +function runtimeCompile( + request: CompilerRequestRuntimeCompile ): RuntimeCompileResult | RuntimeBundleResult { const { bundle, @@ -1741,147 +1336,6 @@ function runtimeCompileNew( } } -async function runtimeCompile( - request: CompilerRequestRuntimeCompile -): Promise<RuntimeCompileResult | RuntimeBundleResult> { - const { bundle, options, rootName, sources, target, unstable } = request; - - util.log(">>> runtime compile start", { - rootName, - bundle, - sources: sources ? Object.keys(sources) : undefined, - }); - - // resolve the root name, if there are sources, the root name does not - // get resolved - const resolvedRootName = sources ? rootName : resolveModules([rootName])[0]; - - // if there are options, convert them into TypeScript compiler options, - // and resolve any external file references - let convertedOptions: ts.CompilerOptions | undefined; - let additionalFiles: string[] | undefined; - if (options) { - const result = convertCompilerOptions(options); - convertedOptions = result.options; - additionalFiles = result.files; - } - - const checkJsImports = - bundle || (convertedOptions && convertedOptions.checkJs); - - // recursively process imports, loading each file into memory. If there - // are sources, these files are pulled out of the there, otherwise the - // files are retrieved from the privileged side - const specifiers = [ - { - original: resolvedRootName, - mapped: resolvedRootName, - }, - ]; - const rootNames = sources - ? processLocalImports(sources, specifiers, undefined, checkJsImports) - : await processImports(specifiers, undefined, checkJsImports); - - if (additionalFiles) { - // any files supplied in the configuration are resolved externally, - // even if sources are provided - const resolvedNames = resolveModules(additionalFiles); - const resolvedSpecifiers = resolvedNames.map((rn) => { - return { - original: rn, - mapped: rn, - }; - }); - const additionalImports = await processImports( - resolvedSpecifiers, - undefined, - checkJsImports - ); - rootNames.push(...additionalImports); - } - - const state: WriteFileState = { - type: request.type, - bundle, - host: undefined, - rootNames, - sources, - emitMap: {}, - bundleOutput: undefined, - }; - let writeFile: WriteFileCallback; - if (bundle) { - writeFile = createBundleWriteFile(state); - } else { - writeFile = createCompileWriteFile(state); - } - - const host = (state.host = new Host({ - bundle, - target, - writeFile, - })); - const compilerOptions = [DEFAULT_RUNTIME_COMPILE_OPTIONS]; - if (convertedOptions) { - compilerOptions.push(convertedOptions); - } - if (unstable) { - compilerOptions.push({ - lib: [ - "deno.unstable", - ...((convertedOptions && convertedOptions.lib) || ["deno.window"]), - ], - }); - } - if (bundle) { - compilerOptions.push(DEFAULT_BUNDLER_OPTIONS); - } - host.mergeOptions(...compilerOptions); - - const program = ts.createProgram({ - rootNames, - options: host.getCompilationSettings(), - host, - oldProgram: TS_SNAPSHOT_PROGRAM, - }); - - if (bundle) { - setRootExports(program, rootNames[0]); - } - - const diagnostics = ts - .getPreEmitDiagnostics(program) - .filter(({ code }) => !ignoredDiagnostics.includes(code)); - - const emitResult = program.emit(); - - assert(emitResult.emitSkipped === false, "Unexpected skip of the emit."); - - assert(state.emitMap); - util.log("<<< runtime compile finish", { - rootName, - sources: sources ? Object.keys(sources) : undefined, - bundle, - emitMap: Object.keys(state.emitMap), - }); - - const maybeDiagnostics = diagnostics.length - ? fromTypeScriptDiagnostic(diagnostics).items - : []; - - if (bundle) { - return { - diagnostics: maybeDiagnostics, - output: state.bundleOutput, - } as RuntimeBundleResult; - } else { - return { - diagnostics: maybeDiagnostics, - emitMap: state.emitMap, - } as RuntimeCompileResult; - } -} - function runtimeTranspile( request: CompilerRequestRuntimeTranspile ): Promise<Record<string, TranspileOnlyResult>> { @@ -1915,13 +1369,8 @@ async function tsCompilerOnMessage({ }): Promise<void> { switch (request.type) { case CompilerRequestType.Compile: { - const result = await compile(request as CompilerRequestCompile); - globalThis.postMessage(result); - break; - } - case CompilerRequestType.CompileNew: { const start = new Date(); - const result = compileNew(request as CompilerRequestCompileNew); + const result = compile(request as CompilerRequestCompile); const end = new Date(); util.log( // @ts-ignore @@ -1931,16 +1380,7 @@ async function tsCompilerOnMessage({ break; } case CompilerRequestType.RuntimeCompile: { - const result = await runtimeCompile( - request as CompilerRequestRuntimeCompile - ); - globalThis.postMessage(result); - break; - } - case CompilerRequestType.RuntimeCompileNew: { - const result = runtimeCompileNew( - request as CompilerRequestRuntimeCompileNew - ); + const result = runtimeCompile(request as CompilerRequestRuntimeCompile); globalThis.postMessage(result); break; } diff --git a/cli/module_graph.rs b/cli/module_graph.rs index 35a996a5c66a2b..313de2b3753b2c 100644 --- a/cli/module_graph.rs +++ b/cli/module_graph.rs @@ -158,6 +158,9 @@ impl ModuleGraphLoader { let mut dummy_prefix = false; + // The resolveModules op only handles fully qualified URLs for referrer. + // However we will have cases where referrer is "/foo.ts". We add this dummy + // prefix "file://" in order to use the op. let module_specifier = if specifier.starts_with('/') { dummy_prefix = true; ModuleSpecifier::resolve_url(&format!("memory://{}", specifier)) diff --git a/cli/msg.rs b/cli/msg.rs index 6c8382268b87a6..186fde42c644e8 100644 --- a/cli/msg.rs +++ b/cli/msg.rs @@ -38,6 +38,4 @@ pub enum CompilerRequestType { Compile = 0, RuntimeCompile = 1, RuntimeTranspile = 2, - CompileNew = 3, - RuntimeCompileNew = 4, } diff --git a/cli/ops/compiler.rs b/cli/ops/compiler.rs index b844011877c390..16ad5e8cb7749a 100644 --- a/cli/ops/compiler.rs +++ b/cli/ops/compiler.rs @@ -1,152 +1,12 @@ // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. -use super::dispatch_json::Deserialize; -use super::dispatch_json::JsonOp; -use super::dispatch_json::Value; -use crate::futures::future::try_join_all; -use crate::op_error::OpError; use crate::state::State; use deno_core::CoreIsolate; -use deno_core::ModuleLoader; -use deno_core::ModuleSpecifier; -use deno_core::ZeroCopyBuf; -use futures::future::FutureExt; -pub fn init(i: &mut CoreIsolate, s: &State) { - i.register_op("op_resolve_modules", s.stateful_json_op(op_resolve_modules)); - i.register_op( - "op_fetch_source_files", - s.stateful_json_op(op_fetch_source_files), - ); +pub fn init(i: &mut CoreIsolate, _s: &State) { let custom_assets = std::collections::HashMap::new(); // TODO(ry) use None. + // TODO(bartlomieju): is this op even required? i.register_op( "op_fetch_asset", deno_typescript::op_fetch_asset(custom_assets), ); } - -#[derive(Deserialize, Debug)] -struct SpecifiersReferrerArgs { - specifiers: Vec<String>, - referrer: Option<String>, -} - -fn op_resolve_modules( - state: &State, - args: Value, - _data: Option<ZeroCopyBuf>, -) -> Result<JsonOp, OpError> { - let args: SpecifiersReferrerArgs = serde_json::from_value(args)?; - let (referrer, is_main) = if let Some(referrer) = args.referrer { - (referrer, false) - } else { - ("<unknown>".to_owned(), true) - }; - - let mut specifiers = vec![]; - - for specifier in &args.specifiers { - let specifier = state - .resolve(specifier, &referrer, is_main) - .map_err(OpError::from)?; - specifiers.push(specifier.as_str().to_owned()); - } - - Ok(JsonOp::Sync(json!(specifiers))) -} - -fn op_fetch_source_files( - state: &State, - args: Value, - _data: Option<ZeroCopyBuf>, -) -> Result<JsonOp, OpError> { - let args: SpecifiersReferrerArgs = serde_json::from_value(args)?; - - let ref_specifier = if let Some(referrer) = args.referrer { - let specifier = ModuleSpecifier::resolve_url(&referrer) - .expect("Referrer is not a valid specifier"); - Some(specifier) - } else { - None - }; - - let s = state.borrow(); - let global_state = s.global_state.clone(); - let permissions = s.permissions.clone(); - let perms_ = permissions.clone(); - drop(s); - let file_fetcher = global_state.file_fetcher.clone(); - let specifiers = args.specifiers.clone(); - let future = async move { - let file_futures: Vec<_> = specifiers - .into_iter() - .map(|specifier| { - let file_fetcher_ = file_fetcher.clone(); - let ref_specifier_ = ref_specifier.clone(); - let perms_ = perms_.clone(); - async move { - let resolved_specifier = ModuleSpecifier::resolve_url(&specifier) - .expect("Invalid specifier"); - // TODO(bartlomieju): duplicated from `state.rs::ModuleLoader::load` - deduplicate - // Verify that remote file doesn't try to statically import local file. - if let Some(referrer) = ref_specifier_.as_ref() { - let referrer_url = referrer.as_url(); - match referrer_url.scheme() { - "http" | "https" => { - let specifier_url = resolved_specifier.as_url(); - match specifier_url.scheme() { - "http" | "https" => {}, - _ => { - let e = OpError::permission_denied("Remote module are not allowed to statically import local modules. Use dynamic import instead.".to_string()); - return Err(e.into()); - } - } - }, - _ => {} - } - } - file_fetcher_ - .fetch_source_file(&resolved_specifier, ref_specifier_, perms_) - .await - } - .boxed_local() - }) - .collect(); - - let files = try_join_all(file_futures).await.map_err(OpError::from)?; - // We want to get an array of futures that resolves to - let v = files.into_iter().map(|f| { - async { - // if the source file contains a `types_url` we need to replace - // the module with the type definition when requested by the compiler - let file = match f.types_url { - Some(types_url) => { - let types_specifier = ModuleSpecifier::from(types_url); - global_state - .file_fetcher - .fetch_source_file( - &types_specifier, - ref_specifier.clone(), - permissions.clone(), - ) - .await - .map_err(OpError::from)? - } - _ => f, - }; - let source_code = String::from_utf8(file.source_code).map_err(|_| OpError::invalid_utf8())?; - Ok::<_, OpError>(json!({ - "url": file.url.to_string(), - "filename": file.filename.to_str().unwrap(), - "mediaType": file.media_type as i32, - "sourceCode": source_code, - })) - } - }); - - let v = try_join_all(v).await?; - Ok(v.into()) - } - .boxed_local(); - - Ok(JsonOp::Async(future)) -} diff --git a/cli/tsc.rs b/cli/tsc.rs index 1651651f4bfe50..6987e0dcf77eec 100644 --- a/cli/tsc.rs +++ b/cli/tsc.rs @@ -379,7 +379,7 @@ impl TsCompiler { let cwd = std::env::current_dir().unwrap(); let j = match (compiler_config.path, compiler_config.content) { (Some(config_path), Some(config_data)) => json!({ - "type": msg::CompilerRequestType::CompileNew as i32, + "type": msg::CompilerRequestType::Compile as i32, "target": target, "rootNames": root_names, "bundle": bundle, @@ -390,7 +390,7 @@ impl TsCompiler { "sourceFileMap": module_graph_json, }), _ => json!({ - "type": msg::CompilerRequestType::CompileNew as i32, + "type": msg::CompilerRequestType::Compile as i32, "target": target, "rootNames": root_names, "bundle": bundle, @@ -544,7 +544,7 @@ impl TsCompiler { let cwd = std::env::current_dir().unwrap(); let j = match (compiler_config.path, compiler_config.content) { (Some(config_path), Some(config_data)) => json!({ - "type": msg::CompilerRequestType::CompileNew as i32, + "type": msg::CompilerRequestType::Compile as i32, "target": target, "rootNames": root_names, "bundle": bundle, @@ -555,7 +555,7 @@ impl TsCompiler { "sourceFileMap": module_graph_json, }), _ => json!({ - "type": msg::CompilerRequestType::CompileNew as i32, + "type": msg::CompilerRequestType::Compile as i32, "target": target, "rootNames": root_names, "bundle": bundle, @@ -941,7 +941,7 @@ pub async fn runtime_compile<S: BuildHasher>( // eprintln!("source graph {:#?}", module_graph_json); let req_msg = json!({ - "type": msg::CompilerRequestType::RuntimeCompileNew as i32, + "type": msg::CompilerRequestType::RuntimeCompile as i32, "target": "runtime", "rootNames": root_names, "sourceFileMap": module_graph_json, From 494a36bcb4cd0359daf472bb9dfeb99e4b1a78d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= <biwanczuk@gmail.com> Date: Sat, 16 May 2020 18:01:01 +0200 Subject: [PATCH 32/47] fix bundle with dynamic imports --- cli/module_graph.rs | 20 ++++++++++++++---- cli/ops/compiler.rs | 2 +- cli/swc_util.rs | 49 ++++++++++++++++++++++++++++++++++++++++++++- cli/tsc.rs | 3 +++ 4 files changed, 68 insertions(+), 6 deletions(-) diff --git a/cli/module_graph.rs b/cli/module_graph.rs index 313de2b3753b2c..2776ac9c50b058 100644 --- a/cli/module_graph.rs +++ b/cli/module_graph.rs @@ -95,6 +95,7 @@ pub struct ModuleGraphLoader { to_visit: Vec<(ModuleSpecifier, Option<ModuleSpecifier>)>, pub graph: ModuleGraph, is_dyn_import: bool, + analyze_dynamic_imports: bool, } impl ModuleGraphLoader { @@ -103,6 +104,7 @@ impl ModuleGraphLoader { maybe_import_map: Option<ImportMap>, permissions: Permissions, is_dyn_import: bool, + analyze_dynamic_imports: bool, ) -> Self { Self { file_fetcher, @@ -112,6 +114,7 @@ impl ModuleGraphLoader { pending_downloads: FuturesUnordered::new(), graph: ModuleGraph(HashMap::new()), is_dyn_import, + analyze_dynamic_imports, } } @@ -168,8 +171,10 @@ impl ModuleGraphLoader { ModuleSpecifier::resolve_url(&specifier) }?; - let (import_descs, ref_descs) = - analyze_dependencies_and_references(&source_code, true)?; + let (import_descs, ref_descs) = analyze_dependencies_and_references( + &source_code, + self.analyze_dynamic_imports, + )?; for import_desc in import_descs { let maybe_resolved = @@ -350,8 +355,10 @@ impl ModuleGraphLoader { type_headers.push(type_header); } - let (import_descs, ref_descs) = - analyze_dependencies_and_references(&source_code, true)?; + let (import_descs, ref_descs) = analyze_dependencies_and_references( + &source_code, + self.analyze_dynamic_imports, + )?; for import_desc in import_descs { let maybe_resolved = @@ -482,6 +489,7 @@ mod tests { None, Permissions::allow_all(), false, + false, ); let graph = graph_loader.build_graph(&module_specifier).await.unwrap(); @@ -554,6 +562,7 @@ mod tests { None, Permissions::allow_all(), false, + false, ); let graph = graph_loader.build_graph(&module_specifier).await.unwrap(); @@ -593,6 +602,7 @@ mod tests { None, Permissions::allow_all(), false, + false, ); let graph = graph_loader.build_graph(&module_specifier).await.unwrap(); @@ -689,6 +699,7 @@ mod tests { None, Permissions::allow_all(), false, + false, ); let graph = graph_loader.build_graph(&module_specifier).await.unwrap(); @@ -754,6 +765,7 @@ mod tests { None, Permissions::allow_all(), false, + false, ); let graph = graph_loader.build_graph(&module_specifier).await.unwrap(); diff --git a/cli/ops/compiler.rs b/cli/ops/compiler.rs index 16ad5e8cb7749a..625f0ac7f9d6a6 100644 --- a/cli/ops/compiler.rs +++ b/cli/ops/compiler.rs @@ -4,7 +4,7 @@ use deno_core::CoreIsolate; pub fn init(i: &mut CoreIsolate, _s: &State) { let custom_assets = std::collections::HashMap::new(); // TODO(ry) use None. - // TODO(bartlomieju): is this op even required? + // TODO(bartlomieju): is this op even required? i.register_op( "op_fetch_asset", deno_typescript::op_fetch_asset(custom_assets), diff --git a/cli/swc_util.rs b/cli/swc_util.rs index 91c155bf6b7f98..570b94a0aa8b9b 100644 --- a/cli/swc_util.rs +++ b/cli/swc_util.rs @@ -332,6 +332,7 @@ const a = await import("./" + "buzz.ts"); #[derive(Clone, Debug, PartialEq)] enum DependencyKind { Import, + DynamicImport, Export, } @@ -387,6 +388,46 @@ impl Visit for NewDependencyVisitor { span: export_all.span, }); } + + fn visit_call_expr( + &mut self, + call_expr: &swc_ecma_ast::CallExpr, + parent: &dyn Node, + ) { + use swc_ecma_ast::Expr::*; + use swc_ecma_ast::ExprOrSuper::*; + + swc_ecma_visit::visit_call_expr(self, call_expr, parent); + let boxed_expr = match call_expr.callee.clone() { + Super(_) => return, + Expr(boxed) => boxed, + }; + + match &*boxed_expr { + Ident(ident) => { + if &ident.sym.to_string() != "import" { + return; + } + } + _ => return, + }; + + if let Some(arg) = call_expr.args.get(0) { + match &*arg.expr { + Lit(lit) => { + if let swc_ecma_ast::Lit::Str(str_) = lit { + let src_str = str_.value.to_string(); + self.dependencies.push(DependencyDescriptor { + specifier: src_str, + kind: DependencyKind::DynamicImport, + span: call_expr.span, + }); + } + } + _ => return, + } + } + } } fn get_deno_types(parser: &AstParser, span: Span) -> Option<String> { @@ -437,7 +478,6 @@ pub struct TsReferenceDescriptor { pub specifier: String, } -#[allow(unused)] pub fn analyze_dependencies_and_references( source_code: &str, analyze_dynamic_imports: bool, @@ -459,6 +499,13 @@ pub fn analyze_dependencies_and_references( // for each import check if there's relevant @deno-types directive let imports = dependency_descriptors .iter() + .filter(|desc| { + if analyze_dynamic_imports { + return true; + } + + desc.kind != DependencyKind::DynamicImport + }) .map(|mut desc| { if desc.kind == DependencyKind::Import { let deno_types = get_deno_types(&parser, desc.span); diff --git a/cli/tsc.rs b/cli/tsc.rs index 6987e0dcf77eec..e2f85c4133cbcd 100644 --- a/cli/tsc.rs +++ b/cli/tsc.rs @@ -365,6 +365,7 @@ impl TsCompiler { import_map, permissions.clone(), false, + true, ); let module_graph = module_graph_loader.build_graph(&module_specifier).await?; @@ -527,6 +528,7 @@ impl TsCompiler { import_map, permissions.clone(), is_dyn_import, + false, ); let module_graph = module_graph_loader.build_graph(&module_specifier).await?; @@ -905,6 +907,7 @@ pub async fn runtime_compile<S: BuildHasher>( None, permissions.clone(), false, + false, ); if let Some(s_map) = sources { From facf6ab70167a41cb943094addbc58bcb2c1ab4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= <biwanczuk@gmail.com> Date: Sat, 16 May 2020 19:17:50 +0200 Subject: [PATCH 33/47] fix some tests --- cli/module_graph.rs | 60 ++++++++++++------- cli/tests/020_json_modules.ts.out | 4 +- .../error_005_missing_dynamic_import.ts.out | 9 +-- ...or_012_bad_dynamic_import_specifier.ts.out | 10 +--- cli/tests/integration_tests.rs | 2 +- core/modules.rs | 7 +++ 6 files changed, 51 insertions(+), 41 deletions(-) diff --git a/cli/module_graph.rs b/cli/module_graph.rs index 2776ac9c50b058..d6f99e8d81c59f 100644 --- a/cli/module_graph.rs +++ b/cli/module_graph.rs @@ -1,5 +1,4 @@ // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. -#![allow(unused)] use crate::file_fetcher::SourceFile; use crate::file_fetcher::SourceFileFetcher; @@ -92,7 +91,6 @@ pub struct ModuleGraphLoader { file_fetcher: SourceFileFetcher, maybe_import_map: Option<ImportMap>, pending_downloads: FuturesUnordered<SourceFileFuture>, - to_visit: Vec<(ModuleSpecifier, Option<ModuleSpecifier>)>, pub graph: ModuleGraph, is_dyn_import: bool, analyze_dynamic_imports: bool, @@ -110,7 +108,6 @@ impl ModuleGraphLoader { file_fetcher, permissions, maybe_import_map, - to_visit: vec![], pending_downloads: FuturesUnordered::new(), graph: ModuleGraph(HashMap::new()), is_dyn_import, @@ -139,11 +136,11 @@ impl ModuleGraphLoader { pub fn build_local_graph<S: BuildHasher>( &mut self, - root_name: &str, + _root_name: &str, source_map: &HashMap<String, String, S>, ) -> Result<(), ErrBox> { for (spec, source_code) in source_map.iter() { - self.visit_memory_module(spec.to_string(), source_code.to_string()); + self.visit_memory_module(spec.to_string(), source_code.to_string())?; } Ok(()) @@ -159,17 +156,17 @@ impl ModuleGraphLoader { let mut lib_directives = vec![]; let mut types_directives = vec![]; - let mut dummy_prefix = false; + // let mut dummy_prefix = false; // The resolveModules op only handles fully qualified URLs for referrer. // However we will have cases where referrer is "/foo.ts". We add this dummy - // prefix "file://" in order to use the op. - let module_specifier = if specifier.starts_with('/') { - dummy_prefix = true; - ModuleSpecifier::resolve_url(&format!("memory://{}", specifier)) - } else { - ModuleSpecifier::resolve_url(&specifier) - }?; + // prefix "memory://" in order to use resolution logic. + let module_specifier = + if let Ok(spec) = ModuleSpecifier::resolve_url(&specifier) { + spec + } else { + ModuleSpecifier::resolve_url(&format!("memory://{}", specifier))? + }; let (import_descs, ref_descs) = analyze_dependencies_and_references( &source_code, @@ -215,10 +212,17 @@ impl ModuleGraphLoader { } for ref_desc in ref_descs { - let resolved_specifier = ModuleSpecifier::resolve_import( + let resolve_result = ModuleSpecifier::resolve_import( &ref_desc.specifier, &module_specifier.to_string(), - )?; + ); + + // Skip for libs like "dom" or "esnext" + let resolved_specifier = match resolve_result { + Ok(s) => s, + Err(_) => continue, + }; + let reference_descriptor = ReferenceDescriptor { specifier: ref_desc.specifier.to_string(), resolved_specifier, @@ -314,9 +318,16 @@ impl ModuleGraphLoader { let perms = self.permissions.clone(); let load_future = async move { - file_fetcher - .new_fetch_source_file(spec, maybe_referrer, perms) - .await + let source_file = file_fetcher + .new_fetch_source_file(spec.clone(), maybe_referrer, perms) + .await?; + // FIXME(bartlomieju): + // because of redirects we may end up with wrong URL, + // substitute with original one + Ok(SourceFile { + url: spec.as_url().to_owned(), + ..source_file + }) } .boxed_local(); @@ -413,10 +424,17 @@ impl ModuleGraphLoader { } for ref_desc in ref_descs { - let resolved_specifier = ModuleSpecifier::resolve_import( + let resolve_result = ModuleSpecifier::resolve_import( &ref_desc.specifier, &module_specifier.to_string(), - )?; + ); + + // Skip for libs like "dom" or "esnext" + let resolved_specifier = match resolve_result { + Ok(s) => s, + Err(_) => continue, + }; + let reference_descriptor = ReferenceDescriptor { specifier: ref_desc.specifier.to_string(), resolved_specifier, @@ -464,7 +482,7 @@ impl ModuleGraphLoader { mod tests { use super::*; use crate::GlobalState; - use std::path::PathBuf; + // use std::path::PathBuf; // fn rel_module_specifier(relpath: &str) -> ModuleSpecifier { // let p = PathBuf::from(env!("CARGO_MANIFEST_DIR")) diff --git a/cli/tests/020_json_modules.ts.out b/cli/tests/020_json_modules.ts.out index cda28038e4e356..7fc60d37eafb76 100644 --- a/cli/tests/020_json_modules.ts.out +++ b/cli/tests/020_json_modules.ts.out @@ -2,8 +2,8 @@ error: Uncaught TypeError: Cannot resolve extension for "[WILDCARD]config.json" with mediaType "Json". at getExtension ($deno$/compiler.ts:[WILDCARD]) at new SourceFile ($deno$/compiler.ts:[WILDCARD]) - at processImports ($deno$/compiler.ts:[WILDCARD]) - at async processImports ($deno$/compiler.ts:[WILDCARD]) + at Function.addToCache ($deno$/compiler.ts:[WILDCARD]) + at buildSourceFileCache ($deno$/compiler.ts:[WILDCARD]) at async compile ($deno$/compiler.ts:[WILDCARD]) at async tsCompilerOnMessage ($deno$/compiler.ts:[WILDCARD]) at async workerMessageRecvCallback ($deno$/runtime_worker.ts:[WILDCARD]) diff --git a/cli/tests/error_005_missing_dynamic_import.ts.out b/cli/tests/error_005_missing_dynamic_import.ts.out index e8c8f8cd090020..3e2c3264a89e11 100644 --- a/cli/tests/error_005_missing_dynamic_import.ts.out +++ b/cli/tests/error_005_missing_dynamic_import.ts.out @@ -1,8 +1 @@ -[WILDCARD]error: Uncaught NotFound: Cannot resolve module "[WILDCARD]/bad-module.ts" from "[WILDCARD]/error_005_missing_dynamic_import.ts" - at unwrapResponse ([WILDCARD]dispatch_json.ts:[WILDCARD]) - at Object.sendAsync ([WILDCARD]dispatch_json.ts:[WILDCARD]) - at async processImports ($deno$/compiler.ts:[WILDCARD]) - at async processImports ($deno$/compiler.ts:[WILDCARD]) - at async compile ($deno$/compiler.ts:[WILDCARD]) - at async tsCompilerOnMessage ($deno$/compiler.ts:[WILDCARD]) - at async workerMessageRecvCallback ([WILDCARD]runtime_worker.ts:[WILDCARD]) +[WILDCARD]error: Uncaught TypeError: Cannot resolve module "[WILDCARD]/bad-module.ts" from "[WILDCARD]/error_005_missing_dynamic_import.ts" diff --git a/cli/tests/error_012_bad_dynamic_import_specifier.ts.out b/cli/tests/error_012_bad_dynamic_import_specifier.ts.out index 900b8f52da0385..57e4003ce4b23b 100644 --- a/cli/tests/error_012_bad_dynamic_import_specifier.ts.out +++ b/cli/tests/error_012_bad_dynamic_import_specifier.ts.out @@ -1,9 +1 @@ -[WILDCARD]error: Uncaught URIError: relative import path "bad-module.ts" not prefixed with / or ./ or ../ Imported from "[WILDCARD]/error_012_bad_dynamic_import_specifier.ts" - at unwrapResponse ($deno$/ops/dispatch_json.ts:[WILDCARD]) - at Object.sendSync ($deno$/ops/dispatch_json.ts:[WILDCARD]) - at resolveModules ($deno$/compiler.ts:[WILDCARD]) - at processImports ($deno$/compiler.ts:[WILDCARD]) - at processImports ($deno$/compiler.ts:[WILDCARD]) - at async compile ($deno$/compiler.ts:[WILDCARD]) - at async tsCompilerOnMessage ($deno$/compiler.ts:[WILDCARD]) - at async workerMessageRecvCallback ([WILDCARD]runtime_worker.ts:[WILDCARD]) +[WILDCARD]error: Uncaught TypeError: relative import path "bad-module.ts" not prefixed with / or ./ or ../ Imported from "[WILDCARD]/error_012_bad_dynamic_import_specifier.ts" diff --git a/cli/tests/integration_tests.rs b/cli/tests/integration_tests.rs index db9ccafd77eae8..309d882c6fd468 100644 --- a/cli/tests/integration_tests.rs +++ b/cli/tests/integration_tests.rs @@ -1368,7 +1368,7 @@ itest!(error_004_missing_module { }); itest!(error_005_missing_dynamic_import { - args: "run --reload error_005_missing_dynamic_import.ts", + args: "run --reload --allow-read error_005_missing_dynamic_import.ts", check_stderr: true, exit_code: 1, output: "error_005_missing_dynamic_import.ts.out", diff --git a/core/modules.rs b/core/modules.rs index 632df2dd000b2f..9aa0981b87bbcc 100644 --- a/core/modules.rs +++ b/core/modules.rs @@ -220,6 +220,13 @@ impl RecursiveModuleLoad { }) .boxed() } + LoadState::ResolveImport(ref _specifier, ref referrer) => { + let ref_spec = ModuleSpecifier::resolve_url(referrer)?; + self + .loader + .load(&module_specifier, Some(ref_spec), self.is_dynamic_import()) + .boxed_local() + } _ => self .loader .load(&module_specifier, None, self.is_dynamic_import()) From 22245437112cc36d2f3c883980cce880807825fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= <biwanczuk@gmail.com> Date: Sat, 16 May 2020 19:33:42 +0200 Subject: [PATCH 34/47] remove unused code --- cli/swc_util.rs | 84 +------------------------------------------------ 1 file changed, 1 insertion(+), 83 deletions(-) diff --git a/cli/swc_util.rs b/cli/swc_util.rs index 570b94a0aa8b9b..e07486cd70b46c 100644 --- a/cli/swc_util.rs +++ b/cli/swc_util.rs @@ -1,7 +1,4 @@ // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. - -#![allow(unused)] - use crate::swc_common; use crate::swc_common::comments::CommentKind; use crate::swc_common::comments::Comments; @@ -10,7 +7,6 @@ use crate::swc_common::errors::DiagnosticBuilder; use crate::swc_common::errors::Emitter; use crate::swc_common::errors::Handler; use crate::swc_common::errors::HandlerFlags; -use crate::swc_common::BytePos; use crate::swc_common::FileName; use crate::swc_common::Globals; use crate::swc_common::SourceMap; @@ -251,84 +247,6 @@ impl Visit for DependencyVisitor { } } -/// Given file name and source code return vector -/// of unresolved import specifiers. -/// -/// Returned vector may contain duplicate entries. -/// -/// Second argument allows to configure if dynamic -/// imports should be analyzed. -/// -/// NOTE: Only statically analyzable dynamic imports -/// are considered; ie. the ones that have plain string specifier: -/// -/// await import("./fizz.ts") -/// -/// These imports will be ignored: -/// -/// await import(`./${dir}/fizz.ts`) -/// await import("./" + "fizz.ts") -#[allow(unused)] -pub fn analyze_dependencies( - source_code: &str, - analyze_dynamic_imports: bool, -) -> Result<Vec<String>, SwcDiagnosticBuffer> { - let parser = AstParser::new(); - parser.parse_module("root.ts", source_code, |parse_result| { - let module = parse_result?; - let mut collector = DependencyVisitor { - dependencies: vec![], - analyze_dynamic_imports, - }; - collector.visit_module(&module, &module); - Ok(collector.dependencies) - }) -} - -#[test] -fn test_analyze_dependencies() { - let source = r#" -import { foo } from "./foo.ts"; -export { bar } from "./foo.ts"; -export * from "./bar.ts"; -"#; - - let dependencies = - analyze_dependencies(source, false).expect("Failed to parse"); - assert_eq!( - dependencies, - vec![ - "./foo.ts".to_string(), - "./foo.ts".to_string(), - "./bar.ts".to_string(), - ] - ); -} - -#[test] -fn test_analyze_dependencies_dyn_imports() { - let source = r#" -import { foo } from "./foo.ts"; -export { bar } from "./foo.ts"; -export * from "./bar.ts"; - -const a = await import("./fizz.ts"); -const a = await import("./" + "buzz.ts"); -"#; - - let dependencies = - analyze_dependencies(source, true).expect("Failed to parse"); - assert_eq!( - dependencies, - vec![ - "./foo.ts".to_string(), - "./foo.ts".to_string(), - "./bar.ts".to_string(), - "./fizz.ts".to_string(), - ] - ); -} - #[derive(Clone, Debug, PartialEq)] enum DependencyKind { Import, @@ -506,7 +424,7 @@ pub fn analyze_dependencies_and_references( desc.kind != DependencyKind::DynamicImport }) - .map(|mut desc| { + .map(|desc| { if desc.kind == DependencyKind::Import { let deno_types = get_deno_types(&parser, desc.span); ImportDescriptor { From 512f535681cb771f1c91eb85ba6572a09696ba67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= <biwanczuk@gmail.com> Date: Sat, 16 May 2020 20:11:25 +0200 Subject: [PATCH 35/47] fix more tests --- cli/module_graph.rs | 33 +++++----- cli/tests/020_json_modules.ts.out | 6 +- .../error_005_missing_dynamic_import.ts.out | 2 +- cli/tsc.rs | 65 +++++++++++++++++++ core/modules.rs | 7 -- 5 files changed, 86 insertions(+), 27 deletions(-) diff --git a/cli/module_graph.rs b/cli/module_graph.rs index d6f99e8d81c59f..78f7d673bbaabd 100644 --- a/cli/module_graph.rs +++ b/cli/module_graph.rs @@ -8,6 +8,7 @@ use crate::op_error::OpError; use crate::permissions::Permissions; use crate::swc_util::analyze_dependencies_and_references; use crate::swc_util::TsReferenceKind; +use crate::tsc::get_available_libs; use deno_core::ErrBox; use deno_core::ModuleSpecifier; use futures::stream::FuturesUnordered; @@ -211,17 +212,17 @@ impl ModuleGraphLoader { imports.push(import_descriptor); } + let available_libs = get_available_libs(); + for ref_desc in ref_descs { - let resolve_result = ModuleSpecifier::resolve_import( + if available_libs.contains(&ref_desc.specifier) { + continue; + } + + let resolved_specifier = ModuleSpecifier::resolve_import( &ref_desc.specifier, &module_specifier.to_string(), - ); - - // Skip for libs like "dom" or "esnext" - let resolved_specifier = match resolve_result { - Ok(s) => s, - Err(_) => continue, - }; + )?; let reference_descriptor = ReferenceDescriptor { specifier: ref_desc.specifier.to_string(), @@ -423,17 +424,17 @@ impl ModuleGraphLoader { imports.push(import_descriptor); } + let available_libs = get_available_libs(); + for ref_desc in ref_descs { - let resolve_result = ModuleSpecifier::resolve_import( + if available_libs.contains(&ref_desc.specifier) { + continue; + } + + let resolved_specifier = ModuleSpecifier::resolve_import( &ref_desc.specifier, &module_specifier.to_string(), - ); - - // Skip for libs like "dom" or "esnext" - let resolved_specifier = match resolve_result { - Ok(s) => s, - Err(_) => continue, - }; + )?; let reference_descriptor = ReferenceDescriptor { specifier: ref_desc.specifier.to_string(), diff --git a/cli/tests/020_json_modules.ts.out b/cli/tests/020_json_modules.ts.out index 7fc60d37eafb76..4369639eb4aa38 100644 --- a/cli/tests/020_json_modules.ts.out +++ b/cli/tests/020_json_modules.ts.out @@ -4,6 +4,6 @@ error: Uncaught TypeError: Cannot resolve extension for "[WILDCARD]config.json" at new SourceFile ($deno$/compiler.ts:[WILDCARD]) at Function.addToCache ($deno$/compiler.ts:[WILDCARD]) at buildSourceFileCache ($deno$/compiler.ts:[WILDCARD]) - at async compile ($deno$/compiler.ts:[WILDCARD]) - at async tsCompilerOnMessage ($deno$/compiler.ts:[WILDCARD]) - at async workerMessageRecvCallback ($deno$/runtime_worker.ts:[WILDCARD]) + at compile ($deno$/compiler.ts:[WILDCARD]) + at tsCompilerOnMessage ($deno$/compiler.ts:[WILDCARD]) +[WILDCARD] \ No newline at end of file diff --git a/cli/tests/error_005_missing_dynamic_import.ts.out b/cli/tests/error_005_missing_dynamic_import.ts.out index 3e2c3264a89e11..0f0e449c40fb34 100644 --- a/cli/tests/error_005_missing_dynamic_import.ts.out +++ b/cli/tests/error_005_missing_dynamic_import.ts.out @@ -1 +1 @@ -[WILDCARD]error: Uncaught TypeError: Cannot resolve module "[WILDCARD]/bad-module.ts" from "[WILDCARD]/error_005_missing_dynamic_import.ts" +[WILDCARD]error: Uncaught TypeError: Cannot resolve module "[WILDCARD]/bad-module.ts" diff --git a/cli/tsc.rs b/cli/tsc.rs index 2503b408f29ea6..e96976a7260b22 100644 --- a/cli/tsc.rs +++ b/cli/tsc.rs @@ -53,6 +53,71 @@ use std::task::Poll; use std::time::Instant; use url::Url; +// TODO(bartlomieju): make static +pub fn get_available_libs() -> Vec<String> { + vec![ + "deno.ns".to_string(), + "deno.window".to_string(), + "deno.worker".to_string(), + "deno.shared_globals".to_string(), + "deno.unstable".to_string(), + "dom".to_string(), + "dom.iterable".to_string(), + "es5".to_string(), + "es6".to_string(), + "esnext".to_string(), + "es2020".to_string(), + "es2020.full".to_string(), + "es2019".to_string(), + "es2019.full".to_string(), + "es2018".to_string(), + "es2018.full".to_string(), + "es2017".to_string(), + "es2017.full".to_string(), + "es2016".to_string(), + "es2016.full".to_string(), + "es2015".to_string(), + "es2015.collection".to_string(), + "es2015.core".to_string(), + "es2015.generator".to_string(), + "es2015.iterable".to_string(), + "es2015.promise".to_string(), + "es2015.proxy".to_string(), + "es2015.reflect".to_string(), + "es2015.symbol".to_string(), + "es2015.symbol.wellknown".to_string(), + "es2016.array.include".to_string(), + "es2017.intl".to_string(), + "es2017.object".to_string(), + "es2017.sharedmemory".to_string(), + "es2017.string".to_string(), + "es2017.typedarrays".to_string(), + "es2018.asyncgenerator".to_string(), + "es2018.asynciterable".to_string(), + "es2018.intl".to_string(), + "es2018.promise".to_string(), + "es2018.regexp".to_string(), + "es2019.array".to_string(), + "es2019.object".to_string(), + "es2019.string".to_string(), + "es2019.symbol".to_string(), + "es2020.bigint".to_string(), + "es2020.promise".to_string(), + "es2020.string".to_string(), + "es2020.symbol.wellknown".to_string(), + "esnext.array".to_string(), + "esnext.asynciterable".to_string(), + "esnext.bigint".to_string(), + "esnext.intl".to_string(), + "esnext.promise".to_string(), + "esnext.string".to_string(), + "esnext.symbol".to_string(), + "scripthost".to_string(), + "webworker".to_string(), + "webworker.importscripts".to_string(), + ] +} + #[derive(Debug, Clone)] pub struct CompiledModule { pub code: String, diff --git a/core/modules.rs b/core/modules.rs index 9aa0981b87bbcc..632df2dd000b2f 100644 --- a/core/modules.rs +++ b/core/modules.rs @@ -220,13 +220,6 @@ impl RecursiveModuleLoad { }) .boxed() } - LoadState::ResolveImport(ref _specifier, ref referrer) => { - let ref_spec = ModuleSpecifier::resolve_url(referrer)?; - self - .loader - .load(&module_specifier, Some(ref_spec), self.is_dynamic_import()) - .boxed_local() - } _ => self .loader .load(&module_specifier, None, self.is_dynamic_import()) From afcfb414ae2974d307dd43b339b37ac945ee3aca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= <biwanczuk@gmail.com> Date: Sat, 16 May 2020 21:52:47 +0200 Subject: [PATCH 36/47] cleanup --- cli/file_fetcher.rs | 12 ------------ cli/module_graph.rs | 5 +++-- cli/ops/compiler.rs | 5 +++-- 3 files changed, 6 insertions(+), 16 deletions(-) diff --git a/cli/file_fetcher.rs b/cli/file_fetcher.rs index 32971e71f06b91..cbfb340ddc6121 100644 --- a/cli/file_fetcher.rs +++ b/cli/file_fetcher.rs @@ -147,18 +147,6 @@ impl SourceFileFetcher { self.source_file_cache.set(specifier.to_string(), file); } - // TODO(bartlomieju): remove - pub async fn new_fetch_source_file( - &self, - specifier: ModuleSpecifier, - maybe_referrer: Option<ModuleSpecifier>, - permissions: Permissions, - ) -> Result<SourceFile, ErrBox> { - self - .fetch_source_file(&specifier, maybe_referrer, permissions) - .await - } - pub async fn fetch_source_file( &self, specifier: &ModuleSpecifier, diff --git a/cli/module_graph.rs b/cli/module_graph.rs index 78f7d673bbaabd..6628beb9795a1c 100644 --- a/cli/module_graph.rs +++ b/cli/module_graph.rs @@ -319,14 +319,15 @@ impl ModuleGraphLoader { let perms = self.permissions.clone(); let load_future = async move { + let spec_ = spec.clone(); let source_file = file_fetcher - .new_fetch_source_file(spec.clone(), maybe_referrer, perms) + .fetch_source_file(&spec_, maybe_referrer, perms) .await?; // FIXME(bartlomieju): // because of redirects we may end up with wrong URL, // substitute with original one Ok(SourceFile { - url: spec.as_url().to_owned(), + url: spec_.as_url().to_owned(), ..source_file }) } diff --git a/cli/ops/compiler.rs b/cli/ops/compiler.rs index 625f0ac7f9d6a6..2e5842c0fd6ef1 100644 --- a/cli/ops/compiler.rs +++ b/cli/ops/compiler.rs @@ -3,8 +3,9 @@ use crate::state::State; use deno_core::CoreIsolate; pub fn init(i: &mut CoreIsolate, _s: &State) { - let custom_assets = std::collections::HashMap::new(); // TODO(ry) use None. - // TODO(bartlomieju): is this op even required? + let custom_assets = std::collections::HashMap::new(); + // TODO(ry) use None. + // TODO(bartlomieju): is this op even required? i.register_op( "op_fetch_asset", deno_typescript::op_fetch_asset(custom_assets), From 6897f7f94559089b9bf0ede11bddbd8c769cca7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= <biwanczuk@gmail.com> Date: Sun, 17 May 2020 01:02:32 +0200 Subject: [PATCH 37/47] run TS compiler worker on the same thread --- cli/js/compiler.ts | 3 ++- cli/tsc.rs | 31 ++++++++++--------------------- 2 files changed, 12 insertions(+), 22 deletions(-) diff --git a/cli/js/compiler.ts b/cli/js/compiler.ts index 8e77ada67f5a41..59562b4133b3ff 100644 --- a/cli/js/compiler.ts +++ b/cli/js/compiler.ts @@ -1398,7 +1398,8 @@ async function tsCompilerOnMessage({ } (${CompilerRequestType[(request as CompilerRequest).type]})` ); } - // Currently Rust shuts down worker after single request + // Shutdown after single request + globalThis.close(); } function bootstrapTsCompilerRuntime(): void { diff --git a/cli/tsc.rs b/cli/tsc.rs index c87a42a51cb0e2..80735059910559 100644 --- a/cli/tsc.rs +++ b/cli/tsc.rs @@ -18,10 +18,8 @@ use crate::source_maps::SourceMapGetter; use crate::startup_data; use crate::state::exit_unstable; use crate::state::State; -use crate::tokio_util; use crate::version; use crate::web_worker::WebWorker; -use crate::web_worker::WebWorkerHandle; use crate::worker::WorkerEvent; use core::task::Context; use deno_core::Buf; @@ -448,7 +446,8 @@ impl TsCompiler { let req_msg = j.to_string().into_boxed_str().into_boxed_bytes(); let msg = - execute_in_thread(global_state.clone(), permissions, req_msg).await?; + execute_in_same_thread(global_state.clone(), permissions, req_msg) + .await?; let json_str = std::str::from_utf8(&msg).unwrap(); debug!("Message: {}", json_str); @@ -604,7 +603,8 @@ impl TsCompiler { let start = Instant::now(); let msg = - execute_in_thread(global_state.clone(), permissions, req_msg).await?; + execute_in_same_thread(global_state.clone(), permissions, req_msg) + .await?; let end = Instant::now(); debug!("time spent in compiler thread {:#?}", end - start); @@ -907,32 +907,21 @@ impl TsCompiler { } } -async fn execute_in_thread( +async fn execute_in_same_thread( global_state: GlobalState, permissions: Permissions, req: Buf, ) -> Result<Buf, ErrBox> { - let (handle_sender, handle_receiver) = - std::sync::mpsc::sync_channel::<Result<WebWorkerHandle, ErrBox>>(1); - let builder = - std::thread::Builder::new().name("deno-ts-compiler".to_string()); - let join_handle = builder.spawn(move || { - let worker = TsCompiler::setup_worker(global_state.clone(), permissions); - handle_sender.send(Ok(worker.thread_safe_handle())).unwrap(); - drop(handle_sender); - tokio_util::run_basic(worker).expect("Panic in event loop"); - })?; - let handle = handle_receiver.recv().unwrap()?; + let worker = TsCompiler::setup_worker(global_state.clone(), permissions); + let handle = worker.thread_safe_handle(); handle.post_message(req)?; + worker.await?; let event = handle.get_event().await.expect("Compiler didn't respond"); let buf = match event { WorkerEvent::Message(buf) => Ok(buf), WorkerEvent::Error(error) => Err(error), WorkerEvent::TerminalError(error) => Err(error), }?; - // Shutdown worker and wait for thread to finish - handle.terminate(); - join_handle.join().unwrap(); Ok(buf) } @@ -1002,7 +991,7 @@ pub async fn runtime_compile<S: BuildHasher>( let compiler = global_state.ts_compiler.clone(); - let msg = execute_in_thread(global_state, permissions, req_msg).await?; + let msg = execute_in_same_thread(global_state, permissions, req_msg).await?; let json_str = std::str::from_utf8(&msg).unwrap(); // TODO(bartlomieju): factor `bundle` path into separate function `runtime_bundle` @@ -1040,7 +1029,7 @@ pub async fn runtime_transpile<S: BuildHasher>( .into_boxed_str() .into_boxed_bytes(); - let msg = execute_in_thread(global_state, permissions, req_msg).await?; + let msg = execute_in_same_thread(global_state, permissions, req_msg).await?; let json_str = std::str::from_utf8(&msg).unwrap(); let v = serde_json::from_str::<serde_json::Value>(json_str) .expect("Error decoding JSON string."); From 0eb065c6ceef93918d95acd62694b4d2c41e9ff7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= <biwanczuk@gmail.com> Date: Sun, 17 May 2020 01:48:34 +0200 Subject: [PATCH 38/47] properly poll ts worker --- cli/tsc.rs | 33 ++++++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/cli/tsc.rs b/cli/tsc.rs index 80735059910559..bd9fa1b71133c2 100644 --- a/cli/tsc.rs +++ b/cli/tsc.rs @@ -26,6 +26,7 @@ use deno_core::Buf; use deno_core::ErrBox; use deno_core::ModuleSpecifier; use deno_core::StartupData; +use futures::future::Either; use futures::future::Future; use futures::future::FutureExt; use log::info; @@ -912,17 +913,31 @@ async fn execute_in_same_thread( permissions: Permissions, req: Buf, ) -> Result<Buf, ErrBox> { - let worker = TsCompiler::setup_worker(global_state.clone(), permissions); + let mut worker = TsCompiler::setup_worker(global_state.clone(), permissions); let handle = worker.thread_safe_handle(); handle.post_message(req)?; - worker.await?; - let event = handle.get_event().await.expect("Compiler didn't respond"); - let buf = match event { - WorkerEvent::Message(buf) => Ok(buf), - WorkerEvent::Error(error) => Err(error), - WorkerEvent::TerminalError(error) => Err(error), - }?; - Ok(buf) + + let mut event_fut = handle.get_event().boxed_local(); + + loop { + let select_result = futures::future::select(event_fut, &mut worker).await; + match select_result { + Either::Left((event_result, _worker)) => { + let event = event_result.expect("Compiler didn't respond"); + + let buf = match event { + WorkerEvent::Message(buf) => Ok(buf), + WorkerEvent::Error(error) => Err(error), + WorkerEvent::TerminalError(error) => Err(error), + }?; + return Ok(buf); + } + Either::Right((worker_result, event_fut_)) => { + event_fut = event_fut_; + worker_result?; + } + } + } } /// This function is used by `Deno.compile()` and `Deno.bundle()` APIs. From cb8ced7b6046256489feb85f36f49783313d83a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= <biwanczuk@gmail.com> Date: Sun, 17 May 2020 02:36:19 +0200 Subject: [PATCH 39/47] don't use oldProgram --- cli/js/compiler.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cli/js/compiler.ts b/cli/js/compiler.ts index 59562b4133b3ff..34f8e5c74edb1a 100644 --- a/cli/js/compiler.ts +++ b/cli/js/compiler.ts @@ -532,7 +532,9 @@ SNAPSHOT_HOST.getSourceFile( ts.ScriptTarget.ESNext ); -const TS_SNAPSHOT_PROGRAM = ts.createProgram({ +// Created to hydrate source file cache with lib +// declaration files during snapshotting process. +const _TS_SNAPSHOT_PROGRAM = ts.createProgram({ rootNames: [`${ASSETS}/bootstrap.ts`], options: SNAPSHOT_COMPILER_OPTIONS, host: SNAPSHOT_HOST, @@ -1182,7 +1184,6 @@ function compile(request: CompilerRequestCompile): CompileResult { rootNames, options, host, - oldProgram: TS_SNAPSHOT_PROGRAM, }); diagnostics = ts @@ -1297,7 +1298,6 @@ function runtimeCompile( rootNames, options: host.getCompilationSettings(), host, - oldProgram: TS_SNAPSHOT_PROGRAM, }); if (bundle) { From 90e903a51b6877e1d2ae4b30af97111d5598e516 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= <biwanczuk@gmail.com> Date: Sun, 17 May 2020 13:19:22 +0200 Subject: [PATCH 40/47] fix after merge --- cli/tsc.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cli/tsc.rs b/cli/tsc.rs index bd9fa1b71133c2..fe82cbc13d4264 100644 --- a/cli/tsc.rs +++ b/cli/tsc.rs @@ -923,7 +923,9 @@ async fn execute_in_same_thread( let select_result = futures::future::select(event_fut, &mut worker).await; match select_result { Either::Left((event_result, _worker)) => { - let event = event_result.expect("Compiler didn't respond"); + let event = event_result + .expect("Compiler didn't respond") + .expect("Empty message"); let buf = match event { WorkerEvent::Message(buf) => Ok(buf), From e3ebbe7347fc5f02e17e1ec6c9e9e2e46e8910ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= <biwanczuk@gmail.com> Date: Sun, 17 May 2020 15:58:38 +0200 Subject: [PATCH 41/47] remove debug code --- cli/tsc.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/cli/tsc.rs b/cli/tsc.rs index fe82cbc13d4264..43d30f3374bfca 100644 --- a/cli/tsc.rs +++ b/cli/tsc.rs @@ -970,7 +970,7 @@ pub async fn runtime_compile<S: BuildHasher>( module_graph_loader.add_to_graph(&module_specifier).await?; } - // TODO: download all additional files from TSconfig and add them to root_names + // download all additional files from TSconfig and add them to root_names if let Some(options) = maybe_options { let options_json: serde_json::Value = serde_json::from_str(options)?; if let Some(types_option) = options_json.get("types") { @@ -992,7 +992,6 @@ pub async fn runtime_compile<S: BuildHasher>( let module_graph_json = serde_json::to_value(module_graph).expect("Failed to serialize data"); - // eprintln!("source graph {:#?}", module_graph_json); let req_msg = json!({ "type": msg::CompilerRequestType::RuntimeCompile as i32, "target": "runtime", @@ -1023,7 +1022,6 @@ pub async fn runtime_compile<S: BuildHasher>( compiler.cache_emitted_files(response.emit_map)?; } - // eprintln!("returned {:#?}", json_str); // We're returning `Ok()` instead of `Err()` because it's not runtime // error if there were diagnostics produces; we want to let user handle // diagnostics in the runtime. From f96c3e20537224d18fc421783af5b03fdf930210 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= <biwanczuk@gmail.com> Date: Sun, 17 May 2020 19:46:12 +0200 Subject: [PATCH 42/47] fix tests --- cli/module_graph.rs | 458 +++++++++++++-------------------- cli/tests/integration_tests.rs | 2 +- cli/tsc.rs | 13 +- 3 files changed, 186 insertions(+), 287 deletions(-) diff --git a/cli/module_graph.rs b/cli/module_graph.rs index 6628beb9795a1c..c3b983bc645025 100644 --- a/cli/module_graph.rs +++ b/cli/module_graph.rs @@ -261,26 +261,6 @@ impl ModuleGraphLoader { Ok(()) } - // TODO: remove - pub async fn build_graph( - mut self, - specifier: &ModuleSpecifier, - ) -> Result<HashMap<String, ModuleGraphFile>, ErrBox> { - self.download_module(specifier.clone(), None)?; - - loop { - let load_result = self.pending_downloads.next().await.unwrap(); - let source_file = load_result?; - let spec = ModuleSpecifier::from(source_file.url.clone()); - self.visit_module(&spec, source_file)?; - if self.pending_downloads.is_empty() { - break; - } - } - - Ok(self.graph.0) - } - pub fn get_graph(self) -> HashMap<String, ModuleGraphFile> { self.graph.0 } @@ -484,343 +464,261 @@ impl ModuleGraphLoader { mod tests { use super::*; use crate::GlobalState; - // use std::path::PathBuf; - // fn rel_module_specifier(relpath: &str) -> ModuleSpecifier { - // let p = PathBuf::from(env!("CARGO_MANIFEST_DIR")) - // .join(relpath) - // .into_os_string(); - // let ps = p.to_str().unwrap(); - // ModuleSpecifier::resolve_url_or_path(ps).unwrap() - // } + async fn build_graph( + module_specifier: &ModuleSpecifier, + ) -> Result<HashMap<String, ModuleGraphFile>, ErrBox> { + let global_state = GlobalState::new(Default::default()).unwrap(); + let mut graph_loader = ModuleGraphLoader::new( + global_state.file_fetcher.clone(), + None, + Permissions::allow_all(), + false, + false, + ); + graph_loader.add_to_graph(&module_specifier).await?; + Ok(graph_loader.get_graph()) + } - #[ignore] #[tokio::test] async fn source_graph_fetch() { let http_server_guard = crate::test_util::http_server(); - let global_state = GlobalState::new(Default::default()).unwrap(); let module_specifier = ModuleSpecifier::resolve_url_or_path( "http://localhost:4545/cli/tests/019_media_types.ts", ) .unwrap(); - let graph_loader = ModuleGraphLoader::new( - global_state.file_fetcher.clone(), - None, - Permissions::allow_all(), - false, - false, - ); - let graph = graph_loader.build_graph(&module_specifier).await.unwrap(); + let graph = build_graph(&module_specifier) + .await + .expect("Failed to build graph"); + + let a = graph + .get("http://localhost:4545/cli/tests/019_media_types.ts") + .unwrap(); + + assert!(graph.contains_key( + "http://localhost:4545/cli/tests/subdir/mt_text_ecmascript.j3.js" + )); + assert!(graph.contains_key( + "http://localhost:4545/cli/tests/subdir/mt_video_vdn.t2.ts" + )); + assert!(graph.contains_key("http://localhost:4545/cli/tests/subdir/mt_application_x_typescript.t4.ts")); + assert!(graph.contains_key( + "http://localhost:4545/cli/tests/subdir/mt_video_mp2t.t3.ts" + )); + assert!(graph.contains_key("http://localhost:4545/cli/tests/subdir/mt_application_x_javascript.j4.js")); + assert!(graph.contains_key( + "http://localhost:4545/cli/tests/subdir/mt_application_ecmascript.j2.js" + )); + assert!(graph.contains_key( + "http://localhost:4545/cli/tests/subdir/mt_text_javascript.j1.js" + )); + assert!(graph.contains_key( + "http://localhost:4545/cli/tests/subdir/mt_text_typescript.t1.ts" + )); assert_eq!( - serde_json::to_value(&graph).unwrap(), - json!({ - "http://localhost:4545/cli/tests/subdir/mt_text_typescript.t1.ts": { + serde_json::to_value(&a.imports).unwrap(), + json!([ + { "specifier": "http://localhost:4545/cli/tests/subdir/mt_text_typescript.t1.ts", - "deps": [] - }, - "http://localhost:4545/cli/tests/019_media_types.ts": { - "specifier": "http://localhost:4545/cli/tests/019_media_types.ts", - "deps": [ - "http://localhost:4545/cli/tests/subdir/mt_text_typescript.t1.ts", - "http://localhost:4545/cli/tests/subdir/mt_video_vdn.t2.ts", - "http://localhost:4545/cli/tests/subdir/mt_video_mp2t.t3.ts", - "http://localhost:4545/cli/tests/subdir/mt_application_x_typescript.t4.ts", - "http://localhost:4545/cli/tests/subdir/mt_text_javascript.j1.js", - "http://localhost:4545/cli/tests/subdir/mt_application_ecmascript.j2.js", - "http://localhost:4545/cli/tests/subdir/mt_text_ecmascript.j3.js", - "http://localhost:4545/cli/tests/subdir/mt_application_x_javascript.j4.js" - ] - }, - "http://localhost:4545/cli/tests/subdir/mt_text_ecmascript.j3.js": { - "specifier": "http://localhost:4545/cli/tests/subdir/mt_text_ecmascript.j3.js", - "deps": [] + "resolvedSpecifier": "http://localhost:4545/cli/tests/subdir/mt_text_typescript.t1.ts", + "typeDirective": null, + "resolvedTypeDirective": null, }, - "http://localhost:4545/cli/tests/subdir/mt_video_vdn.t2.ts": { + { "specifier": "http://localhost:4545/cli/tests/subdir/mt_video_vdn.t2.ts", - "deps": [] - }, - "http://localhost:4545/cli/tests/subdir/mt_application_x_typescript.t4.ts": { - "specifier": "http://localhost:4545/cli/tests/subdir/mt_application_x_typescript.t4.ts", - "deps": [] + "resolvedSpecifier": "http://localhost:4545/cli/tests/subdir/mt_video_vdn.t2.ts", + "typeDirective": null, + "resolvedTypeDirective": null, }, - "http://localhost:4545/cli/tests/subdir/mt_video_mp2t.t3.ts": { + { "specifier": "http://localhost:4545/cli/tests/subdir/mt_video_mp2t.t3.ts", - "deps": [] + "resolvedSpecifier": "http://localhost:4545/cli/tests/subdir/mt_video_mp2t.t3.ts", + "typeDirective": null, + "resolvedTypeDirective": null, }, - "http://localhost:4545/cli/tests/subdir/mt_application_x_javascript.j4.js": { - "specifier": "http://localhost:4545/cli/tests/subdir/mt_application_x_javascript.j4.js", - "deps": [] + { + "specifier": "http://localhost:4545/cli/tests/subdir/mt_application_x_typescript.t4.ts", + "resolvedSpecifier": "http://localhost:4545/cli/tests/subdir/mt_application_x_typescript.t4.ts", + "typeDirective": null, + "resolvedTypeDirective": null, + }, + { + "specifier": "http://localhost:4545/cli/tests/subdir/mt_text_javascript.j1.js", + "resolvedSpecifier": "http://localhost:4545/cli/tests/subdir/mt_text_javascript.j1.js", + "typeDirective": null, + "resolvedTypeDirective": null, }, - "http://localhost:4545/cli/tests/subdir/mt_application_ecmascript.j2.js": { + { "specifier": "http://localhost:4545/cli/tests/subdir/mt_application_ecmascript.j2.js", - "deps": [] + "resolvedSpecifier": "http://localhost:4545/cli/tests/subdir/mt_application_ecmascript.j2.js", + "typeDirective": null, + "resolvedTypeDirective": null, }, - "http://localhost:4545/cli/tests/subdir/mt_text_javascript.j1.js": { - "specifier": "http://localhost:4545/cli/tests/subdir/mt_text_javascript.j1.js", - "deps": [] - } - }) - ); - drop(http_server_guard); - } - - #[ignore] - #[tokio::test] - async fn source_graph_fetch_circular() { - let http_server_guard = crate::test_util::http_server(); - - let global_state = GlobalState::new(Default::default()).unwrap(); - let module_specifier = ModuleSpecifier::resolve_url_or_path( - "http://localhost:4545/cli/tests/circular1.js", - ) - .unwrap(); - - let graph_loader = ModuleGraphLoader::new( - global_state.file_fetcher.clone(), - None, - Permissions::allow_all(), - false, - false, - ); - let graph = graph_loader.build_graph(&module_specifier).await.unwrap(); - - assert_eq!( - serde_json::to_value(&graph).unwrap(), - json!({ - "http://localhost:4545/cli/tests/circular2.js": { - "specifier": "http://localhost:4545/cli/tests/circular2.js", - "deps": [ - "http://localhost:4545/cli/tests/circular1.js" - ] + { + "specifier": "http://localhost:4545/cli/tests/subdir/mt_text_ecmascript.j3.js", + "resolvedSpecifier": "http://localhost:4545/cli/tests/subdir/mt_text_ecmascript.j3.js", + "typeDirective": null, + "resolvedTypeDirective": null, }, - "http://localhost:4545/cli/tests/circular1.js": { - "specifier": "http://localhost:4545/cli/tests/circular1.js", - "deps": [ - "http://localhost:4545/cli/tests/circular2.js" - ] - } - }) + { + "specifier": "http://localhost:4545/cli/tests/subdir/mt_application_x_javascript.j4.js", + "resolvedSpecifier": "http://localhost:4545/cli/tests/subdir/mt_application_x_javascript.j4.js", + "typeDirective": null, + "resolvedTypeDirective": null, + }, + ]) ); drop(http_server_guard); } - #[ignore] #[tokio::test] async fn source_graph_type_references() { let http_server_guard = crate::test_util::http_server(); - let global_state = GlobalState::new(Default::default()).unwrap(); let module_specifier = ModuleSpecifier::resolve_url_or_path( "http://localhost:4545/cli/tests/type_definitions.ts", ) .unwrap(); - let graph_loader = ModuleGraphLoader::new( - global_state.file_fetcher.clone(), - None, - Permissions::allow_all(), - false, - false, - ); - let graph = graph_loader.build_graph(&module_specifier).await.unwrap(); + let graph = build_graph(&module_specifier) + .await + .expect("Failed to build graph"); eprintln!("json {:#?}", serde_json::to_value(&graph).unwrap()); + let a = graph + .get("http://localhost:4545/cli/tests/type_definitions.ts") + .unwrap(); assert_eq!( - serde_json::to_value(&graph).unwrap(), - json!({ - "http://localhost:4545/cli/tests/type_definitions.ts": { - "specifier": "http://localhost:4545/cli/tests/type_definitions.ts", - "imports": [ - { - "specifier": "./type_definitions/foo.js", - "resolvedSpecifier": "http://localhost:4545/cli/tests/type_definitions/foo.js", - "typeDirective": "./type_definitions/foo.d.ts", - "resolvedTypeDirective": "http://localhost:4545/cli/tests/type_definitions/foo.d.ts" - }, - { - "specifier": "./type_definitions/fizz.js", - "resolvedSpecifier": "http://localhost:4545/cli/tests/type_definitions/fizz.js", - "typeDirective": "./type_definitions/fizz.d.ts", - "resolvedTypeDirective": "http://localhost:4545/cli/tests/type_definitions/fizz.d.ts" - }, - { - "specifier": "./type_definitions/qat.ts", - "resolvedSpecifier": "http://localhost:4545/cli/tests/type_definitions/qat.ts", - "typeDirective": null, - "resolvedTypeDirective": null, - }, - ], - "typesDirectives": [], - "referencedFiles": [], - "libDirectives": [], - "typeHeaders": [], - }, - "http://localhost:4545/cli/tests/type_definitions/foo.js": { - "specifier": "http://localhost:4545/cli/tests/type_definitions/foo.js", - "imports": [], - "referencedFiles": [], - "libDirectives": [], - "typesDirectives": [], - "typeHeaders": [], - }, - "http://localhost:4545/cli/tests/type_definitions/foo.d.ts": { - "specifier": "http://localhost:4545/cli/tests/type_definitions/foo.d.ts", - "imports": [], - "referencedFiles": [], - "libDirectives": [], - "typesDirectives": [], - "typeHeaders": [], + serde_json::to_value(&a.imports).unwrap(), + json!([ + { + "specifier": "./type_definitions/foo.js", + "resolvedSpecifier": "http://localhost:4545/cli/tests/type_definitions/foo.js", + "typeDirective": "./type_definitions/foo.d.ts", + "resolvedTypeDirective": "http://localhost:4545/cli/tests/type_definitions/foo.d.ts" }, - "http://localhost:4545/cli/tests/type_definitions/fizz.js": { - "specifier": "http://localhost:4545/cli/tests/type_definitions/fizz.js", - "imports": [], - "referencedFiles": [], - "libDirectives": [], - "typesDirectives": [], - "typeHeaders": [], + { + "specifier": "./type_definitions/fizz.js", + "resolvedSpecifier": "http://localhost:4545/cli/tests/type_definitions/fizz.js", + "typeDirective": "./type_definitions/fizz.d.ts", + "resolvedTypeDirective": "http://localhost:4545/cli/tests/type_definitions/fizz.d.ts" }, - "http://localhost:4545/cli/tests/type_definitions/fizz.d.ts": { - "specifier": "http://localhost:4545/cli/tests/type_definitions/fizz.d.ts", - "imports": [], - "referencedFiles": [], - "libDirectives": [], - "typesDirectives": [], - "typeHeaders": [], + { + "specifier": "./type_definitions/qat.ts", + "resolvedSpecifier": "http://localhost:4545/cli/tests/type_definitions/qat.ts", + "typeDirective": null, + "resolvedTypeDirective": null, }, - "http://localhost:4545/cli/tests/type_definitions/qat.ts": { - "specifier": "http://localhost:4545/cli/tests/type_definitions/qat.ts", - "imports": [], - "referencedFiles": [], - "libDirectives": [], - "typesDirectives": [], - "typeHeaders": [], - } - }) + ]) ); + assert!(graph + .contains_key("http://localhost:4545/cli/tests/type_definitions/foo.js")); + assert!(graph.contains_key( + "http://localhost:4545/cli/tests/type_definitions/foo.d.ts" + )); + assert!(graph.contains_key( + "http://localhost:4545/cli/tests/type_definitions/fizz.js" + )); + assert!(graph.contains_key( + "http://localhost:4545/cli/tests/type_definitions/fizz.d.ts" + )); + assert!(graph + .contains_key("http://localhost:4545/cli/tests/type_definitions/qat.ts")); + drop(http_server_guard); } - #[ignore] #[tokio::test] async fn source_graph_type_references2() { let http_server_guard = crate::test_util::http_server(); - let global_state = GlobalState::new(Default::default()).unwrap(); let module_specifier = ModuleSpecifier::resolve_url_or_path( "http://localhost:4545/cli/tests/type_directives_02.ts", ) .unwrap(); - let graph_loader = ModuleGraphLoader::new( - global_state.file_fetcher.clone(), - None, - Permissions::allow_all(), - false, - false, - ); - let graph = graph_loader.build_graph(&module_specifier).await.unwrap(); + let graph = build_graph(&module_specifier) + .await + .expect("Failed to build graph"); eprintln!("{:#?}", serde_json::to_value(&graph).unwrap()); + let a = graph + .get("http://localhost:4545/cli/tests/type_directives_02.ts") + .unwrap(); assert_eq!( - serde_json::to_value(&graph).unwrap(), - json!({ - "http://localhost:4545/cli/tests/type_directives_02.ts": { - "specifier": "http://localhost:4545/cli/tests/type_directives_02.ts", - "imports": [ - { - "specifier": "./subdir/type_reference.js", - "resolvedSpecifier": "http://localhost:4545/cli/tests/subdir/type_reference.js", - "typeDirective": null, - "resolvedTypeDirective": null, - } - ], - "typesDirectives": [], - "referencedFiles": [], - "libDirectives": [], - "typeHeaders": [], - }, - "http://localhost:4545/cli/tests/subdir/type_reference.d.ts": { - "specifier": "http://localhost:4545/cli/tests/subdir/type_reference.d.ts", - "imports": [], - "referencedFiles": [], - "libDirectives": [], - "typesDirectives": [], - "typeHeaders": [], - }, - "http://localhost:4545/cli/tests/subdir/type_reference.js": { - "specifier": "http://localhost:4545/cli/tests/subdir/type_reference.js", - "imports": [], - "referencedFiles": [], - "libDirectives": [], - "typesDirectives": [ - { - "specifier": "./type_reference.d.ts", - "resolvedSpecifier": "http://localhost:4545/cli/tests/subdir/type_reference.d.ts", - } - ], - "typeHeaders": [], + serde_json::to_value(&a.imports).unwrap(), + json!([ + { + "specifier": "./subdir/type_reference.js", + "resolvedSpecifier": "http://localhost:4545/cli/tests/subdir/type_reference.js", + "typeDirective": null, + "resolvedTypeDirective": null, } - }) + ]) + ); + + assert!(graph.contains_key( + "http://localhost:4545/cli/tests/subdir/type_reference.d.ts" + )); + + let b = graph + .get("http://localhost:4545/cli/tests/subdir/type_reference.js") + .unwrap(); + assert_eq!( + serde_json::to_value(&b.types_directives).unwrap(), + json!([ + { + "specifier": "./type_reference.d.ts", + "resolvedSpecifier": "http://localhost:4545/cli/tests/subdir/type_reference.d.ts", + } + ]) ); drop(http_server_guard); } - #[ignore] #[tokio::test] async fn source_graph_type_references3() { let http_server_guard = crate::test_util::http_server(); - let global_state = GlobalState::new(Default::default()).unwrap(); let module_specifier = ModuleSpecifier::resolve_url_or_path( "http://localhost:4545/cli/tests/type_directives_01.ts", ) .unwrap(); - let graph_loader = ModuleGraphLoader::new( - global_state.file_fetcher.clone(), - None, - Permissions::allow_all(), - false, - false, - ); - let graph = graph_loader.build_graph(&module_specifier).await.unwrap(); + let graph = build_graph(&module_specifier) + .await + .expect("Failed to build graph"); + let ts = graph + .get("http://localhost:4545/cli/tests/type_directives_01.ts") + .unwrap(); assert_eq!( - serde_json::to_value(&graph).unwrap(), - json!({ - "http://localhost:4545/cli/tests/type_directives_01.ts": { - "specifier": "http://localhost:4545/cli/tests/type_directives_01.ts", - "imports": [ - { - "specifier": "http://127.0.0.1:4545/xTypeScriptTypes.js", - "resolvedSpecifier": "http://127.0.0.1:4545/xTypeScriptTypes.js", - "typeDirective": null, - "resolvedTypeDirective": null, - } - ], - "referencedFiles": [], - "libDirectives": [], - "typesDirectives": [], - "typeHeaders": [], - }, - "http://127.0.0.1:4545/xTypeScriptTypes.js": { + serde_json::to_value(&ts.imports).unwrap(), + json!([ + { "specifier": "http://127.0.0.1:4545/xTypeScriptTypes.js", - "typeHeaders": [ - { - "specifier": "./xTypeScriptTypes.d.ts", - "resolvedSpecifier": "http://127.0.0.1:4545/xTypeScriptTypes.d.ts" - } - ], - "imports": [], - "referencedFiles": [], - "libDirectives": [], - "typesDirectives": [], + "resolvedSpecifier": "http://127.0.0.1:4545/xTypeScriptTypes.js", + "typeDirective": null, + "resolvedTypeDirective": null, } - }) + ]) + ); + + let headers = graph + .get("http://127.0.0.1:4545/xTypeScriptTypes.js") + .unwrap(); + assert_eq!( + serde_json::to_value(&headers.type_headers).unwrap(), + json!([ + { + "specifier": "./xTypeScriptTypes.d.ts", + "resolvedSpecifier": "http://127.0.0.1:4545/xTypeScriptTypes.d.ts" + } + ]) ); drop(http_server_guard); } diff --git a/cli/tests/integration_tests.rs b/cli/tests/integration_tests.rs index f83094c79b4cac..476a08224db7d8 100644 --- a/cli/tests/integration_tests.rs +++ b/cli/tests/integration_tests.rs @@ -1353,7 +1353,7 @@ itest!(error_004_missing_module { }); itest!(error_005_missing_dynamic_import { - args: "run --reload --allow-readerror_005_missing_dynamic_import.ts", + args: "run --reload --allow-read error_005_missing_dynamic_import.ts", exit_code: 1, output: "error_005_missing_dynamic_import.ts.out", }); diff --git a/cli/tsc.rs b/cli/tsc.rs index 43d30f3374bfca..41eb7e43cf35ef 100644 --- a/cli/tsc.rs +++ b/cli/tsc.rs @@ -403,15 +403,15 @@ impl TsCompiler { } }; let permissions = Permissions::allow_all(); - let module_graph_loader = ModuleGraphLoader::new( + let mut module_graph_loader = ModuleGraphLoader::new( global_state.file_fetcher.clone(), import_map, permissions.clone(), false, true, ); - let module_graph = - module_graph_loader.build_graph(&module_specifier).await?; + module_graph_loader.add_to_graph(&module_specifier).await?; + let module_graph = module_graph_loader.get_graph(); let module_graph_json = serde_json::to_value(module_graph).expect("Failed to serialize data"); @@ -547,15 +547,16 @@ impl TsCompiler { Some(ImportMap::load(file_path)?) } }; - let module_graph_loader = ModuleGraphLoader::new( + let mut module_graph_loader = ModuleGraphLoader::new( global_state.file_fetcher.clone(), import_map, permissions.clone(), is_dyn_import, false, ); - let module_graph = - module_graph_loader.build_graph(&module_specifier).await?; + + module_graph_loader.add_to_graph(&module_specifier).await?; + let module_graph = module_graph_loader.get_graph(); let module_graph_json = serde_json::to_value(module_graph).expect("Failed to serialize data"); From fa3c8165752c2531d3e2e9fbada3b3791a792b1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= <biwanczuk@gmail.com> Date: Sun, 17 May 2020 22:24:14 +0200 Subject: [PATCH 43/47] review --- cli/module_graph.rs | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/cli/module_graph.rs b/cli/module_graph.rs index c3b983bc645025..fe6221264198ce 100644 --- a/cli/module_graph.rs +++ b/cli/module_graph.rs @@ -116,6 +116,12 @@ impl ModuleGraphLoader { } } + /// This method is used to add specified module and all of its + /// dependencies to the graph. + /// + /// It resolves when all dependent modules have been fetched and analyzed. + /// + /// This method can be called multiple times. pub async fn add_to_graph( &mut self, specifier: &ModuleSpecifier, @@ -123,10 +129,8 @@ impl ModuleGraphLoader { self.download_module(specifier.clone(), None)?; loop { - let load_result = self.pending_downloads.next().await.unwrap(); - let source_file = load_result?; - let spec = ModuleSpecifier::from(source_file.url.clone()); - self.visit_module(&spec, source_file)?; + let source_file = self.pending_downloads.next().await.unwrap()?; + self.visit_module(&source_file.url.into(), source_file)?; if self.pending_downloads.is_empty() { break; } @@ -135,6 +139,9 @@ impl ModuleGraphLoader { Ok(()) } + /// This method is used to create a graph from in-memory files stored in + /// a hash map. Useful for creating module graph for code received from + /// the runtime. pub fn build_local_graph<S: BuildHasher>( &mut self, _root_name: &str, @@ -147,6 +154,11 @@ impl ModuleGraphLoader { Ok(()) } + /// Consumes the loader and returns created graph. + pub fn get_graph(self) -> HashMap<String, ModuleGraphFile> { + self.graph.0 + } + fn visit_memory_module( &mut self, specifier: String, @@ -157,8 +169,6 @@ impl ModuleGraphLoader { let mut lib_directives = vec![]; let mut types_directives = vec![]; - // let mut dummy_prefix = false; - // The resolveModules op only handles fully qualified URLs for referrer. // However we will have cases where referrer is "/foo.ts". We add this dummy // prefix "memory://" in order to use resolution logic. @@ -261,10 +271,6 @@ impl ModuleGraphLoader { Ok(()) } - pub fn get_graph(self) -> HashMap<String, ModuleGraphFile> { - self.graph.0 - } - fn download_module( &mut self, module_specifier: ModuleSpecifier, From 28544c960cba7508f835380acebbc2358012a0a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= <biwanczuk@gmail.com> Date: Sun, 17 May 2020 22:29:59 +0200 Subject: [PATCH 44/47] fix --- cli/module_graph.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/module_graph.rs b/cli/module_graph.rs index fe6221264198ce..e2215742e9bbe6 100644 --- a/cli/module_graph.rs +++ b/cli/module_graph.rs @@ -130,7 +130,7 @@ impl ModuleGraphLoader { loop { let source_file = self.pending_downloads.next().await.unwrap()?; - self.visit_module(&source_file.url.into(), source_file)?; + self.visit_module(&source_file.url.clone().into(), source_file)?; if self.pending_downloads.is_empty() { break; } From 35adb5231109d0681f93e2128b6b11fc086d3676 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= <biwanczuk@gmail.com> Date: Mon, 18 May 2020 11:58:34 +0200 Subject: [PATCH 45/47] review2 --- cli/js/compiler.ts | 19 ++++++------------- cli/module_graph.rs | 1 + 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/cli/js/compiler.ts b/cli/js/compiler.ts index 34f8e5c74edb1a..0a1ffa892d8358 100644 --- a/cli/js/compiler.ts +++ b/cli/js/compiler.ts @@ -450,7 +450,9 @@ class Host implements ts.CompilerHost { return moduleNames.map((specifier) => { const maybeUrl = SourceFile.getResolvedUrl(specifier, containingFile); - util.log("compiler::host.resolveModulenames receiverd", { + util.log("compiler::host.resolveModuleNames maybeUrl", { + specifier, + containingFile, maybeUrl, sf: SourceFile.getCached(maybeUrl!), }); @@ -532,8 +534,9 @@ SNAPSHOT_HOST.getSourceFile( ts.ScriptTarget.ESNext ); -// Created to hydrate source file cache with lib -// declaration files during snapshotting process. +// We never use this program; it's only created +// during snapshotting to hydrate and populate +// source file cache with lib declaration files. const _TS_SNAPSHOT_PROGRAM = ts.createProgram({ rootNames: [`${ASSETS}/bootstrap.ts`], options: SNAPSHOT_COMPILER_OPTIONS, @@ -1175,7 +1178,6 @@ function compile(request: CompilerRequestCompile): CompileResult { } buildSourceFileCache(sourceFileMap); - const start = new Date(); // if there was a configuration and no diagnostics with it, we will continue // to generate the program and possibly emit it. if (diagnostics.length === 0) { @@ -1205,9 +1207,6 @@ function compile(request: CompilerRequestCompile): CompileResult { } } - const end = new Date(); - // @ts-ignore - util.log("time in internal TS", end - start); let bundleOutput = undefined; if (diagnostics && diagnostics.length === 0 && bundle) { @@ -1369,13 +1368,7 @@ async function tsCompilerOnMessage({ }): Promise<void> { switch (request.type) { case CompilerRequestType.Compile: { - const start = new Date(); const result = compile(request as CompilerRequestCompile); - const end = new Date(); - util.log( - // @ts-ignore - `Time spent in TS program "${end - start}"` - ); globalThis.postMessage(result); break; } diff --git a/cli/module_graph.rs b/cli/module_graph.rs index e2215742e9bbe6..c21257d824ca2b 100644 --- a/cli/module_graph.rs +++ b/cli/module_graph.rs @@ -169,6 +169,7 @@ impl ModuleGraphLoader { let mut lib_directives = vec![]; let mut types_directives = vec![]; + // FIXME(bartlomieju): // The resolveModules op only handles fully qualified URLs for referrer. // However we will have cases where referrer is "/foo.ts". We add this dummy // prefix "memory://" in order to use resolution logic. From 5d02e055fcb3a94d281e8df6e86ae1966bf0fc2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= <biwanczuk@gmail.com> Date: Mon, 18 May 2020 12:17:27 +0200 Subject: [PATCH 46/47] fmt --- cli/js/compiler.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cli/js/compiler.ts b/cli/js/compiler.ts index 0a1ffa892d8358..fa4a7526e79d4a 100644 --- a/cli/js/compiler.ts +++ b/cli/js/compiler.ts @@ -534,8 +534,8 @@ SNAPSHOT_HOST.getSourceFile( ts.ScriptTarget.ESNext ); -// We never use this program; it's only created -// during snapshotting to hydrate and populate +// We never use this program; it's only created +// during snapshotting to hydrate and populate // source file cache with lib declaration files. const _TS_SNAPSHOT_PROGRAM = ts.createProgram({ rootNames: [`${ASSETS}/bootstrap.ts`], From aa6733296545978834b0a76b956bf03a62000518 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= <biwanczuk@gmail.com> Date: Mon, 18 May 2020 12:38:44 +0200 Subject: [PATCH 47/47] reset CI