From 8a2d4e27261927c0b0ce0eac5657941c982b9746 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= <biwanczuk@gmail.com>
Date: Thu, 30 Apr 2020 16:16:12 +0200
Subject: [PATCH 01/47] add permissions argument to file fetcher methods

---
 cli/global_state.rs | 4 ++++
 cli/ops/compiler.rs | 1 +
 2 files changed, 5 insertions(+)

diff --git a/cli/global_state.rs b/cli/global_state.rs
index 4e9bdbb99c9411..ecda7c588ebdf4 100644
--- a/cli/global_state.rs
+++ b/cli/global_state.rs
@@ -100,6 +100,10 @@ impl GlobalState {
     let state2 = self.clone();
     let module_specifier = module_specifier.clone();
 
+    // TODO(bartlomieju): currently unused, but file fetcher will
+    // require them in the near future
+    let permissions = DenoPermissions::default();
+
     let out = self
       .file_fetcher
       .fetch_source_file(&module_specifier, maybe_referrer, permissions.clone())
diff --git a/cli/ops/compiler.rs b/cli/ops/compiler.rs
index b844011877c390..948b7c85e9303d 100644
--- a/cli/ops/compiler.rs
+++ b/cli/ops/compiler.rs
@@ -4,6 +4,7 @@ use super::dispatch_json::JsonOp;
 use super::dispatch_json::Value;
 use crate::futures::future::try_join_all;
 use crate::op_error::OpError;
+use crate::permissions::DenoPermissions;
 use crate::state::State;
 use deno_core::CoreIsolate;
 use deno_core::ModuleLoader;

From c7dc06593f2a910a3ff55999beb9b85dcc194f31 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= <biwanczuk@gmail.com>
Date: Sun, 10 May 2020 13:52:59 +0200
Subject: [PATCH 02/47] drop permissions prefix

---
 cli/file_fetcher.rs | 36 ++++++++++++++++++++++++++++++++++++
 cli/global_state.rs |  2 +-
 cli/ops/compiler.rs |  2 +-
 3 files changed, 38 insertions(+), 2 deletions(-)

diff --git a/cli/file_fetcher.rs b/cli/file_fetcher.rs
index 2e75517e632523..ac685fc67376c1 100644
--- a/cli/file_fetcher.rs
+++ b/cli/file_fetcher.rs
@@ -110,7 +110,11 @@ impl SourceFileFetcher {
   pub fn fetch_cached_source_file(
     &self,
     specifier: &ModuleSpecifier,
+<<<<<<< HEAD
     permissions: Permissions,
+=======
+    _permissions: Permissions,
+>>>>>>> drop permissions prefix
   ) -> Option<SourceFile> {
     let maybe_source_file = self.source_file_cache.get(specifier.to_string());
 
@@ -150,7 +154,11 @@ impl SourceFileFetcher {
     &self,
     specifier: &ModuleSpecifier,
     maybe_referrer: Option<ModuleSpecifier>,
+<<<<<<< HEAD
     permissions: Permissions,
+=======
+    _permissions: Permissions,
+>>>>>>> drop permissions prefix
   ) -> Result<SourceFile, ErrBox> {
     let module_url = specifier.as_url().to_owned();
     debug!("fetch_source_file specifier: {} ", &module_url);
@@ -1001,7 +1009,11 @@ mod tests {
 
     // first download
     let r = fetcher
+<<<<<<< HEAD
       .fetch_source_file(&specifier, None, Permissions::allow_all())
+=======
+      .fetch_source_file(&specifier, None, Permissions::default())
+>>>>>>> drop permissions prefix
       .await;
     assert!(r.is_ok());
 
@@ -1019,7 +1031,11 @@ mod tests {
     // header file creation timestamp (should be the same as after first
     // download)
     let r = fetcher
+<<<<<<< HEAD
       .fetch_source_file(&specifier, None, Permissions::allow_all())
+=======
+      .fetch_source_file(&specifier, None, Permissions::default())
+>>>>>>> drop permissions prefix
       .await;
     assert!(r.is_ok());
 
@@ -1493,7 +1509,11 @@ mod tests {
     let specifier =
       ModuleSpecifier::resolve_url(file_url!("/baddir/hello.ts")).unwrap();
     let r = fetcher
+<<<<<<< HEAD
       .fetch_source_file(&specifier, None, Permissions::allow_all())
+=======
+      .fetch_source_file(&specifier, None, Permissions::default())
+>>>>>>> drop permissions prefix
       .await;
     assert!(r.is_err());
 
@@ -1502,7 +1522,11 @@ mod tests {
     let specifier =
       ModuleSpecifier::resolve_url_or_path(p.to_str().unwrap()).unwrap();
     let r = fetcher
+<<<<<<< HEAD
       .fetch_source_file(&specifier, None, Permissions::allow_all())
+=======
+      .fetch_source_file(&specifier, None, Permissions::default())
+>>>>>>> drop permissions prefix
       .await;
     assert!(r.is_ok());
   }
@@ -1516,7 +1540,11 @@ mod tests {
     let specifier =
       ModuleSpecifier::resolve_url(file_url!("/baddir/hello.ts")).unwrap();
     let r = fetcher
+<<<<<<< HEAD
       .fetch_source_file(&specifier, None, Permissions::allow_all())
+=======
+      .fetch_source_file(&specifier, None, Permissions::default())
+>>>>>>> drop permissions prefix
       .await;
     assert!(r.is_err());
 
@@ -1525,7 +1553,11 @@ mod tests {
     let specifier =
       ModuleSpecifier::resolve_url_or_path(p.to_str().unwrap()).unwrap();
     let r = fetcher
+<<<<<<< HEAD
       .fetch_source_file(&specifier, None, Permissions::allow_all())
+=======
+      .fetch_source_file(&specifier, None, Permissions::default())
+>>>>>>> drop permissions prefix
       .await;
     assert!(r.is_ok());
   }
@@ -1540,7 +1572,11 @@ mod tests {
     let specifier =
       ModuleSpecifier::resolve_url_or_path(p.to_str().unwrap()).unwrap();
     let r = fetcher
+<<<<<<< HEAD
       .fetch_source_file(&specifier, None, Permissions::allow_all())
+=======
+      .fetch_source_file(&specifier, None, Permissions::default())
+>>>>>>> drop permissions prefix
       .await;
     assert!(r.is_ok());
   }
diff --git a/cli/global_state.rs b/cli/global_state.rs
index ecda7c588ebdf4..94c4387e845895 100644
--- a/cli/global_state.rs
+++ b/cli/global_state.rs
@@ -102,7 +102,7 @@ impl GlobalState {
 
     // TODO(bartlomieju): currently unused, but file fetcher will
     // require them in the near future
-    let permissions = DenoPermissions::default();
+    let permissions = Permissions::default();
 
     let out = self
       .file_fetcher
diff --git a/cli/ops/compiler.rs b/cli/ops/compiler.rs
index 948b7c85e9303d..3db93ae69f3300 100644
--- a/cli/ops/compiler.rs
+++ b/cli/ops/compiler.rs
@@ -4,7 +4,7 @@ use super::dispatch_json::JsonOp;
 use super::dispatch_json::Value;
 use crate::futures::future::try_join_all;
 use crate::op_error::OpError;
-use crate::permissions::DenoPermissions;
+use crate::permissions::Permissions;
 use crate::state::State;
 use deno_core::CoreIsolate;
 use deno_core::ModuleLoader;

From a8f431de3a64250e19f902ab7aa3a0b2571bd331 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= <biwanczuk@gmail.com>
Date: Sun, 10 May 2020 14:17:19 +0200
Subject: [PATCH 03/47] add permission check to SourceFileFetcher

---
 cli/file_fetcher.rs | 36 ------------------------------------
 cli/permissions.rs  | 16 ++++++++++++++++
 2 files changed, 16 insertions(+), 36 deletions(-)

diff --git a/cli/file_fetcher.rs b/cli/file_fetcher.rs
index ac685fc67376c1..2e75517e632523 100644
--- a/cli/file_fetcher.rs
+++ b/cli/file_fetcher.rs
@@ -110,11 +110,7 @@ impl SourceFileFetcher {
   pub fn fetch_cached_source_file(
     &self,
     specifier: &ModuleSpecifier,
-<<<<<<< HEAD
     permissions: Permissions,
-=======
-    _permissions: Permissions,
->>>>>>> drop permissions prefix
   ) -> Option<SourceFile> {
     let maybe_source_file = self.source_file_cache.get(specifier.to_string());
 
@@ -154,11 +150,7 @@ impl SourceFileFetcher {
     &self,
     specifier: &ModuleSpecifier,
     maybe_referrer: Option<ModuleSpecifier>,
-<<<<<<< HEAD
     permissions: Permissions,
-=======
-    _permissions: Permissions,
->>>>>>> drop permissions prefix
   ) -> Result<SourceFile, ErrBox> {
     let module_url = specifier.as_url().to_owned();
     debug!("fetch_source_file specifier: {} ", &module_url);
@@ -1009,11 +1001,7 @@ mod tests {
 
     // first download
     let r = fetcher
-<<<<<<< HEAD
       .fetch_source_file(&specifier, None, Permissions::allow_all())
-=======
-      .fetch_source_file(&specifier, None, Permissions::default())
->>>>>>> drop permissions prefix
       .await;
     assert!(r.is_ok());
 
@@ -1031,11 +1019,7 @@ mod tests {
     // header file creation timestamp (should be the same as after first
     // download)
     let r = fetcher
-<<<<<<< HEAD
       .fetch_source_file(&specifier, None, Permissions::allow_all())
-=======
-      .fetch_source_file(&specifier, None, Permissions::default())
->>>>>>> drop permissions prefix
       .await;
     assert!(r.is_ok());
 
@@ -1509,11 +1493,7 @@ mod tests {
     let specifier =
       ModuleSpecifier::resolve_url(file_url!("/baddir/hello.ts")).unwrap();
     let r = fetcher
-<<<<<<< HEAD
       .fetch_source_file(&specifier, None, Permissions::allow_all())
-=======
-      .fetch_source_file(&specifier, None, Permissions::default())
->>>>>>> drop permissions prefix
       .await;
     assert!(r.is_err());
 
@@ -1522,11 +1502,7 @@ mod tests {
     let specifier =
       ModuleSpecifier::resolve_url_or_path(p.to_str().unwrap()).unwrap();
     let r = fetcher
-<<<<<<< HEAD
       .fetch_source_file(&specifier, None, Permissions::allow_all())
-=======
-      .fetch_source_file(&specifier, None, Permissions::default())
->>>>>>> drop permissions prefix
       .await;
     assert!(r.is_ok());
   }
@@ -1540,11 +1516,7 @@ mod tests {
     let specifier =
       ModuleSpecifier::resolve_url(file_url!("/baddir/hello.ts")).unwrap();
     let r = fetcher
-<<<<<<< HEAD
       .fetch_source_file(&specifier, None, Permissions::allow_all())
-=======
-      .fetch_source_file(&specifier, None, Permissions::default())
->>>>>>> drop permissions prefix
       .await;
     assert!(r.is_err());
 
@@ -1553,11 +1525,7 @@ mod tests {
     let specifier =
       ModuleSpecifier::resolve_url_or_path(p.to_str().unwrap()).unwrap();
     let r = fetcher
-<<<<<<< HEAD
       .fetch_source_file(&specifier, None, Permissions::allow_all())
-=======
-      .fetch_source_file(&specifier, None, Permissions::default())
->>>>>>> drop permissions prefix
       .await;
     assert!(r.is_ok());
   }
@@ -1572,11 +1540,7 @@ mod tests {
     let specifier =
       ModuleSpecifier::resolve_url_or_path(p.to_str().unwrap()).unwrap();
     let r = fetcher
-<<<<<<< HEAD
       .fetch_source_file(&specifier, None, Permissions::allow_all())
-=======
-      .fetch_source_file(&specifier, None, Permissions::default())
->>>>>>> drop permissions prefix
       .await;
     assert!(r.is_ok());
   }
diff --git a/cli/permissions.rs b/cli/permissions.rs
index bc4d0844e0ec63..e0880ccad83296 100644
--- a/cli/permissions.rs
+++ b/cli/permissions.rs
@@ -344,6 +344,22 @@ fn permission_prompt(message: &str) -> bool {
   }
 }
 
+#[cfg(test)]
+impl Permissions {
+  pub fn allow_all() -> Self {
+    Self {
+      allow_read: PermissionState::from(true),
+      allow_write: PermissionState::from(true),
+      allow_net: PermissionState::from(true),
+      allow_env: PermissionState::from(true),
+      allow_run: PermissionState::from(true),
+      allow_plugin: PermissionState::from(true),
+      allow_hrtime: PermissionState::from(true),
+      ..Default::default()
+    }
+  }
+}
+
 #[cfg(test)]
 lazy_static! {
   /// Lock this when you use `set_prompt_result` in a test case.

From 17d06b3900cc608194979f88f5012f77e2dabdd7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= <biwanczuk@gmail.com>
Date: Sun, 10 May 2020 14:24:05 +0200
Subject: [PATCH 04/47] use Permissions::allow_all

---
 cli/global_state.rs |  2 +-
 cli/permissions.rs  | 16 ----------------
 2 files changed, 1 insertion(+), 17 deletions(-)

diff --git a/cli/global_state.rs b/cli/global_state.rs
index 94c4387e845895..8a0392e44d40bf 100644
--- a/cli/global_state.rs
+++ b/cli/global_state.rs
@@ -102,7 +102,7 @@ impl GlobalState {
 
     // TODO(bartlomieju): currently unused, but file fetcher will
     // require them in the near future
-    let permissions = Permissions::default();
+    let permissions = Permissions::allow_all();
 
     let out = self
       .file_fetcher
diff --git a/cli/permissions.rs b/cli/permissions.rs
index e0880ccad83296..bc4d0844e0ec63 100644
--- a/cli/permissions.rs
+++ b/cli/permissions.rs
@@ -344,22 +344,6 @@ fn permission_prompt(message: &str) -> bool {
   }
 }
 
-#[cfg(test)]
-impl Permissions {
-  pub fn allow_all() -> Self {
-    Self {
-      allow_read: PermissionState::from(true),
-      allow_write: PermissionState::from(true),
-      allow_net: PermissionState::from(true),
-      allow_env: PermissionState::from(true),
-      allow_run: PermissionState::from(true),
-      allow_plugin: PermissionState::from(true),
-      allow_hrtime: PermissionState::from(true),
-      ..Default::default()
-    }
-  }
-}
-
 #[cfg(test)]
 lazy_static! {
   /// Lock this when you use `set_prompt_result` in a test case.

From 38ebecdee89dfc527d3543b88136dce89f5258f0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= <biwanczuk@gmail.com>
Date: Sun, 10 May 2020 14:36:36 +0200
Subject: [PATCH 05/47] propagate proper permissions

---
 cli/global_state.rs | 4 ----
 cli/ops/compiler.rs | 1 -
 2 files changed, 5 deletions(-)

diff --git a/cli/global_state.rs b/cli/global_state.rs
index 8a0392e44d40bf..4e9bdbb99c9411 100644
--- a/cli/global_state.rs
+++ b/cli/global_state.rs
@@ -100,10 +100,6 @@ impl GlobalState {
     let state2 = self.clone();
     let module_specifier = module_specifier.clone();
 
-    // TODO(bartlomieju): currently unused, but file fetcher will
-    // require them in the near future
-    let permissions = Permissions::allow_all();
-
     let out = self
       .file_fetcher
       .fetch_source_file(&module_specifier, maybe_referrer, permissions.clone())
diff --git a/cli/ops/compiler.rs b/cli/ops/compiler.rs
index 3db93ae69f3300..b844011877c390 100644
--- a/cli/ops/compiler.rs
+++ b/cli/ops/compiler.rs
@@ -4,7 +4,6 @@ use super::dispatch_json::JsonOp;
 use super::dispatch_json::Value;
 use crate::futures::future::try_join_all;
 use crate::op_error::OpError;
-use crate::permissions::Permissions;
 use crate::state::State;
 use deno_core::CoreIsolate;
 use deno_core::ModuleLoader;

From 98984e47cf830c8c871ab9b27e1232b8f0f4c77e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= <biwanczuk@gmail.com>
Date: Sun, 10 May 2020 18:07:58 +0200
Subject: [PATCH 06/47] add docs

---
 cli/state.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/cli/state.rs b/cli/state.rs
index 9501ed2866f109..b36331085c0de0 100644
--- a/cli/state.rs
+++ b/cli/state.rs
@@ -318,8 +318,8 @@ impl ModuleLoader for State {
     let permissions = if state.is_main {
       Permissions::allow_all()
     } else {
-      state.permissions.clone()
-    };
+    state.permissions.clone()
+  };
 
     let fut = async move {
       let compiled_module = global_state

From 62e6db4cb1567f61c8ce8da069160a0b91c5abfb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= <biwanczuk@gmail.com>
Date: Thu, 7 May 2020 18:18:24 +0200
Subject: [PATCH 07/47] remove cli/compilers/js.rs

---
 cli/compilers/mod.rs | 23 +++++++++++++++++++++++
 cli/global_state.rs  | 13 +++++++++----
 2 files changed, 32 insertions(+), 4 deletions(-)
 create mode 100644 cli/compilers/mod.rs

diff --git a/cli/compilers/mod.rs b/cli/compilers/mod.rs
new file mode 100644
index 00000000000000..d32ff40eaf5552
--- /dev/null
+++ b/cli/compilers/mod.rs
@@ -0,0 +1,23 @@
+// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license.
+use crate::ops::JsonResult;
+use deno_core::ErrBox;
+use futures::Future;
+
+mod compiler_worker;
+mod ts;
+
+pub use ts::runtime_compile;
+pub use ts::runtime_transpile;
+pub use ts::TargetLib;
+pub use ts::TsCompiler;
+
+pub type CompilationResultFuture = dyn Future<Output = JsonResult>;
+
+#[derive(Debug, Clone)]
+pub struct CompiledModule {
+  pub code: String,
+  pub name: String,
+}
+
+pub type CompiledModuleFuture =
+  dyn Future<Output = Result<CompiledModule, ErrBox>>;
diff --git a/cli/global_state.rs b/cli/global_state.rs
index 4e9bdbb99c9411..50a39092bbdfbb 100644
--- a/cli/global_state.rs
+++ b/cli/global_state.rs
@@ -1,4 +1,7 @@
 // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license.
+use crate::compilers::CompiledModule;
+use crate::compilers::TargetLib;
+use crate::compilers::TsCompiler;
 use crate::deno_dir;
 use crate::file_fetcher::SourceFileFetcher;
 use crate::flags;
@@ -144,10 +147,12 @@ impl GlobalState {
           })
         }
       }
-      _ => Ok(CompiledModule {
-        code: String::from_utf8(out.source_code)?,
-        name: out.url.to_string(),
-      }),
+      _ => {
+        Ok(CompiledModule {
+          code: String::from_utf8(out.source_code)?,
+          name: out.url.to_string(),
+        })
+      },
     }?;
     drop(compile_lock);
 

From 161a36150242dd87ad75db3fea2da1fe0265ae23 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= <biwanczuk@gmail.com>
Date: Thu, 7 May 2020 18:33:25 +0200
Subject: [PATCH 08/47] remove cli/compilers/; add cli/tsc.rs

---
 cli/compilers/mod.rs | 23 -----------------------
 cli/global_state.rs  | 13 ++++---------
 2 files changed, 4 insertions(+), 32 deletions(-)
 delete mode 100644 cli/compilers/mod.rs

diff --git a/cli/compilers/mod.rs b/cli/compilers/mod.rs
deleted file mode 100644
index d32ff40eaf5552..00000000000000
--- a/cli/compilers/mod.rs
+++ /dev/null
@@ -1,23 +0,0 @@
-// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license.
-use crate::ops::JsonResult;
-use deno_core::ErrBox;
-use futures::Future;
-
-mod compiler_worker;
-mod ts;
-
-pub use ts::runtime_compile;
-pub use ts::runtime_transpile;
-pub use ts::TargetLib;
-pub use ts::TsCompiler;
-
-pub type CompilationResultFuture = dyn Future<Output = JsonResult>;
-
-#[derive(Debug, Clone)]
-pub struct CompiledModule {
-  pub code: String,
-  pub name: String,
-}
-
-pub type CompiledModuleFuture =
-  dyn Future<Output = Result<CompiledModule, ErrBox>>;
diff --git a/cli/global_state.rs b/cli/global_state.rs
index 50a39092bbdfbb..4e9bdbb99c9411 100644
--- a/cli/global_state.rs
+++ b/cli/global_state.rs
@@ -1,7 +1,4 @@
 // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license.
-use crate::compilers::CompiledModule;
-use crate::compilers::TargetLib;
-use crate::compilers::TsCompiler;
 use crate::deno_dir;
 use crate::file_fetcher::SourceFileFetcher;
 use crate::flags;
@@ -147,12 +144,10 @@ impl GlobalState {
           })
         }
       }
-      _ => {
-        Ok(CompiledModule {
-          code: String::from_utf8(out.source_code)?,
-          name: out.url.to_string(),
-        })
-      },
+      _ => Ok(CompiledModule {
+        code: String::from_utf8(out.source_code)?,
+        name: out.url.to_string(),
+      }),
     }?;
     drop(compile_lock);
 

From c1e55bc41b4505785238a1fcb9eba1e60ffeb0d4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= <biwanczuk@gmail.com>
Date: Fri, 1 May 2020 16:38:50 +0200
Subject: [PATCH 09/47] prototype module graph loader

---
 cli/lib.rs          |   1 +
 cli/module_graph.rs | 131 ++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 132 insertions(+)
 create mode 100644 cli/module_graph.rs

diff --git a/cli/lib.rs b/cli/lib.rs
index 0b88d734609204..402d54199cc67f 100644
--- a/cli/lib.rs
+++ b/cli/lib.rs
@@ -42,6 +42,7 @@ pub mod installer;
 mod js;
 mod lockfile;
 mod metrics;
+mod module_graph;
 pub mod msg;
 pub mod op_error;
 pub mod ops;
diff --git a/cli/module_graph.rs b/cli/module_graph.rs
new file mode 100644
index 00000000000000..21b5dea47498ee
--- /dev/null
+++ b/cli/module_graph.rs
@@ -0,0 +1,131 @@
+// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license.
+#![allow(unused)]
+
+use crate::file_fetcher::SourceFileFetcher;
+use crate::swc_util::analyze_dependencies;
+use deno_core::ErrBox;
+use deno_core::ModuleSpecifier;
+use serde::Serialize;
+use std::collections::HashMap;
+
+#[derive(Debug, Serialize)]
+struct ModuleGraph(HashMap<String, ModuleGraphFile>);
+
+#[derive(Debug, Serialize)]
+struct ModuleGraphFile {
+  pub specifier: String,
+  pub deps: Vec<String>,
+}
+
+struct ModuleGraphLoader {
+  file_fetcher: SourceFileFetcher,
+  to_visit: Vec<ModuleSpecifier>,
+  pub graph: ModuleGraph,
+}
+
+// struct ModuleGraphFuture {
+//   file_fetcher: SourceFileFetcher,
+//   to_visit: Vec<ModuleSpecifier>,
+//   pending_lods:
+//     FuturesUnordered<Pin<Box<dyn Future<Output = Result<SourceFile, ErrBox>>>>>,
+//   has_loaded: HashSet<ModuleSpecifier>,
+//   pending_analysis: HashSet<ModuleSpecifier>,
+// }
+
+impl ModuleGraphLoader {
+  pub fn new(file_fetcher: SourceFileFetcher) -> Self {
+    Self {
+      file_fetcher,
+      to_visit: vec![],
+      graph: ModuleGraph(HashMap::new()),
+    }
+  }
+
+  pub async fn build_graph(
+    mut self,
+    specifier: &ModuleSpecifier,
+  ) -> Result<HashMap<String, ModuleGraphFile>, ErrBox> {
+    self.to_visit.push(specifier.to_owned());
+    while let Some(spec) = self.to_visit.pop() {
+      self.visit_module(&spec).await?;
+      let file = self.graph.0.get(&spec.to_string()).unwrap();
+      for dep in &file.deps {
+        self
+          .to_visit
+          .push(ModuleSpecifier::resolve_url_or_path(dep).unwrap());
+      }
+    }
+    Ok(self.graph.0)
+  }
+
+  async fn visit_module(
+    &mut self,
+    specifier: &ModuleSpecifier,
+  ) -> Result<(), ErrBox> {
+    if self.graph.0.contains_key(&specifier.to_string()) {
+      return Ok(());
+    }
+
+    let source_file =
+      self.file_fetcher.fetch_source_file(specifier, None).await?;
+
+    let raw_deps =
+      analyze_dependencies(&String::from_utf8(source_file.source_code)?, true)?;
+
+    // TODO(bartlomieju): apply import map, using State
+    //    or should it be passed explicitly
+    let mut deps = vec![];
+    for raw_dep in raw_deps {
+      let specifier =
+        ModuleSpecifier::resolve_import(&raw_dep, &specifier.to_string())?;
+      deps.push(specifier.to_string());
+    }
+
+    self.graph.0.insert(
+      specifier.to_string(),
+      ModuleGraphFile {
+        specifier: specifier.to_string(),
+        deps,
+      },
+    );
+    Ok(())
+  }
+}
+
+#[cfg(test)]
+mod tests {
+  use super::*;
+  use crate::GlobalState;
+  use std::path::PathBuf;
+
+  fn rel_module_specifier(relpath: &str) -> ModuleSpecifier {
+    let p = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
+      .join(relpath)
+      .into_os_string();
+    let ps = p.to_str().unwrap();
+    // TODO(ry) Why doesn't ModuleSpecifier::resolve_path actually take a
+    // Path?!
+    ModuleSpecifier::resolve_url_or_path(ps).unwrap()
+  }
+
+  #[tokio::test]
+  async fn source_graph_fetch() {
+    let http_server_guard = crate::test_util::http_server();
+
+    let global_state = GlobalState::new(Default::default()).unwrap();
+    let module_specifier = rel_module_specifier("tests/019_media_types.ts");
+
+    let graph_loader =
+      ModuleGraphLoader::new(global_state.file_fetcher.clone());
+    let graph = graph_loader.build_graph(&module_specifier).await.unwrap();
+
+    assert_eq!(graph.len(), 9);
+    let r = graph
+      .get("http://localhost:4545/cli/tests/subdir/mt_text_typescript.t1.ts");
+    assert!(r.is_some());
+
+    println!("{}", serde_json::to_string_pretty(&graph).unwrap());
+
+    drop(http_server_guard);
+  }
+}

From ea6e891202e9b0f786c57ae3f1262df89d426d9a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= <biwanczuk@gmail.com>
Date: Fri, 1 May 2020 16:53:11 +0200
Subject: [PATCH 10/47] add test for circular deps

---
 cli/module_graph.rs | 31 ++++++++++++++++++++++++++++---
 1 file changed, 28 insertions(+), 3 deletions(-)

diff --git a/cli/module_graph.rs b/cli/module_graph.rs
index 21b5dea47498ee..1705fbd511b6f1 100644
--- a/cli/module_graph.rs
+++ b/cli/module_graph.rs
@@ -50,9 +50,13 @@ impl ModuleGraphLoader {
       self.visit_module(&spec).await?;
       let file = self.graph.0.get(&spec.to_string()).unwrap();
       for dep in &file.deps {
-        self
-          .to_visit
-          .push(ModuleSpecifier::resolve_url_or_path(dep).unwrap());
+        if let Some(_exists) = self.graph.0.get(dep) {
+          continue;
+        } else {
+          self
+            .to_visit
+            .push(ModuleSpecifier::resolve_url_or_path(dep).unwrap());
+        }
       }
     }
     Ok(self.graph.0)
@@ -128,4 +132,25 @@ mod tests {
 
     drop(http_server_guard);
   }
+
+  #[tokio::test]
+  async fn source_graph_fetch_circular() {
+    let http_server_guard = crate::test_util::http_server();
+
+    let global_state = GlobalState::new(Default::default()).unwrap();
+    let module_specifier = rel_module_specifier("tests/circular1.js");
+
+    let graph_loader =
+      ModuleGraphLoader::new(global_state.file_fetcher.clone());
+    let graph = graph_loader.build_graph(&module_specifier).await.unwrap();
+
+    // assert_eq!(graph.len(), 9);
+    // let r = graph
+    //   .get("http://localhost:4545/cli/tests/subdir/mt_text_typescript.t1.ts");
+    // assert!(r.is_some());
+
+    println!("{}", serde_json::to_string_pretty(&graph).unwrap());
+
+    drop(http_server_guard);
+  }
 }

From 198166a036aad4773dd17bffd147279864febec9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= <biwanczuk@gmail.com>
Date: Mon, 4 May 2020 14:10:34 +0200
Subject: [PATCH 11/47] update tests

---
 cli/module_graph.rs | 117 +++++++++++++++++++++++++++++++-------------
 1 file changed, 82 insertions(+), 35 deletions(-)

diff --git a/cli/module_graph.rs b/cli/module_graph.rs
index 1705fbd511b6f1..0effa13129658b 100644
--- a/cli/module_graph.rs
+++ b/cli/module_graph.rs
@@ -23,15 +23,6 @@ struct ModuleGraphLoader {
   pub graph: ModuleGraph,
 }
 
-// struct ModuleGraphFuture {
-//   file_fetcher: SourceFileFetcher,
-//   to_visit: Vec<ModuleSpecifier>,
-//   pending_lods:
-//     FuturesUnordered<Pin<Box<dyn Future<Output = Result<SourceFile, ErrBox>>>>>,
-//   has_loaded: HashSet<ModuleSpecifier>,
-//   pending_analysis: HashSet<ModuleSpecifier>,
-// }
-
 impl ModuleGraphLoader {
   pub fn new(file_fetcher: SourceFileFetcher) -> Self {
     Self {
@@ -102,34 +93,77 @@ mod tests {
   use crate::GlobalState;
   use std::path::PathBuf;
 
-  fn rel_module_specifier(relpath: &str) -> ModuleSpecifier {
-    let p = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
-      .join(relpath)
-      .into_os_string();
-    let ps = p.to_str().unwrap();
-    // TODO(ry) Why doesn't ModuleSpecifier::resolve_path actually take a
-    // Path?!
-    ModuleSpecifier::resolve_url_or_path(ps).unwrap()
-  }
+  // fn rel_module_specifier(relpath: &str) -> ModuleSpecifier {
+  //   let p = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
+  //     .join(relpath)
+  //     .into_os_string();
+  //   let ps = p.to_str().unwrap();
+  //   ModuleSpecifier::resolve_url_or_path(ps).unwrap()
+  // }
 
   #[tokio::test]
   async fn source_graph_fetch() {
     let http_server_guard = crate::test_util::http_server();
 
     let global_state = GlobalState::new(Default::default()).unwrap();
-    let module_specifier = rel_module_specifier("tests/019_media_types.ts");
-
+    let module_specifier = ModuleSpecifier::resolve_url_or_path(
+      "http://localhost:4545/cli/tests/019_media_types.ts",
+    )
+    .unwrap();
     let graph_loader =
       ModuleGraphLoader::new(global_state.file_fetcher.clone());
     let graph = graph_loader.build_graph(&module_specifier).await.unwrap();
 
-    assert_eq!(graph.len(), 9);
-    let r = graph
-      .get("http://localhost:4545/cli/tests/subdir/mt_text_typescript.t1.ts");
-    assert!(r.is_some());
-
-    println!("{}", serde_json::to_string_pretty(&graph).unwrap());
-
+    assert_eq!(
+      serde_json::to_value(&graph).unwrap(),
+      json!({
+        "http://localhost:4545/cli/tests/subdir/mt_text_typescript.t1.ts": {
+          "specifier": "http://localhost:4545/cli/tests/subdir/mt_text_typescript.t1.ts",
+          "deps": []
+        },
+        "http://localhost:4545/cli/tests/019_media_types.ts": {
+          "specifier": "http://localhost:4545/cli/tests/019_media_types.ts",
+          "deps": [
+            "http://localhost:4545/cli/tests/subdir/mt_text_typescript.t1.ts",
+            "http://localhost:4545/cli/tests/subdir/mt_video_vdn.t2.ts",
+            "http://localhost:4545/cli/tests/subdir/mt_video_mp2t.t3.ts",
+            "http://localhost:4545/cli/tests/subdir/mt_application_x_typescript.t4.ts",
+            "http://localhost:4545/cli/tests/subdir/mt_text_javascript.j1.js",
+            "http://localhost:4545/cli/tests/subdir/mt_application_ecmascript.j2.js",
+            "http://localhost:4545/cli/tests/subdir/mt_text_ecmascript.j3.js",
+            "http://localhost:4545/cli/tests/subdir/mt_application_x_javascript.j4.js"
+          ]
+        },
+        "http://localhost:4545/cli/tests/subdir/mt_text_ecmascript.j3.js": {
+          "specifier": "http://localhost:4545/cli/tests/subdir/mt_text_ecmascript.j3.js",
+          "deps": []
+        },
+        "http://localhost:4545/cli/tests/subdir/mt_video_vdn.t2.ts": {
+          "specifier": "http://localhost:4545/cli/tests/subdir/mt_video_vdn.t2.ts",
+          "deps": []
+        },
+        "http://localhost:4545/cli/tests/subdir/mt_application_x_typescript.t4.ts": {
+          "specifier": "http://localhost:4545/cli/tests/subdir/mt_application_x_typescript.t4.ts",
+          "deps": []
+        },
+        "http://localhost:4545/cli/tests/subdir/mt_video_mp2t.t3.ts": {
+          "specifier": "http://localhost:4545/cli/tests/subdir/mt_video_mp2t.t3.ts",
+          "deps": []
+        },
+        "http://localhost:4545/cli/tests/subdir/mt_application_x_javascript.j4.js": {
+          "specifier": "http://localhost:4545/cli/tests/subdir/mt_application_x_javascript.j4.js",
+          "deps": []
+        },
+        "http://localhost:4545/cli/tests/subdir/mt_application_ecmascript.j2.js": {
+          "specifier": "http://localhost:4545/cli/tests/subdir/mt_application_ecmascript.j2.js",
+          "deps": []
+        },
+        "http://localhost:4545/cli/tests/subdir/mt_text_javascript.j1.js": {
+          "specifier": "http://localhost:4545/cli/tests/subdir/mt_text_javascript.j1.js",
+          "deps": []
+        }
+      })
+    );
     drop(http_server_guard);
   }
 
@@ -138,19 +172,32 @@ mod tests {
     let http_server_guard = crate::test_util::http_server();
 
     let global_state = GlobalState::new(Default::default()).unwrap();
-    let module_specifier = rel_module_specifier("tests/circular1.js");
+    let module_specifier = ModuleSpecifier::resolve_url_or_path(
+      "http://localhost:4545/cli/tests/circular1.js",
+    )
+    .unwrap();
 
     let graph_loader =
       ModuleGraphLoader::new(global_state.file_fetcher.clone());
     let graph = graph_loader.build_graph(&module_specifier).await.unwrap();
 
-    // assert_eq!(graph.len(), 9);
-    // let r = graph
-    //   .get("http://localhost:4545/cli/tests/subdir/mt_text_typescript.t1.ts");
-    // assert!(r.is_some());
-
-    println!("{}", serde_json::to_string_pretty(&graph).unwrap());
-
+    assert_eq!(
+      serde_json::to_value(&graph).unwrap(),
+      json!({
+        "http://localhost:4545/cli/tests/circular2.js": {
+          "specifier": "http://localhost:4545/cli/tests/circular2.js",
+          "deps": [
+            "http://localhost:4545/cli/tests/circular1.js"
+          ]
+        },
+        "http://localhost:4545/cli/tests/circular1.js": {
+          "specifier": "http://localhost:4545/cli/tests/circular1.js",
+          "deps": [
+            "http://localhost:4545/cli/tests/circular2.js"
+          ]
+        }
+      })
+    );
     drop(http_server_guard);
   }
 }

From cf840fb690f1222a6bb3afebeacf0725e9ec29eb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= <biwanczuk@gmail.com>
Date: Wed, 6 May 2020 15:42:01 +0200
Subject: [PATCH 12/47] prototype tests

---
 cli/module_graph.rs | 138 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 138 insertions(+)

diff --git a/cli/module_graph.rs b/cli/module_graph.rs
index 0effa13129658b..a418b4aebd020d 100644
--- a/cli/module_graph.rs
+++ b/cli/module_graph.rs
@@ -15,6 +15,11 @@ struct ModuleGraph(HashMap<String, ModuleGraphFile>);
 struct ModuleGraphFile {
   pub specifier: String,
   pub deps: Vec<String>,
+
+  // pub imports: Vec<ImportDescriptor>,
+  // pub referenced_files: Vec<ReferenceDescriptor>,
+  // pub lib_directives: Vec<LibDirective>,
+  // pub types_directives: Vec<TypeDirective>,
 }
 
 struct ModuleGraphLoader {
@@ -200,4 +205,137 @@ mod tests {
     );
     drop(http_server_guard);
   }
+
+  #[tokio::test]
+  async fn source_graph_type_references() {
+    let http_server_guard = crate::test_util::http_server();
+
+    let global_state = GlobalState::new(Default::default()).unwrap();
+    let module_specifier = ModuleSpecifier::resolve_url_or_path(
+      "http://localhost:4545/cli/tests/type_definitions.ts",
+    )
+    .unwrap();
+
+    let graph_loader =
+      ModuleGraphLoader::new(global_state.file_fetcher.clone());
+    let graph = graph_loader.build_graph(&module_specifier).await.unwrap();
+
+    assert_eq!(
+      serde_json::to_value(&graph).unwrap(),
+      json!({
+        "http://localhost:4545/cli/tests/type_definitions.ts": {
+          "imports": [
+            {
+              "specifier": "./type_definitions/foo.js",
+              "resolvedUrl": "http://localhost:4545/cli/tests/type_definitions/foo.js"
+              "typeDirective": "./type_definitions/foo.d.ts",
+              "resolvedTypeDirective": "http://localhost:4545/cli/tests/type_definitions/foo.d.ts"
+            },
+            {
+              "specifier": "./type_definitions/fizz.js",
+              "resolvedUrl": "http://localhost:4545/cli/tests/type_definitions/fizz.js"
+              "typeDirective": "./type_definitions/fizz.d.ts",
+              "resolvedTypeDirective": "http://localhost:4545/cli/tests/type_definitions/fizz.d.ts"
+            },
+            {
+              "specifier": "./type_definitions/qat.js",
+              "resolvedUrl": "http://localhost:4545/cli/tests/type_definitions/qat.js"
+            },
+          ]
+        },
+        "http://localhost:4545/cli/tests/type_definitions/foo.js": {
+          "imports": [],
+        },
+        "http://localhost:4545/cli/tests/type_definitions/foo.d.ts": {
+          "imports": [],
+        },
+        "http://localhost:4545/cli/tests/type_definitions/fizz.js": {
+          "imports": [],
+        },
+        "http://localhost:4545/cli/tests/type_definitions/fizz.d.ts": {
+          "imports": [],
+        },
+        "http://localhost:4545/cli/tests/type_definitions/qat.js": {
+          "imports": [],
+        }
+      })
+    );
+    drop(http_server_guard);
+  }
+
+  #[tokio::test]
+  async fn source_graph_type_references2() {
+    let http_server_guard = crate::test_util::http_server();
+
+    let global_state = GlobalState::new(Default::default()).unwrap();
+    let module_specifier = ModuleSpecifier::resolve_url_or_path(
+      "http://localhost:4545/cli/tests/type_directives_02.ts",
+    )
+    .unwrap();
+
+    let graph_loader =
+      ModuleGraphLoader::new(global_state.file_fetcher.clone());
+    let graph = graph_loader.build_graph(&module_specifier).await.unwrap();
+
+    assert_eq!(
+      serde_json::to_value(&graph).unwrap(),
+      json!({
+        "http://localhost:4545/cli/tests/type_directives_02.ts": {
+          "imports": [
+            {
+              "specifier": "./subdir/type_reference.js",
+              "resolvedUrl": "http://localhost:4545/cli/tests/subdir/type_reference.js"
+            }
+          ]
+        },
+        "http://localhost:4545/cli/tests/subdir/type_reference.js": {
+          "typeReferences": [
+            {
+              "specifier": "./type_reference.d.ts",
+              "resolvedUrl": "http://localhost:4545/cli/tests/subdir/type_reference.d.ts"
+            }
+          ],
+        }
+      })
+    );
+    drop(http_server_guard);
+  }
+
+  #[tokio::test]
+  async fn source_graph_type_references3() {
+    let http_server_guard = crate::test_util::http_server();
+
+    let global_state = GlobalState::new(Default::default()).unwrap();
+    let module_specifier = ModuleSpecifier::resolve_url_or_path(
+      "http://localhost:4545/cli/tests/type_directives_01.ts",
+    )
+    .unwrap();
+
+    let graph_loader =
+      ModuleGraphLoader::new(global_state.file_fetcher.clone());
+    let graph = graph_loader.build_graph(&module_specifier).await.unwrap();
+
+    assert_eq!(
+      serde_json::to_value(&graph).unwrap(),
+      json!({
+        "http://localhost:4545/cli/tests/type_directives_01.ts": {
+          "imports": [
+            {
+              "specifier": "./xTypeScriptTypes.js",
+              "resolvedUrl": "http://localhost:4545/cli/tests/xTypeScriptTypes.js"
+            }
+          ]
+        },
+        "http://localhost:4545/cli/tests/xTypeScriptTypes.js": {
+          "typeHeaders": [
+            {
+              "specifier": "./xTypeScriptTypes.d.ts",
+              "resolvedUrl": "http://localhost:4545/cli/tests/xTypeScriptTypes.d.ts"
+            }
+          ],
+        }
+      })
+    );
+    drop(http_server_guard);
+  }
 }

From 22e879f27dc0c18e59c3109de587a56ee0efc6e1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= <biwanczuk@gmail.com>
Date: Wed, 6 May 2020 19:59:51 +0200
Subject: [PATCH 13/47] parse imports and type directives

---
 cli/module_graph.rs |  18 ++-
 cli/swc_util.rs     | 278 +++++++++++++++++++++++++++++++++++++++++++-
 2 files changed, 289 insertions(+), 7 deletions(-)

diff --git a/cli/module_graph.rs b/cli/module_graph.rs
index a418b4aebd020d..fddfe0353781da 100644
--- a/cli/module_graph.rs
+++ b/cli/module_graph.rs
@@ -11,11 +11,23 @@ use std::collections::HashMap;
 #[derive(Debug, Serialize)]
 struct ModuleGraph(HashMap<String, ModuleGraphFile>);
 
+// #[derive(Debug, Serialize)]
+// #[serde(rename_all = "camelCase")]
+// struct ImportDescriptor {
+//   specifier: String,
+//   resolved_specifier: ModuleSpecifier,
+//   // These two fields are for support of @deno-types directive
+//   // directly prepending import statement
+//   #[serde(skip_serializing_if = "Option::is_none")]
+//   type_directive: Option<String>,
+//   #[serde(skip_serializing_if = "Option::is_none")]
+//   resolved_type_directive: Option<ModuleSpecifier>,
+// }
+
 #[derive(Debug, Serialize)]
 struct ModuleGraphFile {
   pub specifier: String,
   pub deps: Vec<String>,
-
   // pub imports: Vec<ImportDescriptor>,
   // pub referenced_files: Vec<ReferenceDescriptor>,
   // pub lib_directives: Vec<LibDirective>,
@@ -227,13 +239,13 @@ mod tests {
           "imports": [
             {
               "specifier": "./type_definitions/foo.js",
-              "resolvedUrl": "http://localhost:4545/cli/tests/type_definitions/foo.js"
+              "resolvedUrl": "http://localhost:4545/cli/tests/type_definitions/foo.js",
               "typeDirective": "./type_definitions/foo.d.ts",
               "resolvedTypeDirective": "http://localhost:4545/cli/tests/type_definitions/foo.d.ts"
             },
             {
               "specifier": "./type_definitions/fizz.js",
-              "resolvedUrl": "http://localhost:4545/cli/tests/type_definitions/fizz.js"
+              "resolvedUrl": "http://localhost:4545/cli/tests/type_definitions/fizz.js",
               "typeDirective": "./type_definitions/fizz.d.ts",
               "resolvedTypeDirective": "http://localhost:4545/cli/tests/type_definitions/fizz.d.ts"
             },
diff --git a/cli/swc_util.rs b/cli/swc_util.rs
index a2e5d50fdc5fe2..643a9ffd747abd 100644
--- a/cli/swc_util.rs
+++ b/cli/swc_util.rs
@@ -1,11 +1,16 @@
 // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license.
+
+#![allow(unused)]
+
 use crate::swc_common;
+use crate::swc_common::comments::CommentKind;
 use crate::swc_common::comments::Comments;
 use crate::swc_common::errors::Diagnostic;
 use crate::swc_common::errors::DiagnosticBuilder;
 use crate::swc_common::errors::Emitter;
 use crate::swc_common::errors::Handler;
 use crate::swc_common::errors::HandlerFlags;
+use crate::swc_common::BytePos;
 use crate::swc_common::FileName;
 use crate::swc_common::Globals;
 use crate::swc_common::SourceMap;
@@ -158,10 +163,16 @@ impl AstParser {
     &self,
     span: Span,
   ) -> Vec<swc_common::comments::Comment> {
-    self
-      .comments
-      .take_leading_comments(span.lo())
-      .unwrap_or_else(|| vec![])
+    let maybe_comments = self.comments.take_leading_comments(span.lo());
+
+    if let Some(comments) = maybe_comments {
+      // clone the comments and put them back in map
+      let to_return = comments.clone();
+      self.comments.add_leading(span.lo(), comments);
+      to_return
+    } else {
+      vec![]
+    }
   }
 }
 
@@ -317,3 +328,262 @@ const a = await import("./" + "buzz.ts");
     ]
   );
 }
+
+#[derive(Clone, Debug, PartialEq)]
+enum DependencyKind {
+  Import,
+  Export,
+}
+
+#[derive(Clone, Debug, PartialEq)]
+struct DependencyDescriptor {
+  span: Span,
+  specifier: String,
+  kind: DependencyKind,
+}
+
+struct NewDependencyVisitor {
+  dependencies: Vec<DependencyDescriptor>,
+}
+
+impl Visit for NewDependencyVisitor {
+  fn visit_import_decl(
+    &mut self,
+    import_decl: &swc_ecma_ast::ImportDecl,
+    _parent: &dyn Node,
+  ) {
+    let src_str = import_decl.src.value.to_string();
+    self.dependencies.push(DependencyDescriptor {
+      specifier: src_str,
+      kind: DependencyKind::Import,
+      span: import_decl.span,
+    });
+  }
+
+  fn visit_named_export(
+    &mut self,
+    named_export: &swc_ecma_ast::NamedExport,
+    _parent: &dyn Node,
+  ) {
+    if let Some(src) = &named_export.src {
+      let src_str = src.value.to_string();
+      self.dependencies.push(DependencyDescriptor {
+        specifier: src_str,
+        kind: DependencyKind::Export,
+        span: named_export.span,
+      });
+    }
+  }
+
+  fn visit_export_all(
+    &mut self,
+    export_all: &swc_ecma_ast::ExportAll,
+    _parent: &dyn Node,
+  ) {
+    let src_str = export_all.src.value.to_string();
+    self.dependencies.push(DependencyDescriptor {
+      specifier: src_str,
+      kind: DependencyKind::Export,
+      span: export_all.span,
+    });
+  }
+}
+
+fn get_deno_types(parser: &AstParser, span: Span) -> Option<String> {
+  let comments = parser.get_span_comments(span);
+
+  if comments.is_empty() {
+    return None;
+  }
+
+  // @deno-types must directly prepend import statement - hence
+  // checking last comment for span
+  let last = comments.last().unwrap();
+  let comment = last.text.trim_start();
+
+  if comment.starts_with("@deno-types") {
+    let split: Vec<&str> = comment.split("=").collect();
+    assert_eq!(split.len(), 2);
+    let specifier_in_quotes = split.get(1).unwrap().to_string();
+    let specifier = specifier_in_quotes
+      .trim_start_matches("\"")
+      .trim_start_matches("\'")
+      .trim_end_matches("\"")
+      .trim_end_matches("\'")
+      .to_string();
+    return Some(specifier);
+  }
+
+  None
+}
+
+#[derive(Clone, Debug, PartialEq)]
+struct ImportDescriptor {
+  specifier: String,
+  deno_types: Option<String>,
+}
+
+#[derive(Clone, Debug, PartialEq)]
+enum TsReferenceKind {
+  Lib,
+  Types,
+  Path,
+}
+
+#[derive(Clone, Debug, PartialEq)]
+struct TsReferenceDescriptor {
+  kind: TsReferenceKind,
+  specifier: String,
+}
+
+#[allow(unused)]
+fn analyze_dependencies_and_references(
+  source_code: &str,
+  analyze_dynamic_imports: bool,
+) -> Result<
+  (Vec<ImportDescriptor>, Vec<TsReferenceDescriptor>),
+  SwcDiagnosticBuffer,
+> {
+  let parser = AstParser::new();
+  parser.parse_module("root.ts", source_code, |parse_result| {
+    let module = parse_result?;
+    let mut collector = NewDependencyVisitor {
+      dependencies: vec![],
+    };
+    let module_span = module.span;
+    collector.visit_module(&module, &module);
+
+    let dependency_descriptors = collector.dependencies;
+
+    // for each import check if there's relevant @deno-types directive
+    let imports = dependency_descriptors
+      .iter()
+      .map(|mut desc| {
+        if desc.kind == DependencyKind::Import {
+          let deno_types = get_deno_types(&parser, desc.span);
+          ImportDescriptor {
+            specifier: desc.specifier.to_string(),
+            deno_types,
+          }
+        } else {
+          ImportDescriptor {
+            specifier: desc.specifier.to_string(),
+            deno_types: None,
+          }
+        }
+      })
+      .collect();
+
+    // analyze comment from beginning of the file and find TS directives
+    eprintln!("module span {:?}", module_span);
+    let comments = parser
+      .comments
+      .take_leading_comments(module_span.lo())
+      .unwrap_or_else(|| vec![]);
+
+    let mut references = vec![];
+    for comment in comments {
+      if comment.kind != CommentKind::Line {
+        continue;
+      }
+
+      let text = comment.text.to_string();
+      let (kind, specifier_in_quotes) =
+        if text.starts_with("/ <reference path=") {
+          (
+            TsReferenceKind::Path,
+            text.trim_start_matches("/ <reference path="),
+          )
+        } else if text.starts_with("/ <reference lib=") {
+          (
+            TsReferenceKind::Lib,
+            text.trim_start_matches("/ <reference lib="),
+          )
+        } else if text.starts_with("/ <reference types=") {
+          (
+            TsReferenceKind::Types,
+            text.trim_start_matches("/ <reference types="),
+          )
+        } else {
+          continue;
+        };
+      let specifier = specifier_in_quotes
+        .trim_end_matches("/>")
+        .trim_end()
+        .trim_start_matches("\"")
+        .trim_start_matches("\'")
+        .trim_end_matches("\"")
+        .trim_end_matches("\'")
+        .to_string();
+
+      references.push(TsReferenceDescriptor { kind, specifier });
+    }
+    Ok((imports, references))
+  })
+}
+
+#[test]
+fn test_analyze_dependencies_and_directives() {
+  let source = r#"
+// This comment is placed to make sure that directives are parsed
+// even when they start on non-first line
+  
+/// <reference lib="dom" />
+/// <reference types="./type_reference.d.ts" />
+/// <reference path="./type_reference/dep.ts" />
+// @deno-types="./type_definitions/foo.d.ts"
+import { foo } from "./type_definitions/foo.js";
+// @deno-types="./type_definitions/fizz.d.ts"
+import "./type_definitions/fizz.js";
+
+/// <reference path="./type_reference/dep2.ts" />
+
+import * as qat from "./type_definitions/qat.ts";
+
+console.log(foo);
+console.log(fizz);
+console.log(qat.qat);  
+"#;
+
+  let (imports, references) =
+    analyze_dependencies_and_references(source, true).expect("Failed to parse");
+
+  assert_eq!(
+    imports,
+    vec![
+      ImportDescriptor {
+        specifier: "./type_definitions/foo.js".to_string(),
+        deno_types: Some("./type_definitions/foo.d.ts".to_string())
+      },
+      ImportDescriptor {
+        specifier: "./type_definitions/fizz.js".to_string(),
+        deno_types: Some("./type_definitions/fizz.d.ts".to_string())
+      },
+      ImportDescriptor {
+        specifier: "./type_definitions/qat.ts".to_string(),
+        deno_types: None
+      },
+    ]
+  );
+
+  // According to TS docs (https://www.typescriptlang.org/docs/handbook/triple-slash-directives.html)
+  // directives that are not at the top of the file are ignored, so only
+  // 3 references should be captured instead of 4.
+  assert_eq!(
+    references,
+    vec![
+      TsReferenceDescriptor {
+        specifier: "dom".to_string(),
+        kind: TsReferenceKind::Lib,
+      },
+      TsReferenceDescriptor {
+        specifier: "./type_reference.d.ts".to_string(),
+        kind: TsReferenceKind::Types,
+      },
+      TsReferenceDescriptor {
+        specifier: "./type_reference/dep.ts".to_string(),
+        kind: TsReferenceKind::Path,
+      },
+    ]
+  );
+}

From ac3668c4376b94c7dc039049adfc3e09a134caf2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= <biwanczuk@gmail.com>
Date: Wed, 6 May 2020 22:04:49 +0200
Subject: [PATCH 14/47] type directives analysis

---
 cli/file_fetcher.rs |   7 +
 cli/lib.rs          |   2 +
 cli/module_graph.rs | 313 +++++++++++++++++++++++++++++++++++---------
 cli/swc_util.rs     |  17 +--
 cli/tsc.rs          |   3 +
 5 files changed, 275 insertions(+), 67 deletions(-)

diff --git a/cli/file_fetcher.rs b/cli/file_fetcher.rs
index 2e75517e632523..cbfb340ddc6121 100644
--- a/cli/file_fetcher.rs
+++ b/cli/file_fetcher.rs
@@ -34,6 +34,7 @@ pub struct SourceFile {
   pub url: Url,
   pub filename: PathBuf,
   pub types_url: Option<Url>,
+  pub types_header: Option<String>,
   pub media_type: msg::MediaType,
   pub source_code: Vec<u8>,
 }
@@ -323,6 +324,7 @@ impl SourceFileFetcher {
       media_type,
       source_code,
       types_url,
+      types_header: None,
     })
   }
 
@@ -380,6 +382,7 @@ impl SourceFileFetcher {
       &fake_filepath,
       headers.get("content-type").map(|e| e.as_str()),
     );
+    let types_header = headers.get("x-typescript-types").map(|e| e.to_string());
     let types_url = match media_type {
       msg::MediaType::JavaScript | msg::MediaType::JSX => get_types_url(
         &module_url,
@@ -394,6 +397,7 @@ impl SourceFileFetcher {
       media_type,
       source_code,
       types_url,
+      types_header,
     }))
   }
 
@@ -502,6 +506,8 @@ impl SourceFileFetcher {
             headers.get("content-type").map(String::as_str),
           );
 
+          let types_header =
+            headers.get("x-typescript-types").map(String::to_string);
           let types_url = match media_type {
             msg::MediaType::JavaScript | msg::MediaType::JSX => get_types_url(
               &module_url,
@@ -517,6 +523,7 @@ impl SourceFileFetcher {
             media_type,
             source_code: source,
             types_url,
+            types_header,
           };
 
           Ok(source_file)
diff --git a/cli/lib.rs b/cli/lib.rs
index 402d54199cc67f..bea5d4d84a05f8 100644
--- a/cli/lib.rs
+++ b/cli/lib.rs
@@ -356,6 +356,7 @@ async fn eval_command(
     filename: main_module_url.to_file_path().unwrap(),
     url: main_module_url,
     types_url: None,
+    types_header: None,
     media_type: if as_typescript {
       MediaType::TypeScript
     } else {
@@ -532,6 +533,7 @@ async fn test_command(
     filename: test_file_url.to_file_path().unwrap(),
     url: test_file_url,
     types_url: None,
+    types_header: None,
     media_type: MediaType::TypeScript,
     source_code: test_file.clone().into_bytes(),
   };
diff --git a/cli/module_graph.rs b/cli/module_graph.rs
index fddfe0353781da..3c63829a190ab3 100644
--- a/cli/module_graph.rs
+++ b/cli/module_graph.rs
@@ -2,36 +2,72 @@
 #![allow(unused)]
 
 use crate::file_fetcher::SourceFileFetcher;
-use crate::swc_util::analyze_dependencies;
+use crate::msg::MediaType;
+use crate::swc_util::analyze_dependencies_and_references;
+use crate::swc_util::TsReferenceKind;
 use deno_core::ErrBox;
 use deno_core::ModuleSpecifier;
 use serde::Serialize;
+use serde::Serializer;
 use std::collections::HashMap;
 
+fn serialize_module_specifier<S>(
+  spec: &ModuleSpecifier,
+  s: S,
+) -> Result<S::Ok, S::Error>
+where
+  S: Serializer,
+{
+  s.serialize_str(&spec.to_string())
+}
+
+fn serialize_option_module_specifier<S>(
+  maybe_spec: &Option<ModuleSpecifier>,
+  s: S,
+) -> Result<S::Ok, S::Error>
+where
+  S: Serializer,
+{
+  if let Some(spec) = maybe_spec {
+    serialize_module_specifier(spec, s)
+  } else {
+    s.serialize_none()
+  }
+}
+
 #[derive(Debug, Serialize)]
 struct ModuleGraph(HashMap<String, ModuleGraphFile>);
 
-// #[derive(Debug, Serialize)]
-// #[serde(rename_all = "camelCase")]
-// struct ImportDescriptor {
-//   specifier: String,
-//   resolved_specifier: ModuleSpecifier,
-//   // These two fields are for support of @deno-types directive
-//   // directly prepending import statement
-//   #[serde(skip_serializing_if = "Option::is_none")]
-//   type_directive: Option<String>,
-//   #[serde(skip_serializing_if = "Option::is_none")]
-//   resolved_type_directive: Option<ModuleSpecifier>,
-// }
+#[derive(Debug, Serialize)]
+#[serde(rename_all = "camelCase")]
+struct ImportDescriptor {
+  specifier: String,
+  #[serde(serialize_with = "serialize_module_specifier")]
+  resolved_specifier: ModuleSpecifier,
+  // These two fields are for support of @deno-types directive
+  // directly prepending import statement
+  type_directive: Option<String>,
+  #[serde(serialize_with = "serialize_option_module_specifier")]
+  resolved_type_directive: Option<ModuleSpecifier>,
+}
 
 #[derive(Debug, Serialize)]
+#[serde(rename_all = "camelCase")]
+struct ReferenceDescriptor {
+  specifier: String,
+  #[serde(serialize_with = "serialize_module_specifier")]
+  resolved_specifier: ModuleSpecifier,
+}
+
+#[derive(Debug, Serialize)]
+#[serde(rename_all = "camelCase")]
 struct ModuleGraphFile {
   pub specifier: String,
-  pub deps: Vec<String>,
-  // pub imports: Vec<ImportDescriptor>,
-  // pub referenced_files: Vec<ReferenceDescriptor>,
-  // pub lib_directives: Vec<LibDirective>,
-  // pub types_directives: Vec<TypeDirective>,
+  pub imports: Vec<ImportDescriptor>,
+  pub referenced_files: Vec<ReferenceDescriptor>,
+  pub lib_directives: Vec<ReferenceDescriptor>,
+  pub types_directives: Vec<ReferenceDescriptor>,
+  pub type_headers: Vec<ReferenceDescriptor>,
 }
 
 struct ModuleGraphLoader {
@@ -56,48 +92,139 @@ impl ModuleGraphLoader {
     self.to_visit.push(specifier.to_owned());
     while let Some(spec) = self.to_visit.pop() {
       self.visit_module(&spec).await?;
-      let file = self.graph.0.get(&spec.to_string()).unwrap();
-      for dep in &file.deps {
-        if let Some(_exists) = self.graph.0.get(dep) {
-          continue;
-        } else {
-          self
-            .to_visit
-            .push(ModuleSpecifier::resolve_url_or_path(dep).unwrap());
-        }
-      }
     }
     Ok(self.graph.0)
   }
 
   async fn visit_module(
     &mut self,
-    specifier: &ModuleSpecifier,
+    module_specifier: &ModuleSpecifier,
   ) -> Result<(), ErrBox> {
-    if self.graph.0.contains_key(&specifier.to_string()) {
+    if self.graph.0.contains_key(&module_specifier.to_string()) {
       return Ok(());
     }
 
-    let source_file =
-      self.file_fetcher.fetch_source_file(specifier, None).await?;
+    let source_file = self
+      .file_fetcher
+      .fetch_source_file(module_specifier, None)
+      .await?;
+
+    let mut imports = vec![];
+    let mut referenced_files = vec![];
+    let mut lib_directives = vec![];
+    let mut types_directives = vec![];
+    let mut type_headers = vec![];
+
+    if source_file.media_type == MediaType::JavaScript
+      || source_file.media_type == MediaType::TypeScript
+    {
+      if let Some(types_specifier) = source_file.types_header {
+        let type_header = ReferenceDescriptor {
+          specifier: types_specifier.to_string(),
+          resolved_specifier: ModuleSpecifier::resolve_import(
+            &types_specifier,
+            &module_specifier.to_string(),
+          )?,
+        };
+        type_headers.push(type_header);
+      }
+
+      let (import_descs, ref_descs) = analyze_dependencies_and_references(
+        &String::from_utf8(source_file.source_code)?,
+        true,
+      )?;
+
+      // TODO(bartlomieju): apply import map, using State
+      //    or should it be passed explicitly
+      for import_desc in import_descs {
+        let resolved_specifier = ModuleSpecifier::resolve_import(
+          &import_desc.specifier,
+          &module_specifier.to_string(),
+        )?;
+
+        let resolved_type_directive =
+          if let Some(types_specifier) = import_desc.deno_types.as_ref() {
+            Some(ModuleSpecifier::resolve_import(
+              &types_specifier,
+              &module_specifier.to_string(),
+            )?)
+          } else {
+            None
+          };
+
+        let import_descriptor = ImportDescriptor {
+          specifier: import_desc.specifier.to_string(),
+          resolved_specifier,
+          type_directive: import_desc.deno_types,
+          resolved_type_directive,
+        };
+
+        if self
+          .graph
+          .0
+          .get(&import_descriptor.resolved_specifier.to_string())
+          .is_none()
+        {
+          self
+            .to_visit
+            .push(import_descriptor.resolved_specifier.clone());
+        }
 
-    let raw_deps =
-      analyze_dependencies(&String::from_utf8(source_file.source_code)?, true)?;
+        if let Some(type_dir_url) =
+          import_descriptor.resolved_type_directive.as_ref()
+        {
+          if self.graph.0.get(&type_dir_url.to_string()).is_none() {
+            self.to_visit.push(type_dir_url.clone());
+          }
+        }
 
-    // TODO(bartlomieju): apply import map, using State
-    //    or should it be passed explicitly
-    let mut deps = vec![];
-    for raw_dep in raw_deps {
-      let specifier =
-        ModuleSpecifier::resolve_import(&raw_dep, &specifier.to_string())?;
-      deps.push(specifier.to_string());
+        imports.push(import_descriptor);
+      }
+
+      for ref_desc in ref_descs {
+        let resolved_specifier = ModuleSpecifier::resolve_import(
+          &ref_desc.specifier,
+          &module_specifier.to_string(),
+        )?;
+        let reference_descriptor = ReferenceDescriptor {
+          specifier: ref_desc.specifier.to_string(),
+          resolved_specifier,
+        };
+
+        if self
+          .graph
+          .0
+          .get(&reference_descriptor.resolved_specifier.to_string())
+          .is_none()
+        {
+          self
+            .to_visit
+            .push(reference_descriptor.resolved_specifier.clone());
+        }
+
+        match ref_desc.kind {
+          TsReferenceKind::Lib => {
+            lib_directives.push(reference_descriptor);
+          }
+          TsReferenceKind::Types => {
+            types_directives.push(reference_descriptor);
+          }
+          TsReferenceKind::Path => {
+            referenced_files.push(reference_descriptor);
+          }
+        }
+      }
     }
 
     self.graph.0.insert(
-      specifier.to_string(),
+      module_specifier.to_string(),
       ModuleGraphFile {
-        specifier: specifier.to_string(),
-        deps,
+        specifier: module_specifier.to_string(),
+        imports,
+        referenced_files,
+        lib_directives,
+        types_directives,
+        type_headers,
       },
     );
     Ok(())
@@ -232,43 +359,77 @@ mod tests {
       ModuleGraphLoader::new(global_state.file_fetcher.clone());
     let graph = graph_loader.build_graph(&module_specifier).await.unwrap();
 
+    eprintln!("json {:#?}", serde_json::to_value(&graph).unwrap());
+
     assert_eq!(
       serde_json::to_value(&graph).unwrap(),
       json!({
         "http://localhost:4545/cli/tests/type_definitions.ts": {
+          "specifier": "http://localhost:4545/cli/tests/type_definitions.ts",
           "imports": [
             {
               "specifier": "./type_definitions/foo.js",
-              "resolvedUrl": "http://localhost:4545/cli/tests/type_definitions/foo.js",
+              "resolvedSpecifier": "http://localhost:4545/cli/tests/type_definitions/foo.js",
               "typeDirective": "./type_definitions/foo.d.ts",
               "resolvedTypeDirective": "http://localhost:4545/cli/tests/type_definitions/foo.d.ts"
             },
             {
               "specifier": "./type_definitions/fizz.js",
-              "resolvedUrl": "http://localhost:4545/cli/tests/type_definitions/fizz.js",
+              "resolvedSpecifier": "http://localhost:4545/cli/tests/type_definitions/fizz.js",
               "typeDirective": "./type_definitions/fizz.d.ts",
               "resolvedTypeDirective": "http://localhost:4545/cli/tests/type_definitions/fizz.d.ts"
             },
             {
-              "specifier": "./type_definitions/qat.js",
-              "resolvedUrl": "http://localhost:4545/cli/tests/type_definitions/qat.js"
+              "specifier": "./type_definitions/qat.ts",
+              "resolvedSpecifier": "http://localhost:4545/cli/tests/type_definitions/qat.ts",
+              "typeDirective": null,
+              "resolvedTypeDirective": null,
             },
-          ]
+          ],
+          "typesDirectives": [],
+          "referencedFiles": [],
+          "libDirectives": [],
+          "typeHeaders": [],
         },
         "http://localhost:4545/cli/tests/type_definitions/foo.js": {
+          "specifier": "http://localhost:4545/cli/tests/type_definitions/foo.js",
           "imports": [],
+          "referencedFiles": [],
+          "libDirectives": [],
+          "typesDirectives": [],
+          "typeHeaders": [],
         },
         "http://localhost:4545/cli/tests/type_definitions/foo.d.ts": {
+          "specifier": "http://localhost:4545/cli/tests/type_definitions/foo.d.ts",
           "imports": [],
+          "referencedFiles": [],
+          "libDirectives": [],
+          "typesDirectives": [],
+          "typeHeaders": [],
         },
         "http://localhost:4545/cli/tests/type_definitions/fizz.js": {
+          "specifier": "http://localhost:4545/cli/tests/type_definitions/fizz.js",
           "imports": [],
+          "referencedFiles": [],
+          "libDirectives": [],
+          "typesDirectives": [],
+          "typeHeaders": [],
         },
         "http://localhost:4545/cli/tests/type_definitions/fizz.d.ts": {
+          "specifier": "http://localhost:4545/cli/tests/type_definitions/fizz.d.ts",
           "imports": [],
+          "referencedFiles": [],
+          "libDirectives": [],
+          "typesDirectives": [],
+          "typeHeaders": [],
         },
-        "http://localhost:4545/cli/tests/type_definitions/qat.js": {
+        "http://localhost:4545/cli/tests/type_definitions/qat.ts": {
+          "specifier": "http://localhost:4545/cli/tests/type_definitions/qat.ts",
           "imports": [],
+          "referencedFiles": [],
+          "libDirectives": [],
+          "typesDirectives": [],
+          "typeHeaders": [],
         }
       })
     );
@@ -289,24 +450,46 @@ mod tests {
       ModuleGraphLoader::new(global_state.file_fetcher.clone());
     let graph = graph_loader.build_graph(&module_specifier).await.unwrap();
 
+    eprintln!("{:#?}", serde_json::to_value(&graph).unwrap());
+
     assert_eq!(
       serde_json::to_value(&graph).unwrap(),
       json!({
         "http://localhost:4545/cli/tests/type_directives_02.ts": {
+          "specifier": "http://localhost:4545/cli/tests/type_directives_02.ts",
           "imports": [
             {
               "specifier": "./subdir/type_reference.js",
-              "resolvedUrl": "http://localhost:4545/cli/tests/subdir/type_reference.js"
+              "resolvedSpecifier": "http://localhost:4545/cli/tests/subdir/type_reference.js",
+              "typeDirective": null,
+              "resolvedTypeDirective": null,
             }
-          ]
+          ],
+          "typesDirectives": [],
+          "referencedFiles": [],
+          "libDirectives": [],
+          "typeHeaders": [],
+        },
+        "http://localhost:4545/cli/tests/subdir/type_reference.d.ts": {
+          "specifier": "http://localhost:4545/cli/tests/subdir/type_reference.d.ts",
+          "imports": [],
+          "referencedFiles": [],
+          "libDirectives": [],
+          "typesDirectives": [],
+          "typeHeaders": [],
         },
         "http://localhost:4545/cli/tests/subdir/type_reference.js": {
-          "typeReferences": [
+          "specifier": "http://localhost:4545/cli/tests/subdir/type_reference.js",
+          "imports": [],
+          "referencedFiles": [],
+          "libDirectives": [],
+          "typesDirectives": [
             {
               "specifier": "./type_reference.d.ts",
-              "resolvedUrl": "http://localhost:4545/cli/tests/subdir/type_reference.d.ts"
+              "resolvedSpecifier": "http://localhost:4545/cli/tests/subdir/type_reference.d.ts",
             }
           ],
+          "typeHeaders": [],
         }
       })
     );
@@ -331,20 +514,32 @@ mod tests {
       serde_json::to_value(&graph).unwrap(),
       json!({
         "http://localhost:4545/cli/tests/type_directives_01.ts": {
+          "specifier": "http://localhost:4545/cli/tests/type_directives_01.ts",
           "imports": [
             {
-              "specifier": "./xTypeScriptTypes.js",
-              "resolvedUrl": "http://localhost:4545/cli/tests/xTypeScriptTypes.js"
+              "specifier": "http://127.0.0.1:4545/xTypeScriptTypes.js",
+              "resolvedSpecifier": "http://127.0.0.1:4545/xTypeScriptTypes.js",
+              "typeDirective": null,
+              "resolvedTypeDirective": null,
             }
-          ]
+          ],
+          "referencedFiles": [],
+          "libDirectives": [],
+          "typesDirectives": [],
+          "typeHeaders": [],
         },
-        "http://localhost:4545/cli/tests/xTypeScriptTypes.js": {
+        "http://127.0.0.1:4545/xTypeScriptTypes.js": {
+          "specifier": "http://127.0.0.1:4545/xTypeScriptTypes.js",
           "typeHeaders": [
             {
               "specifier": "./xTypeScriptTypes.d.ts",
-              "resolvedUrl": "http://localhost:4545/cli/tests/xTypeScriptTypes.d.ts"
+              "resolvedSpecifier": "http://127.0.0.1:4545/xTypeScriptTypes.d.ts"
             }
           ],
+          "imports": [],
+          "referencedFiles": [],
+          "libDirectives": [],
+          "typesDirectives": [],
         }
       })
     );
diff --git a/cli/swc_util.rs b/cli/swc_util.rs
index 643a9ffd747abd..fb177614837db9 100644
--- a/cli/swc_util.rs
+++ b/cli/swc_util.rs
@@ -418,26 +418,26 @@ fn get_deno_types(parser: &AstParser, span: Span) -> Option<String> {
 }
 
 #[derive(Clone, Debug, PartialEq)]
-struct ImportDescriptor {
-  specifier: String,
-  deno_types: Option<String>,
+pub struct ImportDescriptor {
+  pub specifier: String,
+  pub deno_types: Option<String>,
 }
 
 #[derive(Clone, Debug, PartialEq)]
-enum TsReferenceKind {
+pub enum TsReferenceKind {
   Lib,
   Types,
   Path,
 }
 
 #[derive(Clone, Debug, PartialEq)]
-struct TsReferenceDescriptor {
-  kind: TsReferenceKind,
-  specifier: String,
+pub struct TsReferenceDescriptor {
+  pub kind: TsReferenceKind,
+  pub specifier: String,
 }
 
 #[allow(unused)]
-fn analyze_dependencies_and_references(
+pub fn analyze_dependencies_and_references(
   source_code: &str,
   analyze_dynamic_imports: bool,
 ) -> Result<
@@ -487,6 +487,7 @@ fn analyze_dependencies_and_references(
         continue;
       }
 
+      // TODO(bartlomieju): you can do better than that...
       let text = comment.text.to_string();
       let (kind, specifier_in_quotes) =
         if text.starts_with("/ <reference path=") {
diff --git a/cli/tsc.rs b/cli/tsc.rs
index f207558a5c6f1b..e69361b10093bd 100644
--- a/cli/tsc.rs
+++ b/cli/tsc.rs
@@ -593,6 +593,7 @@ impl TsCompiler {
       media_type: msg::MediaType::JavaScript,
       source_code: compiled_code,
       types_url: None,
+      types_header: None,
     };
 
     Ok(compiled_module)
@@ -668,6 +669,7 @@ impl TsCompiler {
       media_type: msg::MediaType::JavaScript,
       source_code,
       types_url: None,
+      types_header: None,
     };
 
     Ok(source_map_file)
@@ -876,6 +878,7 @@ mod tests {
       media_type: msg::MediaType::TypeScript,
       source_code: include_bytes!("./tests/002_hello.ts").to_vec(),
       types_url: None,
+      types_header: None,
     };
     let mock_state =
       GlobalState::mock(vec![String::from("deno"), String::from("hello.js")]);

From d6aff9389b6950a5777acb70f449b22d5a247405 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= <biwanczuk@gmail.com>
Date: Thu, 7 May 2020 15:43:45 +0200
Subject: [PATCH 15/47] lint

---
 cli/swc_util.rs | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/cli/swc_util.rs b/cli/swc_util.rs
index fb177614837db9..d602c935bb8563 100644
--- a/cli/swc_util.rs
+++ b/cli/swc_util.rs
@@ -402,14 +402,15 @@ fn get_deno_types(parser: &AstParser, span: Span) -> Option<String> {
   let comment = last.text.trim_start();
 
   if comment.starts_with("@deno-types") {
-    let split: Vec<&str> = comment.split("=").collect();
+    let split: Vec<String> =
+      comment.split('=').map(|s| s.to_string()).collect();
     assert_eq!(split.len(), 2);
     let specifier_in_quotes = split.get(1).unwrap().to_string();
     let specifier = specifier_in_quotes
-      .trim_start_matches("\"")
-      .trim_start_matches("\'")
-      .trim_end_matches("\"")
-      .trim_end_matches("\'")
+      .trim_start_matches('\"')
+      .trim_start_matches('\'')
+      .trim_end_matches('\"')
+      .trim_end_matches('\'')
       .to_string();
     return Some(specifier);
   }
@@ -511,10 +512,10 @@ pub fn analyze_dependencies_and_references(
       let specifier = specifier_in_quotes
         .trim_end_matches("/>")
         .trim_end()
-        .trim_start_matches("\"")
-        .trim_start_matches("\'")
-        .trim_end_matches("\"")
-        .trim_end_matches("\'")
+        .trim_start_matches('\"')
+        .trim_start_matches('\'')
+        .trim_end_matches('\"')
+        .trim_end_matches('\'')
         .to_string();
 
       references.push(TsReferenceDescriptor { kind, specifier });

From 319b94ab322397e8efaf5bfe30a61b86b32981ff Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= <biwanczuk@gmail.com>
Date: Thu, 7 May 2020 16:43:13 +0200
Subject: [PATCH 16/47] basic compilation working

---
 cli/global_state.rs |   4 +-
 cli/js/compiler.ts  | 200 ++++++++++++++++++++++++++++++++++++++++++++
 cli/lib.rs          |   2 +-
 cli/module_graph.rs |  22 +++--
 cli/msg.rs          |   4 +-
 cli/tsc.rs          | 184 +++++++++++++++++++++++++++++++++++++++-
 6 files changed, 399 insertions(+), 17 deletions(-)

diff --git a/cli/global_state.rs b/cli/global_state.rs
index 4e9bdbb99c9411..e8125af731b4b0 100644
--- a/cli/global_state.rs
+++ b/cli/global_state.rs
@@ -115,14 +115,14 @@ impl GlobalState {
       | msg::MediaType::JSX => {
         state1
           .ts_compiler
-          .compile(state1.clone(), &out, target_lib, permissions.clone())
+          .new_compile(state1.clone(), &out, target_lib)
           .await
       }
       msg::MediaType::JavaScript => {
         if state1.ts_compiler.compile_js {
           state2
             .ts_compiler
-            .compile(state1.clone(), &out, target_lib, permissions.clone())
+            .new_compile(state1.clone(), &out, target_lib)
             .await
         } else {
           if let Some(types_url) = out.types_url.clone() {
diff --git a/cli/js/compiler.ts b/cli/js/compiler.ts
index 146529edd1445c..8036be469c52f0 100644
--- a/cli/js/compiler.ts
+++ b/cli/js/compiler.ts
@@ -756,6 +756,59 @@ async function processImports(
   return resolvedSources;
 }
 
+function buildSourceFileCache(sourceFileMap: Record<string, SourceFileMapEntry>): void {
+  for (const entry of Object.values(sourceFileMap)) {
+    assert(entry.sourceCode.length > 0);
+    SourceFile.addToCache({
+      url: entry.url,
+      filename: entry.url,
+      mediaType: entry.mediaType,
+      sourceCode: entry.sourceCode,
+    });
+
+    for (const importDesc of entry.imports) {
+      let mappedUrl = importDesc.resolvedSpecifier;
+      const importedFile = sourceFileMap[importDesc.resolvedSpecifier];
+      assert(importedFile);
+      const isJsOrJsx =
+        importedFile.mediaType === MediaType.JavaScript ||
+        importedFile.mediaType === MediaType.JSX;
+      // If JS or JSX perform substitution for types if available
+      if (isJsOrJsx) {
+        if (importedFile.typeHeaders.length > 0) {
+          const typeHeaders = importedFile.typeHeaders[0];
+          mappedUrl = typeHeaders.resolvedSpecifier;
+        } else if (importDesc.resolvedTypeDirective) {
+          mappedUrl = importDesc.resolvedTypeDirective;
+        } else if (importedFile.typesDirectives.length > 0) {
+          const typeDirective = importedFile.typesDirectives[0];
+          mappedUrl = typeDirective.resolvedSpecifier;
+        }
+      }
+
+      SourceFile.cacheResolvedUrl(
+        mappedUrl,
+        importDesc.specifier,
+        entry.url,
+      );
+    }
+    for (const fileRef of entry.referencedFiles) {
+      SourceFile.cacheResolvedUrl(
+        fileRef.resolvedSpecifier,
+        fileRef.specifier,
+        entry.url,
+      );
+    }
+    for (const fileRef of entry.libDirectives) {
+      SourceFile.cacheResolvedUrl(
+        fileRef.resolvedSpecifier,
+        fileRef.specifier,
+        entry.url,
+      );
+    }
+  }
+}
+
 interface FileReference {
   fileName: string;
   pos: number;
@@ -855,6 +908,7 @@ enum CompilerRequestType {
   Compile = 0,
   RuntimeCompile = 1,
   RuntimeTranspile = 2,
+  CompileNew = 3,
 }
 
 // TODO(bartlomieju): probably could be defined inline?
@@ -1222,6 +1276,43 @@ interface CompilerRequestCompile {
   cwd: string;
 }
 
+interface ImportDescriptor {
+  specifier: string;
+  resolvedSpecifier: string;
+  typeDirective?: string;
+  resolvedTypeDirective?: string;
+}
+
+interface ReferenceDescriptor {
+  specifier: string;
+  resolvedSpecifier: string;
+}
+
+interface SourceFileMapEntry {
+  // fully resolved URL
+  url: string;
+  sourceCode: string;
+  mediaType: MediaType;
+  imports: ImportDescriptor[],
+  referencedFiles: ReferenceDescriptor[],
+  libDirectives: ReferenceDescriptor[],
+  typesDirectives: ReferenceDescriptor[],
+  typeHeaders: ReferenceDescriptor[],
+}
+
+interface CompilerRequestCompileNew {
+  type: CompilerRequestType.CompileNew;
+  target: CompilerHostTarget;
+  rootNames: string[];
+  configPath?: string;
+  config?: string;
+  unstable: boolean;
+  bundle: boolean;
+  cwd: string;
+  // key value is fully resolved URL
+  sourceFileMap: Record<string, SourceFileMapEntry>;
+}
+
 interface CompilerRequestRuntimeCompile {
   type: CompilerRequestType.RuntimeCompile;
   target: CompilerHostTarget;
@@ -1239,6 +1330,7 @@ interface CompilerRequestRuntimeTranspile {
 }
 
 type CompilerRequest =
+  | CompilerRequestCompileNew
   | CompilerRequestCompile
   | CompilerRequestRuntimeCompile
   | CompilerRequestRuntimeTranspile;
@@ -1259,6 +1351,109 @@ interface RuntimeBundleResult {
   diagnostics: DiagnosticItem[];
 }
 
+async function compileNew(
+  request: CompilerRequestCompileNew
+): Promise<CompileResult> {
+  const {
+    bundle,
+    config,
+    configPath,
+    rootNames,
+    target,
+    unstable,
+    cwd,
+    sourceFileMap,
+  } = request;
+  util.log(">>> compile start", {
+    rootNames,
+    type: CompilerRequestType[request.type],
+  });
+
+  // When a programme is emitted, TypeScript will call `writeFile` with
+  // each file that needs to be emitted.  The Deno compiler host delegates
+  // this, to make it easier to perform the right actions, which vary
+  // based a lot on the request.
+  const state: WriteFileState = {
+    type: request.type,
+    emitMap: {},
+    bundle,
+    host: undefined,
+    rootNames,
+  };
+  let writeFile: WriteFileCallback;
+  if (bundle) {
+    writeFile = createBundleWriteFile(state);
+  } else {
+    writeFile = createCompileWriteFile(state);
+  }
+  const host = (state.host = new Host({
+    bundle,
+    target,
+    writeFile,
+    unstable,
+  }));
+  let diagnostics: readonly ts.Diagnostic[] = [];
+
+  // if there is a configuration supplied, we need to parse that
+  if (config && config.length && configPath) {
+    const configResult = host.configure(cwd, configPath, config);
+    diagnostics = processConfigureResponse(configResult, configPath) || [];
+  }
+
+  buildSourceFileCache(sourceFileMap);
+  // if there was a configuration and no diagnostics with it, we will continue
+  // to generate the program and possibly emit it.
+  if (diagnostics.length === 0) {
+    const options = host.getCompilationSettings();
+    const program = ts.createProgram({
+      rootNames,
+      options,
+      host,
+      oldProgram: TS_SNAPSHOT_PROGRAM,
+    });
+
+    diagnostics = ts
+      .getPreEmitDiagnostics(program)
+      .filter(({ code }) => !ignoredDiagnostics.includes(code));
+
+    // We will only proceed with the emit if there are no diagnostics.
+    if (diagnostics && diagnostics.length === 0) {
+      if (bundle) {
+        // we only support a single root module when bundling
+        assert(rootNames.length === 1);
+        setRootExports(program, rootNames[0]);
+      }
+      const emitResult = program.emit();
+      assert(emitResult.emitSkipped === false, "Unexpected skip of the emit.");
+      // emitResult.diagnostics is `readonly` in TS3.5+ and can't be assigned
+      // without casting.
+      diagnostics = emitResult.diagnostics;
+    }
+  }
+
+  let bundleOutput = undefined;
+
+  if (bundle) {
+    assert(state.bundleOutput);
+    bundleOutput = state.bundleOutput;
+  }
+
+  assert(state.emitMap);
+  const result: CompileResult = {
+    emitMap: state.emitMap,
+    bundleOutput,
+    diagnostics: fromTypeScriptDiagnostic(diagnostics),
+  };
+
+  util.log("<<< compile end", {
+    rootNames,
+    type: CompilerRequestType[request.type],
+  });
+
+  return result;
+}
+
+
 async function compile(
   request: CompilerRequestCompile
 ): Promise<CompileResult> {
@@ -1549,6 +1744,11 @@ async function tsCompilerOnMessage({
       globalThis.postMessage(result);
       break;
     }
+    case CompilerRequestType.CompileNew: {
+      const result = await compileNew(request as CompilerRequestCompileNew);
+      globalThis.postMessage(result);
+      break;
+    }
     case CompilerRequestType.RuntimeCompile: {
       const result = await runtimeCompile(
         request as CompilerRequestRuntimeCompile
diff --git a/cli/lib.rs b/cli/lib.rs
index bea5d4d84a05f8..db62e530a4f221 100644
--- a/cli/lib.rs
+++ b/cli/lib.rs
@@ -390,7 +390,7 @@ async fn bundle_command(
   debug!(">>>>> bundle START");
   let bundle_result = global_state
     .ts_compiler
-    .bundle(global_state.clone(), module_name.to_string(), out_file)
+    .new_bundle(global_state.clone(), module_name, out_file)
     .await;
   debug!(">>>>> bundle END");
   bundle_result
diff --git a/cli/module_graph.rs b/cli/module_graph.rs
index 3c63829a190ab3..6005374a59055f 100644
--- a/cli/module_graph.rs
+++ b/cli/module_graph.rs
@@ -36,11 +36,11 @@ where
 }
 
 #[derive(Debug, Serialize)]
-struct ModuleGraph(HashMap<String, ModuleGraphFile>);
+pub struct ModuleGraph(HashMap<String, ModuleGraphFile>);
 
 #[derive(Debug, Serialize)]
 #[serde(rename_all = "camelCase")]
-struct ImportDescriptor {
+pub struct ImportDescriptor {
   specifier: String,
   #[serde(serialize_with = "serialize_module_specifier")]
   resolved_specifier: ModuleSpecifier,
@@ -53,7 +53,7 @@ struct ImportDescriptor {
 
 #[derive(Debug, Serialize)]
 #[serde(rename_all = "camelCase")]
-struct ReferenceDescriptor {
+pub struct ReferenceDescriptor {
   specifier: String,
   #[serde(serialize_with = "serialize_module_specifier")]
   resolved_specifier: ModuleSpecifier,
@@ -61,16 +61,18 @@ struct ReferenceDescriptor {
 
 #[derive(Debug, Serialize)]
 #[serde(rename_all = "camelCase")]
-struct ModuleGraphFile {
+pub struct ModuleGraphFile {
   pub specifier: String,
   pub imports: Vec<ImportDescriptor>,
   pub referenced_files: Vec<ReferenceDescriptor>,
   pub lib_directives: Vec<ReferenceDescriptor>,
   pub types_directives: Vec<ReferenceDescriptor>,
   pub type_headers: Vec<ReferenceDescriptor>,
+  pub media_type: MediaType,
+  pub source_code: String,
 }
 
-struct ModuleGraphLoader {
+pub struct ModuleGraphLoader {
   file_fetcher: SourceFileFetcher,
   to_visit: Vec<ModuleSpecifier>,
   pub graph: ModuleGraph,
@@ -115,6 +117,8 @@ impl ModuleGraphLoader {
     let mut types_directives = vec![];
     let mut type_headers = vec![];
 
+    let source_code = String::from_utf8(source_file.source_code)?;
+
     if source_file.media_type == MediaType::JavaScript
       || source_file.media_type == MediaType::TypeScript
     {
@@ -129,10 +133,8 @@ impl ModuleGraphLoader {
         type_headers.push(type_header);
       }
 
-      let (import_descs, ref_descs) = analyze_dependencies_and_references(
-        &String::from_utf8(source_file.source_code)?,
-        true,
-      )?;
+      let (import_descs, ref_descs) =
+        analyze_dependencies_and_references(&source_code, true)?;
 
       // TODO(bartlomieju): apply import map, using State
       //    or should it be passed explicitly
@@ -220,6 +222,8 @@ impl ModuleGraphLoader {
       module_specifier.to_string(),
       ModuleGraphFile {
         specifier: module_specifier.to_string(),
+        media_type: source_file.media_type,
+        source_code,
         imports,
         referenced_files,
         lib_directives,
diff --git a/cli/msg.rs b/cli/msg.rs
index 146f45ea5485ec..186fde42c644e8 100644
--- a/cli/msg.rs
+++ b/cli/msg.rs
@@ -2,9 +2,11 @@
 
 // Warning! The values in this enum are duplicated in js/compiler.ts
 // Update carefully!
+use serde::Serialize;
+
 #[allow(non_camel_case_types)]
 #[repr(i8)]
-#[derive(Clone, Copy, PartialEq, Debug)]
+#[derive(Clone, Copy, PartialEq, Debug, Serialize)]
 pub enum MediaType {
   JavaScript = 0,
   JSX = 1,
diff --git a/cli/tsc.rs b/cli/tsc.rs
index e69361b10093bd..102abe0edeae43 100644
--- a/cli/tsc.rs
+++ b/cli/tsc.rs
@@ -8,6 +8,7 @@ use crate::file_fetcher::SourceFileFetcher;
 use crate::fmt;
 use crate::fs as deno_fs;
 use crate::global_state::GlobalState;
+use crate::module_graph::ModuleGraphLoader;
 use crate::msg;
 use crate::op_error::OpError;
 use crate::ops;
@@ -370,6 +371,83 @@ impl TsCompiler {
     worker
   }
 
+  pub async fn new_bundle(
+    &self,
+    global_state: GlobalState,
+    module_specifier: ModuleSpecifier,
+    out_file: Option<PathBuf>,
+  ) -> Result<(), ErrBox> {
+    debug!(
+      "Invoking the compiler to bundle. module_name: {}",
+      module_specifier.to_string()
+    );
+    eprintln!("Bundling {}", module_specifier.to_string());
+
+    let module_graph_loader =
+      ModuleGraphLoader::new(global_state.file_fetcher.clone());
+    let module_graph =
+      module_graph_loader.build_graph(&module_specifier).await?;
+    let module_graph_json =
+      serde_json::to_value(module_graph).expect("Failed to serialize data");
+
+    let root_names = vec![module_specifier.to_string()];
+    let bundle = true;
+    let target = "main";
+    let unstable = global_state.flags.unstable;
+    let compiler_config = self.config.clone();
+    let cwd = std::env::current_dir().unwrap();
+    let j = match (compiler_config.path, compiler_config.content) {
+      (Some(config_path), Some(config_data)) => json!({
+        "type": msg::CompilerRequestType::Compile as i32,
+        "target": target,
+        "rootNames": root_names,
+        "bundle": bundle,
+        "unstable": unstable,
+        "configPath": config_path,
+        "config": str::from_utf8(&config_data).unwrap(),
+        "cwd": cwd,
+        "sourceFileMap": module_graph_json,
+      }),
+      _ => json!({
+        "type": msg::CompilerRequestType::Compile as i32,
+        "target": target,
+        "rootNames": root_names,
+        "bundle": bundle,
+        "unstable": unstable,
+        "cwd": cwd,
+        "sourceFileMap": module_graph_json,
+      }),
+    };
+
+    let req_msg = j.to_string().into_boxed_str().into_boxed_bytes();
+
+    let msg = execute_in_thread(global_state.clone(), req_msg).await?;
+    let json_str = std::str::from_utf8(&msg).unwrap();
+    debug!("Message: {}", json_str);
+
+    let bundle_response: BundleResponse = serde_json::from_str(json_str)?;
+
+    if !bundle_response.diagnostics.items.is_empty() {
+      return Err(ErrBox::from(bundle_response.diagnostics));
+    }
+
+    if let Some(out_file_) = out_file.as_ref() {
+      eprintln!("Emitting bundle to {:?}", out_file_);
+
+      let output_bytes = bundle_response.bundle_output.as_bytes();
+      let output_len = output_bytes.len();
+
+      deno_fs::write_file(out_file_, output_bytes, 0o666)?;
+      // TODO(bartlomieju): add "humanFileSize" method
+      eprintln!("{} bytes emmited.", output_len);
+    } else {
+      println!("{}", bundle_response.bundle_output);
+    }
+
+    Ok(())
+  }
+
+  #[allow(unused)]
   pub async fn bundle(
     &self,
     global_state: GlobalState,
@@ -537,6 +615,105 @@ impl TsCompiler {
     None
   }
 
+  #[allow(unused)]
+  pub async fn new_compile(
+    &self,
+    global_state: GlobalState,
+    source_file: &SourceFile,
+    target: TargetLib,
+  ) -> Result<CompiledModule, ErrBox> {
+    if self.has_compiled(&source_file.url) {
+      return self.get_compiled_module(&source_file.url);
+    }
+
+    if self.use_disk_cache {
+      // Try to load cached version:
+      // 1. check if there's 'meta' file
+      if let Some(metadata) = self.get_metadata(&source_file.url) {
+        // 2. compare version hashes
+        // TODO: it would probably be good idea to make it method implemented on SourceFile
+        let version_hash_to_validate = source_code_version_hash(
+          &source_file.source_code,
+          version::DENO,
+          &self.config.hash,
+        );
+
+        if metadata.version_hash == version_hash_to_validate {
+          debug!("load_cache metadata version hash match");
+          if let Ok(compiled_module) =
+            self.get_compiled_module(&source_file.url)
+          {
+            self.mark_compiled(&source_file.url);
+            return Ok(compiled_module);
+          }
+        }
+      }
+    }
+    let source_file_ = source_file.clone();
+    let module_url = source_file.url.clone();
+    let module_specifier = ModuleSpecifier::from(source_file.url.clone());
+    let module_graph_loader =
+      ModuleGraphLoader::new(global_state.file_fetcher.clone());
+    let module_graph =
+      module_graph_loader.build_graph(&module_specifier).await?;
+    let module_graph_json =
+      serde_json::to_value(module_graph).expect("Failed to serialize data");
+
+    let target = match target {
+      TargetLib::Main => "main",
+      TargetLib::Worker => "worker",
+    };
+    let root_names = vec![module_url.to_string()];
+    let bundle = false;
+    let unstable = global_state.flags.unstable;
+    let compiler_config = self.config.clone();
+    let cwd = std::env::current_dir().unwrap();
+    let j = match (compiler_config.path, compiler_config.content) {
+      (Some(config_path), Some(config_data)) => json!({
+        "type": msg::CompilerRequestType::Compile as i32,
+        "target": target,
+        "rootNames": root_names,
+        "bundle": bundle,
+        "unstable": unstable,
+        "configPath": config_path,
+        "config": str::from_utf8(&config_data).unwrap(),
+        "cwd": cwd,
+        "sourceFileMap": module_graph_json,
+      }),
+      _ => json!({
+        "type": msg::CompilerRequestType::Compile as i32,
+        "target": target,
+        "rootNames": root_names,
+        "bundle": bundle,
+        "unstable": unstable,
+        "cwd": cwd,
+        "sourceFileMap": module_graph_json,
+      }),
+    };
+
+    let req_msg = j.to_string().into_boxed_str().into_boxed_bytes();
+
+    let ts_compiler = self.clone();
+
+    info!(
+      "{} {}",
+      colors::green("Compile".to_string()),
+      module_url.to_string()
+    );
+
+    let msg = execute_in_thread(global_state.clone(), req_msg).await?;
+    let json_str = std::str::from_utf8(&msg).unwrap();
+
+    let compile_response: CompileResponse = serde_json::from_str(json_str)?;
+
+    if !compile_response.diagnostics.items.is_empty() {
+      return Err(ErrBox::from(compile_response.diagnostics));
+    }
+
+    self.cache_emitted_files(compile_response.emit_map)?;
+    ts_compiler.get_compiled_module(&source_file_.url)
+  }
+
   fn cache_emitted_files(
     &self,
     emit_map: HashMap<String, EmittedSource>,
@@ -906,9 +1083,8 @@ mod tests {
       .unwrap()
       .join("cli/tests/002_hello.ts");
     use deno_core::ModuleSpecifier;
-    let module_name = ModuleSpecifier::resolve_url_or_path(p.to_str().unwrap())
-      .unwrap()
-      .to_string();
+    let module_name =
+      ModuleSpecifier::resolve_url_or_path(p.to_str().unwrap()).unwrap();
 
     let state = GlobalState::mock(vec![
       String::from("deno"),
@@ -918,7 +1094,7 @@ mod tests {
 
     let result = state
       .ts_compiler
-      .bundle(state.clone(), module_name, None)
+      .new_bundle(state.clone(), module_name, None)
       .await;
     assert!(result.is_ok());
   }

From db5405acac7dbf6f68f411f4d1ac18e072b2de35 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= <biwanczuk@gmail.com>
Date: Thu, 7 May 2020 21:18:18 +0200
Subject: [PATCH 17/47] fmt

---
 cli/js/compiler.ts | 29 +++++++++++++----------------
 1 file changed, 13 insertions(+), 16 deletions(-)

diff --git a/cli/js/compiler.ts b/cli/js/compiler.ts
index 8036be469c52f0..ed1fa15814fa6a 100644
--- a/cli/js/compiler.ts
+++ b/cli/js/compiler.ts
@@ -756,7 +756,9 @@ async function processImports(
   return resolvedSources;
 }
 
-function buildSourceFileCache(sourceFileMap: Record<string, SourceFileMapEntry>): void {
+function buildSourceFileCache(
+  sourceFileMap: Record<string, SourceFileMapEntry>
+): void {
   for (const entry of Object.values(sourceFileMap)) {
     assert(entry.sourceCode.length > 0);
     SourceFile.addToCache({
@@ -786,24 +788,20 @@ function buildSourceFileCache(sourceFileMap: Record<string, SourceFileMapEntry>)
         }
       }
 
-      SourceFile.cacheResolvedUrl(
-        mappedUrl,
-        importDesc.specifier,
-        entry.url,
-      );
+      SourceFile.cacheResolvedUrl(mappedUrl, importDesc.specifier, entry.url);
     }
     for (const fileRef of entry.referencedFiles) {
       SourceFile.cacheResolvedUrl(
         fileRef.resolvedSpecifier,
         fileRef.specifier,
-        entry.url,
+        entry.url
       );
     }
     for (const fileRef of entry.libDirectives) {
       SourceFile.cacheResolvedUrl(
         fileRef.resolvedSpecifier,
         fileRef.specifier,
-        entry.url,
+        entry.url
       );
     }
   }
@@ -1293,11 +1291,11 @@ interface SourceFileMapEntry {
   url: string;
   sourceCode: string;
   mediaType: MediaType;
-  imports: ImportDescriptor[],
-  referencedFiles: ReferenceDescriptor[],
-  libDirectives: ReferenceDescriptor[],
-  typesDirectives: ReferenceDescriptor[],
-  typeHeaders: ReferenceDescriptor[],
+  imports: ImportDescriptor[];
+  referencedFiles: ReferenceDescriptor[];
+  libDirectives: ReferenceDescriptor[];
+  typesDirectives: ReferenceDescriptor[];
+  typeHeaders: ReferenceDescriptor[];
 }
 
 interface CompilerRequestCompileNew {
@@ -1351,7 +1349,7 @@ interface RuntimeBundleResult {
   diagnostics: DiagnosticItem[];
 }
 
-async function compileNew(
+function compileNew(
   request: CompilerRequestCompileNew
 ): Promise<CompileResult> {
   const {
@@ -1453,7 +1451,6 @@ async function compileNew(
   return result;
 }
 
-
 async function compile(
   request: CompilerRequestCompile
 ): Promise<CompileResult> {
@@ -1745,7 +1742,7 @@ async function tsCompilerOnMessage({
       break;
     }
     case CompilerRequestType.CompileNew: {
-      const result = await compileNew(request as CompilerRequestCompileNew);
+      const result = compileNew(request as CompilerRequestCompileNew);
       globalThis.postMessage(result);
       break;
     }

From 9bbd7a5b0caf38edb0a828bf8a3e59641ed3424b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= <biwanczuk@gmail.com>
Date: Thu, 7 May 2020 22:35:05 +0200
Subject: [PATCH 18/47] fix bundle tests

---
 cli/js/compiler.ts  |  4 +---
 cli/module_graph.rs | 45 +++++++++++++++++++++++++++++++++------------
 cli/state.rs        |  2 +-
 cli/swc_util.rs     |  1 -
 cli/tsc.rs          | 32 ++++++++++++++++++++++++++++----
 5 files changed, 63 insertions(+), 21 deletions(-)

diff --git a/cli/js/compiler.ts b/cli/js/compiler.ts
index ed1fa15814fa6a..d959a1e9a0d9e1 100644
--- a/cli/js/compiler.ts
+++ b/cli/js/compiler.ts
@@ -1349,9 +1349,7 @@ interface RuntimeBundleResult {
   diagnostics: DiagnosticItem[];
 }
 
-function compileNew(
-  request: CompilerRequestCompileNew
-): Promise<CompileResult> {
+function compileNew(request: CompilerRequestCompileNew): CompileResult {
   const {
     bundle,
     config,
diff --git a/cli/module_graph.rs b/cli/module_graph.rs
index 6005374a59055f..862a5fd22b3e2e 100644
--- a/cli/module_graph.rs
+++ b/cli/module_graph.rs
@@ -2,6 +2,7 @@
 #![allow(unused)]
 
 use crate::file_fetcher::SourceFileFetcher;
+use crate::import_map::ImportMap;
 use crate::msg::MediaType;
 use crate::swc_util::analyze_dependencies_and_references;
 use crate::swc_util::TsReferenceKind;
@@ -74,14 +75,19 @@ pub struct ModuleGraphFile {
 
 pub struct ModuleGraphLoader {
   file_fetcher: SourceFileFetcher,
+  maybe_import_map: Option<ImportMap>,
   to_visit: Vec<ModuleSpecifier>,
   pub graph: ModuleGraph,
 }
 
 impl ModuleGraphLoader {
-  pub fn new(file_fetcher: SourceFileFetcher) -> Self {
+  pub fn new(
+    file_fetcher: SourceFileFetcher,
+    maybe_import_map: Option<ImportMap>,
+  ) -> Self {
     Self {
       file_fetcher,
+      maybe_import_map,
       to_visit: vec![],
       graph: ModuleGraph(HashMap::new()),
     }
@@ -136,13 +142,23 @@ impl ModuleGraphLoader {
       let (import_descs, ref_descs) =
         analyze_dependencies_and_references(&source_code, true)?;
 
-      // TODO(bartlomieju): apply import map, using State
-      //    or should it be passed explicitly
       for import_desc in import_descs {
-        let resolved_specifier = ModuleSpecifier::resolve_import(
-          &import_desc.specifier,
-          &module_specifier.to_string(),
-        )?;
+        let maybe_resolved =
+          if let Some(import_map) = self.maybe_import_map.as_ref() {
+            import_map
+              .resolve(&import_desc.specifier, &module_specifier.to_string())?
+          } else {
+            None
+          };
+
+        let resolved_specifier = if let Some(resolved) = maybe_resolved {
+          resolved
+        } else {
+          ModuleSpecifier::resolve_import(
+            &import_desc.specifier,
+            &module_specifier.to_string(),
+          )?
+        };
 
         let resolved_type_directive =
           if let Some(types_specifier) = import_desc.deno_types.as_ref() {
@@ -249,6 +265,7 @@ mod tests {
   //   ModuleSpecifier::resolve_url_or_path(ps).unwrap()
   // }
 
+  #[ignore]
   #[tokio::test]
   async fn source_graph_fetch() {
     let http_server_guard = crate::test_util::http_server();
@@ -259,7 +276,7 @@ mod tests {
     )
     .unwrap();
     let graph_loader =
-      ModuleGraphLoader::new(global_state.file_fetcher.clone());
+      ModuleGraphLoader::new(global_state.file_fetcher.clone(), None);
     let graph = graph_loader.build_graph(&module_specifier).await.unwrap();
 
     assert_eq!(
@@ -315,6 +332,7 @@ mod tests {
     drop(http_server_guard);
   }
 
+  #[ignore]
   #[tokio::test]
   async fn source_graph_fetch_circular() {
     let http_server_guard = crate::test_util::http_server();
@@ -326,7 +344,7 @@ mod tests {
     .unwrap();
 
     let graph_loader =
-      ModuleGraphLoader::new(global_state.file_fetcher.clone());
+      ModuleGraphLoader::new(global_state.file_fetcher.clone(), None);
     let graph = graph_loader.build_graph(&module_specifier).await.unwrap();
 
     assert_eq!(
@@ -349,6 +367,7 @@ mod tests {
     drop(http_server_guard);
   }
 
+  #[ignore]
   #[tokio::test]
   async fn source_graph_type_references() {
     let http_server_guard = crate::test_util::http_server();
@@ -360,7 +379,7 @@ mod tests {
     .unwrap();
 
     let graph_loader =
-      ModuleGraphLoader::new(global_state.file_fetcher.clone());
+      ModuleGraphLoader::new(global_state.file_fetcher.clone(), None);
     let graph = graph_loader.build_graph(&module_specifier).await.unwrap();
 
     eprintln!("json {:#?}", serde_json::to_value(&graph).unwrap());
@@ -440,6 +459,7 @@ mod tests {
     drop(http_server_guard);
   }
 
+  #[ignore]
   #[tokio::test]
   async fn source_graph_type_references2() {
     let http_server_guard = crate::test_util::http_server();
@@ -451,7 +471,7 @@ mod tests {
     .unwrap();
 
     let graph_loader =
-      ModuleGraphLoader::new(global_state.file_fetcher.clone());
+      ModuleGraphLoader::new(global_state.file_fetcher.clone(), None);
     let graph = graph_loader.build_graph(&module_specifier).await.unwrap();
 
     eprintln!("{:#?}", serde_json::to_value(&graph).unwrap());
@@ -500,6 +520,7 @@ mod tests {
     drop(http_server_guard);
   }
 
+  #[ignore]
   #[tokio::test]
   async fn source_graph_type_references3() {
     let http_server_guard = crate::test_util::http_server();
@@ -511,7 +532,7 @@ mod tests {
     .unwrap();
 
     let graph_loader =
-      ModuleGraphLoader::new(global_state.file_fetcher.clone());
+      ModuleGraphLoader::new(global_state.file_fetcher.clone(), None);
     let graph = graph_loader.build_graph(&module_specifier).await.unwrap();
 
     assert_eq!(
diff --git a/cli/state.rs b/cli/state.rs
index b36331085c0de0..53fa1b418ad653 100644
--- a/cli/state.rs
+++ b/cli/state.rs
@@ -248,7 +248,7 @@ impl State {
   }
 }
 
-fn exit_unstable(api_name: &str) {
+pub fn exit_unstable(api_name: &str) {
   eprintln!(
     "Unstable API '{}'. The --unstable flag must be provided.",
     api_name
diff --git a/cli/swc_util.rs b/cli/swc_util.rs
index d602c935bb8563..91c155bf6b7f98 100644
--- a/cli/swc_util.rs
+++ b/cli/swc_util.rs
@@ -476,7 +476,6 @@ pub fn analyze_dependencies_and_references(
       .collect();
 
     // analyze comment from beginning of the file and find TS directives
-    eprintln!("module span {:?}", module_span);
     let comments = parser
       .comments
       .take_leading_comments(module_span.lo())
diff --git a/cli/tsc.rs b/cli/tsc.rs
index 102abe0edeae43..383259b33964f9 100644
--- a/cli/tsc.rs
+++ b/cli/tsc.rs
@@ -8,6 +8,7 @@ use crate::file_fetcher::SourceFileFetcher;
 use crate::fmt;
 use crate::fs as deno_fs;
 use crate::global_state::GlobalState;
+use crate::import_map::ImportMap;
 use crate::module_graph::ModuleGraphLoader;
 use crate::msg;
 use crate::op_error::OpError;
@@ -15,6 +16,7 @@ use crate::ops;
 use crate::permissions::Permissions;
 use crate::source_maps::SourceMapGetter;
 use crate::startup_data;
+use crate::state::exit_unstable;
 use crate::state::State;
 use crate::state::*;
 use crate::tokio_util;
@@ -383,8 +385,18 @@ impl TsCompiler {
     );
     eprintln!("Bundling {}", module_specifier.to_string());
 
+    let import_map: Option<ImportMap> =
+      match global_state.flags.import_map_path.as_ref() {
+        None => None,
+        Some(file_path) => {
+          if !global_state.flags.unstable {
+            exit_unstable("--importmap")
+          }
+          Some(ImportMap::load(file_path)?)
+        }
+      };
     let module_graph_loader =
-      ModuleGraphLoader::new(global_state.file_fetcher.clone());
+      ModuleGraphLoader::new(global_state.file_fetcher.clone(), import_map);
     let module_graph =
       module_graph_loader.build_graph(&module_specifier).await?;
     let module_graph_json =
@@ -431,17 +443,19 @@ impl TsCompiler {
       return Err(ErrBox::from(bundle_response.diagnostics));
     }
 
+    let output_string = fmt::format_text(&bundle_response.bundle_output)?;
+
     if let Some(out_file_) = out_file.as_ref() {
       eprintln!("Emitting bundle to {:?}", out_file_);
 
-      let output_bytes = bundle_response.bundle_output.as_bytes();
+      let output_bytes = output_string.as_bytes();
       let output_len = output_bytes.len();
 
       deno_fs::write_file(out_file_, output_bytes, 0o666)?;
       // TODO(bartlomieju): add "humanFileSize" method
       eprintln!("{} bytes emmited.", output_len);
     } else {
-      println!("{}", bundle_response.bundle_output);
+      println!("{}", output_string);
     }
 
     Ok(())
@@ -652,8 +666,18 @@ impl TsCompiler {
     let source_file_ = source_file.clone();
     let module_url = source_file.url.clone();
     let module_specifier = ModuleSpecifier::from(source_file.url.clone());
+    let import_map: Option<ImportMap> =
+      match global_state.flags.import_map_path.as_ref() {
+        None => None,
+        Some(file_path) => {
+          if !global_state.flags.unstable {
+            exit_unstable("--importmap")
+          }
+          Some(ImportMap::load(file_path)?)
+        }
+      };
     let module_graph_loader =
-      ModuleGraphLoader::new(global_state.file_fetcher.clone());
+      ModuleGraphLoader::new(global_state.file_fetcher.clone(), import_map);
     let module_graph =
       module_graph_loader.build_graph(&module_specifier).await?;
     let module_graph_json =

From cfdef1e3553f0627d53c2632be019d4639f671e1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= <biwanczuk@gmail.com>
Date: Mon, 11 May 2020 02:23:32 +0200
Subject: [PATCH 19/47] fixes

---
 cli/global_state.rs |  4 ++--
 cli/module_graph.rs | 16 ++++++++++------
 cli/tsc.rs          | 22 ++++++++++++++++------
 3 files changed, 28 insertions(+), 14 deletions(-)

diff --git a/cli/global_state.rs b/cli/global_state.rs
index e8125af731b4b0..7be983d79dc220 100644
--- a/cli/global_state.rs
+++ b/cli/global_state.rs
@@ -115,14 +115,14 @@ impl GlobalState {
       | msg::MediaType::JSX => {
         state1
           .ts_compiler
-          .new_compile(state1.clone(), &out, target_lib)
+          .new_compile(state1.clone(), &out, target_lib, permissions)
           .await
       }
       msg::MediaType::JavaScript => {
         if state1.ts_compiler.compile_js {
           state2
             .ts_compiler
-            .new_compile(state1.clone(), &out, target_lib)
+            .new_compile(state1.clone(), &out, target_lib, permissions)
             .await
         } else {
           if let Some(types_url) = out.types_url.clone() {
diff --git a/cli/module_graph.rs b/cli/module_graph.rs
index 862a5fd22b3e2e..107777b60a5231 100644
--- a/cli/module_graph.rs
+++ b/cli/module_graph.rs
@@ -4,6 +4,7 @@
 use crate::file_fetcher::SourceFileFetcher;
 use crate::import_map::ImportMap;
 use crate::msg::MediaType;
+use crate::permissions::Permissions;
 use crate::swc_util::analyze_dependencies_and_references;
 use crate::swc_util::TsReferenceKind;
 use deno_core::ErrBox;
@@ -74,6 +75,7 @@ pub struct ModuleGraphFile {
 }
 
 pub struct ModuleGraphLoader {
+  permissions: Permissions,
   file_fetcher: SourceFileFetcher,
   maybe_import_map: Option<ImportMap>,
   to_visit: Vec<ModuleSpecifier>,
@@ -84,9 +86,11 @@ impl ModuleGraphLoader {
   pub fn new(
     file_fetcher: SourceFileFetcher,
     maybe_import_map: Option<ImportMap>,
+    permissions: Permissions,
   ) -> Self {
     Self {
       file_fetcher,
+      permissions,
       maybe_import_map,
       to_visit: vec![],
       graph: ModuleGraph(HashMap::new()),
@@ -114,7 +118,7 @@ impl ModuleGraphLoader {
 
     let source_file = self
       .file_fetcher
-      .fetch_source_file(module_specifier, None)
+      .fetch_source_file(module_specifier, None, self.permissions.clone())
       .await?;
 
     let mut imports = vec![];
@@ -276,7 +280,7 @@ mod tests {
     )
     .unwrap();
     let graph_loader =
-      ModuleGraphLoader::new(global_state.file_fetcher.clone(), None);
+      ModuleGraphLoader::new(global_state.file_fetcher.clone(), None, Permissions::allow_all());
     let graph = graph_loader.build_graph(&module_specifier).await.unwrap();
 
     assert_eq!(
@@ -344,7 +348,7 @@ mod tests {
     .unwrap();
 
     let graph_loader =
-      ModuleGraphLoader::new(global_state.file_fetcher.clone(), None);
+      ModuleGraphLoader::new(global_state.file_fetcher.clone(), None, Permissions::allow_all());
     let graph = graph_loader.build_graph(&module_specifier).await.unwrap();
 
     assert_eq!(
@@ -379,7 +383,7 @@ mod tests {
     .unwrap();
 
     let graph_loader =
-      ModuleGraphLoader::new(global_state.file_fetcher.clone(), None);
+      ModuleGraphLoader::new(global_state.file_fetcher.clone(), None, Permissions::allow_all());
     let graph = graph_loader.build_graph(&module_specifier).await.unwrap();
 
     eprintln!("json {:#?}", serde_json::to_value(&graph).unwrap());
@@ -471,7 +475,7 @@ mod tests {
     .unwrap();
 
     let graph_loader =
-      ModuleGraphLoader::new(global_state.file_fetcher.clone(), None);
+      ModuleGraphLoader::new(global_state.file_fetcher.clone(), None, Permissions::allow_all());
     let graph = graph_loader.build_graph(&module_specifier).await.unwrap();
 
     eprintln!("{:#?}", serde_json::to_value(&graph).unwrap());
@@ -532,7 +536,7 @@ mod tests {
     .unwrap();
 
     let graph_loader =
-      ModuleGraphLoader::new(global_state.file_fetcher.clone(), None);
+      ModuleGraphLoader::new(global_state.file_fetcher.clone(), None, Permissions::allow_all());
     let graph = graph_loader.build_graph(&module_specifier).await.unwrap();
 
     assert_eq!(
diff --git a/cli/tsc.rs b/cli/tsc.rs
index 383259b33964f9..c7a2bcfba7e617 100644
--- a/cli/tsc.rs
+++ b/cli/tsc.rs
@@ -395,8 +395,12 @@ impl TsCompiler {
           Some(ImportMap::load(file_path)?)
         }
       };
-    let module_graph_loader =
-      ModuleGraphLoader::new(global_state.file_fetcher.clone(), import_map);
+    let permissions = Permissions::allow_all();
+    let module_graph_loader = ModuleGraphLoader::new(
+      global_state.file_fetcher.clone(),
+      import_map,
+      permissions.clone(),
+    );
     let module_graph =
       module_graph_loader.build_graph(&module_specifier).await?;
     let module_graph_json =
@@ -433,7 +437,8 @@ impl TsCompiler {
 
     let req_msg = j.to_string().into_boxed_str().into_boxed_bytes();
 
-    let msg = execute_in_thread(global_state.clone(), req_msg).await?;
+    let msg =
+      execute_in_thread(global_state.clone(), permissions, req_msg).await?;
     let json_str = std::str::from_utf8(&msg).unwrap();
     debug!("Message: {}", json_str);
 
@@ -635,6 +640,7 @@ impl TsCompiler {
     global_state: GlobalState,
     source_file: &SourceFile,
     target: TargetLib,
+    permissions: Permissions,
   ) -> Result<CompiledModule, ErrBox> {
     if self.has_compiled(&source_file.url) {
       return self.get_compiled_module(&source_file.url);
@@ -676,8 +682,11 @@ impl TsCompiler {
           Some(ImportMap::load(file_path)?)
         }
       };
-    let module_graph_loader =
-      ModuleGraphLoader::new(global_state.file_fetcher.clone(), import_map);
+    let module_graph_loader = ModuleGraphLoader::new(
+      global_state.file_fetcher.clone(),
+      import_map,
+      permissions.clone(),
+    );
     let module_graph =
       module_graph_loader.build_graph(&module_specifier).await?;
     let module_graph_json =
@@ -725,7 +734,8 @@ impl TsCompiler {
       module_url.to_string()
     );
 
-    let msg = execute_in_thread(global_state.clone(), req_msg).await?;
+    let msg =
+      execute_in_thread(global_state.clone(), permissions, req_msg).await?;
     let json_str = std::str::from_utf8(&msg).unwrap();
 
     let compile_response: CompileResponse = serde_json::from_str(json_str)?;

From 7827a56e7d60c19686ff6c32ac4c4f81f17fa5b1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= <biwanczuk@gmail.com>
Date: Mon, 11 May 2020 02:29:10 +0200
Subject: [PATCH 20/47] fix integration tests

---
 cli/tests/error_011_bad_module_specifier.ts.out | 10 +---------
 cli/tests/error_type_definitions.ts.out         | 11 +----------
 2 files changed, 2 insertions(+), 19 deletions(-)

diff --git a/cli/tests/error_011_bad_module_specifier.ts.out b/cli/tests/error_011_bad_module_specifier.ts.out
index 39726d5c633406..e6f9b2321b0eed 100644
--- a/cli/tests/error_011_bad_module_specifier.ts.out
+++ b/cli/tests/error_011_bad_module_specifier.ts.out
@@ -1,9 +1 @@
-[WILDCARD]error: Uncaught URIError: relative import path "bad-module.ts" not prefixed with / or ./ or ../ Imported from "[WILDCARD]/error_011_bad_module_specifier.ts"
-    at unwrapResponse ($deno$/ops/dispatch_json.ts:[WILDCARD])
-    at Object.sendSync ($deno$/ops/dispatch_json.ts:[WILDCARD])
-    at resolveModules ($deno$/compiler.ts:[WILDCARD])
-    at processImports ($deno$/compiler.ts:[WILDCARD])
-    at processImports ($deno$/compiler.ts:[WILDCARD])
-    at async compile ($deno$/compiler.ts:[WILDCARD])
-    at async tsCompilerOnMessage ($deno$/compiler.ts:[WILDCARD])
-    at async workerMessageRecvCallback ([WILDCARD]runtime_worker.ts:[WILDCARD])
+[WILDCARD]error: relative import path "bad-module.ts" not prefixed with / or ./ or ../ Imported from "[WILDCARD]/error_011_bad_module_specifier.ts"
diff --git a/cli/tests/error_type_definitions.ts.out b/cli/tests/error_type_definitions.ts.out
index bacca4f80e9061..32c3c9b5253b02 100644
--- a/cli/tests/error_type_definitions.ts.out
+++ b/cli/tests/error_type_definitions.ts.out
@@ -1,10 +1 @@
-[WILDCARD]error: Uncaught URIError: relative import path "baz" not prefixed with / or ./ or ../ Imported from "[WILDCARD]/type_definitions/bar.d.ts"
-    at unwrapResponse ($deno$/ops/dispatch_json.ts:[WILDCARD])
-    at Object.sendSync ($deno$/ops/dispatch_json.ts:[WILDCARD])
-    at resolveModules ($deno$/compiler.ts:[WILDCARD])
-    at processImports ($deno$/compiler.ts:[WILDCARD])
-    at processImports ($deno$/compiler.ts:[WILDCARD])
-    at async processImports ($deno$/compiler.ts:[WILDCARD])
-    at async compile ($deno$/compiler.ts:[WILDCARD])
-    at async tsCompilerOnMessage ($deno$/compiler.ts:[WILDCARD])
-    at async workerMessageRecvCallback ([WILDCARD]runtime_worker.ts:[WILDCARD])
+[WILDCARD]error: relative import path "baz" not prefixed with / or ./ or ../ Imported from "[WILDCARD]/type_definitions/bar.d.ts"

From ae1d69321cb84148c0140b391c9a253977c44146 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= <biwanczuk@gmail.com>
Date: Mon, 11 May 2020 02:29:55 +0200
Subject: [PATCH 21/47] fmt

---
 cli/module_graph.rs | 35 +++++++++++++++++++++++++----------
 1 file changed, 25 insertions(+), 10 deletions(-)

diff --git a/cli/module_graph.rs b/cli/module_graph.rs
index 107777b60a5231..15a0889af69c44 100644
--- a/cli/module_graph.rs
+++ b/cli/module_graph.rs
@@ -279,8 +279,11 @@ mod tests {
       "http://localhost:4545/cli/tests/019_media_types.ts",
     )
     .unwrap();
-    let graph_loader =
-      ModuleGraphLoader::new(global_state.file_fetcher.clone(), None, Permissions::allow_all());
+    let graph_loader = ModuleGraphLoader::new(
+      global_state.file_fetcher.clone(),
+      None,
+      Permissions::allow_all(),
+    );
     let graph = graph_loader.build_graph(&module_specifier).await.unwrap();
 
     assert_eq!(
@@ -347,8 +350,11 @@ mod tests {
     )
     .unwrap();
 
-    let graph_loader =
-      ModuleGraphLoader::new(global_state.file_fetcher.clone(), None, Permissions::allow_all());
+    let graph_loader = ModuleGraphLoader::new(
+      global_state.file_fetcher.clone(),
+      None,
+      Permissions::allow_all(),
+    );
     let graph = graph_loader.build_graph(&module_specifier).await.unwrap();
 
     assert_eq!(
@@ -382,8 +388,11 @@ mod tests {
     )
     .unwrap();
 
-    let graph_loader =
-      ModuleGraphLoader::new(global_state.file_fetcher.clone(), None, Permissions::allow_all());
+    let graph_loader = ModuleGraphLoader::new(
+      global_state.file_fetcher.clone(),
+      None,
+      Permissions::allow_all(),
+    );
     let graph = graph_loader.build_graph(&module_specifier).await.unwrap();
 
     eprintln!("json {:#?}", serde_json::to_value(&graph).unwrap());
@@ -474,8 +483,11 @@ mod tests {
     )
     .unwrap();
 
-    let graph_loader =
-      ModuleGraphLoader::new(global_state.file_fetcher.clone(), None, Permissions::allow_all());
+    let graph_loader = ModuleGraphLoader::new(
+      global_state.file_fetcher.clone(),
+      None,
+      Permissions::allow_all(),
+    );
     let graph = graph_loader.build_graph(&module_specifier).await.unwrap();
 
     eprintln!("{:#?}", serde_json::to_value(&graph).unwrap());
@@ -535,8 +547,11 @@ mod tests {
     )
     .unwrap();
 
-    let graph_loader =
-      ModuleGraphLoader::new(global_state.file_fetcher.clone(), None, Permissions::allow_all());
+    let graph_loader = ModuleGraphLoader::new(
+      global_state.file_fetcher.clone(),
+      None,
+      Permissions::allow_all(),
+    );
     let graph = graph_loader.build_graph(&module_specifier).await.unwrap();
 
     assert_eq!(

From ccf641307d65933a0eb0587817084d4a5460688f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= <biwanczuk@gmail.com>
Date: Mon, 11 May 2020 03:04:31 +0200
Subject: [PATCH 22/47] tweak tests

---
 cli/global_state.rs                           | 17 ++++-
 cli/lib.rs                                    |  1 +
 cli/module_graph.rs                           | 62 +++++++++++++++----
 cli/state.rs                                  |  1 +
 cli/tests/037_fetch_multiple.out              |  4 +-
 cli/tests/error_004_missing_module.ts.out     |  9 +--
 cli/tests/error_006_import_ext_failure.ts.out |  9 +--
 ...ror_local_static_import_from_remote.ts.out |  9 +--
 cli/tsc.rs                                    |  3 +
 9 files changed, 75 insertions(+), 40 deletions(-)

diff --git a/cli/global_state.rs b/cli/global_state.rs
index 7be983d79dc220..e4ecf65d5df6d2 100644
--- a/cli/global_state.rs
+++ b/cli/global_state.rs
@@ -95,6 +95,7 @@ impl GlobalState {
     maybe_referrer: Option<ModuleSpecifier>,
     target_lib: TargetLib,
     permissions: Permissions,
+    is_dyn_import: bool,
   ) -> Result<CompiledModule, ErrBox> {
     let state1 = self.clone();
     let state2 = self.clone();
@@ -115,14 +116,26 @@ impl GlobalState {
       | msg::MediaType::JSX => {
         state1
           .ts_compiler
-          .new_compile(state1.clone(), &out, target_lib, permissions)
+          .new_compile(
+            state1.clone(),
+            &out,
+            target_lib,
+            permissions,
+            is_dyn_import,
+          )
           .await
       }
       msg::MediaType::JavaScript => {
         if state1.ts_compiler.compile_js {
           state2
             .ts_compiler
-            .new_compile(state1.clone(), &out, target_lib, permissions)
+            .new_compile(
+              state1.clone(),
+              &out,
+              target_lib,
+              permissions,
+              is_dyn_import,
+            )
             .await
         } else {
           if let Some(types_url) = out.types_url.clone() {
diff --git a/cli/lib.rs b/cli/lib.rs
index db62e530a4f221..65ed6248f7e353 100644
--- a/cli/lib.rs
+++ b/cli/lib.rs
@@ -212,6 +212,7 @@ async fn print_file_info(
       None,
       TargetLib::Main,
       Permissions::allow_all(),
+      false,
     )
     .await?;
 
diff --git a/cli/module_graph.rs b/cli/module_graph.rs
index 15a0889af69c44..ab653aaae79f8f 100644
--- a/cli/module_graph.rs
+++ b/cli/module_graph.rs
@@ -4,6 +4,7 @@
 use crate::file_fetcher::SourceFileFetcher;
 use crate::import_map::ImportMap;
 use crate::msg::MediaType;
+use crate::op_error::OpError;
 use crate::permissions::Permissions;
 use crate::swc_util::analyze_dependencies_and_references;
 use crate::swc_util::TsReferenceKind;
@@ -78,8 +79,9 @@ pub struct ModuleGraphLoader {
   permissions: Permissions,
   file_fetcher: SourceFileFetcher,
   maybe_import_map: Option<ImportMap>,
-  to_visit: Vec<ModuleSpecifier>,
+  to_visit: Vec<(ModuleSpecifier, Option<ModuleSpecifier>)>,
   pub graph: ModuleGraph,
+  is_dyn_import: bool,
 }
 
 impl ModuleGraphLoader {
@@ -87,6 +89,7 @@ impl ModuleGraphLoader {
     file_fetcher: SourceFileFetcher,
     maybe_import_map: Option<ImportMap>,
     permissions: Permissions,
+    is_dyn_import: bool,
   ) -> Self {
     Self {
       file_fetcher,
@@ -94,6 +97,7 @@ impl ModuleGraphLoader {
       maybe_import_map,
       to_visit: vec![],
       graph: ModuleGraph(HashMap::new()),
+      is_dyn_import,
     }
   }
 
@@ -101,9 +105,9 @@ impl ModuleGraphLoader {
     mut self,
     specifier: &ModuleSpecifier,
   ) -> Result<HashMap<String, ModuleGraphFile>, ErrBox> {
-    self.to_visit.push(specifier.to_owned());
-    while let Some(spec) = self.to_visit.pop() {
-      self.visit_module(&spec).await?;
+    self.to_visit.push((specifier.to_owned(), None));
+    while let Some((spec, maybe_referrer)) = self.to_visit.pop() {
+      self.visit_module(&spec, maybe_referrer).await?;
     }
     Ok(self.graph.0)
   }
@@ -111,14 +115,39 @@ impl ModuleGraphLoader {
   async fn visit_module(
     &mut self,
     module_specifier: &ModuleSpecifier,
+    maybe_referrer: Option<ModuleSpecifier>,
   ) -> Result<(), ErrBox> {
     if self.graph.0.contains_key(&module_specifier.to_string()) {
       return Ok(());
     }
 
+    if !self.is_dyn_import {
+      // Verify that remote file doesn't try to statically import local file.
+      if let Some(referrer) = maybe_referrer.as_ref() {
+        let referrer_url = referrer.as_url();
+        match referrer_url.scheme() {
+          "http" | "https" => {
+            let specifier_url = module_specifier.as_url();
+            match specifier_url.scheme() {
+              "http" | "https" => {}
+              _ => {
+                let e = OpError::permission_denied("Remote module are not allowed to statically import local modules. Use dynamic import instead.".to_string());
+                return Err(e.into());
+              }
+            }
+          }
+          _ => {}
+        }
+      }
+    }
+
     let source_file = self
       .file_fetcher
-      .fetch_source_file(module_specifier, None, self.permissions.clone())
+      .fetch_source_file(
+        module_specifier,
+        maybe_referrer,
+        self.permissions.clone(),
+      )
       .await?;
 
     let mut imports = vec![];
@@ -187,16 +216,19 @@ impl ModuleGraphLoader {
           .get(&import_descriptor.resolved_specifier.to_string())
           .is_none()
         {
-          self
-            .to_visit
-            .push(import_descriptor.resolved_specifier.clone());
+          self.to_visit.push((
+            import_descriptor.resolved_specifier.clone(),
+            Some(module_specifier.clone()),
+          ));
         }
 
         if let Some(type_dir_url) =
           import_descriptor.resolved_type_directive.as_ref()
         {
           if self.graph.0.get(&type_dir_url.to_string()).is_none() {
-            self.to_visit.push(type_dir_url.clone());
+            self
+              .to_visit
+              .push((type_dir_url.clone(), Some(module_specifier.clone())));
           }
         }
 
@@ -219,9 +251,10 @@ impl ModuleGraphLoader {
           .get(&reference_descriptor.resolved_specifier.to_string())
           .is_none()
         {
-          self
-            .to_visit
-            .push(reference_descriptor.resolved_specifier.clone());
+          self.to_visit.push((
+            reference_descriptor.resolved_specifier.clone(),
+            Some(module_specifier.clone()),
+          ));
         }
 
         match ref_desc.kind {
@@ -283,6 +316,7 @@ mod tests {
       global_state.file_fetcher.clone(),
       None,
       Permissions::allow_all(),
+      false,
     );
     let graph = graph_loader.build_graph(&module_specifier).await.unwrap();
 
@@ -354,6 +388,7 @@ mod tests {
       global_state.file_fetcher.clone(),
       None,
       Permissions::allow_all(),
+      false,
     );
     let graph = graph_loader.build_graph(&module_specifier).await.unwrap();
 
@@ -392,6 +427,7 @@ mod tests {
       global_state.file_fetcher.clone(),
       None,
       Permissions::allow_all(),
+      false,
     );
     let graph = graph_loader.build_graph(&module_specifier).await.unwrap();
 
@@ -487,6 +523,7 @@ mod tests {
       global_state.file_fetcher.clone(),
       None,
       Permissions::allow_all(),
+      false,
     );
     let graph = graph_loader.build_graph(&module_specifier).await.unwrap();
 
@@ -551,6 +588,7 @@ mod tests {
       global_state.file_fetcher.clone(),
       None,
       Permissions::allow_all(),
+      false,
     );
     let graph = graph_loader.build_graph(&module_specifier).await.unwrap();
 
diff --git a/cli/state.rs b/cli/state.rs
index 53fa1b418ad653..3d2358c8bb1ea9 100644
--- a/cli/state.rs
+++ b/cli/state.rs
@@ -328,6 +328,7 @@ impl ModuleLoader for State {
           maybe_referrer,
           target_lib,
           permissions,
+          is_dyn_import,
         )
         .await?;
       Ok(deno_core::ModuleSource {
diff --git a/cli/tests/037_fetch_multiple.out b/cli/tests/037_fetch_multiple.out
index cdb6fe2ba747d4..1a703a10c42009 100644
--- a/cli/tests/037_fetch_multiple.out
+++ b/cli/tests/037_fetch_multiple.out
@@ -1,5 +1,5 @@
-Compile [WILDCARD]/fetch/test.ts
 Download http://localhost:4545/cli/tests/subdir/mod2.ts
 Download http://localhost:4545/cli/tests/subdir/print_hello.ts
-Compile [WILDCARD]/fetch/other.ts
+Compile [WILDCARD]/fetch/test.ts
 Download http://localhost:4545/cli/tests/subdir/mt_text_typescript.t1.ts
+Compile [WILDCARD]/fetch/other.ts
diff --git a/cli/tests/error_004_missing_module.ts.out b/cli/tests/error_004_missing_module.ts.out
index b93f2e61321b99..d851882ebc7a19 100644
--- a/cli/tests/error_004_missing_module.ts.out
+++ b/cli/tests/error_004_missing_module.ts.out
@@ -1,8 +1 @@
-[WILDCARD]error: Uncaught NotFound: Cannot resolve module "[WILDCARD]/bad-module.ts" from "[WILDCARD]/error_004_missing_module.ts"
-    at unwrapResponse ([WILDCARD]dispatch_json.ts:[WILDCARD])
-    at Object.sendAsync ([WILDCARD]dispatch_json.ts:[WILDCARD])
-    at async processImports ($deno$/compiler.ts:[WILDCARD])
-    at async processImports ($deno$/compiler.ts:[WILDCARD])
-    at async compile ($deno$/compiler.ts:[WILDCARD])
-    at async tsCompilerOnMessage ($deno$/compiler.ts:[WILDCARD])
-    at async workerMessageRecvCallback ([WILDCARD]runtime_worker.ts:[WILDCARD])
+[WILDCARD]error: Cannot resolve module "[WILDCARD]/bad-module.ts" from "[WILDCARD]/error_004_missing_module.ts"
diff --git a/cli/tests/error_006_import_ext_failure.ts.out b/cli/tests/error_006_import_ext_failure.ts.out
index 5c89adeff86dee..c44d5e746940ac 100644
--- a/cli/tests/error_006_import_ext_failure.ts.out
+++ b/cli/tests/error_006_import_ext_failure.ts.out
@@ -1,8 +1 @@
-[WILDCARD]error: Uncaught NotFound: Cannot resolve module "[WILDCARD]/non-existent" from "[WILDCARD]/error_006_import_ext_failure.ts"
-    at unwrapResponse ([WILDCARD]dispatch_json.ts:[WILDCARD])
-    at Object.sendAsync ([WILDCARD]dispatch_json.ts:[WILDCARD])
-    at async processImports ($deno$/compiler.ts:[WILDCARD])
-    at async processImports ($deno$/compiler.ts:[WILDCARD])
-    at async compile ($deno$/compiler.ts:[WILDCARD])
-    at async tsCompilerOnMessage ($deno$/compiler.ts:[WILDCARD])
-    at async workerMessageRecvCallback ([WILDCARD]runtime_worker.ts:[WILDCARD])
+[WILDCARD]error: Cannot resolve module "[WILDCARD]/non-existent" from "[WILDCARD]/error_006_import_ext_failure.ts"
diff --git a/cli/tests/error_local_static_import_from_remote.ts.out b/cli/tests/error_local_static_import_from_remote.ts.out
index af3c8852fe42f5..4e6a9d4e7046ab 100644
--- a/cli/tests/error_local_static_import_from_remote.ts.out
+++ b/cli/tests/error_local_static_import_from_remote.ts.out
@@ -1,9 +1,2 @@
 [WILDCARD]
-error: Uncaught PermissionDenied: Remote module are not allowed to statically import local modules. Use dynamic import instead.
-    at unwrapResponse ($deno$/ops/dispatch_json.ts:[WILDCARD])
-    at Object.sendAsync ($deno$/ops/dispatch_json.ts:[WILDCARD])
-    at async processImports ($deno$/compiler.ts:[WILDCARD])
-    at async processImports ($deno$/compiler.ts:[WILDCARD])
-    at async compile ($deno$/compiler.ts:[WILDCARD])
-    at async tsCompilerOnMessage ($deno$/compiler.ts:[WILDCARD])
-    at async workerMessageRecvCallback ($deno$/runtime_worker.ts:[WILDCARD])
+error: Remote module are not allowed to statically import local modules. Use dynamic import instead.
diff --git a/cli/tsc.rs b/cli/tsc.rs
index c7a2bcfba7e617..afc553fc9b3576 100644
--- a/cli/tsc.rs
+++ b/cli/tsc.rs
@@ -400,6 +400,7 @@ impl TsCompiler {
       global_state.file_fetcher.clone(),
       import_map,
       permissions.clone(),
+      false,
     );
     let module_graph =
       module_graph_loader.build_graph(&module_specifier).await?;
@@ -641,6 +642,7 @@ impl TsCompiler {
     source_file: &SourceFile,
     target: TargetLib,
     permissions: Permissions,
+    is_dyn_import: bool,
   ) -> Result<CompiledModule, ErrBox> {
     if self.has_compiled(&source_file.url) {
       return self.get_compiled_module(&source_file.url);
@@ -686,6 +688,7 @@ impl TsCompiler {
       global_state.file_fetcher.clone(),
       import_map,
       permissions.clone(),
+      is_dyn_import,
     );
     let module_graph =
       module_graph_loader.build_graph(&module_specifier).await?;

From 4ae457c874ba53174b6994dc56090709c694c561 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= <biwanczuk@gmail.com>
Date: Mon, 11 May 2020 13:26:02 +0200
Subject: [PATCH 23/47] fmt

---
 cli/state.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/cli/state.rs b/cli/state.rs
index 3d2358c8bb1ea9..d532906944d7bd 100644
--- a/cli/state.rs
+++ b/cli/state.rs
@@ -318,8 +318,8 @@ impl ModuleLoader for State {
     let permissions = if state.is_main {
       Permissions::allow_all()
     } else {
-    state.permissions.clone()
-  };
+      state.permissions.clone()
+    };
 
     let fut = async move {
       let compiled_module = global_state

From d2eb3b3066bbbd9b2a56204d488010eb84fe97e1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= <biwanczuk@gmail.com>
Date: Thu, 14 May 2020 20:39:38 +0200
Subject: [PATCH 24/47] concurrent downloads

---
 cli/file_fetcher.rs |  12 +++
 cli/module_graph.rs | 195 +++++++++++++++++++++++++++++++++++++++++++-
 2 files changed, 204 insertions(+), 3 deletions(-)

diff --git a/cli/file_fetcher.rs b/cli/file_fetcher.rs
index cbfb340ddc6121..32971e71f06b91 100644
--- a/cli/file_fetcher.rs
+++ b/cli/file_fetcher.rs
@@ -147,6 +147,18 @@ impl SourceFileFetcher {
     self.source_file_cache.set(specifier.to_string(), file);
   }
 
+  // TODO(bartlomieju): remove
+  pub async fn new_fetch_source_file(
+    &self,
+    specifier: ModuleSpecifier,
+    maybe_referrer: Option<ModuleSpecifier>,
+    permissions: Permissions,
+  ) -> Result<SourceFile, ErrBox> {
+    self
+      .fetch_source_file(&specifier, maybe_referrer, permissions)
+      .await
+  }
+
   pub async fn fetch_source_file(
     &self,
     specifier: &ModuleSpecifier,
diff --git a/cli/module_graph.rs b/cli/module_graph.rs
index ab653aaae79f8f..9e46511c34f6a2 100644
--- a/cli/module_graph.rs
+++ b/cli/module_graph.rs
@@ -1,6 +1,7 @@
 // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license.
 #![allow(unused)]
 
+use crate::file_fetcher::SourceFile;
 use crate::file_fetcher::SourceFileFetcher;
 use crate::import_map::ImportMap;
 use crate::msg::MediaType;
@@ -10,9 +11,14 @@ use crate::swc_util::analyze_dependencies_and_references;
 use crate::swc_util::TsReferenceKind;
 use deno_core::ErrBox;
 use deno_core::ModuleSpecifier;
+use futures::stream::FuturesUnordered;
+use futures::stream::StreamExt;
+use futures::Future;
+use futures::FutureExt;
 use serde::Serialize;
 use serde::Serializer;
 use std::collections::HashMap;
+use std::pin::Pin;
 
 fn serialize_module_specifier<S>(
   spec: &ModuleSpecifier,
@@ -75,10 +81,13 @@ pub struct ModuleGraphFile {
   pub source_code: String,
 }
 
+type SourceFileFuture = Pin<Box<dyn Future<Output = Result<SourceFile, ErrBox>>>>;
+
 pub struct ModuleGraphLoader {
   permissions: Permissions,
   file_fetcher: SourceFileFetcher,
   maybe_import_map: Option<ImportMap>,
+  pending_downloads: FuturesUnordered<SourceFileFuture>,
   to_visit: Vec<(ModuleSpecifier, Option<ModuleSpecifier>)>,
   pub graph: ModuleGraph,
   is_dyn_import: bool,
@@ -96,6 +105,7 @@ impl ModuleGraphLoader {
       permissions,
       maybe_import_map,
       to_visit: vec![],
+      pending_downloads: FuturesUnordered::new(),
       graph: ModuleGraph(HashMap::new()),
       is_dyn_import,
     }
@@ -105,13 +115,192 @@ impl ModuleGraphLoader {
     mut self,
     specifier: &ModuleSpecifier,
   ) -> Result<HashMap<String, ModuleGraphFile>, ErrBox> {
-    self.to_visit.push((specifier.to_owned(), None));
-    while let Some((spec, maybe_referrer)) = self.to_visit.pop() {
-      self.visit_module(&spec, maybe_referrer).await?;
+    self.download_module(specifier.clone(), None)?;
+
+    loop {
+      let load_result = self.pending_downloads.next().await.unwrap();
+      let source_file = load_result?;
+      let spec = ModuleSpecifier::from(source_file.url.clone());
+      self.new_visit_module(&spec, source_file)?;
+      if self.pending_downloads.is_empty() {
+        break;
+      }
     }
+
     Ok(self.graph.0)
   }
 
+  fn download_module(
+    &mut self,
+    module_specifier: ModuleSpecifier,
+    maybe_referrer: Option<ModuleSpecifier>,
+  ) -> Result<(), ErrBox> {
+    if self.graph.0.contains_key(&module_specifier.to_string()) {
+      return Ok(());
+    }
+
+    if !self.is_dyn_import {
+      // Verify that remote file doesn't try to statically import local file.
+      if let Some(referrer) = maybe_referrer.as_ref() {
+        let referrer_url = referrer.as_url();
+        match referrer_url.scheme() {
+          "http" | "https" => {
+            let specifier_url = module_specifier.as_url();
+            match specifier_url.scheme() {
+              "http" | "https" => {}
+              _ => {
+                let e = OpError::permission_denied("Remote module are not allowed to statically import local modules. Use dynamic import instead.".to_string());
+                return Err(e.into());
+              }
+            }
+          }
+          _ => {}
+        }
+      }
+    }
+
+    let spec = module_specifier;
+    let file_fetcher = self.file_fetcher.clone();
+    let perms = self.permissions.clone();
+
+    let load_future = async move {
+      file_fetcher
+        .new_fetch_source_file(spec, maybe_referrer, perms)
+        .await
+    }
+    .boxed_local();
+
+    self.pending_downloads.push(load_future);
+    Ok(())
+  }
+
+  fn new_visit_module(
+    &mut self,
+    module_specifier: &ModuleSpecifier,
+    source_file: SourceFile,
+  ) -> Result<(), ErrBox> {
+    let mut imports = vec![];
+    let mut referenced_files = vec![];
+    let mut lib_directives = vec![];
+    let mut types_directives = vec![];
+    let mut type_headers = vec![];
+
+    let source_code = String::from_utf8(source_file.source_code)?;
+
+    if source_file.media_type == MediaType::JavaScript
+      || source_file.media_type == MediaType::TypeScript
+    {
+      if let Some(types_specifier) = source_file.types_header {
+        let type_header = ReferenceDescriptor {
+          specifier: types_specifier.to_string(),
+          resolved_specifier: ModuleSpecifier::resolve_import(
+            &types_specifier,
+            &module_specifier.to_string(),
+          )?,
+        };
+        type_headers.push(type_header);
+      }
+
+      let (import_descs, ref_descs) =
+        analyze_dependencies_and_references(&source_code, true)?;
+
+      for import_desc in import_descs {
+        let maybe_resolved =
+          if let Some(import_map) = self.maybe_import_map.as_ref() {
+            import_map
+              .resolve(&import_desc.specifier, &module_specifier.to_string())?
+          } else {
+            None
+          };
+
+        let resolved_specifier = if let Some(resolved) = maybe_resolved {
+          resolved
+        } else {
+          ModuleSpecifier::resolve_import(
+            &import_desc.specifier,
+            &module_specifier.to_string(),
+          )?
+        };
+
+        let resolved_type_directive =
+          if let Some(types_specifier) = import_desc.deno_types.as_ref() {
+            Some(ModuleSpecifier::resolve_import(
+              &types_specifier,
+              &module_specifier.to_string(),
+            )?)
+          } else {
+            None
+          };
+
+        let import_descriptor = ImportDescriptor {
+          specifier: import_desc.specifier.to_string(),
+          resolved_specifier,
+          type_directive: import_desc.deno_types,
+          resolved_type_directive,
+        };
+
+        self.download_module(
+          import_descriptor.resolved_specifier.clone(),
+          Some(module_specifier.clone()),
+        )?;
+
+        if let Some(type_dir_url) =
+          import_descriptor.resolved_type_directive.as_ref()
+        {
+          self.download_module(
+            type_dir_url.clone(),
+            Some(module_specifier.clone()),
+          )?;
+        }
+
+        imports.push(import_descriptor);
+      }
+
+      for ref_desc in ref_descs {
+        let resolved_specifier = ModuleSpecifier::resolve_import(
+          &ref_desc.specifier,
+          &module_specifier.to_string(),
+        )?;
+        let reference_descriptor = ReferenceDescriptor {
+          specifier: ref_desc.specifier.to_string(),
+          resolved_specifier,
+        };
+
+        self.download_module(
+          reference_descriptor.resolved_specifier.clone(),
+          Some(module_specifier.clone()),
+        )?;
+
+        match ref_desc.kind {
+          TsReferenceKind::Lib => {
+            lib_directives.push(reference_descriptor);
+          }
+          TsReferenceKind::Types => {
+            types_directives.push(reference_descriptor);
+          }
+          TsReferenceKind::Path => {
+            referenced_files.push(reference_descriptor);
+          }
+        }
+      }
+    }
+
+    self.graph.0.insert(
+      module_specifier.to_string(),
+      ModuleGraphFile {
+        specifier: module_specifier.to_string(),
+        media_type: source_file.media_type,
+        source_code,
+        imports,
+        referenced_files,
+        lib_directives,
+        types_directives,
+        type_headers,
+      },
+    );
+    Ok(())
+  }
+
   async fn visit_module(
     &mut self,
     module_specifier: &ModuleSpecifier,

From 10f1ce8e7fc1076eec21e0ebe09f24118bf9dd71 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= <biwanczuk@gmail.com>
Date: Thu, 14 May 2020 20:43:59 +0200
Subject: [PATCH 25/47] fmt

---
 cli/module_graph.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/cli/module_graph.rs b/cli/module_graph.rs
index 9e46511c34f6a2..c04c05fd8a865c 100644
--- a/cli/module_graph.rs
+++ b/cli/module_graph.rs
@@ -81,7 +81,8 @@ pub struct ModuleGraphFile {
   pub source_code: String,
 }
 
-type SourceFileFuture = Pin<Box<dyn Future<Output = Result<SourceFile, ErrBox>>>>;
+type SourceFileFuture =
+  Pin<Box<dyn Future<Output = Result<SourceFile, ErrBox>>>>;
 
 pub struct ModuleGraphLoader {
   permissions: Permissions,

From 28ce2020aa42714b49365d8ec9c0d62ce3706253 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= <biwanczuk@gmail.com>
Date: Thu, 14 May 2020 21:06:02 +0200
Subject: [PATCH 26/47] cleanup

---
 cli/global_state.rs               |  10 +-
 cli/main.rs                       |   2 +-
 cli/module_graph.rs               | 180 +----------------------------
 cli/tests/type_definitions/bar.js |   5 +
 cli/tsc.rs                        | 184 +++---------------------------
 5 files changed, 24 insertions(+), 357 deletions(-)
 create mode 100644 cli/tests/type_definitions/bar.js

diff --git a/cli/global_state.rs b/cli/global_state.rs
index 3bec4a500c7fc7..90961ed6271a8c 100644
--- a/cli/global_state.rs
+++ b/cli/global_state.rs
@@ -116,20 +116,14 @@ impl GlobalState {
       | msg::MediaType::JSX => {
         state1
           .ts_compiler
-          .new_compile(
-            state1.clone(),
-            &out,
-            target_lib,
-            permissions,
-            is_dyn_import,
-          )
+          .compile(state1.clone(), &out, target_lib, permissions, is_dyn_import)
           .await
       }
       msg::MediaType::JavaScript => {
         if state1.ts_compiler.compile_js {
           state2
             .ts_compiler
-            .new_compile(
+            .compile(
               state1.clone(),
               &out,
               target_lib,
diff --git a/cli/main.rs b/cli/main.rs
index 32b25e4f632c3b..cadb76fb4b108b 100644
--- a/cli/main.rs
+++ b/cli/main.rs
@@ -390,7 +390,7 @@ async fn bundle_command(
   debug!(">>>>> bundle START");
   let bundle_result = global_state
     .ts_compiler
-    .new_bundle(global_state.clone(), module_name, out_file)
+    .bundle(global_state.clone(), module_name, out_file)
     .await;
   debug!(">>>>> bundle END");
   bundle_result
diff --git a/cli/module_graph.rs b/cli/module_graph.rs
index c04c05fd8a865c..aedad94f22edcc 100644
--- a/cli/module_graph.rs
+++ b/cli/module_graph.rs
@@ -122,7 +122,7 @@ impl ModuleGraphLoader {
       let load_result = self.pending_downloads.next().await.unwrap();
       let source_file = load_result?;
       let spec = ModuleSpecifier::from(source_file.url.clone());
-      self.new_visit_module(&spec, source_file)?;
+      self.visit_module(&spec, source_file)?;
       if self.pending_downloads.is_empty() {
         break;
       }
@@ -175,7 +175,7 @@ impl ModuleGraphLoader {
     Ok(())
   }
 
-  fn new_visit_module(
+  fn visit_module(
     &mut self,
     module_specifier: &ModuleSpecifier,
     source_file: SourceFile,
@@ -233,6 +233,7 @@ impl ModuleGraphLoader {
             None
           };
 
+        eprintln!("type directive {:#?}", import_desc.deno_types);
         let import_descriptor = ImportDescriptor {
           specifier: import_desc.specifier.to_string(),
           resolved_specifier,
@@ -301,181 +302,6 @@ impl ModuleGraphLoader {
     );
     Ok(())
   }
-
-  async fn visit_module(
-    &mut self,
-    module_specifier: &ModuleSpecifier,
-    maybe_referrer: Option<ModuleSpecifier>,
-  ) -> Result<(), ErrBox> {
-    if self.graph.0.contains_key(&module_specifier.to_string()) {
-      return Ok(());
-    }
-
-    if !self.is_dyn_import {
-      // Verify that remote file doesn't try to statically import local file.
-      if let Some(referrer) = maybe_referrer.as_ref() {
-        let referrer_url = referrer.as_url();
-        match referrer_url.scheme() {
-          "http" | "https" => {
-            let specifier_url = module_specifier.as_url();
-            match specifier_url.scheme() {
-              "http" | "https" => {}
-              _ => {
-                let e = OpError::permission_denied("Remote module are not allowed to statically import local modules. Use dynamic import instead.".to_string());
-                return Err(e.into());
-              }
-            }
-          }
-          _ => {}
-        }
-      }
-    }
-
-    let source_file = self
-      .file_fetcher
-      .fetch_source_file(
-        module_specifier,
-        maybe_referrer,
-        self.permissions.clone(),
-      )
-      .await?;
-
-    let mut imports = vec![];
-    let mut referenced_files = vec![];
-    let mut lib_directives = vec![];
-    let mut types_directives = vec![];
-    let mut type_headers = vec![];
-
-    let source_code = String::from_utf8(source_file.source_code)?;
-
-    if source_file.media_type == MediaType::JavaScript
-      || source_file.media_type == MediaType::TypeScript
-    {
-      if let Some(types_specifier) = source_file.types_header {
-        let type_header = ReferenceDescriptor {
-          specifier: types_specifier.to_string(),
-          resolved_specifier: ModuleSpecifier::resolve_import(
-            &types_specifier,
-            &module_specifier.to_string(),
-          )?,
-        };
-        type_headers.push(type_header);
-      }
-
-      let (import_descs, ref_descs) =
-        analyze_dependencies_and_references(&source_code, true)?;
-
-      for import_desc in import_descs {
-        let maybe_resolved =
-          if let Some(import_map) = self.maybe_import_map.as_ref() {
-            import_map
-              .resolve(&import_desc.specifier, &module_specifier.to_string())?
-          } else {
-            None
-          };
-
-        let resolved_specifier = if let Some(resolved) = maybe_resolved {
-          resolved
-        } else {
-          ModuleSpecifier::resolve_import(
-            &import_desc.specifier,
-            &module_specifier.to_string(),
-          )?
-        };
-
-        let resolved_type_directive =
-          if let Some(types_specifier) = import_desc.deno_types.as_ref() {
-            Some(ModuleSpecifier::resolve_import(
-              &types_specifier,
-              &module_specifier.to_string(),
-            )?)
-          } else {
-            None
-          };
-
-        let import_descriptor = ImportDescriptor {
-          specifier: import_desc.specifier.to_string(),
-          resolved_specifier,
-          type_directive: import_desc.deno_types,
-          resolved_type_directive,
-        };
-
-        if self
-          .graph
-          .0
-          .get(&import_descriptor.resolved_specifier.to_string())
-          .is_none()
-        {
-          self.to_visit.push((
-            import_descriptor.resolved_specifier.clone(),
-            Some(module_specifier.clone()),
-          ));
-        }
-
-        if let Some(type_dir_url) =
-          import_descriptor.resolved_type_directive.as_ref()
-        {
-          if self.graph.0.get(&type_dir_url.to_string()).is_none() {
-            self
-              .to_visit
-              .push((type_dir_url.clone(), Some(module_specifier.clone())));
-          }
-        }
-
-        imports.push(import_descriptor);
-      }
-
-      for ref_desc in ref_descs {
-        let resolved_specifier = ModuleSpecifier::resolve_import(
-          &ref_desc.specifier,
-          &module_specifier.to_string(),
-        )?;
-        let reference_descriptor = ReferenceDescriptor {
-          specifier: ref_desc.specifier.to_string(),
-          resolved_specifier,
-        };
-
-        if self
-          .graph
-          .0
-          .get(&reference_descriptor.resolved_specifier.to_string())
-          .is_none()
-        {
-          self.to_visit.push((
-            reference_descriptor.resolved_specifier.clone(),
-            Some(module_specifier.clone()),
-          ));
-        }
-
-        match ref_desc.kind {
-          TsReferenceKind::Lib => {
-            lib_directives.push(reference_descriptor);
-          }
-          TsReferenceKind::Types => {
-            types_directives.push(reference_descriptor);
-          }
-          TsReferenceKind::Path => {
-            referenced_files.push(reference_descriptor);
-          }
-        }
-      }
-    }
-
-    self.graph.0.insert(
-      module_specifier.to_string(),
-      ModuleGraphFile {
-        specifier: module_specifier.to_string(),
-        media_type: source_file.media_type,
-        source_code,
-        imports,
-        referenced_files,
-        lib_directives,
-        types_directives,
-        type_headers,
-      },
-    );
-    Ok(())
-  }
 }
 
 #[cfg(test)]
diff --git a/cli/tests/type_definitions/bar.js b/cli/tests/type_definitions/bar.js
new file mode 100644
index 00000000000000..616306027d438f
--- /dev/null
+++ b/cli/tests/type_definitions/bar.js
@@ -0,0 +1,5 @@
+export class Bar {
+  constructor() {
+    this.baz = "baz";
+  }
+}
\ No newline at end of file
diff --git a/cli/tsc.rs b/cli/tsc.rs
index c8763d315c8cde..a3ba7aab3177b2 100644
--- a/cli/tsc.rs
+++ b/cli/tsc.rs
@@ -222,39 +222,6 @@ impl CompiledFileMetadata {
     serde_json::to_string(&value_map)
   }
 }
-/// Creates the JSON message send to compiler.ts's onmessage.
-fn req(
-  request_type: msg::CompilerRequestType,
-  root_names: Vec<String>,
-  compiler_config: CompilerConfig,
-  target: &str,
-  bundle: bool,
-  unstable: bool,
-) -> Buf {
-  let cwd = std::env::current_dir().unwrap();
-  let j = match (compiler_config.path, compiler_config.content) {
-    (Some(config_path), Some(config_data)) => json!({
-      "type": request_type as i32,
-      "target": target,
-      "rootNames": root_names,
-      "bundle": bundle,
-      "unstable": unstable,
-      "configPath": config_path,
-      "config": str::from_utf8(&config_data).unwrap(),
-      "cwd": cwd,
-    }),
-    _ => json!({
-      "type": request_type as i32,
-      "target": target,
-      "rootNames": root_names,
-      "bundle": bundle,
-      "unstable": unstable,
-      "cwd": cwd,
-    }),
-  };
-
-  j.to_string().into_boxed_str().into_boxed_bytes()
-}
 
 /// Emit a SHA256 hash based on source code, deno version and TS config.
 /// Used to check if a recompilation for source code is needed.
@@ -369,7 +336,7 @@ impl TsCompiler {
     worker
   }
 
-  pub async fn new_bundle(
+  pub async fn bundle(
     &self,
     global_state: GlobalState,
     module_specifier: ModuleSpecifier,
@@ -463,59 +430,6 @@ impl TsCompiler {
     Ok(())
   }
 
-  #[allow(unused)]
-  pub async fn bundle(
-    &self,
-    global_state: GlobalState,
-    module_name: String,
-    out_file: Option<PathBuf>,
-  ) -> Result<(), ErrBox> {
-    debug!(
-      "Invoking the compiler to bundle. module_name: {}",
-      module_name
-    );
-    eprintln!("Bundling {}", module_name);
-
-    let root_names = vec![module_name];
-    let req_msg = req(
-      msg::CompilerRequestType::Compile,
-      root_names,
-      self.config.clone(),
-      "main",
-      true,
-      global_state.flags.unstable,
-    );
-
-    let permissions = Permissions::allow_all();
-    let msg =
-      execute_in_thread(global_state.clone(), permissions, req_msg).await?;
-    let json_str = std::str::from_utf8(&msg).unwrap();
-    debug!("Message: {}", json_str);
-
-    let bundle_response: BundleResponse = serde_json::from_str(json_str)?;
-
-    if !bundle_response.diagnostics.items.is_empty() {
-      return Err(ErrBox::from(bundle_response.diagnostics));
-    }
-
-    let output_string = fmt::format_text(&bundle_response.bundle_output)?;
-
-    if let Some(out_file_) = out_file.as_ref() {
-      eprintln!("Emitting bundle to {:?}", out_file_);
-
-      let output_bytes = output_string.as_bytes();
-      let output_len = output_bytes.len();
-
-      deno_fs::write_file(out_file_, output_bytes, 0o666)?;
-      // TODO(bartlomieju): add "humanFileSize" method
-      eprintln!("{} bytes emmited.", output_len);
-    } else {
-      println!("{}", output_string);
-    }
-
-    Ok(())
-  }
-
   /// Mark given module URL as compiled to avoid multiple compilations of same
   /// module in single run.
   fn mark_compiled(&self, url: &Url) {
@@ -530,87 +444,6 @@ impl TsCompiler {
     c.contains(url)
   }
 
-  /// Asynchronously compile module and all it's dependencies.
-  ///
-  /// This method compiled every module at most once.
-  ///
-  /// If `--reload` flag was provided then compiler will not on-disk cache and
-  /// force recompilation.
-  ///
-  /// If compilation is required then new V8 worker is spawned with fresh TS
-  /// compiler.
-  pub async fn compile(
-    &self,
-    global_state: GlobalState,
-    source_file: &SourceFile,
-    target: TargetLib,
-    permissions: Permissions,
-  ) -> Result<CompiledModule, ErrBox> {
-    if self.has_compiled(&source_file.url) {
-      return self.get_compiled_module(&source_file.url);
-    }
-
-    if self.use_disk_cache {
-      // Try to load cached version:
-      // 1. check if there's 'meta' file
-      if let Some(metadata) = self.get_metadata(&source_file.url) {
-        // 2. compare version hashes
-        // TODO: it would probably be good idea to make it method implemented on SourceFile
-        let version_hash_to_validate = source_code_version_hash(
-          &source_file.source_code,
-          version::DENO,
-          &self.config.hash,
-        );
-
-        if metadata.version_hash == version_hash_to_validate {
-          debug!("load_cache metadata version hash match");
-          if let Ok(compiled_module) =
-            self.get_compiled_module(&source_file.url)
-          {
-            self.mark_compiled(&source_file.url);
-            return Ok(compiled_module);
-          }
-        }
-      }
-    }
-    let source_file_ = source_file.clone();
-    let module_url = source_file.url.clone();
-    let target = match target {
-      TargetLib::Main => "main",
-      TargetLib::Worker => "worker",
-    };
-    let root_names = vec![module_url.to_string()];
-    let req_msg = req(
-      msg::CompilerRequestType::Compile,
-      root_names,
-      self.config.clone(),
-      target,
-      false,
-      global_state.flags.unstable,
-    );
-
-    let ts_compiler = self.clone();
-
-    info!(
-      "{} {}",
-      colors::green("Compile".to_string()),
-      module_url.to_string()
-    );
-
-    let msg =
-      execute_in_thread(global_state.clone(), permissions, req_msg).await?;
-    let json_str = std::str::from_utf8(&msg).unwrap();
-
-    let compile_response: CompileResponse = serde_json::from_str(json_str)?;
-
-    if !compile_response.diagnostics.items.is_empty() {
-      return Err(ErrBox::from(compile_response.diagnostics));
-    }
-
-    self.cache_emitted_files(compile_response.emit_map)?;
-    ts_compiler.get_compiled_module(&source_file_.url)
-  }
-
   /// Get associated `CompiledFileMetadata` for given module if it exists.
   pub fn get_metadata(&self, url: &Url) -> Option<CompiledFileMetadata> {
     // Try to load cached version:
@@ -631,8 +464,16 @@ impl TsCompiler {
     None
   }
 
-  #[allow(unused)]
-  pub async fn new_compile(
+  /// Asynchronously compile module and all it's dependencies.
+  ///
+  /// This method compiled every module at most once.
+  ///
+  /// If `--reload` flag was provided then compiler will not on-disk cache and
+  /// force recompilation.
+  ///
+  /// If compilation is required then new V8 worker is spawned with fresh TS
+  /// compiler.
+  pub async fn compile(
     &self,
     global_state: GlobalState,
     source_file: &SourceFile,
@@ -1143,6 +984,7 @@ mod tests {
         &out,
         TargetLib::Main,
         Permissions::allow_all(),
+        false,
       )
       .await;
     assert!(result.is_ok());
@@ -1197,7 +1039,7 @@ mod tests {
 
     let result = state
       .ts_compiler
-      .new_bundle(state.clone(), module_name, None)
+      .bundle(state.clone(), module_name, None)
       .await;
     assert!(result.is_ok());
   }

From aa096a97ed89275c621c386f550791812ea5a444 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= <biwanczuk@gmail.com>
Date: Thu, 14 May 2020 23:43:31 +0200
Subject: [PATCH 27/47] cleanup2

---
 cli/js/compiler.ts                | 10 ++++++++++
 cli/module_graph.rs               |  9 ++++++---
 cli/msg.rs                        |  1 +
 cli/tests/type_definitions/bar.js |  2 +-
 cli/tsc.rs                        | 11 +++++++++--
 5 files changed, 27 insertions(+), 6 deletions(-)

diff --git a/cli/js/compiler.ts b/cli/js/compiler.ts
index d959a1e9a0d9e1..38641e438166d0 100644
--- a/cli/js/compiler.ts
+++ b/cli/js/compiler.ts
@@ -1397,6 +1397,7 @@ function compileNew(request: CompilerRequestCompileNew): CompileResult {
   }
 
   buildSourceFileCache(sourceFileMap);
+  const start = new Date();
   // if there was a configuration and no diagnostics with it, we will continue
   // to generate the program and possibly emit it.
   if (diagnostics.length === 0) {
@@ -1427,6 +1428,9 @@ function compileNew(request: CompilerRequestCompileNew): CompileResult {
     }
   }
 
+  const end = new Date();
+  // @ts-ignore
+  util.log("time in internal TS", end - start);
   let bundleOutput = undefined;
 
   if (bundle) {
@@ -1740,7 +1744,13 @@ async function tsCompilerOnMessage({
       break;
     }
     case CompilerRequestType.CompileNew: {
+      const start = new Date();
       const result = compileNew(request as CompilerRequestCompileNew);
+      const end = new Date();
+      util.log(
+        // @ts-ignore
+        `Time spent in TS program "${end - start}"`
+      );
       globalThis.postMessage(result);
       break;
     }
diff --git a/cli/module_graph.rs b/cli/module_graph.rs
index aedad94f22edcc..9561c151ed66e3 100644
--- a/cli/module_graph.rs
+++ b/cli/module_graph.rs
@@ -72,12 +72,14 @@ pub struct ReferenceDescriptor {
 #[serde(rename_all = "camelCase")]
 pub struct ModuleGraphFile {
   pub specifier: String,
+  pub url: String,
+  pub filename: String,
   pub imports: Vec<ImportDescriptor>,
   pub referenced_files: Vec<ReferenceDescriptor>,
   pub lib_directives: Vec<ReferenceDescriptor>,
   pub types_directives: Vec<ReferenceDescriptor>,
   pub type_headers: Vec<ReferenceDescriptor>,
-  pub media_type: MediaType,
+  pub media_type: i32,
   pub source_code: String,
 }
 
@@ -233,7 +235,6 @@ impl ModuleGraphLoader {
             None
           };
 
-        eprintln!("type directive {:#?}", import_desc.deno_types);
         let import_descriptor = ImportDescriptor {
           specifier: import_desc.specifier.to_string(),
           resolved_specifier,
@@ -291,7 +292,9 @@ impl ModuleGraphLoader {
       module_specifier.to_string(),
       ModuleGraphFile {
         specifier: module_specifier.to_string(),
-        media_type: source_file.media_type,
+        url: source_file.url.to_string(),
+        filename: source_file.filename.to_str().unwrap().to_string(),
+        media_type: source_file.media_type as i32,
         source_code,
         imports,
         referenced_files,
diff --git a/cli/msg.rs b/cli/msg.rs
index 186fde42c644e8..dc6765776419df 100644
--- a/cli/msg.rs
+++ b/cli/msg.rs
@@ -38,4 +38,5 @@ pub enum CompilerRequestType {
   Compile = 0,
   RuntimeCompile = 1,
   RuntimeTranspile = 2,
+  CompileNew = 3,
 }
diff --git a/cli/tests/type_definitions/bar.js b/cli/tests/type_definitions/bar.js
index 616306027d438f..e9c2e5193cce21 100644
--- a/cli/tests/type_definitions/bar.js
+++ b/cli/tests/type_definitions/bar.js
@@ -2,4 +2,4 @@ export class Bar {
   constructor() {
     this.baz = "baz";
   }
-}
\ No newline at end of file
+}
diff --git a/cli/tsc.rs b/cli/tsc.rs
index a3ba7aab3177b2..81d8608fb0887b 100644
--- a/cli/tsc.rs
+++ b/cli/tsc.rs
@@ -50,6 +50,7 @@ use std::sync::atomic::Ordering;
 use std::sync::Arc;
 use std::sync::Mutex;
 use std::task::Poll;
+use std::time::Instant;
 use url::Url;
 
 #[derive(Debug, Clone)]
@@ -543,7 +544,7 @@ impl TsCompiler {
     let cwd = std::env::current_dir().unwrap();
     let j = match (compiler_config.path, compiler_config.content) {
       (Some(config_path), Some(config_data)) => json!({
-        "type": msg::CompilerRequestType::Compile as i32,
+        "type": msg::CompilerRequestType::CompileNew as i32,
         "target": target,
         "rootNames": root_names,
         "bundle": bundle,
@@ -554,7 +555,7 @@ impl TsCompiler {
         "sourceFileMap": module_graph_json,
       }),
       _ => json!({
-        "type": msg::CompilerRequestType::Compile as i32,
+        "type": msg::CompilerRequestType::CompileNew as i32,
         "target": target,
         "rootNames": root_names,
         "bundle": bundle,
@@ -574,8 +575,14 @@ impl TsCompiler {
       module_url.to_string()
     );
 
+    let start = Instant::now();
+
     let msg =
       execute_in_thread(global_state.clone(), permissions, req_msg).await?;
+
+    let end = Instant::now();
+    debug!("time spent in compiler thread {:#?}", end - start);
+
     let json_str = std::str::from_utf8(&msg).unwrap();
 
     let compile_response: CompileResponse = serde_json::from_str(json_str)?;

From 3a1c842ac9771a08411b001748e7150d2dd6e0eb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= <biwanczuk@gmail.com>
Date: Sat, 16 May 2020 15:30:39 +0200
Subject: [PATCH 28/47] runtime compile remote files

---
 cli/js/compiler.ts             | 120 ++++++++++++++++
 cli/module_graph.rs            |  55 +++++++
 cli/msg.rs                     |   1 +
 cli/ops/runtime_compiler.rs    |   4 +-
 cli/tests/compiler_api_test.ts | 254 ++++++++++++++++-----------------
 cli/tsc.rs                     |  81 +++++++++++
 6 files changed, 386 insertions(+), 129 deletions(-)

diff --git a/cli/js/compiler.ts b/cli/js/compiler.ts
index 38641e438166d0..7ade3d500f7e7b 100644
--- a/cli/js/compiler.ts
+++ b/cli/js/compiler.ts
@@ -907,6 +907,7 @@ enum CompilerRequestType {
   RuntimeCompile = 1,
   RuntimeTranspile = 2,
   CompileNew = 3,
+  RuntimeCompileNew = 4,
 }
 
 // TODO(bartlomieju): probably could be defined inline?
@@ -1321,6 +1322,16 @@ interface CompilerRequestRuntimeCompile {
   options?: string;
 }
 
+interface CompilerRequestRuntimeCompileNew {
+  type: CompilerRequestType.RuntimeCompileNew;
+  target: CompilerHostTarget;
+  rootNames: string[];
+  sourceFileMap: Record<string, SourceFileMapEntry>,
+  unstable?: boolean;
+  bundle?: boolean;
+  options?: string;
+}
+
 interface CompilerRequestRuntimeTranspile {
   type: CompilerRequestType.RuntimeTranspile;
   sources: Record<string, string>;
@@ -1331,6 +1342,7 @@ type CompilerRequest =
   | CompilerRequestCompileNew
   | CompilerRequestCompile
   | CompilerRequestRuntimeCompile
+  | CompilerRequestRuntimeCompileNew
   | CompilerRequestRuntimeTranspile;
 
 interface CompileResult {
@@ -1565,6 +1577,107 @@ async function compile(
   return result;
 }
 
+async function runtimeCompileNew(
+  request: CompilerRequestRuntimeCompileNew
+): Promise<RuntimeCompileResult | RuntimeBundleResult> {
+  const { bundle, options, rootNames, target, unstable, sourceFileMap } = request;
+
+  util.log(">>> runtime compile start", {
+    rootNames,
+    bundle,
+  });
+
+  // if there are options, convert them into TypeScript compiler options,
+  // and resolve any external file references
+  let convertedOptions: ts.CompilerOptions | undefined;
+  if (options) {
+    const result = convertCompilerOptions(options);
+    convertedOptions = result.options;
+  }
+
+  buildSourceFileCache(sourceFileMap);
+
+  const state: WriteFileState = {
+    type: request.type,
+    bundle,
+    host: undefined,
+    rootNames,
+    emitMap: {},
+    bundleOutput: undefined,
+  };
+  let writeFile: WriteFileCallback;
+  if (bundle) {
+    writeFile = createBundleWriteFile(state);
+  } else {
+    writeFile = createCompileWriteFile(state);
+  }
+
+  const host = (state.host = new Host({
+    bundle,
+    target,
+    writeFile,
+  }));
+  const compilerOptions = [DEFAULT_RUNTIME_COMPILE_OPTIONS];
+  if (convertedOptions) {
+    compilerOptions.push(convertedOptions);
+  }
+  if (unstable) {
+    compilerOptions.push({
+      lib: [
+        "deno.unstable",
+        ...((convertedOptions && convertedOptions.lib) || ["deno.window"]),
+      ],
+    });
+  }
+  if (bundle) {
+    compilerOptions.push(DEFAULT_BUNDLER_OPTIONS);
+  }
+  host.mergeOptions(...compilerOptions);
+
+  const program = ts.createProgram({
+    rootNames,
+    options: host.getCompilationSettings(),
+    host,
+    oldProgram: TS_SNAPSHOT_PROGRAM,
+  });
+
+  if (bundle) {
+    setRootExports(program, rootNames[0]);
+  }
+
+  const diagnostics = ts
+    .getPreEmitDiagnostics(program)
+    .filter(({ code }) => !ignoredDiagnostics.includes(code));
+
+  const emitResult = program.emit();
+
+  assert(emitResult.emitSkipped === false, "Unexpected skip of the emit.");
+
+  assert(state.emitMap);
+  util.log("<<< runtime compile finish", {
+    rootNames,
+    bundle,
+    emitMap: Object.keys(state.emitMap),
+  });
+
+  const maybeDiagnostics = diagnostics.length
+    ? fromTypeScriptDiagnostic(diagnostics).items
+    : [];
+
+  if (bundle) {
+    return {
+      diagnostics: maybeDiagnostics,
+      output: state.bundleOutput,
+    } as RuntimeBundleResult;
+  } else {
+    return {
+      diagnostics: maybeDiagnostics,
+      emitMap: state.emitMap,
+    } as RuntimeCompileResult;
+  }
+}
+
+
 async function runtimeCompile(
   request: CompilerRequestRuntimeCompile
 ): Promise<RuntimeCompileResult | RuntimeBundleResult> {
@@ -1761,6 +1874,13 @@ async function tsCompilerOnMessage({
       globalThis.postMessage(result);
       break;
     }
+    case CompilerRequestType.RuntimeCompileNew: {
+      const result = await runtimeCompileNew(
+        request as CompilerRequestRuntimeCompileNew
+      );
+      globalThis.postMessage(result);
+      break;
+    }
     case CompilerRequestType.RuntimeTranspile: {
       const result = await runtimeTranspile(
         request as CompilerRequestRuntimeTranspile
diff --git a/cli/module_graph.rs b/cli/module_graph.rs
index 9561c151ed66e3..ea3bad98892fa6 100644
--- a/cli/module_graph.rs
+++ b/cli/module_graph.rs
@@ -114,6 +114,51 @@ impl ModuleGraphLoader {
     }
   }
 
+  pub async fn add_to_graph(
+    &mut self,
+    specifier: &ModuleSpecifier,
+  ) -> Result<(), ErrBox> {
+    self.download_module(specifier.clone(), None)?;
+
+    loop {
+      let load_result = self.pending_downloads.next().await.unwrap();
+      let source_file = load_result?;
+      let spec = ModuleSpecifier::from(source_file.url.clone());
+      self.visit_module(&spec, source_file)?;
+      if self.pending_downloads.is_empty() {
+        break;
+      }
+    }
+
+    Ok(())
+  }
+
+  pub fn build_local_graph(
+    &mut self,
+    root_name: &str,
+    source_map: HashMap<String, String>,
+  ) -> Result<(), ErrBox> {
+    for spec, source_code in source_map {
+      
+    }
+
+    self.download_module(specifier.clone(), None)?;
+
+    loop {
+      let load_result = self.pending_downloads.next().await.unwrap();
+      let source_file = load_result?;
+      let spec = ModuleSpecifier::from(source_file.url.clone());
+      self.visit_module(&spec, source_file)?;
+      if self.pending_downloads.is_empty() {
+        break;
+      }
+    }
+
+    Ok(())
+  }
+
+
+  // TODO: remove
   pub async fn build_graph(
     mut self,
     specifier: &ModuleSpecifier,
@@ -133,6 +178,12 @@ impl ModuleGraphLoader {
     Ok(self.graph.0)
   }
 
+  pub fn get_graph(
+    self,
+  ) -> HashMap<String, ModuleGraphFile> {
+    self.graph.0
+  }
+
   fn download_module(
     &mut self,
     module_specifier: ModuleSpecifier,
@@ -201,6 +252,10 @@ impl ModuleGraphLoader {
             &module_specifier.to_string(),
           )?,
         };
+        self.download_module(
+          type_header.resolved_specifier.clone(),
+          Some(module_specifier.clone()),
+        )?;
         type_headers.push(type_header);
       }
 
diff --git a/cli/msg.rs b/cli/msg.rs
index dc6765776419df..6c8382268b87a6 100644
--- a/cli/msg.rs
+++ b/cli/msg.rs
@@ -39,4 +39,5 @@ pub enum CompilerRequestType {
   RuntimeCompile = 1,
   RuntimeTranspile = 2,
   CompileNew = 3,
+  RuntimeCompileNew = 4,
 }
diff --git a/cli/ops/runtime_compiler.rs b/cli/ops/runtime_compiler.rs
index f3b741861a8668..bb958838669ea5 100644
--- a/cli/ops/runtime_compiler.rs
+++ b/cli/ops/runtime_compiler.rs
@@ -3,7 +3,7 @@ use super::dispatch_json::{Deserialize, JsonOp, Value};
 use crate::futures::FutureExt;
 use crate::op_error::OpError;
 use crate::state::State;
-use crate::tsc::runtime_compile;
+use crate::tsc::new_runtime_compile;
 use crate::tsc::runtime_transpile;
 use deno_core::CoreIsolate;
 use deno_core::ZeroCopyBuf;
@@ -34,7 +34,7 @@ fn op_compile(
   let global_state = s.global_state.clone();
   let permissions = s.permissions.clone();
   let fut = async move {
-    runtime_compile(
+    new_runtime_compile(
       global_state,
       permissions,
       &args.root_name,
diff --git a/cli/tests/compiler_api_test.ts b/cli/tests/compiler_api_test.ts
index cdc2be6d28dbe7..66fe2e704cde76 100644
--- a/cli/tests/compiler_api_test.ts
+++ b/cli/tests/compiler_api_test.ts
@@ -3,20 +3,20 @@
 import { assert, assertEquals } from "../../std/testing/asserts.ts";
 const { compile, transpileOnly, bundle, test } = Deno;
 
-test("compilerApiCompileSources", async function () {
-  const [diagnostics, actual] = await compile("/foo.ts", {
-    "/foo.ts": `import * as bar from "./bar.ts";\n\nconsole.log(bar);\n`,
-    "/bar.ts": `export const bar = "bar";\n`,
-  });
-  assert(diagnostics == null);
-  assert(actual);
-  assertEquals(Object.keys(actual), [
-    "/bar.js.map",
-    "/bar.js",
-    "/foo.js.map",
-    "/foo.js",
-  ]);
-});
+// test("compilerApiCompileSources", async function () {
+//   const [diagnostics, actual] = await compile("/foo.ts", {
+//     "/foo.ts": `import * as bar from "./bar.ts";\n\nconsole.log(bar);\n`,
+//     "/bar.ts": `export const bar = "bar";\n`,
+//   });
+//   assert(diagnostics == null);
+//   assert(actual);
+//   assertEquals(Object.keys(actual), [
+//     "/bar.js.map",
+//     "/bar.js",
+//     "/foo.js.map",
+//     "/foo.js",
+//   ]);
+// });
 
 test("compilerApiCompileNoSources", async function () {
   const [diagnostics, actual] = await compile("./subdir/mod1.ts");
@@ -28,125 +28,125 @@ test("compilerApiCompileNoSources", async function () {
   assert(keys[1].endsWith("print_hello.js"));
 });
 
-test("compilerApiCompileOptions", async function () {
-  const [diagnostics, actual] = await compile(
-    "/foo.ts",
-    {
-      "/foo.ts": `export const foo = "foo";`,
-    },
-    {
-      module: "amd",
-      sourceMap: false,
-    }
-  );
-  assert(diagnostics == null);
-  assert(actual);
-  assertEquals(Object.keys(actual), ["/foo.js"]);
-  assert(actual["/foo.js"].startsWith("define("));
-});
+// test("compilerApiCompileOptions", async function () {
+//   const [diagnostics, actual] = await compile(
+//     "/foo.ts",
+//     {
+//       "/foo.ts": `export const foo = "foo";`,
+//     },
+//     {
+//       module: "amd",
+//       sourceMap: false,
+//     }
+//   );
+//   assert(diagnostics == null);
+//   assert(actual);
+//   assertEquals(Object.keys(actual), ["/foo.js"]);
+//   assert(actual["/foo.js"].startsWith("define("));
+// });
 
-test("compilerApiCompileLib", async function () {
-  const [diagnostics, actual] = await compile(
-    "/foo.ts",
-    {
-      "/foo.ts": `console.log(document.getElementById("foo"));
-        console.log(Deno.args);`,
-    },
-    {
-      lib: ["dom", "es2018", "deno.ns"],
-    }
-  );
-  assert(diagnostics == null);
-  assert(actual);
-  assertEquals(Object.keys(actual), ["/foo.js.map", "/foo.js"]);
-});
+// test("compilerApiCompileLib", async function () {
+//   const [diagnostics, actual] = await compile(
+//     "/foo.ts",
+//     {
+//       "/foo.ts": `console.log(document.getElementById("foo"));
+//         console.log(Deno.args);`,
+//     },
+//     {
+//       lib: ["dom", "es2018", "deno.ns"],
+//     }
+//   );
+//   assert(diagnostics == null);
+//   assert(actual);
+//   assertEquals(Object.keys(actual), ["/foo.js.map", "/foo.js"]);
+// });
 
-test("compilerApiCompileTypes", async function () {
-  const [diagnostics, actual] = await compile(
-    "/foo.ts",
-    {
-      "/foo.ts": `console.log(Foo.bar);`,
-    },
-    {
-      types: ["./subdir/foo_types.d.ts"],
-    }
-  );
-  assert(diagnostics == null);
-  assert(actual);
-  assertEquals(Object.keys(actual), ["/foo.js.map", "/foo.js"]);
-});
+// test("compilerApiCompileTypes", async function () {
+//   const [diagnostics, actual] = await compile(
+//     "/foo.ts",
+//     {
+//       "/foo.ts": `console.log(Foo.bar);`,
+//     },
+//     {
+//       types: ["./subdir/foo_types.d.ts"],
+//     }
+//   );
+//   assert(diagnostics == null);
+//   assert(actual);
+//   assertEquals(Object.keys(actual), ["/foo.js.map", "/foo.js"]);
+// });
 
-test("transpileOnlyApi", async function () {
-  const actual = await transpileOnly({
-    "foo.ts": `export enum Foo { Foo, Bar, Baz };\n`,
-  });
-  assert(actual);
-  assertEquals(Object.keys(actual), ["foo.ts"]);
-  assert(actual["foo.ts"].source.startsWith("export var Foo;"));
-  assert(actual["foo.ts"].map);
-});
+// test("transpileOnlyApi", async function () {
+//   const actual = await transpileOnly({
+//     "foo.ts": `export enum Foo { Foo, Bar, Baz };\n`,
+//   });
+//   assert(actual);
+//   assertEquals(Object.keys(actual), ["foo.ts"]);
+//   assert(actual["foo.ts"].source.startsWith("export var Foo;"));
+//   assert(actual["foo.ts"].map);
+// });
 
-test("transpileOnlyApiConfig", async function () {
-  const actual = await transpileOnly(
-    {
-      "foo.ts": `export enum Foo { Foo, Bar, Baz };\n`,
-    },
-    {
-      sourceMap: false,
-      module: "amd",
-    }
-  );
-  assert(actual);
-  assertEquals(Object.keys(actual), ["foo.ts"]);
-  assert(actual["foo.ts"].source.startsWith("define("));
-  assert(actual["foo.ts"].map == null);
-});
+// test("transpileOnlyApiConfig", async function () {
+//   const actual = await transpileOnly(
+//     {
+//       "foo.ts": `export enum Foo { Foo, Bar, Baz };\n`,
+//     },
+//     {
+//       sourceMap: false,
+//       module: "amd",
+//     }
+//   );
+//   assert(actual);
+//   assertEquals(Object.keys(actual), ["foo.ts"]);
+//   assert(actual["foo.ts"].source.startsWith("define("));
+//   assert(actual["foo.ts"].map == null);
+// });
 
-test("bundleApiSources", async function () {
-  const [diagnostics, actual] = await bundle("/foo.ts", {
-    "/foo.ts": `export * from "./bar.ts";\n`,
-    "/bar.ts": `export const bar = "bar";\n`,
-  });
-  assert(diagnostics == null);
-  assert(actual.includes(`__instantiate("foo")`));
-  assert(actual.includes(`__exp["bar"]`));
-});
+// test("bundleApiSources", async function () {
+//   const [diagnostics, actual] = await bundle("/foo.ts", {
+//     "/foo.ts": `export * from "./bar.ts";\n`,
+//     "/bar.ts": `export const bar = "bar";\n`,
+//   });
+//   assert(diagnostics == null);
+//   assert(actual.includes(`__instantiate("foo")`));
+//   assert(actual.includes(`__exp["bar"]`));
+// });
 
-test("bundleApiNoSources", async function () {
-  const [diagnostics, actual] = await bundle("./subdir/mod1.ts");
-  assert(diagnostics == null);
-  assert(actual.includes(`__instantiate("mod1")`));
-  assert(actual.includes(`__exp["printHello3"]`));
-});
+// test("bundleApiNoSources", async function () {
+//   const [diagnostics, actual] = await bundle("./subdir/mod1.ts");
+//   assert(diagnostics == null);
+//   assert(actual.includes(`__instantiate("mod1")`));
+//   assert(actual.includes(`__exp["printHello3"]`));
+// });
 
-test("bundleApiConfig", async function () {
-  const [diagnostics, actual] = await bundle(
-    "/foo.ts",
-    {
-      "/foo.ts": `// random comment\nexport * from "./bar.ts";\n`,
-      "/bar.ts": `export const bar = "bar";\n`,
-    },
-    {
-      removeComments: true,
-    }
-  );
-  assert(diagnostics == null);
-  assert(!actual.includes(`random`));
-});
+// test("bundleApiConfig", async function () {
+//   const [diagnostics, actual] = await bundle(
+//     "/foo.ts",
+//     {
+//       "/foo.ts": `// random comment\nexport * from "./bar.ts";\n`,
+//       "/bar.ts": `export const bar = "bar";\n`,
+//     },
+//     {
+//       removeComments: true,
+//     }
+//   );
+//   assert(diagnostics == null);
+//   assert(!actual.includes(`random`));
+// });
 
-test("bundleApiJsModules", async function () {
-  const [diagnostics, actual] = await bundle("/foo.js", {
-    "/foo.js": `export * from "./bar.js";\n`,
-    "/bar.js": `export const bar = "bar";\n`,
-  });
-  assert(diagnostics == null);
-  assert(actual.includes(`System.register("bar",`));
-});
+// test("bundleApiJsModules", async function () {
+//   const [diagnostics, actual] = await bundle("/foo.js", {
+//     "/foo.js": `export * from "./bar.js";\n`,
+//     "/bar.js": `export const bar = "bar";\n`,
+//   });
+//   assert(diagnostics == null);
+//   assert(actual.includes(`System.register("bar",`));
+// });
 
-test("diagnosticsTest", async function () {
-  const [diagnostics] = await compile("/foo.ts", {
-    "/foo.ts": `document.getElementById("foo");`,
-  });
-  assert(Array.isArray(diagnostics));
-  assert(diagnostics.length === 1);
-});
+// test("diagnosticsTest", async function () {
+//   const [diagnostics] = await compile("/foo.ts", {
+//     "/foo.ts": `document.getElementById("foo");`,
+//   });
+//   assert(Array.isArray(diagnostics));
+//   assert(diagnostics.length === 1);
+// });
diff --git a/cli/tsc.rs b/cli/tsc.rs
index 81d8608fb0887b..d60fc4f4d52ba0 100644
--- a/cli/tsc.rs
+++ b/cli/tsc.rs
@@ -891,6 +891,87 @@ async fn execute_in_thread(
 }
 
 /// This function is used by `Deno.compile()` and `Deno.bundle()` APIs.
+pub async fn new_runtime_compile<S: BuildHasher>(
+  global_state: GlobalState,
+  permissions: Permissions,
+  root_name: &str,
+  sources: &Option<HashMap<String, String, S>>,
+  bundle: bool,
+  maybe_options: &Option<String>,
+) -> Result<Value, OpError> {
+  let mut root_names = vec![];
+  let mut module_graph_loader = ModuleGraphLoader::new(
+    global_state.file_fetcher.clone(),
+    None,
+    permissions.clone(),
+    false,
+  );
+
+  if let Some(s_map) = sources {
+    module_graph_loader.build_local_graph(root_name, s_map)?;
+  } else {
+    let module_specifier = ModuleSpecifier::resolve_import(root_name, "<unknown>")?;
+    root_names.push(module_specifier.to_string());
+    module_graph_loader.add_to_graph(&module_specifier).await?;
+      
+    
+    // TODO: download all additional files from TSconfig and add them to root_names
+    if let Some(options) = maybe_options {
+      let options_json: serde_json::Value = serde_json::from_str(options)?;
+      if let Some(types_option) = options_json.get("types") {
+        let types_arr = types_option.as_array().expect("types is not an array");
+
+        for type_value in types_arr {
+          let type_str = type_value.as_str().expect("type is not a string").to_string();
+          let type_specifier = ModuleSpecifier::resolve_url(&type_str)?;
+          module_graph_loader.add_to_graph(&type_specifier).await?;
+          root_names.push(type_specifier.to_string())
+        }
+      }
+    }
+  }
+  
+  let module_graph = module_graph_loader.get_graph();
+  let module_graph_json =
+    serde_json::to_value(module_graph).expect("Failed to serialize data");
+
+  let req_msg = json!({
+    "type": msg::CompilerRequestType::RuntimeCompileNew as i32,
+    "target": "runtime",
+    "rootNames": root_names,
+    "sourceFileMap": module_graph_json,
+    "options": maybe_options,
+    "bundle": bundle,
+    "unstable": global_state.flags.unstable,
+  })
+  .to_string()
+  .into_boxed_str()
+  .into_boxed_bytes();
+
+  let compiler = global_state.ts_compiler.clone();
+
+  let msg = execute_in_thread(global_state, permissions, req_msg).await?;
+  let json_str = std::str::from_utf8(&msg).unwrap();
+
+  // TODO(bartlomieju): factor `bundle` path into separate function `runtime_bundle`
+  if bundle {
+    let _response: RuntimeBundleResponse = serde_json::from_str(json_str)?;
+    return Ok(serde_json::from_str::<Value>(json_str).unwrap());
+  }
+
+  let response: RuntimeCompileResponse = serde_json::from_str(json_str)?;
+
+  if response.diagnostics.is_empty() && sources.is_none() {
+    compiler.cache_emitted_files(response.emit_map)?;
+  }
+
+  // We're returning `Ok()` instead of `Err()` because it's not runtime
+  // error if there were diagnostics produces; we want to let user handle
+  // diagnostics in the runtime.
+  Ok(serde_json::from_str::<Value>(json_str).unwrap())
+}
+
+#[allow(unused)]
 pub async fn runtime_compile<S: BuildHasher>(
   global_state: GlobalState,
   permissions: Permissions,

From 5fccde9b47eb3dde66e389d77244f18f2e66b016 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= <biwanczuk@gmail.com>
Date: Sat, 16 May 2020 17:00:04 +0200
Subject: [PATCH 29/47] runtime compiler tests passing

---
 cli/js/compiler.ts             |  77 +++++++++-
 cli/module_graph.rs            | 120 +++++++++++++---
 cli/tests/compiler_api_test.ts | 254 ++++++++++++++++-----------------
 cli/tsc.rs                     |  38 ++---
 4 files changed, 323 insertions(+), 166 deletions(-)

diff --git a/cli/js/compiler.ts b/cli/js/compiler.ts
index 7ade3d500f7e7b..147026a8b5f756 100644
--- a/cli/js/compiler.ts
+++ b/cli/js/compiler.ts
@@ -546,6 +546,11 @@ class Host implements ts.CompilerHost {
     return moduleNames.map((specifier) => {
       const maybeUrl = SourceFile.getResolvedUrl(specifier, containingFile);
 
+      util.log("compiler::host.resolveModulenames receiverd", {
+        maybeUrl,
+        sf: SourceFile.getCached(maybeUrl!),
+      });
+
       let sourceFile: SourceFile | undefined = undefined;
 
       if (specifier.startsWith(ASSETS)) {
@@ -756,6 +761,58 @@ async function processImports(
   return resolvedSources;
 }
 
+function buildLocalSourceFileCache(
+  sourceFileMap: Record<string, SourceFileMapEntry>
+): void {
+  for (const entry of Object.values(sourceFileMap)) {
+    assert(entry.sourceCode.length > 0);
+    SourceFile.addToCache({
+      url: entry.url,
+      filename: entry.url,
+      mediaType: getMediaType(entry.url),
+      sourceCode: entry.sourceCode,
+    });
+
+    for (const importDesc of entry.imports) {
+      let mappedUrl = importDesc.resolvedSpecifier;
+      const importedFile = sourceFileMap[importDesc.resolvedSpecifier];
+      assert(importedFile);
+      const isJsOrJsx =
+        importedFile.mediaType === MediaType.JavaScript ||
+        importedFile.mediaType === MediaType.JSX;
+      // If JS or JSX perform substitution for types if available
+      if (isJsOrJsx) {
+        if (importedFile.typeHeaders.length > 0) {
+          const typeHeaders = importedFile.typeHeaders[0];
+          mappedUrl = typeHeaders.resolvedSpecifier;
+        } else if (importDesc.resolvedTypeDirective) {
+          mappedUrl = importDesc.resolvedTypeDirective;
+        } else if (importedFile.typesDirectives.length > 0) {
+          const typeDirective = importedFile.typesDirectives[0];
+          mappedUrl = typeDirective.resolvedSpecifier;
+        }
+      }
+
+      mappedUrl = mappedUrl.replace("memory://", "");
+      SourceFile.cacheResolvedUrl(mappedUrl, importDesc.specifier, entry.url);
+    }
+    for (const fileRef of entry.referencedFiles) {
+      SourceFile.cacheResolvedUrl(
+        fileRef.resolvedSpecifier.replace("memory://", ""),
+        fileRef.specifier,
+        entry.url
+      );
+    }
+    for (const fileRef of entry.libDirectives) {
+      SourceFile.cacheResolvedUrl(
+        fileRef.resolvedSpecifier.replace("memory://", ""),
+        fileRef.specifier,
+        entry.url
+      );
+    }
+  }
+}
+
 function buildSourceFileCache(
   sourceFileMap: Record<string, SourceFileMapEntry>
 ): void {
@@ -1326,7 +1383,7 @@ interface CompilerRequestRuntimeCompileNew {
   type: CompilerRequestType.RuntimeCompileNew;
   target: CompilerHostTarget;
   rootNames: string[];
-  sourceFileMap: Record<string, SourceFileMapEntry>,
+  sourceFileMap: Record<string, SourceFileMapEntry>;
   unstable?: boolean;
   bundle?: boolean;
   options?: string;
@@ -1577,10 +1634,17 @@ async function compile(
   return result;
 }
 
-async function runtimeCompileNew(
+function runtimeCompileNew(
   request: CompilerRequestRuntimeCompileNew
-): Promise<RuntimeCompileResult | RuntimeBundleResult> {
-  const { bundle, options, rootNames, target, unstable, sourceFileMap } = request;
+): RuntimeCompileResult | RuntimeBundleResult {
+  const {
+    bundle,
+    options,
+    rootNames,
+    target,
+    unstable,
+    sourceFileMap,
+  } = request;
 
   util.log(">>> runtime compile start", {
     rootNames,
@@ -1595,7 +1659,7 @@ async function runtimeCompileNew(
     convertedOptions = result.options;
   }
 
-  buildSourceFileCache(sourceFileMap);
+  buildLocalSourceFileCache(sourceFileMap);
 
   const state: WriteFileState = {
     type: request.type,
@@ -1677,7 +1741,6 @@ async function runtimeCompileNew(
   }
 }
 
-
 async function runtimeCompile(
   request: CompilerRequestRuntimeCompile
 ): Promise<RuntimeCompileResult | RuntimeBundleResult> {
@@ -1875,7 +1938,7 @@ async function tsCompilerOnMessage({
       break;
     }
     case CompilerRequestType.RuntimeCompileNew: {
-      const result = await runtimeCompileNew(
+      const result = runtimeCompileNew(
         request as CompilerRequestRuntimeCompileNew
       );
       globalThis.postMessage(result);
diff --git a/cli/module_graph.rs b/cli/module_graph.rs
index ea3bad98892fa6..35a996a5c66a2b 100644
--- a/cli/module_graph.rs
+++ b/cli/module_graph.rs
@@ -18,6 +18,7 @@ use futures::FutureExt;
 use serde::Serialize;
 use serde::Serializer;
 use std::collections::HashMap;
+use std::hash::BuildHasher;
 use std::pin::Pin;
 
 fn serialize_module_specifier<S>(
@@ -133,31 +134,120 @@ impl ModuleGraphLoader {
     Ok(())
   }
 
-  pub fn build_local_graph(
+  pub fn build_local_graph<S: BuildHasher>(
     &mut self,
     root_name: &str,
-    source_map: HashMap<String, String>,
+    source_map: &HashMap<String, String, S>,
   ) -> Result<(), ErrBox> {
-    for spec, source_code in source_map {
-      
+    for (spec, source_code) in source_map.iter() {
+      self.visit_memory_module(spec.to_string(), source_code.to_string());
     }
 
-    self.download_module(specifier.clone(), None)?;
+    Ok(())
+  }
 
-    loop {
-      let load_result = self.pending_downloads.next().await.unwrap();
-      let source_file = load_result?;
-      let spec = ModuleSpecifier::from(source_file.url.clone());
-      self.visit_module(&spec, source_file)?;
-      if self.pending_downloads.is_empty() {
-        break;
+  fn visit_memory_module(
+    &mut self,
+    specifier: String,
+    source_code: String,
+  ) -> Result<(), ErrBox> {
+    let mut imports = vec![];
+    let mut referenced_files = vec![];
+    let mut lib_directives = vec![];
+    let mut types_directives = vec![];
+
+    let mut dummy_prefix = false;
+
+    let module_specifier = if specifier.starts_with('/') {
+      dummy_prefix = true;
+      ModuleSpecifier::resolve_url(&format!("memory://{}", specifier))
+    } else {
+      ModuleSpecifier::resolve_url(&specifier)
+    }?;
+
+    let (import_descs, ref_descs) =
+      analyze_dependencies_and_references(&source_code, true)?;
+
+    for import_desc in import_descs {
+      let maybe_resolved =
+        if let Some(import_map) = self.maybe_import_map.as_ref() {
+          import_map
+            .resolve(&import_desc.specifier, &module_specifier.to_string())?
+        } else {
+          None
+        };
+
+      let resolved_specifier = if let Some(resolved) = maybe_resolved {
+        resolved
+      } else {
+        ModuleSpecifier::resolve_import(
+          &import_desc.specifier,
+          &module_specifier.to_string(),
+        )?
+      };
+
+      let resolved_type_directive =
+        if let Some(types_specifier) = import_desc.deno_types.as_ref() {
+          Some(ModuleSpecifier::resolve_import(
+            &types_specifier,
+            &module_specifier.to_string(),
+          )?)
+        } else {
+          None
+        };
+
+      let import_descriptor = ImportDescriptor {
+        specifier: import_desc.specifier.to_string(),
+        resolved_specifier,
+        type_directive: import_desc.deno_types,
+        resolved_type_directive,
+      };
+
+      imports.push(import_descriptor);
+    }
+
+    for ref_desc in ref_descs {
+      let resolved_specifier = ModuleSpecifier::resolve_import(
+        &ref_desc.specifier,
+        &module_specifier.to_string(),
+      )?;
+      let reference_descriptor = ReferenceDescriptor {
+        specifier: ref_desc.specifier.to_string(),
+        resolved_specifier,
+      };
+
+      match ref_desc.kind {
+        TsReferenceKind::Lib => {
+          lib_directives.push(reference_descriptor);
+        }
+        TsReferenceKind::Types => {
+          types_directives.push(reference_descriptor);
+        }
+        TsReferenceKind::Path => {
+          referenced_files.push(reference_descriptor);
+        }
       }
     }
 
+    self.graph.0.insert(
+      module_specifier.to_string(),
+      ModuleGraphFile {
+        specifier: specifier.to_string(),
+        url: specifier.to_string(),
+        filename: specifier,
+        // ignored, it's set in TS worker
+        media_type: MediaType::JavaScript as i32,
+        source_code,
+        imports,
+        referenced_files,
+        lib_directives,
+        types_directives,
+        type_headers: vec![],
+      },
+    );
     Ok(())
   }
 
-
   // TODO: remove
   pub async fn build_graph(
     mut self,
@@ -178,9 +268,7 @@ impl ModuleGraphLoader {
     Ok(self.graph.0)
   }
 
-  pub fn get_graph(
-    self,
-  ) -> HashMap<String, ModuleGraphFile> {
+  pub fn get_graph(self) -> HashMap<String, ModuleGraphFile> {
     self.graph.0
   }
 
diff --git a/cli/tests/compiler_api_test.ts b/cli/tests/compiler_api_test.ts
index 66fe2e704cde76..cdc2be6d28dbe7 100644
--- a/cli/tests/compiler_api_test.ts
+++ b/cli/tests/compiler_api_test.ts
@@ -3,20 +3,20 @@
 import { assert, assertEquals } from "../../std/testing/asserts.ts";
 const { compile, transpileOnly, bundle, test } = Deno;
 
-// test("compilerApiCompileSources", async function () {
-//   const [diagnostics, actual] = await compile("/foo.ts", {
-//     "/foo.ts": `import * as bar from "./bar.ts";\n\nconsole.log(bar);\n`,
-//     "/bar.ts": `export const bar = "bar";\n`,
-//   });
-//   assert(diagnostics == null);
-//   assert(actual);
-//   assertEquals(Object.keys(actual), [
-//     "/bar.js.map",
-//     "/bar.js",
-//     "/foo.js.map",
-//     "/foo.js",
-//   ]);
-// });
+test("compilerApiCompileSources", async function () {
+  const [diagnostics, actual] = await compile("/foo.ts", {
+    "/foo.ts": `import * as bar from "./bar.ts";\n\nconsole.log(bar);\n`,
+    "/bar.ts": `export const bar = "bar";\n`,
+  });
+  assert(diagnostics == null);
+  assert(actual);
+  assertEquals(Object.keys(actual), [
+    "/bar.js.map",
+    "/bar.js",
+    "/foo.js.map",
+    "/foo.js",
+  ]);
+});
 
 test("compilerApiCompileNoSources", async function () {
   const [diagnostics, actual] = await compile("./subdir/mod1.ts");
@@ -28,125 +28,125 @@ test("compilerApiCompileNoSources", async function () {
   assert(keys[1].endsWith("print_hello.js"));
 });
 
-// test("compilerApiCompileOptions", async function () {
-//   const [diagnostics, actual] = await compile(
-//     "/foo.ts",
-//     {
-//       "/foo.ts": `export const foo = "foo";`,
-//     },
-//     {
-//       module: "amd",
-//       sourceMap: false,
-//     }
-//   );
-//   assert(diagnostics == null);
-//   assert(actual);
-//   assertEquals(Object.keys(actual), ["/foo.js"]);
-//   assert(actual["/foo.js"].startsWith("define("));
-// });
+test("compilerApiCompileOptions", async function () {
+  const [diagnostics, actual] = await compile(
+    "/foo.ts",
+    {
+      "/foo.ts": `export const foo = "foo";`,
+    },
+    {
+      module: "amd",
+      sourceMap: false,
+    }
+  );
+  assert(diagnostics == null);
+  assert(actual);
+  assertEquals(Object.keys(actual), ["/foo.js"]);
+  assert(actual["/foo.js"].startsWith("define("));
+});
 
-// test("compilerApiCompileLib", async function () {
-//   const [diagnostics, actual] = await compile(
-//     "/foo.ts",
-//     {
-//       "/foo.ts": `console.log(document.getElementById("foo"));
-//         console.log(Deno.args);`,
-//     },
-//     {
-//       lib: ["dom", "es2018", "deno.ns"],
-//     }
-//   );
-//   assert(diagnostics == null);
-//   assert(actual);
-//   assertEquals(Object.keys(actual), ["/foo.js.map", "/foo.js"]);
-// });
+test("compilerApiCompileLib", async function () {
+  const [diagnostics, actual] = await compile(
+    "/foo.ts",
+    {
+      "/foo.ts": `console.log(document.getElementById("foo"));
+        console.log(Deno.args);`,
+    },
+    {
+      lib: ["dom", "es2018", "deno.ns"],
+    }
+  );
+  assert(diagnostics == null);
+  assert(actual);
+  assertEquals(Object.keys(actual), ["/foo.js.map", "/foo.js"]);
+});
 
-// test("compilerApiCompileTypes", async function () {
-//   const [diagnostics, actual] = await compile(
-//     "/foo.ts",
-//     {
-//       "/foo.ts": `console.log(Foo.bar);`,
-//     },
-//     {
-//       types: ["./subdir/foo_types.d.ts"],
-//     }
-//   );
-//   assert(diagnostics == null);
-//   assert(actual);
-//   assertEquals(Object.keys(actual), ["/foo.js.map", "/foo.js"]);
-// });
+test("compilerApiCompileTypes", async function () {
+  const [diagnostics, actual] = await compile(
+    "/foo.ts",
+    {
+      "/foo.ts": `console.log(Foo.bar);`,
+    },
+    {
+      types: ["./subdir/foo_types.d.ts"],
+    }
+  );
+  assert(diagnostics == null);
+  assert(actual);
+  assertEquals(Object.keys(actual), ["/foo.js.map", "/foo.js"]);
+});
 
-// test("transpileOnlyApi", async function () {
-//   const actual = await transpileOnly({
-//     "foo.ts": `export enum Foo { Foo, Bar, Baz };\n`,
-//   });
-//   assert(actual);
-//   assertEquals(Object.keys(actual), ["foo.ts"]);
-//   assert(actual["foo.ts"].source.startsWith("export var Foo;"));
-//   assert(actual["foo.ts"].map);
-// });
+test("transpileOnlyApi", async function () {
+  const actual = await transpileOnly({
+    "foo.ts": `export enum Foo { Foo, Bar, Baz };\n`,
+  });
+  assert(actual);
+  assertEquals(Object.keys(actual), ["foo.ts"]);
+  assert(actual["foo.ts"].source.startsWith("export var Foo;"));
+  assert(actual["foo.ts"].map);
+});
 
-// test("transpileOnlyApiConfig", async function () {
-//   const actual = await transpileOnly(
-//     {
-//       "foo.ts": `export enum Foo { Foo, Bar, Baz };\n`,
-//     },
-//     {
-//       sourceMap: false,
-//       module: "amd",
-//     }
-//   );
-//   assert(actual);
-//   assertEquals(Object.keys(actual), ["foo.ts"]);
-//   assert(actual["foo.ts"].source.startsWith("define("));
-//   assert(actual["foo.ts"].map == null);
-// });
+test("transpileOnlyApiConfig", async function () {
+  const actual = await transpileOnly(
+    {
+      "foo.ts": `export enum Foo { Foo, Bar, Baz };\n`,
+    },
+    {
+      sourceMap: false,
+      module: "amd",
+    }
+  );
+  assert(actual);
+  assertEquals(Object.keys(actual), ["foo.ts"]);
+  assert(actual["foo.ts"].source.startsWith("define("));
+  assert(actual["foo.ts"].map == null);
+});
 
-// test("bundleApiSources", async function () {
-//   const [diagnostics, actual] = await bundle("/foo.ts", {
-//     "/foo.ts": `export * from "./bar.ts";\n`,
-//     "/bar.ts": `export const bar = "bar";\n`,
-//   });
-//   assert(diagnostics == null);
-//   assert(actual.includes(`__instantiate("foo")`));
-//   assert(actual.includes(`__exp["bar"]`));
-// });
+test("bundleApiSources", async function () {
+  const [diagnostics, actual] = await bundle("/foo.ts", {
+    "/foo.ts": `export * from "./bar.ts";\n`,
+    "/bar.ts": `export const bar = "bar";\n`,
+  });
+  assert(diagnostics == null);
+  assert(actual.includes(`__instantiate("foo")`));
+  assert(actual.includes(`__exp["bar"]`));
+});
 
-// test("bundleApiNoSources", async function () {
-//   const [diagnostics, actual] = await bundle("./subdir/mod1.ts");
-//   assert(diagnostics == null);
-//   assert(actual.includes(`__instantiate("mod1")`));
-//   assert(actual.includes(`__exp["printHello3"]`));
-// });
+test("bundleApiNoSources", async function () {
+  const [diagnostics, actual] = await bundle("./subdir/mod1.ts");
+  assert(diagnostics == null);
+  assert(actual.includes(`__instantiate("mod1")`));
+  assert(actual.includes(`__exp["printHello3"]`));
+});
 
-// test("bundleApiConfig", async function () {
-//   const [diagnostics, actual] = await bundle(
-//     "/foo.ts",
-//     {
-//       "/foo.ts": `// random comment\nexport * from "./bar.ts";\n`,
-//       "/bar.ts": `export const bar = "bar";\n`,
-//     },
-//     {
-//       removeComments: true,
-//     }
-//   );
-//   assert(diagnostics == null);
-//   assert(!actual.includes(`random`));
-// });
+test("bundleApiConfig", async function () {
+  const [diagnostics, actual] = await bundle(
+    "/foo.ts",
+    {
+      "/foo.ts": `// random comment\nexport * from "./bar.ts";\n`,
+      "/bar.ts": `export const bar = "bar";\n`,
+    },
+    {
+      removeComments: true,
+    }
+  );
+  assert(diagnostics == null);
+  assert(!actual.includes(`random`));
+});
 
-// test("bundleApiJsModules", async function () {
-//   const [diagnostics, actual] = await bundle("/foo.js", {
-//     "/foo.js": `export * from "./bar.js";\n`,
-//     "/bar.js": `export const bar = "bar";\n`,
-//   });
-//   assert(diagnostics == null);
-//   assert(actual.includes(`System.register("bar",`));
-// });
+test("bundleApiJsModules", async function () {
+  const [diagnostics, actual] = await bundle("/foo.js", {
+    "/foo.js": `export * from "./bar.js";\n`,
+    "/bar.js": `export const bar = "bar";\n`,
+  });
+  assert(diagnostics == null);
+  assert(actual.includes(`System.register("bar",`));
+});
 
-// test("diagnosticsTest", async function () {
-//   const [diagnostics] = await compile("/foo.ts", {
-//     "/foo.ts": `document.getElementById("foo");`,
-//   });
-//   assert(Array.isArray(diagnostics));
-//   assert(diagnostics.length === 1);
-// });
+test("diagnosticsTest", async function () {
+  const [diagnostics] = await compile("/foo.ts", {
+    "/foo.ts": `document.getElementById("foo");`,
+  });
+  assert(Array.isArray(diagnostics));
+  assert(diagnostics.length === 1);
+});
diff --git a/cli/tsc.rs b/cli/tsc.rs
index d60fc4f4d52ba0..1bf38e855cbccc 100644
--- a/cli/tsc.rs
+++ b/cli/tsc.rs
@@ -908,33 +908,38 @@ pub async fn new_runtime_compile<S: BuildHasher>(
   );
 
   if let Some(s_map) = sources {
+    root_names.push(root_name.to_string());
     module_graph_loader.build_local_graph(root_name, s_map)?;
   } else {
-    let module_specifier = ModuleSpecifier::resolve_import(root_name, "<unknown>")?;
+    let module_specifier =
+      ModuleSpecifier::resolve_import(root_name, "<unknown>")?;
     root_names.push(module_specifier.to_string());
     module_graph_loader.add_to_graph(&module_specifier).await?;
-      
-    
-    // TODO: download all additional files from TSconfig and add them to root_names
-    if let Some(options) = maybe_options {
-      let options_json: serde_json::Value = serde_json::from_str(options)?;
-      if let Some(types_option) = options_json.get("types") {
-        let types_arr = types_option.as_array().expect("types is not an array");
-
-        for type_value in types_arr {
-          let type_str = type_value.as_str().expect("type is not a string").to_string();
-          let type_specifier = ModuleSpecifier::resolve_url(&type_str)?;
-          module_graph_loader.add_to_graph(&type_specifier).await?;
-          root_names.push(type_specifier.to_string())
-        }
+  }
+
+  // TODO: download all additional files from TSconfig and add them to root_names
+  if let Some(options) = maybe_options {
+    let options_json: serde_json::Value = serde_json::from_str(options)?;
+    if let Some(types_option) = options_json.get("types") {
+      let types_arr = types_option.as_array().expect("types is not an array");
+
+      for type_value in types_arr {
+        let type_str = type_value
+          .as_str()
+          .expect("type is not a string")
+          .to_string();
+        let type_specifier = ModuleSpecifier::resolve_url_or_path(&type_str)?;
+        module_graph_loader.add_to_graph(&type_specifier).await?;
+        root_names.push(type_specifier.to_string())
       }
     }
   }
-  
+
   let module_graph = module_graph_loader.get_graph();
   let module_graph_json =
     serde_json::to_value(module_graph).expect("Failed to serialize data");
 
+  // eprintln!("source graph {:#?}", module_graph_json);
   let req_msg = json!({
     "type": msg::CompilerRequestType::RuntimeCompileNew as i32,
     "target": "runtime",
@@ -965,6 +970,7 @@ pub async fn new_runtime_compile<S: BuildHasher>(
     compiler.cache_emitted_files(response.emit_map)?;
   }
 
+  // eprintln!("returned {:#?}", json_str);
   // We're returning `Ok()` instead of `Err()` because it's not runtime
   // error if there were diagnostics produces; we want to let user handle
   // diagnostics in the runtime.

From e26cc3e3fe80e66024831227e387b15e922dc422 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= <biwanczuk@gmail.com>
Date: Sat, 16 May 2020 17:25:27 +0200
Subject: [PATCH 30/47] fix bundles

---
 cli/js/compiler.ts          |  4 +--
 cli/main.rs                 | 12 ++++++++-
 cli/ops/runtime_compiler.rs |  4 +--
 cli/tsc.rs                  | 51 +++----------------------------------
 4 files changed, 18 insertions(+), 53 deletions(-)

diff --git a/cli/js/compiler.ts b/cli/js/compiler.ts
index 147026a8b5f756..65545ad207746d 100644
--- a/cli/js/compiler.ts
+++ b/cli/js/compiler.ts
@@ -1502,7 +1502,7 @@ function compileNew(request: CompilerRequestCompileNew): CompileResult {
   util.log("time in internal TS", end - start);
   let bundleOutput = undefined;
 
-  if (bundle) {
+  if (diagnostics && diagnostics.length === 0 && bundle) {
     assert(state.bundleOutput);
     bundleOutput = state.bundleOutput;
   }
@@ -1614,7 +1614,7 @@ async function compile(
 
   let bundleOutput = undefined;
 
-  if (bundle) {
+  if (diagnostics && diagnostics.length === 0 && bundle) {
     assert(state.bundleOutput);
     bundleOutput = state.bundleOutput;
   }
diff --git a/cli/main.rs b/cli/main.rs
index cadb76fb4b108b..5985a1040aec72 100644
--- a/cli/main.rs
+++ b/cli/main.rs
@@ -70,6 +70,7 @@ pub use dprint_plugin_typescript::swc_ecma_parser;
 use crate::doc::parser::DocFileLoader;
 use crate::file_fetcher::SourceFile;
 use crate::file_fetcher::SourceFileFetcher;
+use crate::fs as deno_fs;
 use crate::global_state::GlobalState;
 use crate::msg::MediaType;
 use crate::op_error::OpError;
@@ -385,7 +386,16 @@ async fn bundle_command(
   source_file: String,
   out_file: Option<PathBuf>,
 ) -> Result<(), ErrBox> {
-  let module_name = ModuleSpecifier::resolve_url_or_path(&source_file)?;
+  let mut module_name = ModuleSpecifier::resolve_url_or_path(&source_file)?;
+  let url = module_name.as_url();
+
+  // TODO(bartlomieju): fix this hack in ModuleSpecifier
+  if url.scheme() == "file" {
+    let a = deno_fs::normalize_path(&url.to_file_path().unwrap());
+    let u = Url::from_file_path(a).unwrap();
+    module_name = ModuleSpecifier::from(u)
+  }
+
   let global_state = GlobalState::new(flags)?;
   debug!(">>>>> bundle START");
   let bundle_result = global_state
diff --git a/cli/ops/runtime_compiler.rs b/cli/ops/runtime_compiler.rs
index bb958838669ea5..f3b741861a8668 100644
--- a/cli/ops/runtime_compiler.rs
+++ b/cli/ops/runtime_compiler.rs
@@ -3,7 +3,7 @@ use super::dispatch_json::{Deserialize, JsonOp, Value};
 use crate::futures::FutureExt;
 use crate::op_error::OpError;
 use crate::state::State;
-use crate::tsc::new_runtime_compile;
+use crate::tsc::runtime_compile;
 use crate::tsc::runtime_transpile;
 use deno_core::CoreIsolate;
 use deno_core::ZeroCopyBuf;
@@ -34,7 +34,7 @@ fn op_compile(
   let global_state = s.global_state.clone();
   let permissions = s.permissions.clone();
   let fut = async move {
-    new_runtime_compile(
+    runtime_compile(
       global_state,
       permissions,
       &args.root_name,
diff --git a/cli/tsc.rs b/cli/tsc.rs
index 1bf38e855cbccc..1651651f4bfe50 100644
--- a/cli/tsc.rs
+++ b/cli/tsc.rs
@@ -379,7 +379,7 @@ impl TsCompiler {
     let cwd = std::env::current_dir().unwrap();
     let j = match (compiler_config.path, compiler_config.content) {
       (Some(config_path), Some(config_data)) => json!({
-        "type": msg::CompilerRequestType::Compile as i32,
+        "type": msg::CompilerRequestType::CompileNew as i32,
         "target": target,
         "rootNames": root_names,
         "bundle": bundle,
@@ -390,7 +390,7 @@ impl TsCompiler {
         "sourceFileMap": module_graph_json,
       }),
       _ => json!({
-        "type": msg::CompilerRequestType::Compile as i32,
+        "type": msg::CompilerRequestType::CompileNew as i32,
         "target": target,
         "rootNames": root_names,
         "bundle": bundle,
@@ -891,7 +891,7 @@ async fn execute_in_thread(
 }
 
 /// This function is used by `Deno.compile()` and `Deno.bundle()` APIs.
-pub async fn new_runtime_compile<S: BuildHasher>(
+pub async fn runtime_compile<S: BuildHasher>(
   global_state: GlobalState,
   permissions: Permissions,
   root_name: &str,
@@ -977,51 +977,6 @@ pub async fn new_runtime_compile<S: BuildHasher>(
   Ok(serde_json::from_str::<Value>(json_str).unwrap())
 }
 
-#[allow(unused)]
-pub async fn runtime_compile<S: BuildHasher>(
-  global_state: GlobalState,
-  permissions: Permissions,
-  root_name: &str,
-  sources: &Option<HashMap<String, String, S>>,
-  bundle: bool,
-  options: &Option<String>,
-) -> Result<Value, OpError> {
-  let req_msg = json!({
-    "type": msg::CompilerRequestType::RuntimeCompile as i32,
-    "target": "runtime",
-    "rootName": root_name,
-    "sources": sources,
-    "options": options,
-    "bundle": bundle,
-    "unstable": global_state.flags.unstable,
-  })
-  .to_string()
-  .into_boxed_str()
-  .into_boxed_bytes();
-
-  let compiler = global_state.ts_compiler.clone();
-
-  let msg = execute_in_thread(global_state, permissions, req_msg).await?;
-  let json_str = std::str::from_utf8(&msg).unwrap();
-
-  // TODO(bartlomieju): factor `bundle` path into separate function `runtime_bundle`
-  if bundle {
-    let _response: RuntimeBundleResponse = serde_json::from_str(json_str)?;
-    return Ok(serde_json::from_str::<Value>(json_str).unwrap());
-  }
-
-  let response: RuntimeCompileResponse = serde_json::from_str(json_str)?;
-
-  if response.diagnostics.is_empty() && sources.is_none() {
-    compiler.cache_emitted_files(response.emit_map)?;
-  }
-
-  // We're returning `Ok()` instead of `Err()` because it's not runtime
-  // error if there were diagnostics produces; we want to let user handle
-  // diagnostics in the runtime.
-  Ok(serde_json::from_str::<Value>(json_str).unwrap())
-}
-
 /// This function is used by `Deno.transpileOnly()` API.
 pub async fn runtime_transpile<S: BuildHasher>(
   global_state: GlobalState,

From a28794ebac98d97215dd0bc6baf6c985aa3c2f83 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= <biwanczuk@gmail.com>
Date: Sat, 16 May 2020 17:38:36 +0200
Subject: [PATCH 31/47] cleanup compiler

---
 cli/js/compiler.ts  | 576 +-------------------------------------------
 cli/module_graph.rs |   3 +
 cli/msg.rs          |   2 -
 cli/ops/compiler.rs | 144 +----------
 cli/tsc.rs          |  10 +-
 5 files changed, 18 insertions(+), 717 deletions(-)

diff --git a/cli/js/compiler.ts b/cli/js/compiler.ts
index 65545ad207746d..8e77ada67f5a41 100644
--- a/cli/js/compiler.ts
+++ b/cli/js/compiler.ts
@@ -17,39 +17,12 @@ import { CompilerOptions } from "./compiler_options.ts";
 import { Diagnostic, DiagnosticItem } from "./diagnostics.ts";
 import { fromTypeScriptDiagnostic } from "./diagnostics_util.ts";
 import { TranspileOnlyResult } from "./ops/runtime_compiler.ts";
-import { sendAsync, sendSync } from "./ops/dispatch_json.ts";
 import { bootstrapWorkerRuntime } from "./runtime_worker.ts";
-import { assert, log } from "./util.ts";
+import { assert } from "./util.ts";
 import * as util from "./util.ts";
 import { TextDecoder, TextEncoder } from "./web/text_encoding.ts";
 import { core } from "./core.ts";
 
-export function resolveModules(
-  specifiers: string[],
-  referrer?: string
-): string[] {
-  util.log("compiler::resolveModules", { specifiers, referrer });
-  return sendSync("op_resolve_modules", { specifiers, referrer });
-}
-
-export function fetchSourceFiles(
-  specifiers: string[],
-  referrer?: string
-): Promise<
-  Array<{
-    url: string;
-    filename: string;
-    mediaType: number;
-    sourceCode: string;
-  }>
-> {
-  util.log("compiler::fetchSourceFiles", { specifiers, referrer });
-  return sendAsync("op_fetch_source_files", {
-    specifiers,
-    referrer,
-  });
-}
-
 const encoder = new TextEncoder();
 const decoder = new TextDecoder();
 
@@ -263,75 +236,6 @@ class SourceFile {
     this.extension = getExtension(this.url, this.mediaType);
   }
 
-  imports(processJsImports: boolean): SourceFileSpecifierMap[] {
-    if (this.processed) {
-      throw new Error("SourceFile has already been processed.");
-    }
-    assert(this.sourceCode != null);
-    // we shouldn't process imports for files which contain the nocheck pragma
-    // (like bundles)
-    if (this.sourceCode.match(/\/{2}\s+@ts-nocheck/)) {
-      log(`Skipping imports for "${this.filename}"`);
-      return [];
-    }
-
-    const readImportFiles = true;
-    const isJsOrJsx =
-      this.mediaType === MediaType.JavaScript ||
-      this.mediaType === MediaType.JSX;
-    const detectJsImports = isJsOrJsx;
-
-    const preProcessedFileInfo = ts.preProcessFile(
-      this.sourceCode,
-      readImportFiles,
-      detectJsImports
-    );
-    this.processed = true;
-    const files: SourceFileSpecifierMap[] = [];
-
-    function process(references: Array<{ fileName: string }>): void {
-      for (const { fileName } of references) {
-        files.push({ original: fileName, mapped: fileName });
-      }
-    }
-
-    const {
-      importedFiles,
-      referencedFiles,
-      libReferenceDirectives,
-      typeReferenceDirectives,
-    } = preProcessedFileInfo;
-    const typeDirectives = parseTypeDirectives(this.sourceCode);
-
-    if (typeDirectives) {
-      for (const importedFile of importedFiles) {
-        // If there's a type directive for current processed file; then we provide
-        // different `mapped` specifier.
-        const mappedModuleName = getMappedModuleName(
-          importedFile,
-          typeDirectives
-        );
-        files.push({
-          original: importedFile.fileName,
-          mapped: mappedModuleName ?? importedFile.fileName,
-        });
-      }
-    } else if (processJsImports || !isJsOrJsx) {
-      process(importedFiles);
-    }
-    process(referencedFiles);
-    // built in libs comes across as `"dom"` for example, and should be filtered
-    // out during pre-processing as they are either already cached or they will
-    // be lazily fetched by the compiler host.  Ones that contain full files are
-    // not filtered out and will be fetched as normal.
-    const filteredLibs = libReferenceDirectives.filter(
-      ({ fileName }) => !ts.libMap.has(fileName.toLowerCase())
-    );
-    process(filteredLibs);
-    process(typeReferenceDirectives);
-    return files;
-  }
-
   static addToCache(json: SourceFileJson): SourceFile {
     if (SOURCE_FILE_CACHE.has(json.url)) {
       throw new TypeError("SourceFile already exists");
@@ -637,25 +541,6 @@ const TS_SNAPSHOT_PROGRAM = ts.createProgram({
 // This function is called only during snapshotting process
 const SYSTEM_LOADER = getAsset("system_loader.js");
 
-function resolveSpecifier(specifier: string, referrer: string): string {
-  // The resolveModules op only handles fully qualified URLs for referrer.
-  // However we will have cases where referrer is "/foo.ts". We add this dummy
-  // prefix "file://" in order to use the op.
-  // TODO(ry) Maybe we should perhaps ModuleSpecifier::resolve_import() to
-  // handle this situation.
-  let dummyPrefix = false;
-  const prefix = "file://";
-  if (referrer.startsWith("/")) {
-    dummyPrefix = true;
-    referrer = prefix + referrer;
-  }
-  let r = resolveModules([specifier], referrer)[0];
-  if (dummyPrefix) {
-    r = r.replace(prefix, "");
-  }
-  return r;
-}
-
 function getMediaType(filename: string): MediaType {
   const maybeExtension = /\.([a-zA-Z]+)$/.exec(filename);
   if (!maybeExtension) {
@@ -680,87 +565,6 @@ function getMediaType(filename: string): MediaType {
   }
 }
 
-function processLocalImports(
-  sources: Record<string, string>,
-  specifiers: SourceFileSpecifierMap[],
-  referrer?: string,
-  processJsImports = false
-): string[] {
-  if (!specifiers.length) {
-    return [];
-  }
-  const moduleNames = specifiers.map((specifierMap) => {
-    if (referrer) {
-      return resolveSpecifier(specifierMap.mapped, referrer);
-    } else {
-      return specifierMap.mapped;
-    }
-  });
-
-  for (let i = 0; i < moduleNames.length; i++) {
-    const moduleName = moduleNames[i];
-    const specifierMap = specifiers[i];
-    assert(moduleName in sources, `Missing module in sources: "${moduleName}"`);
-    let sourceFile = SourceFile.getCached(moduleName);
-    if (typeof sourceFile === "undefined") {
-      sourceFile = SourceFile.addToCache({
-        url: moduleName,
-        filename: moduleName,
-        sourceCode: sources[moduleName],
-        mediaType: getMediaType(moduleName),
-      });
-    }
-    assert(sourceFile);
-    SourceFile.cacheResolvedUrl(
-      sourceFile.url,
-      specifierMap.original,
-      referrer
-    );
-    if (!sourceFile.processed) {
-      processLocalImports(
-        sources,
-        sourceFile.imports(processJsImports),
-        sourceFile.url,
-        processJsImports
-      );
-    }
-  }
-  return moduleNames;
-}
-
-async function processImports(
-  specifiers: SourceFileSpecifierMap[],
-  referrer?: string,
-  processJsImports = false
-): Promise<string[]> {
-  if (!specifiers.length) {
-    return [];
-  }
-  const sources = specifiers.map(({ mapped }) => mapped);
-  const resolvedSources = resolveModules(sources, referrer);
-  const sourceFiles = await fetchSourceFiles(resolvedSources, referrer);
-  assert(sourceFiles.length === specifiers.length);
-  for (let i = 0; i < sourceFiles.length; i++) {
-    const specifierMap = specifiers[i];
-    const sourceFileJson = sourceFiles[i];
-    let sourceFile = SourceFile.getCached(sourceFileJson.url);
-    if (typeof sourceFile === "undefined") {
-      sourceFile = SourceFile.addToCache(sourceFileJson);
-    }
-    assert(sourceFile);
-    SourceFile.cacheResolvedUrl(
-      sourceFile.url,
-      specifierMap.original,
-      referrer
-    );
-    if (!sourceFile.processed) {
-      const sourceFileImports = sourceFile.imports(processJsImports);
-      await processImports(sourceFileImports, sourceFile.url, processJsImports);
-    }
-  }
-  return resolvedSources;
-}
-
 function buildLocalSourceFileCache(
   sourceFileMap: Record<string, SourceFileMapEntry>
 ): void {
@@ -864,76 +668,6 @@ function buildSourceFileCache(
   }
 }
 
-interface FileReference {
-  fileName: string;
-  pos: number;
-  end: number;
-}
-
-function getMappedModuleName(
-  source: FileReference,
-  typeDirectives: Map<FileReference, string>
-): string | undefined {
-  const { fileName: sourceFileName, pos: sourcePos } = source;
-  for (const [{ fileName, pos }, value] of typeDirectives.entries()) {
-    if (sourceFileName === fileName && sourcePos === pos) {
-      return value;
-    }
-  }
-  return undefined;
-}
-
-const typeDirectiveRegEx = /@deno-types\s*=\s*(["'])((?:(?=(\\?))\3.)*?)\1/gi;
-
-const importExportRegEx = /(?:import|export)(?:\s+|\s+[\s\S]*?from\s+)?(["'])((?:(?=(\\?))\3.)*?)\1/;
-
-function parseTypeDirectives(
-  sourceCode: string | undefined
-): Map<FileReference, string> | undefined {
-  if (!sourceCode) {
-    return;
-  }
-
-  // collect all the directives in the file and their start and end positions
-  const directives: FileReference[] = [];
-  let maybeMatch: RegExpExecArray | null = null;
-  while ((maybeMatch = typeDirectiveRegEx.exec(sourceCode))) {
-    const [matchString, , fileName] = maybeMatch;
-    const { index: pos } = maybeMatch;
-    directives.push({
-      fileName,
-      pos,
-      end: pos + matchString.length,
-    });
-  }
-  if (!directives.length) {
-    return;
-  }
-
-  // work from the last directive backwards for the next `import`/`export`
-  // statement
-  directives.reverse();
-  const results = new Map<FileReference, string>();
-  for (const { end, fileName, pos } of directives) {
-    const searchString = sourceCode.substring(end);
-    const maybeMatch = importExportRegEx.exec(searchString);
-    if (maybeMatch) {
-      const [matchString, , targetFileName] = maybeMatch;
-      const targetPos =
-        end + maybeMatch.index + matchString.indexOf(targetFileName) - 1;
-      const target: FileReference = {
-        fileName: targetFileName,
-        pos: targetPos,
-        end: targetPos + targetFileName.length,
-      };
-      results.set(target, fileName);
-    }
-    sourceCode = sourceCode.substring(0, pos);
-  }
-
-  return results;
-}
-
 interface EmmitedSource {
   // original filename
   filename: string;
@@ -963,8 +697,6 @@ enum CompilerRequestType {
   Compile = 0,
   RuntimeCompile = 1,
   RuntimeTranspile = 2,
-  CompileNew = 3,
-  RuntimeCompileNew = 4,
 }
 
 // TODO(bartlomieju): probably could be defined inline?
@@ -1319,19 +1051,6 @@ function setRootExports(program: ts.Program, rootModule: string): void {
     .map((sym) => sym.getName());
 }
 
-interface CompilerRequestCompile {
-  type: CompilerRequestType.Compile;
-  target: CompilerHostTarget;
-  rootNames: string[];
-  // TODO(ry) add compiler config to this interface.
-  // options: ts.CompilerOptions;
-  configPath?: string;
-  config?: string;
-  unstable: boolean;
-  bundle: boolean;
-  cwd: string;
-}
-
 interface ImportDescriptor {
   specifier: string;
   resolvedSpecifier: string;
@@ -1356,8 +1075,8 @@ interface SourceFileMapEntry {
   typeHeaders: ReferenceDescriptor[];
 }
 
-interface CompilerRequestCompileNew {
-  type: CompilerRequestType.CompileNew;
+interface CompilerRequestCompile {
+  type: CompilerRequestType.Compile;
   target: CompilerHostTarget;
   rootNames: string[];
   configPath?: string;
@@ -1372,16 +1091,6 @@ interface CompilerRequestCompileNew {
 interface CompilerRequestRuntimeCompile {
   type: CompilerRequestType.RuntimeCompile;
   target: CompilerHostTarget;
-  rootName: string;
-  sources?: Record<string, string>;
-  unstable?: boolean;
-  bundle?: boolean;
-  options?: string;
-}
-
-interface CompilerRequestRuntimeCompileNew {
-  type: CompilerRequestType.RuntimeCompileNew;
-  target: CompilerHostTarget;
   rootNames: string[];
   sourceFileMap: Record<string, SourceFileMapEntry>;
   unstable?: boolean;
@@ -1396,10 +1105,8 @@ interface CompilerRequestRuntimeTranspile {
 }
 
 type CompilerRequest =
-  | CompilerRequestCompileNew
   | CompilerRequestCompile
   | CompilerRequestRuntimeCompile
-  | CompilerRequestRuntimeCompileNew
   | CompilerRequestRuntimeTranspile;
 
 interface CompileResult {
@@ -1418,7 +1125,7 @@ interface RuntimeBundleResult {
   diagnostics: DiagnosticItem[];
 }
 
-function compileNew(request: CompilerRequestCompileNew): CompileResult {
+function compile(request: CompilerRequestCompile): CompileResult {
   const {
     bundle,
     config,
@@ -1522,120 +1229,8 @@ function compileNew(request: CompilerRequestCompileNew): CompileResult {
   return result;
 }
 
-async function compile(
-  request: CompilerRequestCompile
-): Promise<CompileResult> {
-  const {
-    bundle,
-    config,
-    configPath,
-    rootNames,
-    target,
-    unstable,
-    cwd,
-  } = request;
-  util.log(">>> compile start", {
-    rootNames,
-    type: CompilerRequestType[request.type],
-  });
-
-  // When a programme is emitted, TypeScript will call `writeFile` with
-  // each file that needs to be emitted.  The Deno compiler host delegates
-  // this, to make it easier to perform the right actions, which vary
-  // based a lot on the request.
-  const state: WriteFileState = {
-    type: request.type,
-    emitMap: {},
-    bundle,
-    host: undefined,
-    rootNames,
-  };
-  let writeFile: WriteFileCallback;
-  if (bundle) {
-    writeFile = createBundleWriteFile(state);
-  } else {
-    writeFile = createCompileWriteFile(state);
-  }
-  const host = (state.host = new Host({
-    bundle,
-    target,
-    writeFile,
-    unstable,
-  }));
-  let diagnostics: readonly ts.Diagnostic[] = [];
-
-  // if there is a configuration supplied, we need to parse that
-  if (config && config.length && configPath) {
-    const configResult = host.configure(cwd, configPath, config);
-    diagnostics = processConfigureResponse(configResult, configPath) || [];
-  }
-
-  // This will recursively analyse all the code for other imports,
-  // requesting those from the privileged side, populating the in memory
-  // cache which will be used by the host, before resolving.
-  const specifiers = rootNames.map((rootName) => {
-    return { original: rootName, mapped: rootName };
-  });
-  const resolvedRootModules = await processImports(
-    specifiers,
-    undefined,
-    bundle || host.getCompilationSettings().checkJs
-  );
-
-  // if there was a configuration and no diagnostics with it, we will continue
-  // to generate the program and possibly emit it.
-  if (diagnostics.length === 0) {
-    const options = host.getCompilationSettings();
-    const program = ts.createProgram({
-      rootNames,
-      options,
-      host,
-      oldProgram: TS_SNAPSHOT_PROGRAM,
-    });
-
-    diagnostics = ts
-      .getPreEmitDiagnostics(program)
-      .filter(({ code }) => !ignoredDiagnostics.includes(code));
-
-    // We will only proceed with the emit if there are no diagnostics.
-    if (diagnostics && diagnostics.length === 0) {
-      if (bundle) {
-        // we only support a single root module when bundling
-        assert(resolvedRootModules.length === 1);
-        setRootExports(program, resolvedRootModules[0]);
-      }
-      const emitResult = program.emit();
-      assert(emitResult.emitSkipped === false, "Unexpected skip of the emit.");
-      // emitResult.diagnostics is `readonly` in TS3.5+ and can't be assigned
-      // without casting.
-      diagnostics = emitResult.diagnostics;
-    }
-  }
-
-  let bundleOutput = undefined;
-
-  if (diagnostics && diagnostics.length === 0 && bundle) {
-    assert(state.bundleOutput);
-    bundleOutput = state.bundleOutput;
-  }
-
-  assert(state.emitMap);
-  const result: CompileResult = {
-    emitMap: state.emitMap,
-    bundleOutput,
-    diagnostics: fromTypeScriptDiagnostic(diagnostics),
-  };
-
-  util.log("<<< compile end", {
-    rootNames,
-    type: CompilerRequestType[request.type],
-  });
-
-  return result;
-}
-
-function runtimeCompileNew(
-  request: CompilerRequestRuntimeCompileNew
+function runtimeCompile(
+  request: CompilerRequestRuntimeCompile
 ): RuntimeCompileResult | RuntimeBundleResult {
   const {
     bundle,
@@ -1741,147 +1336,6 @@ function runtimeCompileNew(
   }
 }
 
-async function runtimeCompile(
-  request: CompilerRequestRuntimeCompile
-): Promise<RuntimeCompileResult | RuntimeBundleResult> {
-  const { bundle, options, rootName, sources, target, unstable } = request;
-
-  util.log(">>> runtime compile start", {
-    rootName,
-    bundle,
-    sources: sources ? Object.keys(sources) : undefined,
-  });
-
-  // resolve the root name, if there are sources, the root name does not
-  // get resolved
-  const resolvedRootName = sources ? rootName : resolveModules([rootName])[0];
-
-  // if there are options, convert them into TypeScript compiler options,
-  // and resolve any external file references
-  let convertedOptions: ts.CompilerOptions | undefined;
-  let additionalFiles: string[] | undefined;
-  if (options) {
-    const result = convertCompilerOptions(options);
-    convertedOptions = result.options;
-    additionalFiles = result.files;
-  }
-
-  const checkJsImports =
-    bundle || (convertedOptions && convertedOptions.checkJs);
-
-  // recursively process imports, loading each file into memory.  If there
-  // are sources, these files are pulled out of the there, otherwise the
-  // files are retrieved from the privileged side
-  const specifiers = [
-    {
-      original: resolvedRootName,
-      mapped: resolvedRootName,
-    },
-  ];
-  const rootNames = sources
-    ? processLocalImports(sources, specifiers, undefined, checkJsImports)
-    : await processImports(specifiers, undefined, checkJsImports);
-
-  if (additionalFiles) {
-    // any files supplied in the configuration are resolved externally,
-    // even if sources are provided
-    const resolvedNames = resolveModules(additionalFiles);
-    const resolvedSpecifiers = resolvedNames.map((rn) => {
-      return {
-        original: rn,
-        mapped: rn,
-      };
-    });
-    const additionalImports = await processImports(
-      resolvedSpecifiers,
-      undefined,
-      checkJsImports
-    );
-    rootNames.push(...additionalImports);
-  }
-
-  const state: WriteFileState = {
-    type: request.type,
-    bundle,
-    host: undefined,
-    rootNames,
-    sources,
-    emitMap: {},
-    bundleOutput: undefined,
-  };
-  let writeFile: WriteFileCallback;
-  if (bundle) {
-    writeFile = createBundleWriteFile(state);
-  } else {
-    writeFile = createCompileWriteFile(state);
-  }
-
-  const host = (state.host = new Host({
-    bundle,
-    target,
-    writeFile,
-  }));
-  const compilerOptions = [DEFAULT_RUNTIME_COMPILE_OPTIONS];
-  if (convertedOptions) {
-    compilerOptions.push(convertedOptions);
-  }
-  if (unstable) {
-    compilerOptions.push({
-      lib: [
-        "deno.unstable",
-        ...((convertedOptions && convertedOptions.lib) || ["deno.window"]),
-      ],
-    });
-  }
-  if (bundle) {
-    compilerOptions.push(DEFAULT_BUNDLER_OPTIONS);
-  }
-  host.mergeOptions(...compilerOptions);
-
-  const program = ts.createProgram({
-    rootNames,
-    options: host.getCompilationSettings(),
-    host,
-    oldProgram: TS_SNAPSHOT_PROGRAM,
-  });
-
-  if (bundle) {
-    setRootExports(program, rootNames[0]);
-  }
-
-  const diagnostics = ts
-    .getPreEmitDiagnostics(program)
-    .filter(({ code }) => !ignoredDiagnostics.includes(code));
-
-  const emitResult = program.emit();
-
-  assert(emitResult.emitSkipped === false, "Unexpected skip of the emit.");
-
-  assert(state.emitMap);
-  util.log("<<< runtime compile finish", {
-    rootName,
-    sources: sources ? Object.keys(sources) : undefined,
-    bundle,
-    emitMap: Object.keys(state.emitMap),
-  });
-
-  const maybeDiagnostics = diagnostics.length
-    ? fromTypeScriptDiagnostic(diagnostics).items
-    : [];
-
-  if (bundle) {
-    return {
-      diagnostics: maybeDiagnostics,
-      output: state.bundleOutput,
-    } as RuntimeBundleResult;
-  } else {
-    return {
-      diagnostics: maybeDiagnostics,
-      emitMap: state.emitMap,
-    } as RuntimeCompileResult;
-  }
-}
-
 function runtimeTranspile(
   request: CompilerRequestRuntimeTranspile
 ): Promise<Record<string, TranspileOnlyResult>> {
@@ -1915,13 +1369,8 @@ async function tsCompilerOnMessage({
 }): Promise<void> {
   switch (request.type) {
     case CompilerRequestType.Compile: {
-      const result = await compile(request as CompilerRequestCompile);
-      globalThis.postMessage(result);
-      break;
-    }
-    case CompilerRequestType.CompileNew: {
       const start = new Date();
-      const result = compileNew(request as CompilerRequestCompileNew);
+      const result = compile(request as CompilerRequestCompile);
       const end = new Date();
       util.log(
         // @ts-ignore
@@ -1931,16 +1380,7 @@ async function tsCompilerOnMessage({
       break;
     }
     case CompilerRequestType.RuntimeCompile: {
-      const result = await runtimeCompile(
-        request as CompilerRequestRuntimeCompile
-      );
-      globalThis.postMessage(result);
-      break;
-    }
-    case CompilerRequestType.RuntimeCompileNew: {
-      const result = runtimeCompileNew(
-        request as CompilerRequestRuntimeCompileNew
-      );
+      const result = runtimeCompile(request as CompilerRequestRuntimeCompile);
       globalThis.postMessage(result);
       break;
     }
diff --git a/cli/module_graph.rs b/cli/module_graph.rs
index 35a996a5c66a2b..313de2b3753b2c 100644
--- a/cli/module_graph.rs
+++ b/cli/module_graph.rs
@@ -158,6 +158,9 @@ impl ModuleGraphLoader {
 
     let mut dummy_prefix = false;
 
+    // The resolveModules op only handles fully qualified URLs for referrer.
+    // However we will have cases where referrer is "/foo.ts". We add this dummy
+    // prefix "file://" in order to use the op.
     let module_specifier = if specifier.starts_with('/') {
       dummy_prefix = true;
       ModuleSpecifier::resolve_url(&format!("memory://{}", specifier))
diff --git a/cli/msg.rs b/cli/msg.rs
index 6c8382268b87a6..186fde42c644e8 100644
--- a/cli/msg.rs
+++ b/cli/msg.rs
@@ -38,6 +38,4 @@ pub enum CompilerRequestType {
   Compile = 0,
   RuntimeCompile = 1,
   RuntimeTranspile = 2,
-  CompileNew = 3,
-  RuntimeCompileNew = 4,
 }
diff --git a/cli/ops/compiler.rs b/cli/ops/compiler.rs
index b844011877c390..16ad5e8cb7749a 100644
--- a/cli/ops/compiler.rs
+++ b/cli/ops/compiler.rs
@@ -1,152 +1,12 @@
 // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license.
-use super::dispatch_json::Deserialize;
-use super::dispatch_json::JsonOp;
-use super::dispatch_json::Value;
-use crate::futures::future::try_join_all;
-use crate::op_error::OpError;
 use crate::state::State;
 use deno_core::CoreIsolate;
-use deno_core::ModuleLoader;
-use deno_core::ModuleSpecifier;
-use deno_core::ZeroCopyBuf;
-use futures::future::FutureExt;
 
-pub fn init(i: &mut CoreIsolate, s: &State) {
-  i.register_op("op_resolve_modules", s.stateful_json_op(op_resolve_modules));
-  i.register_op(
-    "op_fetch_source_files",
-    s.stateful_json_op(op_fetch_source_files),
-  );
+pub fn init(i: &mut CoreIsolate, _s: &State) {
   let custom_assets = std::collections::HashMap::new(); // TODO(ry) use None.
+  // TODO(bartlomieju): is this op even required?
   i.register_op(
     "op_fetch_asset",
     deno_typescript::op_fetch_asset(custom_assets),
   );
 }
-
-#[derive(Deserialize, Debug)]
-struct SpecifiersReferrerArgs {
-  specifiers: Vec<String>,
-  referrer: Option<String>,
-}
-
-fn op_resolve_modules(
-  state: &State,
-  args: Value,
-  _data: Option<ZeroCopyBuf>,
-) -> Result<JsonOp, OpError> {
-  let args: SpecifiersReferrerArgs = serde_json::from_value(args)?;
-  let (referrer, is_main) = if let Some(referrer) = args.referrer {
-    (referrer, false)
-  } else {
-    ("<unknown>".to_owned(), true)
-  };
-
-  let mut specifiers = vec![];
-
-  for specifier in &args.specifiers {
-    let specifier = state
-      .resolve(specifier, &referrer, is_main)
-      .map_err(OpError::from)?;
-    specifiers.push(specifier.as_str().to_owned());
-  }
-
-  Ok(JsonOp::Sync(json!(specifiers)))
-}
-
-fn op_fetch_source_files(
-  state: &State,
-  args: Value,
-  _data: Option<ZeroCopyBuf>,
-) -> Result<JsonOp, OpError> {
-  let args: SpecifiersReferrerArgs = serde_json::from_value(args)?;
-
-  let ref_specifier = if let Some(referrer) = args.referrer {
-    let specifier = ModuleSpecifier::resolve_url(&referrer)
-      .expect("Referrer is not a valid specifier");
-    Some(specifier)
-  } else {
-    None
-  };
-
-  let s = state.borrow();
-  let global_state = s.global_state.clone();
-  let permissions = s.permissions.clone();
-  let perms_ = permissions.clone();
-  drop(s);
-  let file_fetcher = global_state.file_fetcher.clone();
-  let specifiers = args.specifiers.clone();
-  let future = async move {
-    let file_futures: Vec<_> = specifiers
-      .into_iter()
-      .map(|specifier| {
-        let file_fetcher_ = file_fetcher.clone();
-        let ref_specifier_ = ref_specifier.clone();
-        let perms_ = perms_.clone();
-        async move {
-          let resolved_specifier = ModuleSpecifier::resolve_url(&specifier)
-            .expect("Invalid specifier");
-          // TODO(bartlomieju): duplicated from `state.rs::ModuleLoader::load` - deduplicate
-          // Verify that remote file doesn't try to statically import local file.
-          if let Some(referrer) = ref_specifier_.as_ref() {
-            let referrer_url = referrer.as_url();
-            match referrer_url.scheme() {
-              "http" | "https" => {
-                let specifier_url = resolved_specifier.as_url();
-                match specifier_url.scheme() {
-                  "http" | "https" => {},
-                  _ => {
-                    let e = OpError::permission_denied("Remote module are not allowed to statically import local modules. Use dynamic import instead.".to_string());
-                    return Err(e.into());
-                  }
-                }
-              },
-              _ => {}
-            }
-          }
-          file_fetcher_
-            .fetch_source_file(&resolved_specifier, ref_specifier_, perms_)
-            .await
-        }
-        .boxed_local()
-      })
-      .collect();
-
-    let files = try_join_all(file_futures).await.map_err(OpError::from)?;
-    // We want to get an array of futures that resolves to
-    let v = files.into_iter().map(|f| {
-      async {
-        // if the source file contains a `types_url` we need to replace
-        // the module with the type definition when requested by the compiler
-        let file = match f.types_url {
-          Some(types_url) => {
-            let types_specifier = ModuleSpecifier::from(types_url);
-            global_state
-              .file_fetcher
-              .fetch_source_file(
-                &types_specifier,
-                ref_specifier.clone(),
-                permissions.clone(),
-              )
-              .await
-              .map_err(OpError::from)?
-          }
-          _ => f,
-        };
-        let source_code = String::from_utf8(file.source_code).map_err(|_| OpError::invalid_utf8())?;
-        Ok::<_, OpError>(json!({
-          "url": file.url.to_string(),
-          "filename": file.filename.to_str().unwrap(),
-          "mediaType": file.media_type as i32,
-          "sourceCode": source_code,
-        }))
-      }
-    });
-
-    let v = try_join_all(v).await?;
-    Ok(v.into())
-  }
-  .boxed_local();
-
-  Ok(JsonOp::Async(future))
-}
diff --git a/cli/tsc.rs b/cli/tsc.rs
index 1651651f4bfe50..6987e0dcf77eec 100644
--- a/cli/tsc.rs
+++ b/cli/tsc.rs
@@ -379,7 +379,7 @@ impl TsCompiler {
     let cwd = std::env::current_dir().unwrap();
     let j = match (compiler_config.path, compiler_config.content) {
       (Some(config_path), Some(config_data)) => json!({
-        "type": msg::CompilerRequestType::CompileNew as i32,
+        "type": msg::CompilerRequestType::Compile as i32,
         "target": target,
         "rootNames": root_names,
         "bundle": bundle,
@@ -390,7 +390,7 @@ impl TsCompiler {
         "sourceFileMap": module_graph_json,
       }),
       _ => json!({
-        "type": msg::CompilerRequestType::CompileNew as i32,
+        "type": msg::CompilerRequestType::Compile as i32,
         "target": target,
         "rootNames": root_names,
         "bundle": bundle,
@@ -544,7 +544,7 @@ impl TsCompiler {
     let cwd = std::env::current_dir().unwrap();
     let j = match (compiler_config.path, compiler_config.content) {
       (Some(config_path), Some(config_data)) => json!({
-        "type": msg::CompilerRequestType::CompileNew as i32,
+        "type": msg::CompilerRequestType::Compile as i32,
         "target": target,
         "rootNames": root_names,
         "bundle": bundle,
@@ -555,7 +555,7 @@ impl TsCompiler {
         "sourceFileMap": module_graph_json,
       }),
       _ => json!({
-        "type": msg::CompilerRequestType::CompileNew as i32,
+        "type": msg::CompilerRequestType::Compile as i32,
         "target": target,
         "rootNames": root_names,
         "bundle": bundle,
@@ -941,7 +941,7 @@ pub async fn runtime_compile<S: BuildHasher>(
 
   // eprintln!("source graph {:#?}", module_graph_json);
   let req_msg = json!({
-    "type": msg::CompilerRequestType::RuntimeCompileNew as i32,
+    "type": msg::CompilerRequestType::RuntimeCompile as i32,
     "target": "runtime",
     "rootNames": root_names,
     "sourceFileMap": module_graph_json,

From 494a36bcb4cd0359daf472bb9dfeb99e4b1a78d7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= <biwanczuk@gmail.com>
Date: Sat, 16 May 2020 18:01:01 +0200
Subject: [PATCH 32/47] fix bundle with dynamic imports

---
 cli/module_graph.rs | 20 ++++++++++++++----
 cli/ops/compiler.rs |  2 +-
 cli/swc_util.rs     | 49 ++++++++++++++++++++++++++++++++++++++++++++-
 cli/tsc.rs          |  3 +++
 4 files changed, 68 insertions(+), 6 deletions(-)

diff --git a/cli/module_graph.rs b/cli/module_graph.rs
index 313de2b3753b2c..2776ac9c50b058 100644
--- a/cli/module_graph.rs
+++ b/cli/module_graph.rs
@@ -95,6 +95,7 @@ pub struct ModuleGraphLoader {
   to_visit: Vec<(ModuleSpecifier, Option<ModuleSpecifier>)>,
   pub graph: ModuleGraph,
   is_dyn_import: bool,
+  analyze_dynamic_imports: bool,
 }
 
 impl ModuleGraphLoader {
@@ -103,6 +104,7 @@ impl ModuleGraphLoader {
     maybe_import_map: Option<ImportMap>,
     permissions: Permissions,
     is_dyn_import: bool,
+    analyze_dynamic_imports: bool,
   ) -> Self {
     Self {
       file_fetcher,
@@ -112,6 +114,7 @@ impl ModuleGraphLoader {
       pending_downloads: FuturesUnordered::new(),
       graph: ModuleGraph(HashMap::new()),
       is_dyn_import,
+      analyze_dynamic_imports,
     }
   }
 
@@ -168,8 +171,10 @@ impl ModuleGraphLoader {
       ModuleSpecifier::resolve_url(&specifier)
     }?;
 
-    let (import_descs, ref_descs) =
-      analyze_dependencies_and_references(&source_code, true)?;
+    let (import_descs, ref_descs) = analyze_dependencies_and_references(
+      &source_code,
+      self.analyze_dynamic_imports,
+    )?;
 
     for import_desc in import_descs {
       let maybe_resolved =
@@ -350,8 +355,10 @@ impl ModuleGraphLoader {
         type_headers.push(type_header);
       }
 
-      let (import_descs, ref_descs) =
-        analyze_dependencies_and_references(&source_code, true)?;
+      let (import_descs, ref_descs) = analyze_dependencies_and_references(
+        &source_code,
+        self.analyze_dynamic_imports,
+      )?;
 
       for import_desc in import_descs {
         let maybe_resolved =
@@ -482,6 +489,7 @@ mod tests {
       None,
       Permissions::allow_all(),
       false,
+      false,
     );
     let graph = graph_loader.build_graph(&module_specifier).await.unwrap();
 
@@ -554,6 +562,7 @@ mod tests {
       None,
       Permissions::allow_all(),
       false,
+      false,
     );
     let graph = graph_loader.build_graph(&module_specifier).await.unwrap();
 
@@ -593,6 +602,7 @@ mod tests {
       None,
       Permissions::allow_all(),
       false,
+      false,
     );
     let graph = graph_loader.build_graph(&module_specifier).await.unwrap();
 
@@ -689,6 +699,7 @@ mod tests {
       None,
       Permissions::allow_all(),
       false,
+      false,
     );
     let graph = graph_loader.build_graph(&module_specifier).await.unwrap();
 
@@ -754,6 +765,7 @@ mod tests {
       None,
       Permissions::allow_all(),
       false,
+      false,
     );
     let graph = graph_loader.build_graph(&module_specifier).await.unwrap();
 
diff --git a/cli/ops/compiler.rs b/cli/ops/compiler.rs
index 16ad5e8cb7749a..625f0ac7f9d6a6 100644
--- a/cli/ops/compiler.rs
+++ b/cli/ops/compiler.rs
@@ -4,7 +4,7 @@ use deno_core::CoreIsolate;
 
 pub fn init(i: &mut CoreIsolate, _s: &State) {
   let custom_assets = std::collections::HashMap::new(); // TODO(ry) use None.
-  // TODO(bartlomieju): is this op even required?
+                                                        // TODO(bartlomieju): is this op even required?
   i.register_op(
     "op_fetch_asset",
     deno_typescript::op_fetch_asset(custom_assets),
diff --git a/cli/swc_util.rs b/cli/swc_util.rs
index 91c155bf6b7f98..570b94a0aa8b9b 100644
--- a/cli/swc_util.rs
+++ b/cli/swc_util.rs
@@ -332,6 +332,7 @@ const a = await import("./" + "buzz.ts");
 #[derive(Clone, Debug, PartialEq)]
 enum DependencyKind {
   Import,
+  DynamicImport,
   Export,
 }
 
@@ -387,6 +388,46 @@ impl Visit for NewDependencyVisitor {
       span: export_all.span,
     });
   }
+
+  fn visit_call_expr(
+    &mut self,
+    call_expr: &swc_ecma_ast::CallExpr,
+    parent: &dyn Node,
+  ) {
+    use swc_ecma_ast::Expr::*;
+    use swc_ecma_ast::ExprOrSuper::*;
+
+    swc_ecma_visit::visit_call_expr(self, call_expr, parent);
+    let boxed_expr = match call_expr.callee.clone() {
+      Super(_) => return,
+      Expr(boxed) => boxed,
+    };
+
+    match &*boxed_expr {
+      Ident(ident) => {
+        if &ident.sym.to_string() != "import" {
+          return;
+        }
+      }
+      _ => return,
+    };
+
+    if let Some(arg) = call_expr.args.get(0) {
+      match &*arg.expr {
+        Lit(lit) => {
+          if let swc_ecma_ast::Lit::Str(str_) = lit {
+            let src_str = str_.value.to_string();
+            self.dependencies.push(DependencyDescriptor {
+              specifier: src_str,
+              kind: DependencyKind::DynamicImport,
+              span: call_expr.span,
+            });
+          }
+        }
+        _ => return,
+      }
+    }
+  }
 }
 
 fn get_deno_types(parser: &AstParser, span: Span) -> Option<String> {
@@ -437,7 +478,6 @@ pub struct TsReferenceDescriptor {
   pub specifier: String,
 }
 
-#[allow(unused)]
 pub fn analyze_dependencies_and_references(
   source_code: &str,
   analyze_dynamic_imports: bool,
@@ -459,6 +499,13 @@ pub fn analyze_dependencies_and_references(
     // for each import check if there's relevant @deno-types directive
     let imports = dependency_descriptors
       .iter()
+      .filter(|desc| {
+        if analyze_dynamic_imports {
+          return true;
+        }
+
+        desc.kind != DependencyKind::DynamicImport
+      })
       .map(|mut desc| {
         if desc.kind == DependencyKind::Import {
           let deno_types = get_deno_types(&parser, desc.span);
diff --git a/cli/tsc.rs b/cli/tsc.rs
index 6987e0dcf77eec..e2f85c4133cbcd 100644
--- a/cli/tsc.rs
+++ b/cli/tsc.rs
@@ -365,6 +365,7 @@ impl TsCompiler {
       import_map,
       permissions.clone(),
       false,
+      true,
     );
     let module_graph =
       module_graph_loader.build_graph(&module_specifier).await?;
@@ -527,6 +528,7 @@ impl TsCompiler {
       import_map,
       permissions.clone(),
       is_dyn_import,
+      false,
     );
     let module_graph =
       module_graph_loader.build_graph(&module_specifier).await?;
@@ -905,6 +907,7 @@ pub async fn runtime_compile<S: BuildHasher>(
     None,
     permissions.clone(),
     false,
+    false,
   );
 
   if let Some(s_map) = sources {

From facf6ab70167a41cb943094addbc58bcb2c1ab4f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= <biwanczuk@gmail.com>
Date: Sat, 16 May 2020 19:17:50 +0200
Subject: [PATCH 33/47] fix some tests

---
 cli/module_graph.rs                           | 60 ++++++++++++-------
 cli/tests/020_json_modules.ts.out             |  4 +-
 .../error_005_missing_dynamic_import.ts.out   |  9 +--
 ...or_012_bad_dynamic_import_specifier.ts.out | 10 +---
 cli/tests/integration_tests.rs                |  2 +-
 core/modules.rs                               |  7 +++
 6 files changed, 51 insertions(+), 41 deletions(-)

diff --git a/cli/module_graph.rs b/cli/module_graph.rs
index 2776ac9c50b058..d6f99e8d81c59f 100644
--- a/cli/module_graph.rs
+++ b/cli/module_graph.rs
@@ -1,5 +1,4 @@
 // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license.
-#![allow(unused)]
 
 use crate::file_fetcher::SourceFile;
 use crate::file_fetcher::SourceFileFetcher;
@@ -92,7 +91,6 @@ pub struct ModuleGraphLoader {
   file_fetcher: SourceFileFetcher,
   maybe_import_map: Option<ImportMap>,
   pending_downloads: FuturesUnordered<SourceFileFuture>,
-  to_visit: Vec<(ModuleSpecifier, Option<ModuleSpecifier>)>,
   pub graph: ModuleGraph,
   is_dyn_import: bool,
   analyze_dynamic_imports: bool,
@@ -110,7 +108,6 @@ impl ModuleGraphLoader {
       file_fetcher,
       permissions,
       maybe_import_map,
-      to_visit: vec![],
       pending_downloads: FuturesUnordered::new(),
       graph: ModuleGraph(HashMap::new()),
       is_dyn_import,
@@ -139,11 +136,11 @@ impl ModuleGraphLoader {
 
   pub fn build_local_graph<S: BuildHasher>(
     &mut self,
-    root_name: &str,
+    _root_name: &str,
     source_map: &HashMap<String, String, S>,
   ) -> Result<(), ErrBox> {
     for (spec, source_code) in source_map.iter() {
-      self.visit_memory_module(spec.to_string(), source_code.to_string());
+      self.visit_memory_module(spec.to_string(), source_code.to_string())?;
     }
 
     Ok(())
@@ -159,17 +156,17 @@ impl ModuleGraphLoader {
     let mut lib_directives = vec![];
     let mut types_directives = vec![];
 
-    let mut dummy_prefix = false;
+    // let mut dummy_prefix = false;
 
     // The resolveModules op only handles fully qualified URLs for referrer.
     // However we will have cases where referrer is "/foo.ts". We add this dummy
-    // prefix "file://" in order to use the op.
-    let module_specifier = if specifier.starts_with('/') {
-      dummy_prefix = true;
-      ModuleSpecifier::resolve_url(&format!("memory://{}", specifier))
-    } else {
-      ModuleSpecifier::resolve_url(&specifier)
-    }?;
+    // prefix "memory://" in order to use resolution logic.
+    let module_specifier =
+      if let Ok(spec) = ModuleSpecifier::resolve_url(&specifier) {
+        spec
+      } else {
+        ModuleSpecifier::resolve_url(&format!("memory://{}", specifier))?
+      };
 
     let (import_descs, ref_descs) = analyze_dependencies_and_references(
       &source_code,
@@ -215,10 +212,17 @@ impl ModuleGraphLoader {
     }
 
     for ref_desc in ref_descs {
-      let resolved_specifier = ModuleSpecifier::resolve_import(
+      let resolve_result = ModuleSpecifier::resolve_import(
         &ref_desc.specifier,
         &module_specifier.to_string(),
-      )?;
+      );
+
+      // Skip for libs like "dom" or "esnext"
+      let resolved_specifier = match resolve_result {
+        Ok(s) => s,
+        Err(_) => continue,
+      };
+
       let reference_descriptor = ReferenceDescriptor {
         specifier: ref_desc.specifier.to_string(),
         resolved_specifier,
@@ -314,9 +318,16 @@ impl ModuleGraphLoader {
     let perms = self.permissions.clone();
 
     let load_future = async move {
-      file_fetcher
-        .new_fetch_source_file(spec, maybe_referrer, perms)
-        .await
+      let source_file = file_fetcher
+        .new_fetch_source_file(spec.clone(), maybe_referrer, perms)
+        .await?;
+      // FIXME(bartlomieju):
+      // because of redirects we may end up with wrong URL,
+      // substitute with original one
+      Ok(SourceFile {
+        url: spec.as_url().to_owned(),
+        ..source_file
+      })
     }
     .boxed_local();
 
@@ -413,10 +424,17 @@ impl ModuleGraphLoader {
       }
 
       for ref_desc in ref_descs {
-        let resolved_specifier = ModuleSpecifier::resolve_import(
+        let resolve_result = ModuleSpecifier::resolve_import(
           &ref_desc.specifier,
           &module_specifier.to_string(),
-        )?;
+        );
+
+        // Skip for libs like "dom" or "esnext"
+        let resolved_specifier = match resolve_result {
+          Ok(s) => s,
+          Err(_) => continue,
+        };
+
         let reference_descriptor = ReferenceDescriptor {
           specifier: ref_desc.specifier.to_string(),
           resolved_specifier,
@@ -464,7 +482,7 @@ impl ModuleGraphLoader {
 mod tests {
   use super::*;
   use crate::GlobalState;
-  use std::path::PathBuf;
+  // use std::path::PathBuf;
 
   // fn rel_module_specifier(relpath: &str) -> ModuleSpecifier {
   //   let p = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
diff --git a/cli/tests/020_json_modules.ts.out b/cli/tests/020_json_modules.ts.out
index cda28038e4e356..7fc60d37eafb76 100644
--- a/cli/tests/020_json_modules.ts.out
+++ b/cli/tests/020_json_modules.ts.out
@@ -2,8 +2,8 @@
 error: Uncaught TypeError: Cannot resolve extension for "[WILDCARD]config.json" with mediaType "Json".
     at getExtension ($deno$/compiler.ts:[WILDCARD])
     at new SourceFile ($deno$/compiler.ts:[WILDCARD])
-    at processImports ($deno$/compiler.ts:[WILDCARD])
-    at async processImports ($deno$/compiler.ts:[WILDCARD])
+    at Function.addToCache ($deno$/compiler.ts:[WILDCARD])
+    at buildSourceFileCache ($deno$/compiler.ts:[WILDCARD])
     at async compile ($deno$/compiler.ts:[WILDCARD])
     at async tsCompilerOnMessage ($deno$/compiler.ts:[WILDCARD])
     at async workerMessageRecvCallback ($deno$/runtime_worker.ts:[WILDCARD])
diff --git a/cli/tests/error_005_missing_dynamic_import.ts.out b/cli/tests/error_005_missing_dynamic_import.ts.out
index e8c8f8cd090020..3e2c3264a89e11 100644
--- a/cli/tests/error_005_missing_dynamic_import.ts.out
+++ b/cli/tests/error_005_missing_dynamic_import.ts.out
@@ -1,8 +1 @@
-[WILDCARD]error: Uncaught NotFound: Cannot resolve module "[WILDCARD]/bad-module.ts" from "[WILDCARD]/error_005_missing_dynamic_import.ts"
-    at unwrapResponse ([WILDCARD]dispatch_json.ts:[WILDCARD])
-    at Object.sendAsync ([WILDCARD]dispatch_json.ts:[WILDCARD])
-    at async processImports ($deno$/compiler.ts:[WILDCARD])
-    at async processImports ($deno$/compiler.ts:[WILDCARD])
-    at async compile ($deno$/compiler.ts:[WILDCARD])
-    at async tsCompilerOnMessage ($deno$/compiler.ts:[WILDCARD])
-    at async workerMessageRecvCallback ([WILDCARD]runtime_worker.ts:[WILDCARD])
+[WILDCARD]error: Uncaught TypeError: Cannot resolve module "[WILDCARD]/bad-module.ts" from "[WILDCARD]/error_005_missing_dynamic_import.ts"
diff --git a/cli/tests/error_012_bad_dynamic_import_specifier.ts.out b/cli/tests/error_012_bad_dynamic_import_specifier.ts.out
index 900b8f52da0385..57e4003ce4b23b 100644
--- a/cli/tests/error_012_bad_dynamic_import_specifier.ts.out
+++ b/cli/tests/error_012_bad_dynamic_import_specifier.ts.out
@@ -1,9 +1 @@
-[WILDCARD]error: Uncaught URIError: relative import path "bad-module.ts" not prefixed with / or ./ or ../ Imported from "[WILDCARD]/error_012_bad_dynamic_import_specifier.ts"
-    at unwrapResponse ($deno$/ops/dispatch_json.ts:[WILDCARD])
-    at Object.sendSync ($deno$/ops/dispatch_json.ts:[WILDCARD])
-    at resolveModules ($deno$/compiler.ts:[WILDCARD])
-    at processImports ($deno$/compiler.ts:[WILDCARD])
-    at processImports ($deno$/compiler.ts:[WILDCARD])
-    at async compile ($deno$/compiler.ts:[WILDCARD])
-    at async tsCompilerOnMessage ($deno$/compiler.ts:[WILDCARD])
-    at async workerMessageRecvCallback ([WILDCARD]runtime_worker.ts:[WILDCARD])
+[WILDCARD]error: Uncaught TypeError: relative import path "bad-module.ts" not prefixed with / or ./ or ../ Imported from "[WILDCARD]/error_012_bad_dynamic_import_specifier.ts"
diff --git a/cli/tests/integration_tests.rs b/cli/tests/integration_tests.rs
index db9ccafd77eae8..309d882c6fd468 100644
--- a/cli/tests/integration_tests.rs
+++ b/cli/tests/integration_tests.rs
@@ -1368,7 +1368,7 @@ itest!(error_004_missing_module {
 });
 
 itest!(error_005_missing_dynamic_import {
-  args: "run --reload error_005_missing_dynamic_import.ts",
+  args: "run --reload --allow-read error_005_missing_dynamic_import.ts",
   check_stderr: true,
   exit_code: 1,
   output: "error_005_missing_dynamic_import.ts.out",
diff --git a/core/modules.rs b/core/modules.rs
index 632df2dd000b2f..9aa0981b87bbcc 100644
--- a/core/modules.rs
+++ b/core/modules.rs
@@ -220,6 +220,13 @@ impl RecursiveModuleLoad {
         })
         .boxed()
       }
+      LoadState::ResolveImport(ref _specifier, ref referrer) => {
+        let ref_spec = ModuleSpecifier::resolve_url(referrer)?;
+        self
+          .loader
+          .load(&module_specifier, Some(ref_spec), self.is_dynamic_import())
+          .boxed_local()
+      }
       _ => self
         .loader
         .load(&module_specifier, None, self.is_dynamic_import())

From 22245437112cc36d2f3c883980cce880807825fb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= <biwanczuk@gmail.com>
Date: Sat, 16 May 2020 19:33:42 +0200
Subject: [PATCH 34/47] remove unused code

---
 cli/swc_util.rs | 84 +------------------------------------------------
 1 file changed, 1 insertion(+), 83 deletions(-)

diff --git a/cli/swc_util.rs b/cli/swc_util.rs
index 570b94a0aa8b9b..e07486cd70b46c 100644
--- a/cli/swc_util.rs
+++ b/cli/swc_util.rs
@@ -1,7 +1,4 @@
 // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license.
-
-#![allow(unused)]
-
 use crate::swc_common;
 use crate::swc_common::comments::CommentKind;
 use crate::swc_common::comments::Comments;
@@ -10,7 +7,6 @@ use crate::swc_common::errors::DiagnosticBuilder;
 use crate::swc_common::errors::Emitter;
 use crate::swc_common::errors::Handler;
 use crate::swc_common::errors::HandlerFlags;
-use crate::swc_common::BytePos;
 use crate::swc_common::FileName;
 use crate::swc_common::Globals;
 use crate::swc_common::SourceMap;
@@ -251,84 +247,6 @@ impl Visit for DependencyVisitor {
   }
 }
 
-/// Given file name and source code return vector
-/// of unresolved import specifiers.
-///
-/// Returned vector may contain duplicate entries.
-///
-/// Second argument allows to configure if dynamic
-/// imports should be analyzed.
-///
-/// NOTE: Only statically analyzable dynamic imports
-/// are considered; ie. the ones that have plain string specifier:
-///
-///    await import("./fizz.ts")
-///
-/// These imports will be ignored:
-///
-///    await import(`./${dir}/fizz.ts`)
-///    await import("./" + "fizz.ts")
-#[allow(unused)]
-pub fn analyze_dependencies(
-  source_code: &str,
-  analyze_dynamic_imports: bool,
-) -> Result<Vec<String>, SwcDiagnosticBuffer> {
-  let parser = AstParser::new();
-  parser.parse_module("root.ts", source_code, |parse_result| {
-    let module = parse_result?;
-    let mut collector = DependencyVisitor {
-      dependencies: vec![],
-      analyze_dynamic_imports,
-    };
-    collector.visit_module(&module, &module);
-    Ok(collector.dependencies)
-  })
-}
-
-#[test]
-fn test_analyze_dependencies() {
-  let source = r#"
-import { foo } from "./foo.ts";
-export { bar } from "./foo.ts";
-export * from "./bar.ts";
-"#;
-
-  let dependencies =
-    analyze_dependencies(source, false).expect("Failed to parse");
-  assert_eq!(
-    dependencies,
-    vec![
-      "./foo.ts".to_string(),
-      "./foo.ts".to_string(),
-      "./bar.ts".to_string(),
-    ]
-  );
-}
-
-#[test]
-fn test_analyze_dependencies_dyn_imports() {
-  let source = r#"
-import { foo } from "./foo.ts";
-export { bar } from "./foo.ts";
-export * from "./bar.ts";
-
-const a = await import("./fizz.ts");
-const a = await import("./" + "buzz.ts");
-"#;
-
-  let dependencies =
-    analyze_dependencies(source, true).expect("Failed to parse");
-  assert_eq!(
-    dependencies,
-    vec![
-      "./foo.ts".to_string(),
-      "./foo.ts".to_string(),
-      "./bar.ts".to_string(),
-      "./fizz.ts".to_string(),
-    ]
-  );
-}
-
 #[derive(Clone, Debug, PartialEq)]
 enum DependencyKind {
   Import,
@@ -506,7 +424,7 @@ pub fn analyze_dependencies_and_references(
 
         desc.kind != DependencyKind::DynamicImport
       })
-      .map(|mut desc| {
+      .map(|desc| {
         if desc.kind == DependencyKind::Import {
           let deno_types = get_deno_types(&parser, desc.span);
           ImportDescriptor {

From 512f535681cb771f1c91eb85ba6572a09696ba67 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= <biwanczuk@gmail.com>
Date: Sat, 16 May 2020 20:11:25 +0200
Subject: [PATCH 35/47] fix more tests

---
 cli/module_graph.rs                           | 33 +++++-----
 cli/tests/020_json_modules.ts.out             |  6 +-
 .../error_005_missing_dynamic_import.ts.out   |  2 +-
 cli/tsc.rs                                    | 65 +++++++++++++++++++
 core/modules.rs                               |  7 --
 5 files changed, 86 insertions(+), 27 deletions(-)

diff --git a/cli/module_graph.rs b/cli/module_graph.rs
index d6f99e8d81c59f..78f7d673bbaabd 100644
--- a/cli/module_graph.rs
+++ b/cli/module_graph.rs
@@ -8,6 +8,7 @@ use crate::op_error::OpError;
 use crate::permissions::Permissions;
 use crate::swc_util::analyze_dependencies_and_references;
 use crate::swc_util::TsReferenceKind;
+use crate::tsc::get_available_libs;
 use deno_core::ErrBox;
 use deno_core::ModuleSpecifier;
 use futures::stream::FuturesUnordered;
@@ -211,17 +212,17 @@ impl ModuleGraphLoader {
       imports.push(import_descriptor);
     }
 
+    let available_libs = get_available_libs();
+
     for ref_desc in ref_descs {
-      let resolve_result = ModuleSpecifier::resolve_import(
+      if available_libs.contains(&ref_desc.specifier) {
+        continue;
+      }
+
+      let resolved_specifier = ModuleSpecifier::resolve_import(
         &ref_desc.specifier,
         &module_specifier.to_string(),
-      );
-
-      // Skip for libs like "dom" or "esnext"
-      let resolved_specifier = match resolve_result {
-        Ok(s) => s,
-        Err(_) => continue,
-      };
+      )?;
 
       let reference_descriptor = ReferenceDescriptor {
         specifier: ref_desc.specifier.to_string(),
@@ -423,17 +424,17 @@ impl ModuleGraphLoader {
         imports.push(import_descriptor);
       }
 
+      let available_libs = get_available_libs();
+
       for ref_desc in ref_descs {
-        let resolve_result = ModuleSpecifier::resolve_import(
+        if available_libs.contains(&ref_desc.specifier) {
+          continue;
+        }
+
+        let resolved_specifier = ModuleSpecifier::resolve_import(
           &ref_desc.specifier,
           &module_specifier.to_string(),
-        );
-
-        // Skip for libs like "dom" or "esnext"
-        let resolved_specifier = match resolve_result {
-          Ok(s) => s,
-          Err(_) => continue,
-        };
+        )?;
 
         let reference_descriptor = ReferenceDescriptor {
           specifier: ref_desc.specifier.to_string(),
diff --git a/cli/tests/020_json_modules.ts.out b/cli/tests/020_json_modules.ts.out
index 7fc60d37eafb76..4369639eb4aa38 100644
--- a/cli/tests/020_json_modules.ts.out
+++ b/cli/tests/020_json_modules.ts.out
@@ -4,6 +4,6 @@ error: Uncaught TypeError: Cannot resolve extension for "[WILDCARD]config.json"
     at new SourceFile ($deno$/compiler.ts:[WILDCARD])
     at Function.addToCache ($deno$/compiler.ts:[WILDCARD])
     at buildSourceFileCache ($deno$/compiler.ts:[WILDCARD])
-    at async compile ($deno$/compiler.ts:[WILDCARD])
-    at async tsCompilerOnMessage ($deno$/compiler.ts:[WILDCARD])
-    at async workerMessageRecvCallback ($deno$/runtime_worker.ts:[WILDCARD])
+    at compile ($deno$/compiler.ts:[WILDCARD])
+    at tsCompilerOnMessage ($deno$/compiler.ts:[WILDCARD])
+[WILDCARD]
\ No newline at end of file
diff --git a/cli/tests/error_005_missing_dynamic_import.ts.out b/cli/tests/error_005_missing_dynamic_import.ts.out
index 3e2c3264a89e11..0f0e449c40fb34 100644
--- a/cli/tests/error_005_missing_dynamic_import.ts.out
+++ b/cli/tests/error_005_missing_dynamic_import.ts.out
@@ -1 +1 @@
-[WILDCARD]error: Uncaught TypeError: Cannot resolve module "[WILDCARD]/bad-module.ts" from "[WILDCARD]/error_005_missing_dynamic_import.ts"
+[WILDCARD]error: Uncaught TypeError: Cannot resolve module "[WILDCARD]/bad-module.ts"
diff --git a/cli/tsc.rs b/cli/tsc.rs
index 2503b408f29ea6..e96976a7260b22 100644
--- a/cli/tsc.rs
+++ b/cli/tsc.rs
@@ -53,6 +53,71 @@ use std::task::Poll;
 use std::time::Instant;
 use url::Url;
 
+// TODO(bartlomieju): make static
+pub fn get_available_libs() -> Vec<String> {
+  vec![
+    "deno.ns".to_string(),
+    "deno.window".to_string(),
+    "deno.worker".to_string(),
+    "deno.shared_globals".to_string(),
+    "deno.unstable".to_string(),
+    "dom".to_string(),
+    "dom.iterable".to_string(),
+    "es5".to_string(),
+    "es6".to_string(),
+    "esnext".to_string(),
+    "es2020".to_string(),
+    "es2020.full".to_string(),
+    "es2019".to_string(),
+    "es2019.full".to_string(),
+    "es2018".to_string(),
+    "es2018.full".to_string(),
+    "es2017".to_string(),
+    "es2017.full".to_string(),
+    "es2016".to_string(),
+    "es2016.full".to_string(),
+    "es2015".to_string(),
+    "es2015.collection".to_string(),
+    "es2015.core".to_string(),
+    "es2015.generator".to_string(),
+    "es2015.iterable".to_string(),
+    "es2015.promise".to_string(),
+    "es2015.proxy".to_string(),
+    "es2015.reflect".to_string(),
+    "es2015.symbol".to_string(),
+    "es2015.symbol.wellknown".to_string(),
+    "es2016.array.include".to_string(),
+    "es2017.intl".to_string(),
+    "es2017.object".to_string(),
+    "es2017.sharedmemory".to_string(),
+    "es2017.string".to_string(),
+    "es2017.typedarrays".to_string(),
+    "es2018.asyncgenerator".to_string(),
+    "es2018.asynciterable".to_string(),
+    "es2018.intl".to_string(),
+    "es2018.promise".to_string(),
+    "es2018.regexp".to_string(),
+    "es2019.array".to_string(),
+    "es2019.object".to_string(),
+    "es2019.string".to_string(),
+    "es2019.symbol".to_string(),
+    "es2020.bigint".to_string(),
+    "es2020.promise".to_string(),
+    "es2020.string".to_string(),
+    "es2020.symbol.wellknown".to_string(),
+    "esnext.array".to_string(),
+    "esnext.asynciterable".to_string(),
+    "esnext.bigint".to_string(),
+    "esnext.intl".to_string(),
+    "esnext.promise".to_string(),
+    "esnext.string".to_string(),
+    "esnext.symbol".to_string(),
+    "scripthost".to_string(),
+    "webworker".to_string(),
+    "webworker.importscripts".to_string(),
+  ]
+}
+
 #[derive(Debug, Clone)]
 pub struct CompiledModule {
   pub code: String,
diff --git a/core/modules.rs b/core/modules.rs
index 9aa0981b87bbcc..632df2dd000b2f 100644
--- a/core/modules.rs
+++ b/core/modules.rs
@@ -220,13 +220,6 @@ impl RecursiveModuleLoad {
         })
         .boxed()
       }
-      LoadState::ResolveImport(ref _specifier, ref referrer) => {
-        let ref_spec = ModuleSpecifier::resolve_url(referrer)?;
-        self
-          .loader
-          .load(&module_specifier, Some(ref_spec), self.is_dynamic_import())
-          .boxed_local()
-      }
       _ => self
         .loader
         .load(&module_specifier, None, self.is_dynamic_import())

From afcfb414ae2974d307dd43b339b37ac945ee3aca Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= <biwanczuk@gmail.com>
Date: Sat, 16 May 2020 21:52:47 +0200
Subject: [PATCH 36/47] cleanup

---
 cli/file_fetcher.rs | 12 ------------
 cli/module_graph.rs |  5 +++--
 cli/ops/compiler.rs |  5 +++--
 3 files changed, 6 insertions(+), 16 deletions(-)

diff --git a/cli/file_fetcher.rs b/cli/file_fetcher.rs
index 32971e71f06b91..cbfb340ddc6121 100644
--- a/cli/file_fetcher.rs
+++ b/cli/file_fetcher.rs
@@ -147,18 +147,6 @@ impl SourceFileFetcher {
     self.source_file_cache.set(specifier.to_string(), file);
   }
 
-  // TODO(bartlomieju): remove
-  pub async fn new_fetch_source_file(
-    &self,
-    specifier: ModuleSpecifier,
-    maybe_referrer: Option<ModuleSpecifier>,
-    permissions: Permissions,
-  ) -> Result<SourceFile, ErrBox> {
-    self
-      .fetch_source_file(&specifier, maybe_referrer, permissions)
-      .await
-  }
-
   pub async fn fetch_source_file(
     &self,
     specifier: &ModuleSpecifier,
diff --git a/cli/module_graph.rs b/cli/module_graph.rs
index 78f7d673bbaabd..6628beb9795a1c 100644
--- a/cli/module_graph.rs
+++ b/cli/module_graph.rs
@@ -319,14 +319,15 @@ impl ModuleGraphLoader {
     let perms = self.permissions.clone();
 
     let load_future = async move {
+      let spec_ = spec.clone();
       let source_file = file_fetcher
-        .new_fetch_source_file(spec.clone(), maybe_referrer, perms)
+        .fetch_source_file(&spec_, maybe_referrer, perms)
         .await?;
       // FIXME(bartlomieju):
       // because of redirects we may end up with wrong URL,
       // substitute with original one
       Ok(SourceFile {
-        url: spec.as_url().to_owned(),
+        url: spec_.as_url().to_owned(),
         ..source_file
       })
     }
diff --git a/cli/ops/compiler.rs b/cli/ops/compiler.rs
index 625f0ac7f9d6a6..2e5842c0fd6ef1 100644
--- a/cli/ops/compiler.rs
+++ b/cli/ops/compiler.rs
@@ -3,8 +3,9 @@ use crate::state::State;
 use deno_core::CoreIsolate;
 
 pub fn init(i: &mut CoreIsolate, _s: &State) {
-  let custom_assets = std::collections::HashMap::new(); // TODO(ry) use None.
-                                                        // TODO(bartlomieju): is this op even required?
+  let custom_assets = std::collections::HashMap::new();
+  // TODO(ry) use None.
+  // TODO(bartlomieju): is this op even required?
   i.register_op(
     "op_fetch_asset",
     deno_typescript::op_fetch_asset(custom_assets),

From 6897f7f94559089b9bf0ede11bddbd8c769cca7e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= <biwanczuk@gmail.com>
Date: Sun, 17 May 2020 01:02:32 +0200
Subject: [PATCH 37/47] run TS compiler worker on the same thread

---
 cli/js/compiler.ts |  3 ++-
 cli/tsc.rs         | 31 ++++++++++---------------------
 2 files changed, 12 insertions(+), 22 deletions(-)

diff --git a/cli/js/compiler.ts b/cli/js/compiler.ts
index 8e77ada67f5a41..59562b4133b3ff 100644
--- a/cli/js/compiler.ts
+++ b/cli/js/compiler.ts
@@ -1398,7 +1398,8 @@ async function tsCompilerOnMessage({
         } (${CompilerRequestType[(request as CompilerRequest).type]})`
       );
   }
-  // Currently Rust shuts down worker after single request
+  // Shutdown after single request
+  globalThis.close();
 }
 
 function bootstrapTsCompilerRuntime(): void {
diff --git a/cli/tsc.rs b/cli/tsc.rs
index c87a42a51cb0e2..80735059910559 100644
--- a/cli/tsc.rs
+++ b/cli/tsc.rs
@@ -18,10 +18,8 @@ use crate::source_maps::SourceMapGetter;
 use crate::startup_data;
 use crate::state::exit_unstable;
 use crate::state::State;
-use crate::tokio_util;
 use crate::version;
 use crate::web_worker::WebWorker;
-use crate::web_worker::WebWorkerHandle;
 use crate::worker::WorkerEvent;
 use core::task::Context;
 use deno_core::Buf;
@@ -448,7 +446,8 @@ impl TsCompiler {
     let req_msg = j.to_string().into_boxed_str().into_boxed_bytes();
 
     let msg =
-      execute_in_thread(global_state.clone(), permissions, req_msg).await?;
+      execute_in_same_thread(global_state.clone(), permissions, req_msg)
+        .await?;
     let json_str = std::str::from_utf8(&msg).unwrap();
     debug!("Message: {}", json_str);
 
@@ -604,7 +603,8 @@ impl TsCompiler {
     let start = Instant::now();
 
     let msg =
-      execute_in_thread(global_state.clone(), permissions, req_msg).await?;
+      execute_in_same_thread(global_state.clone(), permissions, req_msg)
+        .await?;
 
     let end = Instant::now();
     debug!("time spent in compiler thread {:#?}", end - start);
@@ -907,32 +907,21 @@ impl TsCompiler {
   }
 }
 
-async fn execute_in_thread(
+async fn execute_in_same_thread(
   global_state: GlobalState,
   permissions: Permissions,
   req: Buf,
 ) -> Result<Buf, ErrBox> {
-  let (handle_sender, handle_receiver) =
-    std::sync::mpsc::sync_channel::<Result<WebWorkerHandle, ErrBox>>(1);
-  let builder =
-    std::thread::Builder::new().name("deno-ts-compiler".to_string());
-  let join_handle = builder.spawn(move || {
-    let worker = TsCompiler::setup_worker(global_state.clone(), permissions);
-    handle_sender.send(Ok(worker.thread_safe_handle())).unwrap();
-    drop(handle_sender);
-    tokio_util::run_basic(worker).expect("Panic in event loop");
-  })?;
-  let handle = handle_receiver.recv().unwrap()?;
+  let worker = TsCompiler::setup_worker(global_state.clone(), permissions);
+  let handle = worker.thread_safe_handle();
   handle.post_message(req)?;
+  worker.await?;
   let event = handle.get_event().await.expect("Compiler didn't respond");
   let buf = match event {
     WorkerEvent::Message(buf) => Ok(buf),
     WorkerEvent::Error(error) => Err(error),
     WorkerEvent::TerminalError(error) => Err(error),
   }?;
-  // Shutdown worker and wait for thread to finish
-  handle.terminate();
-  join_handle.join().unwrap();
   Ok(buf)
 }
 
@@ -1002,7 +991,7 @@ pub async fn runtime_compile<S: BuildHasher>(
 
   let compiler = global_state.ts_compiler.clone();
 
-  let msg = execute_in_thread(global_state, permissions, req_msg).await?;
+  let msg = execute_in_same_thread(global_state, permissions, req_msg).await?;
   let json_str = std::str::from_utf8(&msg).unwrap();
 
   // TODO(bartlomieju): factor `bundle` path into separate function `runtime_bundle`
@@ -1040,7 +1029,7 @@ pub async fn runtime_transpile<S: BuildHasher>(
   .into_boxed_str()
   .into_boxed_bytes();
 
-  let msg = execute_in_thread(global_state, permissions, req_msg).await?;
+  let msg = execute_in_same_thread(global_state, permissions, req_msg).await?;
   let json_str = std::str::from_utf8(&msg).unwrap();
   let v = serde_json::from_str::<serde_json::Value>(json_str)
     .expect("Error decoding JSON string.");

From 0eb065c6ceef93918d95acd62694b4d2c41e9ff7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= <biwanczuk@gmail.com>
Date: Sun, 17 May 2020 01:48:34 +0200
Subject: [PATCH 38/47] properly poll ts worker

---
 cli/tsc.rs | 33 ++++++++++++++++++++++++---------
 1 file changed, 24 insertions(+), 9 deletions(-)

diff --git a/cli/tsc.rs b/cli/tsc.rs
index 80735059910559..bd9fa1b71133c2 100644
--- a/cli/tsc.rs
+++ b/cli/tsc.rs
@@ -26,6 +26,7 @@ use deno_core::Buf;
 use deno_core::ErrBox;
 use deno_core::ModuleSpecifier;
 use deno_core::StartupData;
+use futures::future::Either;
 use futures::future::Future;
 use futures::future::FutureExt;
 use log::info;
@@ -912,17 +913,31 @@ async fn execute_in_same_thread(
   permissions: Permissions,
   req: Buf,
 ) -> Result<Buf, ErrBox> {
-  let worker = TsCompiler::setup_worker(global_state.clone(), permissions);
+  let mut worker = TsCompiler::setup_worker(global_state.clone(), permissions);
   let handle = worker.thread_safe_handle();
   handle.post_message(req)?;
-  worker.await?;
-  let event = handle.get_event().await.expect("Compiler didn't respond");
-  let buf = match event {
-    WorkerEvent::Message(buf) => Ok(buf),
-    WorkerEvent::Error(error) => Err(error),
-    WorkerEvent::TerminalError(error) => Err(error),
-  }?;
-  Ok(buf)
+
+  let mut event_fut = handle.get_event().boxed_local();
+
+  loop {
+    let select_result = futures::future::select(event_fut, &mut worker).await;
+    match select_result {
+      Either::Left((event_result, _worker)) => {
+        let event = event_result.expect("Compiler didn't respond");
+
+        let buf = match event {
+          WorkerEvent::Message(buf) => Ok(buf),
+          WorkerEvent::Error(error) => Err(error),
+          WorkerEvent::TerminalError(error) => Err(error),
+        }?;
+        return Ok(buf);
+      }
+      Either::Right((worker_result, event_fut_)) => {
+        event_fut = event_fut_;
+        worker_result?;
+      }
+    }
+  }
 }
 
 /// This function is used by `Deno.compile()` and `Deno.bundle()` APIs.

From cb8ced7b6046256489feb85f36f49783313d83a4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= <biwanczuk@gmail.com>
Date: Sun, 17 May 2020 02:36:19 +0200
Subject: [PATCH 39/47] don't use oldProgram

---
 cli/js/compiler.ts | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/cli/js/compiler.ts b/cli/js/compiler.ts
index 59562b4133b3ff..34f8e5c74edb1a 100644
--- a/cli/js/compiler.ts
+++ b/cli/js/compiler.ts
@@ -532,7 +532,9 @@ SNAPSHOT_HOST.getSourceFile(
   ts.ScriptTarget.ESNext
 );
 
-const TS_SNAPSHOT_PROGRAM = ts.createProgram({
+// Created to hydrate source file cache with lib
+// declaration files during snapshotting process.
+const _TS_SNAPSHOT_PROGRAM = ts.createProgram({
   rootNames: [`${ASSETS}/bootstrap.ts`],
   options: SNAPSHOT_COMPILER_OPTIONS,
   host: SNAPSHOT_HOST,
@@ -1182,7 +1184,6 @@ function compile(request: CompilerRequestCompile): CompileResult {
       rootNames,
       options,
       host,
-      oldProgram: TS_SNAPSHOT_PROGRAM,
     });
 
     diagnostics = ts
@@ -1297,7 +1298,6 @@ function runtimeCompile(
     rootNames,
     options: host.getCompilationSettings(),
     host,
-    oldProgram: TS_SNAPSHOT_PROGRAM,
   });
 
   if (bundle) {

From 90e903a51b6877e1d2ae4b30af97111d5598e516 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= <biwanczuk@gmail.com>
Date: Sun, 17 May 2020 13:19:22 +0200
Subject: [PATCH 40/47] fix after merge

---
 cli/tsc.rs | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/cli/tsc.rs b/cli/tsc.rs
index bd9fa1b71133c2..fe82cbc13d4264 100644
--- a/cli/tsc.rs
+++ b/cli/tsc.rs
@@ -923,7 +923,9 @@ async fn execute_in_same_thread(
     let select_result = futures::future::select(event_fut, &mut worker).await;
     match select_result {
       Either::Left((event_result, _worker)) => {
-        let event = event_result.expect("Compiler didn't respond");
+        let event = event_result
+          .expect("Compiler didn't respond")
+          .expect("Empty message");
 
         let buf = match event {
           WorkerEvent::Message(buf) => Ok(buf),

From e3ebbe7347fc5f02e17e1ec6c9e9e2e46e8910ca Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= <biwanczuk@gmail.com>
Date: Sun, 17 May 2020 15:58:38 +0200
Subject: [PATCH 41/47] remove debug code

---
 cli/tsc.rs | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/cli/tsc.rs b/cli/tsc.rs
index fe82cbc13d4264..43d30f3374bfca 100644
--- a/cli/tsc.rs
+++ b/cli/tsc.rs
@@ -970,7 +970,7 @@ pub async fn runtime_compile<S: BuildHasher>(
     module_graph_loader.add_to_graph(&module_specifier).await?;
   }
 
-  // TODO: download all additional files from TSconfig and add them to root_names
+  // download all additional files from TSconfig and add them to root_names
   if let Some(options) = maybe_options {
     let options_json: serde_json::Value = serde_json::from_str(options)?;
     if let Some(types_option) = options_json.get("types") {
@@ -992,7 +992,6 @@ pub async fn runtime_compile<S: BuildHasher>(
   let module_graph_json =
     serde_json::to_value(module_graph).expect("Failed to serialize data");
 
-  // eprintln!("source graph {:#?}", module_graph_json);
   let req_msg = json!({
     "type": msg::CompilerRequestType::RuntimeCompile as i32,
     "target": "runtime",
@@ -1023,7 +1022,6 @@ pub async fn runtime_compile<S: BuildHasher>(
     compiler.cache_emitted_files(response.emit_map)?;
   }
 
-  // eprintln!("returned {:#?}", json_str);
   // We're returning `Ok()` instead of `Err()` because it's not runtime
   // error if there were diagnostics produces; we want to let user handle
   // diagnostics in the runtime.

From f96c3e20537224d18fc421783af5b03fdf930210 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= <biwanczuk@gmail.com>
Date: Sun, 17 May 2020 19:46:12 +0200
Subject: [PATCH 42/47] fix tests

---
 cli/module_graph.rs            | 458 +++++++++++++--------------------
 cli/tests/integration_tests.rs |   2 +-
 cli/tsc.rs                     |  13 +-
 3 files changed, 186 insertions(+), 287 deletions(-)

diff --git a/cli/module_graph.rs b/cli/module_graph.rs
index 6628beb9795a1c..c3b983bc645025 100644
--- a/cli/module_graph.rs
+++ b/cli/module_graph.rs
@@ -261,26 +261,6 @@ impl ModuleGraphLoader {
     Ok(())
   }
 
-  // TODO: remove
-  pub async fn build_graph(
-    mut self,
-    specifier: &ModuleSpecifier,
-  ) -> Result<HashMap<String, ModuleGraphFile>, ErrBox> {
-    self.download_module(specifier.clone(), None)?;
-
-    loop {
-      let load_result = self.pending_downloads.next().await.unwrap();
-      let source_file = load_result?;
-      let spec = ModuleSpecifier::from(source_file.url.clone());
-      self.visit_module(&spec, source_file)?;
-      if self.pending_downloads.is_empty() {
-        break;
-      }
-    }
-
-    Ok(self.graph.0)
-  }
-
   pub fn get_graph(self) -> HashMap<String, ModuleGraphFile> {
     self.graph.0
   }
@@ -484,343 +464,261 @@ impl ModuleGraphLoader {
 mod tests {
   use super::*;
   use crate::GlobalState;
-  // use std::path::PathBuf;
 
-  // fn rel_module_specifier(relpath: &str) -> ModuleSpecifier {
-  //   let p = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
-  //     .join(relpath)
-  //     .into_os_string();
-  //   let ps = p.to_str().unwrap();
-  //   ModuleSpecifier::resolve_url_or_path(ps).unwrap()
-  // }
+  async fn build_graph(
+    module_specifier: &ModuleSpecifier,
+  ) -> Result<HashMap<String, ModuleGraphFile>, ErrBox> {
+    let global_state = GlobalState::new(Default::default()).unwrap();
+    let mut graph_loader = ModuleGraphLoader::new(
+      global_state.file_fetcher.clone(),
+      None,
+      Permissions::allow_all(),
+      false,
+      false,
+    );
+    graph_loader.add_to_graph(&module_specifier).await?;
+    Ok(graph_loader.get_graph())
+  }
 
-  #[ignore]
   #[tokio::test]
   async fn source_graph_fetch() {
     let http_server_guard = crate::test_util::http_server();
 
-    let global_state = GlobalState::new(Default::default()).unwrap();
     let module_specifier = ModuleSpecifier::resolve_url_or_path(
       "http://localhost:4545/cli/tests/019_media_types.ts",
     )
     .unwrap();
-    let graph_loader = ModuleGraphLoader::new(
-      global_state.file_fetcher.clone(),
-      None,
-      Permissions::allow_all(),
-      false,
-      false,
-    );
-    let graph = graph_loader.build_graph(&module_specifier).await.unwrap();
+    let graph = build_graph(&module_specifier)
+      .await
+      .expect("Failed to build graph");
+
+    let a = graph
+      .get("http://localhost:4545/cli/tests/019_media_types.ts")
+      .unwrap();
+
+    assert!(graph.contains_key(
+      "http://localhost:4545/cli/tests/subdir/mt_text_ecmascript.j3.js"
+    ));
+    assert!(graph.contains_key(
+      "http://localhost:4545/cli/tests/subdir/mt_video_vdn.t2.ts"
+    ));
+    assert!(graph.contains_key("http://localhost:4545/cli/tests/subdir/mt_application_x_typescript.t4.ts"));
+    assert!(graph.contains_key(
+      "http://localhost:4545/cli/tests/subdir/mt_video_mp2t.t3.ts"
+    ));
+    assert!(graph.contains_key("http://localhost:4545/cli/tests/subdir/mt_application_x_javascript.j4.js"));
+    assert!(graph.contains_key(
+      "http://localhost:4545/cli/tests/subdir/mt_application_ecmascript.j2.js"
+    ));
+    assert!(graph.contains_key(
+      "http://localhost:4545/cli/tests/subdir/mt_text_javascript.j1.js"
+    ));
+    assert!(graph.contains_key(
+      "http://localhost:4545/cli/tests/subdir/mt_text_typescript.t1.ts"
+    ));
 
     assert_eq!(
-      serde_json::to_value(&graph).unwrap(),
-      json!({
-        "http://localhost:4545/cli/tests/subdir/mt_text_typescript.t1.ts": {
+      serde_json::to_value(&a.imports).unwrap(),
+      json!([
+        {
           "specifier": "http://localhost:4545/cli/tests/subdir/mt_text_typescript.t1.ts",
-          "deps": []
-        },
-        "http://localhost:4545/cli/tests/019_media_types.ts": {
-          "specifier": "http://localhost:4545/cli/tests/019_media_types.ts",
-          "deps": [
-            "http://localhost:4545/cli/tests/subdir/mt_text_typescript.t1.ts",
-            "http://localhost:4545/cli/tests/subdir/mt_video_vdn.t2.ts",
-            "http://localhost:4545/cli/tests/subdir/mt_video_mp2t.t3.ts",
-            "http://localhost:4545/cli/tests/subdir/mt_application_x_typescript.t4.ts",
-            "http://localhost:4545/cli/tests/subdir/mt_text_javascript.j1.js",
-            "http://localhost:4545/cli/tests/subdir/mt_application_ecmascript.j2.js",
-            "http://localhost:4545/cli/tests/subdir/mt_text_ecmascript.j3.js",
-            "http://localhost:4545/cli/tests/subdir/mt_application_x_javascript.j4.js"
-          ]
-        },
-        "http://localhost:4545/cli/tests/subdir/mt_text_ecmascript.j3.js": {
-          "specifier": "http://localhost:4545/cli/tests/subdir/mt_text_ecmascript.j3.js",
-          "deps": []
+          "resolvedSpecifier": "http://localhost:4545/cli/tests/subdir/mt_text_typescript.t1.ts",
+          "typeDirective": null,
+          "resolvedTypeDirective": null,
         },
-        "http://localhost:4545/cli/tests/subdir/mt_video_vdn.t2.ts": {
+        {
           "specifier": "http://localhost:4545/cli/tests/subdir/mt_video_vdn.t2.ts",
-          "deps": []
-        },
-        "http://localhost:4545/cli/tests/subdir/mt_application_x_typescript.t4.ts": {
-          "specifier": "http://localhost:4545/cli/tests/subdir/mt_application_x_typescript.t4.ts",
-          "deps": []
+          "resolvedSpecifier": "http://localhost:4545/cli/tests/subdir/mt_video_vdn.t2.ts",
+          "typeDirective": null,
+          "resolvedTypeDirective": null,
         },
-        "http://localhost:4545/cli/tests/subdir/mt_video_mp2t.t3.ts": {
+        {
           "specifier": "http://localhost:4545/cli/tests/subdir/mt_video_mp2t.t3.ts",
-          "deps": []
+          "resolvedSpecifier": "http://localhost:4545/cli/tests/subdir/mt_video_mp2t.t3.ts",
+          "typeDirective": null,
+          "resolvedTypeDirective": null,
         },
-        "http://localhost:4545/cli/tests/subdir/mt_application_x_javascript.j4.js": {
-          "specifier": "http://localhost:4545/cli/tests/subdir/mt_application_x_javascript.j4.js",
-          "deps": []
+        {
+          "specifier": "http://localhost:4545/cli/tests/subdir/mt_application_x_typescript.t4.ts",
+          "resolvedSpecifier": "http://localhost:4545/cli/tests/subdir/mt_application_x_typescript.t4.ts",
+          "typeDirective": null,
+          "resolvedTypeDirective": null,
+        },
+        {
+          "specifier": "http://localhost:4545/cli/tests/subdir/mt_text_javascript.j1.js",
+          "resolvedSpecifier": "http://localhost:4545/cli/tests/subdir/mt_text_javascript.j1.js",
+          "typeDirective": null,
+          "resolvedTypeDirective": null,
         },
-        "http://localhost:4545/cli/tests/subdir/mt_application_ecmascript.j2.js": {
+        {
           "specifier": "http://localhost:4545/cli/tests/subdir/mt_application_ecmascript.j2.js",
-          "deps": []
+          "resolvedSpecifier": "http://localhost:4545/cli/tests/subdir/mt_application_ecmascript.j2.js",
+          "typeDirective": null,
+          "resolvedTypeDirective": null,
         },
-        "http://localhost:4545/cli/tests/subdir/mt_text_javascript.j1.js": {
-          "specifier": "http://localhost:4545/cli/tests/subdir/mt_text_javascript.j1.js",
-          "deps": []
-        }
-      })
-    );
-    drop(http_server_guard);
-  }
-
-  #[ignore]
-  #[tokio::test]
-  async fn source_graph_fetch_circular() {
-    let http_server_guard = crate::test_util::http_server();
-
-    let global_state = GlobalState::new(Default::default()).unwrap();
-    let module_specifier = ModuleSpecifier::resolve_url_or_path(
-      "http://localhost:4545/cli/tests/circular1.js",
-    )
-    .unwrap();
-
-    let graph_loader = ModuleGraphLoader::new(
-      global_state.file_fetcher.clone(),
-      None,
-      Permissions::allow_all(),
-      false,
-      false,
-    );
-    let graph = graph_loader.build_graph(&module_specifier).await.unwrap();
-
-    assert_eq!(
-      serde_json::to_value(&graph).unwrap(),
-      json!({
-        "http://localhost:4545/cli/tests/circular2.js": {
-          "specifier": "http://localhost:4545/cli/tests/circular2.js",
-          "deps": [
-            "http://localhost:4545/cli/tests/circular1.js"
-          ]
+        {
+          "specifier": "http://localhost:4545/cli/tests/subdir/mt_text_ecmascript.j3.js",
+          "resolvedSpecifier": "http://localhost:4545/cli/tests/subdir/mt_text_ecmascript.j3.js",
+          "typeDirective": null,
+          "resolvedTypeDirective": null,
         },
-        "http://localhost:4545/cli/tests/circular1.js": {
-          "specifier": "http://localhost:4545/cli/tests/circular1.js",
-          "deps": [
-            "http://localhost:4545/cli/tests/circular2.js"
-          ]
-        }
-      })
+        {
+          "specifier": "http://localhost:4545/cli/tests/subdir/mt_application_x_javascript.j4.js",
+          "resolvedSpecifier": "http://localhost:4545/cli/tests/subdir/mt_application_x_javascript.j4.js",
+          "typeDirective": null,
+          "resolvedTypeDirective": null,
+        },
+      ])
     );
     drop(http_server_guard);
   }
 
-  #[ignore]
   #[tokio::test]
   async fn source_graph_type_references() {
     let http_server_guard = crate::test_util::http_server();
 
-    let global_state = GlobalState::new(Default::default()).unwrap();
     let module_specifier = ModuleSpecifier::resolve_url_or_path(
       "http://localhost:4545/cli/tests/type_definitions.ts",
     )
     .unwrap();
 
-    let graph_loader = ModuleGraphLoader::new(
-      global_state.file_fetcher.clone(),
-      None,
-      Permissions::allow_all(),
-      false,
-      false,
-    );
-    let graph = graph_loader.build_graph(&module_specifier).await.unwrap();
+    let graph = build_graph(&module_specifier)
+      .await
+      .expect("Failed to build graph");
 
     eprintln!("json {:#?}", serde_json::to_value(&graph).unwrap());
 
+    let a = graph
+      .get("http://localhost:4545/cli/tests/type_definitions.ts")
+      .unwrap();
     assert_eq!(
-      serde_json::to_value(&graph).unwrap(),
-      json!({
-        "http://localhost:4545/cli/tests/type_definitions.ts": {
-          "specifier": "http://localhost:4545/cli/tests/type_definitions.ts",
-          "imports": [
-            {
-              "specifier": "./type_definitions/foo.js",
-              "resolvedSpecifier": "http://localhost:4545/cli/tests/type_definitions/foo.js",
-              "typeDirective": "./type_definitions/foo.d.ts",
-              "resolvedTypeDirective": "http://localhost:4545/cli/tests/type_definitions/foo.d.ts"
-            },
-            {
-              "specifier": "./type_definitions/fizz.js",
-              "resolvedSpecifier": "http://localhost:4545/cli/tests/type_definitions/fizz.js",
-              "typeDirective": "./type_definitions/fizz.d.ts",
-              "resolvedTypeDirective": "http://localhost:4545/cli/tests/type_definitions/fizz.d.ts"
-            },
-            {
-              "specifier": "./type_definitions/qat.ts",
-              "resolvedSpecifier": "http://localhost:4545/cli/tests/type_definitions/qat.ts",
-              "typeDirective": null,
-              "resolvedTypeDirective": null,
-            },
-          ],
-          "typesDirectives": [],
-          "referencedFiles": [],
-          "libDirectives": [],
-          "typeHeaders": [],
-        },
-        "http://localhost:4545/cli/tests/type_definitions/foo.js": {
-          "specifier": "http://localhost:4545/cli/tests/type_definitions/foo.js",
-          "imports": [],
-          "referencedFiles": [],
-          "libDirectives": [],
-          "typesDirectives": [],
-          "typeHeaders": [],
-        },
-        "http://localhost:4545/cli/tests/type_definitions/foo.d.ts": {
-          "specifier": "http://localhost:4545/cli/tests/type_definitions/foo.d.ts",
-          "imports": [],
-          "referencedFiles": [],
-          "libDirectives": [],
-          "typesDirectives": [],
-          "typeHeaders": [],
+      serde_json::to_value(&a.imports).unwrap(),
+      json!([
+        {
+          "specifier": "./type_definitions/foo.js",
+          "resolvedSpecifier": "http://localhost:4545/cli/tests/type_definitions/foo.js",
+          "typeDirective": "./type_definitions/foo.d.ts",
+          "resolvedTypeDirective": "http://localhost:4545/cli/tests/type_definitions/foo.d.ts"
         },
-        "http://localhost:4545/cli/tests/type_definitions/fizz.js": {
-          "specifier": "http://localhost:4545/cli/tests/type_definitions/fizz.js",
-          "imports": [],
-          "referencedFiles": [],
-          "libDirectives": [],
-          "typesDirectives": [],
-          "typeHeaders": [],
+        {
+          "specifier": "./type_definitions/fizz.js",
+          "resolvedSpecifier": "http://localhost:4545/cli/tests/type_definitions/fizz.js",
+          "typeDirective": "./type_definitions/fizz.d.ts",
+          "resolvedTypeDirective": "http://localhost:4545/cli/tests/type_definitions/fizz.d.ts"
         },
-        "http://localhost:4545/cli/tests/type_definitions/fizz.d.ts": {
-          "specifier": "http://localhost:4545/cli/tests/type_definitions/fizz.d.ts",
-          "imports": [],
-          "referencedFiles": [],
-          "libDirectives": [],
-          "typesDirectives": [],
-          "typeHeaders": [],
+        {
+          "specifier": "./type_definitions/qat.ts",
+          "resolvedSpecifier": "http://localhost:4545/cli/tests/type_definitions/qat.ts",
+          "typeDirective": null,
+          "resolvedTypeDirective": null,
         },
-        "http://localhost:4545/cli/tests/type_definitions/qat.ts": {
-          "specifier": "http://localhost:4545/cli/tests/type_definitions/qat.ts",
-          "imports": [],
-          "referencedFiles": [],
-          "libDirectives": [],
-          "typesDirectives": [],
-          "typeHeaders": [],
-        }
-      })
+      ])
     );
+    assert!(graph
+      .contains_key("http://localhost:4545/cli/tests/type_definitions/foo.js"));
+    assert!(graph.contains_key(
+      "http://localhost:4545/cli/tests/type_definitions/foo.d.ts"
+    ));
+    assert!(graph.contains_key(
+      "http://localhost:4545/cli/tests/type_definitions/fizz.js"
+    ));
+    assert!(graph.contains_key(
+      "http://localhost:4545/cli/tests/type_definitions/fizz.d.ts"
+    ));
+    assert!(graph
+      .contains_key("http://localhost:4545/cli/tests/type_definitions/qat.ts"));
+
     drop(http_server_guard);
   }
 
-  #[ignore]
   #[tokio::test]
   async fn source_graph_type_references2() {
     let http_server_guard = crate::test_util::http_server();
 
-    let global_state = GlobalState::new(Default::default()).unwrap();
     let module_specifier = ModuleSpecifier::resolve_url_or_path(
       "http://localhost:4545/cli/tests/type_directives_02.ts",
     )
     .unwrap();
 
-    let graph_loader = ModuleGraphLoader::new(
-      global_state.file_fetcher.clone(),
-      None,
-      Permissions::allow_all(),
-      false,
-      false,
-    );
-    let graph = graph_loader.build_graph(&module_specifier).await.unwrap();
+    let graph = build_graph(&module_specifier)
+      .await
+      .expect("Failed to build graph");
 
     eprintln!("{:#?}", serde_json::to_value(&graph).unwrap());
 
+    let a = graph
+      .get("http://localhost:4545/cli/tests/type_directives_02.ts")
+      .unwrap();
     assert_eq!(
-      serde_json::to_value(&graph).unwrap(),
-      json!({
-        "http://localhost:4545/cli/tests/type_directives_02.ts": {
-          "specifier": "http://localhost:4545/cli/tests/type_directives_02.ts",
-          "imports": [
-            {
-              "specifier": "./subdir/type_reference.js",
-              "resolvedSpecifier": "http://localhost:4545/cli/tests/subdir/type_reference.js",
-              "typeDirective": null,
-              "resolvedTypeDirective": null,
-            }
-          ],
-          "typesDirectives": [],
-          "referencedFiles": [],
-          "libDirectives": [],
-          "typeHeaders": [],
-        },
-        "http://localhost:4545/cli/tests/subdir/type_reference.d.ts": {
-          "specifier": "http://localhost:4545/cli/tests/subdir/type_reference.d.ts",
-          "imports": [],
-          "referencedFiles": [],
-          "libDirectives": [],
-          "typesDirectives": [],
-          "typeHeaders": [],
-        },
-        "http://localhost:4545/cli/tests/subdir/type_reference.js": {
-          "specifier": "http://localhost:4545/cli/tests/subdir/type_reference.js",
-          "imports": [],
-          "referencedFiles": [],
-          "libDirectives": [],
-          "typesDirectives": [
-            {
-              "specifier": "./type_reference.d.ts",
-              "resolvedSpecifier": "http://localhost:4545/cli/tests/subdir/type_reference.d.ts",
-            }
-          ],
-          "typeHeaders": [],
+      serde_json::to_value(&a.imports).unwrap(),
+      json!([
+        {
+          "specifier": "./subdir/type_reference.js",
+          "resolvedSpecifier": "http://localhost:4545/cli/tests/subdir/type_reference.js",
+          "typeDirective": null,
+          "resolvedTypeDirective": null,
         }
-      })
+      ])
+    );
+
+    assert!(graph.contains_key(
+      "http://localhost:4545/cli/tests/subdir/type_reference.d.ts"
+    ));
+
+    let b = graph
+      .get("http://localhost:4545/cli/tests/subdir/type_reference.js")
+      .unwrap();
+    assert_eq!(
+      serde_json::to_value(&b.types_directives).unwrap(),
+      json!([
+        {
+          "specifier": "./type_reference.d.ts",
+          "resolvedSpecifier": "http://localhost:4545/cli/tests/subdir/type_reference.d.ts",
+        }
+      ])
     );
     drop(http_server_guard);
   }
 
-  #[ignore]
   #[tokio::test]
   async fn source_graph_type_references3() {
     let http_server_guard = crate::test_util::http_server();
 
-    let global_state = GlobalState::new(Default::default()).unwrap();
     let module_specifier = ModuleSpecifier::resolve_url_or_path(
       "http://localhost:4545/cli/tests/type_directives_01.ts",
     )
     .unwrap();
 
-    let graph_loader = ModuleGraphLoader::new(
-      global_state.file_fetcher.clone(),
-      None,
-      Permissions::allow_all(),
-      false,
-      false,
-    );
-    let graph = graph_loader.build_graph(&module_specifier).await.unwrap();
+    let graph = build_graph(&module_specifier)
+      .await
+      .expect("Failed to build graph");
 
+    let ts = graph
+      .get("http://localhost:4545/cli/tests/type_directives_01.ts")
+      .unwrap();
     assert_eq!(
-      serde_json::to_value(&graph).unwrap(),
-      json!({
-        "http://localhost:4545/cli/tests/type_directives_01.ts": {
-          "specifier": "http://localhost:4545/cli/tests/type_directives_01.ts",
-          "imports": [
-            {
-              "specifier": "http://127.0.0.1:4545/xTypeScriptTypes.js",
-              "resolvedSpecifier": "http://127.0.0.1:4545/xTypeScriptTypes.js",
-              "typeDirective": null,
-              "resolvedTypeDirective": null,
-            }
-          ],
-          "referencedFiles": [],
-          "libDirectives": [],
-          "typesDirectives": [],
-          "typeHeaders": [],
-        },
-        "http://127.0.0.1:4545/xTypeScriptTypes.js": {
+      serde_json::to_value(&ts.imports).unwrap(),
+      json!([
+        {
           "specifier": "http://127.0.0.1:4545/xTypeScriptTypes.js",
-          "typeHeaders": [
-            {
-              "specifier": "./xTypeScriptTypes.d.ts",
-              "resolvedSpecifier": "http://127.0.0.1:4545/xTypeScriptTypes.d.ts"
-            }
-          ],
-          "imports": [],
-          "referencedFiles": [],
-          "libDirectives": [],
-          "typesDirectives": [],
+          "resolvedSpecifier": "http://127.0.0.1:4545/xTypeScriptTypes.js",
+          "typeDirective": null,
+          "resolvedTypeDirective": null,
         }
-      })
+      ])
+    );
+
+    let headers = graph
+      .get("http://127.0.0.1:4545/xTypeScriptTypes.js")
+      .unwrap();
+    assert_eq!(
+      serde_json::to_value(&headers.type_headers).unwrap(),
+      json!([
+        {
+          "specifier": "./xTypeScriptTypes.d.ts",
+          "resolvedSpecifier": "http://127.0.0.1:4545/xTypeScriptTypes.d.ts"
+        }
+      ])
     );
     drop(http_server_guard);
   }
diff --git a/cli/tests/integration_tests.rs b/cli/tests/integration_tests.rs
index f83094c79b4cac..476a08224db7d8 100644
--- a/cli/tests/integration_tests.rs
+++ b/cli/tests/integration_tests.rs
@@ -1353,7 +1353,7 @@ itest!(error_004_missing_module {
 });
 
 itest!(error_005_missing_dynamic_import {
-  args: "run --reload --allow-readerror_005_missing_dynamic_import.ts",
+  args: "run --reload --allow-read error_005_missing_dynamic_import.ts",
   exit_code: 1,
   output: "error_005_missing_dynamic_import.ts.out",
 });
diff --git a/cli/tsc.rs b/cli/tsc.rs
index 43d30f3374bfca..41eb7e43cf35ef 100644
--- a/cli/tsc.rs
+++ b/cli/tsc.rs
@@ -403,15 +403,15 @@ impl TsCompiler {
         }
       };
     let permissions = Permissions::allow_all();
-    let module_graph_loader = ModuleGraphLoader::new(
+    let mut module_graph_loader = ModuleGraphLoader::new(
       global_state.file_fetcher.clone(),
       import_map,
       permissions.clone(),
       false,
       true,
     );
-    let module_graph =
-      module_graph_loader.build_graph(&module_specifier).await?;
+    module_graph_loader.add_to_graph(&module_specifier).await?;
+    let module_graph = module_graph_loader.get_graph();
     let module_graph_json =
       serde_json::to_value(module_graph).expect("Failed to serialize data");
 
@@ -547,15 +547,16 @@ impl TsCompiler {
           Some(ImportMap::load(file_path)?)
         }
       };
-    let module_graph_loader = ModuleGraphLoader::new(
+    let mut module_graph_loader = ModuleGraphLoader::new(
       global_state.file_fetcher.clone(),
       import_map,
       permissions.clone(),
       is_dyn_import,
       false,
     );
-    let module_graph =
-      module_graph_loader.build_graph(&module_specifier).await?;
+
+    module_graph_loader.add_to_graph(&module_specifier).await?;
+    let module_graph = module_graph_loader.get_graph();
     let module_graph_json =
       serde_json::to_value(module_graph).expect("Failed to serialize data");
 

From fa3c8165752c2531d3e2e9fbada3b3791a792b1e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= <biwanczuk@gmail.com>
Date: Sun, 17 May 2020 22:24:14 +0200
Subject: [PATCH 43/47] review

---
 cli/module_graph.rs | 26 ++++++++++++++++----------
 1 file changed, 16 insertions(+), 10 deletions(-)

diff --git a/cli/module_graph.rs b/cli/module_graph.rs
index c3b983bc645025..fe6221264198ce 100644
--- a/cli/module_graph.rs
+++ b/cli/module_graph.rs
@@ -116,6 +116,12 @@ impl ModuleGraphLoader {
     }
   }
 
+  /// This method is used to add specified module and all of its
+  /// dependencies to the graph.
+  ///
+  /// It resolves when all dependent modules have been fetched and analyzed.
+  ///
+  /// This method can be called multiple times.
   pub async fn add_to_graph(
     &mut self,
     specifier: &ModuleSpecifier,
@@ -123,10 +129,8 @@ impl ModuleGraphLoader {
     self.download_module(specifier.clone(), None)?;
 
     loop {
-      let load_result = self.pending_downloads.next().await.unwrap();
-      let source_file = load_result?;
-      let spec = ModuleSpecifier::from(source_file.url.clone());
-      self.visit_module(&spec, source_file)?;
+      let source_file = self.pending_downloads.next().await.unwrap()?;
+      self.visit_module(&source_file.url.into(), source_file)?;
       if self.pending_downloads.is_empty() {
         break;
       }
@@ -135,6 +139,9 @@ impl ModuleGraphLoader {
     Ok(())
   }
 
+  /// This method is used to create a graph from in-memory files stored in
+  /// a hash map. Useful for creating module graph for code received from
+  /// the runtime.
   pub fn build_local_graph<S: BuildHasher>(
     &mut self,
     _root_name: &str,
@@ -147,6 +154,11 @@ impl ModuleGraphLoader {
     Ok(())
   }
 
+  /// Consumes the loader and returns created graph.
+  pub fn get_graph(self) -> HashMap<String, ModuleGraphFile> {
+    self.graph.0
+  }
+
   fn visit_memory_module(
     &mut self,
     specifier: String,
@@ -157,8 +169,6 @@ impl ModuleGraphLoader {
     let mut lib_directives = vec![];
     let mut types_directives = vec![];
 
-    // let mut dummy_prefix = false;
-
     // The resolveModules op only handles fully qualified URLs for referrer.
     // However we will have cases where referrer is "/foo.ts". We add this dummy
     // prefix "memory://" in order to use resolution logic.
@@ -261,10 +271,6 @@ impl ModuleGraphLoader {
     Ok(())
   }
 
-  pub fn get_graph(self) -> HashMap<String, ModuleGraphFile> {
-    self.graph.0
-  }
-
   fn download_module(
     &mut self,
     module_specifier: ModuleSpecifier,

From 28544c960cba7508f835380acebbc2358012a0a1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= <biwanczuk@gmail.com>
Date: Sun, 17 May 2020 22:29:59 +0200
Subject: [PATCH 44/47] fix

---
 cli/module_graph.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cli/module_graph.rs b/cli/module_graph.rs
index fe6221264198ce..e2215742e9bbe6 100644
--- a/cli/module_graph.rs
+++ b/cli/module_graph.rs
@@ -130,7 +130,7 @@ impl ModuleGraphLoader {
 
     loop {
       let source_file = self.pending_downloads.next().await.unwrap()?;
-      self.visit_module(&source_file.url.into(), source_file)?;
+      self.visit_module(&source_file.url.clone().into(), source_file)?;
       if self.pending_downloads.is_empty() {
         break;
       }

From 35adb5231109d0681f93e2128b6b11fc086d3676 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= <biwanczuk@gmail.com>
Date: Mon, 18 May 2020 11:58:34 +0200
Subject: [PATCH 45/47] review2

---
 cli/js/compiler.ts  | 19 ++++++-------------
 cli/module_graph.rs |  1 +
 2 files changed, 7 insertions(+), 13 deletions(-)

diff --git a/cli/js/compiler.ts b/cli/js/compiler.ts
index 34f8e5c74edb1a..0a1ffa892d8358 100644
--- a/cli/js/compiler.ts
+++ b/cli/js/compiler.ts
@@ -450,7 +450,9 @@ class Host implements ts.CompilerHost {
     return moduleNames.map((specifier) => {
       const maybeUrl = SourceFile.getResolvedUrl(specifier, containingFile);
 
-      util.log("compiler::host.resolveModulenames receiverd", {
+      util.log("compiler::host.resolveModuleNames maybeUrl", {
+        specifier,
+        containingFile,
         maybeUrl,
         sf: SourceFile.getCached(maybeUrl!),
       });
@@ -532,8 +534,9 @@ SNAPSHOT_HOST.getSourceFile(
   ts.ScriptTarget.ESNext
 );
 
-// Created to hydrate source file cache with lib
-// declaration files during snapshotting process.
+// We never use this program; it's only created 
+// during snapshotting to hydrate and populate 
+// source file cache with lib declaration files.
 const _TS_SNAPSHOT_PROGRAM = ts.createProgram({
   rootNames: [`${ASSETS}/bootstrap.ts`],
   options: SNAPSHOT_COMPILER_OPTIONS,
@@ -1175,7 +1178,6 @@ function compile(request: CompilerRequestCompile): CompileResult {
   }
 
   buildSourceFileCache(sourceFileMap);
-  const start = new Date();
   // if there was a configuration and no diagnostics with it, we will continue
   // to generate the program and possibly emit it.
   if (diagnostics.length === 0) {
@@ -1205,9 +1207,6 @@ function compile(request: CompilerRequestCompile): CompileResult {
     }
   }
 
-  const end = new Date();
-  // @ts-ignore
-  util.log("time in internal TS", end - start);
   let bundleOutput = undefined;
 
   if (diagnostics && diagnostics.length === 0 && bundle) {
@@ -1369,13 +1368,7 @@ async function tsCompilerOnMessage({
 }): Promise<void> {
   switch (request.type) {
     case CompilerRequestType.Compile: {
-      const start = new Date();
       const result = compile(request as CompilerRequestCompile);
-      const end = new Date();
-      util.log(
-        // @ts-ignore
-        `Time spent in TS program "${end - start}"`
-      );
       globalThis.postMessage(result);
       break;
     }
diff --git a/cli/module_graph.rs b/cli/module_graph.rs
index e2215742e9bbe6..c21257d824ca2b 100644
--- a/cli/module_graph.rs
+++ b/cli/module_graph.rs
@@ -169,6 +169,7 @@ impl ModuleGraphLoader {
     let mut lib_directives = vec![];
     let mut types_directives = vec![];
 
+    // FIXME(bartlomieju):
     // The resolveModules op only handles fully qualified URLs for referrer.
     // However we will have cases where referrer is "/foo.ts". We add this dummy
     // prefix "memory://" in order to use resolution logic.

From 5d02e055fcb3a94d281e8df6e86ae1966bf0fc2b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= <biwanczuk@gmail.com>
Date: Mon, 18 May 2020 12:17:27 +0200
Subject: [PATCH 46/47] fmt

---
 cli/js/compiler.ts | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/cli/js/compiler.ts b/cli/js/compiler.ts
index 0a1ffa892d8358..fa4a7526e79d4a 100644
--- a/cli/js/compiler.ts
+++ b/cli/js/compiler.ts
@@ -534,8 +534,8 @@ SNAPSHOT_HOST.getSourceFile(
   ts.ScriptTarget.ESNext
 );
 
-// We never use this program; it's only created 
-// during snapshotting to hydrate and populate 
+// We never use this program; it's only created
+// during snapshotting to hydrate and populate
 // source file cache with lib declaration files.
 const _TS_SNAPSHOT_PROGRAM = ts.createProgram({
   rootNames: [`${ASSETS}/bootstrap.ts`],

From aa6733296545978834b0a76b956bf03a62000518 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= <biwanczuk@gmail.com>
Date: Mon, 18 May 2020 12:38:44 +0200
Subject: [PATCH 47/47] reset CI