refactor: TS compiler and module graph (denoland#5817)

This PR addresses many problems with module graph loading
introduced in denoland#5029, as well as many long standing issues.

"ModuleGraphLoader" has been wired to "ModuleLoader" implemented
on "State" - that means that dependency analysis and fetching is done
before spinning up TS compiler worker.

Basic dependency tracking for TS compilation has been implemented.

Errors caused by import statements are now annotated with import
location.

Co-authored-by: Ryan Dahl <ry@tinyclouds.org>

cli/doc/node.rs

@@ -32,7 +32,7 @@ pub struct ParamDef {
   pub ts_type: Option<super::ts_type::TsTypeDef>,
 }
 
-#[derive(Debug, Serialize, Clone)]
+#[derive(Debug, Serialize, Clone, PartialEq)]
 pub struct Location {
   pub filename: String,
   pub line: usize,

cli/global_state.rs

@@ -3,9 +3,12 @@ use crate::deno_dir;
 use crate::file_fetcher::SourceFileFetcher;
 use crate::flags;
 use crate::http_cache;
+use crate::import_map::ImportMap;
 use crate::lockfile::Lockfile;
+use crate::module_graph::ModuleGraphLoader;
 use crate::msg;
 use crate::permissions::Permissions;
+use crate::state::exit_unstable;
 use crate::tsc::CompiledModule;
 use crate::tsc::TargetLib;
 use crate::tsc::TsCompiler;
@@ -35,6 +38,7 @@ pub struct GlobalStateInner {
   pub ts_compiler: TsCompiler,
   pub lockfile: Option<Mutex<Lockfile>>,
   pub compiler_starts: AtomicUsize,
+  pub maybe_import_map: Option<ImportMap>,
   compile_lock: AsyncMutex<()>,
 }
 
@@ -75,86 +79,135 @@ impl GlobalState {
       None
     };
 
+    let maybe_import_map: Option<ImportMap> =
+      match flags.import_map_path.as_ref() {
+        None => None,
+        Some(file_path) => {
+          if !flags.unstable {
+            exit_unstable("--importmap")
+          }
+          Some(ImportMap::load(file_path)?)
+        }
+      };
+
     let inner = GlobalStateInner {
       dir,
       permissions: Permissions::from_flags(&flags),
       flags,
       file_fetcher,
       ts_compiler,
       lockfile,
+      maybe_import_map,
       compiler_starts: AtomicUsize::new(0),
       compile_lock: AsyncMutex::new(()),
     };
     Ok(GlobalState(Arc::new(inner)))
   }
 
-  pub async fn fetch_compiled_module(
+  /// This function is called when new module load is
+  /// initialized by the EsIsolate. Its resposibility is to collect
+  /// all dependencies and if it is required then also perform TS typecheck
+  /// and traspilation.
+  pub async fn prepare_module_load(
     &self,
     module_specifier: ModuleSpecifier,
     maybe_referrer: Option<ModuleSpecifier>,
     target_lib: TargetLib,
     permissions: Permissions,
     is_dyn_import: bool,
+    maybe_import_map: Option<ImportMap>,
+  ) -> Result<(), ErrBox> {
+    let module_specifier = module_specifier.clone();
+
+    // TODO(ry) Try to lift compile_lock as high up in the call stack for
+    // sanity.
+    let compile_lock = self.compile_lock.lock().await;
+
+    let mut module_graph_loader = ModuleGraphLoader::new(
+      self.file_fetcher.clone(),
+      maybe_import_map,
+      permissions.clone(),
+      is_dyn_import,
+      false,
+    );
+    module_graph_loader
+      .add_to_graph(&module_specifier, maybe_referrer)
+      .await?;
+    let module_graph = module_graph_loader.get_graph();
+
+    let out = self
+      .file_fetcher
+      .fetch_cached_source_file(&module_specifier, permissions.clone())
+      .expect("Source file not found");
+
+    // Check if we need to compile files
+    let needs_compilation = match out.media_type {
+      msg::MediaType::TypeScript
+      | msg::MediaType::TSX
+      | msg::MediaType::JSX => true,
+      msg::MediaType::JavaScript => self.ts_compiler.compile_js,
+      _ => false,
+    };
+
+    if needs_compilation {
+      self
+        .ts_compiler
+        .compile_module_graph(
+          self.clone(),
+          &out,
+          target_lib,
+          permissions,
+          module_graph,
+        )
+        .await?;
+    }
+
+    drop(compile_lock);
+
+    Ok(())
+  }
+
+  // TODO(bartlomieju): this method doesn't need to be async anymore
+  /// This method is used after `prepare_module_load` finishes and EsIsolate
+  /// starts loading source and executing source code. This method shouldn't
+  /// perform any IO (besides $DENO_DIR) and only operate on sources collected
+  /// during `prepare_module_load`.
+  pub async fn fetch_compiled_module(
+    &self,
+    module_specifier: ModuleSpecifier,
+    _maybe_referrer: Option<ModuleSpecifier>,
   ) -> Result<CompiledModule, ErrBox> {
     let state1 = self.clone();
     let state2 = self.clone();
     let module_specifier = module_specifier.clone();
 
     let out = self
       .file_fetcher
-      .fetch_source_file(&module_specifier, maybe_referrer, permissions.clone())
-      .await?;
+      .fetch_cached_source_file(&module_specifier, Permissions::allow_all())
+      .expect("Cached source file doesn't exist");
 
     // TODO(ry) Try to lift compile_lock as high up in the call stack for
     // sanity.
     let compile_lock = self.compile_lock.lock().await;
 
-    let compiled_module = match out.media_type {
+    // Check if we need to compile files
+    let was_compiled = match out.media_type {
       msg::MediaType::TypeScript
       | msg::MediaType::TSX
-      | msg::MediaType::JSX => {
-        state1
-          .ts_compiler
-          .compile(state1.clone(), &out, target_lib, permissions, is_dyn_import)
-          .await
-      }
-      msg::MediaType::JavaScript => {
-        if state1.ts_compiler.compile_js {
-          state2
-            .ts_compiler
-            .compile(
-              state1.clone(),
-              &out,
-              target_lib,
-              permissions,
-              is_dyn_import,
-            )
-            .await
-        } else {
-          if let Some(types_url) = out.types_url.clone() {
-            let types_specifier = ModuleSpecifier::from(types_url);
-            state1
-              .file_fetcher
-              .fetch_source_file(
-                &types_specifier,
-                Some(module_specifier.clone()),
-                permissions.clone(),
-              )
-              .await
-              .ok();
-          };
-
-          Ok(CompiledModule {
-            code: String::from_utf8(out.source_code.clone())?,
-            name: out.url.to_string(),
-          })
-        }
-      }
-      _ => Ok(CompiledModule {
+      | msg::MediaType::JSX => true,
+      msg::MediaType::JavaScript => self.ts_compiler.compile_js,
+      _ => false,
+    };
+
+    let compiled_module = if was_compiled {
+      state1.ts_compiler.get_compiled_module(&out.url)?
+    } else {
+      CompiledModule {
         code: String::from_utf8(out.source_code.clone())?,
         name: out.url.to_string(),
-      }),
-    }?;
+      }
+    };
+
     drop(compile_lock);
 
     if let Some(ref lockfile) = state2.lockfile {
@@ -193,11 +246,3 @@ fn thread_safe() {
   fn f<S: Send + Sync>(_: S) {}
   f(GlobalState::mock(vec![]));
 }
-
-#[test]
-fn import_map_given_for_repl() {
-  let _result = GlobalState::new(flags::Flags {
-    import_map_path: Some("import_map.json".to_string()),
-    ..flags::Flags::default()
-  });
-}

cli/main.rs

@@ -74,12 +74,10 @@ use crate::file_fetcher::SourceFile;
 use crate::file_fetcher::SourceFileFetcher;
 use crate::fs as deno_fs;
 use crate::global_state::GlobalState;
-use crate::import_map::ImportMap;
 use crate::msg::MediaType;
 use crate::op_error::OpError;
 use crate::ops::io::get_stdio;
 use crate::permissions::Permissions;
-use crate::state::exit_unstable;
 use crate::state::State;
 use crate::tsc::TargetLib;
 use crate::worker::MainWorker;
@@ -158,7 +156,13 @@ fn create_main_worker(
   global_state: GlobalState,
   main_module: ModuleSpecifier,
 ) -> Result<MainWorker, ErrBox> {
-  let state = State::new(global_state, None, main_module, false)?;
+  let state = State::new(
+    global_state.clone(),
+    None,
+    main_module,
+    global_state.maybe_import_map.clone(),
+    false,
+  )?;
 
   let mut worker = MainWorker::new(
     "main".to_string(),
@@ -222,16 +226,21 @@ async fn print_file_info(
   );
 
   let module_specifier_ = module_specifier.clone();
+
   global_state
-    .clone()
-    .fetch_compiled_module(
-      module_specifier_,
+    .prepare_module_load(
+      module_specifier_.clone(),
       None,
       TargetLib::Main,
       Permissions::allow_all(),
       false,
+      global_state.maybe_import_map.clone(),
     )
     .await?;
+  global_state
+    .clone()
+    .fetch_compiled_module(module_specifier_, None)
+    .await?;
 
   if out.media_type == msg::MediaType::TypeScript
     || (out.media_type == msg::MediaType::JavaScript
@@ -403,43 +412,49 @@ async fn bundle_command(
   source_file: String,
   out_file: Option<PathBuf>,
 ) -> Result<(), ErrBox> {
-  let mut module_name = ModuleSpecifier::resolve_url_or_path(&source_file)?;
-  let url = module_name.as_url();
+  let mut module_specifier =
+    ModuleSpecifier::resolve_url_or_path(&source_file)?;
+  let url = module_specifier.as_url();
 
   // TODO(bartlomieju): fix this hack in ModuleSpecifier
   if url.scheme() == "file" {
     let a = deno_fs::normalize_path(&url.to_file_path().unwrap());
     let u = Url::from_file_path(a).unwrap();
-    module_name = ModuleSpecifier::from(u)
+    module_specifier = ModuleSpecifier::from(u)
   }
 
   debug!(">>>>> bundle START");
   let compiler_config = tsc::CompilerConfig::load(flags.config_path.clone())?;
 
-  let maybe_import_map = match flags.import_map_path.as_ref() {
-    None => None,
-    Some(file_path) => {
-      if !flags.unstable {
-        exit_unstable("--importmap")
-      }
-      Some(ImportMap::load(file_path)?)
-    }
-  };
-
   let global_state = GlobalState::new(flags)?;
 
-  let bundle_result = tsc::bundle(
+  info!("Bundling {}", module_specifier.to_string());
+
+  let output = tsc::bundle(
     &global_state,
     compiler_config,
-    module_name,
-    maybe_import_map,
-    out_file,
+    module_specifier,
+    global_state.maybe_import_map.clone(),
     global_state.flags.unstable,
   )
-  .await;
+  .await?;
 
   debug!(">>>>> bundle END");
-  bundle_result
+
+  let output_string = fmt::format_text(&output)?;
+
+  if let Some(out_file_) = out_file.as_ref() {
+    info!("Emitting bundle to {:?}", out_file_);
+    let output_bytes = output_string.as_bytes();
+    let output_len = output_bytes.len();
+    deno_fs::write_file(out_file_, output_bytes, 0o666)?;
+    // TODO(bartlomieju): add "humanFileSize" method
+    info!("{} bytes emitted.", output_len);
+  } else {
+    println!("{}", output_string);
+  }
+
+  Ok(())
 }
 
 async fn doc_command(