build: enable loading internal modules from disk

PR-URL: nodejs#31321
Reviewed-By: Anna Henningsen <anna@addaleax.net>
Reviewed-By: Tobias Nießen <tniessen@tnie.de>
Reviewed-By: Joyee Cheung <joyeec9h3@gmail.com>
Reviewed-By: Rich Trott <rtrott@gmail.com>
Reviewed-By: Colin Ihrig <cjihrig@gmail.com>
Reviewed-By: Matteo Collina <matteo.collina@gmail.com>

configure.py

@@ -630,6 +630,12 @@
     default=False,
     help='compile V8 with minimal optimizations and with runtime checks')
 
+parser.add_option('--node-builtin-modules-path',
+    action='store',
+    dest='node_builtin_modules_path',
+    default=False,
+    help='node will load builtin modules from disk instead of from binary')
+
 # Create compile_commands.json in out/Debug and out/Release.
 parser.add_option('-C',
     action='store_true',
@@ -1023,18 +1029,18 @@ def configure_node(o):
   o['variables']['want_separate_host_toolset'] = int(
       cross_compiling and want_snapshots)
 
-  if not options.without_node_snapshot:
+  if options.without_node_snapshot or options.node_builtin_modules_path:
+    o['variables']['node_use_node_snapshot'] = 'false'
+  else:
     o['variables']['node_use_node_snapshot'] = b(
       not cross_compiling and want_snapshots and not options.shared)
-  else:
-    o['variables']['node_use_node_snapshot'] = 'false'
 
-  if not options.without_node_code_cache:
+  if options.without_node_code_cache or options.node_builtin_modules_path:
+    o['variables']['node_use_node_code_cache'] = 'false'
+  else:
     # TODO(refack): fix this when implementing embedded code-cache when cross-compiling.
     o['variables']['node_use_node_code_cache'] = b(
       not cross_compiling and not options.shared)
-  else:
-    o['variables']['node_use_node_code_cache'] = 'false'
 
   if target_arch == 'arm':
     configure_arm(o)
@@ -1191,6 +1197,10 @@ def configure_node(o):
   else:
     o['variables']['node_target_type'] = 'executable'
 
+  if options.node_builtin_modules_path:
+    print('Warning! Loading builtin modules from disk is for development')
+    o['variables']['node_builtin_modules_path'] = options.node_builtin_modules_path
+
 def configure_napi(output):
   version = getnapibuildversion.get_napi_version()
   output['variables']['napi_build_version'] = version

node.gyp

@@ -24,6 +24,7 @@
     'node_core_target_name%': 'node',
     'node_lib_target_name%': 'libnode',
     'node_intermediate_lib_type%': 'static_library',
+    'node_builtin_modules_path%': '',
     'library_files': [
       'lib/internal/bootstrap/environment.js',
       'lib/internal/bootstrap/loaders.js',
@@ -716,6 +717,9 @@
       'msvs_disabled_warnings!': [4244],
 
       'conditions': [
+        [ 'node_builtin_modules_path!=""', {
+          'defines': [ 'NODE_BUILTIN_MODULES_PATH="<(node_builtin_modules_path)"' ]
+        }],
         [ 'node_shared=="true"', {
           'sources': [
             'src/node_snapshot_stub.cc',

src/node.cc

@@ -27,6 +27,7 @@
 #include "env-inl.h"
 #include "memory_tracker-inl.h"
 #include "node_binding.h"
+#include "node_errors.h"
 #include "node_internals.h"
 #include "node_main_instance.h"
 #include "node_metadata.h"

src/node_native_module.cc

@@ -174,6 +174,64 @@ MaybeLocal<Function> NativeModuleLoader::CompileAsModule(
   return LookupAndCompile(context, id, &parameters, result);
 }
 
+#ifdef NODE_BUILTIN_MODULES_PATH
+static std::string OnDiskFileName(const char* id) {
+  std::string filename = NODE_BUILTIN_MODULES_PATH;
+  filename += "/";
+
+  if (strncmp(id, "internal/deps", strlen("internal/deps")) == 0) {
+    id += strlen("internal/");
+  } else {
+    filename += "lib/";
+  }
+  filename += id;
+  filename += ".js";
+
+  return filename;
+}
+#endif  // NODE_BUILTIN_MODULES_PATH
+
+MaybeLocal<String> NativeModuleLoader::LoadBuiltinModuleSource(Isolate* isolate,
+                                                               const char* id) {
+#ifdef NODE_BUILTIN_MODULES_PATH
+  std::string filename = OnDiskFileName(id);
+
+  uv_fs_t req;
+  uv_file file =
+      uv_fs_open(nullptr, &req, filename.c_str(), O_RDONLY, 0, nullptr);
+  CHECK_GE(req.result, 0);
+  uv_fs_req_cleanup(&req);
+
+  std::shared_ptr<void> defer_close(nullptr, [file](...) {
+    uv_fs_t close_req;
+    CHECK_EQ(0, uv_fs_close(nullptr, &close_req, file, nullptr));
+    uv_fs_req_cleanup(&close_req);
+  });
+
+  std::string contents;
+  char buffer[4096];
+  uv_buf_t buf = uv_buf_init(buffer, sizeof(buffer));
+
+  while (true) {
+    const int r =
+        uv_fs_read(nullptr, &req, file, &buf, 1, contents.length(), nullptr);
+    CHECK_GE(req.result, 0);
+    uv_fs_req_cleanup(&req);
+    if (r <= 0) {
+      break;
+    }
+    contents.append(buf.base, r);
+  }
+
+  return String::NewFromUtf8(
+      isolate, contents.c_str(), v8::NewStringType::kNormal, contents.length());
+#else
+  const auto source_it = source_.find(id);
+  CHECK_NE(source_it, source_.end());
+  return source_it->second.ToStringChecked(isolate);
+#endif  // NODE_BUILTIN_MODULES_PATH
+}
+
 // Returns Local<Function> of the compiled module if return_code_cache
 // is false (we are only compiling the function).
 // Otherwise return a Local<Object> containing the cache.
@@ -185,9 +243,10 @@ MaybeLocal<Function> NativeModuleLoader::LookupAndCompile(
   Isolate* isolate = context->GetIsolate();
   EscapableHandleScope scope(isolate);
 
-  const auto source_it = source_.find(id);
-  CHECK_NE(source_it, source_.end());
-  Local<String> source = source_it->second.ToStringChecked(isolate);
+  Local<String> source;
+  if (!LoadBuiltinModuleSource(isolate, id).ToLocal(&source)) {
+    return {};
+  }
 
   std::string filename_s = id + std::string(".js");
   Local<String> filename =

src/node_native_module.h

@@ -66,6 +66,8 @@ class NativeModuleLoader {
   NativeModuleCacheMap* code_cache();
   v8::ScriptCompiler::CachedData* GetCodeCache(const char* id) const;
   enum class Result { kWithCache, kWithoutCache };
+  v8::MaybeLocal<v8::String> LoadBuiltinModuleSource(v8::Isolate* isolate,
+                                                     const char* id);
   // If an exception is encountered (e.g. source code contains
   // syntax error), the returned value is empty.
   v8::MaybeLocal<v8::Function> LookupAndCompile(