feat(endo): Search for index.js and implied package exports

When asked for a module named "x", Node.js will search for "x.js" then "x/index.js".

For packages that do not supply an "exports" property in their "package.json", any module contained by that package is a valid exported module.

To achieve parity with these two features, Endo uses different techniques when loading from the file system and loading from an archive.

When reading from the file system, Endo will search for a satsifactory candidate in each compartment's asynchronous importHook.  Endo also uses the compartment's new moduleMapHook to search for dependency compartments in the "scope" of a module identifier prefix.  These allow Endo to operate from an incomplete compartment map for the initial load.

The Endo archiver instead creates a more complete compartment map, with every discovered module.  This introduces a new kind of module to the compartment map module descriptor type union: modules with known locations and corresponding parsers.  The archiver erases the "scopes", "types", and "parsers" on each compartment description since they are no longer necessary.

When reading from an archive, Endo uses an importHook that consults the compartment map directly for the locations of all contained modules, and creates a complete moduleMap up front.

packages/endo/DESIGN.md

@@ -200,6 +200,8 @@ type Compartment = {
   location: Location,
   modules: ModuleMap,
   parsers: ParserMap,
+  types: ModuleParserMap,
+  scopes: ScopeMap,
   // The name of the realm to run the compartment within.
   // The default is a single frozen realm that has no name.
   realm: RealmName? // TODO
@@ -213,10 +215,13 @@ type Location string;
 // that do not correspond to source files in the same compartment.
 type ModuleMap = Object<InternalModuleSpecifier, Module>;
 
-// Module describes a module that isn't in the same
+// Module describes a module in a compartment.
+type Module = CompartmentModule | FileModule | ExitModule;
+
+// CompartmentModule describes a module that isn't in the same
 // compartment and how to introduce it to the compartment's
 // module namespace.
-type Module = {
+type CompartmentModule = {
   // The name of the foreign compartment:
   // TODO an absent compartment name may imply either
   // that the module is an internal alias of the
@@ -225,10 +230,32 @@ type Module = {
   // The name of the module in the foreign compartment's
   // module namespace:
   module: ExternalModuleSpecifier?,
-  // Alternately, that this module is not from
-  // any compartment and must be expressly passed
-  // into the compartment graph from the user.
-  parameter: ModuleParameter?, // TODO
+};
+
+// FileLocation is a URL for a module's file relative to the location of the
+// containing compartment.
+type FileLocation = string
+
+// FileModule is a module from a file.
+// When loading modules off a file system (src/import.js), the assembler
+// does not need any explicit FileModules, and instead relies on the
+// compartment to declare a ParserMap and optionally ModuleParserMap and
+// ScopeMap.
+// Endo provides a Compartment importHook and moduleMapHook that will
+// search the filesystem for candidate module files and infer the type from the
+// extension when necessary.
+type FileModule = {
+   location: FileLocation
+   parser: Parser
+};
+
+// ExitName is the name of a built-in module, to be threaded in from the
+// modules passed to the module executor.
+type ExitName string;
+
+// ExitModule refers to a module that comes from outside the compartment map.
+type ExitModule = {
+  exit: ExitName
 };
 
 // InternalModuleSpecifier is the module specifier
@@ -260,6 +287,33 @@ type Extension = string;
 // "json" corresponds to JSON.
 type Parser = "mjs" | "cjs" | "json";
 
+// ModuleParserMap is a table of internal module specifiers
+// to the parser that should be used, regardless of that module's
+// extension.
+// Node.js allows the "module" property in package.json to denote
+// a file that is an ECMAScript module, regardless of its extension.
+// This is the mechanism that allows Endo to respect that behavior.
+type ModuleParserMap = Object<InternalModuleSpecifier, Parser>
+
+// ScopeMap is a map from internal module specifier prefixes
+// like "dependency" or "@organization/dependency" to another
+// compartment.
+// Endo uses this to build a moduleMapHook that can dynamically
+// generate entries for a compartment's moduleMap into
+// Node.js packages that do not explicitly state their "exports".
+// For these modules, any specifier under that prefix corresponds
+// to a link into some internal module of the foreign compartment.
+// When Endo creates an archive, it captures all of the Modules
+// explicitly and erases the scopes entry.
+type ScopeMap = Object<InternalModuleSpecifier, Scope>
+
+// Scope describes the compartment to use for all ad-hoc
+// entries in the compartment's module map.
+type Scope = {
+  compartment: CompartmentName
+}
+
+
 // TODO everything hereafter...
 
 // Realm describes another realm to contain one or more

packages/endo/src/archive.js

@@ -1,54 +1,85 @@
-/* global StaticModuleRecord */
 /* eslint no-shadow: 0 */
 
 import { writeZip } from "./zip.js";
-import { resolve, join } from "./node-module-specifier.js";
+import { resolve } from "./node-module-specifier.js";
 import { parseExtension } from "./extension.js";
 import { compartmentMapForNodeModules } from "./compartmap.js";
 import { search } from "./search.js";
 import { assemble } from "./assemble.js";
 
-const { entries, fromEntries } = Object;
+const { entries, freeze, fromEntries, values } = Object;
 
+// q, as in quote, for quoted strings in error messages.
 const q = JSON.stringify;
 
 const encoder = new TextEncoder();
 const decoder = new TextDecoder();
 
 const resolveLocation = (rel, abs) => new URL(rel, abs).toString();
 
-const makeRecordingImportHookMaker = (read, baseLocation, manifest, errors) => {
+const makeRecordingImportHookMaker = (read, baseLocation, sources) => {
   // per-assembly:
   const makeImportHook = (packageLocation, parse) => {
     // per-compartment:
     packageLocation = resolveLocation(packageLocation, baseLocation);
+    const packageSources = sources[packageLocation] || {};
+    sources[packageLocation] = packageSources;
+
     const importHook = async moduleSpecifier => {
       // per-module:
+
+      // In Node.js, an absolute specifier always indicates a built-in or
+      // third-party dependency.
+      // The `moduleMapHook` captures all third-party dependencies.
+      if (moduleSpecifier !== "." && !moduleSpecifier.startsWith("./")) {
+        packageSources[moduleSpecifier] = {
+          exit: moduleSpecifier
+        };
+        // Return a place-holder.
+        // Archived compartments are not executed.
+        return freeze({ imports: [], execute() {} });
+      }
+
       const candidates = [moduleSpecifier];
       if (parseExtension(moduleSpecifier) === "") {
         candidates.push(`${moduleSpecifier}.js`, `${moduleSpecifier}/index.js`);
       }
       for (const candidate of candidates) {
-        const moduleLocation = new URL(candidate, packageLocation).toString();
+        const moduleLocation = resolveLocation(candidate, packageLocation);
         // eslint-disable-next-line no-await-in-loop
         const moduleBytes = await read(moduleLocation).catch(
           _error => undefined
         );
-        if (moduleBytes === undefined) {
-          errors.push(
-            `missing ${q(candidate)} needed for package ${q(packageLocation)}`
-          );
-        } else {
+        if (moduleBytes !== undefined) {
           const moduleSource = decoder.decode(moduleBytes);
 
-          const packageManifest = manifest[packageLocation] || {};
-          manifest[packageLocation] = packageManifest;
-          packageManifest[moduleSpecifier] = moduleBytes;
+          const { record, parser } = parse(
+            moduleSource,
+            moduleSpecifier,
+            moduleLocation
+          );
 
-          return parse(moduleSource, moduleSpecifier, moduleLocation);
+          const packageRelativeLocation = moduleLocation.slice(
+            packageLocation.length
+          );
+          packageSources[moduleSpecifier] = {
+            location: packageRelativeLocation,
+            parser,
+            bytes: moduleBytes
+          };
+
+          return record;
         }
       }
-      return new StaticModuleRecord("// Module not found", moduleSpecifier);
+
+      // TODO offer breadcrumbs in the error message, or how to construct breadcrumbs with another tool.
+      throw new Error(
+        `Cannot find file for internal module ${q(
+          moduleSpecifier
+        )} (with candidates ${candidates
+          .map(q)
+          .join(", ")}) in package ${packageLocation}`
+      );
     };
     return importHook;
   };
@@ -60,16 +91,18 @@ const renameCompartments = compartments => {
   let n = 0;
   for (const [name, compartment] of entries(compartments)) {
     const { label } = compartment;
-    renames[name] = `${label}#${n}`;
+    renames[name] = `${label}-n${n}`;
     n += 1;
   }
   return renames;
 };
 
-const renameCompartmentMap = (compartments, renames) => {
+const translateCompartmentMap = (compartments, sources, renames) => {
   const result = {};
   for (const [name, compartment] of entries(compartments)) {
-    const { label, parsers, types } = compartment;
+    const { label } = compartment;
+
+    // rename module compartments
     const modules = {};
     for (const [name, module] of entries(compartment.modules || {})) {
       const compartment = module.compartment
@@ -80,14 +113,27 @@ const renameCompartmentMap = (compartments, renames) => {
         compartment
       };
     }
+
+    // integrate sources into modules
+    const compartmentSources = sources[name];
+    for (const [name, source] of entries(compartmentSources || {})) {
+      const { location, parser, exit } = source;
+      modules[name] = {
+        location,
+        parser,
+        exit
+      };
+    }
+
     result[renames[name]] = {
       label,
       location: renames[name],
-      modules,
-      parsers,
-      types
+      modules
+      // `scopes`, `types`, and `parsers` are not necessary since every
+      // loadable module is captured in `modules`.
     };
   }
+
   return result;
 };
 
@@ -99,10 +145,18 @@ const renameSources = (sources, renames) => {
 
 const addSourcesToArchive = async (archive, sources) => {
   for (const [compartment, modules] of entries(sources)) {
-    for (const [module, content] of entries(modules)) {
-      const path = join(compartment, module);
+    const compartmentLocation = resolveLocation(
+      `${encodeURIComponent(compartment)}/`,
+      "file:///"
+    );
+    for (const { location, bytes } of values(modules)) {
+      const moduleLocation = resolveLocation(
+        encodeURIComponent(location),
+        compartmentLocation
+      );
+      const path = new URL(moduleLocation).pathname.slice(1); // elide initial "/"
       // eslint-disable-next-line no-await-in-loop
-      await archive.write(path, content);
+      await archive.write(path, bytes);
     }
   }
 };
@@ -125,22 +179,12 @@ export const makeArchive = async (read, moduleLocation) => {
   const { compartments, main } = compartmentMap;
 
   const sources = {};
-  const errors = [];
   const makeImportHook = makeRecordingImportHookMaker(
     read,
     packageLocation,
-    sources,
-    errors
+    sources
   );
 
-  if (errors.length > 0) {
-    throw new Error(
-      `Cannot assemble compartment for ${errors.length} reasons: ${errors.join(
-        ", "
-      )}`
-    );
-  }
-
   // Induce importHook to record all the necessary modules to import the given module specifier.
   const compartment = assemble({
     name: main,
@@ -151,7 +195,11 @@ export const makeArchive = async (read, moduleLocation) => {
   await compartment.load(moduleSpecifier);
 
   const renames = renameCompartments(compartments);
-  const renamedCompartments = renameCompartmentMap(compartments, renames);
+  const renamedCompartments = translateCompartmentMap(
+    compartments,
+    sources,
+    renames
+  );
   const renamedSources = renameSources(sources, renames);
 
   const manifest = {

packages/endo/src/assemble.js

@@ -2,6 +2,7 @@
 
 import { resolve } from "./node-module-specifier.js";
 import { mapParsers } from "./parse.js";
+import { makeModuleMapHook } from "./module-map-hook.js";
 
 const { entries } = Object;
 
@@ -46,8 +47,41 @@ export const assemble = ({
     throw new Error(`Cannot assemble compartment graph that includes a cycle`);
   }
 
-  for (const [inner, outer] of entries(descriptor.modules || {})) {
-    const { compartment: compartmentName, module: moduleSpecifier } = outer;
+  descriptor.modules = descriptor.modules || {};
+  for (const [inner, outer] of entries(descriptor.modules)) {
+    const {
+      compartment: compartmentName,
+      module: moduleSpecifier,
+      exit
+    } = outer;
+    if (exit !== undefined) {
+      // TODO Currenly, only the entry package can connect to built-in modules.
+      // Policies should be able to allow third-party modules to exit to
+      // built-ins, or have built-ins subverted by modules from specific
+      // compartments.
+      const module = modules[exit];
+      if (module === undefined) {
+        throw new Error(
+          `Cannot assemble module graph with missing external module ${q(exit)}`
+        );
+      }
+      modules[inner] = module;
+    } else if (compartmentName !== undefined) {
+      const compartment = assemble({
+        name: compartmentName,
+        compartments,
+        makeImportHook,
+        parents: [...parents, name],
+        loaded,
+        Compartment
+      });
+      modules[inner] = compartment.module(moduleSpecifier);
+    }
+  }
+
+  const scopes = {};
+  for (const [prefix, scope] of entries(descriptor.scopes || {})) {
+    const { compartment: compartmentName } = scope;
     const compartment = assemble({
       name: compartmentName,
       compartments,
@@ -56,14 +90,16 @@ export const assemble = ({
       loaded,
       Compartment
     });
-    modules[inner] = compartment.module(moduleSpecifier);
+    scopes[prefix] = { compartment, compartmentName };
   }
 
-  const parse = mapParsers(descriptor.parsers, descriptor.types);
+  const parse = mapParsers(descriptor.parsers || {}, descriptor.types || {});
+  // TODO makeResolveHook that filters on file patterns
 
   const compartment = new Compartment(endowments, modules, {
     resolveHook: resolve,
-    importHook: makeImportHook(descriptor.location, parse)
+    importHook: makeImportHook(descriptor.location, parse),
+    moduleMapHook: makeModuleMapHook(scopes, descriptor.modules)
   });
 
   loaded[name] = compartment;