feat(endo): Search for index.js and implied package exports

packages/endo/DESIGN.md

@@ -200,6 +200,8 @@ type Compartment = {
   location: Location,
   modules: ModuleMap,
   parsers: ParserMap,
+  types: ModuleParserMap,
+  scopes: ScopeMap,
   // The name of the realm to run the compartment within.
   // The default is a single frozen realm that has no name.
   realm: RealmName? // TODO
@@ -213,10 +215,13 @@ type Location string;
 // that do not correspond to source files in the same compartment.
 type ModuleMap = Object<InternalModuleSpecifier, Module>;
 
-// Module describes a module that isn't in the same
+// Module describes a module in a compartment.
+type Module = CompartmentModule | FileModule | ExitModule;
+
+// CompartmentModule describes a module that isn't in the same
 // compartment and how to introduce it to the compartment's
 // module namespace.
-type Module = {
+type CompartmentModule = {
   // The name of the foreign compartment:
   // TODO an absent compartment name may imply either
   // that the module is an internal alias of the
@@ -225,10 +230,32 @@ type Module = {
   // The name of the module in the foreign compartment's
   // module namespace:
   module: ExternalModuleSpecifier?,
-  // Alternately, that this module is not from
-  // any compartment and must be expressly passed
-  // into the compartment graph from the user.
-  parameter: ModuleParameter?, // TODO
+};
+
+// FileLocation is a URL for a module's file relative to the location of the
+// containing compartment.
+type FileLocation = string
+
+// FileModule is a module from a file.
+// When loading modules off a file system (src/import.js), the assembler
+// does not need any explicit FileModules, and instead relies on the
+// compartment to declare a ParserMap and optionally ModuleParserMap and
+// ScopeMap.
+// Endo provides a Compartment importHook and moduleMapHook that will
+// search the filesystem for candidate module files and infer the type from the
+// extension when necessary.
+type FileModule = {
+   location: FileLocation
+   parser: Parser
+};
+
+// ExitName is the name of a built-in module, to be threaded in from the
+// modules passed to the module executor.
+type ExitName = string;
+
+// ExitModule refers to a module that comes from outside the compartment map.
+type ExitModule = {
+  exit: ExitName
 };
 
 // InternalModuleSpecifier is the module specifier
@@ -260,6 +287,33 @@ type Extension = string;
 // "json" corresponds to JSON.
 type Parser = "mjs" | "cjs" | "json";
 
+// ModuleParserMap is a table of internal module specifiers
+// to the parser that should be used, regardless of that module's
+// extension.
+// Node.js allows the "module" property in package.json to denote
+// a file that is an ECMAScript module, regardless of its extension.
+// This is the mechanism that allows Endo to respect that behavior.
+type ModuleParserMap = Object<InternalModuleSpecifier, Parser>
+
+// ScopeMap is a map from internal module specifier prefixes
+// like "dependency" or "@organization/dependency" to another
+// compartment.
+// Endo uses this to build a moduleMapHook that can dynamically
+// generate entries for a compartment's moduleMap into
+// Node.js packages that do not explicitly state their "exports".
+// For these modules, any specifier under that prefix corresponds
+// to a link into some internal module of the foreign compartment.
+// When Endo creates an archive, it captures all of the Modules
+// explicitly and erases the scopes entry.
+type ScopeMap = Object<InternalModuleSpecifier, Scope>
+
+// Scope describes the compartment to use for all ad-hoc
+// entries in the compartment's module map.
+type Scope = {
+  compartment: CompartmentName
+}
+
+
 // TODO everything hereafter...
 
 // Realm describes another realm to contain one or more

packages/endo/README.md

@@ -246,6 +246,25 @@ backward compatibility.
 However, packages that have a `type` property that explicitly says `module`
 will treat a `.js` file as an ECMAScript module.
 
+This unforunately conflicts with packages written to work with the ECMAScript
+module system emulator in the `esm` package on npm, which allows every file
+with the `js` extension to be an ECMAScript module that presents itself to
+Node.js as a CommonJS module.
+To overcome such obstacles, Endo will accept a non-standard `parsers` property
+in `package.json` that maps file extensions, specifically `js` to the
+corresponding language name, one of `mjs` for ECMAScript modules, `cjs` for
+CommonJS modules, and `json` for JSON modules.
+All other language names are reserved and the defaults for files with the
+extensions `cjs`, `mjs`, and `json` default to the language of the same name
+unless overridden.
+If Endo sees `parsers`, it ignores `type`, so these can contradict where using
+the `esm` emulator requires.
+
+```json
+{
+  "parsers": {"js": "mjs"}
+}
+```
 
 Many Node.js applications using CommonJS modules expect to be able to `require`
 a JSON file like `package.json`.

packages/endo/bin/endo

@@ -0,0 +1 @@
+endo.js

packages/endo/bin/endo.js

@@ -0,0 +1,6 @@
+#!/usr/bin/env node
+(async () => {
+  const fs = await import("fs");
+  const { main } = await import("../src/cli.js");
+  main(process, { fs: fs.promises });
+})();

packages/endo/mitm/.gitignore

@@ -0,0 +1 @@
+node

packages/endo/package.json

@@ -14,12 +14,16 @@
     "require": "./dist/endo.cjs",
     "browser": "./dist/endo.umd.js"
   },
+  "bin": {
+    "endo": "./bin/endo.js"
+  },
   "scripts": {
     "build": "rollup --config rollup.config.js",
     "clean": "rm -rf dist",
     "depcheck": "depcheck",
     "lint": "eslint '**/*.js'",
     "lint-fix": "eslint --fix '**/*.js'",
+    "postinstall": "node src/postinstall.js",
     "prepublish": "yarn clean && yarn build",
     "test": "yarn build && tap --no-esm --no-coverage --reporter spec 'test/**/*.test.js'"
   },

packages/endo/src/archive.js

@@ -1,54 +1,85 @@
-/* global StaticModuleRecord */
 /* eslint no-shadow: 0 */
 
 import { writeZip } from "./zip.js";
-import { resolve, join } from "./node-module-specifier.js";
+import { resolve } from "./node-module-specifier.js";
 import { parseExtension } from "./extension.js";
 import { compartmentMapForNodeModules } from "./compartmap.js";
 import { search } from "./search.js";
 import { assemble } from "./assemble.js";
 
-const { entries, fromEntries } = Object;
+const { entries, freeze, fromEntries, values } = Object;
 
+// q, as in quote, for quoted strings in error messages.
 const q = JSON.stringify;
 
 const encoder = new TextEncoder();
 const decoder = new TextDecoder();
 
 const resolveLocation = (rel, abs) => new URL(rel, abs).toString();
 
-const makeRecordingImportHookMaker = (read, baseLocation, manifest, errors) => {
+const makeRecordingImportHookMaker = (read, baseLocation, sources) => {
   // per-assembly:
   const makeImportHook = (packageLocation, parse) => {
     // per-compartment:
     packageLocation = resolveLocation(packageLocation, baseLocation);
+    const packageSources = sources[packageLocation] || {};
+    sources[packageLocation] = packageSources;
+
     const importHook = async moduleSpecifier => {
       // per-module:
+
+      // In Node.js, an absolute specifier always indicates a built-in or
+      // third-party dependency.
+      // The `moduleMapHook` captures all third-party dependencies.
+      if (moduleSpecifier !== "." && !moduleSpecifier.startsWith("./")) {
+        packageSources[moduleSpecifier] = {
+          exit: moduleSpecifier
+        };
+        // Return a place-holder.
+        // Archived compartments are not executed.
+        return freeze({ imports: [], execute() {} });
+      }
+
       const candidates = [moduleSpecifier];
       if (parseExtension(moduleSpecifier) === "") {
         candidates.push(`${moduleSpecifier}.js`, `${moduleSpecifier}/index.js`);
       }
       for (const candidate of candidates) {
-        const moduleLocation = new URL(candidate, packageLocation).toString();
+        const moduleLocation = resolveLocation(candidate, packageLocation);
         // eslint-disable-next-line no-await-in-loop
         const moduleBytes = await read(moduleLocation).catch(
           _error => undefined
         );
-        if (moduleBytes === undefined) {
-          errors.push(
-            `missing ${q(candidate)} needed for package ${q(packageLocation)}`
-          );
-        } else {
+        if (moduleBytes !== undefined) {
           const moduleSource = decoder.decode(moduleBytes);
 
-          const packageManifest = manifest[packageLocation] || {};
-          manifest[packageLocation] = packageManifest;
-          packageManifest[moduleSpecifier] = moduleBytes;
+          const { record, parser } = parse(
+            moduleSource,
+            moduleSpecifier,
+            moduleLocation
+          );
 
-          return parse(moduleSource, moduleSpecifier, moduleLocation);
+          const packageRelativeLocation = moduleLocation.slice(
+            packageLocation.length
+          );
+          packageSources[moduleSpecifier] = {
+            location: packageRelativeLocation,
+            parser,
+            bytes: moduleBytes
+          };
+
+          return record;
         }
       }
-      return new StaticModuleRecord("// Module not found", moduleSpecifier);
+
+      // TODO offer breadcrumbs in the error message, or how to construct breadcrumbs with another tool.
+      throw new Error(
+        `Cannot find file for internal module ${q(
+          moduleSpecifier
+        )} (with candidates ${candidates
+          .map(q)
+          .join(", ")}) in package ${packageLocation}`
+      );
     };
     return importHook;
   };
@@ -60,16 +91,18 @@ const renameCompartments = compartments => {
   let n = 0;
   for (const [name, compartment] of entries(compartments)) {
     const { label } = compartment;
-    renames[name] = `${label}#${n}`;
+    renames[name] = `${label}-n${n}`;
     n += 1;
   }
   return renames;
 };
 
-const renameCompartmentMap = (compartments, renames) => {
+const translateCompartmentMap = (compartments, sources, renames) => {
   const result = {};
   for (const [name, compartment] of entries(compartments)) {
-    const { label, parsers, types } = compartment;
+    const { label } = compartment;
+
+    // rename module compartments
     const modules = {};
     for (const [name, module] of entries(compartment.modules || {})) {
       const compartment = module.compartment
@@ -80,14 +113,27 @@ const renameCompartmentMap = (compartments, renames) => {
         compartment
       };
     }
+
+    // integrate sources into modules
+    const compartmentSources = sources[name];
+    for (const [name, source] of entries(compartmentSources || {})) {
+      const { location, parser, exit } = source;
+      modules[name] = {
+        location,
+        parser,
+        exit
+      };
+    }
+
     result[renames[name]] = {
       label,
       location: renames[name],
-      modules,
-      parsers,
-      types
+      modules
+      // `scopes`, `types`, and `parsers` are not necessary since every
+      // loadable module is captured in `modules`.
     };
   }
+
   return result;
 };
 
@@ -99,10 +145,18 @@ const renameSources = (sources, renames) => {
 
 const addSourcesToArchive = async (archive, sources) => {
   for (const [compartment, modules] of entries(sources)) {
-    for (const [module, content] of entries(modules)) {
-      const path = join(compartment, module);
+    const compartmentLocation = resolveLocation(
+      `${encodeURIComponent(compartment)}/`,
+      "file:///"
+    );
+    for (const { location, bytes } of values(modules)) {
+      const moduleLocation = resolveLocation(
+        encodeURIComponent(location),
+        compartmentLocation
+      );
+      const path = new URL(moduleLocation).pathname.slice(1); // elide initial "/"
       // eslint-disable-next-line no-await-in-loop
-      await archive.write(path, content);
+      await archive.write(path, bytes);
     }
   }
 };
@@ -125,22 +179,12 @@ export const makeArchive = async (read, moduleLocation) => {
   const { compartments, main } = compartmentMap;
 
   const sources = {};
-  const errors = [];
   const makeImportHook = makeRecordingImportHookMaker(
     read,
     packageLocation,
-    sources,
-    errors
+    sources
   );
 
-  if (errors.length > 0) {
-    throw new Error(
-      `Cannot assemble compartment for ${errors.length} reasons: ${errors.join(
-        ", "
-      )}`
-    );
-  }
-
   // Induce importHook to record all the necessary modules to import the given module specifier.
   const compartment = assemble({
     name: main,
@@ -151,7 +195,11 @@ export const makeArchive = async (read, moduleLocation) => {
   await compartment.load(moduleSpecifier);
 
   const renames = renameCompartments(compartments);
-  const renamedCompartments = renameCompartmentMap(compartments, renames);
+  const renamedCompartments = translateCompartmentMap(
+    compartments,
+    sources,
+    renames
+  );
   const renamedSources = renameSources(sources, renames);
 
   const manifest = {