From c04592d541ea9b2cc68c930025291b878bd05f44 Mon Sep 17 00:00:00 2001
From: Kris Kowal <kris@agoric.com>
Date: Thu, 30 Jul 2020 19:18:34 -0700
Subject: [PATCH 01/13] feat(endo): Add command line

---
 packages/endo/bin/endo              |   1 +
 packages/endo/bin/endo.js           |   5 ++
 packages/endo/src/cli.js            | 130 ++++++++++++++++++++++++++++
 packages/endo/src/compartmap.js     |   2 +-
 packages/endo/src/import-archive.js |  19 ++--
 packages/endo/src/zip.js            |  12 ++-
 6 files changed, 159 insertions(+), 10 deletions(-)
 create mode 120000 packages/endo/bin/endo
 create mode 100755 packages/endo/bin/endo.js
 create mode 100644 packages/endo/src/cli.js

diff --git a/packages/endo/bin/endo b/packages/endo/bin/endo
new file mode 120000
index 0000000000..8237050ebd
--- /dev/null
+++ b/packages/endo/bin/endo
@@ -0,0 +1 @@
+endo.js
\ No newline at end of file
diff --git a/packages/endo/bin/endo.js b/packages/endo/bin/endo.js
new file mode 100755
index 0000000000..da980e130d
--- /dev/null
+++ b/packages/endo/bin/endo.js
@@ -0,0 +1,5 @@
+#!/usr/bin/env node
+import fs from "fs";
+import { main } from "../src/cli.js";
+
+main(process, { fs: fs.promises });
diff --git a/packages/endo/src/cli.js b/packages/endo/src/cli.js
new file mode 100644
index 0000000000..327bbc47e8
--- /dev/null
+++ b/packages/endo/src/cli.js
@@ -0,0 +1,130 @@
+/* global harden */
+
+import "./lockdown.js";
+import { loadLocation, writeArchive, loadArchive } from "./main.js";
+import { search } from "./search.js";
+import { compartmentMapForNodeModules } from "./compartmap.js";
+
+const iterate = sequence => sequence[Symbol.iterator]();
+
+function usage(message) {
+  console.error(message);
+  return 1;
+}
+
+async function execute(application) {
+  const endowments = harden({
+    console: {
+      log(...args) {
+        console.log(...args);
+      }
+    }
+  });
+
+  await application.execute(endowments);
+  return 0;
+}
+
+async function executeLocation(applicationPath, { cwd, read }) {
+  const currentLocation = new URL(`${cwd()}/`, "file:///");
+  const applicationLocation = new URL(applicationPath, currentLocation);
+  const application = await loadLocation(read, applicationLocation);
+  return execute(application);
+}
+
+async function archive(archivePath, applicationPath, { cwd, read, write }) {
+  const currentLocation = new URL(`${cwd()}/`, "file:///");
+  const archiveLocation = new URL(archivePath, currentLocation);
+  const applicationLocation = new URL(applicationPath, currentLocation);
+  await writeArchive(write, read, archiveLocation, applicationLocation);
+  return 0;
+}
+
+async function executeArchive(archivePath, { read, cwd }) {
+  const currentLocation = new URL(`${cwd()}/`, "file:///");
+  const archiveLocation = new URL(archivePath, currentLocation);
+  const application = await loadArchive(read, archiveLocation);
+  return execute(application);
+}
+
+async function compartmap(applicationPath, { read, cwd }) {
+  const currentLocation = new URL(`${cwd()}/`, "file:///");
+  const applicationLocation = new URL(applicationPath, currentLocation);
+  const { packageLocation } = await search(read, applicationLocation);
+  const compartmentMap = await compartmentMapForNodeModules(
+    read,
+    packageLocation
+  );
+  console.log(JSON.stringify(compartmentMap, null, 2));
+  return 0;
+}
+
+// run parses command line arguments and dispatches to a subcommand.
+async function run(args, powers) {
+  for (const arg of args) {
+    if (arg === "--") {
+      for (const applicationLocation of args) {
+        return executeLocation(applicationLocation, powers);
+      }
+    } else if (arg === "--compartmap") {
+      for (const applicationPath of args) {
+        const rem = Array.from(args);
+        if (rem.length > 0) {
+          return usage(
+            `Unexpected arguments after --compartmap ${applicationPath}: ${rem}`
+          );
+        }
+        return compartmap(applicationPath, powers);
+      }
+    } else if (arg === "-w") {
+      for (const archivePath of args) {
+        for (const applicationPath of args) {
+          const rem = Array.from(args);
+          if (rem.length > 0) {
+            return usage(
+              `Unexpected arguments after --archive ${archivePath} ${applicationPath}: ${rem}`
+            );
+          }
+          return archive(archivePath, applicationPath, powers);
+        }
+        return usage(`Expected application path`);
+      }
+      return usage(`Expected archive path`);
+    } else if (arg === "-x") {
+      for (const archivePath of args) {
+        const rem = Array.from(args);
+        if (rem.length > 0) {
+          return usage(`Unexpected arguments after -x ${archivePath}: ${rem}`);
+        }
+        return executeArchive(archivePath, powers);
+      }
+      return usage(`Expected archive path`);
+    } else if (arg.startsWith("-")) {
+      return usage(`Unrecognized flag ${arg}`);
+    } else {
+      const rem = Array.from(args);
+      if (rem.length > 0) {
+        return usage(`Unexpected arguments after ${arg}: ${rem}`);
+      }
+      return executeLocation(arg, powers);
+    }
+  }
+  return usage(`Expected script path or flag`);
+}
+
+export async function main(process, modules) {
+  const { fs } = modules;
+  const read = async location => fs.readFile(new URL(location).pathname);
+  const write = async (location, content) =>
+    fs.writeFile(new URL(location).pathname, content);
+  try {
+    const args = iterate(process.argv.slice(2));
+    process.exitCode = await run(args, {
+      read,
+      write,
+      cwd: process.cwd
+    });
+  } catch (error) {
+    process.exitCode = usage(error.stack || error.message);
+  }
+}
diff --git a/packages/endo/src/compartmap.js b/packages/endo/src/compartmap.js
index 01f435a175..6b5f9afc4f 100644
--- a/packages/endo/src/compartmap.js
+++ b/packages/endo/src/compartmap.js
@@ -102,7 +102,7 @@ const inferParsers = (type, location) => {
 // that the package exports.
 
 const graphPackage = async (
-  name,
+  name = "",
   readDescriptor,
   graph,
   { packageLocation, packageDescriptor },
diff --git a/packages/endo/src/import-archive.js b/packages/endo/src/import-archive.js
index bf3957c1fd..b6c9f501c8 100644
--- a/packages/endo/src/import-archive.js
+++ b/packages/endo/src/import-archive.js
@@ -22,8 +22,8 @@ const makeArchiveImportHookMaker = archive => {
   return makeImportHook;
 };
 
-export const parseArchive = async archiveBytes => {
-  const archive = await readZip(archiveBytes);
+export const parseArchive = async (archiveBytes, archiveLocation) => {
+  const archive = await readZip(archiveBytes, archiveLocation);
 
   const compartmentMapBytes = await archive.read("compartmap.json");
   const compartmentMapText = decoder.decode(compartmentMapBytes);
@@ -47,12 +47,17 @@ export const parseArchive = async archiveBytes => {
   return { execute };
 };
 
-export const loadArchive = async (read, archivePath) => {
-  const archiveBytes = await read(archivePath);
-  return parseArchive(archiveBytes);
+export const loadArchive = async (read, archiveLocation) => {
+  const archiveBytes = await read(archiveLocation);
+  return parseArchive(archiveBytes, archiveLocation);
 };
 
-export const importArchive = async (read, archivePath, endowments, modules) => {
-  const archive = await loadArchive(read, archivePath);
+export const importArchive = async (
+  read,
+  archiveLocation,
+  endowments,
+  modules
+) => {
+  const archive = await loadArchive(read, archiveLocation);
   return archive.execute(endowments, modules);
 };
diff --git a/packages/endo/src/zip.js b/packages/endo/src/zip.js
index fddc3cc275..022e8d23f5 100644
--- a/packages/endo/src/zip.js
+++ b/packages/endo/src/zip.js
@@ -2,10 +2,18 @@
 
 import JSZip from "jszip";
 
-export const readZip = async data => {
+export const readZip = async (data, location) => {
   const zip = new JSZip();
   await zip.loadAsync(data);
-  const read = async path => zip.file(path).async("uint8array");
+  const read = async path => {
+    const file = zip.file(path);
+    if (file === undefined) {
+      throw new Error(
+        `Cannot find file to read ${path} in archive ${location}`
+      );
+    }
+    return file.async("uint8array");
+  };
   return { read };
 };
 

From 208afd5012bf54352299a4a9d591c8ebb5cc30c1 Mon Sep 17 00:00:00 2001
From: Kris Kowal <kris@agoric.com>
Date: Fri, 31 Jul 2020 20:09:56 -0700
Subject: [PATCH 02/13] Add endo bin

---
 packages/endo/package.json | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/packages/endo/package.json b/packages/endo/package.json
index 04ca0a83b3..f40889199d 100644
--- a/packages/endo/package.json
+++ b/packages/endo/package.json
@@ -14,6 +14,9 @@
     "require": "./dist/endo.cjs",
     "browser": "./dist/endo.umd.js"
   },
+  "bin": {
+    "endo": "./bin/endo"
+  },
   "scripts": {
     "build": "rollup --config rollup.config.js",
     "clean": "rm -rf dist",

From 22eb6c4968e813d59ea6ad9367c90325cf912f65 Mon Sep 17 00:00:00 2001
From: Kris Kowal <kris@agoric.com>
Date: Fri, 31 Jul 2020 20:12:41 -0700
Subject: [PATCH 03/13] Use stdout for compartmap

---
 packages/endo/src/cli.js | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/packages/endo/src/cli.js b/packages/endo/src/cli.js
index 327bbc47e8..9e7a2b0954 100644
--- a/packages/endo/src/cli.js
+++ b/packages/endo/src/cli.js
@@ -47,7 +47,7 @@ async function executeArchive(archivePath, { read, cwd }) {
   return execute(application);
 }
 
-async function compartmap(applicationPath, { read, cwd }) {
+async function compartmap(applicationPath, { read, cwd, stdout }) {
   const currentLocation = new URL(`${cwd()}/`, "file:///");
   const applicationLocation = new URL(applicationPath, currentLocation);
   const { packageLocation } = await search(read, applicationLocation);
@@ -55,7 +55,7 @@ async function compartmap(applicationPath, { read, cwd }) {
     read,
     packageLocation
   );
-  console.log(JSON.stringify(compartmentMap, null, 2));
+  stdout.write(`${JSON.stringify(compartmentMap, null, 2)}\n`);
   return 0;
 }
 
@@ -122,7 +122,8 @@ export async function main(process, modules) {
     process.exitCode = await run(args, {
       read,
       write,
-      cwd: process.cwd
+      cwd: process.cwd,
+      stdout: process.stdout
     });
   } catch (error) {
     process.exitCode = usage(error.stack || error.message);

From c24bc9927f83efb28f247c64d44869e4fe5df4d6 Mon Sep 17 00:00:00 2001
From: Kris Kowal <kris@agoric.com>
Date: Tue, 4 Aug 2020 18:11:23 -0700
Subject: [PATCH 04/13] Revise argument forms for endo bin

---
 packages/endo/src/cli.js | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/packages/endo/src/cli.js b/packages/endo/src/cli.js
index 9e7a2b0954..e06f13f49d 100644
--- a/packages/endo/src/cli.js
+++ b/packages/endo/src/cli.js
@@ -66,7 +66,7 @@ async function run(args, powers) {
       for (const applicationLocation of args) {
         return executeLocation(applicationLocation, powers);
       }
-    } else if (arg === "--compartmap") {
+    } else if (arg === "-m" || arg === "--compartmap") {
       for (const applicationPath of args) {
         const rem = Array.from(args);
         if (rem.length > 0) {
@@ -76,7 +76,7 @@ async function run(args, powers) {
         }
         return compartmap(applicationPath, powers);
       }
-    } else if (arg === "-w") {
+    } else if (arg === "-c" || arg === "--create") {
       for (const archivePath of args) {
         for (const applicationPath of args) {
           const rem = Array.from(args);
@@ -90,11 +90,11 @@ async function run(args, powers) {
         return usage(`Expected application path`);
       }
       return usage(`Expected archive path`);
-    } else if (arg === "-x") {
+    } else if (arg === "-e" || arg === "--execute") {
       for (const archivePath of args) {
         const rem = Array.from(args);
         if (rem.length > 0) {
-          return usage(`Unexpected arguments after -x ${archivePath}: ${rem}`);
+          return usage(`Unexpected arguments after -e ${archivePath}: ${rem}`);
         }
         return executeArchive(archivePath, powers);
       }

From 54bfebf4e41a450eddbc75f5acbc848f37a3ba18 Mon Sep 17 00:00:00 2001
From: Kris Kowal <kris@agoric.com>
Date: Mon, 10 Aug 2020 17:03:06 -0700
Subject: [PATCH 05/13] endo/main Use subcommands, remove dubious ones

---
 packages/endo/src/cli.js | 166 +++++++++++++++++----------------------
 1 file changed, 72 insertions(+), 94 deletions(-)

diff --git a/packages/endo/src/cli.js b/packages/endo/src/cli.js
index e06f13f49d..8ae19aa7cf 100644
--- a/packages/endo/src/cli.js
+++ b/packages/endo/src/cli.js
@@ -5,121 +5,99 @@ import { loadLocation, writeArchive, loadArchive } from "./main.js";
 import { search } from "./search.js";
 import { compartmentMapForNodeModules } from "./compartmap.js";
 
-const iterate = sequence => sequence[Symbol.iterator]();
-
-function usage(message) {
-  console.error(message);
-  return 1;
+async function subcommand([arg, ...rest], handlers) {
+  const keys = Object.keys(handlers);
+  if (arg === undefined || !keys.includes(arg))  {
+    return usage(`expected one of ${keys}`);
+  }
+  return handlers[arg](rest);
 }
 
-async function execute(application) {
-  const endowments = harden({
-    console: {
-      log(...args) {
-        console.log(...args);
-      }
-    }
-  });
-
-  await application.execute(endowments);
-  return 0;
+async function parameter(args, handle, usage) {
+  const [ arg, ...rest ] = args;
+  if (arg === undefined) {
+    return usage(`expected an argument`);
+  }
+  if (arg.startsWith('-')) {
+    return usage(`unexpected flag: ${arg}`);
+  }
+  return handle(arg, rest);
 }
 
-async function executeLocation(applicationPath, { cwd, read }) {
-  const currentLocation = new URL(`${cwd()}/`, "file:///");
-  const applicationLocation = new URL(applicationPath, currentLocation);
-  const application = await loadLocation(read, applicationLocation);
-  return execute(application);
+function usage(message) {
+  console.error(message);
+  return 1;
 }
 
-async function archive(archivePath, applicationPath, { cwd, read, write }) {
-  const currentLocation = new URL(`${cwd()}/`, "file:///");
-  const archiveLocation = new URL(archivePath, currentLocation);
-  const applicationLocation = new URL(applicationPath, currentLocation);
-  await writeArchive(write, read, archiveLocation, applicationLocation);
-  return 0;
+async function noEntryUsage() {
+  return usage(`expected path to program`);
 }
 
-async function executeArchive(archivePath, { read, cwd }) {
-  const currentLocation = new URL(`${cwd()}/`, "file:///");
-  const archiveLocation = new URL(archivePath, currentLocation);
-  const application = await loadArchive(read, archiveLocation);
-  return execute(application);
+async function noArchiveUsage() {
+  return usage(`expected path for archive`);
 }
 
-async function compartmap(applicationPath, { read, cwd, stdout }) {
-  const currentLocation = new URL(`${cwd()}/`, "file:///");
-  const applicationLocation = new URL(applicationPath, currentLocation);
-  const { packageLocation } = await search(read, applicationLocation);
-  const compartmentMap = await compartmentMapForNodeModules(
-    read,
-    packageLocation
-  );
-  stdout.write(`${JSON.stringify(compartmentMap, null, 2)}\n`);
-  return 0;
-}
+async function run(args, { cwd, read, write, stdout }) {
 
-// run parses command line arguments and dispatches to a subcommand.
-async function run(args, powers) {
-  for (const arg of args) {
-    if (arg === "--") {
-      for (const applicationLocation of args) {
-        return executeLocation(applicationLocation, powers);
-      }
-    } else if (arg === "-m" || arg === "--compartmap") {
-      for (const applicationPath of args) {
-        const rem = Array.from(args);
-        if (rem.length > 0) {
-          return usage(
-            `Unexpected arguments after --compartmap ${applicationPath}: ${rem}`
-          );
-        }
-        return compartmap(applicationPath, powers);
-      }
-    } else if (arg === "-c" || arg === "--create") {
-      for (const archivePath of args) {
-        for (const applicationPath of args) {
-          const rem = Array.from(args);
-          if (rem.length > 0) {
-            return usage(
-              `Unexpected arguments after --archive ${archivePath} ${applicationPath}: ${rem}`
-            );
-          }
-          return archive(archivePath, applicationPath, powers);
-        }
-        return usage(`Expected application path`);
+  async function compartmap(args) {
+    async function handleEntry(applicationPath, args) {
+      if (args.length) {
+        return usage(`unexpected arguments: ${JSON.stringify(args)}`);
       }
-      return usage(`Expected archive path`);
-    } else if (arg === "-e" || arg === "--execute") {
-      for (const archivePath of args) {
-        const rem = Array.from(args);
-        if (rem.length > 0) {
-          return usage(`Unexpected arguments after -e ${archivePath}: ${rem}`);
+      const currentLocation = new URL(`${cwd()}/`, "file:///");
+      const applicationLocation = new URL(applicationPath, currentLocation);
+      const { packageLocation } = await search(read, applicationLocation);
+      const compartmentMap = await compartmentMapForNodeModules(
+        read,
+        packageLocation
+      );
+      stdout.write(`${JSON.stringify(compartmentMap, null, 2)}\n`);
+      return 0;
+    }
+    return parameter(args, handleEntry, noEntryUsage);
+  }
+
+  async function archive(args) {
+    async function handleArchive(archivePath, args) {
+      async function handleEntry(applicationPath, args) {
+        if (args.length) {
+          return usage(`unexpected arguments: ${JSON.stringify(args)}`);
         }
-        return executeArchive(archivePath, powers);
+        const currentLocation = new URL(`${cwd()}/`, "file:///");
+        const archiveLocation = new URL(archivePath, currentLocation);
+        const applicationLocation = new URL(applicationPath, currentLocation);
+        await writeArchive(write, read, archiveLocation, applicationLocation);
+        return 0;
       }
-      return usage(`Expected archive path`);
-    } else if (arg.startsWith("-")) {
-      return usage(`Unrecognized flag ${arg}`);
-    } else {
-      const rem = Array.from(args);
-      if (rem.length > 0) {
-        return usage(`Unexpected arguments after ${arg}: ${rem}`);
-      }
-      return executeLocation(arg, powers);
+      return parameter(args, handleEntry, noEntryUsage);
     }
+    return parameter(args, handleArchive, noArchiveUsage);
   }
-  return usage(`Expected script path or flag`);
+
+  return subcommand(args, { compartmap, archive });
 }
 
 export async function main(process, modules) {
   const { fs } = modules;
-  const read = async location => fs.readFile(new URL(location).pathname);
-  const write = async (location, content) =>
-    fs.writeFile(new URL(location).pathname, content);
+
+  async function read(location) {
+    try {
+      return await fs.readFile(new URL(location).pathname);
+    } catch (error) {
+      throw new Error(error.message);
+    }
+  }
+
+  async function write(location, content) {
+    try {
+      return await fs.writeFile(new URL(location).pathname, content);
+    } catch (error) {
+      throw new Error(error.message);
+    }
+  }
+
   try {
-    const args = iterate(process.argv.slice(2));
-    process.exitCode = await run(args, {
+    process.exitCode = await run(process.argv.slice(2), {
       read,
       write,
       cwd: process.cwd,

From 3c8dc532b5f6fbf918b2b1d9272753c686859597 Mon Sep 17 00:00:00 2001
From: Kris Kowal <kris@agoric.com>
Date: Mon, 10 Aug 2020 17:07:12 -0700
Subject: [PATCH 06/13] endo/main Fix lint

---
 packages/endo/src/cli.js | 38 ++++++++++++++++++--------------------
 1 file changed, 18 insertions(+), 20 deletions(-)

diff --git a/packages/endo/src/cli.js b/packages/endo/src/cli.js
index 8ae19aa7cf..3c3057a193 100644
--- a/packages/endo/src/cli.js
+++ b/packages/endo/src/cli.js
@@ -1,44 +1,42 @@
-/* global harden */
-
+/* eslint no-shadow: [0] */
 import "./lockdown.js";
-import { loadLocation, writeArchive, loadArchive } from "./main.js";
+import { writeArchive } from "./main.js";
 import { search } from "./search.js";
 import { compartmentMapForNodeModules } from "./compartmap.js";
 
+function usage(message) {
+  console.error(message);
+  return 1;
+}
+
+async function noEntryUsage() {
+  return usage(`expected path to program`);
+}
+
+async function noArchiveUsage() {
+  return usage(`expected path for archive`);
+}
+
 async function subcommand([arg, ...rest], handlers) {
   const keys = Object.keys(handlers);
-  if (arg === undefined || !keys.includes(arg))  {
+  if (arg === undefined || !keys.includes(arg)) {
     return usage(`expected one of ${keys}`);
   }
   return handlers[arg](rest);
 }
 
 async function parameter(args, handle, usage) {
-  const [ arg, ...rest ] = args;
+  const [arg, ...rest] = args;
   if (arg === undefined) {
     return usage(`expected an argument`);
   }
-  if (arg.startsWith('-')) {
+  if (arg.startsWith("-")) {
     return usage(`unexpected flag: ${arg}`);
   }
   return handle(arg, rest);
 }
 
-function usage(message) {
-  console.error(message);
-  return 1;
-}
-
-async function noEntryUsage() {
-  return usage(`expected path to program`);
-}
-
-async function noArchiveUsage() {
-  return usage(`expected path for archive`);
-}
-
 async function run(args, { cwd, read, write, stdout }) {
-
   async function compartmap(args) {
     async function handleEntry(applicationPath, args) {
       if (args.length) {

From 4cee940e448d8854a0b88b4c493d08b264e3eb0d Mon Sep 17 00:00:00 2001
From: Kris Kowal <kris@agoric.com>
Date: Tue, 11 Aug 2020 13:43:54 -0700
Subject: [PATCH 07/13] endo/cli Nits

---
 packages/endo/src/cli.js | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/packages/endo/src/cli.js b/packages/endo/src/cli.js
index 3c3057a193..868546f3a2 100644
--- a/packages/endo/src/cli.js
+++ b/packages/endo/src/cli.js
@@ -20,7 +20,7 @@ async function noArchiveUsage() {
 async function subcommand([arg, ...rest], handlers) {
   const keys = Object.keys(handlers);
   if (arg === undefined || !keys.includes(arg)) {
-    return usage(`expected one of ${keys}`);
+    return usage(`expected one of ${keys.join(", ")}`);
   }
   return handlers[arg](rest);
 }
@@ -77,6 +77,9 @@ async function run(args, { cwd, read, write, stdout }) {
 
 export async function main(process, modules) {
   const { fs } = modules;
+  const { cwd, stdout } = process;
+
+  // Filesystem errors often don't have stacks:
 
   async function read(location) {
     try {
@@ -98,8 +101,8 @@ export async function main(process, modules) {
     process.exitCode = await run(process.argv.slice(2), {
       read,
       write,
-      cwd: process.cwd,
-      stdout: process.stdout
+      cwd,
+      stdout
     });
   } catch (error) {
     process.exitCode = usage(error.stack || error.message);

From d5bc072223f12eaf4e4e8f6d0f1f1f274fb1e7fb Mon Sep 17 00:00:00 2001
From: Kris Kowal <kris@agoric.com>
Date: Tue, 4 Aug 2020 17:25:42 -0700
Subject: [PATCH 08/13] feat(endo): Support explicit parsers in package.json

---
 packages/endo/src/compartmap.js               | 34 +++++++++++++++----
 packages/endo/test/main.test.js               | 10 ++++--
 packages/endo/test/node_modules/danny/main.js |  3 +-
 .../endo/test/node_modules/danny/package.json |  3 +-
 .../test/node_modules/typeparsers/cjs.cjs     |  1 +
 .../endo/test/node_modules/typeparsers/js.js  |  1 +
 .../test/node_modules/typeparsers/json.json   |  1 +
 .../test/node_modules/typeparsers/main.js     |  6 ++++
 .../test/node_modules/typeparsers/mjs.mjs     |  1 +
 .../node_modules/typeparsers/package.json     |  5 +++
 10 files changed, 54 insertions(+), 11 deletions(-)
 create mode 100644 packages/endo/test/node_modules/typeparsers/cjs.cjs
 create mode 100644 packages/endo/test/node_modules/typeparsers/js.js
 create mode 100644 packages/endo/test/node_modules/typeparsers/json.json
 create mode 100644 packages/endo/test/node_modules/typeparsers/main.js
 create mode 100644 packages/endo/test/node_modules/typeparsers/mjs.mjs
 create mode 100644 packages/endo/test/node_modules/typeparsers/package.json

diff --git a/packages/endo/src/compartmap.js b/packages/endo/src/compartmap.js
index 6b5f9afc4f..0348f95c3b 100644
--- a/packages/endo/src/compartmap.js
+++ b/packages/endo/src/compartmap.js
@@ -2,7 +2,7 @@
 
 import { inferExports } from "./infer-exports.js";
 
-const { create, keys, entries } = Object;
+const { create, entries, keys, values } = Object;
 
 const decoder = new TextDecoder();
 
@@ -73,12 +73,32 @@ const findPackage = async (readDescriptor, directory, name) => {
   }
 };
 
-const commonParsers = { js: "cjs", cjs: "cjs", mjs: "mjs", json: "json" };
-const moduleParsers = { js: "mjs", mjs: "mjs", cjs: "cjs", json: "json" };
+const languages = ["cjs", "mjs", "json"];
+const incontroversialParsers = { cjs: "cjs", mjs: "mjs", json: "json" };
+const commonParsers = { js: "cjs", ...incontroversialParsers };
+const moduleParsers = { js: "mjs", ...incontroversialParsers };
 
-const inferParsers = (type, location) => {
-  if (type === undefined) {
-    return commonParsers;
+const inferParsers = (descriptor, location) => {
+  const { type, parsers } = descriptor;
+  if (parsers !== undefined) {
+    if (typeof parsers !== "object") {
+      throw new Error(
+        `Cannot interpret parser map ${JSON.stringify(
+          parsers
+        )} of package at ${location}, must be an object mapping file extensions to corresponding languages (mjs for ECMAScript modules, cjs for CommonJS modules, or json for JSON modules`
+      );
+    }
+    const invalidLanguages = values(parsers).filter(
+      language => !languages.includes(language)
+    );
+    if (invalidLanguages.length > 0) {
+      throw new Error(
+        `Cannot interpret parser map language values ${JSON.stringify(
+          invalidLanguages
+        )} of package at ${location}, must be an object mapping file extensions to corresponding languages (mjs for ECMAScript modules, cjs for CommonJS modules, or json for JSON modules`
+      );
+    }
+    return { ...incontroversialParsers, ...parsers };
   }
   if (type === "module") {
     return moduleParsers;
@@ -144,7 +164,7 @@ const graphPackage = async (
   result.dependencies = dependencies;
   result.types = {};
   result.exports = inferExports(packageDescriptor, tags, result.types);
-  result.parsers = inferParsers(packageDescriptor.type, packageLocation);
+  result.parsers = inferParsers(packageDescriptor, packageLocation);
 
   return Promise.all(children);
 };
diff --git a/packages/endo/test/main.test.js b/packages/endo/test/main.test.js
index d008d22b67..b7bd5cd61b 100644
--- a/packages/endo/test/main.test.js
+++ b/packages/endo/test/main.test.js
@@ -33,7 +33,8 @@ const assertFixture = (t, namespace) => {
     endowed,
     typecommon,
     typemodule,
-    typehybrid
+    typehybrid,
+    typeparsers
   } = namespace;
   t.equal(avery, "Avery", "exports avery");
   t.equal(brooke, "Brooke", "exports brooke");
@@ -49,10 +50,15 @@ const assertFixture = (t, namespace) => {
     [42, 42, 42, 42],
     "type=module package carries exports"
   );
+  t.deepEqual(
+    typeparsers,
+    [42, 42, 42, 42],
+    "parsers-specifying package carries exports"
+  );
   t.equal(typehybrid, 42, "type=module and module= package carries exports");
 };
 
-const fixtureAssertionCount = 7;
+const fixtureAssertionCount = 8;
 
 // The "create builtin" test prepares a builtin module namespace object that
 // gets threaded into all subsequent tests to satisfy the "builtin" module
diff --git a/packages/endo/test/node_modules/danny/main.js b/packages/endo/test/node_modules/danny/main.js
index cd5f6af4ef..ee9cf84f1b 100644
--- a/packages/endo/test/node_modules/danny/main.js
+++ b/packages/endo/test/node_modules/danny/main.js
@@ -3,8 +3,9 @@
 import typemodule from 'typemodule';
 import typecommon from 'typecommon';
 import typehybrid from 'typehybrid';
+import typeparsers from 'typeparsers';
 
-export { typemodule, typecommon, typehybrid };
+export { typemodule, typecommon, typehybrid, typeparsers };
 
 export { avery } from 'avery';
 export { brooke } from 'brooke';
diff --git a/packages/endo/test/node_modules/danny/package.json b/packages/endo/test/node_modules/danny/package.json
index 41b32ca160..d630fbda75 100644
--- a/packages/endo/test/node_modules/danny/package.json
+++ b/packages/endo/test/node_modules/danny/package.json
@@ -8,6 +8,7 @@
     "brooke": "^1.0.0",
     "typecommon": "^1.0.0",
     "typemodule": "^1.0.0",
-    "typehybrid": "^1.0.0"
+    "typehybrid": "^1.0.0",
+    "typeparsers": "^1.0.0"
   }
 }
diff --git a/packages/endo/test/node_modules/typeparsers/cjs.cjs b/packages/endo/test/node_modules/typeparsers/cjs.cjs
new file mode 100644
index 0000000000..1ecd817300
--- /dev/null
+++ b/packages/endo/test/node_modules/typeparsers/cjs.cjs
@@ -0,0 +1 @@
+exports.default = 42;
diff --git a/packages/endo/test/node_modules/typeparsers/js.js b/packages/endo/test/node_modules/typeparsers/js.js
new file mode 100644
index 0000000000..7a4e8a723a
--- /dev/null
+++ b/packages/endo/test/node_modules/typeparsers/js.js
@@ -0,0 +1 @@
+export default 42;
diff --git a/packages/endo/test/node_modules/typeparsers/json.json b/packages/endo/test/node_modules/typeparsers/json.json
new file mode 100644
index 0000000000..d81cc0710e
--- /dev/null
+++ b/packages/endo/test/node_modules/typeparsers/json.json
@@ -0,0 +1 @@
+42
diff --git a/packages/endo/test/node_modules/typeparsers/main.js b/packages/endo/test/node_modules/typeparsers/main.js
new file mode 100644
index 0000000000..9228517e2b
--- /dev/null
+++ b/packages/endo/test/node_modules/typeparsers/main.js
@@ -0,0 +1,6 @@
+import js from './js.js';
+import mjs from './mjs.mjs';
+import cjs from './cjs.cjs';
+import json from './json.json';
+
+export default [js, mjs, cjs, json];
diff --git a/packages/endo/test/node_modules/typeparsers/mjs.mjs b/packages/endo/test/node_modules/typeparsers/mjs.mjs
new file mode 100644
index 0000000000..7a4e8a723a
--- /dev/null
+++ b/packages/endo/test/node_modules/typeparsers/mjs.mjs
@@ -0,0 +1 @@
+export default 42;
diff --git a/packages/endo/test/node_modules/typeparsers/package.json b/packages/endo/test/node_modules/typeparsers/package.json
new file mode 100644
index 0000000000..400e4491b3
--- /dev/null
+++ b/packages/endo/test/node_modules/typeparsers/package.json
@@ -0,0 +1,5 @@
+{
+  "name": "typeparsers",
+  "parsers": {"js": "mjs"},
+  "exports": "./main.js"
+}

From aff79160b45b17557986f1301600ae272f2d027d Mon Sep 17 00:00:00 2001
From: Kris Kowal <kris@agoric.com>
Date: Tue, 4 Aug 2020 18:09:26 -0700
Subject: [PATCH 09/13] Document parsers

---
 packages/endo/README.md | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/packages/endo/README.md b/packages/endo/README.md
index 00956dfac9..2790187605 100644
--- a/packages/endo/README.md
+++ b/packages/endo/README.md
@@ -246,6 +246,25 @@ backward compatibility.
 However, packages that have a `type` property that explicitly says `module`
 will treat a `.js` file as an ECMAScript module.
 
+This unforunately conflicts with packages written to work with the ECMAScript
+module system emulator in the `esm` package on npm, which allows every file
+with the `js` extension to be an ECMAScript module that presents itself to
+Node.js as a CommonJS module.
+To overcome such obstacles, Endo will accept a non-standard `parsers` property
+in `package.json` that maps file extensions, specifically `js` to the
+corresponding language name, one of `mjs` for ECMAScript modules, `cjs` for
+CommonJS modules, and `json` for JSON modules.
+All other language names are reserved and the defaults for files with the
+extensions `cjs`, `mjs`, and `json` default to the language of the same name
+unless overridden.
+If Endo sees `parsers`, it ignores `type`, so these can contradict where using
+the `esm` emulator requires.
+
+```json
+{
+  "parsers": {"js": "mjs"}
+}
+```
 
 Many Node.js applications using CommonJS modules expect to be able to `require`
 a JSON file like `package.json`.

From c7ad8db3effd7225c45b9a663dcdee0712cbc808 Mon Sep 17 00:00:00 2001
From: Kris Kowal <kris@agoric.com>
Date: Mon, 24 Aug 2020 13:36:51 -0700
Subject: [PATCH 10/13] Nit: uncontroversial parsers

---
 packages/endo/src/compartmap.js | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/packages/endo/src/compartmap.js b/packages/endo/src/compartmap.js
index 0348f95c3b..8d12765b17 100644
--- a/packages/endo/src/compartmap.js
+++ b/packages/endo/src/compartmap.js
@@ -74,9 +74,9 @@ const findPackage = async (readDescriptor, directory, name) => {
 };
 
 const languages = ["cjs", "mjs", "json"];
-const incontroversialParsers = { cjs: "cjs", mjs: "mjs", json: "json" };
-const commonParsers = { js: "cjs", ...incontroversialParsers };
-const moduleParsers = { js: "mjs", ...incontroversialParsers };
+const uncontroversialParsers = { cjs: "cjs", mjs: "mjs", json: "json" };
+const commonParsers = { js: "cjs", ...uncontroversialParsers };
+const moduleParsers = { js: "mjs", ...uncontroversialParsers };
 
 const inferParsers = (descriptor, location) => {
   const { type, parsers } = descriptor;
@@ -98,7 +98,7 @@ const inferParsers = (descriptor, location) => {
         )} of package at ${location}, must be an object mapping file extensions to corresponding languages (mjs for ECMAScript modules, cjs for CommonJS modules, or json for JSON modules`
       );
     }
-    return { ...incontroversialParsers, ...parsers };
+    return { ...uncontroversialParsers, ...parsers };
   }
   if (type === "module") {
     return moduleParsers;

From 9153014adafb73c37288b8c3502e2d5d2ce876b4 Mon Sep 17 00:00:00 2001
From: Kris Kowal <kris@agoric.com>
Date: Tue, 11 Aug 2020 17:26:16 -0700
Subject: [PATCH 11/13] feat(endo): Support exec shell environment

---
 packages/endo/bin/endo.js        |  9 +++++----
 packages/endo/mitm/.gitignore    |  1 +
 packages/endo/package.json       |  3 ++-
 packages/endo/src/cli.js         | 20 ++++++++++++++++----
 packages/endo/src/lockdown.cjs   | 17 +++++++++++++++++
 packages/endo/src/lockdown.js    |  2 ++
 packages/endo/src/postinstall.js | 17 +++++++++++++++++
 7 files changed, 60 insertions(+), 9 deletions(-)
 create mode 100644 packages/endo/mitm/.gitignore
 create mode 100644 packages/endo/src/lockdown.cjs
 create mode 100644 packages/endo/src/postinstall.js

diff --git a/packages/endo/bin/endo.js b/packages/endo/bin/endo.js
index da980e130d..48af66f5a7 100755
--- a/packages/endo/bin/endo.js
+++ b/packages/endo/bin/endo.js
@@ -1,5 +1,6 @@
 #!/usr/bin/env node
-import fs from "fs";
-import { main } from "../src/cli.js";
-
-main(process, { fs: fs.promises });
+(async () => {
+  const fs = await import("fs");
+  const { main } = await import("../src/cli.js");
+  main(process, { fs: fs.promises });
+})();
diff --git a/packages/endo/mitm/.gitignore b/packages/endo/mitm/.gitignore
new file mode 100644
index 0000000000..64f5a0a681
--- /dev/null
+++ b/packages/endo/mitm/.gitignore
@@ -0,0 +1 @@
+node
diff --git a/packages/endo/package.json b/packages/endo/package.json
index f40889199d..06aab6ea2f 100644
--- a/packages/endo/package.json
+++ b/packages/endo/package.json
@@ -15,7 +15,7 @@
     "browser": "./dist/endo.umd.js"
   },
   "bin": {
-    "endo": "./bin/endo"
+    "endo": "./bin/endo.js"
   },
   "scripts": {
     "build": "rollup --config rollup.config.js",
@@ -23,6 +23,7 @@
     "depcheck": "depcheck",
     "lint": "eslint '**/*.js'",
     "lint-fix": "eslint --fix '**/*.js'",
+    "postinstall": "node src/postinstall.js",
     "prepublish": "yarn clean && yarn build",
     "test": "yarn build && tap --no-esm --no-coverage --reporter spec 'test/**/*.test.js'"
   },
diff --git a/packages/endo/src/cli.js b/packages/endo/src/cli.js
index 868546f3a2..014f5bed57 100644
--- a/packages/endo/src/cli.js
+++ b/packages/endo/src/cli.js
@@ -1,9 +1,12 @@
 /* eslint no-shadow: [0] */
 import "./lockdown.js";
+import subprocess from "child_process";
 import { writeArchive } from "./main.js";
 import { search } from "./search.js";
 import { compartmentMapForNodeModules } from "./compartmap.js";
 
+const mitmPath = new URL("../mitm", import.meta.url).pathname;
+
 function usage(message) {
   console.error(message);
   return 1;
@@ -36,7 +39,7 @@ async function parameter(args, handle, usage) {
   return handle(arg, rest);
 }
 
-async function run(args, { cwd, read, write, stdout }) {
+async function run(args, { cwd, read, write, stdout, env }) {
   async function compartmap(args) {
     async function handleEntry(applicationPath, args) {
       if (args.length) {
@@ -72,12 +75,20 @@ async function run(args, { cwd, read, write, stdout }) {
     return parameter(args, handleArchive, noArchiveUsage);
   }
 
-  return subcommand(args, { compartmap, archive });
+  async function exec([arg, ...args]) {
+    const child = subprocess.spawn(arg, args, {
+      env: { ...env, PATH: `${mitmPath}:${env.PATH}` },
+      stdio: "inherit"
+    });
+    return new Promise(resolve => child.on("exit", resolve));
+  }
+
+  return subcommand(args, { compartmap, archive, exec });
 }
 
 export async function main(process, modules) {
   const { fs } = modules;
-  const { cwd, stdout } = process;
+  const { cwd, stdout, env } = process;
 
   // Filesystem errors often don't have stacks:
 
@@ -102,7 +113,8 @@ export async function main(process, modules) {
       read,
       write,
       cwd,
-      stdout
+      stdout,
+      env
     });
   } catch (error) {
     process.exitCode = usage(error.stack || error.message);
diff --git a/packages/endo/src/lockdown.cjs b/packages/endo/src/lockdown.cjs
new file mode 100644
index 0000000000..e63084603c
--- /dev/null
+++ b/packages/endo/src/lockdown.cjs
@@ -0,0 +1,17 @@
+/* global lockdown */
+
+// This is a CommonJS module, to be used like `node -r endo/src/lockdown.cjs`.
+// The `endo exec` command stages a man-in-the-middle `node` shell script that
+// in turn injects this SES lockdown parameter in all Node.js commands in the
+// resulting shell environment.
+// The taming behavior may be overridden with environment variables
+// like `ERROR_TAMING=unsafe endo exec node ...`
+
+require("ses");
+lockdown({
+  dateTaming: process.env.ENDO_DATE_TAMING || 'safe',
+  errorTaming: process.env.ENDO_ERROR_TAMING || 'safe',
+  mathTaming: process.env.ENDO_MATH_TAMING || 'safe',
+  regExpTaming: process.env.ENDO_REGEXP_TAMING || 'safe',
+  localeTaming: process.env.ENDO_LOCAlE_TAMING || 'safe',
+});
diff --git a/packages/endo/src/lockdown.js b/packages/endo/src/lockdown.js
index 7095b6e19a..cb1acb85b9 100644
--- a/packages/endo/src/lockdown.js
+++ b/packages/endo/src/lockdown.js
@@ -1,6 +1,8 @@
 /* global lockdown */
 import "ses";
 
+// This is used by Endo to lockdown its own start compartment.
+
 lockdown({
   errorTaming: "unsafe"
 });
diff --git a/packages/endo/src/postinstall.js b/packages/endo/src/postinstall.js
new file mode 100644
index 0000000000..f1c6c38722
--- /dev/null
+++ b/packages/endo/src/postinstall.js
@@ -0,0 +1,17 @@
+// This postinstall hook creates mitm/node.
+// This in turn interposes SES lockdown for all descendent processes, provided
+// that the mitm directory is appears before Node.js's bin directory on the
+// environment PATH.
+
+import fs from "fs";
+
+const node = process.argv[0];
+const lockdown = new URL("../src/lockdown.cjs", import.meta.url).pathname;
+const mitm = new URL("../mitm/node", import.meta.url).pathname;
+
+const script = `#!/bin/bash
+set -ueo pipefail
+${node} -r ${lockdown} "$@"`;
+
+fs.writeFileSync(mitm, script, "utf-8");
+fs.chmodSync(mitm, 0o755);

From 115e791039c9f6fc39e8b0b6dbb2b48497713500 Mon Sep 17 00:00:00 2001
From: Kris Kowal <kris@agoric.com>
Date: Fri, 21 Aug 2020 18:24:16 -0700
Subject: [PATCH 12/13] feat(endo): Search for index.js and implied package
 exports

When asked for a module named "x", Node.js will search for "x.js" then "x/index.js".

For packages that do not supply an "exports" property in their "package.json", any module contained by that package is a valid exported module.

To achieve parity with these two features, Endo uses different techniques when loading from the file system and loading from an archive.

When reading from the file system, Endo will search for a satsifactory candidate in each compartment's asynchronous importHook.  Endo also uses the compartment's new moduleMapHook to search for dependency compartments in the "scope" of a module identifier prefix.  These allow Endo to operate from an incomplete compartment map for the initial load.

The Endo archiver instead creates a more complete compartment map, with every discovered module.  This introduces a new kind of module to the compartment map module descriptor type union: modules with known locations and corresponding parsers.  The archiver erases the "scopes", "types", and "parsers" on each compartment description since they are no longer necessary.

When reading from an archive, Endo uses an importHook that consults the compartment map directly for the locations of all contained modules, and creates a complete moduleMap up front.
---
 packages/endo/DESIGN.md                       |  66 +++++++++-
 packages/endo/src/archive.js                  | 120 ++++++++++++------
 packages/endo/src/assemble.js                 |  46 ++++++-
 packages/endo/src/compartmap.js               |  34 +++--
 packages/endo/src/import-archive.js           |  20 ++-
 packages/endo/src/import.js                   |  53 ++++++--
 packages/endo/src/module-map-hook.js          |  58 +++++++++
 packages/endo/src/parse.js                    |  17 ++-
 packages/endo/src/zip.js                      |   4 +-
 packages/endo/test/main.test.js               |   4 +-
 packages/endo/test/node_modules/clarke/TODO   |   2 -
 packages/endo/test/node_modules/danny/main.js |   1 +
 .../endo/test/node_modules/danny/package.json |   1 +
 13 files changed, 345 insertions(+), 81 deletions(-)
 create mode 100644 packages/endo/src/module-map-hook.js
 delete mode 100644 packages/endo/test/node_modules/clarke/TODO

diff --git a/packages/endo/DESIGN.md b/packages/endo/DESIGN.md
index 30f04c9652..b67462d57e 100644
--- a/packages/endo/DESIGN.md
+++ b/packages/endo/DESIGN.md
@@ -200,6 +200,8 @@ type Compartment = {
   location: Location,
   modules: ModuleMap,
   parsers: ParserMap,
+  types: ModuleParserMap,
+  scopes: ScopeMap,
   // The name of the realm to run the compartment within.
   // The default is a single frozen realm that has no name.
   realm: RealmName? // TODO
@@ -213,10 +215,13 @@ type Location string;
 // that do not correspond to source files in the same compartment.
 type ModuleMap = Object<InternalModuleSpecifier, Module>;
 
-// Module describes a module that isn't in the same
+// Module describes a module in a compartment.
+type Module = CompartmentModule | FileModule | ExitModule;
+
+// CompartmentModule describes a module that isn't in the same
 // compartment and how to introduce it to the compartment's
 // module namespace.
-type Module = {
+type CompartmentModule = {
   // The name of the foreign compartment:
   // TODO an absent compartment name may imply either
   // that the module is an internal alias of the
@@ -225,10 +230,32 @@ type Module = {
   // The name of the module in the foreign compartment's
   // module namespace:
   module: ExternalModuleSpecifier?,
-  // Alternately, that this module is not from
-  // any compartment and must be expressly passed
-  // into the compartment graph from the user.
-  parameter: ModuleParameter?, // TODO
+};
+
+// FileLocation is a URL for a module's file relative to the location of the
+// containing compartment.
+type FileLocation = string
+
+// FileModule is a module from a file.
+// When loading modules off a file system (src/import.js), the assembler
+// does not need any explicit FileModules, and instead relies on the
+// compartment to declare a ParserMap and optionally ModuleParserMap and
+// ScopeMap.
+// Endo provides a Compartment importHook and moduleMapHook that will
+// search the filesystem for candidate module files and infer the type from the
+// extension when necessary.
+type FileModule = {
+   location: FileLocation
+   parser: Parser
+};
+
+// ExitName is the name of a built-in module, to be threaded in from the
+// modules passed to the module executor.
+type ExitName string;
+
+// ExitModule refers to a module that comes from outside the compartment map.
+type ExitModule = {
+  exit: ExitName
 };
 
 // InternalModuleSpecifier is the module specifier
@@ -260,6 +287,33 @@ type Extension = string;
 // "json" corresponds to JSON.
 type Parser = "mjs" | "cjs" | "json";
 
+// ModuleParserMap is a table of internal module specifiers
+// to the parser that should be used, regardless of that module's
+// extension.
+// Node.js allows the "module" property in package.json to denote
+// a file that is an ECMAScript module, regardless of its extension.
+// This is the mechanism that allows Endo to respect that behavior.
+type ModuleParserMap = Object<InternalModuleSpecifier, Parser>
+
+// ScopeMap is a map from internal module specifier prefixes
+// like "dependency" or "@organization/dependency" to another
+// compartment.
+// Endo uses this to build a moduleMapHook that can dynamically
+// generate entries for a compartment's moduleMap into
+// Node.js packages that do not explicitly state their "exports".
+// For these modules, any specifier under that prefix corresponds
+// to a link into some internal module of the foreign compartment.
+// When Endo creates an archive, it captures all of the Modules
+// explicitly and erases the scopes entry.
+type ScopeMap = Object<InternalModuleSpecifier, Scope>
+
+// Scope describes the compartment to use for all ad-hoc
+// entries in the compartment's module map.
+type Scope = {
+  compartment: CompartmentName
+}
+
+
 // TODO everything hereafter...
 
 // Realm describes another realm to contain one or more
diff --git a/packages/endo/src/archive.js b/packages/endo/src/archive.js
index b84018df87..cb1058a32a 100644
--- a/packages/endo/src/archive.js
+++ b/packages/endo/src/archive.js
@@ -1,15 +1,15 @@
-/* global StaticModuleRecord */
 /* eslint no-shadow: 0 */
 
 import { writeZip } from "./zip.js";
-import { resolve, join } from "./node-module-specifier.js";
+import { resolve } from "./node-module-specifier.js";
 import { parseExtension } from "./extension.js";
 import { compartmentMapForNodeModules } from "./compartmap.js";
 import { search } from "./search.js";
 import { assemble } from "./assemble.js";
 
-const { entries, fromEntries } = Object;
+const { entries, freeze, fromEntries, values } = Object;
 
+// q, as in quote, for quoted strings in error messages.
 const q = JSON.stringify;
 
 const encoder = new TextEncoder();
@@ -17,38 +17,69 @@ const decoder = new TextDecoder();
 
 const resolveLocation = (rel, abs) => new URL(rel, abs).toString();
 
-const makeRecordingImportHookMaker = (read, baseLocation, manifest, errors) => {
+const makeRecordingImportHookMaker = (read, baseLocation, sources) => {
   // per-assembly:
   const makeImportHook = (packageLocation, parse) => {
     // per-compartment:
     packageLocation = resolveLocation(packageLocation, baseLocation);
+    const packageSources = sources[packageLocation] || {};
+    sources[packageLocation] = packageSources;
+
     const importHook = async moduleSpecifier => {
       // per-module:
+
+      // In Node.js, an absolute specifier always indicates a built-in or
+      // third-party dependency.
+      // The `moduleMapHook` captures all third-party dependencies.
+      if (moduleSpecifier !== "." && !moduleSpecifier.startsWith("./")) {
+        packageSources[moduleSpecifier] = {
+          exit: moduleSpecifier
+        };
+        // Return a place-holder.
+        // Archived compartments are not executed.
+        return freeze({ imports: [], execute() {} });
+      }
+
       const candidates = [moduleSpecifier];
       if (parseExtension(moduleSpecifier) === "") {
         candidates.push(`${moduleSpecifier}.js`, `${moduleSpecifier}/index.js`);
       }
       for (const candidate of candidates) {
-        const moduleLocation = new URL(candidate, packageLocation).toString();
+        const moduleLocation = resolveLocation(candidate, packageLocation);
         // eslint-disable-next-line no-await-in-loop
         const moduleBytes = await read(moduleLocation).catch(
           _error => undefined
         );
-        if (moduleBytes === undefined) {
-          errors.push(
-            `missing ${q(candidate)} needed for package ${q(packageLocation)}`
-          );
-        } else {
+        if (moduleBytes !== undefined) {
           const moduleSource = decoder.decode(moduleBytes);
 
-          const packageManifest = manifest[packageLocation] || {};
-          manifest[packageLocation] = packageManifest;
-          packageManifest[moduleSpecifier] = moduleBytes;
+          const { record, parser } = parse(
+            moduleSource,
+            moduleSpecifier,
+            moduleLocation
+          );
 
-          return parse(moduleSource, moduleSpecifier, moduleLocation);
+          const packageRelativeLocation = moduleLocation.slice(
+            packageLocation.length
+          );
+          packageSources[moduleSpecifier] = {
+            location: packageRelativeLocation,
+            parser,
+            bytes: moduleBytes
+          };
+
+          return record;
         }
       }
-      return new StaticModuleRecord("// Module not found", moduleSpecifier);
+
+      // TODO offer breadcrumbs in the error message, or how to construct breadcrumbs with another tool.
+      throw new Error(
+        `Cannot find file for internal module ${q(
+          moduleSpecifier
+        )} (with candidates ${candidates
+          .map(q)
+          .join(", ")}) in package ${packageLocation}`
+      );
     };
     return importHook;
   };
@@ -60,16 +91,18 @@ const renameCompartments = compartments => {
   let n = 0;
   for (const [name, compartment] of entries(compartments)) {
     const { label } = compartment;
-    renames[name] = `${label}#${n}`;
+    renames[name] = `${label}-n${n}`;
     n += 1;
   }
   return renames;
 };
 
-const renameCompartmentMap = (compartments, renames) => {
+const translateCompartmentMap = (compartments, sources, renames) => {
   const result = {};
   for (const [name, compartment] of entries(compartments)) {
-    const { label, parsers, types } = compartment;
+    const { label } = compartment;
+
+    // rename module compartments
     const modules = {};
     for (const [name, module] of entries(compartment.modules || {})) {
       const compartment = module.compartment
@@ -80,14 +113,27 @@ const renameCompartmentMap = (compartments, renames) => {
         compartment
       };
     }
+
+    // integrate sources into modules
+    const compartmentSources = sources[name];
+    for (const [name, source] of entries(compartmentSources || {})) {
+      const { location, parser, exit } = source;
+      modules[name] = {
+        location,
+        parser,
+        exit
+      };
+    }
+
     result[renames[name]] = {
       label,
       location: renames[name],
-      modules,
-      parsers,
-      types
+      modules
+      // `scopes`, `types`, and `parsers` are not necessary since every
+      // loadable module is captured in `modules`.
     };
   }
+
   return result;
 };
 
@@ -99,10 +145,18 @@ const renameSources = (sources, renames) => {
 
 const addSourcesToArchive = async (archive, sources) => {
   for (const [compartment, modules] of entries(sources)) {
-    for (const [module, content] of entries(modules)) {
-      const path = join(compartment, module);
+    const compartmentLocation = resolveLocation(
+      `${encodeURIComponent(compartment)}/`,
+      "file:///"
+    );
+    for (const { location, bytes } of values(modules)) {
+      const moduleLocation = resolveLocation(
+        encodeURIComponent(location),
+        compartmentLocation
+      );
+      const path = new URL(moduleLocation).pathname.slice(1); // elide initial "/"
       // eslint-disable-next-line no-await-in-loop
-      await archive.write(path, content);
+      await archive.write(path, bytes);
     }
   }
 };
@@ -125,22 +179,12 @@ export const makeArchive = async (read, moduleLocation) => {
   const { compartments, main } = compartmentMap;
 
   const sources = {};
-  const errors = [];
   const makeImportHook = makeRecordingImportHookMaker(
     read,
     packageLocation,
-    sources,
-    errors
+    sources
   );
 
-  if (errors.length > 0) {
-    throw new Error(
-      `Cannot assemble compartment for ${errors.length} reasons: ${errors.join(
-        ", "
-      )}`
-    );
-  }
-
   // Induce importHook to record all the necessary modules to import the given module specifier.
   const compartment = assemble({
     name: main,
@@ -151,7 +195,11 @@ export const makeArchive = async (read, moduleLocation) => {
   await compartment.load(moduleSpecifier);
 
   const renames = renameCompartments(compartments);
-  const renamedCompartments = renameCompartmentMap(compartments, renames);
+  const renamedCompartments = translateCompartmentMap(
+    compartments,
+    sources,
+    renames
+  );
   const renamedSources = renameSources(sources, renames);
 
   const manifest = {
diff --git a/packages/endo/src/assemble.js b/packages/endo/src/assemble.js
index 0908e16f7b..069de75070 100644
--- a/packages/endo/src/assemble.js
+++ b/packages/endo/src/assemble.js
@@ -2,6 +2,7 @@
 
 import { resolve } from "./node-module-specifier.js";
 import { mapParsers } from "./parse.js";
+import { makeModuleMapHook } from "./module-map-hook.js";
 
 const { entries } = Object;
 
@@ -46,8 +47,41 @@ export const assemble = ({
     throw new Error(`Cannot assemble compartment graph that includes a cycle`);
   }
 
-  for (const [inner, outer] of entries(descriptor.modules || {})) {
-    const { compartment: compartmentName, module: moduleSpecifier } = outer;
+  descriptor.modules = descriptor.modules || {};
+  for (const [inner, outer] of entries(descriptor.modules)) {
+    const {
+      compartment: compartmentName,
+      module: moduleSpecifier,
+      exit
+    } = outer;
+    if (exit !== undefined) {
+      // TODO Currenly, only the entry package can connect to built-in modules.
+      // Policies should be able to allow third-party modules to exit to
+      // built-ins, or have built-ins subverted by modules from specific
+      // compartments.
+      const module = modules[exit];
+      if (module === undefined) {
+        throw new Error(
+          `Cannot assemble module graph with missing external module ${q(exit)}`
+        );
+      }
+      modules[inner] = module;
+    } else if (compartmentName !== undefined) {
+      const compartment = assemble({
+        name: compartmentName,
+        compartments,
+        makeImportHook,
+        parents: [...parents, name],
+        loaded,
+        Compartment
+      });
+      modules[inner] = compartment.module(moduleSpecifier);
+    }
+  }
+
+  const scopes = {};
+  for (const [prefix, scope] of entries(descriptor.scopes || {})) {
+    const { compartment: compartmentName } = scope;
     const compartment = assemble({
       name: compartmentName,
       compartments,
@@ -56,14 +90,16 @@ export const assemble = ({
       loaded,
       Compartment
     });
-    modules[inner] = compartment.module(moduleSpecifier);
+    scopes[prefix] = { compartment, compartmentName };
   }
 
-  const parse = mapParsers(descriptor.parsers, descriptor.types);
+  const parse = mapParsers(descriptor.parsers || {}, descriptor.types || {});
+  // TODO makeResolveHook that filters on file patterns
 
   const compartment = new Compartment(endowments, modules, {
     resolveHook: resolve,
-    importHook: makeImportHook(descriptor.location, parse)
+    importHook: makeImportHook(descriptor.location, parse),
+    moduleMapHook: makeModuleMapHook(scopes, descriptor.modules)
   });
 
   loaded[name] = compartment;
diff --git a/packages/endo/src/compartmap.js b/packages/endo/src/compartmap.js
index 8d12765b17..e8b36e18fc 100644
--- a/packages/endo/src/compartmap.js
+++ b/packages/endo/src/compartmap.js
@@ -142,7 +142,7 @@ const graphPackage = async (
   const result = {};
   graph[packageLocation] = result;
 
-  const dependencies = [];
+  const dependencies = {};
   const children = [];
   for (const name of keys(packageDescriptor.dependencies || {})) {
     children.push(
@@ -159,12 +159,17 @@ const graphPackage = async (
     );
   }
 
-  const { version = "" } = packageDescriptor;
-  result.label = `${name}@${version}`;
-  result.dependencies = dependencies;
-  result.types = {};
-  result.exports = inferExports(packageDescriptor, tags, result.types);
-  result.parsers = inferParsers(packageDescriptor, packageLocation);
+  const { version = "", exports } = packageDescriptor;
+  const types = {};
+
+  Object.assign(result, {
+    label: `${name}${version ? `-v${version}` : ""}`,
+    explicit: exports !== undefined,
+    exports: inferExports(packageDescriptor, tags, types),
+    dependencies,
+    types,
+    parsers: inferParsers(packageDescriptor, packageLocation)
+  });
 
   return Promise.all(children);
 };
@@ -181,7 +186,7 @@ const gatherDependency = async (
   if (dependency === undefined) {
     throw new Error(`Cannot find dependency ${name} for ${packageLocation}`);
   }
-  dependencies.push(dependency.packageLocation);
+  dependencies[name] = dependency.packageLocation;
   await graphPackage(name, readDescriptor, graph, dependency, tags);
 };
 
@@ -247,22 +252,29 @@ const translateGraph = (mainPackagePath, graph) => {
   // corresponding compartment can import.
   for (const [
     packageLocation,
-    { label, parsers, dependencies, types }
+    { label, dependencies, parsers, types }
   ] of entries(graph)) {
     const modules = {};
-    for (const packageLocation of dependencies) {
-      const { exports } = graph[packageLocation];
+    const scopes = {};
+    for (const [dependencyName, packageLocation] of entries(dependencies)) {
+      const { exports, explicit } = graph[packageLocation];
       for (const [exportName, module] of entries(exports)) {
         modules[exportName] = {
           compartment: packageLocation,
           module
         };
       }
+      if (!explicit) {
+        scopes[dependencyName] = {
+          compartment: packageLocation
+        };
+      }
     }
     compartments[packageLocation] = {
       label,
       location: packageLocation,
       modules,
+      scopes,
       parsers,
       types
     };
diff --git a/packages/endo/src/import-archive.js b/packages/endo/src/import-archive.js
index b6c9f501c8..80123edc09 100644
--- a/packages/endo/src/import-archive.js
+++ b/packages/endo/src/import-archive.js
@@ -1,21 +1,29 @@
 /* eslint no-shadow: 0 */
 
 import { readZip } from "./zip.js";
-import { join } from "./node-module-specifier.js";
 import { assemble } from "./assemble.js";
+import { parserForLanguage } from "./parse.js";
 
 const decoder = new TextDecoder();
 
-const makeArchiveImportHookMaker = archive => {
+const makeArchiveImportHookMaker = (archive, compartments) => {
   // per-assembly:
-  const makeImportHook = (packageLocation, parse) => {
+  const makeImportHook = packageLocation => {
     // per-compartment:
+    const { modules } = compartments[packageLocation];
     const importHook = async moduleSpecifier => {
       // per-module:
-      const moduleLocation = join(packageLocation, moduleSpecifier);
+      const module = modules[moduleSpecifier];
+      const parse = parserForLanguage[module.parser];
+      const moduleLocation = `${packageLocation}/${module.location}`;
       const moduleBytes = await archive.read(moduleLocation);
       const moduleSource = decoder.decode(moduleBytes);
-      return parse(moduleSource, moduleSpecifier, `file:///${moduleLocation}`);
+      return parse(
+        moduleSource,
+        moduleSpecifier,
+        `file:///${moduleLocation}`,
+        packageLocation
+      ).record;
     };
     return importHook;
   };
@@ -31,7 +39,7 @@ export const parseArchive = async (archiveBytes, archiveLocation) => {
 
   const { compartments, main, entry: moduleSpecifier } = compartmentMap;
 
-  const makeImportHook = makeArchiveImportHookMaker(archive);
+  const makeImportHook = makeArchiveImportHookMaker(archive, compartments);
 
   const execute = (endowments, modules) => {
     const compartment = assemble({
diff --git a/packages/endo/src/import.js b/packages/endo/src/import.js
index 4210fa6102..6877d7b71b 100644
--- a/packages/endo/src/import.js
+++ b/packages/endo/src/import.js
@@ -3,9 +3,13 @@
 import { compartmentMapForNodeModules } from "./compartmap.js";
 import { search } from "./search.js";
 import { assemble } from "./assemble.js";
+import { parseExtension } from "./extension.js";
 
 const decoder = new TextDecoder();
 
+// q, as in quote, for quoting strings in error messages.
+const q = JSON.stringify;
+
 const resolveLocation = (rel, abs) => new URL(rel, abs).toString();
 
 const makeImportHookMaker = (read, baseLocation) => {
@@ -15,14 +19,47 @@ const makeImportHookMaker = (read, baseLocation) => {
     packageLocation = resolveLocation(packageLocation, baseLocation);
     const importHook = async moduleSpecifier => {
       // per-module:
-      const moduleLocation = resolveLocation(moduleSpecifier, packageLocation);
-      const moduleBytes = await read(moduleLocation);
-      const moduleSource = decoder.decode(moduleBytes);
-      return parse(
-        moduleSource,
-        moduleSpecifier,
-        moduleLocation,
-        packageLocation
+
+      // Collate candidate locations for the moduleSpecifier per Node.js
+      // conventions.
+      const candidates = [];
+      if (moduleSpecifier === ".") {
+        candidates.push("index.js");
+      } else {
+        candidates.push(moduleSpecifier);
+        if (parseExtension(moduleSpecifier) === "") {
+          candidates.push(
+            `${moduleSpecifier}.js`,
+            `${moduleSpecifier}/index.js`
+          );
+        }
+      }
+
+      for (const candidate of candidates) {
+        const moduleLocation = resolveLocation(candidate, packageLocation);
+        // eslint-disable-next-line no-await-in-loop
+        const moduleBytes = await read(moduleLocation).catch(
+          _error => undefined
+        );
+        if (moduleBytes !== undefined) {
+          const moduleSource = decoder.decode(moduleBytes);
+
+          return parse(
+            moduleSource,
+            moduleSpecifier,
+            moduleLocation,
+            packageLocation
+          ).record;
+        }
+      }
+
+      // TODO offer breadcrumbs in the error message, or how to construct breadcrumbs with another tool.
+      throw new Error(
+        `Cannot find file for internal module ${q(
+          moduleSpecifier
+        )} (with candidates ${candidates
+          .map(q)
+          .join(", ")}) in package ${packageLocation}`
       );
     };
     return importHook;
diff --git a/packages/endo/src/module-map-hook.js b/packages/endo/src/module-map-hook.js
new file mode 100644
index 0000000000..1c4b37df8b
--- /dev/null
+++ b/packages/endo/src/module-map-hook.js
@@ -0,0 +1,58 @@
+const { entries } = Object;
+
+// For a full, absolute module specifier like "dependency",
+// produce the module specifier in the dependency, like ".".
+// For a deeper path like "@org/dep/aux" and a prefix like "@org/dep", produce
+// "./aux".
+const trimModuleSpecifierPrefix = (moduleSpecifier, prefix) => {
+  if (moduleSpecifier === prefix) {
+    return ".";
+  }
+  if (moduleSpecifier.startsWith(`${prefix}/`)) {
+    return `./${moduleSpecifier.slice(prefix.length + 1)}`;
+  }
+  return undefined;
+};
+
+// `makeModuleMapHook` generates a `moduleMapHook` for the `Compartment`
+// constructor, suitable for Node.js style packages where any module in the
+// package might be imported.
+// Since searching for all of these modules up front is either needlessly
+// costly (on a file system) or impossible (from a web service), we
+// let the import graph guide our search.
+// Any module specifier with an absolute prefix should be captured by
+// the `moduleMap` or `moduleMapHook`.
+export const makeModuleMapHook = (scopes, modules) => {
+  const moduleMapHook = moduleSpecifier => {
+    // Search for a scope that shares a prefix with the requested module
+    // specifier.
+    // This might be better with a trie, but only a benchmark on real-world
+    // data would tell us whether the additional complexity would translate to
+    // better performance, so this is left readable and presumed slow for now.
+    for (const [prefix, { compartment, compartmentName }] of entries(scopes)) {
+      const remainder = trimModuleSpecifierPrefix(moduleSpecifier, prefix);
+      if (remainder) {
+        // The following line is weird.
+        // Information is flowing backward.
+        // This moduleMapHook writes back to the `modules` descriptor, from the
+        // original compartment map.
+        // So the compartment map that was used to create the compartment
+        // assembly, can then be captured in an archive, obviating the need for
+        // a moduleMapHook when we assemble compartments from the resulting
+        // archiev.
+        modules[moduleSpecifier] = {
+          compartment: compartmentName,
+          module: remainder
+        };
+
+        return compartment.module(remainder);
+      }
+    }
+
+    // No entry in the module map.
+    // Compartments will fall through to their `importHook`.
+    return undefined;
+  };
+
+  return moduleMapHook;
+};
diff --git a/packages/endo/src/parse.js b/packages/endo/src/parse.js
index 4e7a4dc15d..9cd7c3a992 100644
--- a/packages/endo/src/parse.js
+++ b/packages/endo/src/parse.js
@@ -12,7 +12,10 @@ const q = JSON.stringify;
 // verification.
 
 export const parseMjs = (source, _specifier, location, _packageLocation) => {
-  return new StaticModuleRecord(source, location);
+  return {
+    parser: "mjs",
+    record: new StaticModuleRecord(source, location)
+  };
 };
 
 export const parseCjs = (source, _specifier, location, packageLocation) => {
@@ -61,7 +64,10 @@ export const parseCjs = (source, _specifier, location, packageLocation) => {
       new URL("./", location).toString() // __dirname
     );
   };
-  return freeze({ imports, execute });
+  return {
+    parser: "cjs",
+    record: freeze({ imports, execute })
+  };
 };
 
 export const parseJson = (source, _specifier, location, _packageLocation) => {
@@ -75,7 +81,10 @@ export const parseJson = (source, _specifier, location, _packageLocation) => {
       );
     }
   };
-  return freeze({ imports, execute });
+  return {
+    parser: "json",
+    record: freeze({ imports, execute })
+  };
 };
 
 export const makeExtensionParser = (extensions, types) => {
@@ -96,7 +105,7 @@ export const makeExtensionParser = (extensions, types) => {
   };
 };
 
-const parserForLanguage = {
+export const parserForLanguage = {
   mjs: parseMjs,
   cjs: parseCjs,
   json: parseJson
diff --git a/packages/endo/src/zip.js b/packages/endo/src/zip.js
index 022e8d23f5..ec602d6620 100644
--- a/packages/endo/src/zip.js
+++ b/packages/endo/src/zip.js
@@ -7,9 +7,9 @@ export const readZip = async (data, location) => {
   await zip.loadAsync(data);
   const read = async path => {
     const file = zip.file(path);
-    if (file === undefined) {
+    if (file === undefined || file === null) {
       throw new Error(
-        `Cannot find file to read ${path} in archive ${location}`
+        `Cannot find file to read ${path} in archive ${location || "<unknown>"}`
       );
     }
     return file.async("uint8array");
diff --git a/packages/endo/test/main.test.js b/packages/endo/test/main.test.js
index b7bd5cd61b..996e74f03e 100644
--- a/packages/endo/test/main.test.js
+++ b/packages/endo/test/main.test.js
@@ -29,6 +29,7 @@ const assertFixture = (t, namespace) => {
   const {
     avery,
     brooke,
+    clarke,
     builtin,
     endowed,
     typecommon,
@@ -38,6 +39,7 @@ const assertFixture = (t, namespace) => {
   } = namespace;
   t.equal(avery, "Avery", "exports avery");
   t.equal(brooke, "Brooke", "exports brooke");
+  t.equal(clarke, "Clarke", "exports clarke");
   t.equal(builtin, "builtin", "exports builtin");
   t.equal(endowed, endowments.endowment, "exports endowment");
   t.deepEqual(
@@ -58,7 +60,7 @@ const assertFixture = (t, namespace) => {
   t.equal(typehybrid, 42, "type=module and module= package carries exports");
 };
 
-const fixtureAssertionCount = 8;
+const fixtureAssertionCount = 9;
 
 // The "create builtin" test prepares a builtin module namespace object that
 // gets threaded into all subsequent tests to satisfy the "builtin" module
diff --git a/packages/endo/test/node_modules/clarke/TODO b/packages/endo/test/node_modules/clarke/TODO
deleted file mode 100644
index a9df554009..0000000000
--- a/packages/endo/test/node_modules/clarke/TODO
+++ /dev/null
@@ -1,2 +0,0 @@
-This case is not yet covered because the ability to infer ./index.js from an
-extensionless module specifier has not yet been implemented.
diff --git a/packages/endo/test/node_modules/danny/main.js b/packages/endo/test/node_modules/danny/main.js
index ee9cf84f1b..812dafa78e 100644
--- a/packages/endo/test/node_modules/danny/main.js
+++ b/packages/endo/test/node_modules/danny/main.js
@@ -9,5 +9,6 @@ export { typemodule, typecommon, typehybrid, typeparsers };
 
 export { avery } from 'avery';
 export { brooke } from 'brooke';
+export { clarke } from 'clarke';
 export { builtin } from 'builtin';
 export const endowed = endowment;
diff --git a/packages/endo/test/node_modules/danny/package.json b/packages/endo/test/node_modules/danny/package.json
index d630fbda75..7582696c99 100644
--- a/packages/endo/test/node_modules/danny/package.json
+++ b/packages/endo/test/node_modules/danny/package.json
@@ -6,6 +6,7 @@
   "dependencies": {
     "avery": "^1.0.0",
     "brooke": "^1.0.0",
+    "clarke": "^1.0.0",
     "typecommon": "^1.0.0",
     "typemodule": "^1.0.0",
     "typehybrid": "^1.0.0",

From 444b0633d56fd97fd917d9e967db4ff9a328f148 Mon Sep 17 00:00:00 2001
From: Kris Kowal <kris@agoric.com>
Date: Mon, 24 Aug 2020 14:01:31 -0700
Subject: [PATCH 13/13] Type typo

---
 packages/endo/DESIGN.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/endo/DESIGN.md b/packages/endo/DESIGN.md
index b67462d57e..aef58eefdb 100644
--- a/packages/endo/DESIGN.md
+++ b/packages/endo/DESIGN.md
@@ -251,7 +251,7 @@ type FileModule = {
 
 // ExitName is the name of a built-in module, to be threaded in from the
 // modules passed to the module executor.
-type ExitName string;
+type ExitName = string;
 
 // ExitModule refers to a module that comes from outside the compartment map.
 type ExitModule = {