diff --git a/packages/endo/DESIGN.md b/packages/endo/DESIGN.md index 30f04c9652..aef58eefdb 100644 --- a/packages/endo/DESIGN.md +++ b/packages/endo/DESIGN.md @@ -200,6 +200,8 @@ type Compartment = { location: Location, modules: ModuleMap, parsers: ParserMap, + types: ModuleParserMap, + scopes: ScopeMap, // The name of the realm to run the compartment within. // The default is a single frozen realm that has no name. realm: RealmName? // TODO @@ -213,10 +215,13 @@ type Location string; // that do not correspond to source files in the same compartment. type ModuleMap = Object; -// Module describes a module that isn't in the same +// Module describes a module in a compartment. +type Module = CompartmentModule | FileModule | ExitModule; + +// CompartmentModule describes a module that isn't in the same // compartment and how to introduce it to the compartment's // module namespace. -type Module = { +type CompartmentModule = { // The name of the foreign compartment: // TODO an absent compartment name may imply either // that the module is an internal alias of the @@ -225,10 +230,32 @@ type Module = { // The name of the module in the foreign compartment's // module namespace: module: ExternalModuleSpecifier?, - // Alternately, that this module is not from - // any compartment and must be expressly passed - // into the compartment graph from the user. - parameter: ModuleParameter?, // TODO +}; + +// FileLocation is a URL for a module's file relative to the location of the +// containing compartment. +type FileLocation = string + +// FileModule is a module from a file. +// When loading modules off a file system (src/import.js), the assembler +// does not need any explicit FileModules, and instead relies on the +// compartment to declare a ParserMap and optionally ModuleParserMap and +// ScopeMap. +// Endo provides a Compartment importHook and moduleMapHook that will +// search the filesystem for candidate module files and infer the type from the +// extension when necessary. +type FileModule = { + location: FileLocation + parser: Parser +}; + +// ExitName is the name of a built-in module, to be threaded in from the +// modules passed to the module executor. +type ExitName = string; + +// ExitModule refers to a module that comes from outside the compartment map. +type ExitModule = { + exit: ExitName }; // InternalModuleSpecifier is the module specifier @@ -260,6 +287,33 @@ type Extension = string; // "json" corresponds to JSON. type Parser = "mjs" | "cjs" | "json"; +// ModuleParserMap is a table of internal module specifiers +// to the parser that should be used, regardless of that module's +// extension. +// Node.js allows the "module" property in package.json to denote +// a file that is an ECMAScript module, regardless of its extension. +// This is the mechanism that allows Endo to respect that behavior. +type ModuleParserMap = Object + +// ScopeMap is a map from internal module specifier prefixes +// like "dependency" or "@organization/dependency" to another +// compartment. +// Endo uses this to build a moduleMapHook that can dynamically +// generate entries for a compartment's moduleMap into +// Node.js packages that do not explicitly state their "exports". +// For these modules, any specifier under that prefix corresponds +// to a link into some internal module of the foreign compartment. +// When Endo creates an archive, it captures all of the Modules +// explicitly and erases the scopes entry. +type ScopeMap = Object + +// Scope describes the compartment to use for all ad-hoc +// entries in the compartment's module map. +type Scope = { + compartment: CompartmentName +} + + // TODO everything hereafter... // Realm describes another realm to contain one or more diff --git a/packages/endo/README.md b/packages/endo/README.md index 00956dfac9..2790187605 100644 --- a/packages/endo/README.md +++ b/packages/endo/README.md @@ -246,6 +246,25 @@ backward compatibility. However, packages that have a `type` property that explicitly says `module` will treat a `.js` file as an ECMAScript module. +This unforunately conflicts with packages written to work with the ECMAScript +module system emulator in the `esm` package on npm, which allows every file +with the `js` extension to be an ECMAScript module that presents itself to +Node.js as a CommonJS module. +To overcome such obstacles, Endo will accept a non-standard `parsers` property +in `package.json` that maps file extensions, specifically `js` to the +corresponding language name, one of `mjs` for ECMAScript modules, `cjs` for +CommonJS modules, and `json` for JSON modules. +All other language names are reserved and the defaults for files with the +extensions `cjs`, `mjs`, and `json` default to the language of the same name +unless overridden. +If Endo sees `parsers`, it ignores `type`, so these can contradict where using +the `esm` emulator requires. + +```json +{ + "parsers": {"js": "mjs"} +} +``` Many Node.js applications using CommonJS modules expect to be able to `require` a JSON file like `package.json`. diff --git a/packages/endo/bin/endo b/packages/endo/bin/endo new file mode 120000 index 0000000000..8237050ebd --- /dev/null +++ b/packages/endo/bin/endo @@ -0,0 +1 @@ +endo.js \ No newline at end of file diff --git a/packages/endo/bin/endo.js b/packages/endo/bin/endo.js new file mode 100755 index 0000000000..48af66f5a7 --- /dev/null +++ b/packages/endo/bin/endo.js @@ -0,0 +1,6 @@ +#!/usr/bin/env node +(async () => { + const fs = await import("fs"); + const { main } = await import("../src/cli.js"); + main(process, { fs: fs.promises }); +})(); diff --git a/packages/endo/mitm/.gitignore b/packages/endo/mitm/.gitignore new file mode 100644 index 0000000000..64f5a0a681 --- /dev/null +++ b/packages/endo/mitm/.gitignore @@ -0,0 +1 @@ +node diff --git a/packages/endo/package.json b/packages/endo/package.json index 04ca0a83b3..06aab6ea2f 100644 --- a/packages/endo/package.json +++ b/packages/endo/package.json @@ -14,12 +14,16 @@ "require": "./dist/endo.cjs", "browser": "./dist/endo.umd.js" }, + "bin": { + "endo": "./bin/endo.js" + }, "scripts": { "build": "rollup --config rollup.config.js", "clean": "rm -rf dist", "depcheck": "depcheck", "lint": "eslint '**/*.js'", "lint-fix": "eslint --fix '**/*.js'", + "postinstall": "node src/postinstall.js", "prepublish": "yarn clean && yarn build", "test": "yarn build && tap --no-esm --no-coverage --reporter spec 'test/**/*.test.js'" }, diff --git a/packages/endo/src/archive.js b/packages/endo/src/archive.js index b84018df87..cb1058a32a 100644 --- a/packages/endo/src/archive.js +++ b/packages/endo/src/archive.js @@ -1,15 +1,15 @@ -/* global StaticModuleRecord */ /* eslint no-shadow: 0 */ import { writeZip } from "./zip.js"; -import { resolve, join } from "./node-module-specifier.js"; +import { resolve } from "./node-module-specifier.js"; import { parseExtension } from "./extension.js"; import { compartmentMapForNodeModules } from "./compartmap.js"; import { search } from "./search.js"; import { assemble } from "./assemble.js"; -const { entries, fromEntries } = Object; +const { entries, freeze, fromEntries, values } = Object; +// q, as in quote, for quoted strings in error messages. const q = JSON.stringify; const encoder = new TextEncoder(); @@ -17,38 +17,69 @@ const decoder = new TextDecoder(); const resolveLocation = (rel, abs) => new URL(rel, abs).toString(); -const makeRecordingImportHookMaker = (read, baseLocation, manifest, errors) => { +const makeRecordingImportHookMaker = (read, baseLocation, sources) => { // per-assembly: const makeImportHook = (packageLocation, parse) => { // per-compartment: packageLocation = resolveLocation(packageLocation, baseLocation); + const packageSources = sources[packageLocation] || {}; + sources[packageLocation] = packageSources; + const importHook = async moduleSpecifier => { // per-module: + + // In Node.js, an absolute specifier always indicates a built-in or + // third-party dependency. + // The `moduleMapHook` captures all third-party dependencies. + if (moduleSpecifier !== "." && !moduleSpecifier.startsWith("./")) { + packageSources[moduleSpecifier] = { + exit: moduleSpecifier + }; + // Return a place-holder. + // Archived compartments are not executed. + return freeze({ imports: [], execute() {} }); + } + const candidates = [moduleSpecifier]; if (parseExtension(moduleSpecifier) === "") { candidates.push(`${moduleSpecifier}.js`, `${moduleSpecifier}/index.js`); } for (const candidate of candidates) { - const moduleLocation = new URL(candidate, packageLocation).toString(); + const moduleLocation = resolveLocation(candidate, packageLocation); // eslint-disable-next-line no-await-in-loop const moduleBytes = await read(moduleLocation).catch( _error => undefined ); - if (moduleBytes === undefined) { - errors.push( - `missing ${q(candidate)} needed for package ${q(packageLocation)}` - ); - } else { + if (moduleBytes !== undefined) { const moduleSource = decoder.decode(moduleBytes); - const packageManifest = manifest[packageLocation] || {}; - manifest[packageLocation] = packageManifest; - packageManifest[moduleSpecifier] = moduleBytes; + const { record, parser } = parse( + moduleSource, + moduleSpecifier, + moduleLocation + ); - return parse(moduleSource, moduleSpecifier, moduleLocation); + const packageRelativeLocation = moduleLocation.slice( + packageLocation.length + ); + packageSources[moduleSpecifier] = { + location: packageRelativeLocation, + parser, + bytes: moduleBytes + }; + + return record; } } - return new StaticModuleRecord("// Module not found", moduleSpecifier); + + // TODO offer breadcrumbs in the error message, or how to construct breadcrumbs with another tool. + throw new Error( + `Cannot find file for internal module ${q( + moduleSpecifier + )} (with candidates ${candidates + .map(q) + .join(", ")}) in package ${packageLocation}` + ); }; return importHook; }; @@ -60,16 +91,18 @@ const renameCompartments = compartments => { let n = 0; for (const [name, compartment] of entries(compartments)) { const { label } = compartment; - renames[name] = `${label}#${n}`; + renames[name] = `${label}-n${n}`; n += 1; } return renames; }; -const renameCompartmentMap = (compartments, renames) => { +const translateCompartmentMap = (compartments, sources, renames) => { const result = {}; for (const [name, compartment] of entries(compartments)) { - const { label, parsers, types } = compartment; + const { label } = compartment; + + // rename module compartments const modules = {}; for (const [name, module] of entries(compartment.modules || {})) { const compartment = module.compartment @@ -80,14 +113,27 @@ const renameCompartmentMap = (compartments, renames) => { compartment }; } + + // integrate sources into modules + const compartmentSources = sources[name]; + for (const [name, source] of entries(compartmentSources || {})) { + const { location, parser, exit } = source; + modules[name] = { + location, + parser, + exit + }; + } + result[renames[name]] = { label, location: renames[name], - modules, - parsers, - types + modules + // `scopes`, `types`, and `parsers` are not necessary since every + // loadable module is captured in `modules`. }; } + return result; }; @@ -99,10 +145,18 @@ const renameSources = (sources, renames) => { const addSourcesToArchive = async (archive, sources) => { for (const [compartment, modules] of entries(sources)) { - for (const [module, content] of entries(modules)) { - const path = join(compartment, module); + const compartmentLocation = resolveLocation( + `${encodeURIComponent(compartment)}/`, + "file:///" + ); + for (const { location, bytes } of values(modules)) { + const moduleLocation = resolveLocation( + encodeURIComponent(location), + compartmentLocation + ); + const path = new URL(moduleLocation).pathname.slice(1); // elide initial "/" // eslint-disable-next-line no-await-in-loop - await archive.write(path, content); + await archive.write(path, bytes); } } }; @@ -125,22 +179,12 @@ export const makeArchive = async (read, moduleLocation) => { const { compartments, main } = compartmentMap; const sources = {}; - const errors = []; const makeImportHook = makeRecordingImportHookMaker( read, packageLocation, - sources, - errors + sources ); - if (errors.length > 0) { - throw new Error( - `Cannot assemble compartment for ${errors.length} reasons: ${errors.join( - ", " - )}` - ); - } - // Induce importHook to record all the necessary modules to import the given module specifier. const compartment = assemble({ name: main, @@ -151,7 +195,11 @@ export const makeArchive = async (read, moduleLocation) => { await compartment.load(moduleSpecifier); const renames = renameCompartments(compartments); - const renamedCompartments = renameCompartmentMap(compartments, renames); + const renamedCompartments = translateCompartmentMap( + compartments, + sources, + renames + ); const renamedSources = renameSources(sources, renames); const manifest = { diff --git a/packages/endo/src/assemble.js b/packages/endo/src/assemble.js index 0908e16f7b..069de75070 100644 --- a/packages/endo/src/assemble.js +++ b/packages/endo/src/assemble.js @@ -2,6 +2,7 @@ import { resolve } from "./node-module-specifier.js"; import { mapParsers } from "./parse.js"; +import { makeModuleMapHook } from "./module-map-hook.js"; const { entries } = Object; @@ -46,8 +47,41 @@ export const assemble = ({ throw new Error(`Cannot assemble compartment graph that includes a cycle`); } - for (const [inner, outer] of entries(descriptor.modules || {})) { - const { compartment: compartmentName, module: moduleSpecifier } = outer; + descriptor.modules = descriptor.modules || {}; + for (const [inner, outer] of entries(descriptor.modules)) { + const { + compartment: compartmentName, + module: moduleSpecifier, + exit + } = outer; + if (exit !== undefined) { + // TODO Currenly, only the entry package can connect to built-in modules. + // Policies should be able to allow third-party modules to exit to + // built-ins, or have built-ins subverted by modules from specific + // compartments. + const module = modules[exit]; + if (module === undefined) { + throw new Error( + `Cannot assemble module graph with missing external module ${q(exit)}` + ); + } + modules[inner] = module; + } else if (compartmentName !== undefined) { + const compartment = assemble({ + name: compartmentName, + compartments, + makeImportHook, + parents: [...parents, name], + loaded, + Compartment + }); + modules[inner] = compartment.module(moduleSpecifier); + } + } + + const scopes = {}; + for (const [prefix, scope] of entries(descriptor.scopes || {})) { + const { compartment: compartmentName } = scope; const compartment = assemble({ name: compartmentName, compartments, @@ -56,14 +90,16 @@ export const assemble = ({ loaded, Compartment }); - modules[inner] = compartment.module(moduleSpecifier); + scopes[prefix] = { compartment, compartmentName }; } - const parse = mapParsers(descriptor.parsers, descriptor.types); + const parse = mapParsers(descriptor.parsers || {}, descriptor.types || {}); + // TODO makeResolveHook that filters on file patterns const compartment = new Compartment(endowments, modules, { resolveHook: resolve, - importHook: makeImportHook(descriptor.location, parse) + importHook: makeImportHook(descriptor.location, parse), + moduleMapHook: makeModuleMapHook(scopes, descriptor.modules) }); loaded[name] = compartment; diff --git a/packages/endo/src/cli.js b/packages/endo/src/cli.js new file mode 100644 index 0000000000..014f5bed57 --- /dev/null +++ b/packages/endo/src/cli.js @@ -0,0 +1,122 @@ +/* eslint no-shadow: [0] */ +import "./lockdown.js"; +import subprocess from "child_process"; +import { writeArchive } from "./main.js"; +import { search } from "./search.js"; +import { compartmentMapForNodeModules } from "./compartmap.js"; + +const mitmPath = new URL("../mitm", import.meta.url).pathname; + +function usage(message) { + console.error(message); + return 1; +} + +async function noEntryUsage() { + return usage(`expected path to program`); +} + +async function noArchiveUsage() { + return usage(`expected path for archive`); +} + +async function subcommand([arg, ...rest], handlers) { + const keys = Object.keys(handlers); + if (arg === undefined || !keys.includes(arg)) { + return usage(`expected one of ${keys.join(", ")}`); + } + return handlers[arg](rest); +} + +async function parameter(args, handle, usage) { + const [arg, ...rest] = args; + if (arg === undefined) { + return usage(`expected an argument`); + } + if (arg.startsWith("-")) { + return usage(`unexpected flag: ${arg}`); + } + return handle(arg, rest); +} + +async function run(args, { cwd, read, write, stdout, env }) { + async function compartmap(args) { + async function handleEntry(applicationPath, args) { + if (args.length) { + return usage(`unexpected arguments: ${JSON.stringify(args)}`); + } + const currentLocation = new URL(`${cwd()}/`, "file:///"); + const applicationLocation = new URL(applicationPath, currentLocation); + const { packageLocation } = await search(read, applicationLocation); + const compartmentMap = await compartmentMapForNodeModules( + read, + packageLocation + ); + stdout.write(`${JSON.stringify(compartmentMap, null, 2)}\n`); + return 0; + } + return parameter(args, handleEntry, noEntryUsage); + } + + async function archive(args) { + async function handleArchive(archivePath, args) { + async function handleEntry(applicationPath, args) { + if (args.length) { + return usage(`unexpected arguments: ${JSON.stringify(args)}`); + } + const currentLocation = new URL(`${cwd()}/`, "file:///"); + const archiveLocation = new URL(archivePath, currentLocation); + const applicationLocation = new URL(applicationPath, currentLocation); + await writeArchive(write, read, archiveLocation, applicationLocation); + return 0; + } + return parameter(args, handleEntry, noEntryUsage); + } + return parameter(args, handleArchive, noArchiveUsage); + } + + async function exec([arg, ...args]) { + const child = subprocess.spawn(arg, args, { + env: { ...env, PATH: `${mitmPath}:${env.PATH}` }, + stdio: "inherit" + }); + return new Promise(resolve => child.on("exit", resolve)); + } + + return subcommand(args, { compartmap, archive, exec }); +} + +export async function main(process, modules) { + const { fs } = modules; + const { cwd, stdout, env } = process; + + // Filesystem errors often don't have stacks: + + async function read(location) { + try { + return await fs.readFile(new URL(location).pathname); + } catch (error) { + throw new Error(error.message); + } + } + + async function write(location, content) { + try { + return await fs.writeFile(new URL(location).pathname, content); + } catch (error) { + throw new Error(error.message); + } + } + + try { + process.exitCode = await run(process.argv.slice(2), { + read, + write, + cwd, + stdout, + env + }); + } catch (error) { + process.exitCode = usage(error.stack || error.message); + } +} diff --git a/packages/endo/src/compartmap.js b/packages/endo/src/compartmap.js index 01f435a175..e8b36e18fc 100644 --- a/packages/endo/src/compartmap.js +++ b/packages/endo/src/compartmap.js @@ -2,7 +2,7 @@ import { inferExports } from "./infer-exports.js"; -const { create, keys, entries } = Object; +const { create, entries, keys, values } = Object; const decoder = new TextDecoder(); @@ -73,12 +73,32 @@ const findPackage = async (readDescriptor, directory, name) => { } }; -const commonParsers = { js: "cjs", cjs: "cjs", mjs: "mjs", json: "json" }; -const moduleParsers = { js: "mjs", mjs: "mjs", cjs: "cjs", json: "json" }; +const languages = ["cjs", "mjs", "json"]; +const uncontroversialParsers = { cjs: "cjs", mjs: "mjs", json: "json" }; +const commonParsers = { js: "cjs", ...uncontroversialParsers }; +const moduleParsers = { js: "mjs", ...uncontroversialParsers }; -const inferParsers = (type, location) => { - if (type === undefined) { - return commonParsers; +const inferParsers = (descriptor, location) => { + const { type, parsers } = descriptor; + if (parsers !== undefined) { + if (typeof parsers !== "object") { + throw new Error( + `Cannot interpret parser map ${JSON.stringify( + parsers + )} of package at ${location}, must be an object mapping file extensions to corresponding languages (mjs for ECMAScript modules, cjs for CommonJS modules, or json for JSON modules` + ); + } + const invalidLanguages = values(parsers).filter( + language => !languages.includes(language) + ); + if (invalidLanguages.length > 0) { + throw new Error( + `Cannot interpret parser map language values ${JSON.stringify( + invalidLanguages + )} of package at ${location}, must be an object mapping file extensions to corresponding languages (mjs for ECMAScript modules, cjs for CommonJS modules, or json for JSON modules` + ); + } + return { ...uncontroversialParsers, ...parsers }; } if (type === "module") { return moduleParsers; @@ -102,7 +122,7 @@ const inferParsers = (type, location) => { // that the package exports. const graphPackage = async ( - name, + name = "", readDescriptor, graph, { packageLocation, packageDescriptor }, @@ -122,7 +142,7 @@ const graphPackage = async ( const result = {}; graph[packageLocation] = result; - const dependencies = []; + const dependencies = {}; const children = []; for (const name of keys(packageDescriptor.dependencies || {})) { children.push( @@ -139,12 +159,17 @@ const graphPackage = async ( ); } - const { version = "" } = packageDescriptor; - result.label = `${name}@${version}`; - result.dependencies = dependencies; - result.types = {}; - result.exports = inferExports(packageDescriptor, tags, result.types); - result.parsers = inferParsers(packageDescriptor.type, packageLocation); + const { version = "", exports } = packageDescriptor; + const types = {}; + + Object.assign(result, { + label: `${name}${version ? `-v${version}` : ""}`, + explicit: exports !== undefined, + exports: inferExports(packageDescriptor, tags, types), + dependencies, + types, + parsers: inferParsers(packageDescriptor, packageLocation) + }); return Promise.all(children); }; @@ -161,7 +186,7 @@ const gatherDependency = async ( if (dependency === undefined) { throw new Error(`Cannot find dependency ${name} for ${packageLocation}`); } - dependencies.push(dependency.packageLocation); + dependencies[name] = dependency.packageLocation; await graphPackage(name, readDescriptor, graph, dependency, tags); }; @@ -227,22 +252,29 @@ const translateGraph = (mainPackagePath, graph) => { // corresponding compartment can import. for (const [ packageLocation, - { label, parsers, dependencies, types } + { label, dependencies, parsers, types } ] of entries(graph)) { const modules = {}; - for (const packageLocation of dependencies) { - const { exports } = graph[packageLocation]; + const scopes = {}; + for (const [dependencyName, packageLocation] of entries(dependencies)) { + const { exports, explicit } = graph[packageLocation]; for (const [exportName, module] of entries(exports)) { modules[exportName] = { compartment: packageLocation, module }; } + if (!explicit) { + scopes[dependencyName] = { + compartment: packageLocation + }; + } } compartments[packageLocation] = { label, location: packageLocation, modules, + scopes, parsers, types }; diff --git a/packages/endo/src/import-archive.js b/packages/endo/src/import-archive.js index bf3957c1fd..80123edc09 100644 --- a/packages/endo/src/import-archive.js +++ b/packages/endo/src/import-archive.js @@ -1,29 +1,37 @@ /* eslint no-shadow: 0 */ import { readZip } from "./zip.js"; -import { join } from "./node-module-specifier.js"; import { assemble } from "./assemble.js"; +import { parserForLanguage } from "./parse.js"; const decoder = new TextDecoder(); -const makeArchiveImportHookMaker = archive => { +const makeArchiveImportHookMaker = (archive, compartments) => { // per-assembly: - const makeImportHook = (packageLocation, parse) => { + const makeImportHook = packageLocation => { // per-compartment: + const { modules } = compartments[packageLocation]; const importHook = async moduleSpecifier => { // per-module: - const moduleLocation = join(packageLocation, moduleSpecifier); + const module = modules[moduleSpecifier]; + const parse = parserForLanguage[module.parser]; + const moduleLocation = `${packageLocation}/${module.location}`; const moduleBytes = await archive.read(moduleLocation); const moduleSource = decoder.decode(moduleBytes); - return parse(moduleSource, moduleSpecifier, `file:///${moduleLocation}`); + return parse( + moduleSource, + moduleSpecifier, + `file:///${moduleLocation}`, + packageLocation + ).record; }; return importHook; }; return makeImportHook; }; -export const parseArchive = async archiveBytes => { - const archive = await readZip(archiveBytes); +export const parseArchive = async (archiveBytes, archiveLocation) => { + const archive = await readZip(archiveBytes, archiveLocation); const compartmentMapBytes = await archive.read("compartmap.json"); const compartmentMapText = decoder.decode(compartmentMapBytes); @@ -31,7 +39,7 @@ export const parseArchive = async archiveBytes => { const { compartments, main, entry: moduleSpecifier } = compartmentMap; - const makeImportHook = makeArchiveImportHookMaker(archive); + const makeImportHook = makeArchiveImportHookMaker(archive, compartments); const execute = (endowments, modules) => { const compartment = assemble({ @@ -47,12 +55,17 @@ export const parseArchive = async archiveBytes => { return { execute }; }; -export const loadArchive = async (read, archivePath) => { - const archiveBytes = await read(archivePath); - return parseArchive(archiveBytes); +export const loadArchive = async (read, archiveLocation) => { + const archiveBytes = await read(archiveLocation); + return parseArchive(archiveBytes, archiveLocation); }; -export const importArchive = async (read, archivePath, endowments, modules) => { - const archive = await loadArchive(read, archivePath); +export const importArchive = async ( + read, + archiveLocation, + endowments, + modules +) => { + const archive = await loadArchive(read, archiveLocation); return archive.execute(endowments, modules); }; diff --git a/packages/endo/src/import.js b/packages/endo/src/import.js index 4210fa6102..6877d7b71b 100644 --- a/packages/endo/src/import.js +++ b/packages/endo/src/import.js @@ -3,9 +3,13 @@ import { compartmentMapForNodeModules } from "./compartmap.js"; import { search } from "./search.js"; import { assemble } from "./assemble.js"; +import { parseExtension } from "./extension.js"; const decoder = new TextDecoder(); +// q, as in quote, for quoting strings in error messages. +const q = JSON.stringify; + const resolveLocation = (rel, abs) => new URL(rel, abs).toString(); const makeImportHookMaker = (read, baseLocation) => { @@ -15,14 +19,47 @@ const makeImportHookMaker = (read, baseLocation) => { packageLocation = resolveLocation(packageLocation, baseLocation); const importHook = async moduleSpecifier => { // per-module: - const moduleLocation = resolveLocation(moduleSpecifier, packageLocation); - const moduleBytes = await read(moduleLocation); - const moduleSource = decoder.decode(moduleBytes); - return parse( - moduleSource, - moduleSpecifier, - moduleLocation, - packageLocation + + // Collate candidate locations for the moduleSpecifier per Node.js + // conventions. + const candidates = []; + if (moduleSpecifier === ".") { + candidates.push("index.js"); + } else { + candidates.push(moduleSpecifier); + if (parseExtension(moduleSpecifier) === "") { + candidates.push( + `${moduleSpecifier}.js`, + `${moduleSpecifier}/index.js` + ); + } + } + + for (const candidate of candidates) { + const moduleLocation = resolveLocation(candidate, packageLocation); + // eslint-disable-next-line no-await-in-loop + const moduleBytes = await read(moduleLocation).catch( + _error => undefined + ); + if (moduleBytes !== undefined) { + const moduleSource = decoder.decode(moduleBytes); + + return parse( + moduleSource, + moduleSpecifier, + moduleLocation, + packageLocation + ).record; + } + } + + // TODO offer breadcrumbs in the error message, or how to construct breadcrumbs with another tool. + throw new Error( + `Cannot find file for internal module ${q( + moduleSpecifier + )} (with candidates ${candidates + .map(q) + .join(", ")}) in package ${packageLocation}` ); }; return importHook; diff --git a/packages/endo/src/lockdown.cjs b/packages/endo/src/lockdown.cjs new file mode 100644 index 0000000000..e63084603c --- /dev/null +++ b/packages/endo/src/lockdown.cjs @@ -0,0 +1,17 @@ +/* global lockdown */ + +// This is a CommonJS module, to be used like `node -r endo/src/lockdown.cjs`. +// The `endo exec` command stages a man-in-the-middle `node` shell script that +// in turn injects this SES lockdown parameter in all Node.js commands in the +// resulting shell environment. +// The taming behavior may be overridden with environment variables +// like `ERROR_TAMING=unsafe endo exec node ...` + +require("ses"); +lockdown({ + dateTaming: process.env.ENDO_DATE_TAMING || 'safe', + errorTaming: process.env.ENDO_ERROR_TAMING || 'safe', + mathTaming: process.env.ENDO_MATH_TAMING || 'safe', + regExpTaming: process.env.ENDO_REGEXP_TAMING || 'safe', + localeTaming: process.env.ENDO_LOCAlE_TAMING || 'safe', +}); diff --git a/packages/endo/src/lockdown.js b/packages/endo/src/lockdown.js index 7095b6e19a..cb1acb85b9 100644 --- a/packages/endo/src/lockdown.js +++ b/packages/endo/src/lockdown.js @@ -1,6 +1,8 @@ /* global lockdown */ import "ses"; +// This is used by Endo to lockdown its own start compartment. + lockdown({ errorTaming: "unsafe" }); diff --git a/packages/endo/src/module-map-hook.js b/packages/endo/src/module-map-hook.js new file mode 100644 index 0000000000..1c4b37df8b --- /dev/null +++ b/packages/endo/src/module-map-hook.js @@ -0,0 +1,58 @@ +const { entries } = Object; + +// For a full, absolute module specifier like "dependency", +// produce the module specifier in the dependency, like ".". +// For a deeper path like "@org/dep/aux" and a prefix like "@org/dep", produce +// "./aux". +const trimModuleSpecifierPrefix = (moduleSpecifier, prefix) => { + if (moduleSpecifier === prefix) { + return "."; + } + if (moduleSpecifier.startsWith(`${prefix}/`)) { + return `./${moduleSpecifier.slice(prefix.length + 1)}`; + } + return undefined; +}; + +// `makeModuleMapHook` generates a `moduleMapHook` for the `Compartment` +// constructor, suitable for Node.js style packages where any module in the +// package might be imported. +// Since searching for all of these modules up front is either needlessly +// costly (on a file system) or impossible (from a web service), we +// let the import graph guide our search. +// Any module specifier with an absolute prefix should be captured by +// the `moduleMap` or `moduleMapHook`. +export const makeModuleMapHook = (scopes, modules) => { + const moduleMapHook = moduleSpecifier => { + // Search for a scope that shares a prefix with the requested module + // specifier. + // This might be better with a trie, but only a benchmark on real-world + // data would tell us whether the additional complexity would translate to + // better performance, so this is left readable and presumed slow for now. + for (const [prefix, { compartment, compartmentName }] of entries(scopes)) { + const remainder = trimModuleSpecifierPrefix(moduleSpecifier, prefix); + if (remainder) { + // The following line is weird. + // Information is flowing backward. + // This moduleMapHook writes back to the `modules` descriptor, from the + // original compartment map. + // So the compartment map that was used to create the compartment + // assembly, can then be captured in an archive, obviating the need for + // a moduleMapHook when we assemble compartments from the resulting + // archiev. + modules[moduleSpecifier] = { + compartment: compartmentName, + module: remainder + }; + + return compartment.module(remainder); + } + } + + // No entry in the module map. + // Compartments will fall through to their `importHook`. + return undefined; + }; + + return moduleMapHook; +}; diff --git a/packages/endo/src/parse.js b/packages/endo/src/parse.js index 4e7a4dc15d..9cd7c3a992 100644 --- a/packages/endo/src/parse.js +++ b/packages/endo/src/parse.js @@ -12,7 +12,10 @@ const q = JSON.stringify; // verification. export const parseMjs = (source, _specifier, location, _packageLocation) => { - return new StaticModuleRecord(source, location); + return { + parser: "mjs", + record: new StaticModuleRecord(source, location) + }; }; export const parseCjs = (source, _specifier, location, packageLocation) => { @@ -61,7 +64,10 @@ export const parseCjs = (source, _specifier, location, packageLocation) => { new URL("./", location).toString() // __dirname ); }; - return freeze({ imports, execute }); + return { + parser: "cjs", + record: freeze({ imports, execute }) + }; }; export const parseJson = (source, _specifier, location, _packageLocation) => { @@ -75,7 +81,10 @@ export const parseJson = (source, _specifier, location, _packageLocation) => { ); } }; - return freeze({ imports, execute }); + return { + parser: "json", + record: freeze({ imports, execute }) + }; }; export const makeExtensionParser = (extensions, types) => { @@ -96,7 +105,7 @@ export const makeExtensionParser = (extensions, types) => { }; }; -const parserForLanguage = { +export const parserForLanguage = { mjs: parseMjs, cjs: parseCjs, json: parseJson diff --git a/packages/endo/src/postinstall.js b/packages/endo/src/postinstall.js new file mode 100644 index 0000000000..f1c6c38722 --- /dev/null +++ b/packages/endo/src/postinstall.js @@ -0,0 +1,17 @@ +// This postinstall hook creates mitm/node. +// This in turn interposes SES lockdown for all descendent processes, provided +// that the mitm directory is appears before Node.js's bin directory on the +// environment PATH. + +import fs from "fs"; + +const node = process.argv[0]; +const lockdown = new URL("../src/lockdown.cjs", import.meta.url).pathname; +const mitm = new URL("../mitm/node", import.meta.url).pathname; + +const script = `#!/bin/bash +set -ueo pipefail +${node} -r ${lockdown} "$@"`; + +fs.writeFileSync(mitm, script, "utf-8"); +fs.chmodSync(mitm, 0o755); diff --git a/packages/endo/src/zip.js b/packages/endo/src/zip.js index fddc3cc275..ec602d6620 100644 --- a/packages/endo/src/zip.js +++ b/packages/endo/src/zip.js @@ -2,10 +2,18 @@ import JSZip from "jszip"; -export const readZip = async data => { +export const readZip = async (data, location) => { const zip = new JSZip(); await zip.loadAsync(data); - const read = async path => zip.file(path).async("uint8array"); + const read = async path => { + const file = zip.file(path); + if (file === undefined || file === null) { + throw new Error( + `Cannot find file to read ${path} in archive ${location || ""}` + ); + } + return file.async("uint8array"); + }; return { read }; }; diff --git a/packages/endo/test/main.test.js b/packages/endo/test/main.test.js index d008d22b67..996e74f03e 100644 --- a/packages/endo/test/main.test.js +++ b/packages/endo/test/main.test.js @@ -29,14 +29,17 @@ const assertFixture = (t, namespace) => { const { avery, brooke, + clarke, builtin, endowed, typecommon, typemodule, - typehybrid + typehybrid, + typeparsers } = namespace; t.equal(avery, "Avery", "exports avery"); t.equal(brooke, "Brooke", "exports brooke"); + t.equal(clarke, "Clarke", "exports clarke"); t.equal(builtin, "builtin", "exports builtin"); t.equal(endowed, endowments.endowment, "exports endowment"); t.deepEqual( @@ -49,10 +52,15 @@ const assertFixture = (t, namespace) => { [42, 42, 42, 42], "type=module package carries exports" ); + t.deepEqual( + typeparsers, + [42, 42, 42, 42], + "parsers-specifying package carries exports" + ); t.equal(typehybrid, 42, "type=module and module= package carries exports"); }; -const fixtureAssertionCount = 7; +const fixtureAssertionCount = 9; // The "create builtin" test prepares a builtin module namespace object that // gets threaded into all subsequent tests to satisfy the "builtin" module diff --git a/packages/endo/test/node_modules/clarke/TODO b/packages/endo/test/node_modules/clarke/TODO deleted file mode 100644 index a9df554009..0000000000 --- a/packages/endo/test/node_modules/clarke/TODO +++ /dev/null @@ -1,2 +0,0 @@ -This case is not yet covered because the ability to infer ./index.js from an -extensionless module specifier has not yet been implemented. diff --git a/packages/endo/test/node_modules/danny/main.js b/packages/endo/test/node_modules/danny/main.js index cd5f6af4ef..812dafa78e 100644 --- a/packages/endo/test/node_modules/danny/main.js +++ b/packages/endo/test/node_modules/danny/main.js @@ -3,10 +3,12 @@ import typemodule from 'typemodule'; import typecommon from 'typecommon'; import typehybrid from 'typehybrid'; +import typeparsers from 'typeparsers'; -export { typemodule, typecommon, typehybrid }; +export { typemodule, typecommon, typehybrid, typeparsers }; export { avery } from 'avery'; export { brooke } from 'brooke'; +export { clarke } from 'clarke'; export { builtin } from 'builtin'; export const endowed = endowment; diff --git a/packages/endo/test/node_modules/danny/package.json b/packages/endo/test/node_modules/danny/package.json index 41b32ca160..7582696c99 100644 --- a/packages/endo/test/node_modules/danny/package.json +++ b/packages/endo/test/node_modules/danny/package.json @@ -6,8 +6,10 @@ "dependencies": { "avery": "^1.0.0", "brooke": "^1.0.0", + "clarke": "^1.0.0", "typecommon": "^1.0.0", "typemodule": "^1.0.0", - "typehybrid": "^1.0.0" + "typehybrid": "^1.0.0", + "typeparsers": "^1.0.0" } } diff --git a/packages/endo/test/node_modules/typeparsers/cjs.cjs b/packages/endo/test/node_modules/typeparsers/cjs.cjs new file mode 100644 index 0000000000..1ecd817300 --- /dev/null +++ b/packages/endo/test/node_modules/typeparsers/cjs.cjs @@ -0,0 +1 @@ +exports.default = 42; diff --git a/packages/endo/test/node_modules/typeparsers/js.js b/packages/endo/test/node_modules/typeparsers/js.js new file mode 100644 index 0000000000..7a4e8a723a --- /dev/null +++ b/packages/endo/test/node_modules/typeparsers/js.js @@ -0,0 +1 @@ +export default 42; diff --git a/packages/endo/test/node_modules/typeparsers/json.json b/packages/endo/test/node_modules/typeparsers/json.json new file mode 100644 index 0000000000..d81cc0710e --- /dev/null +++ b/packages/endo/test/node_modules/typeparsers/json.json @@ -0,0 +1 @@ +42 diff --git a/packages/endo/test/node_modules/typeparsers/main.js b/packages/endo/test/node_modules/typeparsers/main.js new file mode 100644 index 0000000000..9228517e2b --- /dev/null +++ b/packages/endo/test/node_modules/typeparsers/main.js @@ -0,0 +1,6 @@ +import js from './js.js'; +import mjs from './mjs.mjs'; +import cjs from './cjs.cjs'; +import json from './json.json'; + +export default [js, mjs, cjs, json]; diff --git a/packages/endo/test/node_modules/typeparsers/mjs.mjs b/packages/endo/test/node_modules/typeparsers/mjs.mjs new file mode 100644 index 0000000000..7a4e8a723a --- /dev/null +++ b/packages/endo/test/node_modules/typeparsers/mjs.mjs @@ -0,0 +1 @@ +export default 42; diff --git a/packages/endo/test/node_modules/typeparsers/package.json b/packages/endo/test/node_modules/typeparsers/package.json new file mode 100644 index 0000000000..400e4491b3 --- /dev/null +++ b/packages/endo/test/node_modules/typeparsers/package.json @@ -0,0 +1,5 @@ +{ + "name": "typeparsers", + "parsers": {"js": "mjs"}, + "exports": "./main.js" +}