diff --git a/packages/compartment-mapper/NEWS.md b/packages/compartment-mapper/NEWS.md index e060bf8a4c..ec11bab989 100644 --- a/packages/compartment-mapper/NEWS.md +++ b/packages/compartment-mapper/NEWS.md @@ -5,6 +5,13 @@ User-visible changes to the compartment mapper: - Fixes incompatible behavior with Node.js package conditional exports #2276. Previously, the last matching tag would override all prior matches, often causing a bundle to adopt the `default` instead of a more specific condition. +- Adds `parserForLanguage` and `languageForExtension` options to all modes of + operation such that the compartment mapper can analyze and bundle languages + apart from the built-in languages, which include `esm` and `cjs`. + The `languageForExtension` option provides defaults for the entire + application and the `"parsers"` property in individual `package.json` + descriptors may extend or override using any of the configured or built-in + language parser names. # 0.9.0 (2023-08-07) diff --git a/packages/compartment-mapper/src/archive.js b/packages/compartment-mapper/src/archive.js index eaa0c6fb8d..a86eb98eac 100644 --- a/packages/compartment-mapper/src/archive.js +++ b/packages/compartment-mapper/src/archive.js @@ -1,12 +1,11 @@ // @ts-check /* eslint no-shadow: 0 */ -/** @import {ArchiveOptions} from './types.js' */ +/** @import {ArchiveOptions, ParserForLanguage} from './types.js' */ /** @import {ArchiveWriter} from './types.js' */ /** @import {CompartmentDescriptor} from './types.js' */ /** @import {CompartmentMapDescriptor} from './types.js' */ /** @import {ModuleDescriptor} from './types.js' */ -/** @import {ParserImplementation} from './types.js' */ /** @import {ReadFn} from './types.js' */ /** @import {CaptureSourceLocationHook} from './types.js' */ /** @import {ReadPowers} from './types.js' */ @@ -39,16 +38,20 @@ import { detectAttenuators } from './policy.js'; const textEncoder = new TextEncoder(); -/** @type {Record} */ -const parserForLanguage = { - mjs: parserArchiveMjs, - 'pre-mjs-json': parserArchiveMjs, - cjs: parserArchiveCjs, - 'pre-cjs-json': parserArchiveCjs, - json: parserJson, - text: parserText, - bytes: parserBytes, -}; +const { assign, create, freeze } = Object; + +/** @satisfies {Readonly} */ +const defaultParserForLanguage = freeze( + /** @type {const} */ ({ + mjs: parserArchiveMjs, + 'pre-mjs-json': parserArchiveMjs, + cjs: parserArchiveCjs, + 'pre-cjs-json': parserArchiveCjs, + json: parserJson, + text: parserText, + bytes: parserBytes, + }), +); /** * @param {string} rel - a relative URL @@ -92,7 +95,7 @@ const { keys, entries, fromEntries } = Object; */ const renameCompartments = compartments => { /** @type {Record} */ - const compartmentRenames = Object.create(null); + const compartmentRenames = create(null); let index = 0; let prev = ''; @@ -133,14 +136,14 @@ const renameCompartments = compartments => { * @param {Record} compartmentRenames */ const translateCompartmentMap = (compartments, sources, compartmentRenames) => { - const result = Object.create(null); + const result = create(null); for (const compartmentName of keys(compartmentRenames)) { const compartment = compartments[compartmentName]; const { name, label, retained, policy } = compartment; if (retained) { // rename module compartments /** @type {Record} */ - const modules = Object.create(null); + const modules = create(null); const compartmentModules = compartment.modules; if (compartment.modules) { for (const name of keys(compartmentModules).sort()) { @@ -291,7 +294,7 @@ export const makeArchiveCompartmentMap = (compartmentMap, sources) => { * @param {ArchiveOptions} [options] * @returns {Promise<{sources: Sources, compartmentMapBytes: Uint8Array, sha512?: string}>} */ -const digestLocation = async (powers, moduleLocation, options) => { +const digestLocation = async (powers, moduleLocation, options = {}) => { const { moduleTransforms, modules: exitModules = {}, @@ -303,7 +306,17 @@ const digestLocation = async (powers, moduleLocation, options) => { importHook: exitModuleImportHook = undefined, policy = undefined, sourceMapHook = undefined, - } = options || {}; + parserForLanguage: parserForLanguageOption = {}, + languageForExtension: languageForExtensionOption = {}, + } = options; + + const parserForLanguage = freeze( + assign(create(null), defaultParserForLanguage, parserForLanguageOption), + ); + const languageForExtension = freeze( + assign(create(null), languageForExtensionOption), + ); + const { read, computeSha512 } = unpackReadPowers(powers); const { packageLocation, @@ -359,6 +372,7 @@ const digestLocation = async (powers, moduleLocation, options) => { makeImportHook, moduleTransforms, parserForLanguage, + languageForExtension, archiveOnly: true, }); await compartment.load(entryModuleSpecifier); diff --git a/packages/compartment-mapper/src/bundle.js b/packages/compartment-mapper/src/bundle.js index 07ae9d905a..99f007d9c5 100644 --- a/packages/compartment-mapper/src/bundle.js +++ b/packages/compartment-mapper/src/bundle.js @@ -1,9 +1,8 @@ // @ts-check /* eslint no-shadow: 0 */ -/** @import {ResolveHook} from 'ses' */ /** @import {PrecompiledStaticModuleInterface} from 'ses' */ -/** @import {ParserImplementation} from './types.js' */ +/** @import {ParserForLanguage} from './types.js' */ /** @import {CompartmentDescriptor} from './types.js' */ /** @import {CompartmentSources} from './types.js' */ /** @import {ReadFn} from './types.js' */ @@ -29,18 +28,23 @@ import cjsSupport from './bundle-cjs.js'; const textEncoder = new TextEncoder(); +const { freeze } = Object; const { quote: q } = assert; -/** @type {Record} */ -const parserForLanguage = { - mjs: parserArchiveMjs, - 'pre-mjs-json': parserArchiveMjs, - cjs: parserArchiveCjs, - 'pre-cjs-json': parserArchiveCjs, - json: parserJson, - text: parserText, - bytes: parserBytes, -}; +/** + * @satisfies {Readonly} + */ +const defaultParserForLanguage = freeze( + /** @type {const} */ ({ + mjs: parserArchiveMjs, + 'pre-mjs-json': parserArchiveMjs, + cjs: parserArchiveCjs, + 'pre-cjs-json': parserArchiveCjs, + json: parserJson, + text: parserText, + bytes: parserBytes, + }), +); /** * @param {Record} compartmentDescriptors @@ -161,13 +165,7 @@ function getBundlerKitForModule(module) { /** * @param {ReadFn} read * @param {string} moduleLocation - * @param {object} [options] - * @param {ModuleTransforms} [options.moduleTransforms] - * @param {boolean} [options.dev] - * @param {Set} [options.tags] - * @param {object} [options.commonDependencies] - * @param {Array} [options.searchSuffixes] - * @param {import('./types.js').SourceMapHook} [options.sourceMapHook] + * @param {ArchiveOptions} [options] * @returns {Promise} */ export const makeBundle = async (read, moduleLocation, options) => { @@ -178,9 +176,22 @@ export const makeBundle = async (read, moduleLocation, options) => { searchSuffixes, commonDependencies, sourceMapHook = undefined, + parserForLanguage: parserForLanguageOption = {}, + languageForExtension: languageForExtensionOption = {}, } = options || {}; const tags = new Set(tagsOption); + const parserForLanguage = Object.freeze( + Object.assign( + Object.create(null), + defaultParserForLanguage, + parserForLanguageOption, + ), + ); + const languageForExtension = Object.freeze( + Object.assign(Object.create(null), languageForExtensionOption), + ); + const { packageLocation, packageDescriptorText, @@ -223,6 +234,7 @@ export const makeBundle = async (read, moduleLocation, options) => { makeImportHook, moduleTransforms, parserForLanguage, + languageForExtension, }); await compartment.load(entryModuleSpecifier); diff --git a/packages/compartment-mapper/src/import-archive.js b/packages/compartment-mapper/src/import-archive.js index 0b5644e7d2..aca46aa547 100644 --- a/packages/compartment-mapper/src/import-archive.js +++ b/packages/compartment-mapper/src/import-archive.js @@ -1,6 +1,24 @@ // @ts-check /* eslint no-shadow: "off" */ +/** @import {ImportHook} from 'ses' */ +/** @import {ModuleExportsNamespace} from 'ses' */ +/** @import {StaticModuleType} from 'ses' */ +/** @import {Application} from './types.js' */ +/** @import {CompartmentDescriptor} from './types.js' */ +/** @import {ComputeSourceLocationHook} from './types.js' */ +/** @import {ComputeSourceMapLocationHook} from './types.js' */ +/** @import {ExecuteFn} from './types.js' */ +/** @import {ExitModuleImportHook} from './types.js' */ +/** @import {HashFn} from './types.js' */ +/** @import {ImportHookMaker} from './types.js' */ +/** @import {LanguageForExtension} from './types.js' */ +/** @import {LoadArchiveOptions} from './types.js' */ +/** @import {ParserForLanguage} from './types.js' */ +/** @import {ReadFn} from './types.js' */ +/** @import {ReadPowers} from './types.js' */ +/** @import {SomeObject} from './types.js' */ + import { ZipReader } from '@endo/zip'; import { link } from './link.js'; import parserPreCjs from './parse-pre-cjs.js'; @@ -15,25 +33,24 @@ import { assertCompartmentMap } from './compartment-map.js'; import { exitModuleImportHookMaker } from './import-hook.js'; import { attenuateModuleHook, enforceModulePolicy } from './policy.js'; -/** @import {StaticModuleType} from 'ses' */ -/** @import {Application, CompartmentDescriptor, ComputeSourceLocationHook, ComputeSourceMapLocationHook, ExecuteFn, ExecuteOptions, ExitModuleImportHook, HashFn, ImportHookMaker, LoadArchiveOptions, ParserImplementation, ReadPowers} from './types.js' */ - const DefaultCompartment = Compartment; const { Fail, quote: q } = assert; const textDecoder = new TextDecoder(); -const { freeze } = Object; +const { assign, create, freeze } = Object; -/** @type {Record} */ -const parserForLanguage = { - 'pre-cjs-json': parserPreCjs, - 'pre-mjs-json': parserPreMjs, - json: parserJson, - text: parserText, - bytes: parserBytes, -}; +/** @satisfies {Readonly} */ +const defaultParserForLanguage = freeze( + /** @type {const} */ ({ + 'pre-cjs-json': parserPreCjs, + 'pre-mjs-json': parserPreMjs, + json: parserJson, + text: parserText, + bytes: parserBytes, + }), +); /** * @param {string} errorMessage - error to throw on execute @@ -60,6 +77,7 @@ const postponeErrorToExecute = errorMessage => { * @param {(path: string) => Uint8Array} get * @param {Record} compartments * @param {string} archiveLocation + * @param {ParserForLanguage} parserForLanguage * @param {HashFn} [computeSha512] * @param {ComputeSourceLocationHook} [computeSourceLocation] * @param {ExitModuleImportHook} [exitModuleImportHook] @@ -70,6 +88,7 @@ const makeArchiveImportHookMaker = ( get, compartments, archiveLocation, + parserForLanguage, computeSha512 = undefined, computeSourceLocation = undefined, exitModuleImportHook = undefined, @@ -87,7 +106,7 @@ const makeArchiveImportHookMaker = ( // per-compartment: const compartmentDescriptor = compartments[packageLocation]; const { modules } = compartmentDescriptor; - /** @type {import('ses').ImportHook} */ + /** @type {ImportHook} */ const importHook = async moduleSpecifier => { // per-module: const module = modules[moduleSpecifier]; @@ -146,14 +165,15 @@ const makeArchiveImportHookMaker = ( )} in archive ${q(archiveLocation)}`, ); } - if (parserForLanguage[module.parser] === undefined) { + const parser = parserForLanguage[module.parser]; + if (parser === undefined) { throw Error( `Cannot parse ${q(module.parser)} module ${q( moduleSpecifier, )} in package ${q(packageLocation)} in archive ${q(archiveLocation)}`, ); } - const { parse } = parserForLanguage[module.parser]; + const { parse } = parser; const moduleLocation = `${packageLocation}/${module.location}`; const moduleBytes = get(moduleLocation); @@ -202,6 +222,7 @@ const makeArchiveImportHookMaker = ( packageLocation, { sourceMapUrl, + compartmentDescriptor, }, ); return { record, specifier: moduleSpecifier }; @@ -215,7 +236,7 @@ const makeArchiveImportHookMaker = ( * Creates a fake module namespace object that passes a brand check. * * @param {typeof Compartment} Compartment - * @returns {import('ses').ModuleExportsNamespace} + * @returns {ModuleExportsNamespace} */ const makeFauxModuleExportsNamespace = Compartment => { const compartment = new Compartment( @@ -253,6 +274,8 @@ const makeFauxModuleExportsNamespace = Compartment => { * @param {CompartmentConstructor} [options.Compartment] * @param {ComputeSourceLocationHook} [options.computeSourceLocation] * @param {ComputeSourceMapLocationHook} [options.computeSourceMapLocation] + * @param {ParserForLanguage} [options.parserForLanguage] + * @param {LanguageForExtension} [options.languageForExtension] * @returns {Promise} */ export const parseArchive = async ( @@ -268,8 +291,17 @@ export const parseArchive = async ( Compartment = DefaultCompartment, modules = undefined, importHook: exitModuleImportHook = undefined, + parserForLanguage: parserForLanguageOption = {}, + languageForExtension: languageForExtensionOption = {}, } = options; + const parserForLanguage = freeze( + assign(create(null), defaultParserForLanguage, parserForLanguageOption), + ); + const languageForExtension = freeze( + assign(create(null), languageForExtensionOption), + ); + const compartmentExitModuleImportHook = exitModuleImportHookMaker({ modules, exitModuleImportHook, @@ -330,6 +362,7 @@ export const parseArchive = async ( get, compartments, archiveLocation, + parserForLanguage, computeSha512, computeSourceLocation, compartmentExitModuleImportHook, @@ -342,6 +375,7 @@ export const parseArchive = async ( const { compartment, pendingJobsPromise } = link(compartmentMap, { makeImportHook, parserForLanguage, + languageForExtension, modules: Object.fromEntries( Object.keys(modules || {}).map(specifier => { return [specifier, makeFauxModuleExportsNamespace(Compartment)]; @@ -378,6 +412,7 @@ export const parseArchive = async ( get, compartments, archiveLocation, + parserForLanguage, computeSha512, computeSourceLocation, compartmentExitModuleImportHook, @@ -386,6 +421,7 @@ export const parseArchive = async ( const { compartment, pendingJobsPromise } = link(compartmentMap, { makeImportHook, parserForLanguage, + languageForExtension, globals, modules, transforms, @@ -403,7 +439,7 @@ export const parseArchive = async ( }; /** - * @param {import('@endo/zip').ReadFn | ReadPowers} readPowers + * @param {ReadFn | ReadPowers} readPowers * @param {string} archiveLocation * @param {LoadArchiveOptions} [options] * @returns {Promise} @@ -419,6 +455,8 @@ export const loadArchive = async ( computeSourceLocation, modules, computeSourceMapLocation, + parserForLanguage, + languageForExtension, } = options; const archiveBytes = await read(archiveLocation); return parseArchive(archiveBytes, archiveLocation, { @@ -427,14 +465,16 @@ export const loadArchive = async ( computeSourceLocation, modules, computeSourceMapLocation, + parserForLanguage, + languageForExtension, }); }; /** - * @param {import('@endo/zip').ReadFn | ReadPowers} readPowers + * @param {ReadFn | ReadPowers} readPowers * @param {string} archiveLocation - * @param {ExecuteOptions & LoadArchiveOptions} options - * @returns {Promise} + * @param {LoadArchiveOptions} options + * @returns {Promise} */ export const importArchive = async (readPowers, archiveLocation, options) => { const archive = await loadArchive(readPowers, archiveLocation, options); diff --git a/packages/compartment-mapper/src/import-hook.js b/packages/compartment-mapper/src/import-hook.js index 4f4526e2f8..63db287814 100644 --- a/packages/compartment-mapper/src/import-hook.js +++ b/packages/compartment-mapper/src/import-hook.js @@ -316,6 +316,7 @@ export const makeImportHookMaker = ( (nextSourceMapObject => { sourceMap = JSON.stringify(nextSourceMapObject); }), + compartmentDescriptor, }, ); const { diff --git a/packages/compartment-mapper/src/import.js b/packages/compartment-mapper/src/import.js index 4d4d3eaf6f..47bff1cb1d 100644 --- a/packages/compartment-mapper/src/import.js +++ b/packages/compartment-mapper/src/import.js @@ -2,12 +2,13 @@ /* eslint no-shadow: "off" */ /** @import {Application} from './types.js' */ -/** @import {ArchiveOptions} from './types.js' */ +/** @import {ImportLocationOptions} from './types.js' */ +/** @import {LoadLocationOptions} from './types.js' */ +/** @import {ParserForLanguage} from './types.js' */ /** @import {ExecuteFn} from './types.js' */ -/** @import {ExecuteOptions} from './types.js' */ -/** @import {ParserImplementation} from './types.js' */ /** @import {ReadFn} from './types.js' */ /** @import {ReadPowers} from './types.js' */ +/** @import {SomeObject} from './types.js' */ import { compartmentMapForNodeModules } from './node-modules.js'; import { search } from './search.js'; @@ -24,22 +25,30 @@ import parserMjs from './parse-mjs.js'; import { parseLocatedJson } from './json.js'; import { unpackReadPowers } from './powers.js'; -/** @type {Record} */ -export const parserForLanguage = { - mjs: parserMjs, - cjs: parserCjs, - json: parserJson, - text: parserText, - bytes: parserBytes, -}; +const { assign, create, freeze } = Object; + +/** @satisfies {Readonly} */ +export const defaultParserForLanguage = freeze( + /** @type {const} */ ({ + mjs: parserMjs, + cjs: parserCjs, + json: parserJson, + text: parserText, + bytes: parserBytes, + }), +); /** * @param {ReadFn | ReadPowers} readPowers * @param {string} moduleLocation - * @param {ArchiveOptions} [options] + * @param {LoadLocationOptions} [options] * @returns {Promise} */ -export const loadLocation = async (readPowers, moduleLocation, options) => { +export const loadLocation = async ( + readPowers, + moduleLocation, + options = {}, +) => { const { moduleTransforms = {}, dev = false, @@ -47,7 +56,16 @@ export const loadLocation = async (readPowers, moduleLocation, options) => { searchSuffixes = undefined, commonDependencies = undefined, policy, - } = options || {}; + parserForLanguage: parserForLanguageOption = {}, + languageForExtension: languageForExtensionOption = {}, + } = options; + + const parserForLanguage = freeze( + assign(create(null), defaultParserForLanguage, parserForLanguageOption), + ); + const languageForExtension = freeze( + assign(create(null), languageForExtensionOption), + ); const { read } = unpackReadPowers(readPowers); @@ -96,6 +114,7 @@ export const loadLocation = async (readPowers, moduleLocation, options) => { const { compartment, pendingJobsPromise } = link(compartmentMap, { makeImportHook, parserForLanguage, + languageForExtension, globals, transforms, moduleTransforms, @@ -114,8 +133,8 @@ export const loadLocation = async (readPowers, moduleLocation, options) => { /** * @param {ReadFn | ReadPowers} readPowers * @param {string} moduleLocation - * @param {ExecuteOptions & ArchiveOptions} [options] - * @returns {Promise} the object of the imported modules exported + * @param {ImportLocationOptions} [options] + * @returns {Promise} the object of the imported modules exported * names. */ export const importLocation = async ( diff --git a/packages/compartment-mapper/src/infer-exports.js b/packages/compartment-mapper/src/infer-exports.js index 5b3a051223..40f0108e53 100644 --- a/packages/compartment-mapper/src/infer-exports.js +++ b/packages/compartment-mapper/src/infer-exports.js @@ -1,6 +1,6 @@ // @ts-check -/** @import {Language} from './types.js' */ +/** @import {LanguageForExtension} from './types.js' */ import { join, relativize } from './node-module-specifier.js'; @@ -105,7 +105,7 @@ function* interpretExports(name, exports, tags) { * @param {object} [packageDescriptor.exports] * @param {Set} tags - build tags about the target environment * for selecting relevant exports, e.g., "browser" or "node". - * @param {Record} types - an object to populate + * @param {LanguageForExtension} types - an object to populate * with any recognized module's type, if implied by a tag. * @yields {[string, string]} */ @@ -146,7 +146,7 @@ export const inferExportsEntries = function* inferExportsEntries( * @param {object} descriptor - the parsed body of a package.json file. * @param {Set} tags - build tags about the target environment * for selecting relevant exports, e.g., "browser" or "node". - * @param {Record} types - an object to populate + * @param {LanguageForExtension} types - an object to populate * with any recognized module's type, if implied by a tag. * @returns {Record} */ diff --git a/packages/compartment-mapper/src/link.js b/packages/compartment-mapper/src/link.js index 6962d3e8ad..8f14309207 100644 --- a/packages/compartment-mapper/src/link.js +++ b/packages/compartment-mapper/src/link.js @@ -1,12 +1,11 @@ // @ts-check /** @import {ModuleMapHook} from 'ses' */ -/** @import {ResolveHook} from 'ses' */ -/** @import {ParseFn} from './types.js' */ +/** @import {ParseFn, ParserForLanguage} from './types.js' */ /** @import {ParserImplementation} from './types.js' */ /** @import {ShouldDeferError} from './types.js' */ /** @import {ModuleTransforms} from './types.js' */ -/** @import {Language} from './types.js' */ +/** @import {LanguageForExtension} from './types.js' */ /** @import {ModuleDescriptor} from './types.js' */ /** @import {CompartmentDescriptor} from './types.js' */ /** @import {CompartmentMapDescriptor} from './types.js' */ @@ -22,7 +21,7 @@ import { makeDeferredAttenuatorsProvider, } from './policy.js'; -const { entries, fromEntries } = Object; +const { assign, create, entries, freeze, fromEntries } = Object; const { hasOwnProperty } = Object.prototype; const { apply } = Reflect; const { allSettled } = Promise; @@ -66,7 +65,7 @@ const extensionImpliesLanguage = extension => extension !== 'js'; * @param {Record} languageForModuleSpecifier - In a rare case, * the type of a module is implied by package.json and should not be inferred * from its extension. - * @param {Record} parserForLanguage + * @param {ParserForLanguage} parserForLanguage * @param {ModuleTransforms} moduleTransforms * @returns {ParseFn} */ @@ -124,7 +123,9 @@ const makeExtensionParser = ( `Cannot parse module ${specifier} at ${location}, no parser configured for the language ${language}`, ); } - const { parse } = parserForLanguage[language]; + const { parse } = /** @type {ParserImplementation} */ ( + parserForLanguage[language] + ); return parse(bytes, specifier, location, packageLocation, { sourceMap, ...options, @@ -133,10 +134,10 @@ const makeExtensionParser = ( }; /** - * @param {Record} languageForExtension + * @param {LanguageForExtension} languageForExtension * @param {Record} languageForModuleSpecifier - In a rare case, the type of a module * is implied by package.json and should not be inferred from its extension. - * @param {Record} parserForLanguage + * @param {ParserForLanguage} parserForLanguage * @param {ModuleTransforms} moduleTransforms * @returns {ParseFn} */ @@ -332,7 +333,8 @@ export const link = ( { resolve = resolveFallback, makeImportHook, - parserForLanguage, + parserForLanguage: parserForLanguageOption = {}, + languageForExtension: languageForExtensionOption = {}, globals = {}, transforms = [], moduleTransforms = {}, @@ -344,7 +346,7 @@ export const link = ( const { compartment: entryCompartmentName } = entry; /** @type {Record} */ - const compartments = Object.create(null); + const compartments = create(null); /** * @param {string} attenuatorSpecifier @@ -356,22 +358,45 @@ export const link = ( const pendingJobs = []; + /** @type {LanguageForExtension} */ + const defaultLanguageForExtension = freeze( + assign(create(null), languageForExtensionOption), + ); + /** @type {ParserForLanguage} */ + const parserForLanguage = freeze( + assign(create(null), parserForLanguageOption), + ); + for (const [compartmentName, compartmentDescriptor] of entries( compartmentDescriptors, )) { + // TODO: The default assignments seem to break type inference const { location, name, - modules = Object.create(null), - parsers: languageForExtension = Object.create(null), - types: languageForModuleSpecifier = Object.create(null), - scopes = Object.create(null), + modules = create(null), + parsers: languageForExtensionOverrides = {}, + types: languageForModuleSpecifierOverrides = {}, + scopes = create(null), } = compartmentDescriptor; // Capture the default. // The `moduleMapHook` writes back to the compartment map. compartmentDescriptor.modules = modules; + /** @type {Record} */ + const languageForModuleSpecifier = freeze( + assign(create(null), languageForModuleSpecifierOverrides), + ); + /** @type {LanguageForExtension} */ + const languageForExtension = freeze( + assign( + create(null), + defaultLanguageForExtension, + languageForExtensionOverrides, + ), + ); + const parse = mapParsers( languageForExtension, languageForModuleSpecifier, @@ -381,7 +406,8 @@ export const link = ( /** @type {ShouldDeferError} */ const shouldDeferError = language => { if (language && has(parserForLanguage, language)) { - return parserForLanguage[language].heuristicImports; + return /** @type {ParserImplementation} */ (parserForLanguage[language]) + .heuristicImports; } else { // If language is undefined or there's no parser, the error we could consider deferring is surely related to // that. Nothing to throw here. @@ -408,7 +434,7 @@ export const link = ( scopes, ); - const compartment = new Compartment(Object.create(null), undefined, { + const compartment = new Compartment(create(null), undefined, { resolveHook, importHook, moduleMapHook, diff --git a/packages/compartment-mapper/src/node-modules.js b/packages/compartment-mapper/src/node-modules.js index d91bf0843a..aaee187034 100644 --- a/packages/compartment-mapper/src/node-modules.js +++ b/packages/compartment-mapper/src/node-modules.js @@ -1,16 +1,18 @@ // @ts-check /* eslint no-shadow: 0 */ -/** @import {Language} from './types.js' */ -/** @import {ReadFn} from './types.js' */ -/** @import {MaybeReadFn} from './types.js' */ /** @import {CanonicalFn} from './types.js' */ +/** @import {CompartmentDescriptor} from './types.js' */ /** @import {CompartmentMapDescriptor} from './types.js' */ +/** @import {Language} from './types.js' */ +/** @import {LanguageForExtension} from './types.js' */ +/** @import {MaybeReadFn} from './types.js' */ +/** @import {MaybeReadPowers} from './types.js' */ /** @import {ModuleDescriptor} from './types.js' */ -/** @import {ScopeDescriptor} from './types.js' */ -/** @import {CompartmentDescriptor} from './types.js' */ +/** @import {ReadFn} from './types.js' */ /** @import {ReadPowers} from './types.js' */ -/** @import {MaybeReadPowers} from './types.js' */ +/** @import {ScopeDescriptor} from './types.js' */ +/** @import {SomePackagePolicy} from './types.js' */ /** * The graph is an intermediate object model that the functions of this module @@ -32,7 +34,7 @@ * @property {Record} externalAliases * @property {Record} dependencyLocations - from module name to * location in storage. - * @property {Record} parsers - the parser for + * @property {LanguageForExtension} parsers - the parser for * modules based on their extension. * @property {Record} types - the parser for specific * modules. @@ -169,23 +171,49 @@ const findPackage = async (readDescriptor, canonical, directory, name) => { } }; -const languages = ['mjs', 'cjs', 'json', 'text', 'bytes']; -const uncontroversialParsers = { +const defaultLanguages = /** @type {const} */ ([ + 'mjs', + 'cjs', + 'json', + 'text', + 'bytes', +]); +const defaultUncontroversialParsers = /** @type {const} */ ({ cjs: 'cjs', mjs: 'mjs', json: 'json', text: 'text', bytes: 'bytes', -}; -const commonParsers = { js: 'cjs', ...uncontroversialParsers }; -const moduleParsers = { js: 'mjs', ...uncontroversialParsers }; +}); +const defaultCommonParsers = /** @type {const} */ ({ + js: 'cjs', + ...defaultUncontroversialParsers, +}); +const defaultModuleParsers = /** @type {const} */ ({ + js: 'mjs', + ...defaultUncontroversialParsers, +}); /** * @param {object} descriptor * @param {string} location + * @param {object} [options] + * @param {readonly string[]|string[]} [options.languages] + * @param {Record} [options.uncontroversialParsers] + * @param {Record} [options.commonParsers] + * @param {Record} [options.moduleParsers] * @returns {Record} */ -const inferParsers = (descriptor, location) => { +const inferParsers = ( + descriptor, + location, + { + languages = defaultLanguages, + uncontroversialParsers = defaultUncontroversialParsers, + commonParsers = defaultCommonParsers, + moduleParsers = defaultModuleParsers, + } = {}, +) => { const { type, module, parsers } = descriptor; let additionalParsers = Object.create(null); if (parsers !== undefined) { @@ -684,7 +712,7 @@ const translateGraph = ( scopes, parsers, types, - policy: packagePolicy, + policy: /** @type {SomePackagePolicy} */ (packagePolicy), }; } diff --git a/packages/compartment-mapper/src/policy-format.js b/packages/compartment-mapper/src/policy-format.js index 3d18f41cc1..346c889ef8 100644 --- a/packages/compartment-mapper/src/policy-format.js +++ b/packages/compartment-mapper/src/policy-format.js @@ -1,5 +1,8 @@ // @ts-check +/** @import {SomePackagePolicy} from './types.js' */ +/** @import {SomePolicy} from './types.js' */ + const { entries, keys } = Object; const { isArray } = Array; const q = JSON.stringify; @@ -134,7 +137,7 @@ const isPolicyItem = item => * @param {unknown} allegedPackagePolicy - Alleged `PackagePolicy` to test * @param {string} path - Path in the `Policy` object; used for error messages only * @param {string} [url] - URL of the policy file; used for error messages only - * @returns {asserts allegedPackagePolicy is import('./types.js').PackagePolicy|undefined} + * @returns {asserts allegedPackagePolicy is SomePackagePolicy|undefined} */ export const assertPackagePolicy = (allegedPackagePolicy, path, url) => { if (allegedPackagePolicy === undefined) { @@ -153,8 +156,10 @@ export const assertPackagePolicy = (allegedPackagePolicy, path, url) => { globals, noGlobalFreeze, defaultAttenuator: _ignore, // a carve out for the default attenuator in compartment map + // eslint-disable-next-line no-unused-vars + options, // any extra options ...extra - } = packagePolicy; + } = /** @type {SomePackagePolicy} */ (packagePolicy); assert( keys(extra).length === 0, @@ -200,7 +205,7 @@ export const assertPackagePolicy = (allegedPackagePolicy, path, url) => { * It also moonlights as a type guard. * * @param {unknown} allegedPolicy - Alleged `Policy` to test - * @returns {asserts allegedPolicy is import('./types.js').Policy|undefined} + * @returns {asserts allegedPolicy is SomePolicy|undefined} */ export const assertPolicy = allegedPolicy => { if (allegedPolicy === undefined) { diff --git a/packages/compartment-mapper/src/types.js b/packages/compartment-mapper/src/types.js index 571bea1559..4b2c36a89c 100644 --- a/packages/compartment-mapper/src/types.js +++ b/packages/compartment-mapper/src/types.js @@ -50,9 +50,9 @@ export {}; * compartment map. * @property {Record} modules * @property {Record} scopes - * @property {Record} parsers - language for extension - * @property {Record} types - language for module specifier - * @property {object} policy - policy specific to compartment + * @property {LanguageForExtension} parsers - language for extension + * @property {LanguageForModuleSpecifier} types - language for module specifier + * @property {SomePackagePolicy} policy - policy specific to compartment */ /** @@ -82,7 +82,15 @@ export {}; */ /** - * @typedef {'mjs' | 'cjs' | 'json' | 'bytes' | 'text' | 'pre-mjs-json' | 'pre-cjs-json'} Language + * Natively-recognized and custom languages + * + * @typedef {LiteralUnion} Language + */ + +/** + * Languages natively recognized by `compartment-mapper` + * + * @typedef {'mjs' | 'cjs' | 'json' | 'bytes' | 'text' | 'pre-mjs-json' | 'pre-cjs-json'} BuiltinLanguage */ // ///////////////////////////////////////////////////////////////////////////// @@ -253,6 +261,7 @@ export {}; * @param {SourceMapHook} [options.sourceMapHook] * @param {string} [options.sourceMapUrl] * @param {ReadFn | ReadPowers} [options.readPowers] + * @param {CompartmentDescriptor} [options.compartmentDescriptor] * @returns {Promise<{ * bytes: Uint8Array, * parser: Language, @@ -284,15 +293,28 @@ export {}; */ /** - * @typedef {object} LoadArchiveOptions + * @see {@link LoadArchiveOptions} + * @typedef {object} ExtraLoadArchiveOptions * @property {string} [expectedSha512] * @property {Record} [modules] * @property {typeof Compartment} [Compartment] * @property {ComputeSourceLocationHook} [computeSourceLocation] * @property {ComputeSourceMapLocationHook} [computeSourceMapLocation] + * @property {ParserForLanguage} [parserForLanguage] + * @property {LanguageForExtension} [languageForExtension] + */ + +/** + * Options for `loadArchive()` + * + * @typedef {ExecuteOptions & ExtraLoadArchiveOptions} LoadArchiveOptions */ /** + * Set of options available in the context of code execution. + * + * May be used only as an intersection with other "options" types + * * @typedef {object} ExecuteOptions * @property {object} [globals] * @property {Array} [transforms] @@ -304,19 +326,43 @@ export {}; */ /** - * @typedef {Record} ParserForLanguage + * Mapping of {@link Language Languages} to {@link ParserImplementation ParserImplementations} + * + * @typedef {Record} ParserForLanguage + */ + +/** + * Mapping of file extension to {@link Language Languages}. + * + * @typedef {Record} LanguageForExtension + */ + +/** + * Mapping of module specifier to {@link Language Languages}. + * + * @typedef {Record} LanguageForModuleSpecifier + */ + +/** + * Options for `loadLocation()` + * + * @typedef {ArchiveOptions} LoadLocationOptions */ /** + * @see {@link LinkOptions} * @typedef {object} ExtraLinkOptions * @property {ResolveHook} [resolve] * @property {ImportHookMaker} makeImportHook - * @property {ParserForLanguage} parserForLanguage + * @property {ParserForLanguage} [parserForLanguage] + * @property {LanguageForExtension} [languageForExtension] * @property {ModuleTransforms} [moduleTransforms] * @property {boolean} [archiveOnly] */ /** + * Options for `link()` + * * @typedef {ExecuteOptions & ExtraLinkOptions} LinkOptions */ @@ -378,13 +424,15 @@ export {}; * @property {ModuleTransforms} [moduleTransforms] * @property {Record} [modules] * @property {boolean} [dev] - * @property {object} [policy] + * @property {SomePolicy} [policy] * @property {Set} [tags] * @property {CaptureSourceLocationHook} [captureSourceLocation] * @property {ExitModuleImportHook} [importHook] * @property {Array} [searchSuffixes] * @property {Record} [commonDependencies] * @property {SourceMapHook} [sourceMapHook] + * @property {Record} [parserForLanguage] + * @property {LanguageForExtension} [languageForExtension] */ // ///////////////////////////////////////////////////////////////////////////// @@ -478,29 +526,96 @@ export {}; /** * An object representing a base package policy. - * @template [PackagePolicyItem=void] - * @template [GlobalsPolicyItem=void] - * @template [BuiltinsPolicyItem=void] + * + * @template [PackagePolicyItem=void] Additional types for a package policy item + * @template [GlobalsPolicyItem=void] Additional types for a global policy item + * @template [BuiltinsPolicyItem=void] Additional types for a builtin policy item + * @template [ExtraOptions=unknown] Additional options * @typedef {object} PackagePolicy * @property {string} [defaultAttenuator] - The default attenuator. * @property {PolicyItem} [packages] - The policy item for packages. * @property {PolicyItem|AttenuationDefinition} [globals] - The policy item or full attenuation definition for globals. * @property {PolicyItem|NestedAttenuationDefinition} [builtins] - The policy item or nested attenuation definition for builtins. * @property {boolean} [noGlobalFreeze] - Whether to disable global freeze. + * @property {ExtraOptions} [options] - Any additional user-defined options can be added to the policy here */ /** * An object representing a base policy. - * @template [PackagePolicyItem=void] - * @template [GlobalsPolicyItem=void] - * @template [BuiltinsPolicyItem=void] + * + * @template [PackagePolicyItem=void] Additional types for a package policy item + * @template [GlobalsPolicyItem=void] Additional types for a global policy item + * @template [BuiltinsPolicyItem=void] Additional types for a builtin policy item + * @template [ExtraOptions=unknown] Additional package-level options * @typedef {object} Policy - * @property {Record>} resources - The package policies for the resources. + * @property {Record>} resources - The package policies for the resources. * @property {string} [defaultAttenuator] - The default attenuator. - * @property {PackagePolicy} [entry] - The package policy for the entry. + * @property {PackagePolicy} [entry] - The package policy for the entry. */ /** * Any object. All objects. Not `null`, though. * @typedef {Record} SomeObject */ + +/** + * Any {@link PackagePolicy} + * + * @typedef {PackagePolicy} SomePackagePolicy + */ + +/** + * Any {@link Policy} + * + * @typedef {Policy} SomePolicy + */ + +/** + * Matches any {@link https://developer.mozilla.org/en-US/docs/Glossary/Primitive primitive value}. + * + * @typedef {null|undefined|string|number|boolean|symbol|bigint} Primitive + * @see {@link https://github.com/sindresorhus/type-fest/blob/main/source/primitive.d.ts original source} + */ + +/** + * Allows creating a union type by combining primitive types and literal + * types without sacrificing auto-completion in IDEs for the literal type part + * of the union. + * + * Currently, when a union type of a primitive type is combined with literal types, + * TypeScript loses all information about the combined literals. Thus, when such + * a type is used in an IDE with autocompletion, no suggestions are made for the + * declared literals. + * + * This type is a workaround for {@link https://github.com/Microsoft/TypeScript/issues/29729 Microsoft/TypeScript#29729}. + * It will be removed as soon as it's not needed anymore. + * + * + * @see {@link https://github.com/sindresorhus/type-fest/blob/main/source/literal-union.d.ts original source} + * @template LiteralType The literal type + * @template {Primitive} PrimitiveType The primitive type + * @typedef {LiteralType | (PrimitiveType & Record)} LiteralUnion + * @example + * ```ts + * // Before + * + * type Pet = 'dog' | 'cat' | string; + * + * const pet: Pet = ''; + * // Start typing in your TypeScript-enabled IDE. + * // You **will not** get auto-completion for `dog` and `cat` literals. + * + * // After + * + * type Pet2 = LiteralUnion<'dog' | 'cat', string>; + * + * const pet: Pet2 = ''; + * // You **will** get auto-completion for `dog` and `cat` literals. + * ``` + */ + +/** + * Options for `importLocation()` + * + * @typedef {ExecuteOptions & ArchiveOptions} ImportLocationOptions + */ diff --git a/packages/compartment-mapper/test/custom-parser.test.js b/packages/compartment-mapper/test/custom-parser.test.js new file mode 100644 index 0000000000..65364ca20d --- /dev/null +++ b/packages/compartment-mapper/test/custom-parser.test.js @@ -0,0 +1,197 @@ +import 'ses'; +import fs from 'fs'; +import test from 'ava'; +import url from 'url'; +import { loadLocation } from '../src/import.js'; +import { makeReadPowers } from '../src/node-powers.js'; + +const { freeze } = Object; +const { quote: q } = assert; + +const readPowers = makeReadPowers({ fs, url }); +const { read } = readPowers; + +test('defining a custom parser works', async t => { + const fixture = new URL( + 'fixtures-0/node_modules/markdown/README.md', + import.meta.url, + ).toString(); + + const application = await loadLocation(read, fixture, { + parserForLanguage: { + markdown: { + // This parser parses markdown files. Use your imagination + parse: async ( + bytes, + _specifier, + _moduleLocation, + _packageLocation, + { compartmentDescriptor } = {}, + ) => { + const execute = moduleEnvironmentRecord => { + moduleEnvironmentRecord.default = new TextDecoder().decode(bytes); + }; + + t.truthy( + compartmentDescriptor, + 'compartmentDescriptor is passed to parser', + ); + + return { + parser: 'markdown', + bytes, + record: freeze({ + imports: [], + exports: ['default'], + reexports: [], + execute, + }), + }; + }, + }, + }, + languageForExtension: { + md: 'markdown', + }, + }); + const { namespace } = await application.import({}); + const { default: value } = namespace; + t.is( + value, + `# A Markdown File + +This fixture is for testing custom parsers. +`, + 'using a custom parser worked', + ); +}); + +test('a custom parser may implement policy enforcement (allowed)', async t => { + const fixture = new URL( + 'fixtures-0/node_modules/markdown/README.md', + import.meta.url, + ).toString(); + + const application = await loadLocation(read, fixture, { + policy: { + resources: {}, + // this was the most straightforward way to do it, sorry! + entry: { + options: { + markdown: true, + }, + }, + }, + parserForLanguage: { + markdown: { + parse: async ( + bytes, + specifier, + _moduleLocation, + _packageLocation, + { compartmentDescriptor = {} } = {}, + ) => { + const execute = moduleEnvironmentRecord => { + moduleEnvironmentRecord.default = new TextDecoder().decode(bytes); + }; + + // it's just a test. just a test. breathe + if ( + !compartmentDescriptor.policy || + !compartmentDescriptor.policy.options || + (compartmentDescriptor.policy && + compartmentDescriptor.policy.options && + compartmentDescriptor.policy.options.markdown !== true) + ) { + throw new Error(`Markdown parsing not allowed for ${q(specifier)}`); + } + + return { + parser: 'markdown', + bytes, + record: freeze({ + imports: [], + exports: ['default'], + reexports: [], + execute, + }), + }; + }, + }, + }, + languageForExtension: { + md: 'markdown', + }, + }); + const { namespace } = await application.import({}); + const { default: value } = namespace; + t.is( + value, + `# A Markdown File + +This fixture is for testing custom parsers. +`, + 'using a custom parser worked', + ); +}); + +test('a custom parser may implement policy enforcement (disallowed)', async t => { + const fixture = new URL( + 'fixtures-0/node_modules/markdown/README.md', + import.meta.url, + ).toString(); + + const application = await loadLocation(read, fixture, { + policy: { + resources: {}, + // this was the most straightforward way to do it, sorry! + entry: { + options: { + markdown: false, + }, + }, + }, + parserForLanguage: { + markdown: { + // This parser parses markdown files. Use your imagination + parse: async ( + bytes, + specifier, + _moduleLocation, + _packageLocation, + { compartmentDescriptor = {} } = {}, + ) => { + const execute = moduleEnvironmentRecord => { + moduleEnvironmentRecord.default = new TextDecoder().decode(bytes); + }; + + // it's just a test. just a test. breathe + if ( + !compartmentDescriptor.policy || + !compartmentDescriptor.policy.options || + (compartmentDescriptor.policy && + compartmentDescriptor.policy.options && + compartmentDescriptor.policy.options.markdown !== true) + ) { + throw new Error(`Markdown parsing not allowed for ${q(specifier)}`); + } + + return { + parser: 'markdown', + bytes, + record: freeze({ + imports: [], + exports: ['default'], + reexports: [], + execute, + }), + }; + }, + }, + }, + languageForExtension: { md: 'markdown' }, + }); + await t.throwsAsync(application.import({}), { + message: /Markdown parsing not allowed for.+README\.md/, + }); +}); diff --git a/packages/compartment-mapper/test/fixtures-0/node_modules/markdown/README.md b/packages/compartment-mapper/test/fixtures-0/node_modules/markdown/README.md new file mode 100644 index 0000000000..f64cf5d08a --- /dev/null +++ b/packages/compartment-mapper/test/fixtures-0/node_modules/markdown/README.md @@ -0,0 +1,3 @@ +# A Markdown File + +This fixture is for testing custom parsers. diff --git a/packages/compartment-mapper/test/policy-format.test.js b/packages/compartment-mapper/test/policy-format.test.js index 34d498d725..0b419d17ab 100644 --- a/packages/compartment-mapper/test/policy-format.test.js +++ b/packages/compartment-mapper/test/policy-format.test.js @@ -62,6 +62,12 @@ const q = JSON.stringify; { globals: ['a', {}], }, + { + options: { + winken: ['blinken', 'nod'], + abc: 123, + }, + }, ].forEach(sample => { test(`assertPackagePolicy(${q(sample)}) -> valid`, t => { t.plan(1);