feat: first-class Deno installs (#146)

src/generator.ts

@@ -74,6 +74,16 @@ export interface GeneratorOptions {
    * New providers can be provided via the `customProviders` option. PRs to merge in providers are welcome as well.
    */
   defaultProvider?: string;
+  /**
+   * The default registry to use when no registry is provided to an install.
+   * Defaults to 'npm:'.
+   * 
+   * Registries are separated from providers because multiple providers can serve
+   * any public registry.
+   * 
+   * Internally, the default providers for registries are handled by the providers object
+   */
+  defaultRegistry?: string;
   /**
    * The conditional environment resolutions to apply.
    * 
@@ -159,13 +169,14 @@ export interface GeneratorOptions {
   /**
    * A map of custom scoped providers.
    * 
-   * The provider map allows setting custom providers for specific package names or package scopes.
+   * The provider map allows setting custom providers for specific package names, package scopes or registries.
    * For example, an organization with private packages with names like `npmpackage` and `@orgscope/...` can define the custom providers to reference these from a custom source:
    * 
    * ```js
    *   providers: {
    *     'npmpackage': 'nodemodules',
-   *     '@orgscope': 'nodemodules'
+   *     '@orgscope': 'nodemodules',
+   *     'npm:': 'nodemodules'
    *   }
    * ```
    * 
@@ -309,6 +320,7 @@ export class Generator {
    *     }
    *   },
    *   defaultProvider: 'jspm',
+   *   defaultRegistry: 'npm',
    *   providers: {
    *     '@orgscope': 'nodemodules'
    *   },
@@ -325,8 +337,9 @@ export class Generator {
     inputMap = undefined,
     env = ['browser', 'development', 'module'],
     defaultProvider = 'jspm',
+    defaultRegistry = 'npm',
     customProviders = undefined,
-    providers = {},
+    providers,
     resolutions = {},
     cache = true,
     stdlib = '@jspm/core',
@@ -392,6 +405,7 @@ export class Generator {
       stdlib,
       env,
       defaultProvider,
+      defaultRegistry,
       providers,
       inputMap,
       ignore,
@@ -644,7 +658,7 @@ export class Generator {
       this.traceMap.startInstall();
     await this.traceMap.processInputMap;
     try {
-      const { alias, target, subpath } = await installToTarget.call(this, install);
+      const { alias, target, subpath } = await installToTarget.call(this, install, this.traceMap.installer.defaultRegistry);
       await this.traceMap.add(alias, target);
       await this.traceMap.visit(alias + subpath.slice(1), { mode: 'new' });
       this.traceMap.pin(alias + subpath.slice(1));
@@ -800,7 +814,7 @@ export async function fetch (url: string, opts: any = {}) {
  */
 export async function lookup (install: string | Install, { provider, cache }: LookupOptions = {}) {
   const generator = new Generator({ cache: !cache, defaultProvider: provider });
-  const { target, subpath, alias } = await installToTarget.call(generator, install);
+  const { target, subpath, alias } = await installToTarget.call(generator, install, generator.traceMap.installer.defaultRegistry);
   if (target instanceof URL)
     throw new Error('URL lookups not supported');
   const resolved = await generator.traceMap.resolver.resolveLatestTarget(target, true, generator.traceMap.installer.defaultProvider);
@@ -892,14 +906,14 @@ export async function getPackageBase (url: string | URL, { provider, cache }: Lo
   return generator.traceMap.resolver.getPackageBase(typeof url === 'string' ? url : url.href);
 }
 
-async function installToTarget (this: Generator, install: Install | string) {
+async function installToTarget (this: Generator, install: Install | string, defaultRegistry: string) {
   if (typeof install === 'string')
     install = { target: install };
   if (typeof install.target !== 'string')
     throw new Error('All installs require a "target" string.');
   if (install.subpath !== undefined && (typeof install.subpath !== 'string' || (install.subpath !== '.' && !install.subpath.startsWith('./'))))
     throw new Error(`Install subpath "${install.subpath}" must be a string equal to "." or starting with "./".${typeof install.subpath === 'string' ? `\nTry setting the subpath to "./${install.subpath}"` : ''}`);
-  const { alias, target, subpath } = await toPackageTarget(this.traceMap.resolver, install.target, this.baseUrl.href);
+  const { alias, target, subpath } = await toPackageTarget(this.traceMap.resolver, install.target, this.baseUrl.href, defaultRegistry);
   return {
     alias: install.alias || alias,
     target,

src/install/installer.ts

@@ -8,6 +8,7 @@ import { JspmError, throwInternalError } from "../common/err.js";
 import { nodeBuiltinSet } from '../providers/node.js';
 import { parseUrlPkg } from '../providers/jspm.js';
 import { getResolution, LockResolutions, pruneResolutions, setResolution, stringResolution } from './lock.js';
+import { registryProviders } from '../providers/index.js';
 
 export interface PackageProvider {
   provider: string;
@@ -77,6 +78,7 @@ export interface InstallOptions {
   resolutions?: Record<string, string>;
 
   defaultProvider?: string;
+  defaultRegistry?: string;
   providers?: Record<string, string>;
 }
 
@@ -92,6 +94,7 @@ export class Installer {
   added = new Map<string, InstallTarget>();
   hasLock = false;
   defaultProvider = { provider: 'jspm', layer: 'default' };
+  defaultRegistry = 'npm';
   providers: Record<string, string>;
   resolutions: Record<string, string>;
   log: Log;
@@ -105,12 +108,16 @@ export class Installer {
     this.opts = opts;
     this.hasLock = !!opts.lock;
     this.installs = opts.lock || {};
+    if (opts.defaultRegistry)
+      this.defaultRegistry = opts.defaultRegistry;
     if (opts.defaultProvider)
       this.defaultProvider = {
         provider: opts.defaultProvider.split('.')[0],
         layer: opts.defaultProvider.split('.')[1] || 'default'
       };
-    this.providers = opts.providers || {};
+    this.providers = Object.assign({}, registryProviders);
+    if (opts.providers)
+      Object.assign(this.providers, opts.providers);
 
     if (opts.stdlib) {
       if (isURL(opts.stdlib) || opts.stdlib[0] === '.') {
@@ -119,7 +126,7 @@ export class Installer {
           this.stdlibTarget.pathname = this.stdlibTarget.pathname.slice(0, -1);
       }
       else {
-        this.stdlibTarget = newPackageTarget(opts.stdlib, this.installBaseUrl);
+        this.stdlibTarget = newPackageTarget(opts.stdlib, this.installBaseUrl, this.defaultRegistry);
       }
     }
   }
@@ -212,7 +219,7 @@ export class Installer {
 
     let provider = this.defaultProvider;
     for (const name of Object.keys(this.providers)) {
-      if (target.name.startsWith(name) && (target.name.length === name.length || target.name[name.length] === '/')) {
+      if (name.endsWith(':') && target.registry === name.slice(0, -1) || target.name.startsWith(name) && (target.name.length === name.length || target.name[name.length] === '/')) {
         provider = { provider: this.providers[name], layer: 'default' };
         const layerIndex = provider.provider.indexOf('.');
         if (layerIndex !== -1) {
@@ -236,7 +243,7 @@ export class Installer {
 
     // resolutions are authoritative at the top-level
     if (this.resolutions[pkgName]) {
-      const resolutionTarget = newPackageTarget(this.resolutions[pkgName], this.opts.baseUrl.href, pkgName);
+      const resolutionTarget = newPackageTarget(this.resolutions[pkgName], this.opts.baseUrl.href, this.defaultRegistry, pkgName);
       if (JSON.stringify(target) !== JSON.stringify(resolutionTarget))
         return this.installTarget(pkgName, resolutionTarget, mode, pkgScope, pjsonPersist, subpath, parentUrl);
     }
@@ -274,7 +281,7 @@ export class Installer {
     }
 
     if (this.resolutions[pkgName]) {
-      return this.installTarget(pkgName, newPackageTarget(this.resolutions[pkgName], this.opts.baseUrl.href, pkgName), mode, pkgUrl, false, null, parentUrl);
+      return this.installTarget(pkgName, newPackageTarget(this.resolutions[pkgName], this.opts.baseUrl.href, this.defaultRegistry, pkgName), mode, pkgUrl, false, null, parentUrl);
     }
 
     // resolution scope cascading for existing only
@@ -296,7 +303,7 @@ export class Installer {
     // package dependencies
     const installTarget = pcfg.dependencies?.[pkgName] || pcfg.peerDependencies?.[pkgName] || pcfg.optionalDependencies?.[pkgName] || pkgUrl === this.installBaseUrl && pcfg.devDependencies?.[pkgName];
     if (installTarget) {
-      const target = newPackageTarget(installTarget, pkgUrl, pkgName);
+      const target = newPackageTarget(installTarget, pkgUrl, this.defaultRegistry, pkgName);
       return this.installTarget(pkgName, target, mode, pkgUrl, false, null, parentUrl);
     }
 
@@ -305,7 +312,7 @@ export class Installer {
       return this.installs[this.installBaseUrl][pkgName];
 
     // global install fallback
-    const target = newPackageTarget('*', pkgUrl, pkgName);
+    const target = newPackageTarget('*', pkgUrl, this.defaultRegistry, pkgName);
     const exactInstall = await this.installTarget(pkgName, target, mode, pkgUrl, true, null, parentUrl);
     return exactInstall;
   }

src/install/package.ts

@@ -71,7 +71,7 @@ export async function parseUrlTarget (resolver: Resolver, targetStr: string, par
 }
 
 // ad-hoc determination of local path v remote package for eg "jspm deno react" v "jspm deno react@2" v "jspm deno ./react.ts" v "jspm deno react.ts"
-const supportedRegistries = ['npm', 'github', 'deno', 'nest'];
+const supportedRegistries = ['npm', 'github', 'deno', 'nest', 'denoland'];
 export function isPackageTarget (targetStr: string): boolean {
   if (isRelative(targetStr))
     return false;
@@ -100,7 +100,7 @@ export function pkgUrlToNiceString (resolver: Resolver, pkgUrl: string) {
   return pkgUrl;
 }
 
-export async function toPackageTarget (resolver: Resolver, targetStr: string, parentPkgUrl: string): Promise<{ alias: string, target: InstallTarget, subpath: '.' | `./${string}` }> {
+export async function toPackageTarget (resolver: Resolver, targetStr: string, parentPkgUrl: string, defaultRegistry: string): Promise<{ alias: string, target: InstallTarget, subpath: '.' | `./${string}` }> {
   const urlTarget = await parseUrlTarget(resolver, targetStr, parentPkgUrl);
   if (urlTarget)
     return urlTarget;
@@ -125,20 +125,20 @@ export async function toPackageTarget (resolver: Resolver, targetStr: string, pa
 
   return {
     alias,
-    target: newPackageTarget(pkg.pkgName, parentPkgUrl),
+    target: newPackageTarget(pkg.pkgName, parentPkgUrl, defaultRegistry),
     subpath: pkg.subpath as '.' | `./{string}`
   };
 }
 
-export function newPackageTarget (target: string, parentPkgUrl: string, depName?: string): InstallTarget {
+export function newPackageTarget (target: string, parentPkgUrl: string, defaultRegistry: string, depName?: string): InstallTarget {
   let registry: string, name: string, ranges: any[];
 
   const registryIndex = target.indexOf(':');
 
   if (target.startsWith('./') || target.startsWith('../') || target.startsWith('/') || registryIndex === 1)
     return new URL(target, parentPkgUrl);
 
-  registry = registryIndex < 1 ? 'npm' : target.substr(0, registryIndex);
+  registry = registryIndex < 1 ? defaultRegistry : target.slice(0, registryIndex);
 
   if (registry === 'file')
     return new URL(target.slice(registry.length + 1), parentPkgUrl);

src/providers/denoland.ts

@@ -4,20 +4,34 @@ import { Resolver } from "../trace/resolver.js";
 import { fetch } from '#fetch';
 
 const cdnUrl = 'https://deno.land/x/';
+const stdlibUrl = 'https://deno.land/std';
 
 export function pkgToUrl (pkg: ExactPackage) {
-  return cdnUrl + pkg.name + '@v' + pkg.version + '/';
+  if (pkg.registry === 'deno')
+    return stdlibUrl + '@' + pkg.version + '/' + pkg.name + '/';
+  if (pkg.registry === 'denoland')
+    return cdnUrl + pkg.name + '@v' + pkg.version + '/';
+  throw new Error(`Deno provider does not support the ${pkg.registry} registry`);
 }
 
 export function parseUrlPkg (url: string): ExactPackage | undefined {
-  if (!url.startsWith(cdnUrl))
-    return;
-  const path = url.slice(cdnUrl.length);
-  const versionIndex = path.indexOf('@v');
-  if (versionIndex === -1)
-    return;
-  const sepIndex = path.indexOf('/', versionIndex);
-  return { registry: 'deno', name: path.slice(0, versionIndex), version: path.slice(versionIndex + 2, sepIndex === -1 ? path.length : sepIndex) };
+  if (url.startsWith(stdlibUrl) && url[stdlibUrl.length] === '@') {
+    const version = url.slice(stdlibUrl.length + 1, url.indexOf('/', stdlibUrl.length + 1));
+    let name = url.slice(stdlibUrl.length + version.length + 2);
+    if (name.endsWith('/mod.ts'))
+      name = name.slice(0, -7);
+    else if (name.endsWith('.ts'))
+      name = name.slice(0, -3);
+    return { registry: 'deno', name, version };
+  }
+  else if (url.startsWith(cdnUrl)) {
+    const path = url.slice(cdnUrl.length);
+    const versionIndex = path.indexOf('@v');
+    if (versionIndex === -1)
+      return;
+    const sepIndex = path.indexOf('/', versionIndex);
+    return { registry: 'denoland', name: path.slice(0, versionIndex), version: path.slice(versionIndex + 2, sepIndex === -1 ? path.length : sepIndex) };
+  }
 }
 
 export async function resolveLatestTarget (this: Resolver, target: LatestPackageTarget, unstable: boolean, _layer: string, parentUrl: string): Promise<ExactPackage | null> {
@@ -29,9 +43,17 @@ export async function resolveLatestTarget (this: Resolver, target: LatestPackage
   if (!range.isWildcard)
     throw new Error(`Version ranges are not supported looking up in the Deno registry currently, until an API is available.`);
 
-  const fetchOpts = { ...this.fetchOpts, headers: Object.assign({}, this.fetchOpts.headers || {}, { 'accept': 'text/html' }) };
-  const res = await fetch(cdnUrl + name, fetchOpts);
+  const fetchOpts = {
+    ...this.fetchOpts,
+    headers: Object.assign({}, this.fetchOpts.headers || {}, {
+      // For some reason, Deno provides different redirect behaviour for the server
+      // Which requires us to use the text/html accept
+      'accept': typeof document === 'undefined' ? 'text/html' : 'text/javascript'
+    })
+  };
+  // "mod.ts" addition is necessary for the browser otherwise not resolving an exact module gives a CORS error
+  const res = await fetch((registry === 'denoland' ? cdnUrl : stdlibUrl + '/') + name + '/mod.ts', fetchOpts);
   if (!res.ok)
-    throw new Error(`Deno: Unable to lookup ${cdnUrl + name}`);
+    throw new Error(`Deno: Unable to lookup ${(registry === 'denoland' ? cdnUrl : stdlibUrl + '/') + name}`);
   return parseUrlPkg(res.url);
 }

src/providers/index.ts

@@ -1,4 +1,4 @@
-import * as deno from './denoland.js';
+import * as denoland from './denoland.js';
 import * as jspm from './jspm.js';
 import * as skypack from './skypack.js';
 import * as jsdelivr from './jsdelivr.js';
@@ -18,7 +18,7 @@ export interface Provider {
 }
 
 export const defaultProviders: Record<string, Provider> = {
-  deno,
+  denoland,
   jsdelivr,
   jspm,
   node,
@@ -33,3 +33,8 @@ export function getProvider (name: string, providers: Record<string, Provider> =
     return provider;
   throw new Error('No ' + name + ' provider is defined.');
 }
+
+export const registryProviders: Record<string, string> = {
+  'denoland:': 'denoland',
+  'deno:': 'denoland'
+};

test/providers/deno.test.js

@@ -1,13 +1,35 @@
 import { Generator } from '@jspm/generator';
 import assert from 'assert';
 
-const generator = new Generator({
-  mapUrl: new URL('../../', import.meta.url),
-  defaultProvider: 'deno'
-});
+{
+  const generator = new Generator({
+    mapUrl: new URL('../../', import.meta.url),
+    defaultRegistry: 'denoland'
+  });
 
-await generator.install('oak@10.6.0');
+  await generator.install('oak@10.6.0');
 
-const json = generator.getMap();
+  const json = generator.getMap();
 
-assert.strictEqual(json.imports['oak'], 'https://deno.land/x/oak@v10.6.0/mod.ts');
+  assert.strictEqual(json.imports['oak'], 'https://deno.land/x/oak@v10.6.0/mod.ts');
+}
+
+{
+  const generator = new Generator();
+
+  await generator.install('denoland:oak');
+
+  const json = generator.getMap();
+
+  assert.strictEqual(json.imports['oak'], 'https://deno.land/x/oak@v10.6.0/mod.ts');
+}
+
+{
+  const generator = new Generator();
+
+  await generator.install('deno:path');
+
+  const json = generator.getMap();
+
+  assert.strictEqual(json.imports['path'], 'https://deno.land/std@0.148.0/path/mod.ts');
+}