Skip to content

Commit

Permalink
Serve linked source maps from loopback server (#660)
Browse files Browse the repository at this point in the history
With cloudflare/workerd#710, `workerd` supports breakpoint debugging!
Support for this in Miniflare just worked, assuming you were using a
plain JavaScript worker, or you had inline source maps. `workerd`
doesn't know where workers are located on disk, it just knows files'
locations relative to each other. This means it's unable to resolve
locations of corresponding linked `.map` files in `sourceMappingURL`
comments.

Miniflare _does_ have this information though. This change detects
linked source maps and rewrites `sourceMappingURL` comments to `http`
URLs pointing to Miniflare's loopback server. This then looks for the
source map relative to the known on-disk source location. Source
maps' `sourceRoot` attributes are updated to ensure correct locations
are displayed in DevTools.

**This enables breakpoint debugging for compiled TypeScript with
linked source maps!** 🎉

Closes DEVX-872
  • Loading branch information
mrbbot committed Oct 31, 2023
1 parent cee744e commit fb0f54a
Show file tree
Hide file tree
Showing 7 changed files with 295 additions and 45 deletions.
1 change: 1 addition & 0 deletions packages/miniflare/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@
"@types/stoppable": "^1.1.1",
"@types/ws": "^8.5.3",
"devalue": "^4.3.0",
"devtools-protocol": "^0.0.1182435",
"semiver": "^1.1.0"
},
"engines": {
Expand Down
10 changes: 10 additions & 0 deletions packages/miniflare/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ import {
QueueConsumers,
QueuesError,
SharedOptions,
SourceMapRegistry,
WorkerOptions,
getGlobalServices,
maybeGetSitesManifestModule,
Expand Down Expand Up @@ -403,6 +404,7 @@ export class Miniflare {
#runtime?: Runtime;
#removeRuntimeExitHook?: () => void;
#runtimeEntryURL?: URL;
#sourceMapRegistry?: SourceMapRegistry;

// Path to temporary directory for use as scratch space/"in-memory" Durable
// Object storage. Note this may not exist, it's up to the consumers to
Expand Down Expand Up @@ -664,6 +666,8 @@ export class Miniflare {
this.#log.debug(`Error parsing response log: ${String(e)}`);
}
response = new Response(null, { status: 204 });
} else if (url.pathname.startsWith(SourceMapRegistry.PATHNAME_PREFIX)) {
response = await this.#sourceMapRegistry?.get(url);
} else {
// TODO: check for proxying/outbound fetch header first (with plans for fetch mocking)
response = await this.#handleLoopbackPlugins(request, url);
Expand Down Expand Up @@ -771,6 +775,7 @@ export class Miniflare {

sharedOpts.core.cf = await setupCf(this.#log, sharedOpts.core.cf);

const sourceMapRegistry = new SourceMapRegistry(this.#log, loopbackPort);
const durableObjectClassNames = getDurableObjectClassNames(allWorkerOpts);
const queueConsumers = getQueueConsumers(allWorkerOpts);
const allWorkerRoutes = getWorkerRoutes(allWorkerOpts);
Expand Down Expand Up @@ -823,6 +828,7 @@ export class Miniflare {
workerIndex: i,
additionalModules,
tmpPath: this.#tmpPath,
sourceMapRegistry,
durableObjectClassNames,
queueConsumers,
};
Expand All @@ -845,6 +851,10 @@ export class Miniflare {
}
}

// Once we've assembled the config, and are about to restart the runtime,
// update the source map registry.
this.#sourceMapRegistry = sourceMapRegistry;

return { services: Array.from(services.values()), sockets };
}

Expand Down
16 changes: 13 additions & 3 deletions packages/miniflare/src/plugins/core/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ import {
HEADER_CF_BLOB,
Plugin,
SERVICE_LOOPBACK,
SourceMapRegistry,
WORKER_BINDING_SERVICE_LOOPBACK,
parseRoutes,
} from "../shared";
Expand Down Expand Up @@ -320,9 +321,14 @@ export const CORE_PLUGIN: Plugin<
workerIndex,
durableObjectClassNames,
additionalModules,
sourceMapRegistry,
}) {
// Define regular user worker
const workerScript = getWorkerScript(options, workerIndex);
const workerScript = getWorkerScript(
sourceMapRegistry,
options,
workerIndex
);
// Add additional modules (e.g. "__STATIC_CONTENT_MANIFEST") if any
if ("modules" in workerScript) {
workerScript.modules.push(...additionalModules);
Expand Down Expand Up @@ -480,6 +486,7 @@ export function getGlobalServices({
}

function getWorkerScript(
sourceMapRegistry: SourceMapRegistry,
options: SourceOptions,
workerIndex: number
): { serviceWorkerScript: string } | { modules: Worker_Module[] } {
Expand All @@ -489,7 +496,7 @@ function getWorkerScript(
("modulesRoot" in options ? options.modulesRoot : undefined) ?? "";
return {
modules: options.modules.map((module) =>
convertModuleDefinition(modulesRoot, module)
convertModuleDefinition(sourceMapRegistry, modulesRoot, module)
),
};
}
Expand All @@ -509,7 +516,7 @@ function getWorkerScript(

if (options.modules) {
// If `modules` is `true`, automatically collect modules...
const locator = new ModuleLocator(options.modulesRules);
const locator = new ModuleLocator(sourceMapRegistry, options.modulesRules);
// If `script` and `scriptPath` are set, resolve modules in `script`
// against `scriptPath`.
locator.visitEntrypoint(
Expand All @@ -520,6 +527,9 @@ function getWorkerScript(
} else {
// ...otherwise, `modules` will either be `false` or `undefined`, so treat
// `code` as a service worker
if ("scriptPath" in options && options.scriptPath !== undefined) {
code = sourceMapRegistry.register(code, options.scriptPath);
}
return { serviceWorkerScript: code };
}
}
Expand Down
17 changes: 13 additions & 4 deletions packages/miniflare/src/plugins/core/modules.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import {
globsToRegExps,
testRegExps,
} from "../../shared";
import { SourceMapRegistry } from "../shared";

const SUGGEST_BUNDLE =
"If you're trying to import an npm package, you'll need to bundle your Worker first.";
Expand Down Expand Up @@ -122,11 +123,13 @@ function getResolveErrorPrefix(referencingPath: string): string {
}

export class ModuleLocator {
readonly #sourceMapRegistry: SourceMapRegistry;
readonly #compiledRules: CompiledModuleRule[];
readonly #visitedPaths = new Set<string>();
readonly modules: Worker_Module[] = [];

constructor(rules?: ModuleRule[]) {
constructor(sourceMapRegistry: SourceMapRegistry, rules?: ModuleRule[]) {
this.#sourceMapRegistry = sourceMapRegistry;
this.#compiledRules = compileModuleRules(rules);
}

Expand All @@ -144,6 +147,7 @@ export class ModuleLocator {
#visitJavaScriptModule(code: string, modulePath: string, esModule = true) {
// Register module
const name = path.relative("", modulePath);
code = this.#sourceMapRegistry.register(code, modulePath);
this.modules.push(
esModule ? { name, esModule: code } : { name, commonJsModule: code }
);
Expand Down Expand Up @@ -305,18 +309,23 @@ function contentsToArray(contents: string | Uint8Array): Uint8Array {
return typeof contents === "string" ? encoder.encode(contents) : contents;
}
export function convertModuleDefinition(
sourceMapRegistry: SourceMapRegistry,
modulesRoot: string,
def: ModuleDefinition
): Worker_Module {
// The runtime requires module identifiers to be relative paths
let name = path.relative(modulesRoot, def.path);
if (path.sep === "\\") name = name.replaceAll("\\", "/");
const contents = def.contents ?? readFileSync(def.path);
let contents = def.contents ?? readFileSync(def.path);
switch (def.type) {
case "ESModule":
return { name, esModule: contentsToString(contents) };
contents = contentsToString(contents);
contents = sourceMapRegistry.register(contents, def.path);
return { name, esModule: contents };
case "CommonJS":
return { name, commonJsModule: contentsToString(contents) };
contents = contentsToString(contents);
contents = sourceMapRegistry.register(contents, def.path);
return { name, commonJsModule: contents };
case "Text":
return { name, text: contentsToString(contents) };
case "Data":
Expand Down
3 changes: 3 additions & 0 deletions packages/miniflare/src/plugins/shared/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import { z } from "zod";
import { Service, Worker_Binding, Worker_Module } from "../../runtime";
import { Awaitable, Log, OptionalZodTypeOf } from "../../shared";
import { GatewayConstructor } from "./gateway";
import { SourceMapRegistry } from "./registry";
import { RouterConstructor } from "./router";

// Maps **service** names to the Durable Object class names exported by them
Expand Down Expand Up @@ -41,6 +42,7 @@ export interface PluginServicesOptions<
workerIndex: number;
additionalModules: Worker_Module[];
tmpPath: string;
sourceMapRegistry: SourceMapRegistry;

// ~~Leaky abstractions~~ "Plugin specific options" :)
durableObjectClassNames: DurableObjectClassNames;
Expand Down Expand Up @@ -91,5 +93,6 @@ export function namespaceEntries(
export * from "./constants";
export * from "./gateway";
export * from "./range";
export * from "./registry";
export * from "./router";
export * from "./routing";
105 changes: 105 additions & 0 deletions packages/miniflare/src/plugins/shared/registry.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,105 @@
import crypto from "crypto";
import fs from "fs/promises";
import path from "path";
import { fileURLToPath, pathToFileURL } from "url";
import type { RawSourceMap } from "source-map";
import { Response } from "../../http";
import { Log } from "../../shared";

function maybeParseURL(url: string): URL | undefined {
if (path.isAbsolute(url)) return;
try {
return new URL(url);
} catch {}
}

export class SourceMapRegistry {
static PATHNAME_PREFIX = "/core/source-map/";

constructor(
private readonly log: Log,
private readonly loopbackPort: number
) {}

readonly #map = new Map<string /* id */, string /* sourceMapPath */>();

register(script: string, scriptPath: string): string /* newScript */ {
// Try to find the last source mapping URL in the file, if none could be
// found, return the script as is
const mappingURLIndex = script.lastIndexOf("//# sourceMappingURL=");
if (mappingURLIndex === -1) return script;

// `pathToFileURL()` will resolve `scriptPath` relative to the current
// working directory if needed
const scriptURL = pathToFileURL(scriptPath);

const sourceSegment = script.substring(0, mappingURLIndex);
const mappingURLSegment = script
.substring(mappingURLIndex)
.replace(/^\/\/# sourceMappingURL=(.+)/, (substring, mappingURL) => {
// If the mapping URL is already a URL (e.g. `data:`), return it as is
if (maybeParseURL(mappingURL) !== undefined) return substring;

// Otherwise, resolve it relative to the script, and register it
const resolvedMappingURL = new URL(mappingURL, scriptURL);
const resolvedMappingPath = fileURLToPath(resolvedMappingURL);

// We intentionally register source maps in a map to prevent arbitrary
// file access via the loopback server.
const id = crypto.randomUUID();
this.#map.set(id, resolvedMappingPath);
mappingURL = `http://localhost:${this.loopbackPort}${SourceMapRegistry.PATHNAME_PREFIX}${id}`;

this.log.verbose(
`Registered source map ${JSON.stringify(
resolvedMappingPath
)} at ${mappingURL}`
);

return `//# sourceMappingURL=${mappingURL}`;
});

return sourceSegment + mappingURLSegment;
}

async get(url: URL): Promise<Response | undefined> {
// Try to get source map from registry
const id = url.pathname.substring(SourceMapRegistry.PATHNAME_PREFIX.length);
const sourceMapPath = this.#map.get(id);
if (sourceMapPath === undefined) return;

// Try to load and parse source map from disk
let contents: string;
try {
contents = await fs.readFile(sourceMapPath, "utf8");
} catch (e) {
this.log.warn(
`Error reading source map ${JSON.stringify(sourceMapPath)}: ${e}`
);
return;
}
let map: RawSourceMap;
try {
map = JSON.parse(contents);
} catch (e) {
this.log.warn(
`Error parsing source map ${JSON.stringify(sourceMapPath)}: ${e}`
);
return;
}

// Modify the `sourceRoot` so source files get the correct paths. Note,
// `sourceMapPath` will always be an absolute path.
const sourceMapDir = path.dirname(sourceMapPath);
map.sourceRoot =
map.sourceRoot === undefined
? sourceMapDir
: path.resolve(sourceMapDir, map.sourceRoot);

return Response.json(map, {
// This source map will be served from the loopback server to DevTools,
// which will likely be on a different origin.
headers: { "Access-Control-Allow-Origin": "*" },
});
}
}
Loading

0 comments on commit fb0f54a

Please sign in to comment.