From 1ff6034119a2c5df0c6f51752d604b0a66921b4f Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Mon, 18 Jan 2021 22:48:49 -0600 Subject: [PATCH 01/23] feat(xsnap): setImmediate and print In addition to detecting XS Machine quiescense so we can safely take snapshots, the supervisor has to detect vat queiscense so it can tell when a delivery is done. I have resorted to ad-hoc `fprintf()` at the C level for debugging enough to justify restoring print. Here we test that it's only available in the start compartment. note print() includes fflush() --- packages/xsnap/src/xsnap.c | 192 +++++++++++++++--------------- packages/xsnap/test/test-xsnap.js | 60 +++++++--- 2 files changed, 140 insertions(+), 112 deletions(-) diff --git a/packages/xsnap/src/xsnap.c b/packages/xsnap/src/xsnap.c index e778e207bff..b44834dceae 100644 --- a/packages/xsnap/src/xsnap.c +++ b/packages/xsnap/src/xsnap.c @@ -47,17 +47,17 @@ static void fx_issueCommand(xsMachine *the); static void fx_Array_prototype_meter(xsMachine* the); // extern void fx_clearTimer(txMachine* the); -// static void fx_destroyTimer(void* data); +static void fx_destroyTimer(void* data); // static void fx_evalScript(xsMachine* the); // static void fx_gc(xsMachine* the); // static void fx_isPromiseJobQueueEmpty(xsMachine* the); -// static void fx_markTimer(txMachine* the, void* it, txMarkRoot markRoot); -// static void fx_print(xsMachine* the); -// static void fx_setImmediate(txMachine* the); +static void fx_markTimer(txMachine* the, void* it, txMarkRoot markRoot); +static void fx_print(xsMachine* the); +static void fx_setImmediate(txMachine* the); // static void fx_setInterval(txMachine* the); // static void fx_setTimeout(txMachine* the); -// static void fx_setTimer(txMachine* the, txNumber interval, txBoolean repeat); -// static void fx_setTimerCallback(txJob* job); +static void fx_setTimer(txMachine* the, txNumber interval, txBoolean repeat); +static void fx_setTimerCallback(txJob* job); static void fxFulfillModuleFile(txMachine* the); static void fxRejectModuleFile(txMachine* the); @@ -73,15 +73,15 @@ static char* fxWriteNetStringError(int code); // The order of the callbacks materially affects how they are introduced to // code that runs from a snapshot, so must be consistent in the face of // upgrade. -#define mxSnapshotCallbackCount 2 +#define mxSnapshotCallbackCount 4 txCallback gxSnapshotCallbacks[mxSnapshotCallbackCount] = { fx_issueCommand, // 0 fx_Array_prototype_meter, // 1 + fx_print, // 2 + fx_setImmediate, // 3 // fx_gc, // fx_evalScript, // fx_isPromiseJobQueueEmpty, - // fx_print, - // fx_setImmediate, // fx_setInterval, // fx_setTimeout, // fx_clearTimer, @@ -467,8 +467,8 @@ void fxBuildAgent(xsMachine* the) // slot = fxNextHostFunctionProperty(the, slot, fx_evalScript, 1, xsID("evalScript"), XS_DONT_ENUM_FLAG); // slot = fxNextHostFunctionProperty(the, slot, fx_gc, 1, xsID("gc"), XS_DONT_ENUM_FLAG); // slot = fxNextHostFunctionProperty(the, slot, fx_isPromiseJobQueueEmpty, 1, xsID("isPromiseJobQueueEmpty"), XS_DONT_ENUM_FLAG); - // slot = fxNextHostFunctionProperty(the, slot, fx_print, 1, xsID("print"), XS_DONT_ENUM_FLAG); - // slot = fxNextHostFunctionProperty(the, slot, fx_setImmediate, 1, xsID("setImmediate"), XS_DONT_ENUM_FLAG); + slot = fxNextHostFunctionProperty(the, slot, fx_print, 1, xsID("print"), XS_DONT_ENUM_FLAG); + slot = fxNextHostFunctionProperty(the, slot, fx_setImmediate, 1, xsID("setImmediate"), XS_DONT_ENUM_FLAG); // slot = fxNextHostFunctionProperty(the, slot, fx_setInterval, 1, xsID("setInterval"), XS_DONT_ENUM_FLAG); // slot = fxNextHostFunctionProperty(the, slot, fx_setTimeout, 1, xsID("setTimeout"), XS_DONT_ENUM_FLAG); @@ -822,24 +822,20 @@ void fxPrintUsage() // xsResult = (the->promiseJobs) ? xsFalse : xsTrue; // } -// void fx_print(xsMachine* the) -// { -// xsIntegerValue c = xsToInteger(xsArgc), i; -// if (gxMeteringPrint) -// fprintf(stdout, "[%u] ", the->meterIndex); -// for (i = 0; i < c; i++) { -// if (i) -// fprintf(stdout, " "); -// fprintf(stdout, "%s", xsToString(xsArg(i))); -// } -// fprintf(stdout, "\n"); -// } +void fx_print(xsMachine* the) +{ + xsIntegerValue c = xsToInteger(xsArgc), i; + if (gxMeteringPrint) + fprintf(stdout, "[%u] ", the->meterIndex); + for (i = 0; i < c; i++) { + if (i) + fprintf(stdout, " "); + fprintf(stdout, "%s", xsToString(xsArg(i))); + } + fprintf(stdout, "\n"); + fflush(stdout); +} -// void fx_setImmediate(txMachine* the) -// { -// fx_setTimer(the, 0, 0); -// } -// // void fx_setInterval(txMachine* the) // { // fx_setTimer(the, fxToNumber(the, mxArgv(1)), 1); @@ -849,11 +845,11 @@ void fxPrintUsage() // { // fx_setTimer(the, fxToNumber(the, mxArgv(1)), 0); // } +void fx_setImmediate(txMachine* the) +{ + fx_setTimer(the, 0, 0); +} -// static txHostHooks gxTimerHooks = { -// fx_destroyTimer, -// fx_markTimer -// }; // void fx_clearTimer(txMachine* the) // { @@ -874,67 +870,73 @@ void fxPrintUsage() // } // } -// void fx_destroyTimer(void* data) -// { -// } +static txHostHooks gxTimerHooks = { + fx_destroyTimer, + fx_markTimer +}; -// void fx_markTimer(txMachine* the, void* it, txMarkRoot markRoot) -// { -// txJob* job = it; -// if (job) { -// (*markRoot)(the, &job->function); -// (*markRoot)(the, &job->argument); -// } -// } -// void fx_setTimer(txMachine* the, txNumber interval, txBoolean repeat) -// { -// c_timeval tv; -// txJob* job; -// txJob** address = (txJob**)&(the->timerJobs); -// while ((job = *address)) -// address = &(job->next); -// job = *address = malloc(sizeof(txJob)); -// c_memset(job, 0, sizeof(txJob)); -// job->the = the; -// job->callback = fx_setTimerCallback; -// c_gettimeofday(&tv, NULL); -// if (repeat) -// job->interval = interval; -// job->when = ((txNumber)(tv.tv_sec) * 1000.0) + ((txNumber)(tv.tv_usec) / 1000.0) + interval; -// job->function.kind = mxArgv(0)->kind; -// job->function.value = mxArgv(0)->value; -// if (mxArgc > 2) { -// job->argument.kind = mxArgv(2)->kind; -// job->argument.value = mxArgv(2)->value; -// } -// fxNewHostObject(the, C_NULL); -// fxSetHostData(the, the->stack, job); -// fxSetHostHooks(the, the->stack, &gxTimerHooks); -// mxPullSlot(mxResult); -// } +void fx_destroyTimer(void* data) +{ +} -// void fx_setTimerCallback(txJob* job) -// { -// txMachine* the = job->the; -// fxBeginHost(the); -// { -// mxTry(the) { -// /* THIS */ -// mxPushUndefined(); -// /* FUNCTION */ -// mxPush(job->function); -// mxCall(); -// mxPush(job->argument); -// /* ARGC */ -// mxRunCount(1); -// mxPop(); -// } -// mxCatch(the) { -// } -// } -// fxEndHost(the); -// } +void fx_markTimer(txMachine* the, void* it, txMarkRoot markRoot) +{ + txJob* job = it; + if (job) { + (*markRoot)(the, &job->function); + (*markRoot)(the, &job->argument); + } +} + +void fx_setTimer(txMachine* the, txNumber interval, txBoolean repeat) +{ + c_timeval tv; + txJob* job; + txJob** address = (txJob**)&(the->timerJobs); + while ((job = *address)) + address = &(job->next); + job = *address = malloc(sizeof(txJob)); + c_memset(job, 0, sizeof(txJob)); + job->the = the; + job->callback = fx_setTimerCallback; + c_gettimeofday(&tv, NULL); + if (repeat) + job->interval = interval; + job->when = ((txNumber)(tv.tv_sec) * 1000.0) + ((txNumber)(tv.tv_usec) / 1000.0) + interval; + job->function.kind = mxArgv(0)->kind; + job->function.value = mxArgv(0)->value; + if (mxArgc > 2) { + job->argument.kind = mxArgv(2)->kind; + job->argument.value = mxArgv(2)->value; + } + fxNewHostObject(the, C_NULL); + fxSetHostData(the, the->stack, job); + fxSetHostHooks(the, the->stack, &gxTimerHooks); + mxPullSlot(mxResult); +} + +void fx_setTimerCallback(txJob* job) +{ + txMachine* the = job->the; + fxBeginHost(the); + { + mxTry(the) { + /* THIS */ + mxPushUndefined(); + /* FUNCTION */ + mxPush(job->function); + mxCall(); + mxPush(job->argument); + /* ARGC */ + mxRunCount(1); + mxPop(); + } + mxCatch(the) { + } + } + fxEndHost(the); +} /* PLATFORM */ @@ -965,11 +967,11 @@ void fxDeleteMachinePlatform(txMachine* the) void fxMarkHost(txMachine* the, txMarkRoot markRoot) { -// txJob* job = the->timerJobs; -// while (job) { -// fx_markTimer(the, job, markRoot); -// job = job->next; -// } + txJob* job = the->timerJobs; + while (job) { + fx_markTimer(the, job, markRoot); + job = job->next; + } } void fxQueuePromiseJobs(txMachine* the) diff --git a/packages/xsnap/test/test-xsnap.js b/packages/xsnap/test/test-xsnap.js index a8657efdf8b..a56ef0a85f6 100644 --- a/packages/xsnap/test/test-xsnap.js +++ b/packages/xsnap/test/test-xsnap.js @@ -17,44 +17,70 @@ const xsnapOptions = { debug: true }; -test('evaluate and issueCommand', async t => { +function options() { const messages = []; async function handleCommand(message) { messages.push(decoder.decode(message)); return new Uint8Array(); } - const vat = xsnap({ ...xsnapOptions, handleCommand }); + return { ...xsnapOptions, handleCommand, messages }; +} + +test('evaluate and issueCommand', async t => { + const opts = options(); + const vat = xsnap(opts); await vat.evaluate(`issueCommand(ArrayBuffer.fromString("Hello, World!"));`); await vat.close(); - t.deepEqual(['Hello, World!'], messages); + t.deepEqual(['Hello, World!'], opts.messages); }); test('evaluate until idle', async t => { - const messages = []; - async function handleCommand(message) { - messages.push(decoder.decode(message)); - return new Uint8Array(); - } - const vat = xsnap({ ...xsnapOptions, handleCommand }); + const opts = options(); + const vat = xsnap(opts); await vat.evaluate(` (async () => { issueCommand(ArrayBuffer.fromString("Hello, World!")); })(); `); await vat.close(); - t.deepEqual(['Hello, World!'], messages); + t.deepEqual(['Hello, World!'], opts.messages); +}); + +test('idle includes setImmediate too', async t => { + const opts = options(); + const vat = xsnap(opts); + await vat.evaluate(` + const send = it => issueCommand(ArrayBuffer.fromString(it)); + setImmediate(() => send("end of crank")); + Promise.resolve("turn 2").then(send); + send("turn 1"); + `); + await vat.close(); + t.deepEqual(['turn 1', 'turn 2', 'end of crank'], opts.messages); +}); + +test('print - start compartment only', async t => { + const opts = options(); + const vat = xsnap(opts); + await vat.evaluate(` + const send = it => issueCommand(ArrayBuffer.fromString(it)); + print(123); + try { + (new Compartment()).evalate('print("456")'); + } catch (_err) { + send('no print in Compartment'); + } + `); + await vat.close(); + t.deepEqual(['no print in Compartment'], opts.messages); }); test('run script until idle', async t => { - const messages = []; - async function handleCommand(message) { - messages.push(decoder.decode(message)); - return new Uint8Array(); - } - const vat = xsnap({ ...xsnapOptions, handleCommand }); + const opts = options(); + const vat = xsnap(opts); await vat.execute(new URL('fixture-xsnap-script.js', importMetaUrl).pathname); await vat.close(); - t.deepEqual(['Hello, World!'], messages); + t.deepEqual(['Hello, World!'], opts.messages); }); test('issueCommand is synchronous inside, async outside', async t => { From 48d9b8b2403803cf0b86d9132fa14b829cecb9dd Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Wed, 20 Jan 2021 17:53:14 -0600 Subject: [PATCH 02/23] build(xsnap): don't set mxDebug in release builds fixes #2216 only tested on lin, not mac nor win --- packages/xsnap/makefiles/lin/xsnap.mk | 3 +-- packages/xsnap/makefiles/mac/xsnap.mk | 3 +-- packages/xsnap/makefiles/win/xsnap.mak | 2 +- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/packages/xsnap/makefiles/lin/xsnap.mk b/packages/xsnap/makefiles/lin/xsnap.mk index ad3d20f580e..249fa8c8a76 100644 --- a/packages/xsnap/makefiles/lin/xsnap.mk +++ b/packages/xsnap/makefiles/lin/xsnap.mk @@ -25,7 +25,6 @@ C_OPTIONS = \ -fno-common \ -DINCLUDE_XSPLATFORM \ -DXSPLATFORM=\"xsnap.h\" \ - -DmxDebug=1 \ -DmxMetering=1 \ -DmxParse=1 \ -DmxRun=1 \ @@ -41,7 +40,7 @@ C_OPTIONS += \ -Wno-misleading-indentation \ -Wno-implicit-fallthrough ifeq ($(GOAL),debug) - C_OPTIONS += -g -O0 -Wall -Wextra -Wno-missing-field-initializers -Wno-unused-parameter + C_OPTIONS += -g -O0 -Wall -Wextra -Wno-missing-field-initializers -Wno-unused-parameter -DmxDebug=1 else C_OPTIONS += -O3 endif diff --git a/packages/xsnap/makefiles/mac/xsnap.mk b/packages/xsnap/makefiles/mac/xsnap.mk index bb77735d16f..ce6bbfa2739 100644 --- a/packages/xsnap/makefiles/mac/xsnap.mk +++ b/packages/xsnap/makefiles/mac/xsnap.mk @@ -28,7 +28,6 @@ C_OPTIONS = \ $(MACOS_VERSION_MIN) \ -DINCLUDE_XSPLATFORM \ -DXSPLATFORM=\"xsnap.h\" \ - -DmxDebug=1 \ -DmxMetering=1 \ -DmxParse=1 \ -DmxRun=1 \ @@ -44,7 +43,7 @@ ifneq ("x$(SDKROOT)", "x") C_OPTIONS += -isysroot $(SDKROOT) endif ifeq ($(GOAL),debug) - C_OPTIONS += -g -O0 -Wall -Wextra -Wno-missing-field-initializers -Wno-unused-parameter + C_OPTIONS += -g -O0 -Wall -Wextra -Wno-missing-field-initializers -Wno-unused-parameter -DmxDebug=1 else C_OPTIONS += -O3 endif diff --git a/packages/xsnap/makefiles/win/xsnap.mak b/packages/xsnap/makefiles/win/xsnap.mak index f3de5faafa9..9907f179d0e 100644 --- a/packages/xsnap/makefiles/win/xsnap.mak +++ b/packages/xsnap/makefiles/win/xsnap.mak @@ -20,7 +20,6 @@ C_OPTIONS = \ /D _CRT_SECURE_NO_DEPRECATE \ /D INCLUDE_XSPLATFORM \ /D XSPLATFORM=\"xsnap.h\" \ - /D mxDebug=1 \ /D mxMetering=1 \ /D mxParse=1 \ /D mxRun=1 \ @@ -35,6 +34,7 @@ C_OPTIONS = \ !IF "$(GOAL)"=="debug" C_OPTIONS = $(C_OPTIONS) \ /D _DEBUG \ + /D mxDebug=1 \ /Fp$(TMP_DIR_DBG)\ \ /Od \ /W3 \ From 6aefa2fa7753e7e41d4208cff3e1dc4e0175c8b6 Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Fri, 22 Jan 2021 15:02:34 -0600 Subject: [PATCH 03/23] build(xsnap): build GOAL=debug too --- packages/xsnap/src/build.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/xsnap/src/build.js b/packages/xsnap/src/build.js index 81a51920265..ab4b9867111 100644 --- a/packages/xsnap/src/build.js +++ b/packages/xsnap/src/build.js @@ -1,7 +1,7 @@ import * as childProcess from 'child_process'; import os from 'os'; -function exec(command, cwd) { +function exec(command, cwd, args=[]) { const child = childProcess.spawn(command, { cwd, stdio: ['inherit', 'inherit', 'inherit'], @@ -25,10 +25,13 @@ function exec(command, cwd) { // Run command depending on the OS if (os.type() === 'Linux') { await exec('make', 'makefiles/lin'); + await exec('make', 'makefiles/lin', ['GOAL=debug']); } else if (os.type() === 'Darwin') { await exec('make', 'makefiles/mac'); + await exec('make', 'makefiles/mac', ['GOAL=debug']); } else if (os.type() === 'Windows_NT') { await exec('nmake', 'makefiles/win'); + await exec('make', 'makefiles/win', ['GOAL=debug']); } else { throw new Error(`Unsupported OS found: ${os.type()}`); } From 7caa380be724eebdb9d98b2f6402c09c0bf6c0b7 Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Fri, 22 Jan 2021 15:08:04 -0600 Subject: [PATCH 04/23] fix(xsnap): don't swallow error message --- packages/xsnap/src/xsnap.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/xsnap/src/xsnap.js b/packages/xsnap/src/xsnap.js index c307508a8a3..a48bc987fc6 100644 --- a/packages/xsnap/src/xsnap.js +++ b/packages/xsnap/src/xsnap.js @@ -123,7 +123,11 @@ export function xsnap(options) { } else if (message[0] === OK) { return message.subarray(1); } else if (message[0] === ERROR) { - throw new Error(`Uncaught exception in ${name}`); + throw new Error( + `Uncaught exception in ${name}: ${decoder.decode( + message.subarray(1), + )}`, + ); } else if (message[0] === QUERY) { await messagesToXsnap.next(await handleCommand(message.subarray(1))); } From 19ad4528d2749c1968130537ff5e8652429b6d31 Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Fri, 22 Jan 2021 15:08:46 -0600 Subject: [PATCH 05/23] feat(xsnap): return data from xsnap.evaluate() Using the .result property of a mutable object rather than the resolution of a promise is a little awkward, but it seems to work. --- packages/xsnap/src/xsnap.c | 67 ++++++++++++++++++------------- packages/xsnap/src/xsnap.js | 6 +-- packages/xsnap/test/test-xsnap.js | 29 ++++++++++++- 3 files changed, 70 insertions(+), 32 deletions(-) diff --git a/packages/xsnap/src/xsnap.c b/packages/xsnap/src/xsnap.c index b44834dceae..d35a406dda2 100644 --- a/packages/xsnap/src/xsnap.c +++ b/packages/xsnap/src/xsnap.c @@ -317,52 +317,63 @@ int main(int argc, char* argv[]) case '?': case 'e': error = 0; - char* response = NULL; - size_t responseLength = 0; + xsSlot report; xsBeginHost(machine); { - xsVars(3); + xsVars(1); xsTry { if (command == '?') { - xsVar(1) = xsArrayBuffer(nsbuf + 1, nslen - 1); - xsVar(2) = xsCall1(xsGlobal, xsID("handleCommand"), xsVar(1)); - if (xsTypeOf(xsVar(2)) != xsUndefinedType) { - responseLength = fxGetArrayBufferLength(machine, &xsVar(2)); - response = malloc(responseLength); - fxGetArrayBufferData(machine, &xsVar(2), 0, response, responseLength); - } + xsVar(0) = xsArrayBuffer(nsbuf + 1, nslen - 1); + report = xsCall1(xsGlobal, xsID("handleCommand"), xsVar(0)); } else { - xsVar(1) = xsStringBuffer(nsbuf + 1, nslen - 1); - xsCall1_noResult(xsGlobal, xsID("eval"), xsVar(1)); + xsVar(0) = xsStringBuffer(nsbuf + 1, nslen - 1); + report = xsCall1(xsGlobal, xsID("eval"), xsVar(0)); } + xsRemember(report); } xsCatch { if (xsTypeOf(xsException) != xsUndefinedType) { - fprintf(stderr, "%s\n", xsToString(xsException)); + // fprintf(stderr, "%c: %s\n", command, xsToString(xsException)); error = 1; + report = xsException; + xsRemember(report); xsException = xsUndefined; } } } xsEndHost(machine); fxRunLoop(machine); - if (error == 0) { - int writeError = fxWriteNetString(toParent, '.', response, responseLength); - if (writeError != 0) { - fprintf(stderr, "%s\n", fxWriteNetStringError(writeError)); - c_exit(1); - } - } else { - // TODO: dynamically build error message including Exception message. - int writeError = fxWriteNetString(toParent, '!', "", 0); - if (writeError != 0) { - fprintf(stderr, "%s\n", fxWriteNetStringError(writeError)); - c_exit(1); + int writeError; + xsBeginHost(machine); + { + if (error) { + xsStringValue message = xsToString(report); + writeError = fxWriteNetString(toParent, '!', message, strlen(message)); + // fprintf(stderr, "error: %d, writeError: %d %s\n", error, writeError, message); + } else { + char* response = NULL; + txInteger responseLength = 0; + // fprintf(stderr, "report: %d %s\n", xsTypeOf(report), xsToString(report)); + xsSlot result; + if (command == 'e' && xsTypeOf(report) != xsUndefinedType) { + result = xsGet(report, xsID("result")); + } else { + result = report; + } + // fprintf(stderr, "result: %d %s\n", xsTypeOf(result), xsToString(result)); + if (xsTypeOf(result) != xsUndefinedType) { + response = xsToArrayBuffer(result); // TODO: catch exceptions from this? + responseLength = xsGetArrayBufferLength(result); + } + // fprintf(stderr, "response of %d bytes\n", responseLength); + writeError = fxWriteNetString(toParent, '.', response, responseLength); } + xsForget(report); } - if (response != NULL) { - free(response); - response = NULL; + xsEndHost(machine); + if (writeError != 0) { + fprintf(stderr, "%s\n", fxWriteNetStringError(writeError)); + c_exit(1); } break; case 's': diff --git a/packages/xsnap/src/xsnap.js b/packages/xsnap/src/xsnap.js index a48bc987fc6..f5660d59e2d 100644 --- a/packages/xsnap/src/xsnap.js +++ b/packages/xsnap/src/xsnap.js @@ -136,14 +136,14 @@ export function xsnap(options) { /** * @param {string} code - * @returns {Promise} + * @returns {Promise} */ async function evaluate(code) { const result = baton.then(async () => { await messagesToXsnap.next(encoder.encode(`e${code}`)); - await runToIdle(); + return runToIdle(); }); - baton = result.catch(() => {}); + baton = result.then(_ => undefined).catch(() => {}); return Promise.race([vatCancelled, result]); } diff --git a/packages/xsnap/test/test-xsnap.js b/packages/xsnap/test/test-xsnap.js index a56ef0a85f6..1873224833a 100644 --- a/packages/xsnap/test/test-xsnap.js +++ b/packages/xsnap/test/test-xsnap.js @@ -14,7 +14,6 @@ const xsnapOptions = { os: os.type(), stderr: 'inherit', stdout: 'inherit', - debug: true }; function options() { @@ -46,6 +45,34 @@ test('evaluate until idle', async t => { t.deepEqual(['Hello, World!'], opts.messages); }); +test('evaluate and report', async t => { + const opts = options(); + const vat = xsnap(opts); + const result = await vat.evaluate(`(() => { + const report = {}; + Promise.resolve('hi').then(v => { + report.result = ArrayBuffer.fromString(v); + }); + return report; + })()`); + await vat.close(); + t.deepEqual('hi', decoder.decode(result)); +}); + +test('evaluate error', async t => { + const opts = options(); + const vat = xsnap(opts); + await vat + .evaluate(`***`) + .then(_ => { + t.fail('should throw'); + }) + .catch(_ => { + t.pass(); + }); + await vat.terminate(); +}); + test('idle includes setImmediate too', async t => { const opts = options(); const vat = xsnap(opts); From 7119d81f7d1120343d85a18a0dfb5dca06a377a2 Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Sat, 16 Jan 2021 12:10:50 -0600 Subject: [PATCH 06/23] chore(xs-vat-worker): prune obsolete dependencies --- packages/xs-vat-worker/package.json | 3 --- 1 file changed, 3 deletions(-) diff --git a/packages/xs-vat-worker/package.json b/packages/xs-vat-worker/package.json index 536ff035268..d00293ea5a9 100644 --- a/packages/xs-vat-worker/package.json +++ b/packages/xs-vat-worker/package.json @@ -28,7 +28,6 @@ "@rollup/plugin-commonjs": "^11.0.2", "@rollup/plugin-node-resolve": "^7.1.1", "ava": "^3.12.1", - "detective-es6": "^2.2.0", "eslint": "^6.8.0", "eslint-config-airbnb-base": "^14.0.0", "eslint-config-jessie": "^0.0.4", @@ -36,7 +35,6 @@ "eslint-plugin-import": "^2.20.0", "eslint-plugin-jsx-a11y": "^6.2.3", "eslint-plugin-prettier": "^3.1.2", - "filing-cabinet": "^2.5.1", "nyc": "^15.1.0", "prettier": "^1.18.2", "rollup": "^1.23.1" @@ -46,7 +44,6 @@ "@agoric/bundle-source": "^1.2.0", "@agoric/eventual-send": "^0.13.0", "@agoric/import-bundle": "^0.2.0", - "@agoric/install-ses": "^0.5.0", "@agoric/marshal": "^0.3.0", "@agoric/promise-kit": "^0.2.0", "@agoric/swingset-vat": "^0.11.0", From 98d0b2cb22ad25ec40a71c9e8330cc394d63d05f Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Sat, 16 Jan 2021 12:08:03 -0600 Subject: [PATCH 07/23] build(xs-vat-worker): moddable submodule is obsolete --- .gitmodules | 4 ---- packages/xs-vat-worker/moddable | 1 - 2 files changed, 5 deletions(-) delete mode 160000 packages/xs-vat-worker/moddable diff --git a/.gitmodules b/.gitmodules index bcc651dbf29..676c50053a5 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,7 +1,3 @@ -[submodule "packages/xs-vat-worker/moddable"] - path = packages/xs-vat-worker/moddable - url = https://github.com/agoric-labs/moddable.git - branch = ag-linux-cli [submodule "packages/xsnap/moddable"] path = packages/xsnap/moddable url = https://github.com/Moddable-OpenSource/moddable.git diff --git a/packages/xs-vat-worker/moddable b/packages/xs-vat-worker/moddable deleted file mode 160000 index 4dcdcffd92d..00000000000 --- a/packages/xs-vat-worker/moddable +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 4dcdcffd92de733d40f91ad34c0c539ec98d331c From 9ba6e6563750d7cafbfafc83786e0714a9614abc Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Sat, 16 Jan 2021 12:11:55 -0600 Subject: [PATCH 08/23] style(xs-vat-worker): use canonical @agoric style --- packages/xs-vat-worker/.eslintignore | 9 ------ packages/xs-vat-worker/.eslintrc.js | 28 ------------------- packages/xs-vat-worker/src/locate.js | 1 + packages/xs-vat-worker/src/lockdown-shim.js | 2 -- .../xs-vat-worker/test/escapeCompartment.js | 2 +- 5 files changed, 2 insertions(+), 40 deletions(-) delete mode 100644 packages/xs-vat-worker/.eslintignore delete mode 100644 packages/xs-vat-worker/.eslintrc.js diff --git a/packages/xs-vat-worker/.eslintignore b/packages/xs-vat-worker/.eslintignore deleted file mode 100644 index 5c0aef66585..00000000000 --- a/packages/xs-vat-worker/.eslintignore +++ /dev/null @@ -1,9 +0,0 @@ -/dist/ -/scripts/ -/xs_modules/ -/swingset/ -/moddable/ -build/ -bundle-*.js -test/bug1/ -src-native/ diff --git a/packages/xs-vat-worker/.eslintrc.js b/packages/xs-vat-worker/.eslintrc.js deleted file mode 100644 index 23d02400696..00000000000 --- a/packages/xs-vat-worker/.eslintrc.js +++ /dev/null @@ -1,28 +0,0 @@ -/* global module */ -module.exports = { - // parser: "babel-eslint", - extends: ['airbnb-base', 'plugin:prettier/recommended'], - env: { - es6: true, // supports new ES6 globals (e.g., new types such as Set) - }, - globals: { - "harden": "readonly", - "globalThis": "writeable", - }, - rules: { - 'implicit-arrow-linebreak': 'off', - 'function-paren-newline': 'off', - 'arrow-parens': 'off', - strict: 'off', - 'prefer-destructuring': 'off', - 'no-else-return': 'off', - 'no-console': 'off', - 'no-unused-vars': ['error', { argsIgnorePattern: '^_' }], - 'no-return-assign': 'off', - 'no-param-reassign': 'off', - 'no-restricted-syntax': ['off', 'ForOfStatement'], - 'no-unused-expressions': 'off', - 'no-loop-func': 'off', - 'import/prefer-default-export': 'off', // contrary to Agoric standard - }, -}; diff --git a/packages/xs-vat-worker/src/locate.js b/packages/xs-vat-worker/src/locate.js index 4f946231069..41cd6a0549b 100644 --- a/packages/xs-vat-worker/src/locate.js +++ b/packages/xs-vat-worker/src/locate.js @@ -2,6 +2,7 @@ * Locate the XS vat worker executable. * * Note: executable is not built by default. + * * @see the `build:xs-lin` script in package.json * * @param {{ resolve: (...string) => string }} filesystem path access diff --git a/packages/xs-vat-worker/src/lockdown-shim.js b/packages/xs-vat-worker/src/lockdown-shim.js index 79a7e30f6d8..a67d383cbeb 100644 --- a/packages/xs-vat-worker/src/lockdown-shim.js +++ b/packages/xs-vat-worker/src/lockdown-shim.js @@ -1,5 +1,3 @@ -/* global lockdown */ - import 'ses/lockdown'; lockdown(); diff --git a/packages/xs-vat-worker/test/escapeCompartment.js b/packages/xs-vat-worker/test/escapeCompartment.js index 9c64ac68e6c..3b77f1a7403 100644 --- a/packages/xs-vat-worker/test/escapeCompartment.js +++ b/packages/xs-vat-worker/test/escapeCompartment.js @@ -1,4 +1,4 @@ -/* global issueCommand, Compartment */ +/* global issueCommand */ const encoder = new TextEncoder(); function send(s) { From 51a30cd832b620a0e5a16d00dd1363a102529a28 Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Sun, 17 Jan 2021 22:21:20 -0600 Subject: [PATCH 09/23] feat(xs-vat-worker): TextDecoder, HandledPromise before lockdown --- packages/xs-vat-worker/src/bootstrap.js | 1 + packages/xs-vat-worker/src/text-shim.js | 9 +++++++++ packages/xs-vat-worker/test/test-boot-lockdown.js | 4 ++-- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/packages/xs-vat-worker/src/bootstrap.js b/packages/xs-vat-worker/src/bootstrap.js index cd615238ba9..57b8feb1712 100644 --- a/packages/xs-vat-worker/src/bootstrap.js +++ b/packages/xs-vat-worker/src/bootstrap.js @@ -1,5 +1,6 @@ import './console-shim'; import './text-shim'; import './lockdown-shim'; +import '@agoric/eventual-send/shim'; harden(console); diff --git a/packages/xs-vat-worker/src/text-shim.js b/packages/xs-vat-worker/src/text-shim.js index 1e8cff5ea1c..75f7ea42629 100644 --- a/packages/xs-vat-worker/src/text-shim.js +++ b/packages/xs-vat-worker/src/text-shim.js @@ -1,7 +1,9 @@ +/* eslint-disable max-classes-per-file */ /* eslint-disable class-methods-use-this */ // Save this XS extension before SES shim deletes it. const { fromString } = ArrayBuffer; +const { fromArrayBuffer } = String; class TextEncoder { encode(s) { @@ -9,4 +11,11 @@ class TextEncoder { } } +class TextDecoder { + decode(bs) { + return fromArrayBuffer(bs); + } +} + globalThis.TextEncoder = TextEncoder; +globalThis.TextDecoder = TextDecoder; diff --git a/packages/xs-vat-worker/test/test-boot-lockdown.js b/packages/xs-vat-worker/test/test-boot-lockdown.js index 85b295d2678..9e9b1ac5393 100644 --- a/packages/xs-vat-worker/test/test-boot-lockdown.js +++ b/packages/xs-vat-worker/test/test-boot-lockdown.js @@ -37,10 +37,10 @@ test('bootstrap to SES lockdown', async t => { globalThis.send = msg => issueCommand(encoder.encode(JSON.stringify(msg)).buffer); `); await vat.evaluate(` - send([ typeof harden, typeof Compartment ]); + send([ typeof harden, typeof Compartment, typeof HandledPromise ]); `); await vat.close(); - t.deepEqual(['["function","function"]'], messages); + t.deepEqual(['["function","function","function"]'], messages); }); test('child compartment cannot access start powers', async t => { From 69d08c75292fd48e52ba8d661ba15f38f66ec300 Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Fri, 15 Jan 2021 13:23:17 -0600 Subject: [PATCH 10/23] refactor(SwingSet): unify vat-worker filenames fixes #2202 --- packages/SwingSet/src/controller.js | 4 ++-- packages/SwingSet/src/kernel/vatManager/factory.js | 6 +++--- .../vatManager/{localVatManager.js => manager-local.js} | 0 .../vatManager/{nodeWorker.js => manager-nodeworker.js} | 0 ...worker-subprocess-node.js => manager-subprocess-node.js} | 0 ...eWorkerSupervisorCJS.js => supervisor-nodeworker-cjs.js} | 2 +- .../{nodeWorkerSupervisor.js => supervisor-nodeworker.js} | 0 ...ubprocessSupervisor.js => supervisor-subprocess-node.js} | 0 8 files changed, 6 insertions(+), 6 deletions(-) rename packages/SwingSet/src/kernel/vatManager/{localVatManager.js => manager-local.js} (100%) rename packages/SwingSet/src/kernel/vatManager/{nodeWorker.js => manager-nodeworker.js} (100%) rename packages/SwingSet/src/kernel/vatManager/{worker-subprocess-node.js => manager-subprocess-node.js} (100%) rename packages/SwingSet/src/kernel/vatManager/{nodeWorkerSupervisorCJS.js => supervisor-nodeworker-cjs.js} (92%) rename packages/SwingSet/src/kernel/vatManager/{nodeWorkerSupervisor.js => supervisor-nodeworker.js} (100%) rename packages/SwingSet/src/kernel/vatManager/{subprocessSupervisor.js => supervisor-subprocess-node.js} (100%) diff --git a/packages/SwingSet/src/controller.js b/packages/SwingSet/src/controller.js index 68babfbd1e3..d5e55b155b2 100644 --- a/packages/SwingSet/src/controller.js +++ b/packages/SwingSet/src/controller.js @@ -144,7 +144,7 @@ export async function makeSwingsetController( // TODO: after we move away from `-r esm` and use real ES6 modules, point // this at nodeWorkerSupervisor.js instead of the CJS intermediate const supercode = require.resolve( - './kernel/vatManager/nodeWorkerSupervisorCJS.js', + './kernel/vatManager/supervisor-nodeworker-cjs.js', ); return new Worker(supercode); } @@ -152,7 +152,7 @@ export async function makeSwingsetController( // launch a worker in a subprocess (which runs Node.js) function startSubprocessWorkerNode() { const supercode = require.resolve( - './kernel/vatManager/subprocessSupervisor.js', + './kernel/vatManager/supervisor-subprocess-node.js', ); return startSubprocessWorker(process.execPath, ['-r', 'esm', supercode]); } diff --git a/packages/SwingSet/src/kernel/vatManager/factory.js b/packages/SwingSet/src/kernel/vatManager/factory.js index 703797fa22b..be1cf384ecd 100644 --- a/packages/SwingSet/src/kernel/vatManager/factory.js +++ b/packages/SwingSet/src/kernel/vatManager/factory.js @@ -1,8 +1,8 @@ import { assert } from '@agoric/assert'; import { assertKnownOptions } from '../../assertOptions'; -import { makeLocalVatManagerFactory } from './localVatManager'; -import { makeNodeWorkerVatManagerFactory } from './nodeWorker'; -import { makeNodeSubprocessFactory } from './worker-subprocess-node'; +import { makeLocalVatManagerFactory } from './manager-local'; +import { makeNodeWorkerVatManagerFactory } from './manager-nodeworker'; +import { makeNodeSubprocessFactory } from './manager-subprocess-node'; export function makeVatManagerFactory({ allVatPowers, diff --git a/packages/SwingSet/src/kernel/vatManager/localVatManager.js b/packages/SwingSet/src/kernel/vatManager/manager-local.js similarity index 100% rename from packages/SwingSet/src/kernel/vatManager/localVatManager.js rename to packages/SwingSet/src/kernel/vatManager/manager-local.js diff --git a/packages/SwingSet/src/kernel/vatManager/nodeWorker.js b/packages/SwingSet/src/kernel/vatManager/manager-nodeworker.js similarity index 100% rename from packages/SwingSet/src/kernel/vatManager/nodeWorker.js rename to packages/SwingSet/src/kernel/vatManager/manager-nodeworker.js diff --git a/packages/SwingSet/src/kernel/vatManager/worker-subprocess-node.js b/packages/SwingSet/src/kernel/vatManager/manager-subprocess-node.js similarity index 100% rename from packages/SwingSet/src/kernel/vatManager/worker-subprocess-node.js rename to packages/SwingSet/src/kernel/vatManager/manager-subprocess-node.js diff --git a/packages/SwingSet/src/kernel/vatManager/nodeWorkerSupervisorCJS.js b/packages/SwingSet/src/kernel/vatManager/supervisor-nodeworker-cjs.js similarity index 92% rename from packages/SwingSet/src/kernel/vatManager/nodeWorkerSupervisorCJS.js rename to packages/SwingSet/src/kernel/vatManager/supervisor-nodeworker-cjs.js index 88b2006efd4..76ac7f1dc70 100644 --- a/packages/SwingSet/src/kernel/vatManager/nodeWorkerSupervisorCJS.js +++ b/packages/SwingSet/src/kernel/vatManager/supervisor-nodeworker-cjs.js @@ -9,4 +9,4 @@ // eslint-disable-next-line no-global-assign require = require('esm')(module); -module.exports = require('./nodeWorkerSupervisor'); +module.exports = require('./supervisor-nodeworker'); diff --git a/packages/SwingSet/src/kernel/vatManager/nodeWorkerSupervisor.js b/packages/SwingSet/src/kernel/vatManager/supervisor-nodeworker.js similarity index 100% rename from packages/SwingSet/src/kernel/vatManager/nodeWorkerSupervisor.js rename to packages/SwingSet/src/kernel/vatManager/supervisor-nodeworker.js diff --git a/packages/SwingSet/src/kernel/vatManager/subprocessSupervisor.js b/packages/SwingSet/src/kernel/vatManager/supervisor-subprocess-node.js similarity index 100% rename from packages/SwingSet/src/kernel/vatManager/subprocessSupervisor.js rename to packages/SwingSet/src/kernel/vatManager/supervisor-subprocess-node.js From 961f0b5b730a20e5d2f6a5c72b047b8d2ec30888 Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Sat, 16 Jan 2021 21:55:21 -0600 Subject: [PATCH 11/23] feat(swingset): xsnap vat manager - build xsnap bootstrap bundles - bytes to tagged array and back - setBundle, importBundle - syscall - delivery success symbol is ok, not deliverDone - Use Tagged type consistently; don't constrain tag to be string. - clean up logging: use parentLog(), trace(), ... - static typing for doProcess: capture dispatch while it's known to be not null - silence parentLog, workerLog for xsnap - no, handleSyscall doesn't return Tagged - inherit stdout, stderr in xsnap - vatid arg on doNotify is no more --- packages/SwingSet/package.json | 1 + packages/SwingSet/src/controller.js | 42 +++- packages/SwingSet/src/kernel/kernel.js | 4 +- .../SwingSet/src/kernel/vatManager/factory.js | 10 +- .../vatManager/lockdown-subprocess-xsnap.js | 1 + .../vatManager/manager-subprocess-xsnap.js | 186 ++++++++++++++ .../vatManager/supervisor-subprocess-xsnap.js | 233 ++++++++++++++++++ packages/SwingSet/src/types.js | 22 ++ packages/SwingSet/test/workers/test-worker.js | 9 +- 9 files changed, 485 insertions(+), 23 deletions(-) create mode 100644 packages/SwingSet/src/kernel/vatManager/lockdown-subprocess-xsnap.js create mode 100644 packages/SwingSet/src/kernel/vatManager/manager-subprocess-xsnap.js create mode 100644 packages/SwingSet/src/kernel/vatManager/supervisor-subprocess-xsnap.js diff --git a/packages/SwingSet/package.json b/packages/SwingSet/package.json index 82c3528f0ab..0ed59d1a1cf 100644 --- a/packages/SwingSet/package.json +++ b/packages/SwingSet/package.json @@ -49,6 +49,7 @@ "@agoric/transform-eventual-send": "^1.4.0", "@agoric/transform-metering": "^1.4.0", "@agoric/xs-vat-worker": "^0.4.0", + "@agoric/xsnap": "^0.1.0", "@babel/core": "^7.5.0", "@babel/generator": "^7.6.4", "anylogger": "^0.21.0", diff --git a/packages/SwingSet/src/controller.js b/packages/SwingSet/src/controller.js index d5e55b155b2..61d2f6cf11c 100644 --- a/packages/SwingSet/src/controller.js +++ b/packages/SwingSet/src/controller.js @@ -1,7 +1,8 @@ import fs from 'fs'; -import path from 'path'; import process from 'process'; import re2 from 're2'; +import { spawn } from 'child_process'; +import { type as osType } from 'os'; import { Worker } from 'worker_threads'; import * as babelCore from '@babel/core'; import * as babelParser from '@agoric/babel-parser'; @@ -11,10 +12,11 @@ import anylogger from 'anylogger'; import { assert } from '@agoric/assert'; import { isTamed, tameMetering } from '@agoric/tame-metering'; import { importBundle } from '@agoric/import-bundle'; +import bundleSource from '@agoric/bundle-source'; import { initSwingStore } from '@agoric/swing-store-simple'; import { makeMeteringTransformer } from '@agoric/transform-metering'; import { makeTransform } from '@agoric/transform-eventual-send'; -import { locateWorkerBin } from '@agoric/xs-vat-worker'; +import { xsnap } from '@agoric/xsnap'; import { WeakRef, FinalizationRegistry } from './weakref'; import { startSubprocessWorker } from './spawnSubprocessWorker'; @@ -36,6 +38,20 @@ function makeConsole(tag) { return harden(cons); } +async function buildXsBundles() { + const zip = (xs, ys) => xs.map((x, i) => [x, ys[i]]); + const { keys, values, fromEntries } = Object; + const allValues = async obj => + fromEntries(zip(keys(obj), await Promise.all(values(obj)))); + const src = rel => bundleSource(require.resolve(rel), 'getExport'); + return harden( + await allValues({ + lockdown: src('./kernel/vatManager/lockdown-subprocess-xsnap.js'), + supervisor: src('./kernel/vatManager/supervisor-subprocess-xsnap.js'), + }), + ); +} + export async function makeSwingsetController( hostStorage = initSwingStore().storage, deviceEndowments = {}, @@ -157,11 +173,21 @@ export async function makeSwingsetController( return startSubprocessWorker(process.execPath, ['-r', 'esm', supercode]); } - let startSubprocessWorkerXS; - const xsWorkerBin = locateWorkerBin({ resolve: path.resolve }); - if (fs.existsSync(xsWorkerBin)) { - startSubprocessWorkerXS = () => startSubprocessWorker(xsWorkerBin); - } + const { xsnapBundles = await buildXsBundles() } = {}; // @@@ options? + const startXSnap = (name, handleCommand) => { + console.log('starting xsnap for', name); + const worker = xsnap({ + os: osType(), + spawn, + handleCommand, + name, + stdout: 'inherit', + stderr: 'inherit', + // debug: true, + }); + + return harden({ worker, bundles: xsnapBundles }); + }; const slogF = slogFile && (await fs.createWriteStream(slogFile, { flags: 'a' })); // append @@ -186,7 +212,7 @@ export async function makeSwingsetController( transformTildot, makeNodeWorker, startSubprocessWorkerNode, - startSubprocessWorkerXS, + startXSnap, writeSlogObject, WeakRef, FinalizationRegistry, diff --git a/packages/SwingSet/src/kernel/kernel.js b/packages/SwingSet/src/kernel/kernel.js index a7f3ab5e7c7..f68df887f63 100644 --- a/packages/SwingSet/src/kernel/kernel.js +++ b/packages/SwingSet/src/kernel/kernel.js @@ -98,7 +98,7 @@ export default function buildKernel( transformTildot, makeNodeWorker, startSubprocessWorkerNode, - startSubprocessWorkerXS, + startXSnap, writeSlogObject, WeakRef, FinalizationRegistry, @@ -560,7 +560,7 @@ export default function buildKernel( waitUntilQuiescent, makeNodeWorker, startSubprocessWorkerNode, - startSubprocessWorkerXS, + startXSnap, gcTools, }); diff --git a/packages/SwingSet/src/kernel/vatManager/factory.js b/packages/SwingSet/src/kernel/vatManager/factory.js index be1cf384ecd..4ed326f5901 100644 --- a/packages/SwingSet/src/kernel/vatManager/factory.js +++ b/packages/SwingSet/src/kernel/vatManager/factory.js @@ -3,6 +3,7 @@ import { assertKnownOptions } from '../../assertOptions'; import { makeLocalVatManagerFactory } from './manager-local'; import { makeNodeWorkerVatManagerFactory } from './manager-nodeworker'; import { makeNodeSubprocessFactory } from './manager-subprocess-node'; +import { makeXsSubprocessFactory } from './manager-subprocess-xsnap'; export function makeVatManagerFactory({ allVatPowers, @@ -13,7 +14,7 @@ export function makeVatManagerFactory({ waitUntilQuiescent, makeNodeWorker, startSubprocessWorkerNode, - startSubprocessWorkerXS, + startXSnap, gcTools, }) { const localFactory = makeLocalVatManagerFactory({ @@ -40,8 +41,8 @@ export function makeVatManagerFactory({ decref: gcTools.decref, }); - const xsWorkerFactory = makeNodeSubprocessFactory({ - startSubprocessWorker: startSubprocessWorkerXS, + const xsWorkerFactory = makeXsSubprocessFactory({ + startXSnap, kernelKeeper, testLog: allVatPowers.testLog, decref: gcTools.decref, @@ -98,9 +99,6 @@ export function makeVatManagerFactory({ } if (managerType === 'xs-worker') { - if (!startSubprocessWorkerXS) { - throw new Error('manager type xs-worker not available'); - } return xsWorkerFactory.createFromBundle(vatID, bundle, managerOptions); } diff --git a/packages/SwingSet/src/kernel/vatManager/lockdown-subprocess-xsnap.js b/packages/SwingSet/src/kernel/vatManager/lockdown-subprocess-xsnap.js new file mode 100644 index 00000000000..c058d91da4e --- /dev/null +++ b/packages/SwingSet/src/kernel/vatManager/lockdown-subprocess-xsnap.js @@ -0,0 +1 @@ +import '@agoric/xs-vat-worker/src/bootstrap'; diff --git a/packages/SwingSet/src/kernel/vatManager/manager-subprocess-xsnap.js b/packages/SwingSet/src/kernel/vatManager/manager-subprocess-xsnap.js new file mode 100644 index 00000000000..d89a53f1428 --- /dev/null +++ b/packages/SwingSet/src/kernel/vatManager/manager-subprocess-xsnap.js @@ -0,0 +1,186 @@ +// @ts-check +import { assert, details } from '@agoric/assert'; +import { makeTranscriptManager } from './transcript'; +import { createSyscall } from './syscall'; + +// eslint-disable-next-line no-unused-vars +function parentLog(first, ...args) { + // console.error(`--parent: ${first}`, ...args); +} + +const encoder = new TextEncoder(); +const decoder = new TextDecoder(); + +/** + * @param {{ + * startXSnap: (name: string, handleCommand: SyncHandler) => { worker: XSnap, bundles: Record }, + * kernelKeeper: KernelKeeper, + * testLog: (...args: unknown[]) => void, + * decref: (vatID: unknown, vref: unknown, count: number) => void, + * }} tools + * @returns { VatManagerFactory } + * + * @typedef { { moduleFormat: 'getExport', source: string } } ExportBundle + * @typedef { (msg: Uint8Array) => Uint8Array } SyncHandler + * @typedef { ReturnType } XSnap + * @typedef { ReturnType } KernelKeeper + * @typedef { ReturnType } VatManagerFactory + * @typedef { [unknown, ...unknown[]] } Tagged + */ +export function makeXsSubprocessFactory({ + startXSnap, + kernelKeeper, + testLog, + decref, +}) { + /** + * @param { unknown } vatID + * @param { unknown } bundle + * @param { ManagerOptions } managerOptions + */ + async function createFromBundle(vatID, bundle, managerOptions) { + parentLog('createFromBundle', { vatID }); + const { vatParameters, virtualObjectCacheSize } = managerOptions; + assert(!managerOptions.metered, 'not supported yet'); + assert(!managerOptions.enableSetup, 'not supported at all'); + if (managerOptions.enableInternalMetering) { + // TODO: warn+ignore, rather than throw, because the kernel enables it + // for all vats, because the Spawner still needs it. When the kernel + // stops doing that, turn this into a regular assert + console.log(`xsnap worker does not support enableInternalMetering`); + } + const vatKeeper = kernelKeeper.getVatKeeper(vatID); + const transcriptManager = makeTranscriptManager( + kernelKeeper, + vatKeeper, + vatID, + ); + + const { doSyscall, setVatSyscallHandler } = createSyscall( + transcriptManager, + ); + + /** @type { (vatSyscallObject: Tagged) => unknown } */ + function handleSyscall(vatSyscallObject) { + return doSyscall(vatSyscallObject); + } + + /** @type { (vref: unknown, count: number) => void } */ + function vatDecref(vref, count) { + decref(vatID, vref, count); + } + + /** @type { (item: Tagged) => unknown } */ + function handleUpstream([type, ...args]) { + parentLog(`handleUpstream`, type, args.length); + switch (type) { + case 'syscall': { + parentLog(`syscall`, args); + const [scTag, ...vatSyscallArgs] = args; + return handleSyscall([scTag, ...vatSyscallArgs]); + } + case 'testLog': + testLog(...args); + return ['OK']; + case 'decref': { + const [vref, count] = args; + assert(typeof count === 'number'); + vatDecref(vref, count); + return ['OK']; + } + default: + throw new Error(`unrecognized uplink message ${type}`); + } + } + + /** @type { (msg: Uint8Array) => Uint8Array } */ + function handleCommand(msg) { + parentLog('handleCommand', { length: msg.byteLength }); + const tagged = handleUpstream(JSON.parse(decoder.decode(msg))); + return encoder.encode(JSON.stringify(tagged)); + } + + // start the worker and establish a connection + const { worker, bundles } = startXSnap(`${vatID}`, handleCommand); + for await (const [it, superCode] of Object.entries(bundles)) { + parentLog('bundle', it); + assert( + superCode.moduleFormat === 'getExport', + details`${it} unexpected: ${superCode.moduleFormat}`, + ); + await worker.evaluate( + `(${superCode.source} + )()`.trim(), + ); + } + + /** @type { (item: Tagged) => Promise } */ + async function issueTagged(item) { + parentLog('issueTagged', item[0]); + const txt = await worker.issueStringCommand(JSON.stringify(item)); + const reply = JSON.parse(txt); + assert(Array.isArray(reply)); + const [tag, ...rest] = reply; + return [tag, ...rest]; + } + + let seq = 0; + /** @type { (item: Tagged) => Promise } */ + async function commandResult(item) { + const [tag, ...rest] = await issueTagged(item); + seq += 1; + if (tag === 'err') return [tag, ...rest]; + return issueTagged(['getResult', seq]); + } + + parentLog(`instructing worker to load bundle..`); + const bundleReply = await commandResult([ + 'setBundle', + vatID, + bundle, + vatParameters, + virtualObjectCacheSize, + ]); + if (bundleReply[0] === 'dispatchReady') { + parentLog(`bundle loaded. dispatch ready.`); + } else { + throw new Error(`failed to setBundle: ${bundleReply}`); + } + + /** @type { (item: Tagged) => Promise } */ + async function deliver(delivery) { + parentLog(`sending delivery`, delivery); + const result = await commandResult(['deliver', ...delivery]); + parentLog(`deliverDone`, result[0], result.length); + return result; + } + + async function replayTranscript() { + transcriptManager.startReplay(); + for (const t of vatKeeper.getTranscript()) { + transcriptManager.checkReplayError(); + transcriptManager.startReplayDelivery(t.syscalls); + // eslint-disable-next-line no-await-in-loop + await deliver(t.d); + } + transcriptManager.checkReplayError(); + transcriptManager.finishReplay(); + } + + function shutdown() { + return worker.close(); + } + + const manager = harden({ + replayTranscript, + setVatSyscallHandler, + deliver, + shutdown, + }); + + parentLog('manager', Object.keys(manager)); + return manager; + } + + return harden({ createFromBundle }); +} diff --git a/packages/SwingSet/src/kernel/vatManager/supervisor-subprocess-xsnap.js b/packages/SwingSet/src/kernel/vatManager/supervisor-subprocess-xsnap.js new file mode 100644 index 00000000000..dcdc35792f0 --- /dev/null +++ b/packages/SwingSet/src/kernel/vatManager/supervisor-subprocess-xsnap.js @@ -0,0 +1,233 @@ +// @ts-check +import { assert, details } from '@agoric/assert'; +import { importBundle } from '@agoric/import-bundle'; +import { Remotable, getInterfaceOf, makeMarshal } from '@agoric/marshal'; +// grumble... waitUntilQuiescent is exported and closes over ambient authority +import { waitUntilQuiescent } from '../../waitUntilQuiescent'; + +import { makeLiveSlots } from '../liveSlots'; + +const encoder = new TextEncoder(); +const decoder = new TextDecoder(); + +// eslint-disable-next-line no-unused-vars +function workerLog(first, ...args) { + // @ts-ignore + // eslint-disable-next-line + // print(`---worker: ${first}`, ...args); +} + +workerLog(`supervisor started`); + +/** + * @typedef { [unknown, ...unknown[]] } Tagged + */ +const Item = { + /** @type { (item: Tagged) => ArrayBuffer } */ + encode: tagged => encoder.encode(JSON.stringify(tagged)).buffer, + + /** @type { (msg: ArrayBuffer) => Tagged } */ + decode(msg) { + const txt = decoder.decode(msg); + + /** @type { Tagged } */ + const item = JSON.parse(txt); + assert(Array.isArray(item), details`expected array`); + assert(item.length > 0, details`empty array lacks tag`); + return item; + }, +}; + +function makeConsole(_tag) { + const log = console; // TODO: anylogger(tag); + const cons = { + debug: log.debug, + log: log.log, + info: log.info, + warn: log.warn, + error: log.error, + }; + return harden(cons); +} + +function testLog(...args) { + // @ts-ignore xsnap provides issueCommand global + // eslint-disable-next-line no-undef + issueCommand(Item.encode(['testLog', ...args])); +} + +/** + * @param { (value: void) => void } f + * @param { string } errmsg + */ +function runAndWait(f, errmsg) { + Promise.resolve() + .then(f) + .then(undefined, err => workerLog(`doProcess: ${errmsg}:`, err)); + return waitUntilQuiescent(); +} + +function makeWorker() { + /** @type { Record void> | null } */ + let dispatch = null; + + /** @type { (dr: Tagged, errmsg: string) => Promise } */ + async function doProcess(dispatchRecord, errmsg) { + assert(dispatch); + const theDispatch = dispatch; + const [dispatchOp, ...dispatchArgs] = dispatchRecord; + assert(typeof dispatchOp === 'string'); + workerLog(`runAndWait`); + await runAndWait(() => theDispatch[dispatchOp](...dispatchArgs), errmsg); + workerLog(`doProcess done`); + /** @type { Tagged } */ + const vatDeliveryResults = harden(['ok']); + return vatDeliveryResults; + } + + /** @type { (ts: unknown, msg: any) => Promise } */ + function doMessage(targetSlot, msg) { + const errmsg = `vat[${targetSlot}].${msg.method} dispatch failed`; + return doProcess( + ['deliver', targetSlot, msg.method, msg.args, msg.result], + errmsg, + ); + } + + /** @type { (rs: unknown) => Promise } */ + function doNotify(resolutions) { + const errmsg = `vat.notify failed`; + return doProcess(['notify', resolutions], errmsg); + } + + /** + * @param {unknown} vatID + * @param {unknown} bundle + * @param {unknown} vatParameters + * @param {unknown} virtualObjectCacheSize + * @returns { Promise } + */ + async function setBundle( + vatID, + bundle, + vatParameters, + virtualObjectCacheSize, + ) { + /** @type { (item: Tagged) => unknown } */ + function doSyscall(vatSyscallObject) { + return JSON.parse( + decoder.decode( + // @ts-ignore + // eslint-disable-next-line no-undef + issueCommand(Item.encode(['syscall', ...vatSyscallObject])), + ), + ); + } + + const syscall = harden({ + send: (...args) => doSyscall(['send', ...args]), + callNow: (...args) => doSyscall(['callNow', ...args]), + subscribe: (...args) => doSyscall(['subscribe', ...args]), + fulfillToData: (...args) => doSyscall(['fulfillToData', ...args]), + fulfillToPresence: (...args) => doSyscall(['fulfillToPresence', ...args]), + reject: (...args) => doSyscall(['reject', ...args]), + }); + + const vatPowers = { + Remotable, + getInterfaceOf, + makeMarshal, + testLog, + }; + + const ls = makeLiveSlots( + syscall, + vatID, + vatPowers, + vatParameters, + // @ts-ignore TODO: defend against non-numeric? + virtualObjectCacheSize, + // TODO: lsgc? API drift? + ); + + const endowments = { + ...ls.vatGlobals, + console: makeConsole(`SwingSet:vatWorker`), + assert, + // @ts-ignore bootstrap provides HandledPromise + // eslint-disable-next-line no-undef + HandledPromise, + }; + const vatNS = await importBundle(bundle, { endowments }); + workerLog(`got vatNS:`, Object.keys(vatNS).join(',')); + ls.setBuildRootObject(vatNS.buildRootObject); + dispatch = ls.dispatch; + assert(dispatch); + workerLog(`got dispatch:`, Object.keys(dispatch).join(',')); + return ['dispatchReady']; + } + + /** @type { (item: Tagged) => Promise } */ + async function handleItem([tag, ...args]) { + switch (tag) { + case 'setBundle': + assert(!dispatch, 'cannot setBundle again'); + return setBundle(args[0], args[1], args[2], args[3]); + case 'deliver': { + assert(dispatch, 'cannot deliver before setBundle'); + const [dtype, ...dargs] = args; + switch (dtype) { + case 'message': + return doMessage(dargs[0], dargs[1]); + case 'notify': + return doNotify(dargs[0]); + default: + throw Error(`bad delivery type ${dtype}`); + } + } + default: + workerLog('handleItem: bad tag', tag, args.length); + return ['bad tag', tag]; + } + } + + let seq = 0; + /** @type { Tagged? } */ + let result = null; + + /** @type { (item: Tagged) => Tagged } */ + function answerItem(item) { + const [tag, ...args] = item; + workerLog('answerItem', tag, args.length); + if (tag === 'getResult') { + const [target] = args; + if (target !== seq) return ['err', 'seq expected', seq, 'got', target]; + const out = result; + result = null; + if (!Array.isArray(out)) return ['no result available', result]; + return out; + } + + handleItem([tag, ...args]).then(out => { + seq += 1; + result = out; + }); + return ['queued', seq, tag, args.length]; + } + + return harden({ + /** @type { (msg: ArrayBuffer) => ArrayBuffer } */ + handleCommand(msg) { + let item; + const { encode } = Item; + try { + item = Item.decode(msg); + } catch (badItem) { + return encode(['bad msg', badItem.message]); + } + return Item.encode(answerItem(item)); + }, + }); +} + +globalThis.handleCommand = makeWorker().handleCommand; diff --git a/packages/SwingSet/src/types.js b/packages/SwingSet/src/types.js index b8e38e1175f..273e2b77dd3 100644 --- a/packages/SwingSet/src/types.js +++ b/packages/SwingSet/src/types.js @@ -3,3 +3,25 @@ * @property {string} body * @property {Array} slots */ + +/** + * @typedef {{ + * bundle: Bundle, + * enableSetup: false, + * }} HasBundle + * @typedef {{ + * setup: unknown, + * enableSetup: true, + * }} HasSetup + * + * TODO: metered... + * + * See validateManagerOptions() in factory.js + * @typedef {{ + * managerType: 'local' | 'nodeWorker' | 'node-subprocess' | 'xs-worker', + * metered?: boolean, + * enableInternalMetering?: boolean, + * vatParameters: Serializable, + * virtualObjectCacheSize: number, + * } & (HasBundle | HasSetup)} ManagerOptions + */ diff --git a/packages/SwingSet/test/workers/test-worker.js b/packages/SwingSet/test/workers/test-worker.js index 6fb6d7184bd..0c01ff85090 100644 --- a/packages/SwingSet/test/workers/test-worker.js +++ b/packages/SwingSet/test/workers/test-worker.js @@ -7,8 +7,7 @@ const expected = [['B good', 'C good', 'F good', 'three good'], 'rp3 good']; async function makeController(managerType) { const config = await loadBasedir(__dirname); config.vats.target.creationOptions = { managerType }; - const canCallNow = ['local'].indexOf(managerType) !== -1; - // const canCallNow = ['local', 'xs-worker'].indexOf(managerType) !== -1; + const canCallNow = ['local', 'xs-worker'].indexOf(managerType) !== -1; config.vats.target.parameters = { canCallNow }; config.devices = { add: { @@ -29,11 +28,7 @@ test('local vat manager', async t => { t.deepEqual(JSON.parse(c.kpResolution(c.bootstrapResult).body), expected); }); -// The XS worker is disabled until the xsnap-based approach is ready for -// testing. Unlike the old approach, I think we'll build xsnap -// unconditionally, so we won't need the old 'maybeTestXS' conditional. - -test.skip('xs vat manager', async t => { +test('xs vat manager', async t => { const c = await makeController('xs-worker'); t.teardown(c.shutdown); From a19da62615f6577115b739f0dbdee020b75bc526 Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Fri, 22 Jan 2021 20:57:48 -0600 Subject: [PATCH 12/23] fix: crank 1 comment in vat-target.js --- packages/SwingSet/test/workers/vat-target.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/SwingSet/test/workers/vat-target.js b/packages/SwingSet/test/workers/vat-target.js index 63e3080c3ae..adcabf5cd97 100644 --- a/packages/SwingSet/test/workers/vat-target.js +++ b/packages/SwingSet/test/workers/vat-target.js @@ -26,9 +26,9 @@ export function buildRootObject(vatPowers, vatParameters) { // dispatch.deliver(target, method="zero", result=pA, args=[callbackObj, pD, pE, adder]) // syscall.subscribe(pD) // syscall.subscribe(pE) + // syscall.callNow(adder, args=[1, 2]) -> 3 // syscall.send(callbackObj, method="callback", result=rp2, args=[11, 12]); // syscall.subscribe(rp2) - // syscall.callNow(adder, args=[1, 2]) -> 3 // syscall.fulfillToData(pA, [pB, pC, 3]); function zero(obj, pD, pE, adder) { callbackObj = obj; From c16d6ce34f8bc4858134b4b4e3e88a6a18c647a7 Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Fri, 22 Jan 2021 20:58:31 -0600 Subject: [PATCH 13/23] fix: supply groupCollapsed etc. in console-shim for SES --- packages/xs-vat-worker/src/console-shim.js | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/packages/xs-vat-worker/src/console-shim.js b/packages/xs-vat-worker/src/console-shim.js index 7c58ec97216..230aa2015b4 100644 --- a/packages/xs-vat-worker/src/console-shim.js +++ b/packages/xs-vat-worker/src/console-shim.js @@ -1,9 +1,17 @@ const noop = _ => undefined; -globalThis.console = { +const console = { debug: noop, log: noop, info: noop, warn: noop, error: noop, + // used by SES + group: noop, + groupCollapsed: noop, + groupEnd: noop, + // others from the MDN / whatwg Console API? + // trace, dirxml, ... }; + +globalThis.console = console; From 454362d0b902c799293b3c9c520d2a0802d41739 Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Fri, 22 Jan 2021 21:07:51 -0600 Subject: [PATCH 14/23] fix(xsnap): handle edge cases in sending replies to e, ? --- packages/xsnap/src/xsnap.c | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/packages/xsnap/src/xsnap.c b/packages/xsnap/src/xsnap.c index d35a406dda2..fc5e70d612d 100644 --- a/packages/xsnap/src/xsnap.c +++ b/packages/xsnap/src/xsnap.c @@ -346,6 +346,7 @@ int main(int argc, char* argv[]) int writeError; xsBeginHost(machine); { + xsSlot result = xsUndefined; if (error) { xsStringValue message = xsToString(report); writeError = fxWriteNetString(toParent, '!', message, strlen(message)); @@ -354,16 +355,26 @@ int main(int argc, char* argv[]) char* response = NULL; txInteger responseLength = 0; // fprintf(stderr, "report: %d %s\n", xsTypeOf(report), xsToString(report)); - xsSlot result; - if (command == 'e' && xsTypeOf(report) != xsUndefinedType) { - result = xsGet(report, xsID("result")); - } else { - result = report; + xsTry { + if (xsTypeOf(report) == xsReferenceType && xsHas(report, xsID("result"))) { + result = xsGet(report, xsID("result")); + } else { + result = report; + } + // fprintf(stderr, "result: %d %s\n", xsTypeOf(result), xsToString(result)); + if (xsIsInstanceOf(result, xsArrayBufferPrototype)) { + response = xsToArrayBuffer(result); + responseLength = xsGetArrayBufferLength(result); + } } - // fprintf(stderr, "result: %d %s\n", xsTypeOf(result), xsToString(result)); - if (xsTypeOf(result) != xsUndefinedType) { - response = xsToArrayBuffer(result); // TODO: catch exceptions from this? - responseLength = xsGetArrayBufferLength(result); + xsCatch { + if (xsTypeOf(xsException) != xsUndefinedType) { + fprintf(stderr, "%c computing response %d %d: %s: %s\n", command, + xsTypeOf(report), xsTypeOf(result), + xsToString(result), + xsToString(xsException)); + xsException = xsUndefined; + } } // fprintf(stderr, "response of %d bytes\n", responseLength); writeError = fxWriteNetString(toParent, '.', response, responseLength); From 63e2eb185bc5e4f25d06a511e64020ecdc5782cb Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Fri, 22 Jan 2021 21:08:41 -0600 Subject: [PATCH 15/23] refactor: avoid 2nd round trip to xsnap - manager: prune commandResult - supervisor: factor out "transport" logic as `ManagerPort`, separate from vat-worker `makeWorker()` - ManagerPort.handler provides `{ result?: ArrayBuffer }` idiom based on Promise - testLog uses ManagerPort.send - clean up redundant 'ok' tag in doMessage, doNotify - refactor: tagged -> item for consistency --- .../vatManager/manager-subprocess-xsnap.js | 13 +-- .../vatManager/supervisor-subprocess-xsnap.js | 106 ++++++++---------- packages/xsnap/src/xsnap.js | 4 +- 3 files changed, 51 insertions(+), 72 deletions(-) diff --git a/packages/SwingSet/src/kernel/vatManager/manager-subprocess-xsnap.js b/packages/SwingSet/src/kernel/vatManager/manager-subprocess-xsnap.js index d89a53f1428..36660b113f8 100644 --- a/packages/SwingSet/src/kernel/vatManager/manager-subprocess-xsnap.js +++ b/packages/SwingSet/src/kernel/vatManager/manager-subprocess-xsnap.js @@ -124,17 +124,8 @@ export function makeXsSubprocessFactory({ return [tag, ...rest]; } - let seq = 0; - /** @type { (item: Tagged) => Promise } */ - async function commandResult(item) { - const [tag, ...rest] = await issueTagged(item); - seq += 1; - if (tag === 'err') return [tag, ...rest]; - return issueTagged(['getResult', seq]); - } - parentLog(`instructing worker to load bundle..`); - const bundleReply = await commandResult([ + const bundleReply = await issueTagged([ 'setBundle', vatID, bundle, @@ -150,7 +141,7 @@ export function makeXsSubprocessFactory({ /** @type { (item: Tagged) => Promise } */ async function deliver(delivery) { parentLog(`sending delivery`, delivery); - const result = await commandResult(['deliver', ...delivery]); + const result = await issueTagged(['deliver', ...delivery]); parentLog(`deliverDone`, result[0], result.length); return result; } diff --git a/packages/SwingSet/src/kernel/vatManager/supervisor-subprocess-xsnap.js b/packages/SwingSet/src/kernel/vatManager/supervisor-subprocess-xsnap.js index dcdc35792f0..05db287be22 100644 --- a/packages/SwingSet/src/kernel/vatManager/supervisor-subprocess-xsnap.js +++ b/packages/SwingSet/src/kernel/vatManager/supervisor-subprocess-xsnap.js @@ -20,23 +20,45 @@ function workerLog(first, ...args) { workerLog(`supervisor started`); /** + * @param { (cmd: ArrayBuffer) => ArrayBuffer } issueCommand * @typedef { [unknown, ...unknown[]] } Tagged + * @typedef { (item: Tagged) => unknown } DataHandler + * @typedef { (item: Tagged) => Promise } AsyncHandler */ -const Item = { +function ManagerPort(issueCommand) { /** @type { (item: Tagged) => ArrayBuffer } */ - encode: tagged => encoder.encode(JSON.stringify(tagged)).buffer, + const encode = item => encoder.encode(JSON.stringify(item)).buffer; - /** @type { (msg: ArrayBuffer) => Tagged } */ - decode(msg) { - const txt = decoder.decode(msg); + /** @type { (msg: ArrayBuffer) => any } */ + const decodeData = msg => JSON.parse(decoder.decode(msg)); + /** @type { (msg: ArrayBuffer) => Tagged } */ + function decode(msg) { /** @type { Tagged } */ - const item = JSON.parse(txt); + const item = decodeData(msg); assert(Array.isArray(item), details`expected array`); assert(item.length > 0, details`empty array lacks tag`); return item; - }, -}; + } + + return harden({ + /** @type { (item: Tagged) => void } */ + send: item => { + issueCommand(encode(item)); + }, + /** @type { DataHandler } */ + call: item => decodeData(issueCommand(encode(item))), + /** @type { (f: AsyncHandler) => ((msg: ArrayBuffer) => { result?: ArrayBuffer })} */ + handler: f => msg => { + const report = {}; + f(decode(msg)).then(item => { + workerLog('result', item); + report.result = encode(item); + }); // TODO: .catch?!?! + return report; + }, + }); +} function makeConsole(_tag) { const log = console; // TODO: anylogger(tag); @@ -50,12 +72,6 @@ function makeConsole(_tag) { return harden(cons); } -function testLog(...args) { - // @ts-ignore xsnap provides issueCommand global - // eslint-disable-next-line no-undef - issueCommand(Item.encode(['testLog', ...args])); -} - /** * @param { (value: void) => void } f * @param { string } errmsg @@ -67,7 +83,10 @@ function runAndWait(f, errmsg) { return waitUntilQuiescent(); } -function makeWorker() { +/** + * @param { ReturnType } port + */ +function makeWorker(port) { /** @type { Record void> | null } */ let dispatch = null; @@ -115,13 +134,10 @@ function makeWorker() { ) { /** @type { (item: Tagged) => unknown } */ function doSyscall(vatSyscallObject) { - return JSON.parse( - decoder.decode( - // @ts-ignore - // eslint-disable-next-line no-undef - issueCommand(Item.encode(['syscall', ...vatSyscallObject])), - ), - ); + workerLog('doSyscall', vatSyscallObject); + const result = port.call(['syscall', ...vatSyscallObject]); + workerLog(' ... syscall result:', result); + return result; } const syscall = harden({ @@ -137,7 +153,7 @@ function makeWorker() { Remotable, getInterfaceOf, makeMarshal, - testLog, + testLog: (...args) => port.send(['testLog', ...args]), }; const ls = makeLiveSlots( @@ -169,6 +185,7 @@ function makeWorker() { /** @type { (item: Tagged) => Promise } */ async function handleItem([tag, ...args]) { + workerLog('handleItem', tag, args.length); switch (tag) { case 'setBundle': assert(!dispatch, 'cannot setBundle again'); @@ -191,43 +208,14 @@ function makeWorker() { } } - let seq = 0; - /** @type { Tagged? } */ - let result = null; - - /** @type { (item: Tagged) => Tagged } */ - function answerItem(item) { - const [tag, ...args] = item; - workerLog('answerItem', tag, args.length); - if (tag === 'getResult') { - const [target] = args; - if (target !== seq) return ['err', 'seq expected', seq, 'got', target]; - const out = result; - result = null; - if (!Array.isArray(out)) return ['no result available', result]; - return out; - } - - handleItem([tag, ...args]).then(out => { - seq += 1; - result = out; - }); - return ['queued', seq, tag, args.length]; - } - return harden({ - /** @type { (msg: ArrayBuffer) => ArrayBuffer } */ - handleCommand(msg) { - let item; - const { encode } = Item; - try { - item = Item.decode(msg); - } catch (badItem) { - return encode(['bad msg', badItem.message]); - } - return Item.encode(answerItem(item)); - }, + /** @type { AsyncHandler } */ + handleItem, }); } -globalThis.handleCommand = makeWorker().handleCommand; +// @ts-ignore xsnap provides issueCommand global +// eslint-disable-next-line no-undef +const port = ManagerPort(issueCommand); +const worker = makeWorker(port); +globalThis.handleCommand = port.handler(worker.handleItem); diff --git a/packages/xsnap/src/xsnap.js b/packages/xsnap/src/xsnap.js index f5660d59e2d..4fb4387e8a8 100644 --- a/packages/xsnap/src/xsnap.js +++ b/packages/xsnap/src/xsnap.js @@ -136,14 +136,14 @@ export function xsnap(options) { /** * @param {string} code - * @returns {Promise} + * @returns {Promise} */ async function evaluate(code) { const result = baton.then(async () => { await messagesToXsnap.next(encoder.encode(`e${code}`)); return runToIdle(); }); - baton = result.then(_ => undefined).catch(() => {}); + baton = result.then(() => {}).catch(() => {}); return Promise.race([vatCancelled, result]); } From 58dfb1776855b765a83b55bad3c094ba88c4c5de Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Fri, 22 Jan 2021 21:22:12 -0600 Subject: [PATCH 16/23] feat(xsnap worker): pass console log messages to manager - prune 'starting xsnap' log msg (per code review) - handle rejection in ManagerPort.handler --- packages/SwingSet/src/controller.js | 1 - .../vatManager/manager-subprocess-xsnap.js | 9 +++++ .../vatManager/supervisor-subprocess-xsnap.js | 37 +++++++++++-------- 3 files changed, 30 insertions(+), 17 deletions(-) diff --git a/packages/SwingSet/src/controller.js b/packages/SwingSet/src/controller.js index 61d2f6cf11c..9a74198879e 100644 --- a/packages/SwingSet/src/controller.js +++ b/packages/SwingSet/src/controller.js @@ -175,7 +175,6 @@ export async function makeSwingsetController( const { xsnapBundles = await buildXsBundles() } = {}; // @@@ options? const startXSnap = (name, handleCommand) => { - console.log('starting xsnap for', name); const worker = xsnap({ os: osType(), spawn, diff --git a/packages/SwingSet/src/kernel/vatManager/manager-subprocess-xsnap.js b/packages/SwingSet/src/kernel/vatManager/manager-subprocess-xsnap.js index 36660b113f8..9313768b6ff 100644 --- a/packages/SwingSet/src/kernel/vatManager/manager-subprocess-xsnap.js +++ b/packages/SwingSet/src/kernel/vatManager/manager-subprocess-xsnap.js @@ -79,6 +79,15 @@ export function makeXsSubprocessFactory({ const [scTag, ...vatSyscallArgs] = args; return handleSyscall([scTag, ...vatSyscallArgs]); } + case 'console': { + const [level, tag, ...rest] = args; + if (typeof level === 'string' && level in console) { + console[level](tag, ...rest); + } else { + console.error('bad console level', level); + } + return ['ok']; + } case 'testLog': testLog(...args); return ['OK']; diff --git a/packages/SwingSet/src/kernel/vatManager/supervisor-subprocess-xsnap.js b/packages/SwingSet/src/kernel/vatManager/supervisor-subprocess-xsnap.js index 05db287be22..dee76357452 100644 --- a/packages/SwingSet/src/kernel/vatManager/supervisor-subprocess-xsnap.js +++ b/packages/SwingSet/src/kernel/vatManager/supervisor-subprocess-xsnap.js @@ -51,27 +51,19 @@ function ManagerPort(issueCommand) { /** @type { (f: AsyncHandler) => ((msg: ArrayBuffer) => { result?: ArrayBuffer })} */ handler: f => msg => { const report = {}; - f(decode(msg)).then(item => { - workerLog('result', item); - report.result = encode(item); - }); // TODO: .catch?!?! + f(decode(msg)) + .then(item => { + workerLog('result', item); + report.result = encode(item); + }) + .catch(err => { + report.result = encode(['err', err.message]); + }); return report; }, }); } -function makeConsole(_tag) { - const log = console; // TODO: anylogger(tag); - const cons = { - debug: log.debug, - log: log.log, - info: log.info, - warn: log.warn, - error: log.error, - }; - return harden(cons); -} - /** * @param { (value: void) => void } f * @param { string } errmsg @@ -119,6 +111,19 @@ function makeWorker(port) { return doProcess(['notify', resolutions], errmsg); } + function makeConsole(tag) { + const log = level => (...args) => + port.send(['console', level, tag, ...args]); + const cons = { + debug: log('debug'), + log: log('log'), + info: log('info'), + warn: log('warn'), + error: log('error'), + }; + return harden(cons); + } + /** * @param {unknown} vatID * @param {unknown} bundle From 7df4d3bfa65324e6006a48e31079115f9193d78a Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Fri, 22 Jan 2021 21:59:58 -0600 Subject: [PATCH 17/23] fix(xsnap): build args --- packages/xsnap/src/build.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/xsnap/src/build.js b/packages/xsnap/src/build.js index ab4b9867111..0745af7afd2 100644 --- a/packages/xsnap/src/build.js +++ b/packages/xsnap/src/build.js @@ -1,10 +1,11 @@ import * as childProcess from 'child_process'; import os from 'os'; -function exec(command, cwd, args=[]) { +function exec(command, cwd, args = []) { const child = childProcess.spawn(command, { cwd, stdio: ['inherit', 'inherit', 'inherit'], + args, }); return new Promise((resolve, reject) => { child.on('close', () => { From f1b20004a84ec072f4508945a94cb379369aa2ea Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Fri, 22 Jan 2021 22:01:15 -0600 Subject: [PATCH 18/23] refactor(xsnap): fold in what's left of xs-vat-worker - prune obsolete locate.js --- packages/SwingSet/package.json | 1 - .../vatManager/lockdown-subprocess-xsnap.js | 2 +- packages/xs-vat-worker/.gitignore | 1 - packages/xs-vat-worker/.npmignore | 11 - packages/xs-vat-worker/CHANGELOG.md | 101 --------- packages/xs-vat-worker/package.json | 81 ------- packages/xs-vat-worker/src/locate.js | 16 -- packages/xs-vat-worker/src/vatWorker.js | 210 ------------------ packages/xsnap/package.json | 6 +- .../{xs-vat-worker => xsnap}/rollup.config.js | 0 .../{xs-vat-worker => xsnap}/src/bootstrap.js | 0 .../src/console-shim.js | 0 .../src/lockdown-shim.js | 1 + .../{xs-vat-worker => xsnap}/src/text-shim.js | 0 .../test/escapeCompartment.js | 0 .../test/test-boot-lockdown.js | 2 +- 16 files changed, 8 insertions(+), 424 deletions(-) delete mode 100644 packages/xs-vat-worker/.gitignore delete mode 100644 packages/xs-vat-worker/.npmignore delete mode 100644 packages/xs-vat-worker/CHANGELOG.md delete mode 100644 packages/xs-vat-worker/package.json delete mode 100644 packages/xs-vat-worker/src/locate.js delete mode 100644 packages/xs-vat-worker/src/vatWorker.js rename packages/{xs-vat-worker => xsnap}/rollup.config.js (100%) rename packages/{xs-vat-worker => xsnap}/src/bootstrap.js (100%) rename packages/{xs-vat-worker => xsnap}/src/console-shim.js (100%) rename packages/{xs-vat-worker => xsnap}/src/lockdown-shim.js (72%) rename packages/{xs-vat-worker => xsnap}/src/text-shim.js (100%) rename packages/{xs-vat-worker => xsnap}/test/escapeCompartment.js (100%) rename packages/{xs-vat-worker => xsnap}/test/test-boot-lockdown.js (97%) diff --git a/packages/SwingSet/package.json b/packages/SwingSet/package.json index 0ed59d1a1cf..6614b9d97ff 100644 --- a/packages/SwingSet/package.json +++ b/packages/SwingSet/package.json @@ -48,7 +48,6 @@ "@agoric/tame-metering": "^1.3.0", "@agoric/transform-eventual-send": "^1.4.0", "@agoric/transform-metering": "^1.4.0", - "@agoric/xs-vat-worker": "^0.4.0", "@agoric/xsnap": "^0.1.0", "@babel/core": "^7.5.0", "@babel/generator": "^7.6.4", diff --git a/packages/SwingSet/src/kernel/vatManager/lockdown-subprocess-xsnap.js b/packages/SwingSet/src/kernel/vatManager/lockdown-subprocess-xsnap.js index c058d91da4e..ed9300404b5 100644 --- a/packages/SwingSet/src/kernel/vatManager/lockdown-subprocess-xsnap.js +++ b/packages/SwingSet/src/kernel/vatManager/lockdown-subprocess-xsnap.js @@ -1 +1 @@ -import '@agoric/xs-vat-worker/src/bootstrap'; +import '@agoric/xsnap/src/bootstrap'; diff --git a/packages/xs-vat-worker/.gitignore b/packages/xs-vat-worker/.gitignore deleted file mode 100644 index 1521c8b7652..00000000000 --- a/packages/xs-vat-worker/.gitignore +++ /dev/null @@ -1 +0,0 @@ -dist diff --git a/packages/xs-vat-worker/.npmignore b/packages/xs-vat-worker/.npmignore deleted file mode 100644 index cd75c6170bf..00000000000 --- a/packages/xs-vat-worker/.npmignore +++ /dev/null @@ -1,11 +0,0 @@ -# Demo -demo - -# scripts -scripts - -# test -test - -# Travis CI -.travis.yml diff --git a/packages/xs-vat-worker/CHANGELOG.md b/packages/xs-vat-worker/CHANGELOG.md deleted file mode 100644 index 55deff9e20d..00000000000 --- a/packages/xs-vat-worker/CHANGELOG.md +++ /dev/null @@ -1,101 +0,0 @@ -# Change Log - -All notable changes to this project will be documented in this file. -See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. - -# [0.4.0](https://github.com/Agoric/agoric-sdk/compare/@agoric/xs-vat-worker@0.3.0...@agoric/xs-vat-worker@0.4.0) (2020-12-10) - - -### Features - -* **import-bundle:** Preliminary support Endo zip hex bundle format ([#1983](https://github.com/Agoric/agoric-sdk/issues/1983)) ([983681b](https://github.com/Agoric/agoric-sdk/commit/983681bfc4bf512b6bd90806ed9220cd4fefc13c)) - - - - - -# [0.3.0](https://github.com/Agoric/agoric-sdk/compare/@agoric/xs-vat-worker@0.2.4-dev.0...@agoric/xs-vat-worker@0.3.0) (2020-11-07) - - -### Bug Fixes - -* further cleanup based on reviews ([2e74cc7](https://github.com/Agoric/agoric-sdk/commit/2e74cc72ce1c898b24c1a2613d7864d97fe383c2)) - - -### Features - -* **assert:** Thread stack traces to console, add entangled assert ([#1884](https://github.com/Agoric/agoric-sdk/issues/1884)) ([5d4f35f](https://github.com/Agoric/agoric-sdk/commit/5d4f35f901f2ca40a2a4d66dab980a5fe8e575f4)) - - - - - -## [0.2.4-dev.0](https://github.com/Agoric/agoric-sdk/compare/@agoric/xs-vat-worker@0.2.3...@agoric/xs-vat-worker@0.2.4-dev.0) (2020-10-19) - -**Note:** Version bump only for package @agoric/xs-vat-worker - - - - - -## [0.2.3](https://github.com/Agoric/agoric-sdk/compare/@agoric/xs-vat-worker@0.2.3-dev.2...@agoric/xs-vat-worker@0.2.3) (2020-10-11) - - -### Bug Fixes - -* handle syscallResult and deliveryResult consistently among workers ([9e6e31a](https://github.com/Agoric/agoric-sdk/commit/9e6e31ac55521893b6fdf31785bb901345ed46af)), closes [#1775](https://github.com/Agoric/agoric-sdk/issues/1775) -* pass testLog to all vatWorkers ([29bc81a](https://github.com/Agoric/agoric-sdk/commit/29bc81a46d057532f51c37bed081d850cf7f31db)), closes [#1776](https://github.com/Agoric/agoric-sdk/issues/1776) -* xs-vat-worker: add makeMarshal to vatPowers ([1edd620](https://github.com/Agoric/agoric-sdk/commit/1edd62015e955d99fef8f75d32d2a5f1032aca38)), closes [#1776](https://github.com/Agoric/agoric-sdk/issues/1776) - - - - - -## [0.2.3-dev.2](https://github.com/Agoric/agoric-sdk/compare/@agoric/xs-vat-worker@0.2.3-dev.1...@agoric/xs-vat-worker@0.2.3-dev.2) (2020-09-18) - -**Note:** Version bump only for package @agoric/xs-vat-worker - - - - - -## [0.2.3-dev.1](https://github.com/Agoric/agoric-sdk/compare/@agoric/xs-vat-worker@0.2.3-dev.0...@agoric/xs-vat-worker@0.2.3-dev.1) (2020-09-18) - -**Note:** Version bump only for package @agoric/xs-vat-worker - - - - - -## [0.2.3-dev.0](https://github.com/Agoric/agoric-sdk/compare/@agoric/xs-vat-worker@0.2.2...@agoric/xs-vat-worker@0.2.3-dev.0) (2020-09-18) - -**Note:** Version bump only for package @agoric/xs-vat-worker - - - - - -## [0.2.2](https://github.com/Agoric/agoric-sdk/compare/@agoric/xs-vat-worker@0.2.1...@agoric/xs-vat-worker@0.2.2) (2020-09-16) - -**Note:** Version bump only for package @agoric/xs-vat-worker - - - - - -## [0.2.1](https://github.com/Agoric/agoric-sdk/compare/@agoric/xs-vat-worker@0.2.0...@agoric/xs-vat-worker@0.2.1) (2020-08-31) - -**Note:** Version bump only for package @agoric/xs-vat-worker - - - - - -# 0.2.0 (2020-08-31) - - -### Features - -* **swingset-vat:** add xs-worker managerType ([2db022d](https://github.com/Agoric/agoric-sdk/commit/2db022d966a416c9b765c18ed543dd5adb31cc6d)) -* **xs-vat-worker:** locateWorkerBin finds built executable ([aecaeb1](https://github.com/Agoric/agoric-sdk/commit/aecaeb143668825183c5aa1b9a5c76d954b51501)) -* use npm style imports with XS build tools via compartmap ([903027a](https://github.com/Agoric/agoric-sdk/commit/903027a30299e9d9b03246bb0476bc4b94fddcf9)) diff --git a/packages/xs-vat-worker/package.json b/packages/xs-vat-worker/package.json deleted file mode 100644 index d00293ea5a9..00000000000 --- a/packages/xs-vat-worker/package.json +++ /dev/null @@ -1,81 +0,0 @@ -{ - "name": "@agoric/xs-vat-worker", - "version": "0.4.0", - "description": "swingset-vat worker for XS js runtime", - "parsers": { - "js": "mjs" - }, - "main": "src/locate.js", - "module": "src/locate.js", - "scripts": { - "test": "ava", - "build": "rollup --config rollup.config.js", - "lint-fix": "eslint --fix '**/*.{js,jsx}'", - "lint-check": "eslint '**/*.{js,jsx}'", - "lint-fix-jessie": "eslint -c '.eslintrc-jessie.js' --fix '**/*.{js,jsx}'", - "lint-check-jessie": "eslint -c '.eslintrc-jessie.js' '**/*.{js,jsx}'" - }, - "ava": { - "files": [ - "test/**/test-*.js" - ], - "require": [ - "esm" - ], - "timeout": "2m" - }, - "devDependencies": { - "@rollup/plugin-commonjs": "^11.0.2", - "@rollup/plugin-node-resolve": "^7.1.1", - "ava": "^3.12.1", - "eslint": "^6.8.0", - "eslint-config-airbnb-base": "^14.0.0", - "eslint-config-jessie": "^0.0.4", - "eslint-config-prettier": "^6.9.0", - "eslint-plugin-import": "^2.20.0", - "eslint-plugin-jsx-a11y": "^6.2.3", - "eslint-plugin-prettier": "^3.1.2", - "nyc": "^15.1.0", - "prettier": "^1.18.2", - "rollup": "^1.23.1" - }, - "dependencies": { - "@agoric/assert": "^0.2.0", - "@agoric/bundle-source": "^1.2.0", - "@agoric/eventual-send": "^0.13.0", - "@agoric/import-bundle": "^0.2.0", - "@agoric/marshal": "^0.3.0", - "@agoric/promise-kit": "^0.2.0", - "@agoric/swingset-vat": "^0.11.0", - "@agoric/xsnap": "^0.1.0", - "anylogger": "^0.21.0", - "esm": "^3.2.5", - "ses": "^0.11.0" - }, - "keywords": [], - "files": [ - "dist" - ], - "repository": { - "type": "git", - "url": "git+https://github.com/Agoric/agoric-sdk.git" - }, - "author": "Agoric", - "license": "Apache-2.0", - "bugs": { - "url": "https://github.com/Agoric/agoric-sdk/issues" - }, - "homepage": "https://github.com/Agoric/agoric-sdk#readme", - "publishConfig": { - "access": "public" - }, - "eslintConfig": { - "extends": [ - "@agoric" - ] - }, - "prettier": { - "trailingComma": "all", - "singleQuote": true - } -} diff --git a/packages/xs-vat-worker/src/locate.js b/packages/xs-vat-worker/src/locate.js deleted file mode 100644 index 41cd6a0549b..00000000000 --- a/packages/xs-vat-worker/src/locate.js +++ /dev/null @@ -1,16 +0,0 @@ -/** - * Locate the XS vat worker executable. - * - * Note: executable is not built by default. - * - * @see the `build:xs-lin` script in package.json - * - * @param {{ resolve: (...string) => string }} filesystem path access - * @returns { string } full path where linux debug executable is built; - * not guaranteed to exist. - */ -export function locateWorkerBin({ resolve }) { - const goal = 'debug'; // ISSUE: support, test release too? - const os = 'lin'; // ISSUE: support, test mac too? - return resolve(__dirname, '../build/bin', os, goal, 'xs-vat-worker'); -} diff --git a/packages/xs-vat-worker/src/vatWorker.js b/packages/xs-vat-worker/src/vatWorker.js deleted file mode 100644 index 0e14053bab6..00000000000 --- a/packages/xs-vat-worker/src/vatWorker.js +++ /dev/null @@ -1,210 +0,0 @@ -/* global HandledPromise */ -import { importBundle } from '@agoric/import-bundle'; -import { Remotable, getInterfaceOf, makeMarshal } from '@agoric/marshal'; -// TODO? import anylogger from 'anylogger'; -import { makeLiveSlots } from '@agoric/swingset-vat/src/kernel/liveSlots'; - -function workerLog(first, ...args) { - console.log(`---worker: ${first}`, ...args); -} - -function assert(ok, whynot) { - if (!ok) { - throw new Error(whynot); - } -} - -function makeConsole(_tag) { - const log = console; // TODO? anylogger(tag); - const cons = {}; - for (const level of ['debug', 'log', 'info', 'warn', 'error']) { - cons[level] = log[level]; - } - return harden(cons); -} - -// see also: detecting an empty vat promise queue (end of "crank") -// https://github.com/Agoric/agoric-sdk/issues/45 -function waitUntilQuiescent(setImmediate) { - return new Promise((resolve, _reject) => { - setImmediate(() => { - // console.log('hello from setImmediate callback. The promise queue is presumably empty.'); - resolve(); - }); - }); -} - -function runAndWait(f, errmsg, setImmediate) { - Promise.resolve() - .then(f) - .then(undefined, err => workerLog(`doProcess: ${errmsg}:`, err)); - return waitUntilQuiescent(setImmediate); -} - -function makeWorker(io, setImmediate) { - let dispatch; - - async function doProcess(dispatchRecord, errmsg) { - const dispatchOp = dispatchRecord[0]; - const dispatchArgs = dispatchRecord.slice(1); - workerLog(`runAndWait`); - await runAndWait( - () => dispatch[dispatchOp](...dispatchArgs), - errmsg, - setImmediate, - ); - workerLog(`doProcess done`); - const vatDeliveryResults = harden(['ok']); - return vatDeliveryResults; - } - - function doMessage(targetSlot, msg) { - const errmsg = `vat[${targetSlot}].${msg.method} dispatch failed`; - return doProcess( - ['deliver', targetSlot, msg.method, msg.args, msg.result], - errmsg, - ); - } - - function doNotify(vpid, vp) { - const errmsg = `vat.promise[${vpid}] ${vp.state} failed`; - switch (vp.state) { - case 'fulfilledToPresence': - return doProcess(['notifyFulfillToPresence', vpid, vp.slot], errmsg); - case 'redirected': - throw new Error('not implemented yet'); - case 'fulfilledToData': - return doProcess(['notifyFulfillToData', vpid, vp.data], errmsg); - case 'rejected': - return doProcess(['notifyReject', vpid, vp.data], errmsg); - default: - throw Error(`unknown promise state '${vp.state}'`); - } - } - - function sendUplink(msg) { - assert(msg instanceof Array, `msg must be an Array`); - io.writeMessage(JSON.stringify(msg)); - } - - // fromParent.on('data', data => { - // workerLog('data from parent', data); - // toParent.write('child ack'); - // }); - - const handle = harden(async ([type, ...margs]) => { - workerLog(`received`, type); - if (type === 'start') { - // TODO: parent should send ['start', vatID] - workerLog(`got start`); - sendUplink(['gotStart']); - } else if (type === 'setBundle') { - const [bundle, vatParameters, virtualObjectCacheSize] = margs; - const endowments = { - console: makeConsole(`SwingSet:vatWorker`), - assert, - HandledPromise, - }; - // ISSUE: this draft code is contorted because it started - // as code that didn't return anything but now it - // has to return a promise to be resolved before - // reading the next input. - return importBundle(bundle, { endowments }).then(vatNS => { - workerLog(`got vatNS:`, Object.keys(vatNS).join(',')); - sendUplink(['gotBundle']); - - function doSyscall(vatSyscallObject) { - sendUplink(['syscall', ...vatSyscallObject]); - } - const syscall = harden({ - send: (...args) => doSyscall(['send', ...args]), - callNow: (..._args) => { - throw Error(`nodeWorker cannot syscall.callNow`); - }, - subscribe: (...args) => doSyscall(['subscribe', ...args]), - fulfillToData: (...args) => doSyscall(['fulfillToData', ...args]), - fulfillToPresence: (...args) => - doSyscall(['fulfillToPresence', ...args]), - reject: (...args) => doSyscall(['reject', ...args]), - }); - - function testLog(...args) { - sendUplink(['testLog', ...args]); - } - const state = null; - const vatID = 'demo-vatID'; - // todo: maybe add transformTildot, makeGetMeter/transformMetering to - // vatPowers, but only if options tell us they're wanted. Maybe - // transformTildot should be async and outsourced to the kernel - // process/thread. - const vatPowers = { - Remotable, - getInterfaceOf, - makeMarshal, - testLog, - }; - dispatch = makeLiveSlots( - syscall, - state, - vatNS.buildRootObject, - vatID, - vatPowers, - vatParameters, - virtualObjectCacheSize, - ); - workerLog(`got dispatch:`, Object.keys(dispatch).join(',')); - sendUplink(['dispatchReady']); - return type; - }); - } else if (type === 'deliver') { - if (!dispatch) { - workerLog(`error: deliver before dispatchReady`); - return undefined; - } - const [dtype, ...dargs] = margs; - if (dtype === 'message') { - await doMessage(...dargs).then(res => - sendUplink(['deliverDone', ...res]), - ); - } else if (dtype === 'notify') { - await doNotify(...dargs).then(res => - sendUplink(['deliverDone', ...res]), - ); - } else { - throw Error(`bad delivery type ${dtype}`); - } - } else { - workerLog(`unrecognized downlink message ${type}`); - } - return type; - }); - - return harden({ handle }); -} - -export async function main({ readMessage, writeMessage, setImmediate }) { - workerLog(`supervisor started`); - - const worker = makeWorker({ readMessage, writeMessage }, setImmediate); - const EOF = new Error('EOF'); - - for (;;) { - let message; - try { - // eslint-disable-next-line no-await-in-loop - message = JSON.parse(readMessage(EOF)); - } catch (noMessage) { - if (noMessage === EOF) { - return; - } - console.warn('problem getting message:', noMessage); - // eslint-disable-next-line no-continue - continue; - } - // eslint-disable-next-line no-await-in-loop - const msgtype = await worker.handle(message); - if (msgtype === 'finish') { - break; - } - } -} diff --git a/packages/xsnap/package.json b/packages/xsnap/package.json index fa36230967c..2088c60d04d 100644 --- a/packages/xsnap/package.json +++ b/packages/xsnap/package.json @@ -13,7 +13,9 @@ }, "scripts": { "repl": "node -r esm src/xsrepl.js", - "build": "git submodule update --init && node -r esm src/build.js", + "build:bundle": "rollup --config rollup.config.js", + "build:bin": "git submodule update --init && node -r esm src/build.js", + "build": "yarn build:bin && yarn build:bundle", "clean": "rm -rf build", "lint": "yarn lint:js && yarn lint:types", "lint:js": "eslint 'src/**/*.js'", @@ -24,6 +26,8 @@ "postinstall": "yarn build" }, "dependencies": { + "@agoric/eventual-send": "^0.13.0", + "ses": "^0.11.0", "esm": "^3.2.5" }, "devDependencies": { diff --git a/packages/xs-vat-worker/rollup.config.js b/packages/xsnap/rollup.config.js similarity index 100% rename from packages/xs-vat-worker/rollup.config.js rename to packages/xsnap/rollup.config.js diff --git a/packages/xs-vat-worker/src/bootstrap.js b/packages/xsnap/src/bootstrap.js similarity index 100% rename from packages/xs-vat-worker/src/bootstrap.js rename to packages/xsnap/src/bootstrap.js diff --git a/packages/xs-vat-worker/src/console-shim.js b/packages/xsnap/src/console-shim.js similarity index 100% rename from packages/xs-vat-worker/src/console-shim.js rename to packages/xsnap/src/console-shim.js diff --git a/packages/xs-vat-worker/src/lockdown-shim.js b/packages/xsnap/src/lockdown-shim.js similarity index 72% rename from packages/xs-vat-worker/src/lockdown-shim.js rename to packages/xsnap/src/lockdown-shim.js index a67d383cbeb..25d481feac8 100644 --- a/packages/xs-vat-worker/src/lockdown-shim.js +++ b/packages/xsnap/src/lockdown-shim.js @@ -1,3 +1,4 @@ +// @ts-ignore import 'ses/lockdown'; lockdown(); diff --git a/packages/xs-vat-worker/src/text-shim.js b/packages/xsnap/src/text-shim.js similarity index 100% rename from packages/xs-vat-worker/src/text-shim.js rename to packages/xsnap/src/text-shim.js diff --git a/packages/xs-vat-worker/test/escapeCompartment.js b/packages/xsnap/test/escapeCompartment.js similarity index 100% rename from packages/xs-vat-worker/test/escapeCompartment.js rename to packages/xsnap/test/escapeCompartment.js diff --git a/packages/xs-vat-worker/test/test-boot-lockdown.js b/packages/xsnap/test/test-boot-lockdown.js similarity index 97% rename from packages/xs-vat-worker/test/test-boot-lockdown.js rename to packages/xsnap/test/test-boot-lockdown.js index 9e9b1ac5393..d649487a7b2 100644 --- a/packages/xs-vat-worker/test/test-boot-lockdown.js +++ b/packages/xsnap/test/test-boot-lockdown.js @@ -3,7 +3,7 @@ import * as childProcess from 'child_process'; import * as os from 'os'; import * as fs from 'fs'; import * as path from 'path'; -import { xsnap } from '@agoric/xsnap'; +import { xsnap } from '../src/xsnap'; const importModuleUrl = `file://${__filename}`; From 1ebf69988f4a8d5016681d92535353f659a2ceae Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Sat, 23 Jan 2021 11:05:21 -0600 Subject: [PATCH 19/23] chore(xsnap): move lockdown-shim out of src/ to avoid tsc errors move lockdown-shim.js and the rest of the SES bootstrap files from src/ to lib/ to avoid many tsc errors of the form... ``` Error: ../../node_modules/ses/src/error/assert.js(24,20): error TS2304: Cannot find name 'StringablePayload'. ``` --- .../src/kernel/vatManager/lockdown-subprocess-xsnap.js | 2 +- packages/xsnap/{src => lib}/bootstrap.js | 0 packages/xsnap/{src => lib}/console-shim.js | 0 packages/xsnap/{src => lib}/lockdown-shim.js | 0 packages/xsnap/{src => lib}/text-shim.js | 0 packages/xsnap/package.json | 4 ++-- packages/xsnap/rollup.config.js | 2 +- 7 files changed, 4 insertions(+), 4 deletions(-) rename packages/xsnap/{src => lib}/bootstrap.js (100%) rename packages/xsnap/{src => lib}/console-shim.js (100%) rename packages/xsnap/{src => lib}/lockdown-shim.js (100%) rename packages/xsnap/{src => lib}/text-shim.js (100%) diff --git a/packages/SwingSet/src/kernel/vatManager/lockdown-subprocess-xsnap.js b/packages/SwingSet/src/kernel/vatManager/lockdown-subprocess-xsnap.js index ed9300404b5..577e316917d 100644 --- a/packages/SwingSet/src/kernel/vatManager/lockdown-subprocess-xsnap.js +++ b/packages/SwingSet/src/kernel/vatManager/lockdown-subprocess-xsnap.js @@ -1 +1 @@ -import '@agoric/xsnap/src/bootstrap'; +import '@agoric/xsnap/lib/bootstrap'; diff --git a/packages/xsnap/src/bootstrap.js b/packages/xsnap/lib/bootstrap.js similarity index 100% rename from packages/xsnap/src/bootstrap.js rename to packages/xsnap/lib/bootstrap.js diff --git a/packages/xsnap/src/console-shim.js b/packages/xsnap/lib/console-shim.js similarity index 100% rename from packages/xsnap/src/console-shim.js rename to packages/xsnap/lib/console-shim.js diff --git a/packages/xsnap/src/lockdown-shim.js b/packages/xsnap/lib/lockdown-shim.js similarity index 100% rename from packages/xsnap/src/lockdown-shim.js rename to packages/xsnap/lib/lockdown-shim.js diff --git a/packages/xsnap/src/text-shim.js b/packages/xsnap/lib/text-shim.js similarity index 100% rename from packages/xsnap/src/text-shim.js rename to packages/xsnap/lib/text-shim.js diff --git a/packages/xsnap/package.json b/packages/xsnap/package.json index 2088c60d04d..6ac559b2b02 100644 --- a/packages/xsnap/package.json +++ b/packages/xsnap/package.json @@ -18,9 +18,9 @@ "build": "yarn build:bin && yarn build:bundle", "clean": "rm -rf build", "lint": "yarn lint:js && yarn lint:types", - "lint:js": "eslint 'src/**/*.js'", + "lint:js": "eslint 'src/**/*.js' 'lib/**/*.js'", "lint:types": "tsc -p jsconfig.json", - "lint-fix": "eslint --fix 'src/**/*.js'", + "lint-fix": "eslint --fix 'src/**/*.js' 'lib/**/*.js'", "lint-check": "yarn lint", "test": "ava", "postinstall": "yarn build" diff --git a/packages/xsnap/rollup.config.js b/packages/xsnap/rollup.config.js index aad39650032..84295a289e7 100644 --- a/packages/xsnap/rollup.config.js +++ b/packages/xsnap/rollup.config.js @@ -3,7 +3,7 @@ import commonjs from '@rollup/plugin-commonjs'; export default [ { - input: 'src/bootstrap.js', + input: 'lib/bootstrap.js', output: { file: `dist/bootstrap.umd.js`, format: 'umd', From aab7d0daf8bba995bf1e811bb29484db46bf696c Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Sat, 23 Jan 2021 11:19:34 -0600 Subject: [PATCH 20/23] docs(xsnap): document XS handleCommand async idiom --- .../vatManager/supervisor-subprocess-xsnap.js | 54 +++++++++++-------- packages/xsnap/README.md | 4 ++ 2 files changed, 36 insertions(+), 22 deletions(-) diff --git a/packages/SwingSet/src/kernel/vatManager/supervisor-subprocess-xsnap.js b/packages/SwingSet/src/kernel/vatManager/supervisor-subprocess-xsnap.js index dee76357452..fb95c892602 100644 --- a/packages/SwingSet/src/kernel/vatManager/supervisor-subprocess-xsnap.js +++ b/packages/SwingSet/src/kernel/vatManager/supervisor-subprocess-xsnap.js @@ -20,12 +20,12 @@ function workerLog(first, ...args) { workerLog(`supervisor started`); /** - * @param { (cmd: ArrayBuffer) => ArrayBuffer } issueCommand - * @typedef { [unknown, ...unknown[]] } Tagged - * @typedef { (item: Tagged) => unknown } DataHandler - * @typedef { (item: Tagged) => Promise } AsyncHandler + * Wrap byte-level protocols with tagged array codec. + * + * @param { (cmd: ArrayBuffer) => ArrayBuffer } issueCommand as from xsnap + * @typedef { [unknown, ...unknown[]] } Tagged tagged array */ -function ManagerPort(issueCommand) { +function managerPort(issueCommand) { /** @type { (item: Tagged) => ArrayBuffer } */ const encode = item => encoder.encode(JSON.stringify(item)).buffer; @@ -46,20 +46,31 @@ function ManagerPort(issueCommand) { send: item => { issueCommand(encode(item)); }, - /** @type { DataHandler } */ + /** @type { (item: Tagged) => unknown } */ call: item => decodeData(issueCommand(encode(item))), - /** @type { (f: AsyncHandler) => ((msg: ArrayBuffer) => { result?: ArrayBuffer })} */ - handler: f => msg => { - const report = {}; - f(decode(msg)) - .then(item => { - workerLog('result', item); - report.result = encode(item); - }) - .catch(err => { - report.result = encode(['err', err.message]); - }); - return report; + + /** + * Wrap an async Tagged handler in the xsnap async reporting idiom. + * + * @param { (item: Tagged) => Promise } f async Tagged handler + * @returns { (msg: ArrayBuffer) => Report } xsnap style handleCommand + * + * @typedef { { result?: T } } Report report T when idle + * @template T + */ + handlerFrom(f) { + return msg => { + const report = {}; + f(decode(msg)) + .then(item => { + workerLog('result', item); + report.result = encode(item); + }) + .catch(err => { + report.result = encode(['err', err.message]); + }); + return report; + }; }, }); } @@ -76,7 +87,7 @@ function runAndWait(f, errmsg) { } /** - * @param { ReturnType } port + * @param { ReturnType } port */ function makeWorker(port) { /** @type { Record void> | null } */ @@ -214,13 +225,12 @@ function makeWorker(port) { } return harden({ - /** @type { AsyncHandler } */ handleItem, }); } // @ts-ignore xsnap provides issueCommand global // eslint-disable-next-line no-undef -const port = ManagerPort(issueCommand); +const port = managerPort(issueCommand); const worker = makeWorker(port); -globalThis.handleCommand = port.handler(worker.handleItem); +globalThis.handleCommand = port.handlerFrom(worker.handleItem); diff --git a/packages/xsnap/README.md b/packages/xsnap/README.md index 9bf906a7121..47074ceb4e8 100644 --- a/packages/xsnap/README.md +++ b/packages/xsnap/README.md @@ -34,6 +34,10 @@ The parent and child communicate using "commands". and receive as response from the Node.js parent. - The XS child can implement a synchronous `handleCommand` function to respond to commands from the Node.js parent. + - The XS child `handleCommand` may be asynchronous after a fashion: it + may return an object and, before the promise queue becomes empty, + set the `result` property of this object to an `ArrayBuffer`. + See the **evaluate and report** test for an example. - The Node.js parent uses an asynchronous `issueCommand` method to send a request and receive a response from the XS child. - The Node.js parent can implement an asynchronous `handleCommand` function to From 701e2f49930306637420e8bb9f350351a3285f6a Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Sat, 23 Jan 2021 11:59:03 -0600 Subject: [PATCH 21/23] refactor: build XS bundles with Kernel bundles --- packages/SwingSet/src/controller.js | 22 +++------- packages/SwingSet/src/initializeSwingset.js | 42 +++++++++++-------- .../vatManager/manager-subprocess-xsnap.js | 5 +-- 3 files changed, 31 insertions(+), 38 deletions(-) diff --git a/packages/SwingSet/src/controller.js b/packages/SwingSet/src/controller.js index 9a74198879e..fbd75661a5b 100644 --- a/packages/SwingSet/src/controller.js +++ b/packages/SwingSet/src/controller.js @@ -12,7 +12,6 @@ import anylogger from 'anylogger'; import { assert } from '@agoric/assert'; import { isTamed, tameMetering } from '@agoric/tame-metering'; import { importBundle } from '@agoric/import-bundle'; -import bundleSource from '@agoric/bundle-source'; import { initSwingStore } from '@agoric/swing-store-simple'; import { makeMeteringTransformer } from '@agoric/transform-metering'; import { makeTransform } from '@agoric/transform-eventual-send'; @@ -38,20 +37,6 @@ function makeConsole(tag) { return harden(cons); } -async function buildXsBundles() { - const zip = (xs, ys) => xs.map((x, i) => [x, ys[i]]); - const { keys, values, fromEntries } = Object; - const allValues = async obj => - fromEntries(zip(keys(obj), await Promise.all(values(obj)))); - const src = rel => bundleSource(require.resolve(rel), 'getExport'); - return harden( - await allValues({ - lockdown: src('./kernel/vatManager/lockdown-subprocess-xsnap.js'), - supervisor: src('./kernel/vatManager/supervisor-subprocess-xsnap.js'), - }), - ); -} - export async function makeSwingsetController( hostStorage = initSwingStore().storage, deviceEndowments = {}, @@ -173,7 +158,6 @@ export async function makeSwingsetController( return startSubprocessWorker(process.execPath, ['-r', 'esm', supercode]); } - const { xsnapBundles = await buildXsBundles() } = {}; // @@@ options? const startXSnap = (name, handleCommand) => { const worker = xsnap({ os: osType(), @@ -185,7 +169,11 @@ export async function makeSwingsetController( // debug: true, }); - return harden({ worker, bundles: xsnapBundles }); + const bundles = { + lockdown: JSON.parse(hostStorage.get('lockdownBundle')), + supervisor: JSON.parse(hostStorage.get('supervisorBundle')), + }; + return harden({ worker, bundles }); }; const slogF = diff --git a/packages/SwingSet/src/initializeSwingset.js b/packages/SwingSet/src/initializeSwingset.js index 5a290f36f43..bad03ff08ac 100644 --- a/packages/SwingSet/src/initializeSwingset.js +++ b/packages/SwingSet/src/initializeSwingset.js @@ -8,27 +8,33 @@ import { initSwingStore } from '@agoric/swing-store-simple'; import { insistStorageAPI } from './storageAPI'; import { initializeKernel } from './kernel/initializeKernel'; +const zip = (xs, ys) => xs.map((x, i) => [x, ys[i]]); +const { keys, values, fromEntries } = Object; +const allValues = async obj => + fromEntries(zip(keys(obj), await Promise.all(values(obj)))); + /** - * Build the kernel source bundles. - * + * Build the source bundles for the kernel and xsnap vat worker. */ export async function buildKernelBundles() { // this takes 2.7s on my computer - const sources = { - kernel: require.resolve('./kernel/kernel.js'), - adminDevice: require.resolve('./kernel/vatAdmin/vatAdmin-src'), - adminVat: require.resolve('./kernel/vatAdmin/vatAdminWrapper'), - comms: require.resolve('./vats/comms'), - vattp: require.resolve('./vats/vat-tp'), - timer: require.resolve('./vats/vat-timerWrapper'), - }; - const kernelBundles = {}; - for (const name of Object.keys(sources)) { - // this was harder to read with Promise.all - // eslint-disable-next-line no-await-in-loop - kernelBundles[name] = await bundleSource(sources[name]); - } - return harden(kernelBundles); + + const src = rel => bundleSource(require.resolve(rel)); + const srcGE = rel => bundleSource(require.resolve(rel), 'getExport'); + + const bundles = await allValues({ + kernel: src('./kernel/kernel.js'), + adminDevice: src('./kernel/vatAdmin/vatAdmin-src'), + adminVat: src('./kernel/vatAdmin/vatAdminWrapper'), + comms: src('./vats/comms'), + vattp: src('./vats/vat-tp'), + timer: src('./vats/vat-timerWrapper'), + + lockdown: srcGE('./kernel/vatManager/lockdown-subprocess-xsnap.js'), + supervisor: srcGE('./kernel/vatManager/supervisor-subprocess-xsnap.js'), + }); + + return harden(bundles); } function byName(a, b) { @@ -246,6 +252,8 @@ export async function initializeSwingset( const { kernelBundles = await buildKernelBundles() } = initializationOptions; hostStorage.set('kernelBundle', JSON.stringify(kernelBundles.kernel)); + hostStorage.set('lockdownBundle', JSON.stringify(kernelBundles.lockdown)); + hostStorage.set('supervisorBundle', JSON.stringify(kernelBundles.supervisor)); if (config.bootstrap && argv) { if (config.vats[config.bootstrap]) { diff --git a/packages/SwingSet/src/kernel/vatManager/manager-subprocess-xsnap.js b/packages/SwingSet/src/kernel/vatManager/manager-subprocess-xsnap.js index 9313768b6ff..677d72a16b1 100644 --- a/packages/SwingSet/src/kernel/vatManager/manager-subprocess-xsnap.js +++ b/packages/SwingSet/src/kernel/vatManager/manager-subprocess-xsnap.js @@ -117,10 +117,7 @@ export function makeXsSubprocessFactory({ superCode.moduleFormat === 'getExport', details`${it} unexpected: ${superCode.moduleFormat}`, ); - await worker.evaluate( - `(${superCode.source} - )()`.trim(), - ); + await worker.evaluate(`(${superCode.source}\n)()`.trim()); } /** @type { (item: Tagged) => Promise } */ From d200428c336ff1b0fc1dfdfd49b10d7d1c9f81a1 Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Sat, 23 Jan 2021 20:07:11 -0600 Subject: [PATCH 22/23] fix(xsnap worker): update syscall API to use .resolve() --- .../src/kernel/vatManager/supervisor-subprocess-xsnap.js | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/SwingSet/src/kernel/vatManager/supervisor-subprocess-xsnap.js b/packages/SwingSet/src/kernel/vatManager/supervisor-subprocess-xsnap.js index fb95c892602..e778b6188d2 100644 --- a/packages/SwingSet/src/kernel/vatManager/supervisor-subprocess-xsnap.js +++ b/packages/SwingSet/src/kernel/vatManager/supervisor-subprocess-xsnap.js @@ -160,9 +160,7 @@ function makeWorker(port) { send: (...args) => doSyscall(['send', ...args]), callNow: (...args) => doSyscall(['callNow', ...args]), subscribe: (...args) => doSyscall(['subscribe', ...args]), - fulfillToData: (...args) => doSyscall(['fulfillToData', ...args]), - fulfillToPresence: (...args) => doSyscall(['fulfillToPresence', ...args]), - reject: (...args) => doSyscall(['reject', ...args]), + resolve: (...args) => doSyscall(['resolve', ...args]), }); const vatPowers = { From 37a658cf3ae9e0afa3286ce1da2d5a1c23ed1e3c Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Sat, 23 Jan 2021 20:35:09 -0600 Subject: [PATCH 23/23] chore(xsnap): provide non-trivial console in start compartment add TODO re other console methods with pointer to https://github.com/Agoric/agoric-sdk/issues/2146 --- .../vatManager/supervisor-subprocess-xsnap.js | 6 ++++ packages/xsnap/lib/console-shim.js | 35 ++++++++++++++----- 2 files changed, 32 insertions(+), 9 deletions(-) diff --git a/packages/SwingSet/src/kernel/vatManager/supervisor-subprocess-xsnap.js b/packages/SwingSet/src/kernel/vatManager/supervisor-subprocess-xsnap.js index e778b6188d2..5080039435c 100644 --- a/packages/SwingSet/src/kernel/vatManager/supervisor-subprocess-xsnap.js +++ b/packages/SwingSet/src/kernel/vatManager/supervisor-subprocess-xsnap.js @@ -122,6 +122,12 @@ function makeWorker(port) { return doProcess(['notify', resolutions], errmsg); } + /** + * TODO: consider other methods per SES VirtualConsole. + * See https://github.com/Agoric/agoric-sdk/issues/2146 + * + * @param { string } tag + */ function makeConsole(tag) { const log = level => (...args) => port.send(['console', level, tag, ...args]); diff --git a/packages/xsnap/lib/console-shim.js b/packages/xsnap/lib/console-shim.js index 230aa2015b4..91479c61ddd 100644 --- a/packages/xsnap/lib/console-shim.js +++ b/packages/xsnap/lib/console-shim.js @@ -1,17 +1,34 @@ -const noop = _ => undefined; +function tryPrint(...args) { + try { + // eslint-disable-next-line + print(...args); + } catch (err) { + // eslint-disable-next-line + print('cannot print:', err.message); + } +} +const noop = _ => {}; + +/** + * Since SES expects (requires?) a console, + * provide one based on xsnap's print. + * Note that this runs in the start compartment, + * before lockdown. + * + * TODO: consider other methods per SES VirtualConsole. + * See https://github.com/Agoric/agoric-sdk/issues/2146 + */ const console = { - debug: noop, - log: noop, - info: noop, - warn: noop, - error: noop, - // used by SES + debug: tryPrint, + log: tryPrint, + info: tryPrint, + warn: tryPrint, + error: tryPrint, + group: noop, groupCollapsed: noop, groupEnd: noop, - // others from the MDN / whatwg Console API? - // trace, dirxml, ... }; globalThis.console = console;