feat(swingset): persist xsnap bundle IDs at vat creation/upgrade

When launching an xsnap worker, we must load two special bundles into
the worker's empty JS engine: xsnap-lockdown (which turns it into
a SES environment), and swingset-xsnap-supervisor (which loads
liveslots and the command handlers). We get these bundles from
packages of the same names.

Previously, we would load the current versions of these bundles each
time the controller/kernel started, and use them for all vats started
or upgraded during that incarnation of the controller (i.e. until the
process exited). This did not maintain stable behavior for a vat which
e.g. does not record a heap snapshot before its initial controller
incarnation finishes: if the xsnap-lockdown package were updated
between controller incarnations, the second incarnation would launch
the worker with a different bundle than the first, potentially causing
divergence between the vat's original behavior (as recorded in its
transcript) and the replay. If there were any local-manager liveslots
-based vats, these would suffer the same problem on every kernel
reboot, since only xsnap-hosted vats can use heap snapshots.

With this commit, the lockdown and supervisor bundles are stored in
the swing-store `bundleStore`, and their IDs are recorded in the
metadata for each vat. We check the bundle-providing packages each
time we start a new vat (or upgrade an existing one), so the new
worker will use the latest bundles. But by recording the IDs
separately for each vat, we ensure that every launch of that vat
worker will the same bundles. This stability will help us maintain the
deterministic behavior of a vat despite changes to the packages
depended upon by any given version of the kernel.

Unit tests and replay tools can use `options.overrideBundles=[..]` to
replace the usual bundles with alternates: either no bundles, or
custom lockdown/supervisor bundles.

closes #7208

packages/SwingSet/misc-tools/replay-transcript.js

@@ -158,10 +158,11 @@ async function replay(transcriptFile) {
       xsnapPID = child.pid;
       return child;
     };
-    const startXSnap = makeStartXSnap(bundles, {
+    const startXSnap = makeStartXSnap({
       snapStore,
       spawn: capturePIDSpawn,
       workerTraceRootPath: RECORD_XSNAP_TRACE ? process.cwd() : undefined,
+      overrideBundles: bundles,
     });
     factory = makeXsSubprocessFactory({
       kernelKeeper: fakeKernelKeeper,

packages/SwingSet/src/controller/bundle-handler.js

@@ -0,0 +1,59 @@
+import { getLockdownBundleSHA256, getLockdownBundle } from '@agoric/xsnap-lockdown';
+import { getSupervisorBundleSHA256, getSupervisorBundle } from '@agoric/swingset-xsnap-supervisor';
+
+
+export const makeXsnapBundleData = harden(() => {
+  return harden({
+    getLockdownBundleSHA256,
+    getLockdownBundle,
+    getSupervisorBundleSHA256,
+    getSupervisorBundle,
+  });
+});
+
+/**
+ * @typedef {import('../types-external.js').BundleID} BundleID
+ * @typedef {import('../types-external.js').Bundle} Bundle
+ *
+ * @typedef {object} BundleHandler
+ * @property {() => Promise<BundleID[]>} getCurrentBundleIDs
+ * @property {(id: BundleID) => Promise<Bundle>} getBundle
+ */
+
+/**
+ * @param {import('@agoric/swing-store').BundleStore} bundleStore
+ * @param {ReturnType<makeXsnapBundleData>} bundleData
+ * @returns {BundleHandler}
+ */
+export const makeWorkerBundleHandler = harden((bundleStore, bundleData) => {
+  const {
+    getLockdownBundleSHA256,
+    getLockdownBundle,
+    getSupervisorBundleSHA256,
+    getSupervisorBundle,
+  } = bundleData;
+
+  return harden({
+    getCurrentBundleIDs: async () => {
+      const lockdownHash = await getLockdownBundleSHA256();
+      const lockdownID = `b0-${lockdownHash}`;
+      if (!bundleStore.hasBundle(lockdownID)) {
+        const lockdownBundle = await getLockdownBundle();
+        bundleStore.addBundle(lockdownID, lockdownBundle);
+      }
+
+      const supervisorHash = await getSupervisorBundleSHA256();
+      const supervisorID = `b0-${supervisorHash}`;
+      if (!bundleStore.hasBundle(supervisorID)) {
+        const supervisorBundle = await getSupervisorBundle();
+        bundleStore.addBundle(supervisorID, supervisorBundle);
+      }
+
+      return [lockdownID, supervisorID]; // order is important
+    },
+    getBundle: async id => {
+      return bundleStore.getBundle(id);
+    },
+  });
+});
+

packages/SwingSet/src/controller/controller.js

@@ -23,6 +23,10 @@ import {
   swingsetIsInitialized,
   initializeSwingset,
 } from './initializeSwingset.js';
+import {
+  makeWorkerBundleHandler,
+  makeXsnapBundleData,
+} from './bundle-handler.js';
 import { makeStartXSnap } from './startXSnap.js';
 
 /** @param {Uint8Array} bytes */
@@ -87,6 +91,8 @@ function unhandledRejectionHandler(e, pr) {
  *   spawn?: typeof import('child_process').spawn,
  *   env?: Record<string, string | undefined>,
  *   kernelBundle?: Bundle
+ *   xsnapBundleData?: ReturnType<import('./bundle-handler.js').makeXsnapBundleData>,
+ *   bundleHandler?: import('./bundle-handler.js').BundleHandler,
  * }} runtimeOptions
  */
 export async function makeSwingsetController(
@@ -109,11 +115,27 @@ export async function makeSwingsetController(
     spawn = ambientSpawn,
     warehousePolicy = {},
     overrideVatManagerOptions = {},
+    xsnapBundleData = makeXsnapBundleData(),
+  } = runtimeOptions;
+  const {
+    bundleHandler = makeWorkerBundleHandler(
+      kernelStorage.bundleStore,
+      xsnapBundleData,
+    ),
   } = runtimeOptions;
+
   if (typeof Compartment === 'undefined') {
     throw Error('SES must be installed before calling makeSwingsetController');
   }
 
+  const startXSnap = makeStartXSnap({
+    bundleHandler,
+    snapStore: kernelStorage.snapStore,
+    spawn,
+    debug: !!env.XSNAP_DEBUG,
+    workerTraceRootPath: env.XSNAP_TEST_RECORD,
+  });
+
   function writeSlogObject(obj) {
     if (!slogSender) {
       // Fast path; nothing to do.
@@ -180,19 +202,6 @@ export async function makeSwingsetController(
   // all vats get these in their global scope, plus a vat-specific 'console'
   const vatEndowments = harden({});
 
-  const bundles = [
-    // @ts-ignore assume lockdownBundle is set
-    JSON.parse(kvStore.get('lockdownBundle')),
-    // @ts-ignore assume supervisorBundle is set
-    JSON.parse(kvStore.get('supervisorBundle')),
-  ];
-  const startXSnap = makeStartXSnap(bundles, {
-    snapStore: kernelStorage.snapStore,
-    spawn,
-    debug: !!env.XSNAP_DEBUG,
-    workerTraceRootPath: env.XSNAP_TEST_RECORD,
-  });
-
   const kernelEndowments = {
     waitUntilQuiescent,
     kernelStorage,
@@ -205,6 +214,7 @@ export async function makeSwingsetController(
     WeakRef,
     FinalizationRegistry,
     gcAndFinalize: makeGcAndFinalize(engineGC),
+    bundleHandler,
   };
 
   const kernelRuntimeOptions = {

packages/SwingSet/src/controller/initializeKernel.js

@@ -17,8 +17,10 @@ function makeVatRootObjectSlot() {
 }
 
 export async function initializeKernel(config, kernelStorage, options = {}) {
-  await 0; // no sync prelude
-  const { verbose = false } = options;
+  const {
+    verbose = false,
+    bundleHandler, // required if config has xsnap-based static vats
+  } = options;
   const logStartup = verbose ? console.debug : () => 0;
   insistStorageAPI(kernelStorage.kvStore);
 
@@ -90,7 +92,11 @@ export async function initializeKernel(config, kernelStorage, options = {}) {
         ...otherOptions
       } = creationOptions;
       // eslint-disable-next-line @jessie.js/no-nested-await,no-await-in-loop
-      const workerOptions = await makeWorkerOptions(kernelKeeper, managerType);
+      const workerOptions = await makeWorkerOptions(
+        kernelKeeper,
+        bundleHandler,
+        managerType,
+      );
       /** @type {import('../types-internal.js').RecordedVatOptions} */
       const vatOptions = harden({
         name,

packages/SwingSet/src/controller/initializeSwingset.js

@@ -4,13 +4,15 @@ import path from 'path';
 
 import { assert, Fail } from '@agoric/assert';
 import { makeTracer } from '@agoric/internal';
-import { getLockdownBundle } from '@agoric/xsnap-lockdown';
-import { getSupervisorBundle } from '@agoric/swingset-xsnap-supervisor';
 import bundleSource from '@endo/bundle-source';
 import { resolve as resolveModuleSpecifier } from 'import-meta-resolve';
 import { kdebugEnable } from '../lib/kdebug.js';
 import { insistStorageAPI } from '../lib/storageAPI.js';
 import { initializeKernel } from './initializeKernel.js';
+import {
+  makeWorkerBundleHandler,
+  makeXsnapBundleData,
+} from './bundle-handler.js';
 
 import '../types-ambient.js';
 import { makeNodeBundleCache } from '../../tools/bundleTool.js';
@@ -53,17 +55,12 @@ export async function buildKernelBundle() {
  * xsnap vat worker.
  */
 export async function buildVatAndDeviceBundles() {
-  const lockdownP = getLockdownBundle(); // throws if bundle is not built
-  const supervisorP = getSupervisorBundle(); // ditto
   const bundles = await allValues({
     adminDevice: bundleRelative('../devices/vat-admin/device-vat-admin.js'),
     adminVat: bundleRelative('../vats/vat-admin/vat-vat-admin.js'),
     comms: bundleRelative('../vats/comms/index.js'),
     vattp: bundleRelative('../vats/vattp/vat-vattp.js'),
     timer: bundleRelative('../vats/timer/vat-timer.js'),
-
-    lockdown: lockdownP,
-    supervisor: supervisorP,
   });
 
   return harden(bundles);
@@ -290,7 +287,9 @@ function sortObjectProperties(obj, firsts = []) {
  * @param {unknown} bootstrapArgs
  * @param {SwingStoreKernelStorage} kernelStorage
  * @param {InitializationOptions} initializationOptions
- * @param {{ env?: Record<string, string | undefined > }} runtimeOptions
+ * @param {{ env?: Record<string, string | undefined >,
+ *           bundleHandler?: import('./bundle-handler.js').BundleHandler,
+ *         }} runtimeOptions
  * @returns {Promise<string | undefined>} KPID of the bootstrap message result promise
  */
 export async function initializeSwingset(
@@ -304,6 +303,12 @@ export async function initializeSwingset(
   insistStorageAPI(kvStore);
   !swingsetIsInitialized(kernelStorage) ||
     Fail`kernel store already initialized`;
+  const {
+    bundleHandler = makeWorkerBundleHandler(
+      kernelStorage.bundleStore,
+      makeXsnapBundleData(),
+    ),
+  } = runtimeOptions;
 
   // copy config so we can safely mess with it even if it's shared or hardened
   config = JSON.parse(JSON.stringify(config));
@@ -346,11 +351,6 @@ export async function initializeSwingset(
     addTimer = true,
   } = initializationOptions;
 
-  assert.typeof(kernelBundles.lockdown, 'object');
-  assert.typeof(kernelBundles.supervisor, 'object');
-  kvStore.set('lockdownBundle', JSON.stringify(kernelBundles.lockdown));
-  kvStore.set('supervisorBundle', JSON.stringify(kernelBundles.supervisor));
-
   if (config.bootstrap && bootstrapArgs) {
     const bootConfig = config.vats[config.bootstrap];
     if (bootConfig) {
@@ -575,7 +575,8 @@ export async function initializeSwingset(
     kdebugEnable(true);
   }
 
+  const kopts = { bundleHandler };
   // returns the kpid of the bootstrap() result
-  const bootKpid = await initializeKernel(kconfig, kernelStorage);
+  const bootKpid = await initializeKernel(kconfig, kernelStorage, kopts);
   return bootKpid;
 }

packages/SwingSet/src/controller/startXSnap.js

@@ -7,18 +7,21 @@ import { xsnap, recordXSnap } from '@agoric/xsnap';
 const NETSTRING_MAX_CHUNK_SIZE = 12_000_000;
 
 /**
- * @param {{ moduleFormat: string, source: string }[]} bundles
  * @param {{
- *   snapStore?: SnapStore,
+ *   bundleHandler: import('./bundle-handler.js').BundleHandler,
+ *   snapStore?: import('@agoric/swing-store').SnapStore,
  *   spawn: typeof import('child_process').spawn
  *   debug?: boolean,
  *   workerTraceRootPath?: string,
+ *   overrideBundles?: import('../types-external.js').Bundle[],
  * }} options
  */
-export function makeStartXSnap(bundles, options) {
+export function makeStartXSnap(options) {
   // our job is to simply curry some authorities and settings into the
   // 'startXSnap' function we return
+  const { bundleHandler } = options; // required unless bundleIDs is empty
   const { snapStore, spawn, debug = false, workerTraceRootPath } = options;
+  const { overrideBundles } = options;
 
   let doXSnap = xsnap;
 
@@ -53,13 +56,15 @@ export function makeStartXSnap(bundles, options) {
   /**
    * @param {string} vatID
    * @param {string} name
+   * @param {import('../types-external.js').BundleID[]} bundleIDs
    * @param {(request: Uint8Array) => Promise<Uint8Array>} handleCommand
    * @param {boolean} [metered]
    * @param {boolean} [reload]
    */
   async function startXSnap(
     vatID,
     name,
+    bundleIDs,
     handleCommand,
     metered,
     reload = false,
@@ -82,10 +87,17 @@ export function makeStartXSnap(bundles, options) {
     // console.log('fresh xsnap', { snapStore: snapStore });
     const worker = doXSnap({ handleCommand, name, ...meterOpts, ...xsnapOpts });
 
+    const bundlePs = bundleIDs.map(id => bundleHandler.getBundle(id));
+    let bundles = await Promise.all(bundlePs);
+    if (overrideBundles) {
+      bundles = overrideBundles; // replace the usual bundles
+    }
+
     for (const bundle of bundles) {
-      bundle.moduleFormat === 'getExport' ||
-        bundle.moduleFormat === 'nestedEvaluate' ||
-        Fail`unexpected: ${bundle.moduleFormat}`;
+      const { moduleFormat } = bundle;
+      if (moduleFormat !== 'getExport' && moduleFormat !== 'nestedEvaluate') {
+        throw Fail`unexpected: ${moduleFormat}`;
+      }
       // eslint-disable-next-line no-await-in-loop, @jessie.js/no-nested-await
       await worker.evaluate(`(${bundle.source}\n)()`.trim());
     }

packages/SwingSet/src/kernel/kernel.js

@@ -19,7 +19,10 @@ import { parseVatSlot } from '../lib/parseVatSlots.js';
 import { extractSingleSlot, insistCapData } from '../lib/capdata.js';
 import { insistMessage, insistVatDeliveryResult } from '../lib/message.js';
 import { insistDeviceID, insistVatID } from '../lib/id.js';
-import { makeWorkerOptions } from '../lib/workerOptions.js';
+import {
+  makeWorkerOptions,
+  updateWorkerOptions,
+} from '../lib/workerOptions.js';
 import { makeKernelQueueHandler } from './kernelQueue.js';
 import { makeKernelSyscallHandler } from './kernelSyscall.js';
 import { makeSlogger, makeDummySlogger } from './slogger.js';
@@ -95,6 +98,7 @@ export default function buildKernel(
     WeakRef,
     FinalizationRegistry,
     gcAndFinalize,
+    bundleHandler,
   } = kernelEndowments;
   deviceEndowments = { ...deviceEndowments }; // copy so we can modify
   const {
@@ -688,7 +692,11 @@ export default function buildKernel(
       reapInterval = kernelKeeper.getDefaultReapInterval(),
       ...otherOptions
     } = dynamicOptions;
-    const workerOptions = await makeWorkerOptions(kernelKeeper, managerType);
+    const workerOptions = await makeWorkerOptions(
+      kernelKeeper,
+      bundleHandler,
+      managerType,
+    );
     /** @type {import('../types-internal.js').RecordedVatOptions} */
     const vatOptions = harden({ workerOptions, reapInterval, ...otherOptions });
     vatKeeper.setSourceAndOptions(source, vatOptions);
@@ -914,9 +922,15 @@ export default function buildKernel(
     // transcript or snapshot and prime everything for the next incarnation.
 
     await vatWarehouse.resetWorker(vatID);
+    // update source and bundleIDs, store back to vat metadata
     const source = { bundleID };
-    const { options } = vatKeeper.getSourceAndOptions();
-    vatKeeper.setSourceAndOptions(source, options);
+    const origOptions = vatKeeper.getOptions();
+    const workerOptions = await updateWorkerOptions(
+      bundleHandler,
+      origOptions.workerOptions,
+    );
+    const vatOptions = harden({ ...origOptions, workerOptions });
+    vatKeeper.setSourceAndOptions(source, vatOptions);
     const incarnationNumber = vatKeeper.incIncarnationNumber();
     // TODO: decref the bundleID once setSourceAndOptions increfs it
 
@@ -1572,7 +1586,11 @@ export default function buildKernel(
     const vatID = kernelKeeper.allocateVatIDForNameIfNeeded(name);
     logStartup(`assigned VatID ${vatID} for test vat ${name}`);
 
-    const workerOptions = await makeWorkerOptions(kernelKeeper, 'local');
+    const workerOptions = await makeWorkerOptions(
+      kernelKeeper,
+      bundleHandler,
+      'local',
+    );
     /** @type {import('../types-internal.js').RecordedVatOptions} */
     const vatOptions = harden({
       name,