From 73b9b145c399f3cdaaed73f65e4913840e318238 Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Wed, 31 Mar 2021 18:26:54 -0500 Subject: [PATCH 1/3] feat(xsnap): snapstore with compressed snapshots --- packages/xsnap/package.json | 2 +- packages/xsnap/src/index.js | 2 + packages/xsnap/src/snapStore.js | 129 +++++++++++++++++++++++ packages/xsnap/test/test-snapstore.js | 144 ++++++++++++++++++++++++++ 4 files changed, 276 insertions(+), 1 deletion(-) create mode 100644 packages/xsnap/src/index.js create mode 100644 packages/xsnap/src/snapStore.js create mode 100644 packages/xsnap/test/test-snapstore.js diff --git a/packages/xsnap/package.json b/packages/xsnap/package.json index 38e4c2cc976..ff0e8ef883a 100644 --- a/packages/xsnap/package.json +++ b/packages/xsnap/package.json @@ -7,7 +7,7 @@ "parsers": { "js": "mjs" }, - "main": "./src/xsnap.js", + "main": "./src/index.js", "bin": { "ava-xs": "./src/ava-xs.js", "xsrepl": "./src/xsrepl" diff --git a/packages/xsnap/src/index.js b/packages/xsnap/src/index.js new file mode 100644 index 00000000000..e25ebe7d81c --- /dev/null +++ b/packages/xsnap/src/index.js @@ -0,0 +1,2 @@ +export { xsnap } from './xsnap'; +export { makeSnapstore } from './snapStore'; diff --git a/packages/xsnap/src/snapStore.js b/packages/xsnap/src/snapStore.js new file mode 100644 index 00000000000..2aa81b75d5c --- /dev/null +++ b/packages/xsnap/src/snapStore.js @@ -0,0 +1,129 @@ +// @ts-check +import { createHash } from 'crypto'; +import { pipeline } from 'stream'; +import { createGzip, createGunzip } from 'zlib'; +import { assert, details as d } from '@agoric/assert'; +import { promisify } from 'util'; + +const pipe = promisify(pipeline); + +const { freeze } = Object; + +/** + * @param {string} root + * @param {{ + * tmpName: typeof import('tmp').tmpName, + * existsSync: typeof import('fs').existsSync + * createReadStream: typeof import('fs').createReadStream, + * createWriteStream: typeof import('fs').createWriteStream, + * resolve: typeof import('path').resolve, + * rename: typeof import('fs').promises.rename, + * unlink: typeof import('fs').promises.unlink, + * }} io + */ +export function makeSnapstore( + root, + { + tmpName, + existsSync, + createReadStream, + createWriteStream, + resolve, + rename, + unlink, + }, +) { + /** @type {(opts: unknown) => Promise} */ + const ptmpName = promisify(tmpName); + const tmpOpts = { tmpdir: root, template: 'tmp-XXXXXX.xss' }; + /** + * @param { (name: string) => Promise } thunk + * @returns { Promise } + * @template T + */ + async function withTempName(thunk) { + const name = await ptmpName(tmpOpts); + let result; + try { + result = await thunk(name); + } finally { + try { + await unlink(name); + } catch (ignore) { + // ignore + } + } + return result; + } + + /** + * @param {string} dest + * @param { (name: string) => Promise } thunk + * @returns { Promise } + * @template T + */ + async function atomicWrite(dest, thunk) { + const tmp = await ptmpName(tmpOpts); + let result; + try { + result = await thunk(tmp); + await rename(tmp, resolve(root, dest)); + } finally { + try { + await unlink(tmp); + } catch (ignore) { + // ignore + } + } + return result; + } + + /** @type {(input: string, f: NodeJS.ReadWriteStream, output: string) => Promise} */ + async function filter(input, f, output) { + const source = createReadStream(input); + const destination = createWriteStream(output); + await pipe(source, f, destination); + } + + /** @type {(filename: string) => Promise} */ + async function fileHash(filename) { + const hash = createHash('sha256'); + const input = createReadStream(filename); + await pipe(input, hash); + return hash.digest('hex'); + } + + /** + * @param {(fn: string) => Promise} saveRaw + * @returns { Promise } sha256 hash of (uncompressed) snapshot + */ + async function save(saveRaw) { + return withTempName(async snapFile => { + await saveRaw(snapFile); + const h = await fileHash(snapFile); + if (existsSync(`${h}.gz`)) return h; + await atomicWrite(`${h}.gz`, gztmp => + filter(snapFile, createGzip(), gztmp), + ); + return h; + }); + } + + /** + * @param {string} hash + * @param {(fn: string) => Promise} loadRaw + * @template T + */ + async function load(hash, loadRaw) { + return withTempName(async raw => { + await filter(resolve(root, `${hash}.gz`), createGunzip(), raw); + const actual = await fileHash(raw); + assert(actual === hash, d`actual hash ${actual} !== expected ${hash}`); + // be sure to await loadRaw before exiting withTempName + const result = await loadRaw(raw); + return result; + }); + } + + return freeze({ load, save }); +} diff --git a/packages/xsnap/test/test-snapstore.js b/packages/xsnap/test/test-snapstore.js new file mode 100644 index 00000000000..76f53678d51 --- /dev/null +++ b/packages/xsnap/test/test-snapstore.js @@ -0,0 +1,144 @@ +/* global __dirname, __filename */ +// @ts-check + +import '@agoric/install-ses'; +import { spawn } from 'child_process'; +import { type as osType } from 'os'; +import fs from 'fs'; +import path from 'path'; + +// eslint-disable-next-line import/no-extraneous-dependencies +import test from 'ava'; +// eslint-disable-next-line import/no-extraneous-dependencies +import tmp from 'tmp'; +import { xsnap } from '../src/xsnap'; +import { makeSnapstore } from '../src/snapStore'; + +const importModuleUrl = `file://${__filename}`; + +const asset = async (...segments) => + fs.promises.readFile( + path.join(importModuleUrl.replace('file:/', ''), '..', ...segments), + 'utf-8', + ); + +/** + * @param {string} name + * @param {(request:Uint8Array) => Promise} handleCommand + */ +async function bootWorker(name, handleCommand) { + const worker = xsnap({ + os: osType(), + spawn, + handleCommand, + name, + stdout: 'inherit', + stderr: 'inherit', + // debug: !!env.XSNAP_DEBUG, + }); + + const bootScript = await asset('..', 'dist', 'bundle-ses-boot.umd.js'); + await worker.evaluate(bootScript); + return worker; +} + +test('build temp file; compress to cache file', async t => { + const pool = path.resolve(__dirname, './fixture-snap-pool/'); + await fs.promises.mkdir(pool, { recursive: true }); + const store = makeSnapstore(pool, { + ...tmp, + ...path, + ...fs, + ...fs.promises, + }); + let keepTmp = ''; + const hash = await store.save(async fn => { + t.falsy(fs.existsSync(fn)); + fs.writeFileSync(fn, 'abc'); + keepTmp = fn; + }); + t.is( + 'ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad', + hash, + ); + t.falsy( + fs.existsSync(keepTmp), + 'temp file should have been deleted after withTempName', + ); + const dest = path.resolve(pool, `${hash}.gz`); + t.truthy(fs.existsSync(dest)); + const gz = fs.readFileSync(dest); + t.is(gz.toString('hex'), '1f8b08000000000000034b4c4a0600c241243503000000'); +}); + +test('bootstrap, save, compress', async t => { + const vat = await bootWorker('ses-boot1', async m => m); + t.teardown(() => vat.close()); + + const pool = path.resolve(__dirname, './fixture-snap-pool/'); + await fs.promises.mkdir(pool, { recursive: true }); + + const store = makeSnapstore(pool, { + ...tmp, + ...path, + ...fs, + ...fs.promises, + }); + + await vat.evaluate('globalThis.x = harden({a: 1})'); + + /** @type {(fn: string) => number} */ + const Kb = fn => Math.round(fs.statSync(fn).size / 1024); + /** @type {(fn: string, fullSize: number) => number} */ + const relativeSize = (fn, fullSize) => + Math.round((fs.statSync(fn).size / 1024 / fullSize) * 10) / 10; + + const snapSize = { + raw: 857, + compression: 0.1, + }; + + const h = await store.save(async snapFile => { + await vat.snapshot(snapFile); + t.is(snapSize.raw, Kb(snapFile), 'raw snapshots are large-ish'); + }); + + const zfile = path.resolve(pool, `${h}.gz`); + t.is( + relativeSize(zfile, snapSize.raw), + snapSize.compression, + 'compressed snapshots are smaller', + ); +}); + +test('create, save, restore, resume', async t => { + const pool = path.resolve(__dirname, './fixture-snap-pool/'); + await fs.promises.mkdir(pool, { recursive: true }); + + const store = makeSnapstore(pool, { + ...tmp, + ...path, + ...fs, + ...fs.promises, + }); + + const vat0 = await bootWorker('ses-boot2', async m => m); + t.teardown(() => vat0.close()); + await vat0.evaluate('globalThis.x = harden({a: 1})'); + const h = await store.save(vat0.snapshot); + + const worker = await store.load(h, async snapshot => { + const xs = xsnap({ name: 'ses-resume', snapshot, os: osType(), spawn }); + await xs.evaluate('0'); + return xs; + }); + t.teardown(() => worker.close()); + await worker.evaluate('x.a'); + t.pass(); +}); + +// see https://github.com/Agoric/agoric-sdk/issues/2776 +test.failing('xs snapshots should be deterministic', t => { + const h = 'abc'; + t.is('66244b4bfe92ae9138d24a9b50b492d231f6a346db0cf63543d200860b423724', h); +}); From 90d90f3938746cf05767aa889caa15865169b4f7 Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Wed, 31 Mar 2021 19:27:01 -0500 Subject: [PATCH 2/3] feat(swingset): boot xsnap workers from snapshot --- packages/SwingSet/package.json | 2 + packages/SwingSet/src/controller.js | 80 ++++++++++++++----- .../vatManager/manager-subprocess-xsnap.js | 12 +-- 3 files changed, 66 insertions(+), 28 deletions(-) diff --git a/packages/SwingSet/package.json b/packages/SwingSet/package.json index 06294ea34c0..2925158b4ac 100644 --- a/packages/SwingSet/package.json +++ b/packages/SwingSet/package.json @@ -53,6 +53,7 @@ "@babel/core": "^7.5.0", "@babel/generator": "^7.6.4", "@endo/base64": "^0.1.0", + "@types/tmp": "^0.2.0", "anylogger": "^0.21.0", "esm": "^3.2.5", "re2": "^1.10.5", @@ -60,6 +61,7 @@ "rollup-plugin-node-resolve": "^5.2.0", "semver": "^6.3.0", "ses": "^0.12.6", + "tmp": "^0.2.1", "yargs": "^14.2.0" }, "files": [ diff --git a/packages/SwingSet/src/controller.js b/packages/SwingSet/src/controller.js index 48081e194e4..793e12d59b5 100644 --- a/packages/SwingSet/src/controller.js +++ b/packages/SwingSet/src/controller.js @@ -1,5 +1,6 @@ /* global require */ import fs from 'fs'; +import path from 'path'; import process from 'process'; import re2 from 're2'; import { spawn } from 'child_process'; @@ -9,6 +10,7 @@ import * as babelCore from '@babel/core'; import * as babelParser from '@agoric/babel-parser'; import babelGenerate from '@babel/generator'; import anylogger from 'anylogger'; +import { tmpName } from 'tmp'; import { assert, details as X } from '@agoric/assert'; import { isTamed, tameMetering } from '@agoric/tame-metering'; @@ -16,7 +18,7 @@ import { importBundle } from '@agoric/import-bundle'; import { initSwingStore } from '@agoric/swing-store-simple'; import { makeMeteringTransformer } from '@agoric/transform-metering'; import { makeTransform } from '@agoric/transform-eventual-send'; -import { xsnap } from '@agoric/xsnap'; +import { xsnap, makeSnapstore } from '@agoric/xsnap'; import { WeakRef, FinalizationRegistry } from './weakref'; import { startSubprocessWorker } from './spawnSubprocessWorker'; @@ -42,6 +44,57 @@ function unhandledRejectionHandler(e) { console.error('UnhandledPromiseRejectionWarning:', e); } +export function makeStartXSnap(bundles, { snapstorePath, env }) { + const xsnapOpts = { + os: osType(), + spawn, + stdout: 'inherit', + stderr: 'inherit', + debug: !!env.XSNAP_DEBUG, + }; + + let snapStore; + + if (snapstorePath) { + fs.mkdirSync(snapstorePath, { recursive: true }); + + snapStore = makeSnapstore(snapstorePath, { + tmpName, + existsSync: fs.existsSync, + createReadStream: fs.createReadStream, + createWriteStream: fs.createWriteStream, + rename: fs.promises.rename, + unlink: fs.promises.unlink, + resolve: path.resolve, + }); + } + + let supervisorHash = ''; + return async function startXSnap(name, handleCommand) { + if (supervisorHash) { + return snapStore.load(supervisorHash, async snapshot => { + const xs = xsnap({ snapshot, name, handleCommand, ...xsnapOpts }); + await xs.evaluate('null'); // ensure that spawn is done + return xs; + }); + } + const worker = xsnap({ handleCommand, name, ...xsnapOpts }); + + for (const bundle of bundles) { + assert( + bundle.moduleFormat === 'getExport', + X`unexpected: ${bundle.moduleFormat}`, + ); + // eslint-disable-next-line no-await-in-loop + await worker.evaluate(`(${bundle.source}\n)()`.trim()); + } + if (snapStore) { + supervisorHash = await snapStore.save(async fn => worker.snapshot(fn)); + } + return worker; + }; +} + export async function makeSwingsetController( hostStorage = initSwingStore().storage, deviceEndowments = {}, @@ -59,6 +112,7 @@ export async function makeSwingsetController( slogCallbacks, slogFile, testTrackDecref, + snapstorePath, } = runtimeOptions; if (typeof Compartment === 'undefined') { throw Error('SES must be installed before calling makeSwingsetController'); @@ -177,23 +231,11 @@ export async function makeSwingsetController( return startSubprocessWorker(process.execPath, ['-r', 'esm', supercode]); } - const startXSnap = (name, handleCommand) => { - const worker = xsnap({ - os: osType(), - spawn, - handleCommand, - name, - stdout: 'inherit', - stderr: 'inherit', - debug: !!env.XSNAP_DEBUG, - }); - - const bundles = { - lockdown: JSON.parse(hostStorage.get('lockdownBundle')), - supervisor: JSON.parse(hostStorage.get('supervisorBundle')), - }; - return harden({ worker, bundles }); - }; + const bundles = [ + JSON.parse(hostStorage.get('lockdownBundle')), + JSON.parse(hostStorage.get('supervisorBundle')), + ]; + const startXSnap = makeStartXSnap(bundles, { snapstorePath, env }); const slogF = slogFile && (await fs.createWriteStream(slogFile, { flags: 'a' })); // append @@ -323,12 +365,14 @@ export async function buildVatController( debugPrefix, slogCallbacks, testTrackDecref, + snapstorePath, } = runtimeOptions; const actualRuntimeOptions = { verbose, debugPrefix, testTrackDecref, slogCallbacks, + snapstorePath, }; const initializationOptions = { verbose, kernelBundles }; let bootstrapResult; diff --git a/packages/SwingSet/src/kernel/vatManager/manager-subprocess-xsnap.js b/packages/SwingSet/src/kernel/vatManager/manager-subprocess-xsnap.js index 104350dbb0d..109c3ab86cd 100644 --- a/packages/SwingSet/src/kernel/vatManager/manager-subprocess-xsnap.js +++ b/packages/SwingSet/src/kernel/vatManager/manager-subprocess-xsnap.js @@ -18,7 +18,7 @@ const decoder = new TextDecoder(); * @param {{ * allVatPowers: VatPowers, * kernelKeeper: KernelKeeper, - * startXSnap: (name: string, handleCommand: SyncHandler) => { worker: XSnap, bundles: Record }, + * startXSnap: (name: string, handleCommand: SyncHandler) => Promise, * testLog: (...args: unknown[]) => void, * decref: (vatID: unknown, vref: unknown, count: number) => void, * }} tools @@ -128,15 +128,7 @@ export function makeXsSubprocessFactory({ } // start the worker and establish a connection - const { worker, bundles } = startXSnap(`${vatID}:${name}`, handleCommand); - for await (const [it, superCode] of Object.entries(bundles)) { - parentLog(vatID, 'eval bundle', it); - assert( - superCode.moduleFormat === 'getExport', - X`${it} unexpected: ${superCode.moduleFormat}`, - ); - await worker.evaluate(`(${superCode.source}\n)()`.trim()); - } + const worker = await startXSnap(`${vatID}:${name}`, handleCommand); /** @type { (item: Tagged) => Promise } */ async function issueTagged(item) { From a4f234cd54b6477a693bf03a45df492eb99c1693 Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Thu, 1 Apr 2021 14:45:10 -0500 Subject: [PATCH 3/3] test(swingset): test:xs runs all of swingset tests --- packages/SwingSet/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/SwingSet/package.json b/packages/SwingSet/package.json index 2925158b4ac..0ec99fa81ad 100644 --- a/packages/SwingSet/package.json +++ b/packages/SwingSet/package.json @@ -16,7 +16,7 @@ "scripts": { "build": "exit 0", "test": "ava", - "test:xs": "exit 0", + "test:xs": "SWINGSET_WORKER_TYPE=xs-worker ava", "test:xs-worker": "ava test/workers/test-worker.js -m 'xs vat manager'", "pretty-fix": "prettier --write '**/*.js'", "pretty-check": "prettier --check '**/*.js'",