testnet + test fixes (#335)

* added dht bootstrap to tests

* removed system pear in ci

* removed artifact folder in CI

* pass bootstrap to pear-updater-bootstraper

* clean up

* added verbose flag

* smoke working

* added stream teardown in teardown tests

* isolate

* pear-updater-bootstap 1.2.0

* only lin

* enc test fix

* fix

* artifact to artefact rename

* handle streams in update tests

* corrected teardown

* corrected teardown

* corrected teardown

* more fixes

* more fixes

* more fixes

* fix

* enable all machines

* fix

* fix

* explicit bump pear-ipc

* renamed --bootstrap to --dht-bootstrap

* flag parsing in sidecar subsystem instead

* default bootstrap fix

* refactor default bootstrap fix

---------

Co-authored-by: dmc <david.mark.clements@gmail.com>

.github/workflows/ci.yml

@@ -36,61 +36,17 @@ jobs:
       with:
         node-version: lts/*
 
-    - name: Determine Local Pear Command per OS
-      run: echo "PEAR=$( [[ \"$RUNNER_OS\" == \"Windows\" ]] && echo 'pwsh -File ./pear.ps1' || echo './pear.dev' )" >> $GITHUB_ENV
-      shell: bash
-
     - name: Install w/ dev deps
       run: npm install
 
-    - name: System Pear
-      run: npm i -g pear && pear
-
     - name: Bootstrap
       run: npm run bootstrap
 
-    - name: Remove node_modules
-      run: rm -fr node_modules
-      shell: bash
-
-    - name: Install w/out dev deps
-      run: npm i --omit=dev
-
-    - name: Self-Stage
-      run: $PEAR stage ci
-      shell: bash
-
-    - name: Artifact Pear
-      run: $PEAR dump $($PEAR info ci --key) $(node -p "fs.realpathSync(os.tmpdir())")/artifact-pear
-      shell: bash
-
-    - name: Shutdown Local Pear Sidecar
-      run: $PEAR sidecar shutdown
-      shell: bash
-
-    - name: Remove node_modules
-      run: rm -fr node_modules
-      shell: bash
-
-    - name: Install w/ dev deps
-      run: npm install
-
-    - name: Install fixture deps
-      run: |
-        cd ./test/fixtures/harness
-        npm install
-        cd -
-        cd ./test/fixtures/encrypted
-        npm install
-        cd -
-      shell: bash
+    - name: Install Bare
+      run: npm i -g bare-runtime
 
     - name: Test
       run: npm test
 
-    - name: Shutdown Local Pear Sidecar
-      run: $PEAR sidecar shutdown
-      shell: bash
-
     - name: Lint
       run: npm run lint

lib/api.js

@@ -58,12 +58,20 @@ class API {
     this.#unloading()
 
     const MAX_TEARDOWN_WAIT = 5000
-    const timeout = new Promise(resolve => setTimeout(() => {
+    let timeout = null
+    let timedout = false
+    const countdown = new Promise((resolve) => {
+      timeout = setTimeout(() => {
+        timedout = true
+        resolve()
+      }, MAX_TEARDOWN_WAIT)
+    })
+    this.#teardowns.finally(() => { clearTimeout(timeout) })
+    await Promise.race([this.#teardowns, countdown])
+    if (timedout) {
       console.error(`Max teardown wait reached after ${MAX_TEARDOWN_WAIT} ms. Exiting...`)
-      resolve()
-    }, MAX_TEARDOWN_WAIT))
-
-    await Promise.race([this.#teardowns, timeout])
+      Bare.exit()
+    }
   }
 
   async #reftrack (promise) {

lib/tryboot.js

@@ -6,7 +6,12 @@ const { RUNTIME, PLATFORM_DIR } = require('../constants.js')
 module.exports = function tryboot () {
   const verbose = (global.Bare || global.process).argv.includes('--verbose')
   const args = ['--sidecar']
+  const dhtBootstrap = Bare.argv.includes('--dht-bootstrap') ? Bare.argv[Bare.argv.indexOf('--dht-bootstrap') + 1] : null
   if (verbose) args.push('--verbose')
+  if (dhtBootstrap) {
+    args.push('--dht-bootstrap')
+    args.push(dhtBootstrap)
+  }
   const sc = spawn(RUNTIME, args, {
     detached: !verbose,
     stdio: verbose ? 'inherit' : 'ignore',

package.json

@@ -5,8 +5,7 @@
   "private": true,
   "scripts": {
     "test:gen": "brittle -r test/index.js test/*.test.js",
-    "pretest": "pear run -t scripts/pretest.js",
-    "test": "pear run -t scripts/test.js",
+    "test": "bare scripts/test.mjs",
     "lint": "standard",
     "lint:fix": "standard --fix",
     "archdump": "node scripts/bootstrap.js --archdump",
@@ -98,10 +97,10 @@
     "paparam": "^1.4.0",
     "pear-changelog": "^1.0.1",
     "pear-interface": "^1.0.0",
-    "pear-ipc": "^2.2.1",
+    "pear-ipc": "^2.2.8",
     "pear-link": "^2.0.1",
     "pear-updater": "^3.1.0",
-    "pear-updater-bootstrap": "^1.1.3",
+    "pear-updater-bootstrap": "^1.2.0",
     "protomux": "^3.6.0",
     "pw-to-ek": "^1.0.0",
     "quickbit-universal": "^2.2.0",
@@ -121,6 +120,7 @@
     "z32": "^1.1.0"
   },
   "devDependencies": {
+    "@hyperswarm/testnet": "^3.1.4",
     "bare-inspector": "^2.3.0",
     "brittle": "^3.5.2",
     "graceful-goodbye": "^1.3.0",

run/definition.js

@@ -19,6 +19,7 @@ module.exports = [
   flag('--checkout <n|release|staged>', 'Run a checkout from version length'),
   flag('--detached', 'Wakeup existing app or run detached'),
   flag('--no-ask', 'Suppress permissions dialog'),
+  hiddenFlag('--dht-bootstrap <nodes>', 'DHT boostrap'),
   hiddenFlag('--encryption-key <name>', 'Application encryption key'),
   hiddenFlag('--trusted'),
   hiddenFlag('--detach'),

run/index.js

@@ -125,8 +125,13 @@ module.exports = async function run ({ ipc, args, cmdArgs, link, storage, detach
       await new Promise((resolve, reject) => fs.close(fd, (err) => err ? reject(err) : resolve(fd)))
 
       await global.Pear.restart()
-    })
+    }).on('error', () => {})
 
+    global.Pear.teardown(async () => {
+      if (reloadSubscriber.destroyed) return
+      reloadSubscriber.end()
+      return new Promise((resolve) => reloadSubscriber.on('close', resolve))
+    })
     const protocol = new Module.Protocol({
       exists (url) {
         return Object.hasOwn(bundle.sources, url.href)

scripts/test.mjs

@@ -0,0 +1,38 @@
+'use strict'
+import url from 'bare-url'
+import path from 'bare-path'
+import { spawn, spawnSync } from 'bare-subprocess'
+import { RUNTIME } from '../constants'
+import createTestnet from '@hyperswarm/testnet'
+import fs from 'bare-fs'
+import { isWindows } from 'which-runtime'
+
+const filename = url.fileURLToPath(import.meta.url)
+const dirname = path.dirname(filename)
+const root = path.dirname(dirname)
+
+const force = Bare.argv.includes('--force-install')
+
+const dirs = [
+  path.join(root, 'test', 'fixtures', 'harness', 'node_modules'),
+  path.join(root, 'test', 'fixtures', 'encrypted', 'node_modules')
+]
+for (const dir of dirs) {
+  if (force === false && fs.existsSync(dir)) continue
+  console.log(force ? 'reinstalling node_modules in' : 'node_modules not found in', path.dirname(dir))
+  console.log('Running npm install...')
+  if (isWindows) spawnSync('pwsh', ['-Command', 'npm install'], { cwd: path.dirname(dir), stdio: 'inherit' })
+  else spawnSync('npm', ['install'], { cwd: path.dirname(dir), stdio: 'inherit' })
+}
+
+const testnet = await createTestnet(10)
+const dhtBootstrap = testnet.nodes.map(e => `${e.host}:${e.port}`).join(',')
+
+const verbose = Bare.argv.includes('--verbose') ? '--verbose' : ''
+
+const tests = spawn(RUNTIME, ['run', '--dht-bootstrap', dhtBootstrap, '-t', 'test', verbose], { cwd: root, stdio: 'inherit' })
+
+tests.on('exit', async (code) => {
+  await testnet.destroy()
+  Bare.exitCode = code
+})

sidecar.js

@@ -19,11 +19,8 @@ const {
 } = require('./constants.js')
 const registerUrlHandler = require('./url-handler.js')
 const gunk = require('./gunk')
-const verbose = Bare.argv.includes('--verbose')
 crasher('sidecar', SWAP)
-module.exports = bootSidecar().then(() => {
-  if (verbose) console.log('- Sidecar booted')
-}).catch((err) => {
+module.exports = bootSidecar().catch((err) => {
   console.error(err.stack)
   Bare.exit(1)
 })
@@ -45,7 +42,7 @@ async function bootSidecar () {
   const Sidecar = await subsystem(drive, '/subsystems/sidecar/index.js')
 
   const updater = createUpdater()
-  const sidecar = new Sidecar({ updater, drive, corestore, gunk, verbose })
+  const sidecar = new Sidecar({ updater, drive, corestore, gunk })
   await sidecar.ipc.ready()
 
   registerUrlHandler(WAKEUP)

state.js

@@ -36,6 +36,7 @@ module.exports = class State {
   entrypoints = null
   applink = null
   dht = null
+  dhtBootstrap = null
   static injestPackage (state, pkg, overrides = {}) {
     state.manifest = pkg
     state.main = pkg?.main || 'index.html'
@@ -80,9 +81,9 @@ module.exports = class State {
   }
 
   static configFrom (state) {
-    const { id, key, links, alias, env, options, checkpoint, flags, dev, tier, stage, storage, trace, name, main, dependencies, args, channel, release, applink, fragment, link, linkData, entrypoint, dir, dht } = state
+    const { id, key, links, alias, env, options, checkpoint, flags, dev, tier, stage, storage, trace, name, main, dependencies, args, channel, release, applink, fragment, link, linkData, entrypoint, dir, dht, dhtBootstrap } = state
     const pearDir = PLATFORM_DIR
-    return { id, key, links, alias, env, options, checkpoint, flags, dev, tier, stage, storage, trace, name, main, dependencies, args, channel, release, applink, fragment, link, linkData, entrypoint, dir, dht, pearDir }
+    return { id, key, links, alias, env, options, checkpoint, flags, dev, tier, stage, storage, trace, name, main, dependencies, args, channel, release, applink, fragment, link, linkData, entrypoint, dir, dht, dhtBootstrap, pearDir }
   }
 
   static isKeetInvite (segment) {
@@ -121,6 +122,7 @@ module.exports = class State {
     this.#onupdate = onupdate
     this.startId = startId || null
     this.dht = dht
+    this.dhtBootstrap = flags.dhtBootstrap?.split(',').map(e => ({ host: e.split(':')[0], port: Number(e.split(':')[1]) }))
     this.store = store
     this.args = args
     this.appling = appling

subsystems/sidecar/index.js

@@ -17,7 +17,8 @@ const sodium = require('sodium-native')
 const Updater = require('pear-updater')
 const IPC = require('pear-ipc')
 const { isMac, isWindows } = require('which-runtime')
-const { command } = require('paparam')
+const { command, flag, arg, rest } = require('paparam')
+const deriveEncryptionKey = require('pw-to-ek')
 const reports = require('./lib/reports')
 const Store = require('./lib/store')
 const Applings = require('./lib/applings')
@@ -29,7 +30,6 @@ const registerUrlHandler = require('../../url-handler')
 const parseLink = require('../../lib/parse-link')
 const runDefinition = require('../../run/definition')
 const { version } = require('../../package.json')
-const deriveEncryptionKey = require('pw-to-ek')
 const {
   PLATFORM_DIR, PLATFORM_LOCK, SOCKET_PATH, CHECKOUT, APPLINGS_PATH,
   SWAP, RUNTIME, DESKTOP_RUNTIME, ALIASES, SPINDOWN_TIMEOUT, WAKEUP, SALT
@@ -79,8 +79,32 @@ class Sidecar extends ReadyResource {
 
   teardown () { global.Bare.exit() }
 
-  constructor ({ updater, drive, corestore, gunk, verbose = false }) {
+  #parsePlatformFlags () {
+    const argv = Bare.argv.slice(1)
+    let verbose = false
+    let dhtBootstrap = ''
+    const dhtBootstrapFlag = flag('--dht-bootstrap <nodes>')
+    try {
+      const result = command('pear', flag('--sidecar'), flag('--verbose'), dhtBootstrapFlag, arg('<cmd>'), rest('rest')).parse(argv).flags
+      verbose = result.verbose
+      dhtBootstrap = result.dhtBootstrap || undefined
+    } catch {
+      const result = command('pear', flag('--sidecar'), flag('--verbose'), arg('<cmd>'), rest('rest')).parse(argv).flags
+      verbose = result.verbose
+    }
+    return {
+      verbose,
+      dhtBootstrap: typeof dhtBootstrap === 'string' ? dhtBootstrap.split(',').map(e => ({ host: e.split(':')[0], port: Number(e.split(':')[1]) })) : dhtBootstrap
+    }
+  }
+
+  constructor ({ updater, drive, corestore, gunk }) {
     super()
+
+    const { verbose, dhtBootstrap } = this.#parsePlatformFlags()
+    this.verbose = verbose
+    this.dhtBootstrap = dhtBootstrap
+
     this.bus = new Iambus()
     this.version = CHECKOUT
 
@@ -125,6 +149,8 @@ class Sidecar extends ReadyResource {
     this.http = new Http(this)
     this.running = new Map()
 
+    this.dhtBootstrap = dhtBootstrap
+
     const sidecar = this
     this.App = class App {
       sidecar = sidecar
@@ -282,6 +308,7 @@ class Sidecar extends ReadyResource {
     await this.applings.set('runtime', DESKTOP_RUNTIME)
     await this.http.ready()
     await this.#ensureSwarm()
+    if (this.verbose) console.log('- Sidecar booted')
   }
 
   get clients () { return this.ipc.clients }
@@ -863,7 +890,7 @@ class Sidecar extends ReadyResource {
       throw err
     }
     this.keyPair = await this.corestore.createKeyPair('holepunch')
-    this.swarm = new Hyperswarm({ keyPair: this.keyPair })
+    this.swarm = new Hyperswarm({ keyPair: this.keyPair, bootstrap: this.dhtBootstrap })
     this.swarm.once('close', () => { this.swarm = null })
     this.swarm.on('connection', (connection) => { this.corestore.replicate(connection) })
     if (this.replicator !== null) this.replicator.join(this.swarm, { server: false, client: true }).catch(safetyCatch)

test/01-smoke.test.js

@@ -5,26 +5,28 @@ const hypercoreid = require('hypercore-id-encoding')
 const Helper = require('./helper')
 const harness = path.join(Helper.localDir, 'test', 'fixtures', 'harness')
 
-test('smoke', async function ({ ok, is, plan, comment, teardown, timeout }) {
+test('smoke', async function ({ ok, is, plan, comment, teardown, timeout, end }) {
   timeout(180000)
   plan(5)
   const stager = new Helper()
-  teardown(() => stager.close())
+  teardown(() => stager.close(), { order: Infinity })
   await stager.ready()
   const dir = harness
 
   const id = Math.floor(Math.random() * 10000)
 
   comment('staging')
   const staging = stager.stage({ channel: `test-${id}`, name: `test-${id}`, dir, dryRun: false, bare: true })
+  teardown(() => Helper.teardownStream(staging))
   const final = await Helper.pick(staging, { tag: 'final' })
   ok(final.success, 'stage succeeded')
 
   comment('seeding')
   const seeder = new Helper()
-  teardown(() => seeder.close())
+  teardown(() => seeder.close(), { order: Infinity })
   await seeder.ready()
   const seeding = seeder.seed({ channel: `test-${id}`, name: `test-${id}`, dir, key: null, cmdArgs: [] })
+  teardown(() => Helper.teardownStream(seeding))
   const until = await Helper.pick(seeding, [{ tag: 'key' }, { tag: 'announced' }])
 
   const key = await until.key