Complete the Jest DI with crypto (#234)

* Use ipfs-core instead of ipfs for a lighter dependency

* Make yarn node --version be 15.x in nix-shell

* Implement the complete crypto DI for node

* Fix cbor encode/decode errors on node

Switch to using Uint8Arrays instead of node Buffers.
This is something the js-ipfs folks apparently have gone through, too:
ipfs/js-ipfs#3220

* ESLint fixes

* Fix eslint errors in index.ts

* Revert "Fix cbor encode/decode errors on node"

This reverts commit dbc3b3d.

* Fix type

* Test cbor en/decoding

* Implement helper for totally in-memory ipfs

* Add integration test & fix filesystem running in node

* Remove logging

* Use types added to the workaround repo

* Fix CBOR decoding errors of encrypted data in node

* Switch back to borc

* Update jest & puppeteer & base58-universal

* Add jest puppeteer types

* Make it possible to run multiple ipfs in-memory

If I didn't disable swarm addresses, then multiple instances of ipfs would request socket access and block each other.

* Actually switch back from cborg to borc

* Remove unneeded Buffer.from call

* Make jest properly exit after all tests

* wtfnode

* Revert "wtfnode"

This reverts commit 5cbfd03.

* Force jest to exit, even if ipfs holds resources

* Revert "Make jest properly exit after all tests"

This reverts commit eab6bc8.

package.json

@@ -36,7 +36,7 @@
     "prebuild": "rimraf dist && node scripts/gen-version.js",
     "build": "tsc && rollup -c rollup.config.ts",
     "start": "tsc -w",
-    "test": "jest",
+    "test": "jest --forceExit",
     "test:watch": "jest --watch",
     "test:prod": "yarn run build && yarn run lint && yarn run test -- --no-cache",
     "test:types": "cp -RT tests/types/ dist/ && yarn run tsd",
@@ -61,12 +61,14 @@
     "@babel/core": "^7.12.10",
     "@babel/preset-env": "^7.12.7",
     "@babel/preset-typescript": "^7.12.7",
+    "@ipld/car": "https://github.com/matheus23/ipld-car-jest-fix#28fa48a8d0e701ddaf6e4785d0d9f08735b67bc8",
     "@rollup/plugin-babel": "^5.2.2",
     "@rollup/plugin-commonjs": "^17.0.0",
     "@rollup/plugin-inject": "^4.0.2",
     "@rollup/plugin-json": "^4.1.0",
     "@rollup/plugin-node-resolve": "^11.1.0",
     "@types/jest": "^26.0.20",
+    "@types/jest-environment-puppeteer": "^4.4.1",
     "@types/node": "^13.7.4",
     "@types/throttle-debounce": "^2.1.0",
     "@typescript-eslint/eslint-plugin": "^4.24.0",
@@ -75,15 +77,16 @@
     "braces": "^3.0.2",
     "eslint": "^7.26.0",
     "fast-check": "^2.14.0",
-    "ipfs": "^0.54.4",
+    "interface-datastore": "^4.0.1",
+    "ipfs-repo": "^9.1.6",
     "jest": "^26.6.3",
     "jest-config": "^26.6.3",
-    "jest-puppeteer": "^4.4.0",
+    "jest-puppeteer": "^5.0.3",
     "lint-staged": "^10.5.3",
     "multihashing-async": "^2.1.2",
     "prettier": "^1.19.1",
     "prompt": "^1.1.0",
-    "puppeteer": "^5.5.0",
+    "puppeteer": "^9.1.1",
     "replace-in-file": "^5.0.2",
     "rimraf": "^3.0.2",
     "rollup": "^2.37.1",
@@ -99,11 +102,12 @@
     "yarn": "^1.22.4"
   },
   "dependencies": {
-    "base58-universal": "^1.0.0",
-    "borc": "^2.1.1",
+    "base58-universal": "https://github.com/digitalbazaar/base58-universal#de970560f005de0f7054723c35ef6e0ff4b328b7",
+    "borc": "^3.0.0",
     "buffer": "^6.0.3",
     "cids": "^1.1.5",
     "fission-bloom-filters": "1.6.0",
+    "ipfs-core": "^0.6.1",
     "ipfs-message-port-client": "https://ipfs.runfission.com/ipfs/bafybeigx6q4aezve7my76s5vvfuiinbxtepapqvmjf2jbgrozrut6cjape/p/ipfs-message-port-client.tar.gz",
     "ipfs-message-port-protocol": "https://ipfs.runfission.com/ipfs/bafybeigx6q4aezve7my76s5vvfuiinbxtepapqvmjf2jbgrozrut6cjape/p/ipfs-message-port-protocol.tar.gz",
     "ipld-dag-pb": "^0.20.0",

shell.nix

@@ -6,7 +6,8 @@ in
 
 pkgs.mkShell {
   buildInputs = [
-    unstable.yarn
+    # https://github.com/NixOS/nixpkgs/issues/53820#issuecomment-617973476
+    (unstable.yarn.override { nodejs = null; })
     unstable.nodejs-15_x
     unstable.niv
   ];

src/@types/index.d.ts

@@ -1,3 +1,6 @@
-declare module 'base58-universal/main.js'
-declare module 'borc'
+declare module 'base58-universal' {
+    export function encode(input: Uint8Array, maxline?: number): string
+    export function decode(input: string): Uint8Array
+}
 declare module 'ipld-dag-pb'
+declare module 'borc'

src/did/transformers.ts

@@ -1,5 +1,4 @@
-import * as base58 from 'base58-universal/main.js'
-
+import * as base58 from 'base58-universal'
 import * as utils from 'keystore-idb/utils'
 
 import { BASE58_DID_PREFIX, magicBytes, parseMagicBytes } from './util'

src/fs/filesystem.ts

@@ -56,7 +56,7 @@ type MutationOptions = {
 export class FileSystem {
 
   root: RootTree
-  localOnly: boolean
+  readonly localOnly: boolean
 
   appPath: AppPath | undefined
   proofs: { [_: string]: string }
@@ -115,11 +115,13 @@ export class FileSystem {
     this.publishHooks.push(logCid)
     this.publishHooks.push(updateDataRootWhenOnline)
 
-    // Publish when coming back online
-    globalThis.addEventListener('online', this._whenOnline)
-
-    // Show an alert when leaving the page while updating the data root
-    globalThis.addEventListener('beforeunload', this._beforeLeaving)
+    if (!this.localOnly) {
+      // Publish when coming back online
+      globalThis.addEventListener('online', this._whenOnline)
+
+      // Show an alert when leaving the page while updating the data root
+      globalThis.addEventListener('beforeunload', this._beforeLeaving)
+    }
   }
 
 
@@ -170,6 +172,7 @@ export class FileSystem {
    * The only function of this is to stop listing to online/offline events.
    */
   deactivate(): void {
+    if (this.localOnly) return
     const globe = (globalThis as any)
     globe.filesystems = globe.filesystems.filter((a: FileSystem) => a !== this)
     globe.removeEventListener('online', this._whenOnline)

src/fs/protocol/private/mmpt.test.ts

@@ -1,8 +1,9 @@
 import crypto from "crypto"
-import Ipfs, { IPFS } from "ipfs"
+import { IPFS } from "ipfs-core"
 
 import MMPT from "./mmpt"
 import * as ipfsConfig from "../../../ipfs/config"
+import { createInMemoryIPFS } from "../../../../tests/helpers/in-memory-ipfs"
 
 function sha256Str(str: string): string {
   return crypto.createHash('sha256').update(str).digest('hex')
@@ -12,7 +13,20 @@ function encode(str: string): Uint8Array {
   return (new TextEncoder()).encode(str)
 }
 
-let ipfs: IPFS;
+
+let ipfs: IPFS | null = null
+
+beforeAll(async done => {
+  ipfs = await createInMemoryIPFS()
+  ipfsConfig.set(ipfs)
+  done()
+})
+
+afterAll(async done => {
+  if (ipfs == null) return
+  await ipfs.stop()
+  done()
+})
 
 /*
 Generates lots of entries for insertion into the MMPT.
@@ -22,8 +36,8 @@ The MMPT is a glorified key-value store.
 This returns an array of key-values sorted by the key,
 so that key collisions are more likely to be tested.
 */
-async function generateExampleEntries(amount: number): Promise<{ name: string, cid: string }[]> {
-  let entries: { name: string, cid: string }[] = []
+async function generateExampleEntries(amount: number): Promise<{ name: string; cid: string }[]> {
+  const entries: { name: string; cid: string }[] = []
 
   for (const i of Array(amount).keys()) {
     const hash = sha256Str(`${i}`)
@@ -39,17 +53,6 @@ async function generateExampleEntries(amount: number): Promise<{ name: string, c
 
 
 
-beforeAll(async done => {
-  ipfs = await Ipfs.create({ offline: true, silent: true })
-  ipfsConfig.set(ipfs)
-  done()
-})
-
-afterAll(async done => {
-  await ipfs.stop()
-  done()
-})
-
 describe("the mmpt", () => {
   it("can handle concurrent adds", async () => {
     const mmpt = MMPT.create()

src/index.ts

@@ -72,7 +72,7 @@ export type Continuation = {
   permissions: Maybe<Permissions>
 
   authenticated: true
-  newUser: false,
+  newUser: false
   throughLobby: false
   username: string
 
@@ -172,7 +172,7 @@ export async function initialise(
     )
 
   } else if (cancellation) {
-    const c = (_ => {
+    const c = (() => {
       switch (cancellation) {
         case "DENIED": return "User denied authorisation"
         default: return "Unknown reason"
@@ -256,6 +256,7 @@ export * as ipfs from './ipfs'
 export * as keystore from './keystore'
 export * as machinery from './common'
 export * as crypto from './crypto'
+export * as cbor from 'borc'
 
 
 
@@ -345,7 +346,7 @@ async function importClassifiedInfo(
   const secretsStr = await crypto.aes.decryptGCM(info.secrets, rawSessionKey, info.iv)
   const secrets = JSON.parse(secretsStr)
 
-  const fsSecrets: Record<string, { key: string, bareNameFilter: string }> = secrets.fs
+  const fsSecrets: Record<string, { key: string; bareNameFilter: string }> = secrets.fs
   const ucans = secrets.ucans
 
   // Import read keys and bare name filters
@@ -365,7 +366,7 @@ async function importClassifiedInfo(
 }
 
 async function getClassifiedViaPostMessage(): Promise<string> {
-  const iframe: HTMLIFrameElement = await new Promise((resolve, reject) => {
+  const iframe: HTMLIFrameElement = await new Promise(resolve => {
     const iframe = document.createElement("iframe")
     iframe.id = "webnative-secret-exchange"
     iframe.style.width = "0"

src/ipfs/types.ts

@@ -1,5 +1,5 @@
 export type IPFS = {
-  add(data: FileContent, options?: unknown): UnixFSFile
+  add(data: FileContent, options?: unknown): Promise<UnixFSFile>
   cat(cid: CID): AsyncIterable<FileContentRaw>
   ls(cid: CID): AsyncIterable<UnixFSFile>
   dns(domain: string): Promise<CID>