deps: update it-length-prefix, uint8arraylist etc (#1317)

In order to support no-copy operations in streams, update all deps to support streaming Uint8ArrayLists.
libp2p · Aug 3, 2022 · 564f4b8 · 564f4b8
1 parent 05e8e7e
commit 564f4b8
Show file tree

Hide file tree

Showing 27 changed files with 69 additions and 58 deletions.
diff --git a/examples/auto-relay/test.js b/examples/auto-relay/test.js
@@ -1,5 +1,5 @@
 import path from 'path'
-import execa from 'execa'
+import { execa } from 'execa'
 import pDefer from 'p-defer'
 import { toString as uint8ArrayToString } from 'uint8arrays/to-string'
 import { fileURLToPath } from 'url'

diff --git a/examples/chat/src/stream.js b/examples/chat/src/stream.js
@@ -28,7 +28,7 @@ export function streamToConsole(stream) {
     // Decode length-prefixed data
     lp.decode(),
     // Turn buffers into strings
-    (source) => map(source, (buf) => uint8ArrayToString(buf)),
+    (source) => map(source, (buf) => uint8ArrayToString(buf.subarray())),
     // Sink function
     async function (source) {
       // For each chunk of data

diff --git a/examples/chat/test.js b/examples/chat/test.js
@@ -1,5 +1,5 @@
 import path from 'path'
-import execa from 'execa'
+import { execa } from 'execa'
 import pDefer from 'p-defer'
 import { toString as uint8ArrayToString } from 'uint8arrays/to-string'
 import { fileURLToPath } from 'url'

diff --git a/examples/discovery-mechanisms/test-2.js b/examples/discovery-mechanisms/test-2.js
@@ -1,5 +1,5 @@
 import path from 'path'
-import execa from 'execa'
+import { execa } from 'execa'
 import pWaitFor from 'p-wait-for'
 import { toString as uint8ArrayToString } from 'uint8arrays/to-string'
 import { fileURLToPath } from 'url'

diff --git a/examples/discovery-mechanisms/test-3.js b/examples/discovery-mechanisms/test-3.js
@@ -1,5 +1,5 @@
 import path from 'path'
-import execa from 'execa'
+import { execa } from 'execa'
 import pWaitFor from 'p-wait-for'
 import { toString as uint8ArrayToString } from 'uint8arrays/to-string'
 import { fileURLToPath } from 'url'

diff --git a/examples/echo/test.js b/examples/echo/test.js
@@ -1,5 +1,5 @@
 import path from 'path'
-import execa from 'execa'
+import { execa } from 'execa'
 import pDefer from 'p-defer'
 import { toString as uint8ArrayToString } from 'uint8arrays/to-string'
 import { fileURLToPath } from 'url'

diff --git a/examples/libp2p-in-the-browser/README.md b/examples/libp2p-in-the-browser/README.md
@@ -1,6 +1,6 @@
 # libp2p in the browser
 
-This example leverages the [Parcel.js bundler](https://parceljs.org/) to compile and serve the libp2p code in the browser. Parcel uses [Babel](https://babeljs.io/) to handle transpilation of the code. You can use other bundlers such as Webpack or Browserify, but we will not be covering them here.
+This example leverages the [vite bundler](https://vitejs.dev/) to compile and serve the libp2p code in the browser. You can use other bundlers such as Webpack, but we will not be covering them here.
 
 ## Setup
 
@@ -18,7 +18,7 @@ npm install
 
 ## Running the examples
 
-Start by running the Parcel server:
+Start by running the vite server:
 
 ```
 npm start
@@ -30,7 +30,7 @@ The output should look something like this:
 $ npm start
 
 > libp2p-in-browser@1.0.0 start
-> parcel index.html
+> vite index.html
 
 Server running at http://localhost:1234
 ✨  Built in 1000ms.
@@ -40,7 +40,7 @@ This will compile the code and start a server listening on port [http://localhos
 
 Now, if you open a second browser tab to `http://localhost:1234`, you should discover your node from the previous tab. This is due to the fact that the `libp2p-webrtc-star` transport also acts as a Peer Discovery interface. Your node will be notified of any peer that connects to the same signaling server you are connected to. Once libp2p discovers this new peer, it will attempt to establish a direct WebRTC connection.
 
-**Note**: In the example we assign libp2p to `window.libp2p`, in case you would like to play around with the API directly in the browser. You can of course make changes to `index.js` and Parcel will automatically rebuild and reload the browser tabs.
+**Note**: In the example we assign libp2p to `window.libp2p`, in case you would like to play around with the API directly in the browser. You can of course make changes to `index.js` and vite will automatically rebuild and reload the browser tabs.
 
 ## Going to production?
 

diff --git a/examples/libp2p-in-the-browser/index.html b/examples/libp2p-in-the-browser/index.html
@@ -4,7 +4,7 @@
 
   <head>
     <meta charset="utf-8">
-    <title>js-libp2p parcel.js browser example</title>
+    <title>js-libp2p vite browser example</title>
   </head>
 
   <body>

diff --git a/examples/libp2p-in-the-browser/test.js b/examples/libp2p-in-the-browser/test.js
@@ -1,4 +1,4 @@
-import execa from 'execa'
+import { execa } from 'execa'
 import { chromium } from 'playwright'
 import path from 'path'
 import { fileURLToPath } from 'url'

diff --git a/examples/package.json b/examples/package.json
@@ -9,13 +9,13 @@
   },
   "license": "MIT",
   "dependencies": {
-    "@libp2p/pubsub-peer-discovery": "^6.0.0",
-    "@libp2p/floodsub": "^3.0.0",
+    "@libp2p/pubsub-peer-discovery": "^6.0.1",
+    "@libp2p/floodsub": "^3.0.3",
     "@nodeutils/defaults-deep": "^1.1.0",
-    "execa": "^2.1.0",
-    "fs-extra": "^8.1.0",
+    "execa": "^6.1.0",
+    "fs-extra": "^10.1.0",
     "libp2p": "../",
-    "p-defer": "^3.0.0",
+    "p-defer": "^4.0.0",
     "uint8arrays": "^3.0.0",
     "which": "^2.0.1"
   },

diff --git a/examples/pubsub/message-filtering/test.js b/examples/pubsub/message-filtering/test.js
@@ -1,5 +1,5 @@
 import path from 'path'
-import execa from 'execa'
+import { execa } from 'execa'
 import pDefer from 'p-defer'
 import { toString as uint8ArrayToString } from 'uint8arrays/to-string'
 import { fileURLToPath } from 'url'

diff --git a/examples/pubsub/test-1.js b/examples/pubsub/test-1.js
@@ -1,5 +1,5 @@
 import path from 'path'
-import execa from 'execa'
+import { execa } from 'execa'
 import pDefer from 'p-defer'
 import { toString as uint8ArrayToString } from 'uint8arrays/to-string'
 import { fileURLToPath } from 'url'

diff --git a/examples/test.js b/examples/test.js
@@ -3,7 +3,7 @@ process.env.CI = true // needed for some "clever" build tools
 
 import fs from 'fs-extra'
 import path from 'path'
-import execa from 'execa'
+import { execa } from 'execa'
 import { fileURLToPath } from 'url'
 
 const __dirname = path.dirname(fileURLToPath(import.meta.url))
@@ -36,7 +36,8 @@ async function installDeps (dir) {
     return
   }
 
-  const proc = execa.command('npm install', {
+  const proc = execa('npm', ['install'], {
+    all: true,
     cwd: dir
   })
   proc.all.on('data', (data) => {
@@ -71,6 +72,7 @@ async function build (dir) {
   }
 
   const proc = execa('npm', ['run', build], {
+    all: true,
     cwd: dir
   })
   proc.all.on('data', (data) => {

diff --git a/examples/utils.js b/examples/utils.js
@@ -1,4 +1,4 @@
-import execa from 'execa'
+import { execa } from 'execa'
 import fs from 'fs-extra'
 import which from 'which'
 
@@ -26,7 +26,10 @@ export async function waitForOutput (expectedOutput, command, args = [], opts =
     command = 'node'
   }
 
-  const proc = execa(command, args, opts)
+  const proc = execa(command, args, {
+    ...opts,
+    all: true
+  })
   let output = ''
   let time = 600000
 

diff --git a/examples/webrtc-direct/index.html b/examples/webrtc-direct/index.html
@@ -2,7 +2,7 @@
 <html lang="en">
 <head>
   <meta charset="UTF-8">
-  <title>js-libp2p parcel.js browser example</title>
+  <title>js-libp2p vite browser example</title>
 </head>
 
 <body>

diff --git a/examples/webrtc-direct/package.json b/examples/webrtc-direct/package.json
@@ -10,9 +10,9 @@
   "license": "ISC",
   "dependencies": {
     "@libp2p/webrtc-direct": "^2.0.0",
-    "@chainsafe/libp2p-noise": "^7.0.1",
+    "@chainsafe/libp2p-noise": "^7.0.3",
     "@libp2p/bootstrap": "^2.0.0",
-    "@libp2p/mplex": "^4.0.2",
+    "@libp2p/mplex": "^4.0.3",
     "libp2p": "../../",
     "wrtc": "^0.4.7"
   },

diff --git a/examples/webrtc-direct/test.js b/examples/webrtc-direct/test.js
@@ -1,5 +1,5 @@
 import path from 'path'
-import execa from 'execa'
+import { execa } from 'execa'
 import pDefer from 'p-defer'
 import { toString as uint8ArrayToString } from 'uint8arrays/to-string'
 import { chromium } from 'playwright'

diff --git a/package.json b/package.json
@@ -87,7 +87,6 @@
     "generate:proto:fetch": "protons ./src/fetch/pb/proto.proto",
     "generate:proto:identify": "protons ./src/identify/pb/message.proto",
     "generate:proto:plaintext": "protons ./src/insecure/pb/proto.proto",
-    "generate:proto:tags": "protons ./src/connection-manager/tags/tags.proto",
     "test": "aegir test",
     "test:node": "aegir test -t node -f \"./dist/test/**/*.{node,spec}.js\" --cov",
     "test:chrome": "aegir test -t browser -f \"./dist/test/**/*.spec.js\" --cov",
@@ -113,7 +112,7 @@
     "@libp2p/interface-peer-info": "^1.0.1",
     "@libp2p/interface-peer-routing": "^1.0.0",
     "@libp2p/interface-peer-store": "^1.2.0",
-    "@libp2p/interface-pubsub": "^1.0.3",
+    "@libp2p/interface-pubsub": "^2.0.0",
     "@libp2p/interface-registrar": "^2.0.0",
     "@libp2p/interface-stream-muxer": "^2.0.1",
     "@libp2p/interface-transport": "^1.0.0",
@@ -123,7 +122,7 @@
     "@libp2p/peer-collections": "^2.0.0",
     "@libp2p/peer-id": "^1.1.10",
     "@libp2p/peer-id-factory": "^1.0.9",
-    "@libp2p/peer-record": "^3.0.0",
+    "@libp2p/peer-record": "^4.0.0",
     "@libp2p/peer-store": "^3.0.0",
     "@libp2p/tracked-map": "^2.0.1",
     "@libp2p/utils": "^3.0.0",
@@ -142,7 +141,7 @@
     "it-first": "^1.0.6",
     "it-foreach": "^0.1.1",
     "it-handshake": "^4.0.0",
-    "it-length-prefixed": "^7.0.1",
+    "it-length-prefixed": "^8.0.2",
     "it-map": "^1.0.6",
     "it-merge": "^1.0.3",
     "it-pair": "^2.0.2",
@@ -157,17 +156,18 @@
     "p-retry": "^5.0.0",
     "p-settle": "^5.0.0",
     "private-ip": "^2.3.3",
-    "protons-runtime": "^1.0.4",
+    "protons-runtime": "^2.0.2",
     "retimer": "^3.0.0",
     "sanitize-filename": "^1.6.3",
     "set-delayed-interval": "^1.0.0",
     "timeout-abort-controller": "^3.0.0",
+    "uint8arraylist": "^2.0.0",
     "uint8arrays": "^3.0.0",
     "wherearewe": "^1.0.0",
     "xsalsa20": "^1.1.0"
   },
   "devDependencies": {
-    "@chainsafe/libp2p-noise": "^7.0.1",
+    "@chainsafe/libp2p-noise": "^7.0.2",
     "@libp2p/bootstrap": "^2.0.0",
     "@libp2p/daemon-client": "^2.0.0",
     "@libp2p/daemon-server": "^2.0.0",
@@ -204,8 +204,8 @@
     "p-defer": "^4.0.0",
     "p-event": "^5.0.1",
     "p-times": "^4.0.0",
-    "p-wait-for": "^4.1.0",
-    "protons": "^3.0.4",
+    "p-wait-for": "^5.0.0",
+    "protons": "^4.0.1",
     "rimraf": "^3.0.2",
     "sinon": "^14.0.0",
     "ts-sinon": "^2.0.2"

diff --git a/src/circuit/circuit/stream-handler.ts b/src/circuit/circuit/stream-handler.ts
@@ -4,6 +4,7 @@ import { Handshake, handshake } from 'it-handshake'
 import { CircuitRelay } from '../pb/index.js'
 import type { Stream } from '@libp2p/interface-connection'
 import type { Source } from 'it-stream-types'
+import type { Uint8ArrayList } from 'uint8arraylist'
 
 const log = logger('libp2p:circuit:stream-handler')
 
@@ -22,7 +23,7 @@ export interface StreamHandlerOptions {
 export class StreamHandler {
   private readonly stream: Stream
   private readonly shake: Handshake
-  private readonly decoder: Source<Uint8Array>
+  private readonly decoder: Source<Uint8ArrayList>
 
   constructor (options: StreamHandlerOptions) {
     const { stream, maxLength = 4096 } = options
@@ -40,7 +41,7 @@ export class StreamHandler {
     const msg = await this.decoder.next()
 
     if (msg.value != null) {
-      const value = CircuitRelay.decode(msg.value.slice())
+      const value = CircuitRelay.decode(msg.value)
       log('read message type', value.type)
       return value
     }

diff --git a/src/circuit/pb/index.ts b/src/circuit/pb/index.ts
@@ -3,6 +3,7 @@
 
 import { enumeration, encodeMessage, decodeMessage, message, bytes } from 'protons-runtime'
 import type { Codec } from 'protons-runtime'
+import type { Uint8ArrayList } from 'uint8arraylist'
 
 export interface CircuitRelay {
   type?: CircuitRelay.Type
@@ -89,11 +90,11 @@ export namespace CircuitRelay {
       })
     }
 
-    export const encode = (obj: Peer): Uint8Array => {
+    export const encode = (obj: Peer): Uint8ArrayList => {
       return encodeMessage(obj, Peer.codec())
     }
 
-    export const decode = (buf: Uint8Array): Peer => {
+    export const decode = (buf: Uint8Array | Uint8ArrayList): Peer => {
       return decodeMessage(buf, Peer.codec())
     }
   }
@@ -107,11 +108,11 @@ export namespace CircuitRelay {
     })
   }
 
-  export const encode = (obj: CircuitRelay): Uint8Array => {
+  export const encode = (obj: CircuitRelay): Uint8ArrayList => {
     return encodeMessage(obj, CircuitRelay.codec())
   }
 
-  export const decode = (buf: Uint8Array): CircuitRelay => {
+  export const decode = (buf: Uint8Array | Uint8ArrayList): CircuitRelay => {
     return decodeMessage(buf, CircuitRelay.codec())
   }
 }
diff --git a/src/fetch/pb/proto.ts b/src/fetch/pb/proto.ts
@@ -3,6 +3,7 @@
 
 import { encodeMessage, decodeMessage, message, string, enumeration, bytes } from 'protons-runtime'
 import type { Codec } from 'protons-runtime'
+import type { Uint8ArrayList } from 'uint8arraylist'
 
 export interface FetchRequest {
   identifier: string
@@ -15,11 +16,11 @@ export namespace FetchRequest {
     })
   }
 
-  export const encode = (obj: FetchRequest): Uint8Array => {
+  export const encode = (obj: FetchRequest): Uint8ArrayList => {
     return encodeMessage(obj, FetchRequest.codec())
   }
 
-  export const decode = (buf: Uint8Array): FetchRequest => {
+  export const decode = (buf: Uint8Array | Uint8ArrayList): FetchRequest => {
     return decodeMessage(buf, FetchRequest.codec())
   }
 }
@@ -55,11 +56,11 @@ export namespace FetchResponse {
     })
   }
 
-  export const encode = (obj: FetchResponse): Uint8Array => {
+  export const encode = (obj: FetchResponse): Uint8ArrayList => {
     return encodeMessage(obj, FetchResponse.codec())
   }
 
-  export const decode = (buf: Uint8Array): FetchResponse => {
+  export const decode = (buf: Uint8Array | Uint8ArrayList): FetchResponse => {
     return decodeMessage(buf, FetchResponse.codec())
   }
 }
diff --git a/src/identify/index.ts b/src/identify/index.ts
@@ -404,7 +404,7 @@ export class IdentifyService implements Startable {
 
         const envelope = await RecordEnvelope.seal(peerRecord, this.components.getPeerId())
         await this.components.getPeerStore().addressBook.consumePeerRecord(envelope)
-        signedPeerRecord = envelope.marshal()
+        signedPeerRecord = envelope.marshal().subarray()
       }
 
       const message = Identify.encode({

diff --git a/src/identify/pb/message.ts b/src/identify/pb/message.ts
@@ -3,6 +3,7 @@
 
 import { encodeMessage, decodeMessage, message, string, bytes } from 'protons-runtime'
 import type { Codec } from 'protons-runtime'
+import type { Uint8ArrayList } from 'uint8arraylist'
 
 export interface Identify {
   protocolVersion?: string
@@ -27,11 +28,11 @@ export namespace Identify {
     })
   }
 
-  export const encode = (obj: Identify): Uint8Array => {
+  export const encode = (obj: Identify): Uint8ArrayList => {
     return encodeMessage(obj, Identify.codec())
   }
 
-  export const decode = (buf: Uint8Array): Identify => {
+  export const decode = (buf: Uint8Array | Uint8ArrayList): Identify => {
     return decodeMessage(buf, Identify.codec())
   }
 }
-Original file line number
+Diff line change
@@ Expand Up / @@ -4,7 +4,7 @@ @@
       <head>
         <meta charset="utf-8">
-        <title>js-libp2p parcel.js browser example</title>
+        <title>js-libp2p vite browser example</title>
       </head>
       <body>
@@ Expand Down @@