From 564f4b8aa77dffc7c3744aa97c5cb217f42c56bc Mon Sep 17 00:00:00 2001 From: Alex Potsides Date: Wed, 3 Aug 2022 14:15:35 +0100 Subject: [PATCH] deps: update it-length-prefix, uint8arraylist etc (#1317) In order to support no-copy operations in streams, update all deps to support streaming Uint8ArrayLists. --- examples/auto-relay/test.js | 2 +- examples/chat/src/stream.js | 2 +- examples/chat/test.js | 2 +- examples/discovery-mechanisms/test-2.js | 2 +- examples/discovery-mechanisms/test-3.js | 2 +- examples/echo/test.js | 2 +- examples/libp2p-in-the-browser/README.md | 8 ++++---- examples/libp2p-in-the-browser/index.html | 2 +- examples/libp2p-in-the-browser/test.js | 2 +- examples/package.json | 10 +++++----- examples/pubsub/message-filtering/test.js | 2 +- examples/pubsub/test-1.js | 2 +- examples/test.js | 6 ++++-- examples/utils.js | 7 +++++-- examples/webrtc-direct/index.html | 2 +- examples/webrtc-direct/package.json | 4 ++-- examples/webrtc-direct/test.js | 2 +- package.json | 16 ++++++++-------- src/circuit/circuit/stream-handler.ts | 5 +++-- src/circuit/pb/index.ts | 9 +++++---- src/fetch/pb/proto.ts | 9 +++++---- src/identify/index.ts | 2 +- src/identify/pb/message.ts | 5 +++-- src/insecure/index.ts | 2 +- src/insecure/pb/proto.ts | 9 +++++---- test/configuration/utils.ts | 9 +++++---- test/relay/auto-relay.node.ts | 2 +- 27 files changed, 69 insertions(+), 58 deletions(-) diff --git a/examples/auto-relay/test.js b/examples/auto-relay/test.js index 4c7541b962..646dbce2b3 100644 --- a/examples/auto-relay/test.js +++ b/examples/auto-relay/test.js @@ -1,5 +1,5 @@ import path from 'path' -import execa from 'execa' +import { execa } from 'execa' import pDefer from 'p-defer' import { toString as uint8ArrayToString } from 'uint8arrays/to-string' import { fileURLToPath } from 'url' diff --git a/examples/chat/src/stream.js b/examples/chat/src/stream.js index 391ac9868d..9263c128aa 100644 --- a/examples/chat/src/stream.js +++ b/examples/chat/src/stream.js @@ -28,7 +28,7 @@ export function streamToConsole(stream) { // Decode length-prefixed data lp.decode(), // Turn buffers into strings - (source) => map(source, (buf) => uint8ArrayToString(buf)), + (source) => map(source, (buf) => uint8ArrayToString(buf.subarray())), // Sink function async function (source) { // For each chunk of data diff --git a/examples/chat/test.js b/examples/chat/test.js index dd75cb91b7..179bf3ead6 100644 --- a/examples/chat/test.js +++ b/examples/chat/test.js @@ -1,5 +1,5 @@ import path from 'path' -import execa from 'execa' +import { execa } from 'execa' import pDefer from 'p-defer' import { toString as uint8ArrayToString } from 'uint8arrays/to-string' import { fileURLToPath } from 'url' diff --git a/examples/discovery-mechanisms/test-2.js b/examples/discovery-mechanisms/test-2.js index b40da4a332..9cef3bfd24 100644 --- a/examples/discovery-mechanisms/test-2.js +++ b/examples/discovery-mechanisms/test-2.js @@ -1,5 +1,5 @@ import path from 'path' -import execa from 'execa' +import { execa } from 'execa' import pWaitFor from 'p-wait-for' import { toString as uint8ArrayToString } from 'uint8arrays/to-string' import { fileURLToPath } from 'url' diff --git a/examples/discovery-mechanisms/test-3.js b/examples/discovery-mechanisms/test-3.js index 375ef6f8ab..24a033110d 100644 --- a/examples/discovery-mechanisms/test-3.js +++ b/examples/discovery-mechanisms/test-3.js @@ -1,5 +1,5 @@ import path from 'path' -import execa from 'execa' +import { execa } from 'execa' import pWaitFor from 'p-wait-for' import { toString as uint8ArrayToString } from 'uint8arrays/to-string' import { fileURLToPath } from 'url' diff --git a/examples/echo/test.js b/examples/echo/test.js index 927b8e986c..2fbae5c111 100644 --- a/examples/echo/test.js +++ b/examples/echo/test.js @@ -1,5 +1,5 @@ import path from 'path' -import execa from 'execa' +import { execa } from 'execa' import pDefer from 'p-defer' import { toString as uint8ArrayToString } from 'uint8arrays/to-string' import { fileURLToPath } from 'url' diff --git a/examples/libp2p-in-the-browser/README.md b/examples/libp2p-in-the-browser/README.md index 9c5513f2d6..d2185b2d49 100644 --- a/examples/libp2p-in-the-browser/README.md +++ b/examples/libp2p-in-the-browser/README.md @@ -1,6 +1,6 @@ # libp2p in the browser -This example leverages the [Parcel.js bundler](https://parceljs.org/) to compile and serve the libp2p code in the browser. Parcel uses [Babel](https://babeljs.io/) to handle transpilation of the code. You can use other bundlers such as Webpack or Browserify, but we will not be covering them here. +This example leverages the [vite bundler](https://vitejs.dev/) to compile and serve the libp2p code in the browser. You can use other bundlers such as Webpack, but we will not be covering them here. ## Setup @@ -18,7 +18,7 @@ npm install ## Running the examples -Start by running the Parcel server: +Start by running the vite server: ``` npm start @@ -30,7 +30,7 @@ The output should look something like this: $ npm start > libp2p-in-browser@1.0.0 start -> parcel index.html +> vite index.html Server running at http://localhost:1234 ✨ Built in 1000ms. @@ -40,7 +40,7 @@ This will compile the code and start a server listening on port [http://localhos Now, if you open a second browser tab to `http://localhost:1234`, you should discover your node from the previous tab. This is due to the fact that the `libp2p-webrtc-star` transport also acts as a Peer Discovery interface. Your node will be notified of any peer that connects to the same signaling server you are connected to. Once libp2p discovers this new peer, it will attempt to establish a direct WebRTC connection. -**Note**: In the example we assign libp2p to `window.libp2p`, in case you would like to play around with the API directly in the browser. You can of course make changes to `index.js` and Parcel will automatically rebuild and reload the browser tabs. +**Note**: In the example we assign libp2p to `window.libp2p`, in case you would like to play around with the API directly in the browser. You can of course make changes to `index.js` and vite will automatically rebuild and reload the browser tabs. ## Going to production? diff --git a/examples/libp2p-in-the-browser/index.html b/examples/libp2p-in-the-browser/index.html index 1b1f48da60..2a1fbe7820 100644 --- a/examples/libp2p-in-the-browser/index.html +++ b/examples/libp2p-in-the-browser/index.html @@ -4,7 +4,7 @@ - js-libp2p parcel.js browser example + js-libp2p vite browser example diff --git a/examples/libp2p-in-the-browser/test.js b/examples/libp2p-in-the-browser/test.js index 0b3a16a6fe..4097ad8544 100644 --- a/examples/libp2p-in-the-browser/test.js +++ b/examples/libp2p-in-the-browser/test.js @@ -1,4 +1,4 @@ -import execa from 'execa' +import { execa } from 'execa' import { chromium } from 'playwright' import path from 'path' import { fileURLToPath } from 'url' diff --git a/examples/package.json b/examples/package.json index a699ca30c0..6e86102b1e 100644 --- a/examples/package.json +++ b/examples/package.json @@ -9,13 +9,13 @@ }, "license": "MIT", "dependencies": { - "@libp2p/pubsub-peer-discovery": "^6.0.0", - "@libp2p/floodsub": "^3.0.0", + "@libp2p/pubsub-peer-discovery": "^6.0.1", + "@libp2p/floodsub": "^3.0.3", "@nodeutils/defaults-deep": "^1.1.0", - "execa": "^2.1.0", - "fs-extra": "^8.1.0", + "execa": "^6.1.0", + "fs-extra": "^10.1.0", "libp2p": "../", - "p-defer": "^3.0.0", + "p-defer": "^4.0.0", "uint8arrays": "^3.0.0", "which": "^2.0.1" }, diff --git a/examples/pubsub/message-filtering/test.js b/examples/pubsub/message-filtering/test.js index 4cb9fbaa15..97736f4985 100644 --- a/examples/pubsub/message-filtering/test.js +++ b/examples/pubsub/message-filtering/test.js @@ -1,5 +1,5 @@ import path from 'path' -import execa from 'execa' +import { execa } from 'execa' import pDefer from 'p-defer' import { toString as uint8ArrayToString } from 'uint8arrays/to-string' import { fileURLToPath } from 'url' diff --git a/examples/pubsub/test-1.js b/examples/pubsub/test-1.js index c818a573d9..e863307154 100644 --- a/examples/pubsub/test-1.js +++ b/examples/pubsub/test-1.js @@ -1,5 +1,5 @@ import path from 'path' -import execa from 'execa' +import { execa } from 'execa' import pDefer from 'p-defer' import { toString as uint8ArrayToString } from 'uint8arrays/to-string' import { fileURLToPath } from 'url' diff --git a/examples/test.js b/examples/test.js index 5de9963449..e6d625c8be 100644 --- a/examples/test.js +++ b/examples/test.js @@ -3,7 +3,7 @@ process.env.CI = true // needed for some "clever" build tools import fs from 'fs-extra' import path from 'path' -import execa from 'execa' +import { execa } from 'execa' import { fileURLToPath } from 'url' const __dirname = path.dirname(fileURLToPath(import.meta.url)) @@ -36,7 +36,8 @@ async function installDeps (dir) { return } - const proc = execa.command('npm install', { + const proc = execa('npm', ['install'], { + all: true, cwd: dir }) proc.all.on('data', (data) => { @@ -71,6 +72,7 @@ async function build (dir) { } const proc = execa('npm', ['run', build], { + all: true, cwd: dir }) proc.all.on('data', (data) => { diff --git a/examples/utils.js b/examples/utils.js index 914d91e9bb..e6118cc5bb 100644 --- a/examples/utils.js +++ b/examples/utils.js @@ -1,4 +1,4 @@ -import execa from 'execa' +import { execa } from 'execa' import fs from 'fs-extra' import which from 'which' @@ -26,7 +26,10 @@ export async function waitForOutput (expectedOutput, command, args = [], opts = command = 'node' } - const proc = execa(command, args, opts) + const proc = execa(command, args, { + ...opts, + all: true + }) let output = '' let time = 600000 diff --git a/examples/webrtc-direct/index.html b/examples/webrtc-direct/index.html index 3fcf9c35d7..0da84a2f5f 100644 --- a/examples/webrtc-direct/index.html +++ b/examples/webrtc-direct/index.html @@ -2,7 +2,7 @@ - js-libp2p parcel.js browser example + js-libp2p vite browser example diff --git a/examples/webrtc-direct/package.json b/examples/webrtc-direct/package.json index f355efd3c5..664488dd28 100644 --- a/examples/webrtc-direct/package.json +++ b/examples/webrtc-direct/package.json @@ -10,9 +10,9 @@ "license": "ISC", "dependencies": { "@libp2p/webrtc-direct": "^2.0.0", - "@chainsafe/libp2p-noise": "^7.0.1", + "@chainsafe/libp2p-noise": "^7.0.3", "@libp2p/bootstrap": "^2.0.0", - "@libp2p/mplex": "^4.0.2", + "@libp2p/mplex": "^4.0.3", "libp2p": "../../", "wrtc": "^0.4.7" }, diff --git a/examples/webrtc-direct/test.js b/examples/webrtc-direct/test.js index 513a21e928..23181b879f 100644 --- a/examples/webrtc-direct/test.js +++ b/examples/webrtc-direct/test.js @@ -1,5 +1,5 @@ import path from 'path' -import execa from 'execa' +import { execa } from 'execa' import pDefer from 'p-defer' import { toString as uint8ArrayToString } from 'uint8arrays/to-string' import { chromium } from 'playwright' diff --git a/package.json b/package.json index 93188b34de..a8071c1ec5 100644 --- a/package.json +++ b/package.json @@ -87,7 +87,6 @@ "generate:proto:fetch": "protons ./src/fetch/pb/proto.proto", "generate:proto:identify": "protons ./src/identify/pb/message.proto", "generate:proto:plaintext": "protons ./src/insecure/pb/proto.proto", - "generate:proto:tags": "protons ./src/connection-manager/tags/tags.proto", "test": "aegir test", "test:node": "aegir test -t node -f \"./dist/test/**/*.{node,spec}.js\" --cov", "test:chrome": "aegir test -t browser -f \"./dist/test/**/*.spec.js\" --cov", @@ -113,7 +112,7 @@ "@libp2p/interface-peer-info": "^1.0.1", "@libp2p/interface-peer-routing": "^1.0.0", "@libp2p/interface-peer-store": "^1.2.0", - "@libp2p/interface-pubsub": "^1.0.3", + "@libp2p/interface-pubsub": "^2.0.0", "@libp2p/interface-registrar": "^2.0.0", "@libp2p/interface-stream-muxer": "^2.0.1", "@libp2p/interface-transport": "^1.0.0", @@ -123,7 +122,7 @@ "@libp2p/peer-collections": "^2.0.0", "@libp2p/peer-id": "^1.1.10", "@libp2p/peer-id-factory": "^1.0.9", - "@libp2p/peer-record": "^3.0.0", + "@libp2p/peer-record": "^4.0.0", "@libp2p/peer-store": "^3.0.0", "@libp2p/tracked-map": "^2.0.1", "@libp2p/utils": "^3.0.0", @@ -142,7 +141,7 @@ "it-first": "^1.0.6", "it-foreach": "^0.1.1", "it-handshake": "^4.0.0", - "it-length-prefixed": "^7.0.1", + "it-length-prefixed": "^8.0.2", "it-map": "^1.0.6", "it-merge": "^1.0.3", "it-pair": "^2.0.2", @@ -157,17 +156,18 @@ "p-retry": "^5.0.0", "p-settle": "^5.0.0", "private-ip": "^2.3.3", - "protons-runtime": "^1.0.4", + "protons-runtime": "^2.0.2", "retimer": "^3.0.0", "sanitize-filename": "^1.6.3", "set-delayed-interval": "^1.0.0", "timeout-abort-controller": "^3.0.0", + "uint8arraylist": "^2.0.0", "uint8arrays": "^3.0.0", "wherearewe": "^1.0.0", "xsalsa20": "^1.1.0" }, "devDependencies": { - "@chainsafe/libp2p-noise": "^7.0.1", + "@chainsafe/libp2p-noise": "^7.0.2", "@libp2p/bootstrap": "^2.0.0", "@libp2p/daemon-client": "^2.0.0", "@libp2p/daemon-server": "^2.0.0", @@ -204,8 +204,8 @@ "p-defer": "^4.0.0", "p-event": "^5.0.1", "p-times": "^4.0.0", - "p-wait-for": "^4.1.0", - "protons": "^3.0.4", + "p-wait-for": "^5.0.0", + "protons": "^4.0.1", "rimraf": "^3.0.2", "sinon": "^14.0.0", "ts-sinon": "^2.0.2" diff --git a/src/circuit/circuit/stream-handler.ts b/src/circuit/circuit/stream-handler.ts index 5dcb36ebf3..0935d5d689 100644 --- a/src/circuit/circuit/stream-handler.ts +++ b/src/circuit/circuit/stream-handler.ts @@ -4,6 +4,7 @@ import { Handshake, handshake } from 'it-handshake' import { CircuitRelay } from '../pb/index.js' import type { Stream } from '@libp2p/interface-connection' import type { Source } from 'it-stream-types' +import type { Uint8ArrayList } from 'uint8arraylist' const log = logger('libp2p:circuit:stream-handler') @@ -22,7 +23,7 @@ export interface StreamHandlerOptions { export class StreamHandler { private readonly stream: Stream private readonly shake: Handshake - private readonly decoder: Source + private readonly decoder: Source constructor (options: StreamHandlerOptions) { const { stream, maxLength = 4096 } = options @@ -40,7 +41,7 @@ export class StreamHandler { const msg = await this.decoder.next() if (msg.value != null) { - const value = CircuitRelay.decode(msg.value.slice()) + const value = CircuitRelay.decode(msg.value) log('read message type', value.type) return value } diff --git a/src/circuit/pb/index.ts b/src/circuit/pb/index.ts index 71ee1b9d29..ec47246aab 100644 --- a/src/circuit/pb/index.ts +++ b/src/circuit/pb/index.ts @@ -3,6 +3,7 @@ import { enumeration, encodeMessage, decodeMessage, message, bytes } from 'protons-runtime' import type { Codec } from 'protons-runtime' +import type { Uint8ArrayList } from 'uint8arraylist' export interface CircuitRelay { type?: CircuitRelay.Type @@ -89,11 +90,11 @@ export namespace CircuitRelay { }) } - export const encode = (obj: Peer): Uint8Array => { + export const encode = (obj: Peer): Uint8ArrayList => { return encodeMessage(obj, Peer.codec()) } - export const decode = (buf: Uint8Array): Peer => { + export const decode = (buf: Uint8Array | Uint8ArrayList): Peer => { return decodeMessage(buf, Peer.codec()) } } @@ -107,11 +108,11 @@ export namespace CircuitRelay { }) } - export const encode = (obj: CircuitRelay): Uint8Array => { + export const encode = (obj: CircuitRelay): Uint8ArrayList => { return encodeMessage(obj, CircuitRelay.codec()) } - export const decode = (buf: Uint8Array): CircuitRelay => { + export const decode = (buf: Uint8Array | Uint8ArrayList): CircuitRelay => { return decodeMessage(buf, CircuitRelay.codec()) } } diff --git a/src/fetch/pb/proto.ts b/src/fetch/pb/proto.ts index 919f94a73f..608559358b 100644 --- a/src/fetch/pb/proto.ts +++ b/src/fetch/pb/proto.ts @@ -3,6 +3,7 @@ import { encodeMessage, decodeMessage, message, string, enumeration, bytes } from 'protons-runtime' import type { Codec } from 'protons-runtime' +import type { Uint8ArrayList } from 'uint8arraylist' export interface FetchRequest { identifier: string @@ -15,11 +16,11 @@ export namespace FetchRequest { }) } - export const encode = (obj: FetchRequest): Uint8Array => { + export const encode = (obj: FetchRequest): Uint8ArrayList => { return encodeMessage(obj, FetchRequest.codec()) } - export const decode = (buf: Uint8Array): FetchRequest => { + export const decode = (buf: Uint8Array | Uint8ArrayList): FetchRequest => { return decodeMessage(buf, FetchRequest.codec()) } } @@ -55,11 +56,11 @@ export namespace FetchResponse { }) } - export const encode = (obj: FetchResponse): Uint8Array => { + export const encode = (obj: FetchResponse): Uint8ArrayList => { return encodeMessage(obj, FetchResponse.codec()) } - export const decode = (buf: Uint8Array): FetchResponse => { + export const decode = (buf: Uint8Array | Uint8ArrayList): FetchResponse => { return decodeMessage(buf, FetchResponse.codec()) } } diff --git a/src/identify/index.ts b/src/identify/index.ts index f868a631ac..ba82aa8178 100644 --- a/src/identify/index.ts +++ b/src/identify/index.ts @@ -404,7 +404,7 @@ export class IdentifyService implements Startable { const envelope = await RecordEnvelope.seal(peerRecord, this.components.getPeerId()) await this.components.getPeerStore().addressBook.consumePeerRecord(envelope) - signedPeerRecord = envelope.marshal() + signedPeerRecord = envelope.marshal().subarray() } const message = Identify.encode({ diff --git a/src/identify/pb/message.ts b/src/identify/pb/message.ts index 04aad2a8b9..1f9072d677 100644 --- a/src/identify/pb/message.ts +++ b/src/identify/pb/message.ts @@ -3,6 +3,7 @@ import { encodeMessage, decodeMessage, message, string, bytes } from 'protons-runtime' import type { Codec } from 'protons-runtime' +import type { Uint8ArrayList } from 'uint8arraylist' export interface Identify { protocolVersion?: string @@ -27,11 +28,11 @@ export namespace Identify { }) } - export const encode = (obj: Identify): Uint8Array => { + export const encode = (obj: Identify): Uint8ArrayList => { return encodeMessage(obj, Identify.codec()) } - export const decode = (buf: Uint8Array): Identify => { + export const decode = (buf: Uint8Array | Uint8ArrayList): Identify => { return decodeMessage(buf, Identify.codec()) } } diff --git a/src/insecure/index.ts b/src/insecure/index.ts index 446104e0ca..3ffd6618be 100644 --- a/src/insecure/index.ts +++ b/src/insecure/index.ts @@ -39,7 +39,7 @@ async function encrypt (localId: PeerId, conn: Duplex, remoteId?: Pe Type: type, Data: localId.publicKey ?? new Uint8Array(0) } - }).slice() + }).subarray() ) log('write pubkey exchange to peer %p', remoteId) diff --git a/src/insecure/pb/proto.ts b/src/insecure/pb/proto.ts index 24b0020a7a..ec84391110 100644 --- a/src/insecure/pb/proto.ts +++ b/src/insecure/pb/proto.ts @@ -3,6 +3,7 @@ import { encodeMessage, decodeMessage, message, bytes, enumeration } from 'protons-runtime' import type { Codec } from 'protons-runtime' +import type { Uint8ArrayList } from 'uint8arraylist' export interface Exchange { id?: Uint8Array @@ -17,11 +18,11 @@ export namespace Exchange { }) } - export const encode = (obj: Exchange): Uint8Array => { + export const encode = (obj: Exchange): Uint8ArrayList => { return encodeMessage(obj, Exchange.codec()) } - export const decode = (buf: Uint8Array): Exchange => { + export const decode = (buf: Uint8Array | Uint8ArrayList): Exchange => { return decodeMessage(buf, Exchange.codec()) } } @@ -58,11 +59,11 @@ export namespace PublicKey { }) } - export const encode = (obj: PublicKey): Uint8Array => { + export const encode = (obj: PublicKey): Uint8ArrayList => { return encodeMessage(obj, PublicKey.codec()) } - export const decode = (buf: Uint8Array): PublicKey => { + export const decode = (buf: Uint8Array | Uint8ArrayList): PublicKey => { return decodeMessage(buf, PublicKey.codec()) } } diff --git a/test/configuration/utils.ts b/test/configuration/utils.ts index a34385c9e3..bc587e5cd9 100644 --- a/test/configuration/utils.ts +++ b/test/configuration/utils.ts @@ -10,6 +10,7 @@ import type { Libp2pInit, Libp2pOptions } from '../../src/index.js' import type { PeerId } from '@libp2p/interface-peer-id' import * as cborg from 'cborg' import { peerIdFromString } from '@libp2p/peer-id' +import { Uint8ArrayList } from 'uint8arraylist' const relayAddr = MULTIADDRS_WEBSOCKETS[0] @@ -32,16 +33,16 @@ class MockPubSub extends PubSubBaseProtocol { return cborg.decode(bytes) } - encodeRpc (rpc: PubSubRPC): Uint8Array { - return cborg.encode(rpc) + encodeRpc (rpc: PubSubRPC): Uint8ArrayList { + return new Uint8ArrayList(cborg.encode(rpc)) } decodeMessage (bytes: Uint8Array): PubSubRPCMessage { return cborg.decode(bytes) } - encodeMessage (rpc: PubSubRPCMessage): Uint8Array { - return cborg.encode(rpc) + encodeMessage (rpc: PubSubRPCMessage): Uint8ArrayList { + return new Uint8ArrayList(cborg.encode(rpc)) } async publishMessage (from: PeerId, message: Message): Promise { diff --git a/test/relay/auto-relay.node.ts b/test/relay/auto-relay.node.ts index 1363144e85..6dd8e0f484 100644 --- a/test/relay/auto-relay.node.ts +++ b/test/relay/auto-relay.node.ts @@ -16,7 +16,7 @@ import type Sinon from 'sinon' import { createRelayOptions, createNodeOptions } from './utils.js' import { protocols } from '@multiformats/multiaddr' -async function usingAsRelay (node: Libp2pNode, relay: Libp2pNode, opts?: PWaitForOptions) { +async function usingAsRelay (node: Libp2pNode, relay: Libp2pNode, opts?: PWaitForOptions) { // Wait for peer to be used as a relay await pWaitFor(() => { for (const addr of node.getMultiaddrs()) {