From 83b6f6058e9eb652b0651fb76223455022953943 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Thu, 14 Jan 2021 12:53:29 +0000 Subject: [PATCH 1/3] feat: allow passing a http.Agent to the grpc client Follows on from #3474 and allows using http.Agents with node.js to control the behaviour of the underlying node http client. --- packages/ipfs-client/README.md | 1 + packages/ipfs-grpc-client/README.md | 1 + .../ipfs-grpc-client/src/core-api/add-all.js | 3 +- .../ipfs-grpc-client/src/core-api/files/ls.js | 3 +- .../src/core-api/files/write.js | 3 +- packages/ipfs-grpc-client/src/core-api/id.js | 3 +- .../src/grpc/transport.node.js | 31 ++++++++++-- packages/ipfs-grpc-client/src/index.js | 9 ++-- .../src/utils/bidi-to-duplex.js | 9 +++- .../src/utils/client-stream-to-promise.js | 1 + .../src/utils/server-stream-to-iterator.js | 1 + .../src/utils/unary-to-promise.js | 1 + packages/ipfs-grpc-client/test/agent.js | 49 +++++++++++++++++++ packages/ipfs-grpc-client/test/node.js | 3 ++ packages/ipfs-http-client/test/node/agent.js | 5 ++ 15 files changed, 110 insertions(+), 13 deletions(-) create mode 100644 packages/ipfs-grpc-client/test/agent.js create mode 100644 packages/ipfs-grpc-client/test/node.js diff --git a/packages/ipfs-client/README.md b/packages/ipfs-client/README.md index 23d7f8500e..3750448ddb 100644 --- a/packages/ipfs-client/README.md +++ b/packages/ipfs-client/README.md @@ -28,6 +28,7 @@ An optional object which may have the following keys: | ---- | ---- | ------- | ----------- | | grpc | `Multiaddr` or `string` or `URL` | `undefined` | The address of a [ipfs-grpc-server][] to connect to | | http | `Multiaddr` or `string` or `URL` | `undefined` | The address of a [ipfs-http-server][] to connect to | +| agent | [http.Agent](https://nodejs.org/api/http.html#http_class_http_agent) | `undefined` | A http.Agent used to control HTTP client behaviour (node.js only) | ### Returns diff --git a/packages/ipfs-grpc-client/README.md b/packages/ipfs-grpc-client/README.md index 1c2be2b0d9..2de71d5b73 100644 --- a/packages/ipfs-grpc-client/README.md +++ b/packages/ipfs-grpc-client/README.md @@ -35,6 +35,7 @@ An optional object which may have the following keys: | Name | Type | Default | Description | | ---- | ---- | ------- | ----------- | | url | `Multiaddr` or `string` or `URL` | `undefined` | The address of a [ipfs-grpc-server][] to connect to | +| agent | [http.Agent](https://nodejs.org/api/http.html#http_class_http_agent) | `undefined` | A http.Agent used to control HTTP client behaviour (node.js only) | ### Returns diff --git a/packages/ipfs-grpc-client/src/core-api/add-all.js b/packages/ipfs-grpc-client/src/core-api/add-all.js index af11d3e4c5..a27d14a3fc 100644 --- a/packages/ipfs-grpc-client/src/core-api/add-all.js +++ b/packages/ipfs-grpc-client/src/core-api/add-all.js @@ -79,7 +79,8 @@ module.exports = function grpcAddAll (grpc, service, opts = {}) { } = bidiToDuplex(grpc, service, { host: opts.url, debug: Boolean(process.env.DEBUG), - metadata: options + metadata: options, + agent: opts.agent }) sendFiles(stream, sink) diff --git a/packages/ipfs-grpc-client/src/core-api/files/ls.js b/packages/ipfs-grpc-client/src/core-api/files/ls.js index b3be56c0b0..7a7a9945c2 100644 --- a/packages/ipfs-grpc-client/src/core-api/files/ls.js +++ b/packages/ipfs-grpc-client/src/core-api/files/ls.js @@ -13,7 +13,8 @@ module.exports = function grpcMfsLs (grpc, service, opts = {}) { for await (const result of serverStreamToIterator(grpc, service, request, { host: opts.url, debug: Boolean(process.env.DEBUG), - metadata: options + metadata: options, + agent: opts.agent })) { yield { name: result.name, diff --git a/packages/ipfs-grpc-client/src/core-api/files/write.js b/packages/ipfs-grpc-client/src/core-api/files/write.js index 30856247a2..f9c686c947 100644 --- a/packages/ipfs-grpc-client/src/core-api/files/write.js +++ b/packages/ipfs-grpc-client/src/core-api/files/write.js @@ -38,7 +38,8 @@ module.exports = function grpcMfsWrite (grpc, service, opts = {}) { await clientStreamToPromise(grpc, service, stream(path, content), { host: opts.url, debug: Boolean(process.env.DEBUG), - metadata: options + metadata: options, + agent: opts.agent }) } diff --git a/packages/ipfs-grpc-client/src/core-api/id.js b/packages/ipfs-grpc-client/src/core-api/id.js index b5486cbb68..b3af2a623d 100644 --- a/packages/ipfs-grpc-client/src/core-api/id.js +++ b/packages/ipfs-grpc-client/src/core-api/id.js @@ -11,7 +11,8 @@ module.exports = function grpcId (grpc, service, opts = {}) { const res = await unaryToPromise(grpc, service, request, { host: opts.url, - metadata: toHeaders(options) + metadata: toHeaders(options), + agent: opts.agent }) return { diff --git a/packages/ipfs-grpc-client/src/grpc/transport.node.js b/packages/ipfs-grpc-client/src/grpc/transport.node.js index 21a3dd53aa..863f1a1e52 100644 --- a/packages/ipfs-grpc-client/src/grpc/transport.node.js +++ b/packages/ipfs-grpc-client/src/grpc/transport.node.js @@ -13,12 +13,32 @@ const WebsocketSignal = { const finishSendFrame = new Uint8Array([1]) -function WebsocketTransport () { - return (opts) => { - return websocketRequest(opts) +/** + * @param {object} options + * @param {import('http').Agent} [options.agent] - http.Agent used to control HTTP client behaviour + */ +function WebsocketTransport (options) { + /** + * @param {import('@improbable-eng/grpc-web').grpc.TransportOptions} opts + */ + const websocketTransportFactory = (opts) => { + return websocketRequest({ + ...options, + ...opts + }) } + + return websocketTransportFactory } +/** + * @typedef {object} NodeTransportOptions + * @property {import('http').Agent} [options.agent] + * + * @typedef {NodeTransportOptions & import('@improbable-eng/grpc-web').grpc.TransportOptions} WebSocketTransportOptions + * + * @param {WebSocketTransportOptions} options + */ function websocketRequest (options) { const webSocketAddress = constructWebSocketAddress(options.url) @@ -54,7 +74,7 @@ function websocketRequest (options) { } }, start: (metadata) => { - ws = new WebSocket(webSocketAddress, ['grpc-websockets']) + ws = new WebSocket(webSocketAddress, ['grpc-websockets'], options) ws.binaryType = 'arraybuffer' ws.onopen = function () { options.debug && debug('websocketRequest.onopen') @@ -93,7 +113,8 @@ function constructWebSocketAddress (url) { } else if (url.substr(0, 7) === 'http://') { return `ws://${url.substr(7)}` } - throw new Error('Websocket transport constructed with non-https:// or http:// host.') + + throw new Error('Websocket transport url must start with ws:// or wss:// or http:// or https://') } function headersToBytes (headers) { diff --git a/packages/ipfs-grpc-client/src/index.js b/packages/ipfs-grpc-client/src/index.js index 3b0c5d2088..0d34276fde 100644 --- a/packages/ipfs-grpc-client/src/index.js +++ b/packages/ipfs-grpc-client/src/index.js @@ -1,10 +1,8 @@ 'use strict' -const transport = require('./grpc/transport') const toUrlString = require('ipfs-core-utils/src/to-url-string') const loadServices = require('./utils/load-services') const { grpc } = require('@improbable-eng/grpc-web') -grpc.setDefaultTransport(transport()) const service = loadServices() @@ -21,7 +19,12 @@ function normaliseUrls (opts) { }) } -module.exports = function createClient (opts = {}) { +/** + * @param {object} opts + * @param {string} opts.url - The URL to connect to as a URL or Multiaddr + * @param {import('http').Agent} [opts.agent] - http.Agent used to control HTTP client behaviour (node.js only) + */ +module.exports = function createClient (opts = { url: '' }) { opts.url = toUrlString(opts.url) // @improbable-eng/grpc-web requires http:// protocol URLs, not ws:// diff --git a/packages/ipfs-grpc-client/src/utils/bidi-to-duplex.js b/packages/ipfs-grpc-client/src/utils/bidi-to-duplex.js index 2bb0c16ab6..2ec54bedc6 100644 --- a/packages/ipfs-grpc-client/src/utils/bidi-to-duplex.js +++ b/packages/ipfs-grpc-client/src/utils/bidi-to-duplex.js @@ -3,6 +3,7 @@ const pushable = require('it-pushable') const errCode = require('err-code') const toHeaders = require('./to-headers') +const transport = require('../grpc/transport') async function sendMessages (service, client, source) { for await (const obj of source) { @@ -23,6 +24,7 @@ async function sendMessages (service, client, source) { * @param {string} options.host - The remote host * @param {boolean} [options.debug] - Whether to print debug messages * @param {object} [options.metadata] - Metadata sent as headers + * @param {import('http').Agent} [options.agent] - http.Agent used to control HTTP client behaviour (node.js only) * @returns {{ source: AsyncIterable, sink: { push: Function, end: Function } }} **/ module.exports = function bidiToDuplex (grpc, service, options) { @@ -32,7 +34,12 @@ module.exports = function bidiToDuplex (grpc, service, options) { // @ts-ignore const sink = pushable() - const client = grpc.client(service, options) + const client = grpc.client(service, { + ...options, + transport: transport({ + agent: options.agent + }) + }) client.onMessage(message => { sink.push(message) }) diff --git a/packages/ipfs-grpc-client/src/utils/client-stream-to-promise.js b/packages/ipfs-grpc-client/src/utils/client-stream-to-promise.js index ba107f23a1..8828badabe 100644 --- a/packages/ipfs-grpc-client/src/utils/client-stream-to-promise.js +++ b/packages/ipfs-grpc-client/src/utils/client-stream-to-promise.js @@ -15,6 +15,7 @@ const bidiToDuplex = require('./bidi-to-duplex') * @param {string} options.host - The remote host * @param {boolean} [options.debug] - Whether to print debug messages * @param {object} [options.metadata] - Metadata sent as headers + * @param {import('http').Agent} [options.agent] - http.Agent used to control HTTP client behaviour (node.js only) * @returns {Promise} - A promise that resolves to a response object **/ module.exports = async function clientStreamToPromise (grpc, service, source, options) { diff --git a/packages/ipfs-grpc-client/src/utils/server-stream-to-iterator.js b/packages/ipfs-grpc-client/src/utils/server-stream-to-iterator.js index 295408755e..61ac9372ac 100644 --- a/packages/ipfs-grpc-client/src/utils/server-stream-to-iterator.js +++ b/packages/ipfs-grpc-client/src/utils/server-stream-to-iterator.js @@ -14,6 +14,7 @@ const bidiToDuplex = require('./bidi-to-duplex') * @param {string} options.host - The remote host * @param {boolean} [options.debug] - Whether to print debug messages * @param {object} [options.metadata] - Metadata sent as headers + * @param {import('http').Agent} [options.agent] - http.Agent used to control HTTP client behaviour (node.js only) * @returns {AsyncIterable} **/ module.exports = function serverStreamToIterator (grpc, service, request, options) { diff --git a/packages/ipfs-grpc-client/src/utils/unary-to-promise.js b/packages/ipfs-grpc-client/src/utils/unary-to-promise.js index 7404d9e8ec..4053b4ee50 100644 --- a/packages/ipfs-grpc-client/src/utils/unary-to-promise.js +++ b/packages/ipfs-grpc-client/src/utils/unary-to-promise.js @@ -15,6 +15,7 @@ const bidiToDuplex = require('./bidi-to-duplex') * @param {string} options.host - The remote host * @param {boolean} [options.debug] - Whether to print debug messages * @param {object} [options.metadata] - Metadata sent as headers + * @param {import('http').Agent} [options.agent] - http.Agent used to control HTTP client behaviour (node.js only) * @returns {Promise} - A promise that resolves to a response object **/ module.exports = function unaryToPromise (grpc, service, request, options) { diff --git a/packages/ipfs-grpc-client/test/agent.js b/packages/ipfs-grpc-client/test/agent.js new file mode 100644 index 0000000000..4553ef5c89 --- /dev/null +++ b/packages/ipfs-grpc-client/test/agent.js @@ -0,0 +1,49 @@ +/* eslint-env mocha */ +'use strict' + +const grpcClient = require('../src') +const WebSocket = require('ws') + +function startServer () { + return new Promise((resolve) => { + const wss = new WebSocket.Server({ port: 0 }) + + wss.on('listening', () => { + resolve({ + port: wss.address().port, + close: () => wss.close() + }) + }) + + wss.on('connection', (ws) => { + ws.once('message', () => { + ws.send('') + ws.end() + }) + }) + }) +} + +describe('agent', function () { + it('uses the passed agent', async () => { + const server = await startServer() + + try { + await new Promise((resolve) => { + const ipfs = grpcClient({ + url: `http://localhost:${server.port}`, + agent: { + addRequest () { + // an agent method was invoked + resolve() + } + } + }) + + ipfs.id().catch(() => {}) + }) + } finally { + server.close() + } + }) +}) diff --git a/packages/ipfs-grpc-client/test/node.js b/packages/ipfs-grpc-client/test/node.js new file mode 100644 index 0000000000..c2f3f0a9bb --- /dev/null +++ b/packages/ipfs-grpc-client/test/node.js @@ -0,0 +1,3 @@ +'use strict' + +require('./agent') diff --git a/packages/ipfs-http-client/test/node/agent.js b/packages/ipfs-http-client/test/node/agent.js index 3a4e18cd9c..6cc98cbe5b 100644 --- a/packages/ipfs-http-client/test/node/agent.js +++ b/packages/ipfs-http-client/test/node/agent.js @@ -82,6 +82,11 @@ describe('agent', function () { break } + + if (i === 4) { + // should have first two responses by now + expect(responses).to.have.lengthOf(2) + } } // wait for the final request to arrive From 3a8e34fa428b37286349bee0badcc5ce85b0be6a Mon Sep 17 00:00:00 2001 From: achingbrain Date: Thu, 14 Jan 2021 13:38:09 +0000 Subject: [PATCH 2/3] chore: typecheck errors --- packages/ipfs-http-client/src/dag/put.js | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/ipfs-http-client/src/dag/put.js b/packages/ipfs-http-client/src/dag/put.js index c06e43f625..1f58b32278 100644 --- a/packages/ipfs-http-client/src/dag/put.js +++ b/packages/ipfs-http-client/src/dag/put.js @@ -28,7 +28,6 @@ module.exports = configure((api, opts) => { const cid = new CID(options.cid) encodingOptions = { ...options, - // @ts-expect-error - https://github.com/multiformats/js-cid/pull/138 format: multicodec.getName(cid.code), hashAlg: multihash.decode(cid.multihash).name } From 4bf0f88aae2d3b6d761323c25863dc1e9ff0ca96 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Thu, 14 Jan 2021 14:25:14 +0000 Subject: [PATCH 3/3] chore: upgrade cids module --- examples/custom-ipld-formats/package.json | 2 +- examples/traverse-ipld-graphs/package.json | 2 +- packages/interface-ipfs-core/package.json | 2 +- packages/ipfs-cli/package.json | 2 +- packages/ipfs-core-types/package.json | 2 +- packages/ipfs-core-utils/package.json | 2 +- packages/ipfs-core/package.json | 2 +- packages/ipfs-grpc-client/package.json | 2 +- packages/ipfs-http-client/package.json | 2 +- packages/ipfs-http-gateway/package.json | 2 +- packages/ipfs-http-server/package.json | 2 +- packages/ipfs-message-port-protocol/package.json | 2 +- packages/ipfs-message-port-server/package.json | 2 +- 13 files changed, 13 insertions(+), 13 deletions(-) diff --git a/examples/custom-ipld-formats/package.json b/examples/custom-ipld-formats/package.json index 3e4b5f3c01..a8079f40a2 100644 --- a/examples/custom-ipld-formats/package.json +++ b/examples/custom-ipld-formats/package.json @@ -11,7 +11,7 @@ "test-ipfs-example": "^2.0.3" }, "dependencies": { - "cids": "^1.0.0", + "cids": "^1.1.5", "ipfs-daemon": "^0.3.1", "ipfs-core": "^0.3.0", "ipfs-http-client": "^48.1.2", diff --git a/examples/traverse-ipld-graphs/package.json b/examples/traverse-ipld-graphs/package.json index 27747ba097..19a2176739 100644 --- a/examples/traverse-ipld-graphs/package.json +++ b/examples/traverse-ipld-graphs/package.json @@ -13,7 +13,7 @@ "test-ipfs-example": "^2.0.3" }, "dependencies": { - "cids": "^1.0.0", + "cids": "^1.1.5", "ipfs": "^0.52.2", "ipld-block": "^0.11.0", "ipld-dag-pb": "^0.20.0", diff --git a/packages/interface-ipfs-core/package.json b/packages/interface-ipfs-core/package.json index a8782fab51..7990ac9b30 100644 --- a/packages/interface-ipfs-core/package.json +++ b/packages/interface-ipfs-core/package.json @@ -34,7 +34,7 @@ "chai": "^4.2.0", "chai-as-promised": "^7.1.1", "chai-subset": "^1.6.0", - "cids": "^1.0.0", + "cids": "^1.1.5", "delay": "^4.4.0", "dirty-chai": "^2.0.1", "err-code": "^2.0.3", diff --git a/packages/ipfs-cli/package.json b/packages/ipfs-cli/package.json index 58ea6f5fab..cb472d96e1 100644 --- a/packages/ipfs-cli/package.json +++ b/packages/ipfs-cli/package.json @@ -32,7 +32,7 @@ "bignumber.js": "^9.0.0", "byteman": "^1.3.5", "cid-tool": "^1.0.0", - "cids": "^1.0.0", + "cids": "^1.1.5", "debug": "^4.1.1", "err-code": "^2.0.3", "execa": "^5.0.0", diff --git a/packages/ipfs-core-types/package.json b/packages/ipfs-core-types/package.json index 5330b4b88b..1061d8c6bc 100644 --- a/packages/ipfs-core-types/package.json +++ b/packages/ipfs-core-types/package.json @@ -29,7 +29,7 @@ ], "license": "(Apache-2.0 OR MIT)", "dependencies": { - "cids": "^1.0.0", + "cids": "^1.1.5", "multiaddr": "^8.0.0", "peer-id": "^0.14.1" }, diff --git a/packages/ipfs-core-utils/package.json b/packages/ipfs-core-utils/package.json index 617b0462dc..13faf44536 100644 --- a/packages/ipfs-core-utils/package.json +++ b/packages/ipfs-core-utils/package.json @@ -41,7 +41,7 @@ "any-signal": "^2.0.0", "blob-to-it": "^1.0.1", "browser-readablestream-to-it": "^1.0.1", - "cids": "^1.0.0", + "cids": "^1.1.5", "err-code": "^2.0.3", "ipfs-utils": "^5.0.0", "ipfs-core-types": "^0.1.0", diff --git a/packages/ipfs-core/package.json b/packages/ipfs-core/package.json index 92d2d7e28d..b5d7210546 100644 --- a/packages/ipfs-core/package.json +++ b/packages/ipfs-core/package.json @@ -57,7 +57,7 @@ "array-shuffle": "^1.0.1", "bignumber.js": "^9.0.0", "cbor": "^5.1.0", - "cids": "^1.0.0", + "cids": "^1.1.5", "class-is": "^1.1.0", "dag-cbor-links": "^2.0.0", "datastore-core": "^2.0.0", diff --git a/packages/ipfs-grpc-client/package.json b/packages/ipfs-grpc-client/package.json index 6d4b0e392b..d901d64748 100644 --- a/packages/ipfs-grpc-client/package.json +++ b/packages/ipfs-grpc-client/package.json @@ -42,7 +42,7 @@ "dependencies": { "@improbable-eng/grpc-web": "^0.13.0", "change-case": "^4.1.1", - "cids": "^1.0.0", + "cids": "^1.1.5", "debug": "^4.1.1", "err-code": "^2.0.3", "ipfs-core-utils": "^0.5.0", diff --git a/packages/ipfs-http-client/package.json b/packages/ipfs-http-client/package.json index d8a35f7083..08138bed6c 100644 --- a/packages/ipfs-http-client/package.json +++ b/packages/ipfs-http-client/package.json @@ -53,7 +53,7 @@ "dependencies": { "any-signal": "^2.0.0", "bignumber.js": "^9.0.0", - "cids": "^1.0.0", + "cids": "^1.1.5", "debug": "^4.1.1", "form-data": "^3.0.0", "ipfs-core-utils": "^0.5.4", diff --git a/packages/ipfs-http-gateway/package.json b/packages/ipfs-http-gateway/package.json index 3af7d78d64..ac84fe2f13 100644 --- a/packages/ipfs-http-gateway/package.json +++ b/packages/ipfs-http-gateway/package.json @@ -32,7 +32,7 @@ "@hapi/ammo": "^5.0.1", "@hapi/boom": "^9.1.0", "@hapi/hapi": "^20.0.0", - "cids": "^1.0.0", + "cids": "^1.1.5", "debug": "^4.1.1", "hapi-pino": "^8.3.0", "ipfs-core-utils": "^0.5.4", diff --git a/packages/ipfs-http-server/package.json b/packages/ipfs-http-server/package.json index b792c563a2..3367e648d9 100644 --- a/packages/ipfs-http-server/package.json +++ b/packages/ipfs-http-server/package.json @@ -32,7 +32,7 @@ "@hapi/boom": "^9.1.0", "@hapi/content": "^5.0.2", "@hapi/hapi": "^20.0.0", - "cids": "^1.0.0", + "cids": "^1.1.5", "debug": "^4.1.1", "dlv": "^1.1.3", "err-code": "^2.0.3", diff --git a/packages/ipfs-message-port-protocol/package.json b/packages/ipfs-message-port-protocol/package.json index 89513ec3d4..04c7ff98d2 100644 --- a/packages/ipfs-message-port-protocol/package.json +++ b/packages/ipfs-message-port-protocol/package.json @@ -42,7 +42,7 @@ "dep-check": "aegir dep-check -i typescript -i rimraf" }, "dependencies": { - "cids": "^1.0.0", + "cids": "^1.1.5", "ipld-block": "^0.11.0" }, "devDependencies": { diff --git a/packages/ipfs-message-port-server/package.json b/packages/ipfs-message-port-server/package.json index c405182dec..ddaef833c1 100644 --- a/packages/ipfs-message-port-server/package.json +++ b/packages/ipfs-message-port-server/package.json @@ -50,7 +50,7 @@ "devDependencies": { "@types/it-all": "^1.0.0", "aegir": "^29.2.2", - "cids": "^1.0.0", + "cids": "^1.1.5", "ipfs-utils": "^5.0.0", "rimraf": "^3.0.2", "typescript": "4.0.x"