-
Notifications
You must be signed in to change notification settings - Fork 445
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: add connection monitor (#2644)
Adds a connection monitor that periodically ensures remote peers are still online and contactable by trying to send a single byte via the ping protocol, and sets the `.rtt` property of the connection to how long it took. If the ping protocol is not supported by the remote, it tries to infer the round trip time by how long it took to fail. If the remote is unresponsive or opening the stream fails for any other reason, the connection is aborted with the throw error. It's possible to configure the ping interval, how long we wait before considering a peer to be inactive and whether or not to close the connection on failure. Closes #2643 --------- Co-authored-by: Chad Nehemiah <chad.nehemiah94@gmail.com>
- Loading branch information
1 parent
c5dba70
commit 7939dbd
Showing
6 changed files
with
270 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,134 @@ | ||
| import { serviceCapabilities } from '@libp2p/interface' | ||
| import { AdaptiveTimeout } from '@libp2p/utils/adaptive-timeout' | ||
| import { byteStream } from 'it-byte-stream' | ||
| import type { ComponentLogger, Logger, Metrics, Startable } from '@libp2p/interface' | ||
| import type { ConnectionManager } from '@libp2p/interface-internal' | ||
| import type { AdaptiveTimeoutInit } from '@libp2p/utils/adaptive-timeout' | ||
|
|
||
| const DEFAULT_PING_INTERVAL_MS = 10000 | ||
|
|
||
| export interface ConnectionMonitorInit { | ||
| /** | ||
| * Whether the connection monitor is enabled | ||
| * | ||
| * @default true | ||
| */ | ||
| enabled?: boolean | ||
|
|
||
| /** | ||
| * How often to ping remote peers in ms | ||
| * | ||
| * @default 10000 | ||
| */ | ||
| pingInterval?: number | ||
|
|
||
| /** | ||
| * Timeout settings for how long the ping is allowed to take before the | ||
| * connection will be judged inactive and aborted. | ||
| * | ||
| * The timeout is adaptive to cope with slower networks or nodes that | ||
| * have changing network characteristics, such as mobile. | ||
| */ | ||
| pingTimeout?: Omit<AdaptiveTimeoutInit, 'metricsName' | 'metrics'> | ||
|
|
||
| /** | ||
| * If true, any connection that fails the ping will be aborted | ||
| * | ||
| * @default true | ||
| */ | ||
| abortConnectionOnPingFailure?: boolean | ||
| } | ||
|
|
||
| export interface ConnectionMonitorComponents { | ||
| logger: ComponentLogger | ||
| connectionManager: ConnectionManager | ||
| metrics?: Metrics | ||
| } | ||
|
|
||
| export class ConnectionMonitor implements Startable { | ||
| private readonly components: ConnectionMonitorComponents | ||
| private readonly log: Logger | ||
| private heartbeatInterval?: ReturnType<typeof setInterval> | ||
| private readonly pingIntervalMs: number | ||
| private abortController?: AbortController | ||
| private readonly timeout: AdaptiveTimeout | ||
|
|
||
| constructor (components: ConnectionMonitorComponents, init: ConnectionMonitorInit = {}) { | ||
| this.components = components | ||
|
|
||
| this.log = components.logger.forComponent('libp2p:connection-monitor') | ||
| this.pingIntervalMs = init.pingInterval ?? DEFAULT_PING_INTERVAL_MS | ||
|
|
||
| this.timeout = new AdaptiveTimeout({ | ||
| ...(init.pingTimeout ?? {}), | ||
| metrics: components.metrics, | ||
| metricName: 'libp2p_connection_monitor_ping_time_milliseconds' | ||
| }) | ||
| } | ||
|
|
||
| readonly [Symbol.toStringTag] = '@libp2p/connection-monitor' | ||
|
|
||
| readonly [serviceCapabilities]: string[] = [ | ||
| '@libp2p/connection-monitor' | ||
| ] | ||
|
|
||
| start (): void { | ||
| this.abortController = new AbortController() | ||
|
|
||
| this.heartbeatInterval = setInterval(() => { | ||
| this.components.connectionManager.getConnections().forEach(conn => { | ||
| Promise.resolve().then(async () => { | ||
| let start = Date.now() | ||
| try { | ||
| const signal = this.timeout.getTimeoutSignal({ | ||
| signal: this.abortController?.signal | ||
| }) | ||
| const stream = await conn.newStream('/ipfs/ping/1.0.0', { | ||
| signal, | ||
| runOnTransientConnection: true | ||
| }) | ||
| const bs = byteStream(stream) | ||
| start = Date.now() | ||
|
|
||
| await Promise.all([ | ||
| bs.write(new Uint8Array(1), { | ||
| signal | ||
| }), | ||
| bs.read(1, { | ||
| signal | ||
| }) | ||
| ]) | ||
|
|
||
| conn.rtt = Date.now() - start | ||
|
|
||
| await bs.unwrap().close({ | ||
| signal | ||
| }) | ||
| } catch (err: any) { | ||
| if (err.code !== 'ERR_UNSUPPORTED_PROTOCOL') { | ||
| throw err | ||
| } | ||
|
|
||
| // protocol was unsupported, but that's ok as it means the remote | ||
| // peer was still alive. We ran multistream-select which means two | ||
| // round trips (e.g. 1x for the mss header, then another for the | ||
| // protocol) so divide the time it took by two | ||
| conn.rtt = (Date.now() - start) / 2 | ||
| } | ||
| }) | ||
| .catch(err => { | ||
| this.log.error('error during heartbeat, aborting connection', err) | ||
| conn.abort(err) | ||
| }) | ||
| }) | ||
| }, this.pingIntervalMs) | ||
| } | ||
|
|
||
| stop (): void { | ||
| this.abortController?.abort() | ||
|
|
||
| if (this.heartbeatInterval != null) { | ||
| clearInterval(this.heartbeatInterval) | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,115 @@ | ||
| /* eslint-env mocha */ | ||
|
|
||
| import { CodeError, start, stop } from '@libp2p/interface' | ||
| import { defaultLogger } from '@libp2p/logger' | ||
| import { expect } from 'aegir/chai' | ||
| import delay from 'delay' | ||
| import { pair } from 'it-pair' | ||
| import { type StubbedInstance, stubInterface } from 'sinon-ts' | ||
| import { ConnectionMonitor } from '../../src/connection-monitor.js' | ||
| import type { ComponentLogger, Stream, Connection } from '@libp2p/interface' | ||
| import type { ConnectionManager } from '@libp2p/interface-internal' | ||
|
|
||
| interface StubbedConnectionMonitorComponents { | ||
| logger: ComponentLogger | ||
| connectionManager: StubbedInstance<ConnectionManager> | ||
| } | ||
|
|
||
| describe('connection monitor', () => { | ||
| let monitor: ConnectionMonitor | ||
| let components: StubbedConnectionMonitorComponents | ||
|
|
||
| beforeEach(() => { | ||
| components = { | ||
| logger: defaultLogger(), | ||
| connectionManager: stubInterface<ConnectionManager>() | ||
| } | ||
| }) | ||
|
|
||
| afterEach(async () => { | ||
| await stop(monitor) | ||
| }) | ||
|
|
||
| it('should monitor the liveness of a connection', async () => { | ||
| monitor = new ConnectionMonitor(components, { | ||
| pingInterval: 10 | ||
| }) | ||
|
|
||
| await start(monitor) | ||
|
|
||
| const connection = stubInterface<Connection>() | ||
| const stream = stubInterface<Stream>({ | ||
| ...pair<any>() | ||
| }) | ||
| connection.newStream.withArgs('/ipfs/ping/1.0.0').resolves(stream) | ||
|
|
||
| components.connectionManager.getConnections.returns([connection]) | ||
|
|
||
| await delay(100) | ||
|
|
||
| expect(connection.rtt).to.be.gte(0) | ||
| }) | ||
|
|
||
| it('should monitor the liveness of a connection that does not support ping', async () => { | ||
| monitor = new ConnectionMonitor(components, { | ||
| pingInterval: 10 | ||
| }) | ||
|
|
||
| await start(monitor) | ||
|
|
||
| const connection = stubInterface<Connection>() | ||
| connection.newStream.withArgs('/ipfs/ping/1.0.0').callsFake(async () => { | ||
| await delay(10) | ||
| throw new CodeError('Unsupported protocol', 'ERR_UNSUPPORTED_PROTOCOL') | ||
| }) | ||
|
|
||
| components.connectionManager.getConnections.returns([connection]) | ||
|
|
||
| await delay(100) | ||
|
|
||
| expect(connection.rtt).to.be.gte(0) | ||
| }) | ||
|
|
||
| it('should abort a connection that times out', async () => { | ||
| monitor = new ConnectionMonitor(components, { | ||
| pingInterval: 50, | ||
| pingTimeout: { | ||
| initialValue: 10 | ||
| } | ||
| }) | ||
|
|
||
| await start(monitor) | ||
|
|
||
| const connection = stubInterface<Connection>() | ||
| connection.newStream.withArgs('/ipfs/ping/1.0.0').callsFake(async (protocols, opts) => { | ||
| await delay(200) | ||
| opts?.signal?.throwIfAborted() | ||
| return stubInterface<Stream>() | ||
| }) | ||
|
|
||
| components.connectionManager.getConnections.returns([connection]) | ||
|
|
||
| await delay(500) | ||
|
|
||
| expect(connection.abort).to.have.property('called', true) | ||
| }) | ||
|
|
||
| it('should abort a connection that fails', async () => { | ||
| monitor = new ConnectionMonitor(components, { | ||
| pingInterval: 10 | ||
| }) | ||
|
|
||
| await start(monitor) | ||
|
|
||
| const connection = stubInterface<Connection>() | ||
| connection.newStream.withArgs('/ipfs/ping/1.0.0').callsFake(async (protocols, opts) => { | ||
| throw new CodeError('Connection closed', 'ERR_CONNECTION_CLOSED') | ||
| }) | ||
|
|
||
| components.connectionManager.getConnections.returns([connection]) | ||
|
|
||
| await delay(100) | ||
|
|
||
| expect(connection.abort).to.have.property('called', true) | ||
| }) | ||
| }) |