From 6ff1b981f353449a15627ec0ec724e6e4a3fbb8d Mon Sep 17 00:00:00 2001 From: Peter Somogyvari Date: Thu, 25 Jun 2020 12:21:26 -0700 Subject: [PATCH] fix(test-tooling): bind test ledgers to port zero for macOS This makes the ledger container based integration tests pass on macOS that also leads to the CI script finally passing on Macs in general. Yaaay! Fixes #186 Signed-off-by: Peter Somogyvari --- .../main/typescript/besu/besu-test-ledger.ts | 131 +++++++++++------- .../http-echo/http-echo-container.ts | 72 +++++++--- .../typescript/quorum/quorum-test-ledger.ts | 112 +++++++++------ .../constructor-validates-options.ts | 19 ++- .../constructor-validates-options.ts | 14 +- 5 files changed, 229 insertions(+), 119 deletions(-) diff --git a/packages/cactus-test-tooling/src/main/typescript/besu/besu-test-ledger.ts b/packages/cactus-test-tooling/src/main/typescript/besu/besu-test-ledger.ts index 10fd47a5bb..2330e111c5 100644 --- a/packages/cactus-test-tooling/src/main/typescript/besu/besu-test-ledger.ts +++ b/packages/cactus-test-tooling/src/main/typescript/besu/besu-test-ledger.ts @@ -1,5 +1,5 @@ -import Docker, { Container } from "dockerode"; -import isPortReachable from "is-port-reachable"; +import Docker, { Container, ContainerInfo } from "dockerode"; +import axios from "axios"; import Joi from "joi"; import tar from "tar-stream"; import { EventEmitter } from "events"; @@ -56,10 +56,9 @@ export class BesuTestLedger implements ITestLedger { } public getContainer(): Container { + const fnTag = "BesuTestLedger#getContainer()"; if (!this.container) { - throw new Error( - `BesuTestLedger#getBesuKeyPair() container wasn't started by this instance yet.` - ); + throw new Error(`${fnTag} container not yet started by this instance.`); } else { return this.container; } @@ -70,8 +69,9 @@ export class BesuTestLedger implements ITestLedger { } public async getRpcApiHttpHost(): Promise { - const ipAddress: string = await this.getContainerIpAddress(); - return `http://${ipAddress}:${this.rpcApiHttpPort}`; + const ipAddress: string = "127.0.0.1"; + const hostPort: number = await this.getRpcApiPublicPort(); + return `http://${ipAddress}:${hostPort}`; } public async getFileContents(filePath: string): Promise { @@ -137,16 +137,10 @@ export class BesuTestLedger implements ITestLedger { "9001/tcp": {}, // supervisord - HTTP "9545/tcp": {}, // besu metrics }, - Hostconfig: { - PortBindings: { - // [`${this.rpcApiHttpPort}/tcp`]: [{ HostPort: '8545', }], - // '8546/tcp': [{ HostPort: '8546', }], - // '8080/tcp': [{ HostPort: '8080', }], - // '8888/tcp': [{ HostPort: '8888', }], - // '9001/tcp': [{ HostPort: '9001', }], - // '9545/tcp': [{ HostPort: '9545', }], - }, - }, + // This is a workaround needed for macOS which has issues with routing + // to docker container's IP addresses directly... + // https://stackoverflow.com/a/39217691 + PublishAllPorts: true, }, {}, (err: any) => { @@ -158,15 +152,8 @@ export class BesuTestLedger implements ITestLedger { eventEmitter.once("start", async (container: Container) => { this.container = container; - // once the container has started, we wait until the the besu RPC API starts listening on the designated port - // which we determine by continously trying to establish a socket until it actually works - const host: string = await this.getContainerIpAddress(); try { - let reachable: boolean = false; - do { - reachable = await isPortReachable(this.rpcApiHttpPort, { host }); - await new Promise((resolve2) => setTimeout(resolve2, 100)); - } while (!reachable); + await this.waitForHealthCheck(); resolve(container); } catch (ex) { reject(ex); @@ -175,7 +162,27 @@ export class BesuTestLedger implements ITestLedger { }); } + public async waitForHealthCheck(timeoutMs: number = 120000): Promise { + const fnTag = "BesuTestLedger#waitForHealthCheck()"; + const httpUrl = await this.getRpcApiHttpHost(); + const startedAt = Date.now(); + let reachable: boolean = false; + do { + try { + const res = await axios.get(httpUrl); + reachable = res.status > 199 && res.status < 300; + } catch (ex) { + reachable = false; + if (Date.now() >= startedAt + timeoutMs) { + throw new Error(`${fnTag} timed out (${timeoutMs}ms) -> ${ex.stack}`); + } + } + await new Promise((resolve2) => setTimeout(resolve2, 100)); + } while (!reachable); + } + public stop(): Promise { + const fnTag = "BesuTestLedger#stop()"; return new Promise((resolve, reject) => { if (this.container) { this.container.stop({}, (err: any, result: any) => { @@ -186,54 +193,74 @@ export class BesuTestLedger implements ITestLedger { } }); } else { - return reject( - new Error( - `BesuTestLedger#stop() Container was not running to begin with.` - ) - ); + return reject(new Error(`${fnTag} Container was not running.`)); } }); } public destroy(): Promise { + const fnTag = "BesuTestLedger#destroy()"; if (this.container) { return this.container.remove(); } else { - return Promise.reject( - new Error( - `BesuTestLedger#destroy() Container was never created, nothing to destroy.` - ) - ); + const ex = new Error(`${fnTag} Container not found, nothing to destroy.`); + return Promise.reject(ex); } } - public async getContainerIpAddress(): Promise { + protected async getContainerInfo(): Promise { const docker = new Docker(); - const containerImageName = this.getContainerImageName(); - const containerInfos: Docker.ContainerInfo[] = await docker.listContainers( - {} - ); + const image = this.getContainerImageName(); + const containerInfos = await docker.listContainers({}); + + const aContainerInfo = containerInfos.find((ci) => ci.Image === image); + + if (aContainerInfo) { + return aContainerInfo; + } else { + throw new Error(`BesuTestLedger#getContainerInfo() no image "${image}"`); + } + } + + public async getRpcApiPublicPort(): Promise { + const fnTag = "BesuTestLedger#getRpcApiPublicPort()"; + const aContainerInfo = await this.getContainerInfo(); + const { rpcApiHttpPort: thePort } = this; + const { Ports: ports } = aContainerInfo; + + if (ports.length < 1) { + throw new Error(`${fnTag} no ports exposed or mapped at all`); + } + const mapping = ports.find((x) => x.PrivatePort === thePort); + if (mapping) { + if (!mapping.PublicPort) { + throw new Error(`${fnTag} port ${thePort} mapped but not public`); + } else if (mapping.IP !== "0.0.0.0") { + throw new Error(`${fnTag} port ${thePort} mapped to localhost`); + } else { + return mapping.PublicPort; + } + } else { + throw new Error(`${fnTag} no mapping found for ${thePort}`); + } + } + + public async getContainerIpAddress(): Promise { + const fnTag = "BesuTestLedger#getContainerIpAddress()"; + const aContainerInfo = await this.getContainerInfo(); - const aContainerInfo = containerInfos.find( - (ci) => ci.Image === containerImageName - ); if (aContainerInfo) { const { NetworkSettings } = aContainerInfo; const networkNames: string[] = Object.keys(NetworkSettings.Networks); if (networkNames.length < 1) { - throw new Error( - `BesuTestLedger#getContainerIpAddress() no network found: ${JSON.stringify( - NetworkSettings - )}` - ); + throw new Error(`${fnTag} container not connected to any networks`); } else { - // return IP address of container on the first network that we found it connected to. Make this configurable? + // return IP address of container on the first network that we found + // it connected to. Make this configurable? return NetworkSettings.Networks[networkNames[0]].IPAddress; } } else { - throw new Error( - `BesuTestLedger#getContainerIpAddress() cannot find container image ${this.containerImageName}` - ); + throw new Error(`${fnTag} cannot find image: ${this.containerImageName}`); } } diff --git a/packages/cactus-test-tooling/src/main/typescript/http-echo/http-echo-container.ts b/packages/cactus-test-tooling/src/main/typescript/http-echo/http-echo-container.ts index 24277dbaf7..fe7958ed9c 100644 --- a/packages/cactus-test-tooling/src/main/typescript/http-echo/http-echo-container.ts +++ b/packages/cactus-test-tooling/src/main/typescript/http-echo/http-echo-container.ts @@ -1,4 +1,4 @@ -import Docker, { Container } from "dockerode"; +import Docker, { Container, ContainerInfo } from "dockerode"; import isPortReachable from "is-port-reachable"; import Joi from "joi"; import { EventEmitter } from "events"; @@ -80,10 +80,13 @@ export class HttpEchoContainer implements ITestLedger { ["--port", this.httpPort.toString(10)], [], { - ExposedPorts: {}, - Hostconfig: { - PortBindings: {}, + ExposedPorts: { + [`${this.httpPort}/tcp`]: {}, }, + // This is a workaround needed for macOS which has issues with routing + // to docker container's IP addresses directly... + // https://stackoverflow.com/a/39217691 + PublishAllPorts: true, }, {}, (err: any) => { @@ -95,11 +98,12 @@ export class HttpEchoContainer implements ITestLedger { eventEmitter.once("start", async (container: Container) => { this.container = container; - const host: string = await this.getContainerIpAddress(); + const host: string = "127.0.0.1"; + const hostPort = await this.getPublicHttpPort(); try { let reachable: boolean = false; do { - reachable = await isPortReachable(this.httpPort, { host }); + reachable = await isPortReachable(hostPort, { host }); await new Promise((resolve2) => setTimeout(resolve2, 100)); } while (!reachable); resolve(container); @@ -142,31 +146,59 @@ export class HttpEchoContainer implements ITestLedger { } } - public async getContainerIpAddress(): Promise { + protected async getContainerInfo(): Promise { + const fnTag = "HttpEchoContainer#getContainerInfo()"; const docker = new Docker(); - const imageName = this.getImageName(); - const containerInfos: Docker.ContainerInfo[] = await docker.listContainers( - {} - ); + const image = this.getImageName(); + const containerInfos = await docker.listContainers({}); + + const aContainerInfo = containerInfos.find((ci) => ci.Image === image); + + if (aContainerInfo) { + return aContainerInfo; + } else { + throw new Error(`${fnTag} no image found: "${image}"`); + } + } + + public async getPublicHttpPort(): Promise { + const fnTag = "HttpEchoContainer#getRpcApiPublicPort()"; + const aContainerInfo = await this.getContainerInfo(); + const { httpPort: thePort } = this; + const { Ports: ports } = aContainerInfo; + + if (ports.length < 1) { + throw new Error(`${fnTag} no ports exposed or mapped at all`); + } + const mapping = ports.find((x) => x.PrivatePort === thePort); + if (mapping) { + if (!mapping.PublicPort) { + throw new Error(`${fnTag} port ${thePort} mapped but not public`); + } else if (mapping.IP !== "0.0.0.0") { + throw new Error(`${fnTag} port ${thePort} mapped to localhost`); + } else { + return mapping.PublicPort; + } + } else { + throw new Error(`${fnTag} no mapping found for ${thePort}`); + } + } + + public async getContainerIpAddress(): Promise { + const fnTag = "HttpEchoContainer#getContainerIpAddress()"; + const aContainerInfo = await this.getContainerInfo(); - const aContainerInfo = containerInfos.find((ci) => ci.Image === imageName); if (aContainerInfo) { const { NetworkSettings } = aContainerInfo; const networkNames: string[] = Object.keys(NetworkSettings.Networks); if (networkNames.length < 1) { - throw new Error( - `HttpEchoContainer#getContainerIpAddress() no network found: ${JSON.stringify( - NetworkSettings - )}` - ); + throw new Error(`${fnTag} container not on any networks`); } else { // return IP address of container on the first network that we found it connected to. Make this configurable? return NetworkSettings.Networks[networkNames[0]].IPAddress; } } else { - throw new Error( - `HttpEchoContainer#getContainerIpAddress() cannot find container image ${this.imageName}` - ); + throw new Error(`${fnTag} cannot find container image ${this.imageName}`); } } diff --git a/packages/cactus-test-tooling/src/main/typescript/quorum/quorum-test-ledger.ts b/packages/cactus-test-tooling/src/main/typescript/quorum/quorum-test-ledger.ts index 0f649761d4..45743dc8e9 100644 --- a/packages/cactus-test-tooling/src/main/typescript/quorum/quorum-test-ledger.ts +++ b/packages/cactus-test-tooling/src/main/typescript/quorum/quorum-test-ledger.ts @@ -1,8 +1,8 @@ -import Docker, { Container } from "dockerode"; -import isPortReachable from "is-port-reachable"; +import { EventEmitter } from "events"; +import axios from "axios"; +import Docker, { Container, ContainerInfo } from "dockerode"; import Joi from "joi"; import tar from "tar-stream"; -import { EventEmitter } from "events"; import { ITestLedger } from "../i-test-ledger"; import { Streams } from "../common/streams"; import { IKeyPair } from "../i-key-pair"; @@ -60,10 +60,9 @@ export class QuorumTestLedger implements ITestLedger { } public getContainer(): Container { + const fnTag = "QuorumTestLedger#getQuorumKeyPair()"; if (!this.container) { - throw new Error( - `QuorumTestLedger#getQuorumKeyPair() container wasn't started by this instance yet.` - ); + throw new Error(`${fnTag} container not started by this instance yet.`); } else { return this.container; } @@ -74,8 +73,9 @@ export class QuorumTestLedger implements ITestLedger { } public async getRpcApiHttpHost(): Promise { - const ipAddress: string = await this.getContainerIpAddress(); - return `http://${ipAddress}:${this.rpcApiHttpPort}`; + const ipAddress: string = "127.0.0.1"; + const hostPort = await this.getRpcApiPublicPort(); + return `http://${ipAddress}:${hostPort}`; } public async getFileContents(filePath: string): Promise { @@ -151,16 +151,10 @@ export class QuorumTestLedger implements ITestLedger { "9001/tcp": {}, // supervisord - HTTP "9545/tcp": {}, // quorum metrics }, - Hostconfig: { - PortBindings: { - // [`${this.rpcApiHttpPort}/tcp`]: [{ HostPort: '8545', }], - // '8546/tcp': [{ HostPort: '8546', }], - // '8080/tcp': [{ HostPort: '8080', }], - // '8888/tcp': [{ HostPort: '8888', }], - // '9001/tcp': [{ HostPort: '9001', }], - // '9545/tcp': [{ HostPort: '9545', }], - }, - }, + // This is a workaround needed for macOS which has issues with routing + // to docker container's IP addresses directly... + // https://stackoverflow.com/a/39217691 + PublishAllPorts: true, }, {}, (err: any) => { @@ -172,15 +166,8 @@ export class QuorumTestLedger implements ITestLedger { eventEmitter.once("start", async (container: Container) => { this.container = container; - // once the container has started, we wait until the the quorum RPC API starts listening on the designated port - // which we determine by continously trying to establish a socket until it actually works - const host: string = await this.getContainerIpAddress(); try { - let reachable: boolean = false; - do { - reachable = await isPortReachable(this.rpcApiHttpPort, { host }); - await new Promise((resolve2) => setTimeout(resolve2, 100)); - } while (!reachable); + await this.waitForHealthCheck(); resolve(container); } catch (ex) { reject(ex); @@ -189,6 +176,25 @@ export class QuorumTestLedger implements ITestLedger { }); } + public async waitForHealthCheck(timeoutMs: number = 120000): Promise { + const fnTag = "QuorumTestLedger#waitForHealthCheck()"; + const httpUrl = await this.getRpcApiHttpHost(); + const startedAt = Date.now(); + let reachable: boolean = false; + do { + try { + const res = await axios.get(httpUrl); + reachable = res.status > 199 && res.status < 300; + } catch (ex) { + reachable = false; + if (Date.now() >= startedAt + timeoutMs) { + throw new Error(`${fnTag} timed out (${timeoutMs}ms) -> ${ex.stack}`); + } + } + await new Promise((resolve2) => setTimeout(resolve2, 100)); + } while (!reachable); + } + public stop(): Promise { return new Promise((resolve, reject) => { if (this.container) { @@ -221,25 +227,53 @@ export class QuorumTestLedger implements ITestLedger { } } - public async getContainerIpAddress(): Promise { + protected async getContainerInfo(): Promise { + const fnTag = "QuorumTestLedger#getContainerInfo()"; const docker = new Docker(); - const containerImageName = this.getContainerImageName(); - const containerInfos: Docker.ContainerInfo[] = await docker.listContainers( - {} - ); + const image = this.getContainerImageName(); + const containerInfos = await docker.listContainers({}); + + const aContainerInfo = containerInfos.find((ci) => ci.Image === image); + + if (aContainerInfo) { + return aContainerInfo; + } else { + throw new Error(`${fnTag} no image found: "${image}"`); + } + } + + public async getRpcApiPublicPort(): Promise { + const fnTag = "QuorumTestLedger#getRpcApiPublicPort()"; + const aContainerInfo = await this.getContainerInfo(); + const { rpcApiHttpPort: thePort } = this; + const { Ports: ports } = aContainerInfo; + + if (ports.length < 1) { + throw new Error(`${fnTag} no ports exposed or mapped at all`); + } + const mapping = ports.find((x) => x.PrivatePort === thePort); + if (mapping) { + if (!mapping.PublicPort) { + throw new Error(`${fnTag} port ${thePort} mapped but not public`); + } else if (mapping.IP !== "0.0.0.0") { + throw new Error(`${fnTag} port ${thePort} mapped to localhost`); + } else { + return mapping.PublicPort; + } + } else { + throw new Error(`${fnTag} no mapping found for ${thePort}`); + } + } + + public async getContainerIpAddress(): Promise { + const fnTag = "QuorumTestLedger#getContainerIpAddress()"; + const aContainerInfo = await this.getContainerInfo(); - const aContainerInfo = containerInfos.find( - (ci) => ci.Image === containerImageName - ); if (aContainerInfo) { const { NetworkSettings } = aContainerInfo; const networkNames: string[] = Object.keys(NetworkSettings.Networks); if (networkNames.length < 1) { - throw new Error( - `QuorumTestLedger#getContainerIpAddress() no network found: ${JSON.stringify( - NetworkSettings - )}` - ); + throw new Error(`${fnTag} container not connected to any network`); } else { // return IP address of container on the first network that we found it connected to. Make this configurable? return NetworkSettings.Networks[networkNames[0]].IPAddress; diff --git a/packages/cactus-test-tooling/src/test/typescript/integration/besu/besu-test-ledger/constructor-validates-options.ts b/packages/cactus-test-tooling/src/test/typescript/integration/besu/besu-test-ledger/constructor-validates-options.ts index 68b45c611a..1e2781eed2 100644 --- a/packages/cactus-test-tooling/src/test/typescript/integration/besu/besu-test-ledger/constructor-validates-options.ts +++ b/packages/cactus-test-tooling/src/test/typescript/integration/besu/besu-test-ledger/constructor-validates-options.ts @@ -1,11 +1,12 @@ // tslint:disable-next-line: no-var-requires const tap = require("tap"); +import isPortReachable from "is-port-reachable"; +import { Container } from "dockerode"; import { BesuTestLedger, IKeyPair, isIKeyPair, } from "../../../../../main/typescript/public-api"; -import { Container } from "dockerode"; tap.test("constructor throws if invalid input is provided", (assert: any) => { assert.ok(BesuTestLedger); @@ -24,21 +25,29 @@ tap.test( tap.test("starts/stops/destroys a docker container", async (assert: any) => { const besuTestLedger = new BesuTestLedger(); + assert.tearDown(() => besuTestLedger.stop()); + assert.tearDown(() => besuTestLedger.destroy()); + const container: Container = await besuTestLedger.start(); assert.ok(container); const ipAddress: string = await besuTestLedger.getContainerIpAddress(); assert.ok(ipAddress); assert.ok(ipAddress.length); + const hostPort: number = await besuTestLedger.getRpcApiPublicPort(); + assert.ok(hostPort, "getRpcApiPublicPort() returns truthy OK"); + assert.ok(isFinite(hostPort), "getRpcApiPublicPort() returns finite OK"); + + const isReachable = await isPortReachable(hostPort, { host: "localhost" }); + assert.ok(isReachable, `HostPort ${hostPort} is reachable via localhost`); + const besuKeyPair: IKeyPair = await besuTestLedger.getBesuKeyPair(); - assert.ok(besuKeyPair); + assert.ok(besuKeyPair, "getBesuKeyPair() returns truthy OK"); assert.ok(isIKeyPair(besuKeyPair)); const orionKeyPair: IKeyPair = await besuTestLedger.getOrionKeyPair(); - assert.ok(orionKeyPair); + assert.ok(orionKeyPair, "getOrionKeyPair() returns truthy OK"); assert.ok(isIKeyPair(orionKeyPair)); - await besuTestLedger.stop(); - await besuTestLedger.destroy(); assert.end(); }); diff --git a/packages/cactus-test-tooling/src/test/typescript/integration/quorum/quorum-test-ledger/constructor-validates-options.ts b/packages/cactus-test-tooling/src/test/typescript/integration/quorum/quorum-test-ledger/constructor-validates-options.ts index 34cec60685..d82e979409 100644 --- a/packages/cactus-test-tooling/src/test/typescript/integration/quorum/quorum-test-ledger/constructor-validates-options.ts +++ b/packages/cactus-test-tooling/src/test/typescript/integration/quorum/quorum-test-ledger/constructor-validates-options.ts @@ -1,11 +1,12 @@ // tslint:disable-next-line: no-var-requires const tap = require("tap"); +import isPortReachable from "is-port-reachable"; +import { Container } from "dockerode"; import { QuorumTestLedger, IKeyPair, isIKeyPair, } from "../../../../../main/typescript/public-api"; -import { Container } from "dockerode"; tap.test("constructor throws if invalid input is provided", (assert: any) => { assert.ok(QuorumTestLedger); @@ -24,12 +25,21 @@ tap.test( tap.test("starts/stops/destroys a docker container", async (assert: any) => { const ledger = new QuorumTestLedger(); + assert.tearDown(() => ledger.stop()); + assert.tearDown(() => ledger.destroy()); const container: Container = await ledger.start(); assert.ok(container); const ipAddress: string = await ledger.getContainerIpAddress(); assert.ok(ipAddress); assert.ok(ipAddress.length); + const hostPort: number = await ledger.getRpcApiPublicPort(); + assert.ok(hostPort, "getRpcApiPublicPort() returns truthy OK"); + assert.ok(isFinite(hostPort), "getRpcApiPublicPort() returns finite OK"); + + const isReachable = await isPortReachable(hostPort, { host: "localhost" }); + assert.ok(isReachable, `HostPort ${hostPort} is reachable via localhost`); + const quorumKeyPair: IKeyPair = await ledger.getQuorumKeyPair(); assert.ok(quorumKeyPair); assert.ok(isIKeyPair(quorumKeyPair)); @@ -38,7 +48,5 @@ tap.test("starts/stops/destroys a docker container", async (assert: any) => { assert.ok(tesseraKeyPair); assert.ok(isIKeyPair(tesseraKeyPair)); - await ledger.stop(); - await ledger.destroy(); assert.end(); });