diff --git a/packages/cactus-test-tooling/package-lock.json b/packages/cactus-test-tooling/package-lock.json index f42d35db2cc..0621ec6258f 100644 --- a/packages/cactus-test-tooling/package-lock.json +++ b/packages/cactus-test-tooling/package-lock.json @@ -93,6 +93,11 @@ "@types/node": "*" } }, + "@types/retry": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/@types/retry/-/retry-0.12.0.tgz", + "integrity": "sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA==" + }, "@types/secp256k1": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/@types/secp256k1/-/secp256k1-4.0.1.tgz", @@ -3064,6 +3069,15 @@ "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz", "integrity": "sha1-P7z7FbiZpEEjs0ttzBi3JDNqLK4=" }, + "p-retry": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/p-retry/-/p-retry-4.4.0.tgz", + "integrity": "sha512-gVB/tBsG+3AHI1SyDHRrX6n9ZL0Bcbifps9W9/Bgu3Oyu4/OrAh8SvDzDsvpP0oxfCt3oWNT+0fQ9LyUGwBTLg==", + "requires": { + "@types/retry": "^0.12.0", + "retry": "^0.12.0" + } + }, "p-timeout": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-1.2.1.tgz", @@ -3316,6 +3330,11 @@ "lowercase-keys": "^1.0.0" } }, + "retry": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/retry/-/retry-0.12.0.tgz", + "integrity": "sha1-G0KmJmoh8HQh0bC1S33BZ7AcATs=" + }, "rimraf": { "version": "2.7.1", "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.7.1.tgz", diff --git a/packages/cactus-test-tooling/package.json b/packages/cactus-test-tooling/package.json index 7abec1e5cb3..6c33b777947 100644 --- a/packages/cactus-test-tooling/package.json +++ b/packages/cactus-test-tooling/package.json @@ -31,7 +31,10 @@ "ignore": [ "src/**/generated/*" ], - "extensions": ["ts", "json"], + "extensions": [ + "ts", + "json" + ], "quiet": true, "verbose": false, "runOnChangeOnly": true @@ -91,6 +94,7 @@ "loglevel": "1.6.7", "loglevel-plugin-prefix": "0.8.4", "node-ssh": "11.1.1", + "p-retry": "4.4.0", "tar-stream": "2.1.2", "typescript-optional": "2.0.1", "web3": "1.2.7" @@ -100,8 +104,8 @@ "@types/extract-zip": "1.6.2", "@types/fs-extra": "8.1.0", "@types/joi": "14.3.4", - "@types/ssh2": "0.5.44", "@types/node-ssh": "7.0.0", + "@types/ssh2": "0.5.44", "@types/tar-stream": "2.1.0" } } diff --git a/packages/cactus-test-tooling/src/main/typescript/common/containers.ts b/packages/cactus-test-tooling/src/main/typescript/common/containers.ts index 78ded092aa7..f3d14f917c2 100644 --- a/packages/cactus-test-tooling/src/main/typescript/common/containers.ts +++ b/packages/cactus-test-tooling/src/main/typescript/common/containers.ts @@ -5,9 +5,15 @@ import { Container, ContainerInfo } from "dockerode"; import Dockerode from "dockerode"; import tar from "tar-stream"; import fs from "fs-extra"; +import pRetry from "p-retry"; import { Streams } from "../common/streams"; -import { Checks, LoggerProvider, Strings } from "@hyperledger/cactus-common"; -import { LogLevelDesc } from "loglevel"; +import { + Checks, + LogLevelDesc, + LoggerProvider, + Strings, + ILoggerOptions, +} from "@hyperledger/cactus-common"; export interface IPruneDockerResourcesRequest { logLevel?: LogLevelDesc; @@ -297,33 +303,62 @@ export class Containers { return NetworkSettings.Networks[networkNames[0]].IPAddress; } } - public static pullImage( - containerNameAndTag: string, - pullOptions: any = {}, + imageFqn: string, + options: any = {}, + logLevel?: LogLevelDesc, + ): Promise { + const defualtLoggerOptions: ILoggerOptions = { + label: "containers#pullImage()", + level: logLevel || "INFO", + }; + const log = LoggerProvider.getOrCreate(defualtLoggerOptions); + const task = () => Containers.tryPullImage(imageFqn, options, logLevel); + const retryOptions: pRetry.Options = { + retries: 6, + onFailedAttempt: async (ex) => { + log.debug(`Failed attempt at pulling container image ${imageFqn}`, ex); + }, + }; + return pRetry(task, retryOptions); + } + + public static tryPullImage( + imageFqn: string, + options: any = {}, + logLevel?: LogLevelDesc, ): Promise { return new Promise((resolve, reject) => { + const loggerOptions: ILoggerOptions = { + label: "containers#tryPullImage()", + level: logLevel || "INFO", + }; + const log = LoggerProvider.getOrCreate(loggerOptions); + const docker = new Dockerode(); - docker.pull( - containerNameAndTag, - pullOptions, - (pullError: any, stream: any) => { - if (pullError) { - reject(pullError); - } else { - docker.modem.followProgress( - stream, - (progressError: any, output: any[]) => { - if (progressError) { - reject(progressError); - } else { - resolve(output); - } - }, - ); - } - }, - ); + + const pullStreamStartedHandler = (pullError: any, stream: any) => { + if (pullError) { + log.error(`Could not even start ${imageFqn} pull:`, pullError); + reject(pullError); + } else { + log.debug(`Started ${imageFqn} pull progress stream OK`); + docker.modem.followProgress( + stream, + (progressError: any, output: any[]) => { + if (progressError) { + log.error(`Failed to finish ${imageFqn} pull:`, progressError); + reject(progressError); + } else { + log.debug(`Finished ${imageFqn} pull completely OK`); + resolve(output); + } + }, + ); + } + }; + + docker.pull(imageFqn, options, pullStreamStartedHandler); }); }