From 5063e81f341192e782bf5b38c0d1628a496b50de Mon Sep 17 00:00:00 2001 From: Josh Usiskin Date: Sun, 21 Mar 2021 02:11:08 +0000 Subject: [PATCH] fix(deadline): Windows Workers fail to deploy waiting for Deadline launcher service to restart Fixes #353 Fixed #312 --- .github/workflows/ci.yml | 4 +- .../lib/deadline/lib/render-queue-ref.ts | 7 ++++ .../lib/deadline/lib/rq-connection.ts | 40 ++++++++++++++----- .../lib/deadline/lib/worker-configuration.ts | 9 ++++- .../deadline/scripts/bash/configureWorker.sh | 2 +- .../scripts/powershell/configureWorker.ps1 | 2 +- .../lib/deadline/test/asset-constants.ts | 8 ++-- .../lib/deadline/test/render-queue.test.ts | 4 +- .../lib/deadline/test/worker-fleet.test.ts | 6 --- 9 files changed, 55 insertions(+), 27 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f3e5d0d2d..0d6439184 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,7 +26,7 @@ jobs: uses: actions/setup-node@v2.1.5 with: node-version: ${{ matrix.node-version }} + - run: npm install --global yarn - run: yarn global add typescript - - run: yarn install --frozen-lockfile - - run: ./build.sh + - run: yarn build - run: ./pack.sh diff --git a/packages/aws-rfdk/lib/deadline/lib/render-queue-ref.ts b/packages/aws-rfdk/lib/deadline/lib/render-queue-ref.ts index 8426d0f6c..a8f7a3621 100644 --- a/packages/aws-rfdk/lib/deadline/lib/render-queue-ref.ts +++ b/packages/aws-rfdk/lib/deadline/lib/render-queue-ref.ts @@ -361,4 +361,11 @@ export interface InstanceConnectOptions { * The Instance/UserData which will directly connect to the Repository */ readonly host: IHost; + + /** + * Whether or not to start or restart the Deadline Launcher after configuring the connection. + * + * @default true + */ + readonly restartLauncher?: boolean; } diff --git a/packages/aws-rfdk/lib/deadline/lib/rq-connection.ts b/packages/aws-rfdk/lib/deadline/lib/rq-connection.ts index f1cab7100..e62e58704 100644 --- a/packages/aws-rfdk/lib/deadline/lib/rq-connection.ts +++ b/packages/aws-rfdk/lib/deadline/lib/rq-connection.ts @@ -39,10 +39,20 @@ export interface HTTPSConnectionOptions extends BaseConnectionOptions { } interface ConnectionScriptArguments { - + /** + * The address of the Render Queue + */ readonly address: string; + /** + * The ARN to the AWS Secrets Manager secret containing the X509 CA Certificate in PEM format. + */ readonly tlsCaArn?: string; + + /** + * Whether to restart the Deadline launcher after configuring the Render Queue connection. + */ + readonly restartLauncher?: boolean; } /** @@ -114,23 +124,31 @@ export abstract class RenderQueueConnection { `"\${DEADLINE_PATH}/deadlinecommand" -executeScriptNoGui "${configureScriptPath}" --render-queue "${args.address}" ${dlExtraCommands.join(' ')}`, // Cleanup `rm -f "${configureScriptPath}"`, - 'if service --status-all | grep -q "Deadline 10 Launcher"; then', - ' service deadline10launcher restart', - 'fi', ); + if (args.restartLauncher ?? true) { + host.userData.addCommands( + 'if service --status-all | grep -q "Deadline 10 Launcher"; then', + ' service deadline10launcher restart', + 'fi', + ); + } } else if ( host.osType === OperatingSystemType.WINDOWS ) { host.userData.addCommands( '$ErrorActionPreference = "Stop"', '$DEADLINE_PATH = (get-item env:"DEADLINE_PATH").Value', `& "$DEADLINE_PATH/deadlinecommand.exe" -executeScriptNoGui "${configureScriptPath}" --render-queue "${args.address}" ${dlExtraCommands.join(' ')} 2>&1`, `Remove-Item -Path "${configureScriptPath}"`, - 'If (Get-Service "deadline10launcherservice" -ErrorAction SilentlyContinue) {', - ' Restart-Service "deadline10launcherservice"', - '} Else {', - ' & "$DEADLINE_PATH/deadlinelauncher.exe" -shutdownall 2>&1', - ' & "$DEADLINE_PATH/deadlinelauncher.exe" 2>&1', - '}', ); + if (args.restartLauncher ?? true) { + host.userData.addCommands( + 'If (Get-Service "deadline10launcherservice" -ErrorAction SilentlyContinue) {', + ' Restart-Service "deadline10launcherservice"', + '} Else {', + ' & "$DEADLINE_PATH/deadlinelauncher.exe" -shutdownall 2>&1', + ' & "$DEADLINE_PATH/deadlinelauncher.exe" -nogui 2>&1', + '}', + ); + } } } @@ -163,6 +181,7 @@ class HTTPConnection extends RenderQueueConnection { params.host, { address: `http://${this.config.endpoint.socketAddress}`, + restartLauncher: params.restartLauncher, }, ); } @@ -200,6 +219,7 @@ class HTTPSConnection extends RenderQueueConnection { { address: `https://${this.config.endpoint.socketAddress}`, tlsCaArn: this.config.caCert.secretArn, + restartLauncher: params.restartLauncher, }, ); diff --git a/packages/aws-rfdk/lib/deadline/lib/worker-configuration.ts b/packages/aws-rfdk/lib/deadline/lib/worker-configuration.ts index 9c17f53ff..f001bf810 100644 --- a/packages/aws-rfdk/lib/deadline/lib/worker-configuration.ts +++ b/packages/aws-rfdk/lib/deadline/lib/worker-configuration.ts @@ -221,7 +221,14 @@ export class WorkerInstanceConfiguration extends Construct { ); } props.userDataProvider?.preRenderQueueConfiguration(props.worker); - props.renderQueue?.configureClientInstance({ host: props.worker }); + props.renderQueue?.configureClientInstance({ + host: props.worker, + // Don't restart the Deadline Launcher service after configuring the connection to the Render Queue. We need to + // restart it later anyways, and the Windows service for the Deadline Launcher can get locked in the "stopping" + // state if you attempt to stop or restart it while it is still restarting. This can cause the user data execution + // to get locked waiting for the service to finish stopping/restarting. + restartLauncher: false, + }); props.userDataProvider?.preWorkerConfiguration(props.worker); this.listenerPort = props.workerSettings?.listenerPort ?? WorkerInstanceConfiguration.DEFAULT_LISTENER_PORT; diff --git a/packages/aws-rfdk/lib/deadline/scripts/bash/configureWorker.sh b/packages/aws-rfdk/lib/deadline/scripts/bash/configureWorker.sh index 173abc410..713fb5345 100644 --- a/packages/aws-rfdk/lib/deadline/scripts/bash/configureWorker.sh +++ b/packages/aws-rfdk/lib/deadline/scripts/bash/configureWorker.sh @@ -133,7 +133,7 @@ else DEADLINE_LAUNCHER="$DEADLINE_PATH/deadlinelauncher" "$DEADLINE_LAUNCHER" -shutdownall sudo killall -w deadlineworker || true - "$DEADLINE_LAUNCHER" + "$DEADLINE_LAUNCHER" -nogui fi echo "Script completed successfully." diff --git a/packages/aws-rfdk/lib/deadline/scripts/powershell/configureWorker.ps1 b/packages/aws-rfdk/lib/deadline/scripts/powershell/configureWorker.ps1 index ea58beb0c..ce481b1ef 100644 --- a/packages/aws-rfdk/lib/deadline/scripts/powershell/configureWorker.ps1 +++ b/packages/aws-rfdk/lib/deadline/scripts/powershell/configureWorker.ps1 @@ -127,7 +127,7 @@ If (Get-Service $serviceName -ErrorAction SilentlyContinue) { $DEADLINE_LAUNCHER = $DEADLINE_PATH + '/deadlinelauncher.exe' & $DEADLINE_LAUNCHER -shutdownall | Out-Default taskkill /f /fi "IMAGENAME eq deadlineworker.exe" - & $DEADLINE_LAUNCHER + & $DEADLINE_LAUNCHER -nogui } Write-Host "Script completed successfully." diff --git a/packages/aws-rfdk/lib/deadline/test/asset-constants.ts b/packages/aws-rfdk/lib/deadline/test/asset-constants.ts index be8ccd004..54f752640 100644 --- a/packages/aws-rfdk/lib/deadline/test/asset-constants.ts +++ b/packages/aws-rfdk/lib/deadline/test/asset-constants.ts @@ -18,8 +18,8 @@ export { // configureWorker.sh export const CONFIG_WORKER_ASSET_LINUX = { - Bucket: 'AssetParameters21c2af3bc1d4fd78061765b059dcc8e32568828e5cf479b08115489651491c8fS3BucketF10C60A7', - Key: 'AssetParameters21c2af3bc1d4fd78061765b059dcc8e32568828e5cf479b08115489651491c8fS3VersionKey7FDCC89A', + Bucket: 'AssetParameters1cfdffe73bb016717ba1f43d64fe528af27b3784f524a97bb36533a6e6d057ffS3Bucket9FBDD688', + Key: 'AssetParameters1cfdffe73bb016717ba1f43d64fe528af27b3784f524a97bb36533a6e6d057ffS3VersionKey02A3157B', }; // configureWorker.ps1 @@ -29,8 +29,8 @@ export const CONFIG_WORKER_ASSET_WINDOWS = { }; export const CONFIG_WORKER_PORT_ASSET_WINDOWS = { - Bucket: 'AssetParameters0b4fe3ffb7177773bb2781f92b37d9b01b3bd37ee60ea1715c0ad407f141005dS3BucketE7B32C3E', - Key: 'AssetParameters0b4fe3ffb7177773bb2781f92b37d9b01b3bd37ee60ea1715c0ad407f141005dS3VersionKey843794E3', + Bucket: 'AssetParameters3227efc256da3ae31791b7c80e1532cac975116846f179f118a20843e0c2ee80S3Bucket6583BE37', + Key: 'AssetParameters3227efc256da3ae31791b7c80e1532cac975116846f179f118a20843e0c2ee80S3VersionKey6C80977B', }; // installDeadlineRepository.sh diff --git a/packages/aws-rfdk/lib/deadline/test/render-queue.test.ts b/packages/aws-rfdk/lib/deadline/test/render-queue.test.ts index aabee299c..91ef099f6 100644 --- a/packages/aws-rfdk/lib/deadline/test/render-queue.test.ts +++ b/packages/aws-rfdk/lib/deadline/test/render-queue.test.ts @@ -1227,7 +1227,7 @@ describe('RenderQueue', () => { ' Restart-Service "deadline10launcherservice"\n' + '} Else {\n' + ' & "$DEADLINE_PATH/deadlinelauncher.exe" -shutdownall 2>&1\n' + - ' & "$DEADLINE_PATH/deadlinelauncher.exe" 2>&1\n' + + ' & "$DEADLINE_PATH/deadlinelauncher.exe" -nogui 2>&1\n' + '}', ], ], @@ -1635,7 +1635,7 @@ describe('RenderQueue', () => { ' Restart-Service "deadline10launcherservice"\n' + '} Else {\n' + ' & "$DEADLINE_PATH/deadlinelauncher.exe" -shutdownall 2>&1\n' + - ' & "$DEADLINE_PATH/deadlinelauncher.exe" 2>&1\n' + + ' & "$DEADLINE_PATH/deadlinelauncher.exe" -nogui 2>&1\n' + '}', ], ], diff --git a/packages/aws-rfdk/lib/deadline/test/worker-fleet.test.ts b/packages/aws-rfdk/lib/deadline/test/worker-fleet.test.ts index 9e919ad3b..3e6e22b9a 100644 --- a/packages/aws-rfdk/lib/deadline/test/worker-fleet.test.ts +++ b/packages/aws-rfdk/lib/deadline/test/worker-fleet.test.ts @@ -721,9 +721,6 @@ test('default worker fleet is created correctly custom subnet values', () => { ], }, '\"\n' + - 'if service --status-all | grep -q "Deadline 10 Launcher"; then\n' + - ' service deadline10launcher restart\n' + - 'fi\n' + "mkdir -p $(dirname '/tmp/", { 'Fn::Select': [ @@ -1214,9 +1211,6 @@ test('default worker fleet is created correctly with groups, pools and region', ], }, '\"\n' + - 'if service --status-all | grep -q "Deadline 10 Launcher"; then\n' + - ' service deadline10launcher restart\n' + - 'fi\n' + "mkdir -p $(dirname '/tmp/", { 'Fn::Select': [