Skip to content

Commit

Permalink
Fix tyger run cancel race condition
Browse files Browse the repository at this point in the history
  • Loading branch information
johnstairs committed Nov 26, 2024
1 parent 70aa3c3 commit 8457562
Show file tree
Hide file tree
Showing 3 changed files with 5 additions and 7 deletions.
4 changes: 0 additions & 4 deletions cli/integrationtest/controlplane_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -1311,10 +1311,6 @@ func TestCancelJob(t *testing.T) {

runTygerSucceeds(t, "run", "cancel", runId)

// force the sweep to run to terminate the pod
_, err := controlplane.InvokeRequest(context.Background(), http.MethodPost, "v1/runs/_sweep", nil, nil)
require.NoError(err)

waitForRunCanceled(t, runId)

// Check that the run failed because it was canceled.
Expand Down
4 changes: 3 additions & 1 deletion cli/internal/cmd/run.go
Original file line number Diff line number Diff line change
Expand Up @@ -311,7 +311,9 @@ beginWatch:
}

end:
mainWg.Wait()
if runFailedErr == nil {
mainWg.Wait()
}

if logs {
// The run has completed and we have received all data. We just need to wait for the logs to finish streaming,
Expand Down
4 changes: 2 additions & 2 deletions server/ControlPlane/Database/Repository.cs
Original file line number Diff line number Diff line change
Expand Up @@ -534,7 +534,7 @@ await _resiliencePipeline.ExecuteAsync(async cancellationToken =>
SELECT run
FROM runs
WHERE id = $1
AND status NOT IN ('Failed', 'Succeeded', 'Canceled')
AND status NOT IN ('Failed', 'Succeeded', 'Canceled', 'Canceling')
FOR UPDATE
""", conn)
{
Expand All @@ -550,7 +550,7 @@ FOR UPDATE
await using var reader = await readRun.ExecuteReaderAsync(cancellationToken);
if (!await reader.ReadAsync(cancellationToken))
{
// The run is already in a terminal state so we don't do anything
// The run is already in a terminal or canceling state so we don't do anything
return;
}

Expand Down

0 comments on commit 8457562

Please sign in to comment.