diff --git a/commands/apply_changes.go b/commands/apply_changes.go index 51ef77b1..5344b7f3 100644 --- a/commands/apply_changes.go +++ b/commands/apply_changes.go @@ -167,10 +167,24 @@ func (ac ApplyChanges) Execute(args []string) error { } func (ac ApplyChanges) waitForApplyChangesCompletion(installation api.InstallationsServiceOutput) error { + const maxRetries = 3 + for { - current, err := ac.service.GetInstallation(installation.ID) + var current api.InstallationsServiceOutput + var err error + + for i := 0; i < maxRetries; i++ { + current, err = ac.service.GetInstallation(installation.ID) + if err == nil { + break + } + if i < maxRetries-1 { + time.Sleep(ac.waitDuration) + } + } + if err != nil { - return fmt.Errorf("installation failed to get status: %s", err) + return fmt.Errorf("installation failed to get status after %d attempts: %s", maxRetries, err) } install, err := ac.service.GetInstallationLogs(installation.ID) diff --git a/commands/apply_changes_test.go b/commands/apply_changes_test.go index 27711f53..5fc95ecd 100644 --- a/commands/apply_changes_test.go +++ b/commands/apply_changes_test.go @@ -78,6 +78,35 @@ var _ = Describe("ApplyChanges", func() { Expect(writer.FlushArgsForCall(2)).To(Equal("some other logs")) }) + It("retries apply changes to the Ops Manager", func() { + service.GetInstallationReturnsOnCall(0, api.InstallationsServiceOutput{}, errors.New("some error")) + service.GetInstallationReturnsOnCall(1, api.InstallationsServiceOutput{Status: "running"}, nil) + service.GetInstallationReturnsOnCall(2, api.InstallationsServiceOutput{Status: "succeeded"}, nil) + + command := commands.NewApplyChanges(service, pendingService, writer, logger, 1) + + err := executeCommand(command, []string{}) + Expect(err).ToNot(HaveOccurred()) + + Expect(service.CreateInstallationCallCount()).To(Equal(1)) + + ignoreWarnings, deployProducts, _, _ := service.CreateInstallationArgsForCall(0) + Expect(ignoreWarnings).To(Equal(false)) + Expect(deployProducts).To(Equal(true)) + + Expect(stderr).To(gbytes.Say("attempting to apply changes to the targeted Ops Manager")) + + Expect(service.GetInstallationArgsForCall(0)).To(Equal(311)) + Expect(service.GetInstallationCallCount()).To(Equal(3)) + + Expect(service.GetInstallationLogsArgsForCall(0)).To(Equal(311)) + Expect(service.GetInstallationLogsCallCount()).To(Equal(2)) + + Expect(writer.FlushCallCount()).To(Equal(2)) + Expect(writer.FlushArgsForCall(0)).To(Equal("start of logs")) + Expect(writer.FlushArgsForCall(1)).To(Equal("these logs")) + }) + When("passed the ignore-warnings flag", func() { It("applies changes while ignoring warnings", func() { service.InfoReturns(api.Info{Version: "2.3-build43"}, nil) @@ -473,12 +502,14 @@ errands: lolololol When("getting the installation status has an error", func() { It("returns an error", func() { service.CreateInstallationReturns(api.InstallationsServiceOutput{ID: 311}, nil) - service.GetInstallationReturnsOnCall(0, api.InstallationsServiceOutput{}, errors.New("another error")) + service.GetInstallationReturnsOnCall(0, api.InstallationsServiceOutput{}, errors.New("first error")) + service.GetInstallationReturnsOnCall(1, api.InstallationsServiceOutput{}, errors.New("second error")) + service.GetInstallationReturnsOnCall(2, api.InstallationsServiceOutput{}, errors.New("third error")) command := commands.NewApplyChanges(service, pendingService, writer, logger, 1) err := executeCommand(command, []string{}) - Expect(err).To(MatchError("installation failed to get status: another error")) + Expect(err).To(MatchError("installation failed to get status after 3 attempts: third error")) }) })