Add retry to apply changes

commands/apply_changes.go

@@ -167,10 +167,24 @@ func (ac ApplyChanges) Execute(args []string) error {
 }
 
 func (ac ApplyChanges) waitForApplyChangesCompletion(installation api.InstallationsServiceOutput) error {
+	const maxRetries = 3
+
 	for {
-		current, err := ac.service.GetInstallation(installation.ID)
+		var current api.InstallationsServiceOutput
+		var err error
+
+		for i := 0; i < maxRetries; i++ {
+			current, err = ac.service.GetInstallation(installation.ID)
+			if err == nil {
+				break
+			}
+			if i < maxRetries-1 {
+				time.Sleep(ac.waitDuration)
+			}
+		}
+
 		if err != nil {
-			return fmt.Errorf("installation failed to get status: %s", err)
+			return fmt.Errorf("installation failed to get status after %d attempts: %s", maxRetries, err)
 		}
 
 		install, err := ac.service.GetInstallationLogs(installation.ID)

commands/apply_changes_test.go

@@ -78,6 +78,35 @@ var _ = Describe("ApplyChanges", func() {
 			Expect(writer.FlushArgsForCall(2)).To(Equal("some other logs"))
 		})
 
+		It("retries apply changes to the Ops Manager", func() {
+			service.GetInstallationReturnsOnCall(0, api.InstallationsServiceOutput{}, errors.New("some error"))
+			service.GetInstallationReturnsOnCall(1, api.InstallationsServiceOutput{Status: "running"}, nil)
+			service.GetInstallationReturnsOnCall(2, api.InstallationsServiceOutput{Status: "succeeded"}, nil)
+
+			command := commands.NewApplyChanges(service, pendingService, writer, logger, 1)
+
+			err := executeCommand(command, []string{})
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(service.CreateInstallationCallCount()).To(Equal(1))
+
+			ignoreWarnings, deployProducts, _, _ := service.CreateInstallationArgsForCall(0)
+			Expect(ignoreWarnings).To(Equal(false))
+			Expect(deployProducts).To(Equal(true))
+
+			Expect(stderr).To(gbytes.Say("attempting to apply changes to the targeted Ops Manager"))
+
+			Expect(service.GetInstallationArgsForCall(0)).To(Equal(311))
+			Expect(service.GetInstallationCallCount()).To(Equal(3))
+
+			Expect(service.GetInstallationLogsArgsForCall(0)).To(Equal(311))
+			Expect(service.GetInstallationLogsCallCount()).To(Equal(2))
+
+			Expect(writer.FlushCallCount()).To(Equal(2))
+			Expect(writer.FlushArgsForCall(0)).To(Equal("start of logs"))
+			Expect(writer.FlushArgsForCall(1)).To(Equal("these logs"))
+		})
+
 		When("passed the ignore-warnings flag", func() {
 			It("applies changes while ignoring warnings", func() {
 				service.InfoReturns(api.Info{Version: "2.3-build43"}, nil)
@@ -473,12 +502,14 @@ errands: lolololol
 		When("getting the installation status has an error", func() {
 			It("returns an error", func() {
 				service.CreateInstallationReturns(api.InstallationsServiceOutput{ID: 311}, nil)
-				service.GetInstallationReturnsOnCall(0, api.InstallationsServiceOutput{}, errors.New("another error"))
+				service.GetInstallationReturnsOnCall(0, api.InstallationsServiceOutput{}, errors.New("first error"))
+				service.GetInstallationReturnsOnCall(1, api.InstallationsServiceOutput{}, errors.New("second error"))
+				service.GetInstallationReturnsOnCall(2, api.InstallationsServiceOutput{}, errors.New("third error"))
 
 				command := commands.NewApplyChanges(service, pendingService, writer, logger, 1)
 
 				err := executeCommand(command, []string{})
-				Expect(err).To(MatchError("installation failed to get status: another error"))
+				Expect(err).To(MatchError("installation failed to get status after 3 attempts: third error"))
 			})
 		})