(chocolateyGH-445) Only fail scripts on non-zero exit code

Add a new switch and feature flag called `failOnStandardError`, that
will allow the older behavior of non-terminating errors to fail a
package. Otherwise log the error to stderr and give a warning message
that only an exit code of 1 will fail a package without the switch.

src/chocolatey.tests.integration/chocolatey.tests.integration.csproj

@@ -312,6 +312,12 @@
     <None Include="context\dependencies\isexactversiondependency\2.0.0\tools\chocolateyuninstall.ps1">
       <CopyToOutputDirectory>Always</CopyToOutputDirectory>
     </None>
+    <None Include="context\nonterminatingerror\1.0\nonterminatingerror.nuspec">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+    <None Include="context\nonterminatingerror\1.0\tools\chocolateyInstall.ps1">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
     <None Include="context\testing.packages.config">
       <CopyToOutputDirectory>Always</CopyToOutputDirectory>
     </None>

src/chocolatey.tests.integration/context/nonterminatingerror/1.0/nonterminatingerror.nuspec

@@ -0,0 +1,13 @@
+<?xml version="1.0"?>
+<package xmlns="http://schemas.microsoft.com/packaging/2010/07/nuspec.xsd">
+  <metadata>
+    <id>nonterminatingerror</id>
+    <version>1.0</version>
+    <authors>Rob Reynolds</authors>
+    <owners>Rob Reynolds</owners>
+    <description>nonterminating - This package errors during install with a non-terminating error</description>
+  </metadata>
+  <files>
+    <file src="tools\**" target="tools" />
+  </files>
+</package>

src/chocolatey.tests.integration/context/nonterminatingerror/1.0/tools/chocolateyInstall.ps1

@@ -0,0 +1 @@
+Write-Error "Oh no! An error"

src/chocolatey.tests.integration/scenarios/InstallScenarios.cs

@@ -1142,6 +1142,189 @@ public void should_have_expected_error_in_package_result()
             }
         }
 
+        [Concern(typeof(ChocolateyInstallCommand))]
+        public class when_installing_a_package_that_has_nonterminating_errors : ScenariosBase
+        {
+            private PackageResult packageResult;
+
+            public override void Context()
+            {
+                base.Context();
+                Configuration.PackageNames = Configuration.Input = "nonterminatingerror";
+                Configuration.Features.FailOnStandardError = false; //the default
+
+                Scenario.add_packages_to_source_location(Configuration, Configuration.Input + "*" + Constants.PackageExtension);
+            }
+
+            public override void Because()
+            {
+                Results = Service.install_run(Configuration);
+                packageResult = Results.FirstOrDefault().Value;
+            }
+
+            [Fact]
+            public void should_install_where_install_location_reports()
+            {
+                Directory.Exists(packageResult.InstallLocation).ShouldBeTrue();
+            }
+
+            [Fact]
+            public void should_install_the_package_in_the_lib_directory()
+            {
+                var packageDir = Path.Combine(Scenario.get_top_level(), "lib", Configuration.Input);
+
+                Directory.Exists(packageDir).ShouldBeTrue();
+            }
+
+            [Fact]
+            public void should_install_the_expected_version_of_the_package()
+            {
+                var packageFile = Path.Combine(Scenario.get_top_level(), "lib", Configuration.Input, Configuration.Input + Constants.PackageExtension);
+                var package = new OptimizedZipPackage(packageFile);
+                package.Version.Version.to_string().ShouldEqual("1.0.0.0");
+            }
+
+            [Fact]
+            public void should_contain_a_message_that_it_installed_successfully()
+            {
+                bool installedSuccessfully = false;
+                foreach (var message in MockLogger.MessagesFor(LogLevel.Warn).or_empty_list_if_null())
+                {
+                    if (message.Contains("1/1")) installedSuccessfully = true;
+                }
+
+                installedSuccessfully.ShouldBeTrue();
+            }
+
+            [Fact]
+            public void should_have_a_successful_package_result()
+            {
+                packageResult.Success.ShouldBeTrue();
+            }
+
+            [Fact]
+            public void should_not_have_inconclusive_package_result()
+            {
+                packageResult.Inconclusive.ShouldBeFalse();
+            }
+
+            [Fact]
+            public void should_not_have_warning_package_result()
+            {
+                packageResult.Warning.ShouldBeFalse();
+            }
+
+            [Fact]
+            public void config_should_match_package_result_name()
+            {
+                packageResult.Name.ShouldEqual(Configuration.Input);
+            }
+
+            [Fact]
+            public void should_have_a_version_of_one_dot_zero()
+            {
+                packageResult.Version.ShouldEqual("1.0");
+            }
+        }
+
+
+        [Concern(typeof(ChocolateyInstallCommand))]
+        public class when_installing_a_package_that_has_nonterminating_errors_with_fail_on_stderr : ScenariosBase
+        {
+            private PackageResult packageResult;
+
+            public override void Context()
+            {
+                base.Context();
+                Configuration.PackageNames = Configuration.Input = "nonterminatingerror";
+                Configuration.Features.FailOnStandardError = true;
+
+                Scenario.add_packages_to_source_location(Configuration, Configuration.Input + "*" + Constants.PackageExtension);
+            }
+
+            public override void Because()
+            {
+                Results = Service.install_run(Configuration);
+                packageResult = Results.FirstOrDefault().Value;
+            }
+
+            [Fact]
+            public void should_not_install_a_package_in_the_lib_directory()
+            {
+                var packageDir = Path.Combine(Scenario.get_top_level(), "lib", Configuration.PackageNames);
+
+                Directory.Exists(packageDir).ShouldBeFalse();
+            }
+
+            [Fact]
+            public void should_put_a_package_in_the_lib_bad_directory()
+            {
+                var packageDir = Path.Combine(Scenario.get_top_level(), "lib-bad", Configuration.PackageNames);
+
+                Directory.Exists(packageDir).ShouldBeTrue();
+            }
+
+            [Fact]
+            public void should_contain_a_warning_message_that_it_was_unable_to_install_a_package()
+            {
+                bool installedSuccessfully = false;
+                foreach (var message in MockLogger.MessagesFor(LogLevel.Warn).or_empty_list_if_null())
+                {
+                    if (message.Contains("0/1")) installedSuccessfully = true;
+                }
+
+                installedSuccessfully.ShouldBeTrue();
+            }
+
+            [Fact]
+            public void should_not_have_a_successful_package_result()
+            {
+                packageResult.Success.ShouldBeFalse();
+            }
+
+            [Fact]
+            public void should_not_have_inconclusive_package_result()
+            {
+                packageResult.Inconclusive.ShouldBeFalse();
+            }
+
+            [Fact]
+            public void should_not_have_warning_package_result()
+            {
+                packageResult.Warning.ShouldBeFalse();
+            }
+
+            [Fact]
+            public void should_have_an_error_package_result()
+            {
+                bool errorFound = false;
+                foreach (var message in packageResult.Messages)
+                {
+                    if (message.MessageType == ResultType.Error)
+                    {
+                        errorFound = true;
+                    }
+                }
+
+                errorFound.ShouldBeTrue();
+            }
+
+            [Fact]
+            public void should_have_expected_error_in_package_result()
+            {
+                bool errorFound = false;
+                foreach (var message in packageResult.Messages)
+                {
+                    if (message.MessageType == ResultType.Error)
+                    {
+                        if (message.Message.Contains("chocolateyInstall.ps1")) errorFound = true;
+                    }
+                }
+
+                errorFound.ShouldBeTrue();
+            }
+        }
+
         [Concern(typeof(ChocolateyInstallCommand))]
         public class when_installing_a_side_by_side_package : ScenariosBase
         {

src/chocolatey/infrastructure.app/ApplicationParameters.cs

@@ -100,6 +100,7 @@ public static class Features
             public static readonly string AutoUninstaller = "autoUninstaller";
             public static readonly string FailOnAutoUninstaller = "failOnAutoUninstaller";
             public static readonly string AllowGlobalConfirmation = "allowGlobalConfirmation";
+            public static readonly string FailOnStandardError = "failOnStandardError";
         }
 
         public static class Messages

src/chocolatey/infrastructure.app/builders/ConfigurationBuilder.cs

@@ -187,6 +187,7 @@ private static void set_feature_flags(ChocolateyConfiguration config, ConfigFile
             config.Features.CheckSumFiles = set_feature_flag(ApplicationParameters.Features.CheckSumFiles, configFileSettings, defaultEnabled: true, description: "Checksum files when pulled in from internet (based on package).");
             config.Features.AutoUninstaller = set_feature_flag(ApplicationParameters.Features.AutoUninstaller, configFileSettings, defaultEnabled: true, description: "Uninstall from programs and features without requiring an explicit uninstall script.");
             config.Features.FailOnAutoUninstaller = set_feature_flag(ApplicationParameters.Features.FailOnAutoUninstaller, configFileSettings, defaultEnabled: false, description: "Fail if automatic uninstaller fails.");
+            config.Features.FailOnStandardError = set_feature_flag(ApplicationParameters.Features.FailOnStandardError, configFileSettings, defaultEnabled: false, description: "Fail if install provider writes to stderr.");
             config.PromptForConfirmation = !set_feature_flag(ApplicationParameters.Features.AllowGlobalConfirmation, configFileSettings, defaultEnabled: false, description: "Prompt for confirmation in scripts or bypass.");
         }
 
@@ -259,7 +260,10 @@ private static void set_global_options(IList<string> args, ChocolateyConfigurati
                                  option => config.CacheLocation = option.remove_surrounding_quotes())
                             .Add("allowunofficial|allow-unofficial|allowunofficialbuild|allow-unofficial-build",
                                  "AllowUnofficialBuild - When not using the official build you must set this flag for choco to continue.",
-                                 option => config.AllowUnofficialBuild = option != null)
+                                 option => config.AllowUnofficialBuild = option != null) 
+                            .Add("failstderr|failonstderr|fail-on-stderr|fail-on-standard-error|fail-on-error-output",
+                                 "FailOnStandardError - Fail on standard error output (stderr), typically received when running external commands during install providers. This overrides the feature failOnStandardError.",
+                                 option => config.Features.FailOnStandardError = option != null)
                             ;
                     },
                 (unparsedArgs) =>

src/chocolatey/infrastructure.app/configuration/ChocolateyConfiguration.cs

@@ -335,6 +335,7 @@ public sealed class FeaturesConfiguration
         public bool AutoUninstaller { get; set; }
         public bool CheckSumFiles { get; set; }
         public bool FailOnAutoUninstaller { get; set; }
+        public bool FailOnStandardError { get; set; }
     }
 
     //todo: retrofit other command configs this way

src/chocolatey/infrastructure.app/configuration/chocolatey.config

@@ -16,5 +16,6 @@
     <feature name="autoUninstaller" enabled="false" />
     <feature name="allowGlobalConfirmation" enabled="false" />
     <feature name="failOnAutoUninstaller" enabled="false" />
+    <feature name="failOnStandardError" enabled="false" />
   </features>
 </chocolatey>

src/chocolatey/infrastructure.app/services/PowershellService.cs

@@ -234,6 +234,7 @@ public bool run_action(ChocolateyConfiguration configuration, PackageResult pack
                 if (shouldRun)
                 {
                     installerRun = true;
+                    var errorMessagesLogged = false;
                     var exitCode = PowershellExecutor.execute(
                         wrap_script_with_module(chocoPowerShellScript, configuration),
                         _fileSystem,
@@ -268,7 +269,8 @@ public bool run_action(ChocolateyConfiguration configuration, PackageResult pack
                                 }
                                 else
                                 {
-                                    failure = true;
+                                    errorMessagesLogged = true;
+                                    if (configuration.Features.FailOnStandardError) failure = true;
                                     this.Log().Error(() => " " + e.Data);
                                 }
                             });
@@ -278,6 +280,14 @@ public bool run_action(ChocolateyConfiguration configuration, PackageResult pack
                         failure = true;
                     }
 
+                    if (!configuration.Features.FailOnStandardError && errorMessagesLogged)
+                    {
+                        this.Log().Warn(() =>
+@"Only an exit code of non-zero will fail the package by default. Set 
+ `--failonstderr` if you want error messages to also fail a script. See 
+ `choco -h` for details.");
+                    }
+
                     if (failure)
                     {
                         Environment.ExitCode = exitCode;