disable pip red error messages; add pip warning

CHANGELOG.md

@@ -5,25 +5,28 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+### Added
+- `future.strict_environments` top-level configuration option
+  - modifies how `deployment.environments`/`module.environments` are handled
+- notice about *false alarm* compatibility errors when using pip with the aws_lambda hook
 
 ### Changed
 - `sls-tsc` sample updated to use eslint in favor of deprecated tslint
+- log format of module skip information no longer includes extra characters and is now prefixed by the module name
+- aws_lambda hook will no longer use colorized pip output so its *false alarm* compatibility errors are less menacing
 
 ## [1.8.1] - 2020-06-04
 ### Added
 - `destroy_stack` is now aware of `action=diff` and prints a different confirmation prompt
 - `-no-color`/`--no-color` option automatically added to cdk, npm, sls, and tf commands
   - looks at `RUNWAY_COLORIZE` env var for an explicit enable/disable
   - if not set, checks `sys.stdout.isatty()` to determine if option should be provided
-- `future.strict_environments` top-level configuration option
-  - modifies how `deployment.environments`/`module.environments` are handled
 
 ### Changed
 - a@e check_auth will now try to refresh tokens 5 minutes before expiration instead of waiting for it to expire
 - `runway test` will now return a non-zero exit code if any non-required tests failed
 - `static-react` sample uses npm instead of yarn
 - `yamllint` is now invoked using `runpy` instead of using `runway run-python`
-- log format of module skip information no longer includes extra characters and is now prefixed by the module name
 
 ### Fixed
 - issue where `yamllint` and `cfnlint` could not be imported/executed from the Pyinstaller executables

integration_tests/test_cfngin/tests/16_aws_lambda_hook.py

@@ -1,61 +1,81 @@
-"""Test AWS Lambda hook."""
-# flake8: noqa
-# pylint: disable=invalid-name
-from os import path
-from os.path import basename
-
-import boto3
-from send2trash import send2trash
-
-from integration_tests.test_cfngin.test_cfngin import Cfngin
-
-FILE_BASENAME = '.'.join(basename(__file__).split('.')[:-1])
-
-
-class TestAwsLambda(Cfngin):
-    """Test AWS Lambda Hook from cfngin.
-
-    Requires valid AWS Credentials.
-
-    """
-
-    REQUIRED_FIXTURE_FILES = [FILE_BASENAME + '.yaml']
-    TEST_NAME = __name__
-
-    def invoke_lambda(self, client, func_name):
-        """Verify lambda function deployed successfully."""
-        self.logger.info('Invoking lambda function: %s', func_name)
-        resp = client.invoke(
-            FunctionName=func_name,
-            InvocationType='RequestResponse'
-        )
-
-        return resp['StatusCode']
-
-    def _build(self):
-        """Execute and assert initial build."""
-        self.set_environment('dev')
-        code, _stdout, _stderr = self.runway_cmd('deploy')
-        assert code == 0, 'exit code should be zero'
-
-    def run(self):
-        """Run test."""
-        self.copy_fixtures()
-        self._build()
-
-        client = boto3.client('lambda', region_name=self.region)
-
-        functions = ['dockerizepip-integrationtest',
-                     'nondockerizepip-integrationtest']
-        for func_name in functions:
-            assert self.invoke_lambda(client, func_name) == 200, \
-                f'{self.TEST_NAME}: Execution of lambda {func_name} failed'
-
-    def teardown(self):
-        """Teardown test."""
-        self.runway_cmd('destroy')
-        venv_dir = path.join(self.fixture_dir,
-                             'lambda_src/dockerize_src/.venv')
-        if path.isdir(venv_dir):
-            send2trash(venv_dir)
-        self.cleanup_fixtures()
+"""Test AWS Lambda hook."""
+# flake8: noqa
+# pylint: disable=invalid-name
+import os
+import pty
+import tempfile
+
+import boto3
+from send2trash import send2trash
+
+from integration_tests.test_cfngin.test_cfngin import Cfngin
+from runway.util import environ
+
+FILE_BASENAME = '.'.join(os.path.basename(__file__).split('.')[:-1])
+
+
+class TestAwsLambda(Cfngin):
+    """Test AWS Lambda Hook from cfngin.
+
+    Requires valid AWS Credentials.
+
+    """
+
+    REQUIRED_FIXTURE_FILES = [FILE_BASENAME + '.yaml']
+    TEST_NAME = __name__
+
+    def invoke_lambda(self, client, func_name):
+        """Verify lambda function deployed successfully."""
+        self.logger.info('Invoking lambda function: %s', func_name)
+        resp = client.invoke(
+            FunctionName=func_name,
+            InvocationType='RequestResponse'
+        )
+
+        return resp['StatusCode']
+
+    def _build(self):
+        """Execute and assert initial build.
+
+        Explicitly spawning with a tty here to ensure output
+        (e.g. from pip) includes color
+
+        """
+        with environ({**os.environ, **{'DEPLOY_ENVIRONMENT': 'dev'}}):
+            with tempfile.TemporaryFile() as pty_output:
+                def read_pty(fds):
+                    """Append tty output to file descriptor."""
+                    data = os.read(fds, 1024)
+                    pty_output.write(data)
+                    return data
+
+                spawn_result = pty.spawn(['runway', 'deploy'], read_pty)
+                assert spawn_result == 0, 'exit code should be zero'
+                pty_output.seek(0)
+                combined_output = pty_output.read().decode()
+                assert '\x1b[31mERROR: ' not in combined_output, (
+                    'no red ERROR should be present'
+                )
+
+    def run(self):
+        """Run test."""
+        self.copy_fixtures()
+        self._build()
+
+        client = boto3.client('lambda', region_name=self.region)
+
+        functions = ['dockerizepip-integrationtest',
+                     'nondockerizepip-integrationtest',
+                     'authatedge-integrationtest']
+        for func_name in functions:
+            assert self.invoke_lambda(client, func_name) == 200, \
+                f'{self.TEST_NAME}: Execution of lambda {func_name} failed'
+
+    def teardown(self):
+        """Teardown test."""
+        self.runway_cmd('destroy')
+        venv_dir = os.path.join(self.fixture_dir,
+                                'lambda_src/dockerize_src/.venv')
+        if os.path.isdir(venv_dir):
+            send2trash(venv_dir)
+        self.cleanup_fixtures()

integration_tests/test_cfngin/tests/fixtures/16_aws_lambda_hook.yaml

@@ -1,52 +1,61 @@
-namespace: ${CFNGIN_NAMESPACE}
-cfngin_bucket: ${CFNGIN_NAMESPACE}-${region}
-
-sys_path: ./
-
-pre_build:
-  upload_lambdas:
-    path: runway.cfngin.hooks.aws_lambda.upload_lambda_functions
-    required: true
-    data_key: lambda
-    args:
-      functions:
-        dockerize:
-          # this won't actually use docker at the moment.
-          # it will take additional work to get docker running in codebuild for this.
-          # but, for the time being we are at least testing the use of pipenv.
-          # note: leaving it set to `non-linux` will allow local testing to use docker even though codebuild testing does not.
-          dockerize_pip: non-linux
-          use_pipenv: true
-          runtime: python3.7
-          path: ./fixtures/lambda_src/dockerize_src
-          include:
-            - '*.py'
-          exclude:
-            - '*.pyc'
-        nondockerize:
-            path: ./fixtures/lambda_src/nondockerize_src
-            include:
-              - '*.py'
-            exclude:
-              - '*.pyc'
-
-stacks:
-  test-dockerize:
-    class_path: fixtures.lambda_function.BlueprintClass
-    variables:
-      Code: ${hook_data lambda::dockerize}
-      Entrypoint: dockerize.handler
-      AppName: dockerizepip
-  test-nondockerize:
-    class_path: fixtures.lambda_function.BlueprintClass
-    variables:
-      Code: ${hook_data lambda::nondockerize}
-      Entrypoint: nondockerize.handler
-      AppName: nondockerizepip
-
-post_destroy:
-  purge_bucket:
-    path: runway.hooks.cleanup_s3.purge_bucket
-    required: True
-    args:
-      bucket_name: ${CFNGIN_NAMESPACE}-${region}
+namespace: ${CFNGIN_NAMESPACE}
+cfngin_bucket: ${CFNGIN_NAMESPACE}-${region}
+
+sys_path: ./
+
+pre_build:
+  upload_lambdas:
+    path: runway.cfngin.hooks.aws_lambda.upload_lambda_functions
+    required: true
+    data_key: lambda
+    args:
+      functions:
+        dockerize:
+          # this won't actually use docker at the moment.
+          # it will take additional work to get docker running in codebuild for this.
+          # but, for the time being we are at least testing the use of pipenv.
+          # note: leaving it set to `non-linux` will allow local testing to use docker even though codebuild testing does not.
+          dockerize_pip: non-linux
+          use_pipenv: true
+          runtime: python3.7
+          path: ./fixtures/lambda_src/dockerize_src
+          include:
+            - '*.py'
+          exclude:
+            - '*.pyc'
+        nondockerize:
+            path: ./fixtures/lambda_src/nondockerize_src
+            include:
+              - '*.py'
+            exclude:
+              - '*.pyc'
+        authatedge:
+            path: ./fixtures/lambda_src/authatedge_src
+            exclude:
+              - '*.pyc'
+
+stacks:
+  test-dockerize:
+    class_path: fixtures.lambda_function.BlueprintClass
+    variables:
+      Code: ${hook_data lambda::dockerize}
+      Entrypoint: dockerize.handler
+      AppName: dockerizepip
+  test-nondockerize:
+    class_path: fixtures.lambda_function.BlueprintClass
+    variables:
+      Code: ${hook_data lambda::nondockerize}
+      Entrypoint: nondockerize.handler
+      AppName: nondockerizepip
+  test-authatedge:
+    class_path: fixtures.lambda_function.BlueprintClass
+    variables:
+      Code: ${hook_data lambda::authatedge}
+      AppName: authatedge
+
+post_destroy:
+  purge_bucket:
+    path: runway.hooks.cleanup_s3.purge_bucket
+    required: True
+    args:
+      bucket_name: ${CFNGIN_NAMESPACE}-${region}

integration_tests/test_cfngin/tests/fixtures/lambda_src/authatedge_src/index.py

@@ -0,0 +1,19 @@
+"""Test handler."""
+import rsa
+
+
+def handler(event, context):
+    """Handle lambda."""
+    try:
+        prime = rsa.prime.getprime(4)
+        if isinstance(prime, int):
+            return {
+                'statusCode': 200,
+                'body': str(prime)
+            }
+        raise ValueError
+    except:
+        return {
+            'statusCode': 500,
+            'body': 'fail'
+        }

integration_tests/test_cfngin/tests/fixtures/lambda_src/authatedge_src/requirements.txt

@@ -0,0 +1,7 @@
+-i https://pypi.org/simple
+ecdsa==0.15
+pyasn1==0.4.8
+pyjwt==1.7.1
+python-jose==3.1.0
+rsa==4.0
+six==1.14.0

runway/cfngin/hooks/aws_lambda.py

@@ -1,4 +1,4 @@
-"""AWS Lambda hook."""
+"""AWS Lambda hook."""  # pylint: disable=too-many-lines
 import hashlib
 import json
 import logging
@@ -464,6 +464,41 @@ def dockerized_pip(work_dir, client=None, runtime=None, docker_file=None,
         LOGGER.info('lambda.docker: %s', log.decode().strip())
 
 
+def _pip_has_no_color_option(python_path):
+    """Return boolean on whether pip is new enough to have --no-color option.
+
+    pip v10 introduced this option, and it's used to minimize the effect of
+    pip falsely indicating errors like the following:
+
+    "ERROR: awscli 1.18.64 has requirement rsa<=3.5.0,>=3.1.2, but you'll
+    have rsa 4.0 which is incompatible."
+
+    An error like that (colored in red) will appear when there's a conflict
+    between the host & target environments, which is not helpful in the
+    context of building a Lambda zip that will never interact with the
+    running python system.
+
+    Ideally this mitigation (and this function by assocation) can be removed
+    by an enhancement or replacement of pip for building the packages.
+
+    """
+    try:
+        pip_version_string = subprocess.check_output(
+            [python_path,
+             '-c',
+             'from __future__ import print_function;'
+             'import pip;'
+             'print(pip.__version__)']
+        )
+        if sys.version_info[0] > 2 and isinstance(pip_version_string, bytes):
+            pip_version_string = pip_version_string.decode()
+        if int(pip_version_string.split('.')[0]) > 10:
+            return True
+    except (AttributeError, ValueError, subprocess.CalledProcessError):
+        LOGGER.debug('Error checking pip version; assuming it to be pre-v10')
+    return False
+
+
 def _zip_package(package_root, includes, excludes=None, dockerize_pip=False,
                  follow_symlinks=False, python_path=None,
                  requirements_files=None, use_pipenv=False, **kwargs):
@@ -521,7 +556,8 @@ def _zip_package(package_root, includes, excludes=None, dockerize_pip=False,
             pip_cmd = [python_path or sys.executable, '-m',
                        'pip', 'install',
                        '--target', tmpdir,
-                       '--requirement', tmp_req]
+                       '--requirement', tmp_req,
+                       '--no-color']
 
             # Pyinstaller build or explicit python path
             if getattr(sys, 'frozen', False) and not python_path:
@@ -537,8 +573,17 @@ def _zip_package(package_root, includes, excludes=None, dockerize_pip=False,
                                       else script_contents.decode('UTF-8'))
                 cmd = [sys.executable, 'run-python', str(tmp_script)]
             else:
+                if not _pip_has_no_color_option(pip_cmd[0]):
+                    pip_cmd.remove('--no-color')
                 cmd = pip_cmd
 
+            LOGGER.info(
+                'The following output from pip may include incompatibility errors. '
+                'These can generally be ignored (pip will erroneously warn '
+                'about conflicts between the packages in your Lambda zip and '
+                'your host system).'
+            )
+
             try:
                 subprocess.check_call(cmd)
             except subprocess.CalledProcessError: