bump pylint and astroid versions, fix linting errors (#258)

* bump pylint and astroid versions, fix linting errors

* update make sync_two, add lint_two, update cicd workflow

.github/workflows/cicd.yml

@@ -75,20 +75,22 @@ jobs:
         if: matrix.python-version == '2.7' && matrix.os != 'windows-latest'
         run: |
           rm -rf Pipfile.lock
-          pipenv install "pylint<2.0" --dev
-          pipenv sync --dev
+          make sync_two
       - # &pipenv_sync_27_windows
         name: Setup Python Virtual Environment (2.7) (windows)
         if: matrix.python-version == '2.7' && matrix.os == 'windows-latest'
         run: |
           Remove-Item -Path $Env:GITHUB_WORKSPACE\Pipfile.lock -Force
-          pipenv install "pylint<2.0" --dev
-          pipenv sync --dev
+          make sync_two
       - # &pipenv_sync
         name: Setup Python Virtual Environment (3.x)
         if: matrix.python-version != '2.7'
         run: pipenv sync --dev
-      - name: Run Linters
+      - name: Run Linters (2.7)
+        if: matrix.python-version == '2.7'
+        run: make lint_two
+      - name: Run Linters (3.x)
+        if: matrix.python-version != '2.7'
         run: make lint
       - name: Run Unit Tests
         if: matrix.os != 'windows-latest'

Makefile

@@ -1,9 +1,10 @@
 sync:
 	PIPENV_VENV_IN_PROJECT=1 pipenv sync -d
 
-# not actually a sync since we need to skip-lock but maintains naming
+# changes that need to be made inorder to sync python two (may also require deletion of the existing lock file)
 sync_two:
-	PIPENV_VENV_IN_PROJECT=1 pipenv install --dev --two --skip-lock
+	pipenv install "astroid<2.0" "pylint<2.0" --dev
+	PIPENV_VENV_IN_PROJECT=1 pipenv install --dev
 
 # sync all virtual environments used by this project with their Pipfile.lock
 sync_all:
@@ -32,6 +33,12 @@ lint:
 	find runway -name '*.py' -not -path 'runway/embedded*' -not -path 'runway/templates/stacker/*' -not -path 'runway/templates/cdk-py/*' -not -path 'runway/blueprints/*' | xargs pipenv run pylint --rcfile=.pylintrc
 	find runway/blueprints -name '*.py' | xargs pipenv run pylint --disable=duplicate-code
 
+# linting for python 2, requires additional disables
+lint_two:
+	pipenv run flake8 --exclude=runway/embedded,runway/templates runway
+	find runway -name '*.py' -not -path 'runway/embedded*' -not -path 'runway/templates/stacker/*' -not -path 'runway/templates/cdk-py/*' -not -path 'runway/blueprints/*' | xargs pipenv run pylint --rcfile=.pylintrc --disable=bad-option-value,relative-import
+	find runway/blueprints -name '*.py' | xargs pipenv run pylint --disable=duplicate-code
+
 test:
 	pipenv run pytest
 

Pipfile

@@ -20,8 +20,8 @@ flake8-docstrings = "*"
 flake8 = "*"
 # pylint and astroid pinned to avoid "Instance of '' has no 'Bucket' member"
 # error when using boto3. https://github.com/PyCQA/pylint/issues/3134
-pylint = "<2.3.1"
-astroid = "<2.2.5"
+pylint = ">=2.5.0"
+astroid = ">=2.4.0"
 # stuck at 3.5 till this is resolved https://github.com/pyinstaller/pyinstaller/issues/4674
 pyinstaller = "==3.5"
 pytest = "<5.0"  # last version that supports 2.7 - allows install with 2/3

runway/cfngin/plan.py

@@ -290,6 +290,7 @@ def from_stack_name(cls, stack_name, context, requires=None, fn=None,
             :class:`Step`
 
         """
+        # pylint: disable=import-outside-toplevel
         from runway.cfngin.config import Stack as StackConfig
         from runway.cfngin.stack import Stack
 

runway/cfngin/providers/aws/default.py

@@ -677,8 +677,7 @@ def _log_func(event):
                     if cancel.wait(TAIL_RETRY_SLEEP):
                         return
                     continue
-                else:
-                    raise
+                raise
 
     @staticmethod
     def _tail_print(event):

runway/cfngin/util.py

@@ -335,7 +335,7 @@ def get_config_directory():
 
     """
     # avoid circular import
-    from .commands.stacker import Stacker
+    from .commands.stacker import Stacker  # pylint: disable=import-outside-toplevel
     deprecation_msg = ('get_config_directory has been deprecated and will be '
                        'removed in the next major release')
     warnings.warn(deprecation_msg, DeprecationWarning)
@@ -715,7 +715,7 @@ def fetch_git_package(self, config):
         """
         # only loading git here when needed to avoid load errors on systems
         # without git installed
-        from git import Repo
+        from git import Repo  # pylint: disable=import-outside-toplevel
 
         ref = self.determine_git_ref(config)
         dir_name = self.sanitize_git_path(uri=config['uri'], ref=ref)

runway/commands/runway/gen_sample.py

@@ -83,6 +83,7 @@
 
 def generate_tfstate_cfn_template():
     """Return rendered CFN template yaml."""
+    # pylint: disable=import-outside-toplevel
     from runway.blueprints.tf_state import TfState
 
     return to_yaml(TfState('test',
@@ -169,6 +170,7 @@ def generate_sample_k8s_cfn_repo(env_root):
                      repo_dir)
         sys.exit(1)
 
+    # pylint: disable=import-outside-toplevel
     from runway.blueprints.k8s.k8s_master import Cluster
     from runway.blueprints.k8s.k8s_iam import Iam
     from runway.blueprints.k8s.k8s_workers import NodeGroup as WorkerNodeGroup

runway/commands/runway_command.py

@@ -126,8 +126,8 @@ def get_env(path, ignore_git_branch=False, prompt_if_unexpected=False):
     else:
         # These are not located with the top imports because they throw an
         # error if git isn't installed
-        from git import Repo as GitRepo
-        from git.exc import InvalidGitRepositoryError
+        from git import Repo as GitRepo  # pylint: disable=import-outside-toplevel
+        from git.exc import InvalidGitRepositoryError  # pylint: disable=import-outside-toplevel
 
         try:
             b_name = GitRepo(

runway/env_mgr/kbenv.py

@@ -7,8 +7,8 @@
 import tempfile
 
 # Old pylint on py2.7 incorrectly flags these
-from six.moves.urllib.request import urlretrieve  # noqa pylint: disable=import-error,line-too-long
-from six.moves.urllib.error import URLError  # noqa pylint: disable=import-error,relative-import,line-too-long
+from six.moves.urllib.request import urlretrieve  # pylint: disable=E
+from six.moves.urllib.error import URLError  # pylint: disable=E
 
 from . import EnvManager, ensure_versions_dir_exists, handle_bin_download_error
 from ..util import md5sum

runway/env_mgr/tfenv.py

@@ -14,8 +14,8 @@
 import hcl
 import requests
 # Old pylint on py2.7 incorrectly flags these
-from six.moves.urllib.request import urlretrieve  # noqa pylint: disable=import-error,line-too-long
-from six.moves.urllib.error import URLError  # noqa pylint: disable=import-error,relative-import,line-too-long
+from six.moves.urllib.request import urlretrieve  # pylint: disable=E
+from six.moves.urllib.error import URLError  # pylint: disable=E
 
 from . import EnvManager, ensure_versions_dir_exists, handle_bin_download_error
 from ..util import get_hash_for_filename, sha256sum

runway/sources/git.py

@@ -53,7 +53,7 @@ def __init__(self, uri='', location='', options=None, **kwargs):
     def fetch(self):
         # type: () -> str
         """Retrieve the git repository from it's remote location."""
-        from git import Repo
+        from git import Repo  # pylint: disable=import-outside-toplevel
 
         ref = self.__determine_git_ref()  # type: str
         dir_name = '_'.join([self.sanitize_git_path(self.uri), ref])  # type: str

runway/templates/k8s-tf-repo/eks-base.tf/get_idp_root_cert_thumbprint.py

@@ -4,7 +4,7 @@
 import socket
 import sys
 # This false pylint error is only an issue on py2
-from six.moves.urllib.parse import urlparse  # pylint: disable=relative-import
+from six.moves.urllib.parse import urlparse  # pylint: disable=E
 import requests
 from cryptography.hazmat.primitives import serialization
 from OpenSSL import SSL

runway/variables.py

@@ -699,7 +699,7 @@ def _resolve_legacy(self, context, provider):
 
         """
         if isinstance(self.handler, type):
-            import warnings
+            import warnings  # pylint: disable=import-outside-toplevel
             warn_msg = ('Old style lookup in use. Please upgrade to use '
                         'the new style of Lookups that accepts '
                         '"**kwargs".')