Enable integration testing as if cfnlint is a module (#1234)

aws-cloudformation · Dec 2, 2019 · efeb44e · efeb44e
1 parent 635b962
commit efeb44e
Show file tree

Hide file tree

Showing 4 changed files with 81 additions and 32 deletions.
diff --git a/test/integration/__init__.py b/test/integration/__init__.py
@@ -6,6 +6,7 @@
 import json
 import subprocess
 from test.testlib.testcase import BaseTestCase
+import cfnlint.core
 
 
 class BaseCliTestCase(BaseTestCase):
@@ -61,3 +62,30 @@ def run_scenarios(self, extra_params=None):
                 else:
                     self.assertItemsEqual(expected_results, matches, 'Expected {} failures, got {} on {}'.format(
                         len(expected_results), len(matches), filename))
+
+    def run_module_integration_scenarios(self, rules):
+        """ Test using cfnlint as a module integrated into another package"""
+
+        cfnlint.core.configure_logging(None)
+        regions = ['us-east-1']
+        for scenario in self.scenarios:
+            filename = scenario.get('filename')
+            results_filename = scenario.get('results_filename')
+            expected_results = scenario.get('results', [])
+
+            if results_filename and not expected_results:
+                with open(results_filename) as json_data:
+                    expected_results = json.load(json_data)
+
+            template = cfnlint.decode.cfn_yaml.load(filename)
+
+            matches = cfnlint.core.run_checks(
+                filename,
+                template,
+                rules,
+                regions
+            )
+
+            # Only check that the error count matches as the formats are different
+            self.assertEqual(len(expected_results), len(matches), 'Expected {} failures, got {} on {}'.format(
+                len(expected_results), len(matches), filename))
diff --git a/test/integration/test_good_templates.py b/test/integration/test_good_templates.py
@@ -3,30 +3,31 @@
 SPDX-License-Identifier: MIT-0
 """
 from test.integration import BaseCliTestCase
+import cfnlint.core
 
 
 class TestQuickStartTemplates(BaseCliTestCase):
     """Test QuickStart Templates Parsing """
 
     scenarios = [
         {
-            "filename": 'test/fixtures/templates/good/generic.yaml',
-            "results": [],
-            "exit_code": 0,
+            'filename': 'test/fixtures/templates/good/generic.yaml',
+            'results': [],
+            'exit_code': 0,
         },
         {
-            "filename": 'test/fixtures/templates/good/minimal.yaml',
-            "results": [],
-            "exit_code": 0,
+            'filename': 'test/fixtures/templates/good/minimal.yaml',
+            'results': [],
+            'exit_code': 0,
         },
         {
-            "filename": 'test/fixtures/templates/good/transform.yaml',
-            "results": [],
-            "exit_code": 0,
+            'filename': 'test/fixtures/templates/good/transform.yaml',
+            'results': [],
+            'exit_code': 0,
         },
         {
-            "filename": 'test/fixtures/templates/bad/transform_serverless_template.yaml',
-            "results": [
+            'filename': 'test/fixtures/templates/bad/transform_serverless_template.yaml',
+            'results': [
                 {
                     'Filename': 'test/fixtures/templates/bad/transform_serverless_template.yaml',
                     'Location': {
@@ -94,31 +95,31 @@ class TestQuickStartTemplates(BaseCliTestCase):
                     'Message': "Error transforming template: Resource with id [myFunctionMyTimer] is invalid. Missing required property 'Schedule'."
                 }
             ],
-            "exit_code": 2,
+            'exit_code': 2,
         },
         {
-            "filename": 'test/fixtures/templates/good/conditions.yaml',
-            "results": [],
-            "exit_code": 0,
+            'filename': 'test/fixtures/templates/good/conditions.yaml',
+            'results': [],
+            'exit_code': 0,
         },
         {
             'filename': 'test/fixtures/templates/good/resources_codepipeline.yaml',
-            "results": [],
-            "exit_code": 0,
+            'results': [],
+            'exit_code': 0,
         },
         {
             'filename': 'test/fixtures/templates/good/transform_serverless_api.yaml',
-            "results": [],
-            "exit_code": 0,
+            'results': [],
+            'exit_code': 0,
         },
         {
             'filename': 'test/fixtures/templates/good/transform_serverless_function.yaml',
-            "results": [],
-            "exit_code": 0,
+            'results': [],
+            'exit_code': 0,
         },
         {
             'filename': 'test/fixtures/templates/good/transform_serverless_globals.yaml',
-            "results": [
+            'results': [
                 {
                     'Level': 'Error',
                     'Message': 'Deprecated runtime (nodejs6.10) specified. Updating disabled since 2019-08-12, please consider to update to nodejs10.x',
@@ -136,25 +137,31 @@ class TestQuickStartTemplates(BaseCliTestCase):
                     }
                 }
             ],
-            "exit_code": 2,
+            'exit_code': 2,
         },
         {
             'filename': 'test/fixtures/templates/good/transform/list_transform.yaml',
-            "results": [],
-            "exit_code": 0,
+            'results': [],
+            'exit_code': 0,
         },
         {
             'filename': 'test/fixtures/templates/good/transform/list_transform_many.yaml',
-            "results": [],
-            "exit_code": 0,
+            'results': [],
+            'exit_code': 0,
         },
         {
             'filename': 'test/fixtures/templates/good/transform/list_transform_not_sam.yaml',
-            "results": [],
-            "exit_code": 0,
+            'results': [],
+            'exit_code': 0,
         }
     ]
 
     def test_templates(self):
         """Test Successful JSON Parsing"""
         self.run_scenarios()
+
+    def test_module_integration(self):
+        """ Test same templates using integration approach"""
+        rules = cfnlint.core.get_rules(
+            [], [], ['E', 'I', 'W'], {}, False)
+        self.run_module_integration_scenarios(rules)
diff --git a/test/integration/test_quickstart_templates.py b/test/integration/test_quickstart_templates.py
@@ -3,6 +3,7 @@
 SPDX-License-Identifier: MIT-0
 """
 from test.integration import BaseCliTestCase
+import cfnlint.core
 
 
 class TestQuickStartTemplates(BaseCliTestCase):
@@ -72,6 +73,7 @@ class TestQuickStartTemplates(BaseCliTestCase):
     ]
 
     def test_templates(self):
-        """Test Successful JSON Parsing"""
-        self.maxDiff = None
-        self.run_scenarios(['--include-checks', 'I', '--include-expiremental'])
+        """ Test same templates using integration approach"""
+        rules = cfnlint.core.get_rules(
+            [], [], ['I', 'E', 'W'], {}, True)
+        self.run_module_integration_scenarios(rules)
diff --git a/test/integration/test_quickstart_templates_non_strict.py b/test/integration/test_quickstart_templates_non_strict.py
@@ -3,6 +3,7 @@
 SPDX-License-Identifier: MIT-0
 """
 from test.integration import BaseCliTestCase
+import cfnlint.core
 
 
 class TestQuickStartTemplates(BaseCliTestCase):
@@ -39,3 +40,14 @@ def test_templates(self):
             '--include-expiremental',
             '--configure-rule', 'E3012:strict=false',
         ])
+
+    def test_module_integration(self):
+        """ Test same templates using integration approach"""
+        rules = cfnlint.core.get_rules(
+            [], [], ['I', 'E', 'W'],
+            {
+                'E3012': {
+                    'strict': False,
+                }
+            }, True)
+        self.run_module_integration_scenarios(rules)