Add manifest units to expand output (New) (#1489)

* Also print manifest to expand

* Document and partially de-functionalize code

* Add tests for the new export units

* Also test text exporter for manifest exporting

* Check null tempalte-id

checkbox-ng/checkbox_ng/launcher/subcommands.py

@@ -24,8 +24,8 @@
 from collections import defaultdict
 from string import Formatter
 from tempfile import TemporaryDirectory
-import textwrap
 import fnmatch
+import itertools
 import contextlib
 import gettext
 import json
@@ -1324,6 +1324,37 @@ def register_arguments(self, parser):
             help=_("output format: 'text' or 'json' (default: %(default)s)"),
         )
 
+    def _get_relevant_manifest_units(self, jobs_and_templates_list):
+        """
+        Get all manifest units that are cited in the jobs_and_templates_list
+        resource expressions
+        """
+        # get all manifest units
+        manifest_units = filter(
+            lambda unit: unit.unit == "manifest entry",
+            self.sa._context.unit_list,
+        )
+        # get all jobs/templates that have a requires and do require a manifest
+        # entry
+        job_requires = [
+            requires
+            for requires in map(
+                lambda x: x.get_record_value("requires"),
+                jobs_and_templates_list,
+            )
+            if requires and "manifest" in requires
+        ]
+
+        # only return manifest entries that are actually required by any job in
+        # the list
+        return filter(
+            lambda manifest_unit: any(
+                "manifest.{}".format(manifest_unit.partial_id) in require
+                for require in job_requires
+            ),
+            manifest_units,
+        )
+
     def invoked(self, ctx):
         self.ctx = ctx
         session_title = "checkbox-expand-{}".format(ctx.args.TEST_PLAN)
@@ -1340,31 +1371,48 @@ def invoked(self, ctx):
         tp = self.sa._context._test_plan_list[0]
         tp_us = TestPlanUnitSupport(tp)
         self.override_list = tp_us.override_list
+
         jobs_and_templates_list = select_units(
             all_jobs_and_templates,
             [tp.get_mandatory_qualifier()] + [tp.get_qualifier()],
         )
+        relevant_manifest_units = self._get_relevant_manifest_units(
+            jobs_and_templates_list
+        )
 
+        units_to_print = itertools.chain(
+            relevant_manifest_units, iter(jobs_and_templates_list)
+        )
         obj_list = []
-        for unit in jobs_and_templates_list:
+        for unit in units_to_print:
             obj = unit._raw_data.copy()
             obj["unit"] = unit.unit
             obj["id"] = unit.id  # To get the fully qualified id
-            obj["certification-status"] = (
-                self.get_effective_certification_status(unit)
-            )
-            if unit.template_id:
-                obj["template-id"] = unit.template_id
+            # these two don't make sense for manifest units
+            if unit.unit != "manifest entry":
+                obj["certification-status"] = (
+                    self.get_effective_certification_status(unit)
+                )
+                if unit.template_id:
+                    obj["template-id"] = unit.template_id
             obj_list.append(obj)
-        obj_list.sort(key=lambda x: x.get("template-id", x["id"]))
+
+        obj_list.sort(key=lambda x: x.get("template-id", x["id"]) or x["id"])
+
         if ctx.args.format == "json":
-            print(json.dumps(obj_list, sort_keys=True))
+            json.dump(obj_list, sys.stdout, sort_keys=True)
         else:
             for obj in obj_list:
                 if obj["unit"] == "template":
                     print("Template '{}'".format(obj["template-id"]))
-                else:
+                elif obj["unit"] == "manifest entry":
+                    print("Manifest '{}'".format(obj["id"]))
+                elif obj["unit"] == "job":
                     print("Job '{}'".format(obj["id"]))
+                else:
+                    raise AssertionError(
+                        "Unknown unit type {}".format(obj["unit"])
+                    )
 
     def get_effective_certification_status(self, unit):
         if unit.unit == "template":

checkbox-ng/checkbox_ng/launcher/test_subcommands.py

@@ -813,6 +813,16 @@ def setUp(self):
         self.launcher = Expand()
         self.ctx = Mock()
         self.ctx.args = Mock(TEST_PLAN="", format="")
+
+        selected_1 = Mock(unit="manifest entry", id="some", partial_id="some")
+        selected_1._raw_data.copy.return_value = {}
+        selected_2 = Mock(
+            unit="manifest entry", id="other", partial_id="other"
+        )
+        selected_2._raw_data.copy.return_value = {}
+        not_selected = Mock(unit="manifest entry", partial_id="not_selected")
+        not_selected._raw_data.copy.return_value = {}
+
         self.ctx.sa = Mock(
             start_new_session=Mock(),
             get_test_plans=Mock(return_value=["test-plan1", "test-plan2"]),
@@ -821,6 +831,7 @@ def setUp(self):
             _context=Mock(
                 state=Mock(unit_list=[]),
                 _test_plan_list=[Mock()],
+                unit_list=[selected_1, selected_2, not_selected],
             ),
         )
 
@@ -844,17 +855,22 @@ def test_invoke__text(self, mock_select_units, mock_tpus, stdout):
                 "template-id": "test-template",
                 "id": "test-{res}",
                 "template-summary": "Test Template Summary",
+                "requires": "manifest.some == 'True'",
             }
         )
         job1 = JobDefinition(
             {
                 "id": "job1",
+                "requires": "manifest.other == 'Other'",
             }
         )
         mock_select_units.return_value = [job1, template1]
         self.ctx.args.TEST_PLAN = "test-plan1"
         self.launcher.invoked(self.ctx)
         self.assertIn("Template 'test-template'", stdout.getvalue())
+        self.assertIn("Manifest 'some'", stdout.getvalue())
+        self.assertIn("Manifest 'other'", stdout.getvalue())
+        self.assertNotIn("Manifest 'not_selected'", stdout.getvalue())
 
     @patch("sys.stdout", new_callable=StringIO)
     @patch("checkbox_ng.launcher.subcommands.TestPlanUnitSupport")
@@ -865,18 +881,24 @@ def test_invoke__json(self, mock_select_units, mock_tpus, stdout):
                 "template-id": "test-template",
                 "id": "test-{res}",
                 "template-summary": "Test Template Summary",
+                "requires": "manifest.some == 'True'",
             }
         )
         job1 = JobDefinition(
             {
                 "id": "job1",
+                "requires": "manifest.other == 'Other'",
             }
         )
+
         mock_select_units.return_value = [job1, template1]
         self.ctx.args.TEST_PLAN = "test-plan1"
         self.ctx.args.format = "json"
         self.launcher.invoked(self.ctx)
         self.assertIn('"template-id": "test-template"', stdout.getvalue())
+        self.assertIn('"id": "some"', stdout.getvalue())
+        self.assertIn('"id": "other"', stdout.getvalue())
+        self.assertNotIn('"id": "not_selected"', stdout.getvalue())
 
     def test_get_effective_certificate_status(self):
         job1 = JobDefinition(