Fix first_run and add friendlier component import by refactoring source catalog handling

CHANGELOG.md

@@ -1,8 +1,8 @@
 GovReady-Q Release Notes
 ========================
 
-v0.9.9-dev (August xx, 2021)
-----------------------------
+v0.9.8.1-dev (August xx, 2021)
+------------------------------
 
 **UI changes**
 
@@ -11,7 +11,14 @@ v0.9.9-dev (August xx, 2021)
 
 **Developer changes**
 
+* Move creation of users in first_run to earlier in script.
+* Use faster bulk_create importing components.
+
+**Data changes**
+
+* Update sample components to OSCAL 1.0.0.
 * Change CatalogData JSONFields to Django JSONField for better searching options.
+* Import components and their statements even when catalog not found or statement control ids are not found in referenced catalog.
 
 
 v0.9.8 (August 09, 2021)

VERSION

@@ -1 +1 @@
-v0.9.9-dev
+v0.9.8.1-dev

controls/utilities.py

@@ -68,26 +68,27 @@ def oscalize_control_id(cl_id):
 
     return cl_id
 
-
-def oscalize_catalog_key(catalogkey):
-    """ Covers empty catalog key case. Otherwise, outputs an oscal standard catalog key from various common formats for catalog keys
-    NIST_SP_800_53_rev4 --> NIST_SP-800-53_rev4
+def oscalize_catalog_key(catalogkey=None):
+    """Outputs an oscal standard catalog key from various common formats for catalog keys
+    Covers empty catalog key case
+    Example: NIST_SP_800_53_rev4 --> NIST_SP-800-53_rev4
     """
+    DEFAULT_CATALOG_KEY = 'NIST_SP-800-53_rev5'
+    if catalogkey is None or catalogkey=='':
+        return DEFAULT_CATALOG_KEY
     # If coming with reference to some path to catalog json file
     # (e.g. '../../../nist.gov/SP800-53/rev4/json/NIST_SP-800-53_rev4_catalog.json', 'FedRAMP_rev4_HIGH-baseline_profile.json')
     if ".json" in catalogkey:
         catalogkey = catalogkey.split('/')[-1].split('_catalog.json')[0].split(".json")[0]
-    # A default catalog key
-    if catalogkey=='':
-        catalogkey = 'NIST_SP-800-53_rev4'
     # Handle the default improperly formatted control id
     if catalogkey.count("_") > 2 and "800" in catalogkey:
         split_key_list = catalogkey.split("_800_")
         catalogkey = split_key_list[0] + "-800-" + split_key_list[1]
+    # TODO: Handle other cases
+    #if catalogkey in ['NIST_SP-800-53_rev4', 'NIST_SP-800-53_rev4', 'CMMC_ver1', 'NIST_SP-800-171_rev1']:
 
     return catalogkey
 
-
 def get_control_statement_part(control_stmnt_id):
     """ Parses part from control statement id
     ra-5_smt.a --> a

controls/views.py

@@ -738,6 +738,15 @@ def statement_id_from_control(control_id, part_id):
 
         return f"{control_id}"
 
+    def generate_source(self, src_str):
+        """Return a valid catalog source given string"""
+        DEFAULT_SOURCE = "NIST_SP-800-53_rev5"
+        if not src_str:
+            return DEFAULT_SOURCE
+        # TODO: Handle other cases
+        source = src_str
+        return source
+
     def as_json(self):
         # Build OSCAL
         # Example: https://github.com/usnistgov/OSCAL/blob/master/src/content/ssp-example/json/example-component.json
@@ -765,10 +774,10 @@ def as_json(self):
                 "components": [
                    {
                         "uuid": comp_uuid,
-                       "type": self.element.component_type.lower() if self.element.component_type is not None else "software",
-                       "title": self.element.full_name or self.element.name,
+                        "type": self.element.component_type.lower() if self.element.component_type is not None else "software",
+                        "title": self.element.full_name or self.element.name,
                         "description": self.element.description,
-                         "responsible-roles": responsible_roles, # TODO: gathering party-uuids, just filling for now
+                        "responsible-roles": responsible_roles, # TODO: gathering party-uuids, just filling for now
                         "control-implementations": control_implementations
                     }
                 ]
@@ -813,7 +822,7 @@ def as_json(self):
         for sid_class, requirements in by_class.items():
             control_implementation = {
                 "uuid":str(uuid4()),# TODO: Not sure if this should implemented or just generated here.
-                "source": smt.source if smt.source else "Govready",
+                "source": self.generate_source(smt.source if smt.source else None),
                 "description": f"This is a partial implementation of the {sid_class} catalog, focusing on the control enhancement {requirements[0].get('control-id')}.",
                 "implemented-requirements": [req for req in requirements]
             }
@@ -897,17 +906,6 @@ def import_components_as_json(self, import_name, json_object, request=None, exis
         new_import_record = self.create_import_record(import_name, created_components, existing_import_record=existing_import_record)
         return new_import_record
 
-    # def find_import_record_by_name(self, import_name):
-    #     """Returns most recent existing import record by name
-
-    #     @type import_name: str
-    #     @param import_name: Name of import file (if it exists)
-    #     """
-
-    #     found_import_record = ImportRecord.objects.filter(name=import_name).last()
-
-    #     return found_import_record
-
     def create_import_record(self, import_name, components, existing_import_record=False):
         """Associates components and statements to an import record
 
@@ -969,15 +967,11 @@ def create_component(self, component_json):
 
         logger.info(f"Component {new_component.name} created with UUID {new_component.uuid}.")
         control_implementation_statements = component_json.get('control-implementations', None)
-        catalog = "missing"
-        # If there is an data in the control-implementations key
+        # catalog = "missing"
+        # If there data exists the OSCAL component's control-implementations key
         if control_implementation_statements:
-            # For each element if there is a source and the oscalized key is in the available keys
-            # Then create statements otherwise it will return an empty list
             for control_element in control_implementation_statements:
-                if 'source' in control_element:
-                    if oscalize_catalog_key(control_element['source']) in Catalogs().catalog_keys:
-                        catalog = oscalize_catalog_key(control_element['source'])
+                catalog = oscalize_catalog_key(control_element.get('source', None))
                 created_statements = self.create_control_implementation_statements(catalog, control_element, new_component)
         # If there are no valid statements in the json object
         if created_statements == []:
@@ -1000,55 +994,27 @@ def create_control_implementation_statements(self, catalog_key, control_element,
         @returns: New statement objects created
         """
 
-        statements_created = []
-        if catalog_key == "missing":
-            logger.info(f"Control Catalog {catalog_key} missing skipping Statement creation...")
-            return statements_created
+        new_statements = []
         implemented_reqs = control_element['implemented-requirements'] if 'implemented-requirements' in control_element else []
         for implemented_control in implemented_reqs:
-
-            control_id = implemented_control['control-id'] if 'control-id' in implemented_control else ''
-            #statements = implemented_control['statements'] if 'statements' in implemented_control else ''
-            if self.control_exists_in_catalog(catalog_key, control_id):
-                new_statement = Statement.objects.create(
-                    sid=control_id,
-                    sid_class=catalog_key,
-                    pid=get_control_statement_part(control_id),
-                    source=control_element['source'] if 'source' in control_element else catalog_key,
-                    uuid=control_element['uuid'] if 'uuid' in control_element else uuid.uuid4(),
-                    body=implemented_control['description'] if 'description' in implemented_control else '',
-                    statement_type=StatementTypeEnum.CONTROL_IMPLEMENTATION_PROTOTYPE.name,
-                    remarks=implemented_control['remarks'] if 'remarks' in implemented_control else '',
-                    status=implemented_control['status'] if 'status' in implemented_control else None,
-                    producer_element=parent_component,
-                )
-
-                logger.info(f"New statement with UUID {new_statement.uuid} created.")
-                statements_created.append(new_statement)
-
-            else:
-                logger.info(f"Control {control_id} doesn't exist in catalog {catalog_key}. Skipping Statement...")
-
+            control_id = implemented_control['control-id'] if 'control-id' in implemented_control else 'missing'
+            new_statement = Statement(
+                sid=control_id,
+                sid_class=catalog_key,
+                pid=get_control_statement_part(control_id),
+                source=catalog_key,
+                uuid=control_element['uuid'] if 'uuid' in control_element else uuid.uuid4(),
+                body=implemented_control['description'] if 'description' in implemented_control else '',
+                statement_type=StatementTypeEnum.CONTROL_IMPLEMENTATION_PROTOTYPE.name,
+                remarks=implemented_control['remarks'] if 'remarks' in implemented_control else '',
+                status=implemented_control['status'] if 'status' in implemented_control else None,
+                producer_element=parent_component,
+            )
+            logger.info(f"New statement with UUID {new_statement.uuid} being created.")
+            new_statements.append(new_statement)
+        statements_created = Statement.objects.bulk_create(new_statements)
         return statements_created
 
-    def control_exists_in_catalog(self, catalog_key, control_id):
-        """Searches for the presence of a specific control id in a catalog.
-
-        @type catalog_key: str
-        @param catalog_key: Catalog Key
-        @type control_id: str
-        @param control_id: Control id
-        @rtype: bool
-        @returns: True if control id exists in the catalog. False otherwise
-        """
-
-        if catalog_key not in Catalogs()._list_catalog_keys():
-            return False
-        else:
-            catalog = Catalog.GetInstance(catalog_key)
-            control = catalog.get_control_by_id(control_id)
-            return True if control is not None else False
-
 @login_required
 def add_selected_components(system, import_record):
         """Add a component from the library or a compliance app to the project and its statements using the import record"""
@@ -1159,7 +1125,6 @@ def system_element_control(request, system_id, element_id, catalog_key, control_
         }
         return render(request, "systems/element_detail_control.html", context)
 
-
 def edit_component_state(request, system_id, element_id):
     """
     Edit system component state

q-files/vendors/govready/components/OSCAL/cybrary.json

@@ -1,74 +1,82 @@
 {
   "component-definition": {
-    "uuid": "e7b37ba4-ce0d-46bb-8900-231003dba10c",
+    "uuid": "343d3b0a-30a7-4196-b2b9-4bb3df85e04d",
     "metadata": {
-      "title": "Cybrary Component-to-Control Narratives",
-      "published": "2021-02-15T10:10:57+00:00",
-      "last-modified": "2021-01-05T03:40:57+00:00",
-      "version": "string",
-      "oscal-version": "1.0.0-rc1"
+      "title": "Cybrary",
+      "last-modified": "2021-03-23T23:45:28+00:00",
+      "version": "2021-03-23T23:45:28+00:00",
+      "oscal-version": "1.0.0",
+      "parties": [
+        {
+          "uuid": "49e75032-c17a-4b57-9afd-cf7f23fc1dee",
+          "type": "organization",
+          "name": "Main"
+        }
+      ]
     },
-    "components": {
-      "a42932ab-7aa9-4c34-a643-2840b6630a19": {
+    "components": [
+      {
+        "uuid": "a42932ab-7aa9-4c34-a643-2840b6630a19",
+        "type": "software",
         "title": "Cybrary",
-        "type": "system_element",
         "description": "Training course",
+        "responsible-roles": [
+          {
+            "role-id": "supplier",
+            "party-uuids": [
+              "49e75032-c17a-4b57-9afd-cf7f23fc1dee"
+            ]
+          }
+        ],
         "control-implementations": [
           {
-            "uuid": "8a8f9643-2c68-4f87-a47d-70d8af924e73",
+            "uuid": "f986c0f5-47b3-4c06-aa74-e7d0d3683f8f",
             "source": "NIST_SP-800-53_rev4",
-            "description": "Partial implementation of NIST_SP-800-53_rev4",
+            "description": "This is a partial implementation of the at-2 catalog, focusing on the control enhancement at-2.",
             "implemented-requirements": [
               {
-                "uuid": "8a9856df-69b4-4b5c-9709-169b34ef06b9",
-                "control-id": "at-2",
-                "description": "",
-                "remarks": "",
-                "statements": {
-                  "at-2_smt": {
-                    "uuid": "3d9091a6-1315-4fb9-94c7-f138e0d93191",
-                    "description": "Cybrary provides a complete catalog of security awareness and role-based security training. Cybrary also tracks the training completed by each user. Cybrary content can be tailored for annual security awareness.",
-                    "remarks": ""
-                  }
-                }
+                "uuid": "3d9091a6-1315-4fb9-94c7-f138e0d93191",
+                "description": "Cybrary provides a complete catalog of security awareness and role-based security training. Cybrary also tracks the training completed by each user. Cybrary content can be tailored for annual security awareness.",
+                "control-id": "at-2"
               },
               {
-                "uuid": "a25c123b-b78d-4833-b7f9-7826676699be",
-                "control-id": "at-3",
-                "description": "",
-                "remarks": "",
-                "statements": {
-                  "at-3_smt": {
-                    "uuid": "08a5bcb1-80ff-4c44-8f5f-28d24c796e63",
-                    "description": "Role base training implementation narrative.",
-                    "remarks": ""
-                  }
-                }
+                "uuid": "3d9091a6-1315-4fb9-94c7-f138e0d93191",
+                "description": "Cybrary provides a complete catalog of security awareness, role-based security training, and vendor training. Cybrary also tracks the training completed by each user. Cybrary content can be tailored for annual security awareness. \r\nCybrary version 4.5 is being used by system Admins.",
+                "control-id": "at-2"
               }
             ]
           },
           {
-            "uuid": "ed191423-96b6-4a8f-98b9-de07b953d36d",
-            "source": "NIST_SP-800-53_rev5",
-            "description": "Partial implementation of NIST_SP-800-53_rev5",
+            "uuid": "5a8952e7-1846-49e1-9a4d-bd58f23610e4",
+            "source": "NIST_SP-800-53_rev4",
+            "description": "This is a partial implementation of the at-2.2 catalog, focusing on the control enhancement at-2.2.",
             "implemented-requirements": [
               {
-                "uuid": "28383ff1-8870-484e-b31b-3d00bfd6fbae",
-                "control-id": "at-2.2",
-                "description": "",
-                "remarks": "",
-                "statements": {
-                  "at-2.2_smt": {
-                    "uuid": "ddba3705-efbd-446f-8ac1-2c68f1b28e55",
-                    "description": "THis is how Cybrary helps with Insider Threat.",
-                    "remarks": ""
-                  }
-                }
+                "uuid": "ddba3705-efbd-446f-8ac1-2c68f1b28e55",
+                "description": "THis is how Cybrary helps with Insider Threat.",
+                "control-id": "at-2.2"
+              }
+            ]
+          },
+          {
+            "uuid": "04507036-0a47-4612-b39a-576e60df614c",
+            "source": "NIST_SP-800-53_rev4",
+            "description": "This is a partial implementation of the at-3 catalog, focusing on the control enhancement at-3.",
+            "implemented-requirements": [
+              {
+                "uuid": "08a5bcb1-80ff-4c44-8f5f-28d24c796e63",
+                "description": "Role base training implementation narrative.",
+                "control-id": "at-3"
+              },
+              {
+                "uuid": "08a5bcb1-80ff-4c44-8f5f-28d24c796e63",
+                "description": "Role base training implementation narrative. sdfsdfdf",
+                "control-id": "at-3"
               }
             ]
           }
         ]
       }
-    }
+    ]
   }
 }