Feature/0.9.10

CHANGELOG.md

@@ -1,6 +1,23 @@
 GovReady-Q Release Notes
 ========================
 
+v0.9.10 (August 16, 2021)
+-------------------------
+
+**Developer changes**
+
+* Component tags now correctly included on OSCAL component export and included on OSCAL component import.
+* Component tags now correctly included on OSCAL SSP generation.
+
+**Bug fix**
+
+* Add the catalog_key to statement's `sid_class` and `source` fields when adding new statement to a component in library.
+
+**Data fix**
+
+* Add migration in controls to load default control catalogs into CatalogData in database. Remove loading of catalogs via first_run command.
+
+
 v0.9.9 (August 12, 2021)
 ------------------------
 

VERSION

@@ -1 +1 @@
-v0.9.9
+v0.9.10

controls/migrations/0060_auto_20210816_1634.py

@@ -0,0 +1,51 @@
+# Generated by Django 3.2.5 on 2021-08-16 16:34
+
+from django.db import migrations
+import os.path
+import json
+
+from controls.models import Element
+from controls.oscal import CatalogData
+
+
+def load_catalog_data(apps, schema_editor):
+    """Load control catalog data into database"""
+
+    # Load the default control catalogs and baselines
+    CATALOG_PATH = os.path.join(os.path.dirname(__file__),'..','data','catalogs')
+    BASELINE_PATH = os.path.join(os.path.dirname(__file__),'..','data','baselines')
+
+    # TODO: Check directory exists
+    catalog_files = [file for file in os.listdir(CATALOG_PATH) if file.endswith('.json')]
+    # Load catalog and baseline data into database records from source files if data records do not exist in database
+    for cf in catalog_files:
+        catalog_key = cf.replace("_catalog.json", "")
+        with open(os.path.join(CATALOG_PATH,cf), 'r') as json_file:
+            catalog_json = json.load(json_file)
+        baseline_filename = cf.replace("_catalog.json", "_baselines.json")
+        if os.path.isfile(os.path.join(BASELINE_PATH, baseline_filename)):
+            with open(os.path.join(BASELINE_PATH, baseline_filename), 'r') as json_file:
+                baselines_json = json.load(json_file)
+        else:
+            baselines_json = {}
+
+        catalog, created = CatalogData.objects.get_or_create(
+                catalog_key=catalog_key,
+                catalog_json=catalog_json,
+                baselines_json=baselines_json
+            )
+        if created:
+            print(f"{catalog_key} record created into database")
+        else:
+                print(f"{catalog_key} record found in database")
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('controls', '0059_auto_20210811_0001'),
+    ]
+
+    operations = [
+         migrations.RunPython(load_catalog_data)
+    ]

controls/views.py

@@ -30,7 +30,7 @@
 from django.views.generic import ListView
 from simple_history.utils import update_change_reason
 
-from siteapp.models import Project, Organization
+from siteapp.models import Project, Organization, Tag
 from siteapp.settings import GOVREADY_URL
 from system_settings.models import SystemSettings
 from .forms import ElementEditForm
@@ -651,12 +651,21 @@ def as_json(self):
 
         of["system-security-plan"]["metadata"]['roles'] =  [{"id": auths.get('role', "member").split(';')[0], "title": auths.get('role', "member").split(';')[0].capitalize()  } for user, auths in users]
         of["system-security-plan"]["system-implementation"]['users'] = [{"uuid":user_party_uuid, "title":user.username, "role-ids": [auths.get('role', "member").split(';')[0]]} for user, auths in users]
-        of["system-security-plan"]["system-implementation"]['components'] = [{"uuid":str(comp_ele.uuid), "title":comp_ele.name, "description":comp_ele.description, "status": {"state": comp_ele.component_state}, "type":comp_ele.component_type, "responsible-roles": [{
-              "role-id": "asset-owner",
-              "party-uuids": [
-                user_party_uuid
-              ]
-            }]} for comp_ele in components]# TODO: responsible-roles
+        of["system-security-plan"]["system-implementation"]['components'] = [{"uuid":str(comp_ele.uuid),
+                                                                              "title":comp_ele.name,
+                                                                              "description":comp_ele.description,
+                                                                              "status": {"state": comp_ele.component_state},
+                                                                              "type":comp_ele.component_type,
+                                                                              "responsible-roles": [{
+                                                                                "role-id": "asset-owner",
+                                                                                "party-uuids": [user_party_uuid]
+                                                                                }],
+                                                                              "props": [{"name": "tag",
+                                                                                         "ns": "https://govready.com/ns/oscal",
+                                                                                         "value": tag.label} for tag in
+                                                                                        comp_ele.tags.all()]
+                                                                              } for comp_ele in components]# TODO: responsible-roles
+
         # System characteristics
         # TODO: status remarks, authorization-boundary
         security_body = project.system.get_security_impact_level
@@ -768,8 +777,7 @@ def as_json(self):
                     "last-modified": self.element.updated.replace(microsecond=0).isoformat(),
                     "version": self.element.updated.replace(microsecond=0).isoformat(),
                     "oscal-version": self.element.oscal_version,
-                    "parties": parties,
-                    "props": props
+                    "parties": parties
                 },
                 "components": [
                    {
@@ -778,6 +786,7 @@ def as_json(self):
                         "title": self.element.full_name or self.element.name,
                         "description": self.element.description,
                         "responsible-roles": responsible_roles, # TODO: gathering party-uuids, just filling for now
+                        "props": props,
                         "control-implementations": control_implementations
                     }
                 ]
@@ -966,8 +975,17 @@ def create_component(self, component_json):
         )
 
         logger.info(f"Component {new_component.name} created with UUID {new_component.uuid}.")
+
+        component_props = component_json.get('props', None)
+        if component_props is not None:
+            desired_tags = set([prop['value'] for prop in component_props if prop['name'] == 'tag' and 'ns' in prop and prop['ns'] == "https://govready.com/ns/oscal"])
+            existing_tags = Tag.objects.filter(label__in=desired_tags).values('id', 'label')
+            tags_to_create = desired_tags.difference(set([tag['label'] for tag in existing_tags]))
+            new_tags = Tag.objects.bulk_create([Tag(label=tag) for tag in tags_to_create])
+            all_tag_ids = [tag.id for tag in new_tags] + [tag['id'] for tag in existing_tags]
+            new_component.add_tags(all_tag_ids)
+            new_component.save()
         control_implementation_statements = component_json.get('control-implementations', None)
-        # catalog = "missing"
         # If there data exists the OSCAL component's control-implementations key
         if control_implementation_statements:
             for control_element in control_implementation_statements:
@@ -1250,6 +1268,7 @@ def component_library_component(request, element_id):
             "impl_smts": impl_smts,
             "is_admin": request.user.is_superuser,
             "enable_experimental_opencontrol": SystemSettings.enable_experimental_opencontrol,
+            "form_source": "component_library"
         }
         return render(request, "components/element_detail_tabs.html", context)
 
@@ -1298,6 +1317,7 @@ def component_library_component(request, element_id):
         "enable_experimental_opencontrol": SystemSettings.enable_experimental_opencontrol,
         "enable_experimental_oscal": SystemSettings.enable_experimental_oscal,
         "opencontrol": opencontrol_string,
+        "form_source": "component_library"
     }
     return render(request, "components/element_detail_tabs.html", context)
 
@@ -1312,6 +1332,8 @@ def api_controls_select(request):
     cxs = []
     for catalog in catalogs_containing_cl_id:
         catalog_key_display = catalog.catalog_key.replace("_", " ")
+        # TODO: get control title effectively from CatalogData
+        # title = "catalog.ctl_title"
         cxs.append({"id": oscal_ctl_id, 'catalog_key_display': catalog_key_display, 'display_text': f"{oscal_ctl_id} - {catalog_key_display} - {cl_id}"})
     status = "success"
     message = "Sending list."
@@ -1998,20 +2020,18 @@ def save_smt(request):
         else:
             new_statement_type_enum = StatementTypeEnum[form_values['statement_type'].upper()]
             # Create new Statement object
+            new_sid_class = form_values['sid_class'].replace(" ","_") # convert displayed catalog name to catalog_key
             statement = Statement(
                 sid=oscalize_control_id(form_values['sid']),
-                sid_class=form_values['sid_class'],
+                sid_class=new_sid_class,
+                source=new_sid_class,
                 body=form_values['body'],
                 pid=form_values['pid'],
                 statement_type=new_statement_type_enum.name,
                 status=form_values['status'],
                 remarks=form_values['remarks'],
             )
             new_statement = True
-            # Convert the human readable catalog name to proper catalog key, if needed
-            # from human readable `NIST SP-800-53 rev4` to `NIST_SP-800-53_rev4`
-            statement.sid_class = statement.sid_class.replace(" ","_")
-
 
         # Updating or saving a new producer_element?
         try:
@@ -2048,8 +2068,7 @@ def save_smt(request):
             except Exception as e:
                 statement_status = "error"
                 statement_msg = "Statement save failed while saving statement prototype. Error reported {}".format(e)
-                return JsonResponse({"status": "error", "message": statement_msg})
-
+                return JsonResponse({"status": statement_status, "message": statement_msg})
         # Retain only prototype statement if statement is created in the component library
         # A statement of type `control_implementation` should only exists if associated a consumer_element.
         # When the statement is created in the component library, no consuming_element will exist.
@@ -2058,13 +2077,13 @@ def save_smt(request):
         # - Skip the associating the statement with the system's root_element because we do not have a system identified
         statement_del_msg = ""
         if "form_source" in form_values and form_values['form_source'] == 'component_library':
-            # Form source is part of form
             # Form received from component library
             from django.core import serializers
             serialized_obj = serializers.serialize('json', [statement, ])
             # Delete statement
+            Statement.objects.filter(pk=statement.id).delete()
             statement.delete()
-            statement_del_msg = "Statement unassociated with System/Consumer Element deleted."
+            statement_del_msg = "Orphaned Control_Implementation Statement deleted."
         else:
             # Associate Statement and System's root_element
             system_id = form_values['system_id']
@@ -2087,13 +2106,13 @@ def save_smt(request):
 
     # Save Statement object
     try:
-        statement.save()
+        if not new_statement:
+            statement.save()
         statement_msg = "Statement saved."
         messages.add_message(request, messages.INFO, f"Statement {smt_id} Saved")
     except Exception as e:
         statement_status = "error"
         statement_msg = "Statement save failed. Error reported {}".format(e)
-
         return JsonResponse({"status": statement_status, "message": statement_msg})
     # Return successful save result to web page's Ajax request
     return JsonResponse(

q-files/vendors/govready/components/OSCAL/cybrary.json

@@ -20,6 +20,18 @@
         "type": "software",
         "title": "Cybrary",
         "description": "Training course",
+        "props": [
+                {
+                  "name": "tag",
+                  "ns": "https://govready.com/ns/oscal",
+                  "value": "sample"
+                },
+                {
+                  "name": "tag",
+                  "ns": "https://govready.com/ns/oscal",
+                  "value": "training"
+                }
+              ],
         "responsible-roles": [
           {
             "role-id": "supplier",

q-files/vendors/govready/components/OSCAL/ilias.json

@@ -20,6 +20,13 @@
         "type": "software",
         "title": "ILIAS",
         "description": "ILIAS Training course",
+        "props": [
+                {
+                  "name": "tag",
+                  "ns": "https://govready.com/ns/oscal",
+                  "value": "sample"
+                }
+              ],
         "responsible-roles": [
           {
             "role-id": "supplier",

siteapp/admin.py

@@ -155,8 +155,9 @@ class SupportAdmin(admin.ModelAdmin):
   fields = ('text', 'email', 'phone', 'url')
 
 class TagAdmin(admin.ModelAdmin):
-  list_display = ('label', 'system_created')
-  fields = ('label', 'system_created')
+  list_display = ('id', 'label', 'system_created')
+  fields = ('id', 'label', 'system_created')
+  readonly_fields = ('id',)
 
 class ProjectAssetAdmin(admin.ModelAdmin):
     list_display = ('uuid', 'asset_type', 'description', 'project', 'default', 'title', 'filename', 'created', 'updated')

siteapp/management/commands/first_run.py

@@ -123,34 +123,6 @@ def handle(self, *args, **options):
             # One or more superusers already exists
             print("\n[INFO] Superuser(s) already exists, not creating default admin superuser. Did you specify 'govready_admins' in 'local/environment.json'? Did you specify an admin or are you connecting to a persistent database?\n")
 
-        # Load the default control catalogs and baselines
-        CATALOG_PATH = os.path.join(os.path.dirname(__file__),'..','..','..','controls','data','catalogs')
-        BASELINE_PATH = os.path.join(os.path.dirname(__file__),'..','..','..','controls','data','baselines')
-
-        # TODO: Check directory exists
-        catalog_files = [file for file in os.listdir(CATALOG_PATH) if file.endswith('.json')]
-        # Load catalog and baseline data into database records from source files if data records do not exist in database
-        for cf in catalog_files:
-            catalog_key = cf.replace("_catalog.json", "")
-            with open(os.path.join(CATALOG_PATH,cf), 'r') as json_file:
-                catalog_json = json.load(json_file)
-            baseline_filename = cf.replace("_catalog.json", "_baselines.json")
-            if os.path.isfile(os.path.join(BASELINE_PATH, baseline_filename)):
-                with open(os.path.join(BASELINE_PATH, baseline_filename), 'r') as json_file:
-                    baselines_json = json.load(json_file)
-            else:
-                baselines_json = {}
-
-            catalog, created = CatalogData.objects.get_or_create(
-                    catalog_key=catalog_key,
-                    catalog_json=catalog_json,
-                    baselines_json=baselines_json
-                )
-            if created:
-                print(f"{catalog_key} record created into database")
-            else:
-                print(f"{catalog_key} record found in database")
-
         # Install default AppSources and compliance apps if no AppSources installed
         if not AppSource.objects.filter(slug="govready-q-files-startpack").exists():
             # Create AppSources that we want.

siteapp/settings.py

@@ -256,9 +256,14 @@ def make_secret_key():
 		'ENGINE': 'django.db.backends.sqlite3',
 		'NAME': local('db.sqlite3'),
 		'CONN_MAX_AGE': 60*5, # 5 min
-		'timeout': 30,
+		'OPTIONS': {
+			'timeout': 30,  # in seconds
+			# see also
+			# https://docs.python.org/3.7/library/sqlite3.html#sqlite3.connect
+		}
 	}
 }
+
 if not environment.get('db'):
 	# Ensure the 'local' directory exists for the default Sqlite
 	# database and then try touching the path to check for write

siteapp/tests.py

@@ -324,6 +324,7 @@ def setUp(self):
                 }
             }
         )
+        var_sleep(1)
         load_modules().handle() # load system modules
 
         AppSource.objects.get_or_create(
@@ -340,6 +341,8 @@ def setUp(self):
         # tests. The Selenium tests require a separate log in via the
         # headless browser.
 
+        var_sleep(2)
+
         # self.user = User.objects.create_superuser(
         self.user = wait_for_sleep_after(lambda: User.objects.get_or_create(
             username="me",

templates/components/element_detail_tabs.html

@@ -388,16 +388,13 @@ <h3>Systems</h3>
         }
 
         function set_catalog_key(display_text, element) {
-          // Determine the catalog from the selected display_text
+          // Set the sid_class field to the catalogkey based on splitting the selected display_text
           // Split text
             displaytext = display_text.split(' - ');
             var first = displaytext.shift(); //or arr[arr.length-1];
-            var last = displaytext.pop(); //or arr[0];
-            dtv = [first, displaytext.join(" - "), last]; // [control_id, title, catalog_key]
-
-          ck = dtv[2]
-
-          $(element).val(ck)
+            var catalog = displaytext.shift(); //or arr[0];
+            dtv = [first, displaytext.join(" - "), catalog]; // [control_id, catalog_key - match string]
+          $(element).val(dtv[2])
         }
 
         function save_smt(smt_panel_num) {

templates/controls/add_smt_form.html

@@ -21,7 +21,7 @@
                                     <option value='' disabled selected>Enter control ID (e.g., ac-2, CM-3(1), 3.1.3)</option>
                                 </select>
                                 <input type="hidden" id="sid_class_panel_num" name="sid_class" prompt="Enter catalog_key" value="{{ catalog_key }}">
-                                <input type="hidden" id="form_source_panel_num" name="form_source" prompt="Form source" value="component_library">
+                                <input type="hidden" id="form_source_panel_num" name="form_source" prompt="Form source" value="{{ form_source }}">
                             </div>
                             <div class="form-group">
                                 <input type="hidden" id="smt_id" name="smt_id" value="">