From eb7e98f1efe40c7cd84c222845b001c47cdb0243 Mon Sep 17 00:00:00 2001 From: Greg Elin Date: Thu, 5 Aug 2021 00:19:23 -0500 Subject: [PATCH 1/7] Move Catalog data into database, faster control select autocomplete Move control catalog data into database so catalogs shipped with GovReady and user added catalogs all accessed in database. Also faster reading in containers. Adjust first_run to load catalogs. Speed up auocomplete by only selecting matching controls. --- CHANGELOG.md | 9 +- controls/admin.py | 6 +- controls/migrations/0057_catalogdata.py | 33 +++++++ controls/models.py | 2 + controls/oscal.py | 86 +++++++------------ controls/views.py | 22 +++-- siteapp/management/commands/first_run.py | 23 ++++- siteapp/settings.py | 2 +- templates/components/element_detail_tabs.html | 2 +- 9 files changed, 113 insertions(+), 72 deletions(-) create mode 100644 controls/migrations/0057_catalogdata.py diff --git a/CHANGELOG.md b/CHANGELOG.md index cbb14968b..3281eef74 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,11 +15,12 @@ v0.9.7-dev (July xx, 2021) * Support datagrid specifying select type cell. * Added new function OSCAL_ssp_export in order to export a system's security plan in OSCAL, this replaces the usual JSON export. Added a several fields of data for OSCAL SSP. * If a component to be imported has a catalog key that is not found in the internal or external catalog list then it will be skipped and logged -* If no statements are created the resulting element/component is deleted -* Component and System Security Plan exports pass OSCAL 1.0.0 schema validation -* Added a proxy for parties and responsible parties for component OSCAL export +* If no statements are created the resulting element/component is deleted. +* Component and System Security Plan exports pass OSCAL 1.0.0 schema validation. +* Added a proxy for parties and responsible parties for component OSCAL export. * Coverage 6.0b1 starts to use a modern hash algorithm (sha256) when fingerprinting for high-security environments, upgrading to avoid this safety fail. -* Validate Component import and SSP with trestle the package +* Validate Component import and SSP with trestle the package. +* Retrieve Catalog data from database instead of file system with new controls.models.CatalogData model. * **Bug fixes** diff --git a/controls/admin.py b/controls/admin.py index fa0d7584b..68d6c2db3 100644 --- a/controls/admin.py +++ b/controls/admin.py @@ -2,6 +2,7 @@ from django.contrib import admin from django.http import HttpResponse from .models import ImportRecord, Statement, Element, ElementControl, ElementRole, System, CommonControlProvider, CommonControl, ElementCommonControl, Poam, Deployment, SystemAssessmentResult +from .oscal import CatalogData from guardian.admin import GuardedModelAdmin from simple_history.admin import SimpleHistoryAdmin @@ -89,6 +90,9 @@ class SystemAssessmentResultAdmin(admin.ModelAdmin): list_display = ('id', 'name', 'system', 'deployment', 'uuid') search_fields = ('id', 'name', 'system', 'deployment', 'uuid') +class CatalogDataAdmin(admin.ModelAdmin): + list_display = ('catalog_key',) + search_fields = ('catalog_key',) admin.site.register(ImportRecord, ImportRecordAdmin) admin.site.register(Statement, StatementAdmin) @@ -102,4 +106,4 @@ class SystemAssessmentResultAdmin(admin.ModelAdmin): admin.site.register(Poam, PoamAdmin) admin.site.register(Deployment, DeploymentAdmin) admin.site.register(SystemAssessmentResult, SystemAssessmentResultAdmin) - +admin.site.register(CatalogData, CatalogDataAdmin) diff --git a/controls/migrations/0057_catalogdata.py b/controls/migrations/0057_catalogdata.py new file mode 100644 index 000000000..198bedaed --- /dev/null +++ b/controls/migrations/0057_catalogdata.py @@ -0,0 +1,33 @@ +# Generated by Django 3.2.4 on 2021-08-05 02:12 + +from django.db import migrations, models +import django.db.models.manager +import jsonfield.fields + + +class Migration(migrations.Migration): + + dependencies = [ + ('controls', '0056_element_oscal_version'), + ] + + operations = [ + migrations.CreateModel( + name='CatalogData', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('catalog_key', models.CharField(help_text='Unique key for catalog', max_length=100, unique=True)), + ('catalog_json', jsonfield.fields.JSONField(blank=True, help_text='JSON object representing the OSCAL-formatted control catalog.', null=True)), + ('created', models.DateTimeField(auto_now_add=True, db_index=True)), + ('updated', models.DateTimeField(auto_now=True, db_index=True)), + ], + options={ + 'abstract': False, + 'base_manager_name': 'prefetch_manager', + }, + managers=[ + ('objects', django.db.models.manager.Manager()), + ('prefetch_manager', django.db.models.manager.Manager()), + ], + ), + ] diff --git a/controls/models.py b/controls/models.py index a4cd0d19c..e8dac5d9f 100644 --- a/controls/models.py +++ b/controls/models.py @@ -1016,3 +1016,5 @@ def __repr__(self): # # For debugging. # return "" % (self.statement, self.id) + + diff --git a/controls/oscal.py b/controls/oscal.py index 271543b16..7243b605e 100644 --- a/controls/oscal.py +++ b/controls/oscal.py @@ -6,56 +6,40 @@ from pathlib import Path import sys +import auto_prefetch +from django.db import models +from django.utils.functional import cached_property +from jsonfield import JSONField + + CATALOG_PATH = os.path.join(os.path.dirname(__file__), 'data', 'catalogs') EXTERNAL_CATALOG_PATH = os.path.join(f"{os.getcwd()}",'local', 'controls', 'data', 'catalogs') -class Catalogs(object): - """Represent list of catalogs""" +class CatalogData(auto_prefetch.Model): + catalog_key = models.CharField(max_length=100, help_text="Unique key for catalog", unique=True, blank=False, null=False) + catalog_json = JSONField(blank=True, null=True, help_text="JSON object representing the OSCAL-formatted control catalog.") + created = models.DateTimeField(auto_now_add=True, db_index=True) + updated = models.DateTimeField(auto_now=True, db_index=True) - # well known catalog identifiers + def __str__(self): + return "'%s id=%d'" % (self.catalog_key, self.id) - NIST_SP_800_53_rev4 = 'NIST_SP-800-53_rev4' - NIST_SP_800_53_rev5 = 'NIST_SP-800-53_rev5' - NIST_SP_800_171_rev1 = 'NIST_SP-800-171_rev1' - CMMC_ver1 = 'CMMC_ver1' + def __repr__(self): + # For debugging. + return "'%s id=%d'" % (self.catalog_key, self.id) + +class Catalogs(object): + """Represent list of catalogs""" def __init__(self): - self.catalog_path = CATALOG_PATH - # self.catalog = None self.catalog_keys = self._list_catalog_keys() self.index = self._build_index() - def extend_external_catalogs(self, catalog_info, extendtype): - """ - Add external catalogs to list of catalogs - """ - os.makedirs(EXTERNAL_CATALOG_PATH, exist_ok=True) - external_catalogs = [file for file in os.listdir(EXTERNAL_CATALOG_PATH) if - file.endswith('.json')] - catalog_info = check_and_extend(catalog_info, external_catalogs, extendtype, "_catalog") - - return catalog_info - - def _list_catalog_files(self): - return self.extend_external_catalogs([ - 'NIST_SP-800-53_rev4_catalog.json', - 'NIST_SP-800-53_rev5_catalog.json', - 'NIST_SP-800-171_rev1_catalog.json', - 'CMMC_ver1_catalog.json' - ], "files") - def _list_catalog_keys(self): - - return self.extend_external_catalogs([ - Catalogs.NIST_SP_800_53_rev4, - Catalogs.NIST_SP_800_53_rev5, - Catalogs.NIST_SP_800_171_rev1, - Catalogs.CMMC_ver1 - ], "keys") + return [item['catalog_key'] for item in CatalogData.objects.order_by().values('catalog_key').distinct()] def _load_catalog_json(self, catalog_key): catalog = Catalog(catalog_key) - #print(catalog_key, catalog._load_catalog_json()) return catalog._load_catalog_json() def _build_index(self): @@ -101,10 +85,10 @@ class Catalog(object): # Create a singleton instance of this class per catalog. GetInstance returns # that singleton instance. Instead of doing - # `cg = Catalog(catalog_key=Catalogs.NIST_SP_800_53_rev4)`, - # do `cg = Catalog.GetInstance(catalog_key=Catalogs.NIST_SP_800_53_rev4')`. + # `cg = Catalog(catalog_key='NIST_SP-800-53_rev4')`, + # do `cg = Catalog.GetInstance(catalog_key='NIST_SP-800-53_rev4')`. @classmethod - def GetInstance(cls, catalog_key=Catalogs.NIST_SP_800_53_rev4, parameter_values=dict()): + def GetInstance(cls, catalog_key='NIST_SP-800-53_rev4', parameter_values=dict()): # Create a new instance of Catalog() the first time for each # catalog key / parameter combo # this method is called. Keep it in memory indefinitely. @@ -125,7 +109,7 @@ def _catalog_instance_key(catalog_key, parameter_values): catalog_instance_key += '_' + str(parameter_values_hash) return catalog_instance_key.replace('-', '_') - def __init__(self, catalog_key=Catalogs.NIST_SP_800_53_rev4, parameter_values=dict()): + def __init__(self, catalog_key='NIST_SP-800-53_rev4', parameter_values=dict()): self.catalog_key = catalog_key self.catalog_key_display = catalog_key.replace("_", " ") self.catalog_path = CATALOG_PATH @@ -156,22 +140,12 @@ def __init__(self, catalog_key=Catalogs.NIST_SP_800_53_rev4, parameter_values=di def _load_catalog_json(self): """Read catalog file - JSON""" - catalog_file = os.path.join(self.catalog_path, self.catalog_file) - catalog_file_external = os.path.join(self.external_catalog_path, self.catalog_file) - # Get catalog file from internal or "external" catalog files - if os.path.isfile(catalog_file): - with open(catalog_file, 'r') as json_file: - data = json.load(json_file) - oscal = data['catalog'] - return oscal - elif os.path.isfile(catalog_file_external): - with open(catalog_file_external, 'r') as json_file: - data = json.load(json_file) - oscal = data['catalog'] - return oscal - else: - # Catalog file doesn't exist - return False + + # Get catalog from database + # TODO: check for DB miss + catalog_record = CatalogData.objects.get(catalog_key=self.catalog_key) + oscal = catalog_record.catalog_json['catalog'] + return oscal def find_dict_by_value(self, search_array, search_key, search_value): """Return the dictionary in an array of dictionaries with a key matching a value""" diff --git a/controls/views.py b/controls/views.py index d66adeb30..a79938239 100644 --- a/controls/views.py +++ b/controls/views.py @@ -1289,20 +1289,26 @@ def component_library_component(request, element_id): def api_controls_select(request): """Return list of controls in json for select2 options from all control catalogs""" + cl_id = request.GET.get('q', None).lower() + print("\n\n*** request", request) # Create array to hold accumulated controls cxs = [] - # Loop through control catalogs + # Search control catalogs in a loop catalogs = Catalogs() for ck in catalogs._list_catalog_keys(): cx = Catalog.GetInstance(catalog_key=ck) - # Get controls - ctl_list = cx.get_flattened_controls_all_as_dict() - # Build objects for rendering Select2 auto complete list from catalog - select_list = [{'id': ctl_list[ctl]['id'], 'title': ctl_list[ctl]['title'], 'class': ctl_list[ctl]['class'], 'catalog_key_display': cx.catalog_key_display, 'display_text': f"{ctl_list[ctl]['label']} - {ctl_list[ctl]['title']} - {cx.catalog_key_display}"} for ctl in ctl_list] - # Extend array of accumuated controls with catalog's control list - cxs.extend(select_list) + ctr = cx.get_control_by_id(cl_id) + if ctr: + cxs.append({'id': ctr['id'], 'title': ctr['title'], 'class': ctr['class'], 'catalog_key_display': cx.catalog_key_display, 'display_text': f"{ctr['id']} - {ctr['title']} - {cx.catalog_key_display} - ({ctr['id'].upper()})"}) + + # # Get controls + # ctl_list = cx.get_flattened_controls_all_as_dict() + # # Build objects for rendering Select2 auto complete list from catalog + # select_list = [{'id': ctl_list[ctl]['id'], 'title': ctl_list[ctl]['title'], 'class': ctl_list[ctl]['class'], 'catalog_key_display': cx.catalog_key_display, 'display_text': f"{ctl_list[ctl]['label']} - {ctl_list[ctl]['title']} - {cx.catalog_key_display}"} for ctl in ctl_list] + # # Extend array of accumuated controls with catalog's control list + # cxs.extend(select_list) # Sort the accummulated list - cxs.sort(key = operator.itemgetter('id', 'catalog_key_display')) + # cxs.sort(key = operator.itemgetter('id', 'catalog_key_display')) status = "success" message = "Sending list." diff --git a/siteapp/management/commands/first_run.py b/siteapp/management/commands/first_run.py index 36828c606..4dceed914 100644 --- a/siteapp/management/commands/first_run.py +++ b/siteapp/management/commands/first_run.py @@ -1,5 +1,7 @@ import sys import os.path +import json + from django.core.management import call_command from django.core.management.base import BaseCommand, CommandError @@ -10,6 +12,7 @@ from guidedmodules.models import AppSource, Module from siteapp.models import User, Organization, Portfolio from controls.models import Element +from controls.oscal import CatalogData from django.contrib.auth.management.commands import createsuperuser import fs, fs.errors @@ -38,8 +41,26 @@ def handle(self, *args, **options): if not Organization.objects.all().exists() and not Organization.objects.filter(name="main").exists(): org = Organization.objects.create(name="main", slug="main") + # Load the default control catalogs + CATALOG_PATH = os.path.join(os.path.dirname(__file__),'..','..','..','controls','data','catalogs') + # TODO: Check directory exists + catalog_files = [file for file in os.listdir(CATALOG_PATH) if file.endswith('.json')] + # conditionally load catalog files + for cf in catalog_files: + catalog_key = cf.replace("_catalog.json", "") + with open(os.path.join(CATALOG_PATH,cf), 'r') as json_file: + catalog_json = json.load(json_file) + catalog, created = CatalogData.objects.get_or_create( + catalog_key=catalog_key, + catalog_json=catalog_json + ) + if created: + print(f"{catalog_key} record created into database") + else: + print(f"{catalog_key} record found in database") + # Install default AppSources and compliance apps if no AppSources installed - if AppSource.objects.all().exists(): + if not AppSource.objects.filter(slug="govready-q-files-startpack").exists(): # Create AppSources that we want. if os.path.exists("/mnt/q-files-host"): # For our docker image. diff --git a/siteapp/settings.py b/siteapp/settings.py index 122034db6..58c2265ad 100644 --- a/siteapp/settings.py +++ b/siteapp/settings.py @@ -331,7 +331,7 @@ def make_secret_key(): } SILENCED_SYSTEM_CHECKS = [] -DATA_UPLOAD_MAX_MEMORY_SIZE = 2621440 +DATA_UPLOAD_MAX_MEMORY_SIZE = 8242880 # Settings that have normal values based on the primary app # (the app this file resides in). diff --git a/templates/components/element_detail_tabs.html b/templates/components/element_detail_tabs.html index b1ebfbf3e..1a8510eb0 100644 --- a/templates/components/element_detail_tabs.html +++ b/templates/components/element_detail_tabs.html @@ -342,7 +342,7 @@

Systems

ajax: { url: '{% url 'api_controls_select' %}', dataType: 'json', - delay: 250, + delay: 150, data: function(params) { return { q: params.term // search term From 3234ca500972499b46accb54e46c6d4490ee3688 Mon Sep 17 00:00:00 2001 From: Greg Elin Date: Thu, 5 Aug 2021 07:14:42 -0500 Subject: [PATCH 2/7] Fix tests: add catalogs to db in setup --- controls/oscal.py | 9 ++- controls/tests.py | 91 ++++++------------------ controls/views.py | 17 ++--- siteapp/management/commands/first_run.py | 1 - siteapp/tests.py | 19 +++++ 5 files changed, 50 insertions(+), 87 deletions(-) diff --git a/controls/oscal.py b/controls/oscal.py index 7243b605e..738272fe4 100644 --- a/controls/oscal.py +++ b/controls/oscal.py @@ -31,6 +31,12 @@ def __repr__(self): class Catalogs(object): """Represent list of catalogs""" + # well known catalog identifiers + NIST_SP_800_53_rev4 = 'NIST_SP-800-53_rev4' + NIST_SP_800_53_rev5 = 'NIST_SP-800-53_rev5' + NIST_SP_800_171_rev1 = 'NIST_SP-800-171_rev1' + CMMC_ver1 = 'CMMC_ver1' + def __init__(self): self.catalog_keys = self._list_catalog_keys() self.index = self._build_index() @@ -60,8 +66,7 @@ def list_catalogs(self): """ List catalog objects """ - return [Catalog(key) for key in Catalogs()._list_catalog_keys()] - + return [Catalog.GetInstance(catalog_key=key) for key in Catalogs()._list_catalog_keys()] def uhash(obj): """Return a positive hash code""" diff --git a/controls/tests.py b/controls/tests.py index ae5cf3b9d..d41685ef7 100644 --- a/controls/tests.py +++ b/controls/tests.py @@ -29,7 +29,7 @@ from system_settings.models import SystemSettings from controls.models import * from controls.enums.statements import StatementTypeEnum -from controls.oscal import Catalogs, Catalog, EXTERNAL_CATALOG_PATH +from controls.oscal import Catalogs, Catalog, CatalogData from siteapp.models import User, Project, Portfolio from system_settings.models import SystemSettings @@ -38,10 +38,10 @@ ##################################################################### - # Control Tests -class Oscal80053Tests(TestCase): +class Oscal80053Tests(SeleniumTest): + # Test def test_catalog_load_control(self): cg = Catalog.GetInstance(Catalogs.NIST_SP_800_53_rev4) @@ -122,6 +122,7 @@ def test_statement_id_from_control(self): ##################################################################### class ControlUITests(SeleniumTest): + def test_homepage(self): self.browser.get(self.url("/controls/")) @@ -136,76 +137,11 @@ def test_control_enhancement_lookup(self): self.assertInNodeText("Automated Audit Actions", "#control-heading") def test_catalog_list(self): - """ - Check the catalog listing method has the 3 default catalogs - """ - - os.makedirs(EXTERNAL_CATALOG_PATH, exist_ok=True) catalog_list = Catalogs().list_catalogs() self.assertEqual(len(catalog_list), 4) - def test_extend_external_catalogs(self): - """ - Extending catalog file and key list - """ - os.makedirs(EXTERNAL_CATALOG_PATH, exist_ok=True) - with tempfile.TemporaryFile() as d: - temp_file_name = os.path.join(EXTERNAL_CATALOG_PATH, f'{d.name}_revtest_catalog.json') - - # finding fixture data and dumping in the temp file - test_catalog = os.getcwd() + "/fixtures/test_catalog.json" - with open(test_catalog, 'r') as json_file: - catalog_data = json.load(json_file) - with open(temp_file_name, 'w') as cred: - json.dump(catalog_data, cred) - - extended_files = Catalogs.extend_external_catalogs(self, [ - 'NIST_SP-800-53_rev4_catalog.json', - 'NIST_SP-800-53_rev5_catalog.json', - 'NIST_SP-800-171_rev1_catalog.json' - ], "files") - - self.assertEqual(len(extended_files), 4) - - extended_keys = Catalogs.extend_external_catalogs(self, [ - Catalogs.NIST_SP_800_53_rev4, - Catalogs.NIST_SP_800_53_rev5, - Catalogs.NIST_SP_800_171_rev1 - ], "keys") - self.assertEqual(len(extended_keys), 4) - # Delete temp file - os.remove(temp_file_name) - - def test_extend_external_baseline(self): - """ - Extending baseline file and key list - """ - os.makedirs(EXTERNAL_BASELINE_PATH, exist_ok=True) - with tempfile.TemporaryFile() as d: - temp_file_name = os.path.join(EXTERNAL_BASELINE_PATH, f'{d.name}_revtest_baseline.json') - - # finding fixture data and dumping in the temp file - test_baseline = os.getcwd() + "/fixtures/test_baselines.json" - with open(test_baseline, 'r') as json_file: - baseline_data = json.load(json_file) - with open(temp_file_name, 'w') as cred: - json.dump(baseline_data, cred) - - extended_files = Baselines.extend_external_baselines(self, [ - 'NIST_SP-800-53_rev4_baselines.json', - 'NIST_SP-800-171_rev1_baselines.json' - ], "files") - - self.assertEqual(len(extended_files), 3) - - extended_keys = Baselines.extend_external_baselines(self, [ - 'NIST_SP-800-53_rev4', - 'NIST_SP-800-171_rev1' - ], "keys") - - self.assertEqual(len(extended_keys), 3) - # Delete temp file - os.remove(temp_file_name) + # TODO: Create tests for uploading catalog + ##################################################################### class ComponentUITests(OrganizationSiteFunctionalTests): @@ -215,6 +151,19 @@ class ComponentUITests(OrganizationSiteFunctionalTests): def setUp(self): super().setUp() + # Add catalogs to database + CATALOG_PATH = os.path.join(os.path.dirname(__file__),'..','controls','data','catalogs') + catalog_files = [file for file in os.listdir(CATALOG_PATH) if file.endswith('.json')] + for cf in catalog_files: + catalog_key = cf.replace("_catalog.json", "") + with open(os.path.join(CATALOG_PATH,cf), 'r') as json_file: + catalog_json = json.load(json_file) + # cls.foo = Foo.objects.create(bar="Test") + catalog, created = CatalogData.objects.get_or_create( + catalog_key=catalog_key, + catalog_json=catalog_json + ) + self.json_download = \ self.download_path / PurePath(slugify(self.component_name)).with_suffix(".json") # print("********* self.json_download", self.json_download) @@ -791,7 +740,7 @@ def test_element_create(self): # poam.delete() # self.assertTrue(poam.uuid is None) -class OrgParamTests(TestCase): +class OrgParamTests(SeleniumTest): """Class for OrgParam Unit Tests""" def test_org_params(self): diff --git a/controls/views.py b/controls/views.py index a79938239..2b928ca68 100644 --- a/controls/views.py +++ b/controls/views.py @@ -1290,26 +1290,17 @@ def api_controls_select(request): """Return list of controls in json for select2 options from all control catalogs""" cl_id = request.GET.get('q', None).lower() - print("\n\n*** request", request) - # Create array to hold accumulated controls + # Search control catalogs in a loop and add results to an array cxs = [] - # Search control catalogs in a loop catalogs = Catalogs() for ck in catalogs._list_catalog_keys(): cx = Catalog.GetInstance(catalog_key=ck) ctr = cx.get_control_by_id(cl_id) + # TODO: Better representation of control ids for case-insensitive searching insteading of listing ids in both cases + # TODO: OSCALizing control id? if ctr: cxs.append({'id': ctr['id'], 'title': ctr['title'], 'class': ctr['class'], 'catalog_key_display': cx.catalog_key_display, 'display_text': f"{ctr['id']} - {ctr['title']} - {cx.catalog_key_display} - ({ctr['id'].upper()})"}) - - # # Get controls - # ctl_list = cx.get_flattened_controls_all_as_dict() - # # Build objects for rendering Select2 auto complete list from catalog - # select_list = [{'id': ctl_list[ctl]['id'], 'title': ctl_list[ctl]['title'], 'class': ctl_list[ctl]['class'], 'catalog_key_display': cx.catalog_key_display, 'display_text': f"{ctl_list[ctl]['label']} - {ctl_list[ctl]['title']} - {cx.catalog_key_display}"} for ctl in ctl_list] - # # Extend array of accumuated controls with catalog's control list - # cxs.extend(select_list) - # Sort the accummulated list - # cxs.sort(key = operator.itemgetter('id', 'catalog_key_display')) - + cxs.sort(key = operator.itemgetter('id', 'catalog_key_display')) status = "success" message = "Sending list." return JsonResponse( {"status": status, "message": message, "data": {"controls": cxs} }) diff --git a/siteapp/management/commands/first_run.py b/siteapp/management/commands/first_run.py index 4dceed914..ffe9ca520 100644 --- a/siteapp/management/commands/first_run.py +++ b/siteapp/management/commands/first_run.py @@ -167,7 +167,6 @@ def handle(self, *args, **options): username )) - # Create the first user. if not User.objects.filter(is_superuser=True).exists(): if not options['non_interactive']: diff --git a/siteapp/tests.py b/siteapp/tests.py index 56dec9086..0a4e9b553 100644 --- a/siteapp/tests.py +++ b/siteapp/tests.py @@ -17,6 +17,7 @@ import tempfile import time import unittest +import json from django.contrib.auth import authenticate from django.test.client import RequestFactory @@ -35,6 +36,7 @@ from siteapp.models import (Organization, Portfolio, Project, ProjectMembership, User) from controls.models import Statement, Element, System +from controls.oscal import CatalogData, Catalogs, Catalog from siteapp.settings import HEADLESS, DOS from siteapp.views import project_edit from tools.utils.linux_to_dos import convert_w @@ -140,6 +142,23 @@ def tearDownClass(cls): def setUp(self): # clear the browser's cookies before each test self.browser.delete_all_cookies() + # Add catalogs to database + CATALOG_PATH = os.path.join(os.path.dirname(__file__),'..','controls','data','catalogs') + catalog_files = [file for file in os.listdir(CATALOG_PATH) if file.endswith('.json')] + for cf in catalog_files: + catalog_key = cf.replace("_catalog.json", "") + with open(os.path.join(CATALOG_PATH,cf), 'r') as json_file: + catalog_json = json.load(json_file) + # cls.foo = Foo.objects.create(bar="Test") + catalog, created = CatalogData.objects.get_or_create( + catalog_key=catalog_key, + catalog_json=catalog_json + ) + # if created: + # print(f"{catalog_key} record created into database") + # else: + # print(f"{catalog_key} record found in database") + def navigateToPage(self, path): self.browser.get(self.url(path)) From 72ec070b130e93d14ec3f19bd96912375d67bd4d Mon Sep 17 00:00:00 2001 From: Greg Elin Date: Thu, 5 Aug 2021 19:19:04 -0500 Subject: [PATCH 3/7] Read baselines from database --- .../0058_catalogdata_baselines_json.py | 19 ++++++ controls/models.py | 62 +++---------------- controls/oscal.py | 16 +---- controls/tests.py | 2 +- siteapp/management/commands/first_run.py | 16 ++++- siteapp/settings.py | 2 +- siteapp/tests.py | 12 +++- 7 files changed, 55 insertions(+), 74 deletions(-) create mode 100644 controls/migrations/0058_catalogdata_baselines_json.py diff --git a/controls/migrations/0058_catalogdata_baselines_json.py b/controls/migrations/0058_catalogdata_baselines_json.py new file mode 100644 index 000000000..31145bb82 --- /dev/null +++ b/controls/migrations/0058_catalogdata_baselines_json.py @@ -0,0 +1,19 @@ +# Generated by Django 3.2.4 on 2021-08-05 15:19 + +from django.db import migrations +import jsonfield.fields + + +class Migration(migrations.Migration): + + dependencies = [ + ('controls', '0057_catalogdata'), + ] + + operations = [ + migrations.AddField( + model_name='catalogdata', + name='baselines_json', + field=jsonfield.fields.JSONField(blank=True, help_text='JSON object representing the baselines for the catalog.', null=True), + ), + ] diff --git a/controls/models.py b/controls/models.py index e8dac5d9f..5b25fb7e1 100644 --- a/controls/models.py +++ b/controls/models.py @@ -15,7 +15,7 @@ from controls.enums.components import ComponentTypeEnum, ComponentStateEnum from siteapp.model_mixins.tags import TagModelMixin from controls.enums.statements import StatementTypeEnum -from controls.oscal import Catalogs, Catalog, check_and_extend +from controls.oscal import Catalogs, Catalog, CatalogData import uuid import tools.diff_match_patch.python3 as dmp_module from copy import deepcopy @@ -783,9 +783,7 @@ class Baselines (object): def __init__(self): self.file_path = BASELINE_PATH - self.external_file_path = EXTERNAL_BASELINE_PATH self.baselines_keys = self._list_keys() - # self.index = self._build_index() # Usage # from controls.models import Baselines @@ -796,44 +794,18 @@ def __init__(self): # # Returns ['ac-1', 'ac-2', 'ac-2.1', 'ac-2.2', ...] # bs.get_baseline_controls('NIST_SP-800-53_rev4', 'moderate') - def _list_files(self): - return self.extend_external_baselines([ - 'NIST_SP-800-53_rev4_baselines.json', - # 'NIST_SP-800-53_rev5_baselines.json', - 'NIST_SP-800-171_rev1_baselines.json', - 'CMMC_ver1_baselines.json' - ], "files") - - def _list_keys(self): - return self.extend_external_baselines([ - 'NIST_SP-800-53_rev4', - # 'NIST_SP-800-53_rev5', - 'NIST_SP-800-171_rev1', - 'CMMC_ver1' - ], "keys") - + # TODO: only return keys for records that have baselines? + return [item['catalog_key'] for item in CatalogData.objects.order_by().values('catalog_key').distinct()] def _load_json(self, baselines_key): """Read baseline file - JSON""" - # TODO Escape baselines_key - self.data_file = baselines_key + "_baselines.json" - data_file = os.path.join(self.file_path, self.data_file) - # Does file exist? - if not os.path.isfile(data_file): - # Check if there any external oscal baseline files - try: - data_file = os.path.join(self.external_file_path, self.data_file) - except: - print("ERROR: {} does not exist".format(data_file)) - return False - # Load file as json - try: - with open(data_file, 'r') as json_file: - data = json.load(json_file) - return data - except: - print("ERROR: {} could not be read or could not be read as json".format(data_file)) + + catalog_record = CatalogData.objects.get(catalog_key=baselines_key) + baselines = catalog_record.baselines_json + if baselines: + return baselines + else: return False def get_baseline_controls(self, baselines_key, baseline_name): @@ -849,22 +821,6 @@ def get_baseline_controls(self, baselines_key, baseline_name): print("Requested baseline name not found in baselines_key data file") return False - @property - def body(self): - return self.legacy_imp_smt - - - def extend_external_baselines(self, baseline_info, extendtype): - """ - Add external baselines to list of baselines - """ - os.makedirs(EXTERNAL_BASELINE_PATH, exist_ok=True) - external_baselines = [file for file in os.listdir(EXTERNAL_BASELINE_PATH) if - file.endswith('.json')] - - baseline_info = check_and_extend(baseline_info, external_baselines, extendtype, "_baselines") - return baseline_info - class OrgParams(object): """ Represent list of organizational defined parameters. Temporary diff --git a/controls/oscal.py b/controls/oscal.py index 738272fe4..1065c609f 100644 --- a/controls/oscal.py +++ b/controls/oscal.py @@ -13,11 +13,12 @@ CATALOG_PATH = os.path.join(os.path.dirname(__file__), 'data', 'catalogs') -EXTERNAL_CATALOG_PATH = os.path.join(f"{os.getcwd()}",'local', 'controls', 'data', 'catalogs') +BASELINE_PATH = os.path.join(os.path.dirname(__file__),'data','baselines') class CatalogData(auto_prefetch.Model): catalog_key = models.CharField(max_length=100, help_text="Unique key for catalog", unique=True, blank=False, null=False) catalog_json = JSONField(blank=True, null=True, help_text="JSON object representing the OSCAL-formatted control catalog.") + baselines_json = JSONField(blank=True, null=True, help_text="JSON object representing the baselines for the catalog.") created = models.DateTimeField(auto_now_add=True, db_index=True) updated = models.DateTimeField(auto_now=True, db_index=True) @@ -73,18 +74,6 @@ def uhash(obj): h = hash(obj) return h + sys.maxsize + 1 -def check_and_extend(values, external_values, extendtype, splitter): - """ - Modularize value to extend - """ - if extendtype == "keys": - keys = [key.split(f'{splitter}.json')[0] for key in external_values] - values.extend(keys) - elif extendtype == "files": - files = [file for file in external_values] - values.extend(files) - return values - class Catalog(object): """Represent a catalog""" @@ -118,7 +107,6 @@ def __init__(self, catalog_key='NIST_SP-800-53_rev4', parameter_values=dict()): self.catalog_key = catalog_key self.catalog_key_display = catalog_key.replace("_", " ") self.catalog_path = CATALOG_PATH - self.external_catalog_path = EXTERNAL_CATALOG_PATH self.catalog_file = catalog_key + "_catalog.json" try: self.oscal = self._load_catalog_json() diff --git a/controls/tests.py b/controls/tests.py index d41685ef7..56bee4c1d 100644 --- a/controls/tests.py +++ b/controls/tests.py @@ -558,7 +558,7 @@ def test_component_type_state(self): self.assertTrue(e2.component_type == "hardware") self.assertTrue(e2.component_state == "disposition") -class ElementControlUnitTests(TestCase): +class ElementControlUnitTests(SeleniumTest): def test_assign_baseline(self): diff --git a/siteapp/management/commands/first_run.py b/siteapp/management/commands/first_run.py index ffe9ca520..e3601dcd6 100644 --- a/siteapp/management/commands/first_run.py +++ b/siteapp/management/commands/first_run.py @@ -41,18 +41,28 @@ def handle(self, *args, **options): if not Organization.objects.all().exists() and not Organization.objects.filter(name="main").exists(): org = Organization.objects.create(name="main", slug="main") - # Load the default control catalogs + # Load the default control catalogs and baselines CATALOG_PATH = os.path.join(os.path.dirname(__file__),'..','..','..','controls','data','catalogs') + BASELINE_PATH = os.path.join(os.path.dirname(__file__),'..','..','..','controls','data','baselines') + # TODO: Check directory exists catalog_files = [file for file in os.listdir(CATALOG_PATH) if file.endswith('.json')] - # conditionally load catalog files + # Load catalog and baseline data into database records from source files if data records do not exist in database for cf in catalog_files: catalog_key = cf.replace("_catalog.json", "") with open(os.path.join(CATALOG_PATH,cf), 'r') as json_file: catalog_json = json.load(json_file) + baseline_filename = cf.replace("_catalog.json", "_baselines.json") + if os.path.isfile(os.path.join(BASELINE_PATH, baseline_filename)): + with open(os.path.join(BASELINE_PATH, baseline_filename), 'r') as json_file: + baselines_json = json.load(json_file) + else: + baselines_json = {} + catalog, created = CatalogData.objects.get_or_create( catalog_key=catalog_key, - catalog_json=catalog_json + catalog_json=catalog_json, + baselines_json=baselines_json ) if created: print(f"{catalog_key} record created into database") diff --git a/siteapp/settings.py b/siteapp/settings.py index 58c2265ad..7288ab1bf 100644 --- a/siteapp/settings.py +++ b/siteapp/settings.py @@ -331,7 +331,7 @@ def make_secret_key(): } SILENCED_SYSTEM_CHECKS = [] -DATA_UPLOAD_MAX_MEMORY_SIZE = 8242880 +DATA_UPLOAD_MAX_MEMORY_SIZE = 10242880 # Settings that have normal values based on the primary app # (the app this file resides in). diff --git a/siteapp/tests.py b/siteapp/tests.py index 0a4e9b553..79b93ea5b 100644 --- a/siteapp/tests.py +++ b/siteapp/tests.py @@ -144,15 +144,23 @@ def setUp(self): self.browser.delete_all_cookies() # Add catalogs to database CATALOG_PATH = os.path.join(os.path.dirname(__file__),'..','controls','data','catalogs') + BASELINE_PATH = os.path.join(os.path.dirname(__file__),'..','controls','data','baselines') catalog_files = [file for file in os.listdir(CATALOG_PATH) if file.endswith('.json')] for cf in catalog_files: catalog_key = cf.replace("_catalog.json", "") with open(os.path.join(CATALOG_PATH,cf), 'r') as json_file: catalog_json = json.load(json_file) - # cls.foo = Foo.objects.create(bar="Test") + baseline_filename = cf.replace("_catalog.json", "_baselines.json") + if os.path.isfile(os.path.join(BASELINE_PATH, baseline_filename)): + with open(os.path.join(BASELINE_PATH, baseline_filename), 'r') as json_file: + baselines_json = json.load(json_file) + else: + baselines_json = {} + catalog, created = CatalogData.objects.get_or_create( catalog_key=catalog_key, - catalog_json=catalog_json + catalog_json=catalog_json, + baselines_json=baselines_json ) # if created: # print(f"{catalog_key} record created into database") From 2cb6189771e10a9378202c79b5ebddd2070aa9f2 Mon Sep 17 00:00:00 2001 From: Greg Elin Date: Sun, 8 Aug 2021 10:48:19 -0500 Subject: [PATCH 4/7] Sync with 0.9.7 --- CHANGELOG.md | 8 ++- VERSION | 2 +- controls/models.py | 2 +- controls/oscal.py | 1 + controls/tests.py | 130 ++++++++++++++++++++++----------------------- 5 files changed, 75 insertions(+), 68 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8bb6a0865..b49283d34 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,13 @@ GovReady-Q Release Notes ======================== +v0.9.8-dev (August xx, 2021) +---------------------------- + +**Developer changes** + +* Retrieve Catalog data from database instead of file system with new controls.models.CatalogData model. + v0.9.7 (August 06, 2021) ------------------------ @@ -20,7 +27,6 @@ v0.9.7 (August 06, 2021) * Added a proxy for parties and responsible parties for component OSCAL export. * Coverage 6.0b1 starts to use a modern hash algorithm (sha256) when fingerprinting for high-security environments, upgrading to avoid this safety fail. * Validate Component import and SSP with trestle the package. -* Retrieve Catalog data from database instead of file system with new controls.models.CatalogData model. * **Bug fixes** diff --git a/VERSION b/VERSION index b2cf0d104..6257c9ff0 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -v0.9.7 +v0.9.8-dev diff --git a/controls/models.py b/controls/models.py index 5b25fb7e1..f4f4a61a6 100644 --- a/controls/models.py +++ b/controls/models.py @@ -22,7 +22,6 @@ from django.db import transaction BASELINE_PATH = os.path.join(os.path.dirname(__file__),'data','baselines') -EXTERNAL_BASELINE_PATH = os.path.join(f"{os.getcwd()}",'local', 'controls', 'data', 'baselines') ORGPARAM_PATH = os.path.join(os.path.dirname(__file__),'data','org_defined_parameters') class ImportRecord(models.Model): @@ -797,6 +796,7 @@ def __init__(self): def _list_keys(self): # TODO: only return keys for records that have baselines? return [item['catalog_key'] for item in CatalogData.objects.order_by().values('catalog_key').distinct()] + # return CatalogData.objects.order_by('catalog_key').values_list('catalog_key', flat=True).distinct() def _load_json(self, baselines_key): """Read baseline file - JSON""" diff --git a/controls/oscal.py b/controls/oscal.py index f9305b7d5..feb885350 100644 --- a/controls/oscal.py +++ b/controls/oscal.py @@ -44,6 +44,7 @@ def __init__(self): def _list_catalog_keys(self): return [item['catalog_key'] for item in CatalogData.objects.order_by().values('catalog_key').distinct()] + # return CatalogData.objects.order_by('catalog_key').values_list('catalog_key', flat=True).distinct() def _load_catalog_json(self, catalog_key): catalog = Catalog(catalog_key) diff --git a/controls/tests.py b/controls/tests.py index 0aff5a12a..9658b785d 100644 --- a/controls/tests.py +++ b/controls/tests.py @@ -29,7 +29,7 @@ from system_settings.models import SystemSettings from controls.models import * from controls.enums.statements import StatementTypeEnum -from controls.oscal import Catalogs, Catalog, EXTERNAL_CATALOG_PATH, de_oscalize_control +from controls.oscal import Catalogs, Catalog, de_oscalize_control from siteapp.models import User, Project, Portfolio from system_settings.models import SystemSettings @@ -41,7 +41,7 @@ # Control Tests -class Oscal80053Tests(TestCase): +class Oscal80053Tests(SeleniumTest): # Test def test_catalog_load_control(self): cg = Catalog.GetInstance(Catalogs.NIST_SP_800_53_rev4) @@ -140,72 +140,72 @@ def test_catalog_list(self): Check the catalog listing method has the 3 default catalogs """ - os.makedirs(EXTERNAL_CATALOG_PATH, exist_ok=True) + # os.makedirs(EXTERNAL_CATALOG_PATH, exist_ok=True) catalog_list = Catalogs().list_catalogs() self.assertEqual(len(catalog_list), 4) - def test_extend_external_catalogs(self): - """ - Extending catalog file and key list - """ - os.makedirs(EXTERNAL_CATALOG_PATH, exist_ok=True) - with tempfile.TemporaryFile() as d: - temp_file_name = os.path.join(EXTERNAL_CATALOG_PATH, f'{d.name}_revtest_catalog.json') - - # finding fixture data and dumping in the temp file - test_catalog = os.getcwd() + "/fixtures/test_catalog.json" - with open(test_catalog, 'r') as json_file: - catalog_data = json.load(json_file) - with open(temp_file_name, 'w') as cred: - json.dump(catalog_data, cred) - - extended_files = Catalogs.extend_external_catalogs(self, [ - 'NIST_SP-800-53_rev4_catalog.json', - 'NIST_SP-800-53_rev5_catalog.json', - 'NIST_SP-800-171_rev1_catalog.json' - ], "files") - - self.assertEqual(len(extended_files), 4) - - extended_keys = Catalogs.extend_external_catalogs(self, [ - Catalogs.NIST_SP_800_53_rev4, - Catalogs.NIST_SP_800_53_rev5, - Catalogs.NIST_SP_800_171_rev1 - ], "keys") - self.assertEqual(len(extended_keys), 4) - # Delete temp file - os.remove(temp_file_name) - - def test_extend_external_baseline(self): - """ - Extending baseline file and key list - """ - os.makedirs(EXTERNAL_BASELINE_PATH, exist_ok=True) - with tempfile.TemporaryFile() as d: - temp_file_name = os.path.join(EXTERNAL_BASELINE_PATH, f'{d.name}_revtest_baseline.json') - - # finding fixture data and dumping in the temp file - test_baseline = os.getcwd() + "/fixtures/test_baselines.json" - with open(test_baseline, 'r') as json_file: - baseline_data = json.load(json_file) - with open(temp_file_name, 'w') as cred: - json.dump(baseline_data, cred) - - extended_files = Baselines.extend_external_baselines(self, [ - 'NIST_SP-800-53_rev4_baselines.json', - 'NIST_SP-800-171_rev1_baselines.json' - ], "files") - - self.assertEqual(len(extended_files), 3) - - extended_keys = Baselines.extend_external_baselines(self, [ - 'NIST_SP-800-53_rev4', - 'NIST_SP-800-171_rev1' - ], "keys") - - self.assertEqual(len(extended_keys), 3) - # Delete temp file - os.remove(temp_file_name) + # def test_extend_external_catalogs(self): + # """ + # Extending catalog file and key list + # """ + # os.makedirs(EXTERNAL_CATALOG_PATH, exist_ok=True) + # with tempfile.TemporaryFile() as d: + # temp_file_name = os.path.join(EXTERNAL_CATALOG_PATH, f'{d.name}_revtest_catalog.json') + + # # finding fixture data and dumping in the temp file + # test_catalog = os.getcwd() + "/fixtures/test_catalog.json" + # with open(test_catalog, 'r') as json_file: + # catalog_data = json.load(json_file) + # with open(temp_file_name, 'w') as cred: + # json.dump(catalog_data, cred) + + # extended_files = Catalogs.extend_external_catalogs(self, [ + # 'NIST_SP-800-53_rev4_catalog.json', + # 'NIST_SP-800-53_rev5_catalog.json', + # 'NIST_SP-800-171_rev1_catalog.json' + # ], "files") + + # self.assertEqual(len(extended_files), 4) + + # extended_keys = Catalogs.extend_external_catalogs(self, [ + # Catalogs.NIST_SP_800_53_rev4, + # Catalogs.NIST_SP_800_53_rev5, + # Catalogs.NIST_SP_800_171_rev1 + # ], "keys") + # self.assertEqual(len(extended_keys), 4) + # # Delete temp file + # os.remove(temp_file_name) +# + # def test_extend_external_baseline(self): + # """ + # Extending baseline file and key list + # """ + # os.makedirs(EXTERNAL_BASELINE_PATH, exist_ok=True) + # with tempfile.TemporaryFile() as d: + # temp_file_name = os.path.join(EXTERNAL_BASELINE_PATH, f'{d.name}_revtest_baseline.json') + + # # finding fixture data and dumping in the temp file + # test_baseline = os.getcwd() + "/fixtures/test_baselines.json" + # with open(test_baseline, 'r') as json_file: + # baseline_data = json.load(json_file) + # with open(temp_file_name, 'w') as cred: + # json.dump(baseline_data, cred) + + # extended_files = Baselines.extend_external_baselines(self, [ + # 'NIST_SP-800-53_rev4_baselines.json', + # 'NIST_SP-800-171_rev1_baselines.json' + # ], "files") + + # self.assertEqual(len(extended_files), 3) + + # extended_keys = Baselines.extend_external_baselines(self, [ + # 'NIST_SP-800-53_rev4', + # 'NIST_SP-800-171_rev1' + # ], "keys") + + # self.assertEqual(len(extended_keys), 3) + # # Delete temp file + # os.remove(temp_file_name) ##################################################################### class ComponentUITests(OrganizationSiteFunctionalTests): From dec583bca12a8b48a8adb81e1fe303eb3f34a3f7 Mon Sep 17 00:00:00 2001 From: Greg Elin Date: Sun, 8 Aug 2021 11:24:04 -0500 Subject: [PATCH 5/7] Fix tests --- controls/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/controls/tests.py b/controls/tests.py index 9658b785d..6f00b228d 100644 --- a/controls/tests.py +++ b/controls/tests.py @@ -609,7 +609,7 @@ def test_component_type_state(self): self.assertTrue(e2.component_type == "hardware") self.assertTrue(e2.component_state == "disposition") -class ElementControlUnitTests(TestCase): +class ElementControlUnitTests(SeleniumTest): def test_assign_baseline(self): From 53660a19d95fedcb7dcbd57d407dc66c1b0b9a08 Mon Sep 17 00:00:00 2001 From: Greg Elin Date: Sun, 8 Aug 2021 11:36:36 -0500 Subject: [PATCH 6/7] Fix tests 2 --- controls/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/controls/tests.py b/controls/tests.py index 6f00b228d..4f422075e 100644 --- a/controls/tests.py +++ b/controls/tests.py @@ -791,7 +791,7 @@ def test_element_create(self): # poam.delete() # self.assertTrue(poam.uuid is None) -class OrgParamTests(TestCase): +class OrgParamTests(SeleniumTest): """Class for OrgParam Unit Tests""" def test_org_params(self): From 22f0b937b89e56f21ddfe2253bbfc155cd52c20f Mon Sep 17 00:00:00 2001 From: Greg Elin Date: Sun, 8 Aug 2021 12:09:14 -0500 Subject: [PATCH 7/7] Use better DB query instead of closure --- controls/models.py | 3 +-- controls/oscal.py | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/controls/models.py b/controls/models.py index f4f4a61a6..12bcc812f 100644 --- a/controls/models.py +++ b/controls/models.py @@ -795,8 +795,7 @@ def __init__(self): def _list_keys(self): # TODO: only return keys for records that have baselines? - return [item['catalog_key'] for item in CatalogData.objects.order_by().values('catalog_key').distinct()] - # return CatalogData.objects.order_by('catalog_key').values_list('catalog_key', flat=True).distinct() + return list(CatalogData.objects.order_by('catalog_key').values_list('catalog_key', flat=True).distinct()) def _load_json(self, baselines_key): """Read baseline file - JSON""" diff --git a/controls/oscal.py b/controls/oscal.py index feb885350..a769286a8 100644 --- a/controls/oscal.py +++ b/controls/oscal.py @@ -43,8 +43,7 @@ def __init__(self): self.index = self._build_index() def _list_catalog_keys(self): - return [item['catalog_key'] for item in CatalogData.objects.order_by().values('catalog_key').distinct()] - # return CatalogData.objects.order_by('catalog_key').values_list('catalog_key', flat=True).distinct() + return list(CatalogData.objects.order_by('catalog_key').values_list('catalog_key', flat=True).distinct()) def _load_catalog_json(self, catalog_key): catalog = Catalog(catalog_key)