From 5a5513a7811a8f722fb91ee4b03ab2cdd7395bc2 Mon Sep 17 00:00:00 2001 From: Greg Elin Date: Sun, 8 Aug 2021 18:08:31 +0000 Subject: [PATCH] Move Catalog data into database, faster control select autocomplete (#1673) * Move Catalog data into database, faster control select autocomplete Move control catalog data into database so catalogs shipped with GovReady and user added catalogs all accessed in database. Also faster reading in containers. Adjust first_run to load catalogs. Speed up auocomplete by only selecting matching controls. * Fix tests: add catalogs to db in setup * Read baselines from database * Sync with 0.9.7 * Fix tests * Fix tests 2 * Use better DB query instead of closure Co-authored-by: Greg Elin --- CHANGELOG.md | 1 + controls/admin.py | 6 +- controls/migrations/0057_catalogdata.py | 33 +++++ .../0058_catalogdata_baselines_json.py | 19 +++ controls/models.py | 65 ++------- controls/oscal.py | 99 +++++-------- controls/tests.py | 134 +++++++++--------- controls/views.py | 17 +-- siteapp/management/commands/first_run.py | 34 ++++- siteapp/settings.py | 2 +- siteapp/tests.py | 27 ++++ templates/components/element_detail_tabs.html | 2 +- 12 files changed, 237 insertions(+), 202 deletions(-) create mode 100644 controls/migrations/0057_catalogdata.py create mode 100644 controls/migrations/0058_catalogdata_baselines_json.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 475e569db..b30f700d0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ v0.9.8-dev (August xx, 2021) **Developer changes** * Support autostart of a project, taking user to first question of a project when starting a new project. +* Retrieve Catalog data from database instead of file system with new controls.models.CatalogData model. v0.9.7 (August 06, 2021) diff --git a/controls/admin.py b/controls/admin.py index fa0d7584b..68d6c2db3 100644 --- a/controls/admin.py +++ b/controls/admin.py @@ -2,6 +2,7 @@ from django.contrib import admin from django.http import HttpResponse from .models import ImportRecord, Statement, Element, ElementControl, ElementRole, System, CommonControlProvider, CommonControl, ElementCommonControl, Poam, Deployment, SystemAssessmentResult +from .oscal import CatalogData from guardian.admin import GuardedModelAdmin from simple_history.admin import SimpleHistoryAdmin @@ -89,6 +90,9 @@ class SystemAssessmentResultAdmin(admin.ModelAdmin): list_display = ('id', 'name', 'system', 'deployment', 'uuid') search_fields = ('id', 'name', 'system', 'deployment', 'uuid') +class CatalogDataAdmin(admin.ModelAdmin): + list_display = ('catalog_key',) + search_fields = ('catalog_key',) admin.site.register(ImportRecord, ImportRecordAdmin) admin.site.register(Statement, StatementAdmin) @@ -102,4 +106,4 @@ class SystemAssessmentResultAdmin(admin.ModelAdmin): admin.site.register(Poam, PoamAdmin) admin.site.register(Deployment, DeploymentAdmin) admin.site.register(SystemAssessmentResult, SystemAssessmentResultAdmin) - +admin.site.register(CatalogData, CatalogDataAdmin) diff --git a/controls/migrations/0057_catalogdata.py b/controls/migrations/0057_catalogdata.py new file mode 100644 index 000000000..198bedaed --- /dev/null +++ b/controls/migrations/0057_catalogdata.py @@ -0,0 +1,33 @@ +# Generated by Django 3.2.4 on 2021-08-05 02:12 + +from django.db import migrations, models +import django.db.models.manager +import jsonfield.fields + + +class Migration(migrations.Migration): + + dependencies = [ + ('controls', '0056_element_oscal_version'), + ] + + operations = [ + migrations.CreateModel( + name='CatalogData', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('catalog_key', models.CharField(help_text='Unique key for catalog', max_length=100, unique=True)), + ('catalog_json', jsonfield.fields.JSONField(blank=True, help_text='JSON object representing the OSCAL-formatted control catalog.', null=True)), + ('created', models.DateTimeField(auto_now_add=True, db_index=True)), + ('updated', models.DateTimeField(auto_now=True, db_index=True)), + ], + options={ + 'abstract': False, + 'base_manager_name': 'prefetch_manager', + }, + managers=[ + ('objects', django.db.models.manager.Manager()), + ('prefetch_manager', django.db.models.manager.Manager()), + ], + ), + ] diff --git a/controls/migrations/0058_catalogdata_baselines_json.py b/controls/migrations/0058_catalogdata_baselines_json.py new file mode 100644 index 000000000..31145bb82 --- /dev/null +++ b/controls/migrations/0058_catalogdata_baselines_json.py @@ -0,0 +1,19 @@ +# Generated by Django 3.2.4 on 2021-08-05 15:19 + +from django.db import migrations +import jsonfield.fields + + +class Migration(migrations.Migration): + + dependencies = [ + ('controls', '0057_catalogdata'), + ] + + operations = [ + migrations.AddField( + model_name='catalogdata', + name='baselines_json', + field=jsonfield.fields.JSONField(blank=True, help_text='JSON object representing the baselines for the catalog.', null=True), + ), + ] diff --git a/controls/models.py b/controls/models.py index a4cd0d19c..12bcc812f 100644 --- a/controls/models.py +++ b/controls/models.py @@ -15,14 +15,13 @@ from controls.enums.components import ComponentTypeEnum, ComponentStateEnum from siteapp.model_mixins.tags import TagModelMixin from controls.enums.statements import StatementTypeEnum -from controls.oscal import Catalogs, Catalog, check_and_extend +from controls.oscal import Catalogs, Catalog, CatalogData import uuid import tools.diff_match_patch.python3 as dmp_module from copy import deepcopy from django.db import transaction BASELINE_PATH = os.path.join(os.path.dirname(__file__),'data','baselines') -EXTERNAL_BASELINE_PATH = os.path.join(f"{os.getcwd()}",'local', 'controls', 'data', 'baselines') ORGPARAM_PATH = os.path.join(os.path.dirname(__file__),'data','org_defined_parameters') class ImportRecord(models.Model): @@ -783,9 +782,7 @@ class Baselines (object): def __init__(self): self.file_path = BASELINE_PATH - self.external_file_path = EXTERNAL_BASELINE_PATH self.baselines_keys = self._list_keys() - # self.index = self._build_index() # Usage # from controls.models import Baselines @@ -796,44 +793,18 @@ def __init__(self): # # Returns ['ac-1', 'ac-2', 'ac-2.1', 'ac-2.2', ...] # bs.get_baseline_controls('NIST_SP-800-53_rev4', 'moderate') - def _list_files(self): - return self.extend_external_baselines([ - 'NIST_SP-800-53_rev4_baselines.json', - # 'NIST_SP-800-53_rev5_baselines.json', - 'NIST_SP-800-171_rev1_baselines.json', - 'CMMC_ver1_baselines.json' - ], "files") - - def _list_keys(self): - return self.extend_external_baselines([ - 'NIST_SP-800-53_rev4', - # 'NIST_SP-800-53_rev5', - 'NIST_SP-800-171_rev1', - 'CMMC_ver1' - ], "keys") - + # TODO: only return keys for records that have baselines? + return list(CatalogData.objects.order_by('catalog_key').values_list('catalog_key', flat=True).distinct()) def _load_json(self, baselines_key): """Read baseline file - JSON""" - # TODO Escape baselines_key - self.data_file = baselines_key + "_baselines.json" - data_file = os.path.join(self.file_path, self.data_file) - # Does file exist? - if not os.path.isfile(data_file): - # Check if there any external oscal baseline files - try: - data_file = os.path.join(self.external_file_path, self.data_file) - except: - print("ERROR: {} does not exist".format(data_file)) - return False - # Load file as json - try: - with open(data_file, 'r') as json_file: - data = json.load(json_file) - return data - except: - print("ERROR: {} could not be read or could not be read as json".format(data_file)) + + catalog_record = CatalogData.objects.get(catalog_key=baselines_key) + baselines = catalog_record.baselines_json + if baselines: + return baselines + else: return False def get_baseline_controls(self, baselines_key, baseline_name): @@ -849,22 +820,6 @@ def get_baseline_controls(self, baselines_key, baseline_name): print("Requested baseline name not found in baselines_key data file") return False - @property - def body(self): - return self.legacy_imp_smt - - - def extend_external_baselines(self, baseline_info, extendtype): - """ - Add external baselines to list of baselines - """ - os.makedirs(EXTERNAL_BASELINE_PATH, exist_ok=True) - external_baselines = [file for file in os.listdir(EXTERNAL_BASELINE_PATH) if - file.endswith('.json')] - - baseline_info = check_and_extend(baseline_info, external_baselines, extendtype, "_baselines") - return baseline_info - class OrgParams(object): """ Represent list of organizational defined parameters. Temporary @@ -1016,3 +971,5 @@ def __repr__(self): # # For debugging. # return "" % (self.statement, self.id) + + diff --git a/controls/oscal.py b/controls/oscal.py index 2bf44d213..a769286a8 100644 --- a/controls/oscal.py +++ b/controls/oscal.py @@ -6,56 +6,47 @@ from pathlib import Path import sys +import auto_prefetch +from django.db import models +from django.utils.functional import cached_property +from jsonfield import JSONField + + CATALOG_PATH = os.path.join(os.path.dirname(__file__), 'data', 'catalogs') -EXTERNAL_CATALOG_PATH = os.path.join(f"{os.getcwd()}",'local', 'controls', 'data', 'catalogs') +BASELINE_PATH = os.path.join(os.path.dirname(__file__),'data','baselines') + +class CatalogData(auto_prefetch.Model): + catalog_key = models.CharField(max_length=100, help_text="Unique key for catalog", unique=True, blank=False, null=False) + catalog_json = JSONField(blank=True, null=True, help_text="JSON object representing the OSCAL-formatted control catalog.") + baselines_json = JSONField(blank=True, null=True, help_text="JSON object representing the baselines for the catalog.") + created = models.DateTimeField(auto_now_add=True, db_index=True) + updated = models.DateTimeField(auto_now=True, db_index=True) + + def __str__(self): + return "'%s id=%d'" % (self.catalog_key, self.id) + + def __repr__(self): + # For debugging. + return "'%s id=%d'" % (self.catalog_key, self.id) class Catalogs(object): """Represent list of catalogs""" # well known catalog identifiers - NIST_SP_800_53_rev4 = 'NIST_SP-800-53_rev4' NIST_SP_800_53_rev5 = 'NIST_SP-800-53_rev5' NIST_SP_800_171_rev1 = 'NIST_SP-800-171_rev1' CMMC_ver1 = 'CMMC_ver1' def __init__(self): - self.catalog_path = CATALOG_PATH - # self.catalog = None self.catalog_keys = self._list_catalog_keys() self.index = self._build_index() - def extend_external_catalogs(self, catalog_info, extendtype): - """ - Add external catalogs to list of catalogs - """ - os.makedirs(EXTERNAL_CATALOG_PATH, exist_ok=True) - external_catalogs = [file for file in os.listdir(EXTERNAL_CATALOG_PATH) if - file.endswith('.json')] - catalog_info = check_and_extend(catalog_info, external_catalogs, extendtype, "_catalog") - - return catalog_info - - def _list_catalog_files(self): - return self.extend_external_catalogs([ - 'NIST_SP-800-53_rev4_catalog.json', - 'NIST_SP-800-53_rev5_catalog.json', - 'NIST_SP-800-171_rev1_catalog.json', - 'CMMC_ver1_catalog.json' - ], "files") - def _list_catalog_keys(self): - - return self.extend_external_catalogs([ - Catalogs.NIST_SP_800_53_rev4, - Catalogs.NIST_SP_800_53_rev5, - Catalogs.NIST_SP_800_171_rev1, - Catalogs.CMMC_ver1 - ], "keys") + return list(CatalogData.objects.order_by('catalog_key').values_list('catalog_key', flat=True).distinct()) def _load_catalog_json(self, catalog_key): catalog = Catalog(catalog_key) - #print(catalog_key, catalog._load_catalog_json()) return catalog._load_catalog_json() def _build_index(self): @@ -76,26 +67,13 @@ def list_catalogs(self): """ List catalog objects """ - return [Catalog(key) for key in Catalogs()._list_catalog_keys()] - + return [Catalog.GetInstance(catalog_key=key) for key in Catalogs()._list_catalog_keys()] def uhash(obj): """Return a positive hash code""" h = hash(obj) return h + sys.maxsize + 1 -def check_and_extend(values, external_values, extendtype, splitter): - """ - Modularize value to extend - """ - if extendtype == "keys": - keys = [key.split(f'{splitter}.json')[0] for key in external_values] - values.extend(keys) - elif extendtype == "files": - files = [file for file in external_values] - values.extend(files) - return values - def de_oscalize_control(control_id): """ Returns the regular control formatting from an oscalized version of the control number. @@ -108,10 +86,10 @@ class Catalog(object): # Create a singleton instance of this class per catalog. GetInstance returns # that singleton instance. Instead of doing - # `cg = Catalog(catalog_key=Catalogs.NIST_SP_800_53_rev4)`, - # do `cg = Catalog.GetInstance(catalog_key=Catalogs.NIST_SP_800_53_rev4')`. + # `cg = Catalog(catalog_key='NIST_SP-800-53_rev4')`, + # do `cg = Catalog.GetInstance(catalog_key='NIST_SP-800-53_rev4')`. @classmethod - def GetInstance(cls, catalog_key=Catalogs.NIST_SP_800_53_rev4, parameter_values=dict()): + def GetInstance(cls, catalog_key='NIST_SP-800-53_rev4', parameter_values=dict()): # Create a new instance of Catalog() the first time for each # catalog key / parameter combo # this method is called. Keep it in memory indefinitely. @@ -132,11 +110,10 @@ def _catalog_instance_key(catalog_key, parameter_values): catalog_instance_key += '_' + str(parameter_values_hash) return catalog_instance_key.replace('-', '_') - def __init__(self, catalog_key=Catalogs.NIST_SP_800_53_rev4, parameter_values=dict()): + def __init__(self, catalog_key='NIST_SP-800-53_rev4', parameter_values=dict()): self.catalog_key = catalog_key self.catalog_key_display = catalog_key.replace("_", " ") self.catalog_path = CATALOG_PATH - self.external_catalog_path = EXTERNAL_CATALOG_PATH self.catalog_file = catalog_key + "_catalog.json" try: self.oscal = self._load_catalog_json() @@ -163,22 +140,12 @@ def __init__(self, catalog_key=Catalogs.NIST_SP_800_53_rev4, parameter_values=di def _load_catalog_json(self): """Read catalog file - JSON""" - catalog_file = os.path.join(self.catalog_path, self.catalog_file) - catalog_file_external = os.path.join(self.external_catalog_path, self.catalog_file) - # Get catalog file from internal or "external" catalog files - if os.path.isfile(catalog_file): - with open(catalog_file, 'r') as json_file: - data = json.load(json_file) - oscal = data['catalog'] - return oscal - elif os.path.isfile(catalog_file_external): - with open(catalog_file_external, 'r') as json_file: - data = json.load(json_file) - oscal = data['catalog'] - return oscal - else: - # Catalog file doesn't exist - return False + + # Get catalog from database + # TODO: check for DB miss + catalog_record = CatalogData.objects.get(catalog_key=self.catalog_key) + oscal = catalog_record.catalog_json['catalog'] + return oscal def find_dict_by_value(self, search_array, search_key, search_value): """Return the dictionary in an array of dictionaries with a key matching a value""" diff --git a/controls/tests.py b/controls/tests.py index 0aff5a12a..4f422075e 100644 --- a/controls/tests.py +++ b/controls/tests.py @@ -29,7 +29,7 @@ from system_settings.models import SystemSettings from controls.models import * from controls.enums.statements import StatementTypeEnum -from controls.oscal import Catalogs, Catalog, EXTERNAL_CATALOG_PATH, de_oscalize_control +from controls.oscal import Catalogs, Catalog, de_oscalize_control from siteapp.models import User, Project, Portfolio from system_settings.models import SystemSettings @@ -41,7 +41,7 @@ # Control Tests -class Oscal80053Tests(TestCase): +class Oscal80053Tests(SeleniumTest): # Test def test_catalog_load_control(self): cg = Catalog.GetInstance(Catalogs.NIST_SP_800_53_rev4) @@ -140,72 +140,72 @@ def test_catalog_list(self): Check the catalog listing method has the 3 default catalogs """ - os.makedirs(EXTERNAL_CATALOG_PATH, exist_ok=True) + # os.makedirs(EXTERNAL_CATALOG_PATH, exist_ok=True) catalog_list = Catalogs().list_catalogs() self.assertEqual(len(catalog_list), 4) - def test_extend_external_catalogs(self): - """ - Extending catalog file and key list - """ - os.makedirs(EXTERNAL_CATALOG_PATH, exist_ok=True) - with tempfile.TemporaryFile() as d: - temp_file_name = os.path.join(EXTERNAL_CATALOG_PATH, f'{d.name}_revtest_catalog.json') - - # finding fixture data and dumping in the temp file - test_catalog = os.getcwd() + "/fixtures/test_catalog.json" - with open(test_catalog, 'r') as json_file: - catalog_data = json.load(json_file) - with open(temp_file_name, 'w') as cred: - json.dump(catalog_data, cred) - - extended_files = Catalogs.extend_external_catalogs(self, [ - 'NIST_SP-800-53_rev4_catalog.json', - 'NIST_SP-800-53_rev5_catalog.json', - 'NIST_SP-800-171_rev1_catalog.json' - ], "files") - - self.assertEqual(len(extended_files), 4) - - extended_keys = Catalogs.extend_external_catalogs(self, [ - Catalogs.NIST_SP_800_53_rev4, - Catalogs.NIST_SP_800_53_rev5, - Catalogs.NIST_SP_800_171_rev1 - ], "keys") - self.assertEqual(len(extended_keys), 4) - # Delete temp file - os.remove(temp_file_name) - - def test_extend_external_baseline(self): - """ - Extending baseline file and key list - """ - os.makedirs(EXTERNAL_BASELINE_PATH, exist_ok=True) - with tempfile.TemporaryFile() as d: - temp_file_name = os.path.join(EXTERNAL_BASELINE_PATH, f'{d.name}_revtest_baseline.json') - - # finding fixture data and dumping in the temp file - test_baseline = os.getcwd() + "/fixtures/test_baselines.json" - with open(test_baseline, 'r') as json_file: - baseline_data = json.load(json_file) - with open(temp_file_name, 'w') as cred: - json.dump(baseline_data, cred) - - extended_files = Baselines.extend_external_baselines(self, [ - 'NIST_SP-800-53_rev4_baselines.json', - 'NIST_SP-800-171_rev1_baselines.json' - ], "files") - - self.assertEqual(len(extended_files), 3) - - extended_keys = Baselines.extend_external_baselines(self, [ - 'NIST_SP-800-53_rev4', - 'NIST_SP-800-171_rev1' - ], "keys") - - self.assertEqual(len(extended_keys), 3) - # Delete temp file - os.remove(temp_file_name) + # def test_extend_external_catalogs(self): + # """ + # Extending catalog file and key list + # """ + # os.makedirs(EXTERNAL_CATALOG_PATH, exist_ok=True) + # with tempfile.TemporaryFile() as d: + # temp_file_name = os.path.join(EXTERNAL_CATALOG_PATH, f'{d.name}_revtest_catalog.json') + + # # finding fixture data and dumping in the temp file + # test_catalog = os.getcwd() + "/fixtures/test_catalog.json" + # with open(test_catalog, 'r') as json_file: + # catalog_data = json.load(json_file) + # with open(temp_file_name, 'w') as cred: + # json.dump(catalog_data, cred) + + # extended_files = Catalogs.extend_external_catalogs(self, [ + # 'NIST_SP-800-53_rev4_catalog.json', + # 'NIST_SP-800-53_rev5_catalog.json', + # 'NIST_SP-800-171_rev1_catalog.json' + # ], "files") + + # self.assertEqual(len(extended_files), 4) + + # extended_keys = Catalogs.extend_external_catalogs(self, [ + # Catalogs.NIST_SP_800_53_rev4, + # Catalogs.NIST_SP_800_53_rev5, + # Catalogs.NIST_SP_800_171_rev1 + # ], "keys") + # self.assertEqual(len(extended_keys), 4) + # # Delete temp file + # os.remove(temp_file_name) +# + # def test_extend_external_baseline(self): + # """ + # Extending baseline file and key list + # """ + # os.makedirs(EXTERNAL_BASELINE_PATH, exist_ok=True) + # with tempfile.TemporaryFile() as d: + # temp_file_name = os.path.join(EXTERNAL_BASELINE_PATH, f'{d.name}_revtest_baseline.json') + + # # finding fixture data and dumping in the temp file + # test_baseline = os.getcwd() + "/fixtures/test_baselines.json" + # with open(test_baseline, 'r') as json_file: + # baseline_data = json.load(json_file) + # with open(temp_file_name, 'w') as cred: + # json.dump(baseline_data, cred) + + # extended_files = Baselines.extend_external_baselines(self, [ + # 'NIST_SP-800-53_rev4_baselines.json', + # 'NIST_SP-800-171_rev1_baselines.json' + # ], "files") + + # self.assertEqual(len(extended_files), 3) + + # extended_keys = Baselines.extend_external_baselines(self, [ + # 'NIST_SP-800-53_rev4', + # 'NIST_SP-800-171_rev1' + # ], "keys") + + # self.assertEqual(len(extended_keys), 3) + # # Delete temp file + # os.remove(temp_file_name) ##################################################################### class ComponentUITests(OrganizationSiteFunctionalTests): @@ -609,7 +609,7 @@ def test_component_type_state(self): self.assertTrue(e2.component_type == "hardware") self.assertTrue(e2.component_state == "disposition") -class ElementControlUnitTests(TestCase): +class ElementControlUnitTests(SeleniumTest): def test_assign_baseline(self): @@ -791,7 +791,7 @@ def test_element_create(self): # poam.delete() # self.assertTrue(poam.uuid is None) -class OrgParamTests(TestCase): +class OrgParamTests(SeleniumTest): """Class for OrgParam Unit Tests""" def test_org_params(self): diff --git a/controls/views.py b/controls/views.py index d66adeb30..2b928ca68 100644 --- a/controls/views.py +++ b/controls/views.py @@ -1289,21 +1289,18 @@ def component_library_component(request, element_id): def api_controls_select(request): """Return list of controls in json for select2 options from all control catalogs""" - # Create array to hold accumulated controls + cl_id = request.GET.get('q', None).lower() + # Search control catalogs in a loop and add results to an array cxs = [] - # Loop through control catalogs catalogs = Catalogs() for ck in catalogs._list_catalog_keys(): cx = Catalog.GetInstance(catalog_key=ck) - # Get controls - ctl_list = cx.get_flattened_controls_all_as_dict() - # Build objects for rendering Select2 auto complete list from catalog - select_list = [{'id': ctl_list[ctl]['id'], 'title': ctl_list[ctl]['title'], 'class': ctl_list[ctl]['class'], 'catalog_key_display': cx.catalog_key_display, 'display_text': f"{ctl_list[ctl]['label']} - {ctl_list[ctl]['title']} - {cx.catalog_key_display}"} for ctl in ctl_list] - # Extend array of accumuated controls with catalog's control list - cxs.extend(select_list) - # Sort the accummulated list + ctr = cx.get_control_by_id(cl_id) + # TODO: Better representation of control ids for case-insensitive searching insteading of listing ids in both cases + # TODO: OSCALizing control id? + if ctr: + cxs.append({'id': ctr['id'], 'title': ctr['title'], 'class': ctr['class'], 'catalog_key_display': cx.catalog_key_display, 'display_text': f"{ctr['id']} - {ctr['title']} - {cx.catalog_key_display} - ({ctr['id'].upper()})"}) cxs.sort(key = operator.itemgetter('id', 'catalog_key_display')) - status = "success" message = "Sending list." return JsonResponse( {"status": status, "message": message, "data": {"controls": cxs} }) diff --git a/siteapp/management/commands/first_run.py b/siteapp/management/commands/first_run.py index 36828c606..e3601dcd6 100644 --- a/siteapp/management/commands/first_run.py +++ b/siteapp/management/commands/first_run.py @@ -1,5 +1,7 @@ import sys import os.path +import json + from django.core.management import call_command from django.core.management.base import BaseCommand, CommandError @@ -10,6 +12,7 @@ from guidedmodules.models import AppSource, Module from siteapp.models import User, Organization, Portfolio from controls.models import Element +from controls.oscal import CatalogData from django.contrib.auth.management.commands import createsuperuser import fs, fs.errors @@ -38,8 +41,36 @@ def handle(self, *args, **options): if not Organization.objects.all().exists() and not Organization.objects.filter(name="main").exists(): org = Organization.objects.create(name="main", slug="main") + # Load the default control catalogs and baselines + CATALOG_PATH = os.path.join(os.path.dirname(__file__),'..','..','..','controls','data','catalogs') + BASELINE_PATH = os.path.join(os.path.dirname(__file__),'..','..','..','controls','data','baselines') + + # TODO: Check directory exists + catalog_files = [file for file in os.listdir(CATALOG_PATH) if file.endswith('.json')] + # Load catalog and baseline data into database records from source files if data records do not exist in database + for cf in catalog_files: + catalog_key = cf.replace("_catalog.json", "") + with open(os.path.join(CATALOG_PATH,cf), 'r') as json_file: + catalog_json = json.load(json_file) + baseline_filename = cf.replace("_catalog.json", "_baselines.json") + if os.path.isfile(os.path.join(BASELINE_PATH, baseline_filename)): + with open(os.path.join(BASELINE_PATH, baseline_filename), 'r') as json_file: + baselines_json = json.load(json_file) + else: + baselines_json = {} + + catalog, created = CatalogData.objects.get_or_create( + catalog_key=catalog_key, + catalog_json=catalog_json, + baselines_json=baselines_json + ) + if created: + print(f"{catalog_key} record created into database") + else: + print(f"{catalog_key} record found in database") + # Install default AppSources and compliance apps if no AppSources installed - if AppSource.objects.all().exists(): + if not AppSource.objects.filter(slug="govready-q-files-startpack").exists(): # Create AppSources that we want. if os.path.exists("/mnt/q-files-host"): # For our docker image. @@ -146,7 +177,6 @@ def handle(self, *args, **options): username )) - # Create the first user. if not User.objects.filter(is_superuser=True).exists(): if not options['non_interactive']: diff --git a/siteapp/settings.py b/siteapp/settings.py index 122034db6..7288ab1bf 100644 --- a/siteapp/settings.py +++ b/siteapp/settings.py @@ -331,7 +331,7 @@ def make_secret_key(): } SILENCED_SYSTEM_CHECKS = [] -DATA_UPLOAD_MAX_MEMORY_SIZE = 2621440 +DATA_UPLOAD_MAX_MEMORY_SIZE = 10242880 # Settings that have normal values based on the primary app # (the app this file resides in). diff --git a/siteapp/tests.py b/siteapp/tests.py index 56dec9086..79b93ea5b 100644 --- a/siteapp/tests.py +++ b/siteapp/tests.py @@ -17,6 +17,7 @@ import tempfile import time import unittest +import json from django.contrib.auth import authenticate from django.test.client import RequestFactory @@ -35,6 +36,7 @@ from siteapp.models import (Organization, Portfolio, Project, ProjectMembership, User) from controls.models import Statement, Element, System +from controls.oscal import CatalogData, Catalogs, Catalog from siteapp.settings import HEADLESS, DOS from siteapp.views import project_edit from tools.utils.linux_to_dos import convert_w @@ -140,6 +142,31 @@ def tearDownClass(cls): def setUp(self): # clear the browser's cookies before each test self.browser.delete_all_cookies() + # Add catalogs to database + CATALOG_PATH = os.path.join(os.path.dirname(__file__),'..','controls','data','catalogs') + BASELINE_PATH = os.path.join(os.path.dirname(__file__),'..','controls','data','baselines') + catalog_files = [file for file in os.listdir(CATALOG_PATH) if file.endswith('.json')] + for cf in catalog_files: + catalog_key = cf.replace("_catalog.json", "") + with open(os.path.join(CATALOG_PATH,cf), 'r') as json_file: + catalog_json = json.load(json_file) + baseline_filename = cf.replace("_catalog.json", "_baselines.json") + if os.path.isfile(os.path.join(BASELINE_PATH, baseline_filename)): + with open(os.path.join(BASELINE_PATH, baseline_filename), 'r') as json_file: + baselines_json = json.load(json_file) + else: + baselines_json = {} + + catalog, created = CatalogData.objects.get_or_create( + catalog_key=catalog_key, + catalog_json=catalog_json, + baselines_json=baselines_json + ) + # if created: + # print(f"{catalog_key} record created into database") + # else: + # print(f"{catalog_key} record found in database") + def navigateToPage(self, path): self.browser.get(self.url(path)) diff --git a/templates/components/element_detail_tabs.html b/templates/components/element_detail_tabs.html index b1ebfbf3e..1a8510eb0 100644 --- a/templates/components/element_detail_tabs.html +++ b/templates/components/element_detail_tabs.html @@ -342,7 +342,7 @@

Systems

ajax: { url: '{% url 'api_controls_select' %}', dataType: 'json', - delay: 250, + delay: 150, data: function(params) { return { q: params.term // search term