From 3d8c891e5e2457c108010a307110138603537c33 Mon Sep 17 00:00:00 2001 From: DefectDojo release bot Date: Mon, 29 Jan 2024 16:25:43 +0000 Subject: [PATCH 1/9] Update versions in application files --- components/package.json | 2 +- dojo/__init__.py | 2 +- helm/defectdojo/Chart.yaml | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/components/package.json b/components/package.json index 3f7fe72f6d..262ef7e6f3 100644 --- a/components/package.json +++ b/components/package.json @@ -1,6 +1,6 @@ { "name": "defectdojo", - "version": "2.30.4", + "version": "2.31.0-dev", "license" : "BSD-3-Clause", "private": true, "dependencies": { diff --git a/dojo/__init__.py b/dojo/__init__.py index f5f755a23a..9406e56f47 100644 --- a/dojo/__init__.py +++ b/dojo/__init__.py @@ -4,6 +4,6 @@ # Django starts so that shared_task will use this app. from .celery import app as celery_app # noqa -__version__ = '2.30.4' +__version__ = '2.31.0-dev' __url__ = 'https://github.com/DefectDojo/django-DefectDojo' __docs__ = 'https://documentation.defectdojo.com' diff --git a/helm/defectdojo/Chart.yaml b/helm/defectdojo/Chart.yaml index 5ac25d323c..c723762ee5 100644 --- a/helm/defectdojo/Chart.yaml +++ b/helm/defectdojo/Chart.yaml @@ -1,8 +1,8 @@ apiVersion: v2 -appVersion: "2.30.4" +appVersion: "2.31.0-dev" description: A Helm chart for Kubernetes to install DefectDojo name: defectdojo -version: 1.6.107 +version: 1.6.108-dev icon: https://www.defectdojo.org/img/favicon.ico maintainers: - name: madchap From 4499b7b95c782fecce490f73a923deb12cb1b8f3 Mon Sep 17 00:00:00 2001 From: Cody Maffucci <46459665+Maffooch@users.noreply.github.com> Date: Tue, 30 Jan 2024 16:00:21 -0600 Subject: [PATCH 2/9] Update jira-description.tpl (#9403) --- dojo/templates/issue-trackers/jira_full/jira-description.tpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dojo/templates/issue-trackers/jira_full/jira-description.tpl b/dojo/templates/issue-trackers/jira_full/jira-description.tpl index b4e60a64a8..6fd326efb5 100644 --- a/dojo/templates/issue-trackers/jira_full/jira-description.tpl +++ b/dojo/templates/issue-trackers/jira_full/jira-description.tpl @@ -25,7 +25,7 @@ {% endif %} {% if finding.cvssv3_score %} -*CVSSv3 Score:* {{ finding.cvssv3_score }} +*CVSSv3 Score:* {{ finding.cvssv3_score }} {% if finding.cvssv3 %}({{ finding.cvssv3 }}){% endif %} {% endif %} *Product/Engagement/Test:* [{{ finding.test.engagement.product.name }}|{{ product_url|full_url }}] / [{{ finding.test.engagement.name }}|{{ engagement_url|full_url }}] / [{{ finding.test }}|{{ test_url|full_url }}] From c263392440eca2809ee322d1377998af017c20d7 Mon Sep 17 00:00:00 2001 From: Paul Osinski <42211303+paulOsinski@users.noreply.github.com> Date: Wed, 31 Jan 2024 14:06:47 -0700 Subject: [PATCH 3/9] Update and rename whitesource.md to mend.md (#9348) * Update and rename whitesource.md to mend.md * Update docs/content/en/integrations/parsers/file/mend.md Co-authored-by: Charles Neill <1749665+cneill@users.noreply.github.com> --------- Co-authored-by: Charles Neill <1749665+cneill@users.noreply.github.com> --- docs/content/en/integrations/parsers/file/mend.md | 15 +++++++++++++++ .../en/integrations/parsers/file/whitesource.md | 5 ----- 2 files changed, 15 insertions(+), 5 deletions(-) create mode 100644 docs/content/en/integrations/parsers/file/mend.md delete mode 100644 docs/content/en/integrations/parsers/file/whitesource.md diff --git a/docs/content/en/integrations/parsers/file/mend.md b/docs/content/en/integrations/parsers/file/mend.md new file mode 100644 index 0000000000..e45b2d16b8 --- /dev/null +++ b/docs/content/en/integrations/parsers/file/mend.md @@ -0,0 +1,15 @@ +--- +title: "Mend Scan" +toc_hide: true +--- + +### File Types +Accepts a JSON file, generated from the Mend* Unified Agent. + +### Sample Scan Data / Unit Tests +Unit tests for Mend JSON files can be found at https://github.com/DefectDojo/django-DefectDojo/tree/master/unittests/scans/mend + +### Link To Tool +See documentation: https://docs.mend.io/bundle/unified_agent/page/example_of_a_unified_agent_json_report.html + +*Formerly known as Whitesource. diff --git a/docs/content/en/integrations/parsers/file/whitesource.md b/docs/content/en/integrations/parsers/file/whitesource.md deleted file mode 100644 index d647d7cc96..0000000000 --- a/docs/content/en/integrations/parsers/file/whitesource.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -title: "Whitesource Scan" -toc_hide: true ---- -Import JSON report From 202fcb60f95224304c317fb62221e071f2c94ab3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 1 Feb 2024 20:06:48 -0600 Subject: [PATCH 4/9] Bump boto3 from 1.34.31 to 1.34.32 (#9455) Bumps [boto3](https://github.com/boto/boto3) from 1.34.31 to 1.34.32. - [Release notes](https://github.com/boto/boto3/releases) - [Changelog](https://github.com/boto/boto3/blob/develop/CHANGELOG.rst) - [Commits](https://github.com/boto/boto3/compare/1.34.31...1.34.32) --- updated-dependencies: - dependency-name: boto3 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 6ada4ca73a..23d70825b8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -79,7 +79,7 @@ django-ratelimit==4.1.0 argon2-cffi==23.1.0 blackduck==1.1.0 pycurl==7.45.2 # Required for Celery Broker AWS (SQS) support -boto3==1.34.31 # Required for Celery Broker AWS (SQS) support +boto3==1.34.32 # Required for Celery Broker AWS (SQS) support netaddr==0.10.1 vulners==2.1.2 fontawesomefree==6.5.1 From 47603755972411695c5fc5bf4f0da87a5e199596 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 1 Feb 2024 20:07:14 -0600 Subject: [PATCH 5/9] Bump drf-spectacular-sidecar from 2024.1.1 to 2024.2.1 (#9456) Bumps [drf-spectacular-sidecar](https://github.com/tfranzel/drf-spectacular-sidecar) from 2024.1.1 to 2024.2.1. - [Commits](https://github.com/tfranzel/drf-spectacular-sidecar/compare/2024.1.1...2024.2.1) --- updated-dependencies: - dependency-name: drf-spectacular-sidecar dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 23d70825b8..75e16b58db 100644 --- a/requirements.txt +++ b/requirements.txt @@ -74,7 +74,7 @@ hyperlink==21.0.0 django-test-migrations==1.3.0 djangosaml2==1.9.1 drf-spectacular==0.27.1 -drf-spectacular-sidecar==2024.1.1 +drf-spectacular-sidecar==2024.2.1 django-ratelimit==4.1.0 argon2-cffi==23.1.0 blackduck==1.1.0 From 39b68dcf6dc6b1d1199c753233a2fb8002338227 Mon Sep 17 00:00:00 2001 From: kiblik Date: Mon, 5 Feb 2024 16:19:48 +0100 Subject: [PATCH 6/9] API: Remote v2 OpenAPI2 Docs from menu (#9469) --- dojo/templates/base.html | 6 ------ 1 file changed, 6 deletions(-) diff --git a/dojo/templates/base.html b/dojo/templates/base.html index 7157a73896..8e42e4278a 100644 --- a/dojo/templates/base.html +++ b/dojo/templates/base.html @@ -170,12 +170,6 @@ {% endif %} {% endif %} -
  • - - - {% trans "API v2 OpenAPI2 Docs (Deprecated)" %} - -
  • From 3e81d6dfffe523c099d94455b21d2cea8d04afbf Mon Sep 17 00:00:00 2001 From: manuelsommer <47991713+manuel-sommer@users.noreply.github.com> Date: Mon, 5 Feb 2024 16:34:37 +0100 Subject: [PATCH 7/9] :bug: fix migration (#9467) --- dojo/db_migrations/0197_parser_merge.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dojo/db_migrations/0197_parser_merge.py b/dojo/db_migrations/0197_parser_merge.py index b00dac78a4..613ebea02f 100644 --- a/dojo/db_migrations/0197_parser_merge.py +++ b/dojo/db_migrations/0197_parser_merge.py @@ -76,7 +76,7 @@ def migrate_clairklar_parsers(apps, schema_editor): clair_test_type, _ = test_type_model.objects.get_or_create(name="Clair Scan", active=True) clairklar_test_type = test_type_model.objects.filter(name="Clair Klar Scan").first() # Get all the findings found by Clair Klar Scan - findings = finding_model.objects.filter(test__scan_type__in=OPENVAS_REFERENCES) + findings = finding_model.objects.filter(test__scan_type__in=CLAIRKLAR_REFERENCES) logger.warning(f'We identified {findings.count()} Clair Klar Scan findings to migrate to Clair Scan findings') # Iterate over all findings and change for finding in findings: From 19ecb49b94696cac9718fe836e485c91d56fa422 Mon Sep 17 00:00:00 2001 From: Blake Owens <76979297+blakeaowens@users.noreply.github.com> Date: Mon, 5 Feb 2024 14:53:04 -0600 Subject: [PATCH 8/9] finding sla expiration date field (part one) (#9473) * addition of sla expiration date field on the finding model * add migration and fix indentation issue * fix mitigated finding remaining sla days calculation * fix sla violation filter to return only active, sla violating findings * migration system settings fix * fix mitigation date vs datetime discrepancy * fix breaking unit test * move product save check to signal * fix unit test failure * make signal operations async, fix sla config delete 500 error * add unit tests to test sla expiration date functionality * restarting without signals * add async updating flags, redo migration * move signal logic to overriden save * fix errors for non-existing objects at creation * clean up comments and a few logical expressions * fix flake8 error * addition of new unit tests * fix unit test error * add message to form fields when async updating flag is true * fix save location, reword form messages, reword redirect messages * remove commented lines from unit tests * add a bit more description to API validation errors * migration fix * migration performance improvements * fix datetime - str comparison issue * clean up for part one of sla expiration date field * fix flake8 * Update dojo/db_migrations/0200_finding_sla_expiration_date_product_async_updating_and_more.py Co-authored-by: Charles Neill <1749665+cneill@users.noreply.github.com> * Update dojo/models.py Co-authored-by: Charles Neill <1749665+cneill@users.noreply.github.com> --------- Co-authored-by: Charles Neill <1749665+cneill@users.noreply.github.com> --- dojo/api_v2/serializers.py | 28 +++- dojo/apps.py | 1 + ...on_date_product_async_updating_and_more.py | 31 ++++ dojo/filters.py | 21 ++- dojo/forms.py | 40 ++++- dojo/models.py | 146 ++++++++++++++--- dojo/product/helpers.py | 26 ++- dojo/product/views.py | 7 +- dojo/sla_config/helpers.py | 26 +++ dojo/sla_config/views.py | 29 ++-- dojo/templates/dojo/form_fields.html | 3 +- dojo/templatetags/display_tags.py | 2 +- dojo/utils.py | 2 +- unittests/dojo_test_case.py | 7 +- unittests/test_finding_model.py | 148 +++++++++++++++++- unittests/tools/test_veracode_parser.py | 2 - 16 files changed, 454 insertions(+), 65 deletions(-) create mode 100644 dojo/db_migrations/0200_finding_sla_expiration_date_product_async_updating_and_more.py create mode 100644 dojo/sla_config/helpers.py diff --git a/dojo/api_v2/serializers.py b/dojo/api_v2/serializers.py index 49e3486fe2..45d2707a6e 100644 --- a/dojo/api_v2/serializers.py +++ b/dojo/api_v2/serializers.py @@ -2004,8 +2004,20 @@ class Meta: exclude = ( "tid", "updated", + "async_updating" ) + def validate(self, data): + async_updating = getattr(self.instance, 'async_updating', None) + if async_updating: + new_sla_config = data.get('sla_configuration', None) + old_sla_config = getattr(self.instance, 'sla_configuration', None) + if new_sla_config and old_sla_config and new_sla_config != old_sla_config: + raise serializers.ValidationError( + 'Finding SLA expiration dates are currently being recalculated. The SLA configuration for this product cannot be changed until the calculation is complete.' + ) + return data + def get_findings_count(self, obj) -> int: return obj.findings_count @@ -3031,7 +3043,21 @@ class Meta: class SLAConfigurationSerializer(serializers.ModelSerializer): class Meta: model = SLA_Configuration - fields = "__all__" + exclude = ( + "async_updating", + ) + + def validate(self, data): + async_updating = getattr(self.instance, 'async_updating', None) + if async_updating: + for field in ['critical', 'high', 'medium', 'low']: + old_days = getattr(self.instance, field, None) + new_days = data.get(field, None) + if old_days and new_days and (old_days != new_days): + raise serializers.ValidationError( + 'Finding SLA expiration dates are currently being calculated. The SLA days for this SLA configuration cannot be changed until the calculation is complete.' + ) + return data class UserProfileSerializer(serializers.Serializer): diff --git a/dojo/apps.py b/dojo/apps.py index 30a1711b19..6c84a420de 100644 --- a/dojo/apps.py +++ b/dojo/apps.py @@ -74,6 +74,7 @@ def ready(self): import dojo.announcement.signals # noqa import dojo.product.signals # noqa import dojo.test.signals # noqa + import dojo.sla_config.helpers # noqa def get_model_fields_with_extra(model, extra_fields=()): diff --git a/dojo/db_migrations/0200_finding_sla_expiration_date_product_async_updating_and_more.py b/dojo/db_migrations/0200_finding_sla_expiration_date_product_async_updating_and_more.py new file mode 100644 index 0000000000..20ef3e4f68 --- /dev/null +++ b/dojo/db_migrations/0200_finding_sla_expiration_date_product_async_updating_and_more.py @@ -0,0 +1,31 @@ +# Generated by Django 4.1.13 on 2024-01-17 03:07 + +from django.db import migrations, models +import logging + +logger = logging.getLogger(__name__) + + +class Migration(migrations.Migration): + + dependencies = [ + ('dojo', '0199_whitesource_to_mend'), + ] + + operations = [ + migrations.AddField( + model_name='finding', + name='sla_expiration_date', + field=models.DateField(blank=True, help_text="(readonly)The date SLA expires for this finding. Empty by default, causing a fallback to 'date'.", null=True, verbose_name='SLA Expiration Date'), + ), + migrations.AddField( + model_name='product', + name='async_updating', + field=models.BooleanField(default=False, help_text='Findings under this Product or SLA configuration are asynchronously being updated'), + ), + migrations.AddField( + model_name='sla_configuration', + name='async_updating', + field=models.BooleanField(default=False, help_text='Findings under this SLA configuration are asynchronously being updated'), + ), + ] diff --git a/dojo/filters.py b/dojo/filters.py index 20db8bddcf..51279d76a9 100644 --- a/dojo/filters.py +++ b/dojo/filters.py @@ -147,13 +147,13 @@ class FindingSLAFilter(ChoiceFilter): def any(self, qs, name): return qs - def satisfies_sla(self, qs, name): + def sla_satisfied(self, qs, name): for finding in qs: if finding.violates_sla: qs = qs.exclude(id=finding.id) return qs - def violates_sla(self, qs, name): + def sla_violated(self, qs, name): for finding in qs: if not finding.violates_sla: qs = qs.exclude(id=finding.id) @@ -161,8 +161,8 @@ def violates_sla(self, qs, name): options = { None: (_('Any'), any), - 0: (_('False'), satisfies_sla), - 1: (_('True'), violates_sla), + 0: (_('False'), sla_satisfied), + 1: (_('True'), sla_violated), } def __init__(self, *args, **kwargs): @@ -182,13 +182,13 @@ class ProductSLAFilter(ChoiceFilter): def any(self, qs, name): return qs - def satisfies_sla(self, qs, name): + def sla_satisifed(self, qs, name): for product in qs: if product.violates_sla: qs = qs.exclude(id=product.id) return qs - def violates_sla(self, qs, name): + def sla_violated(self, qs, name): for product in qs: if not product.violates_sla: qs = qs.exclude(id=product.id) @@ -196,8 +196,8 @@ def violates_sla(self, qs, name): options = { None: (_('Any'), any), - 0: (_('False'), satisfies_sla), - 1: (_('True'), violates_sla), + 0: (_('False'), sla_satisifed), + 1: (_('True'), sla_violated), } def __init__(self, *args, **kwargs): @@ -1465,9 +1465,8 @@ class Meta: 'endpoints', 'references', 'thread_id', 'notes', 'scanner_confidence', 'numerical_severity', 'line', 'duplicate_finding', - 'hash_code', - 'reviewers', - 'created', 'files', 'sla_start_date', 'cvssv3', + 'hash_code', 'reviewers', 'created', 'files', + 'sla_start_date', 'sla_expiration_date', 'cvssv3', 'severity_justification', 'steps_to_reproduce'] def __init__(self, *args, **kwargs): diff --git a/dojo/forms.py b/dojo/forms.py index b544c09d05..558c09ae69 100755 --- a/dojo/forms.py +++ b/dojo/forms.py @@ -263,6 +263,12 @@ def __init__(self, *args, **kwargs): super(ProductForm, self).__init__(*args, **kwargs) self.fields['prod_type'].queryset = get_authorized_product_types(Permissions.Product_Type_Add_Product) + # if this product has findings being asynchronously updated, disable the sla config field + if self.instance.async_updating: + self.fields['sla_configuration'].disabled = True + self.fields['sla_configuration'].widget.attrs['message'] = 'Finding SLA expiration dates are currently being recalculated. ' + \ + 'This field cannot be changed until the calculation is complete.' + class Meta: model = Product fields = ['name', 'description', 'tags', 'product_manager', 'technical_contact', 'team_manager', 'prod_type', 'sla_configuration', 'regulations', @@ -1073,7 +1079,7 @@ class AdHocFindingForm(forms.ModelForm): # the only reliable way without hacking internal fields to get predicatble ordering is to make it explicit field_order = ('title', 'date', 'cwe', 'vulnerability_ids', 'severity', 'cvssv3', 'description', 'mitigation', 'impact', 'request', 'response', 'steps_to_reproduce', 'severity_justification', 'endpoints', 'endpoints_to_add', 'references', 'active', 'verified', 'false_p', 'duplicate', 'out_of_scope', - 'risk_accepted', 'under_defect_review', 'sla_start_date') + 'risk_accepted', 'under_defect_review', 'sla_start_date', 'sla_expiration_date') def __init__(self, *args, **kwargs): req_resp = kwargs.pop('req_resp') @@ -1113,7 +1119,8 @@ def clean(self): class Meta: model = Finding exclude = ('reporter', 'url', 'numerical_severity', 'under_review', 'reviewers', 'cve', 'inherited_tags', - 'review_requested_by', 'is_mitigated', 'jira_creation', 'jira_change', 'endpoint_status', 'sla_start_date') + 'review_requested_by', 'is_mitigated', 'jira_creation', 'jira_change', 'endpoints', 'sla_start_date', + 'sla_expiration_date') class PromoteFindingForm(forms.ModelForm): @@ -1139,9 +1146,9 @@ class PromoteFindingForm(forms.ModelForm): references = forms.CharField(widget=forms.Textarea, required=False) # the onyl reliable way without hacking internal fields to get predicatble ordering is to make it explicit - field_order = ('title', 'group', 'date', 'sla_start_date', 'cwe', 'vulnerability_ids', 'severity', 'cvssv3', 'cvssv3_score', 'description', 'mitigation', 'impact', - 'request', 'response', 'steps_to_reproduce', 'severity_justification', 'endpoints', 'endpoints_to_add', 'references', - 'active', 'mitigated', 'mitigated_by', 'verified', 'false_p', 'duplicate', + field_order = ('title', 'group', 'date', 'sla_start_date', 'sla_expiration_date', 'cwe', 'vulnerability_ids', 'severity', 'cvssv3', + 'cvssv3_score', 'description', 'mitigation', 'impact', 'request', 'response', 'steps_to_reproduce', 'severity_justification', + 'endpoints', 'endpoints_to_add', 'references', 'active', 'mitigated', 'mitigated_by', 'verified', 'false_p', 'duplicate', 'out_of_scope', 'risk_accept', 'under_defect_review') def __init__(self, *args, **kwargs): @@ -1211,9 +1218,9 @@ class FindingForm(forms.ModelForm): 'invalid_choice': EFFORT_FOR_FIXING_INVALID_CHOICE}) # the only reliable way without hacking internal fields to get predicatble ordering is to make it explicit - field_order = ('title', 'group', 'date', 'sla_start_date', 'cwe', 'vulnerability_ids', 'severity', 'cvssv3', 'cvssv3_score', 'description', 'mitigation', 'impact', - 'request', 'response', 'steps_to_reproduce', 'severity_justification', 'endpoints', 'endpoints_to_add', 'references', - 'active', 'mitigated', 'mitigated_by', 'verified', 'false_p', 'duplicate', + field_order = ('title', 'group', 'date', 'sla_start_date', 'sla_expiration_date', 'cwe', 'vulnerability_ids', 'severity', 'cvssv3', + 'cvssv3_score', 'description', 'mitigation', 'impact', 'request', 'response', 'steps_to_reproduce', 'severity_justification', + 'endpoints', 'endpoints_to_add', 'references', 'active', 'mitigated', 'mitigated_by', 'verified', 'false_p', 'duplicate', 'out_of_scope', 'risk_accept', 'under_defect_review') def __init__(self, *args, **kwargs): @@ -1251,6 +1258,7 @@ def __init__(self, *args, **kwargs): self.fields['duplicate'].help_text = "You can mark findings as duplicate only from the view finding page." self.fields['sla_start_date'].disabled = True + self.fields['sla_expiration_date'].disabled = True if self.can_edit_mitigated_data: if hasattr(self, 'instance'): @@ -2436,6 +2444,22 @@ def clean(self): class SLAConfigForm(forms.ModelForm): + def __init__(self, *args, **kwargs): + super(SLAConfigForm, self).__init__(*args, **kwargs) + + # if this sla config has findings being asynchronously updated, disable the days by severity fields + if self.instance.async_updating: + msg = 'Finding SLA expiration dates are currently being recalculated. ' + \ + 'This field cannot be changed until the calculation is complete.' + self.fields['critical'].disabled = True + self.fields['critical'].widget.attrs['message'] = msg + self.fields['high'].disabled = True + self.fields['high'].widget.attrs['message'] = msg + self.fields['medium'].disabled = True + self.fields['medium'].widget.attrs['message'] = msg + self.fields['low'].disabled = True + self.fields['low'].widget.attrs['message'] = msg + class Meta: model = SLA_Configuration fields = ['name', 'description', 'critical', 'high', 'medium', 'low'] diff --git a/dojo/models.py b/dojo/models.py index 64fc813716..7bda3997c0 100755 --- a/dojo/models.py +++ b/dojo/models.py @@ -31,6 +31,7 @@ from django import forms from django.utils.translation import gettext as _ from dateutil.relativedelta import relativedelta +from datetime import datetime from tagulous.models import TagField from tagulous.models.managers import FakeTagRelatedManager import tagulous.admin @@ -856,9 +857,7 @@ class Meta: class SLA_Configuration(models.Model): name = models.CharField(max_length=128, unique=True, blank=False, verbose_name=_('Custom SLA Name'), - help_text=_('A unique name for the set of SLAs.') - ) - + help_text=_('A unique name for the set of SLAs.')) description = models.CharField(max_length=512, null=True, blank=True) critical = models.IntegerField(default=7, verbose_name=_('Critical Finding SLA Days'), help_text=_('number of days to remediate a critical finding.')) @@ -868,15 +867,56 @@ class SLA_Configuration(models.Model): help_text=_('number of days to remediate a medium finding.')) low = models.IntegerField(default=120, verbose_name=_('Low Finding SLA Days'), help_text=_('number of days to remediate a low finding.')) + async_updating = models.BooleanField(default=False, + help_text=_('Findings under this SLA configuration are asynchronously being updated')) def clean(self): - sla_days = [self.critical, self.high, self.medium, self.low] for sla_day in sla_days: if sla_day < 1: raise ValidationError('SLA Days must be at least 1') + def save(self, *args, **kwargs): + # get the initial sla config before saving (if this is an existing sla config) + initial_sla_config = None + if self.pk is not None: + initial_sla_config = SLA_Configuration.objects.get(pk=self.pk) + # if initial config exists and async finding update is already running, revert sla config before saving + if initial_sla_config and self.async_updating: + self.critical = initial_sla_config.critical + self.high = initial_sla_config.high + self.medium = initial_sla_config.medium + self.low = initial_sla_config.low + + super(SLA_Configuration, self).save(*args, **kwargs) + + # if the initial sla config exists and async finding update is not running + if initial_sla_config is not None and not self.async_updating: + # check which sla days fields changed based on severity + severities = [] + if initial_sla_config.critical != self.critical: + severities.append('Critical') + if initial_sla_config.high != self.high: + severities.append('High') + if initial_sla_config.medium != self.medium: + severities.append('Medium') + if initial_sla_config.low != self.low: + severities.append('Low') + # if severities have changed, update finding sla expiration dates with those severities + if len(severities): + # set the async updating flag to true for this sla config + self.async_updating = True + super(SLA_Configuration, self).save(*args, **kwargs) + # set the async updating flag to true for all products using this sla config + products = Product.objects.filter(sla_configuration=self) + for product in products: + product.async_updating = True + super(Product, product).save() + # launch the async task to update all finding sla expiration dates + from dojo.sla_config.helpers import update_sla_expiration_dates_sla_config_async + update_sla_expiration_dates_sla_config_async(self, tuple(severities), products) + def __str__(self): return self.name @@ -998,6 +1038,37 @@ class Product(models.Model): blank=False, verbose_name=_("Disable SLA breach notifications"), help_text=_("Disable SLA breach notifications if configured in the global settings")) + async_updating = models.BooleanField(default=False, + help_text=_('Findings under this Product or SLA configuration are asynchronously being updated')) + + def save(self, *args, **kwargs): + # get the product's sla config before saving (if this is an existing product) + initial_sla_config = None + if self.pk is not None: + initial_sla_config = getattr(Product.objects.get(pk=self.pk), 'sla_configuration', None) + # if initial sla config exists and async finding update is already running, revert sla config before saving + if initial_sla_config and self.async_updating: + self.sla_configuration = initial_sla_config + + super(Product, self).save(*args, **kwargs) + + # if the initial sla config exists and async finding update is not running + if initial_sla_config is not None and not self.async_updating: + # get the new sla config from the saved product + new_sla_config = getattr(self, 'sla_configuration', None) + # if the sla config has changed, update finding sla expiration dates within this product + if new_sla_config and (initial_sla_config != new_sla_config): + # set the async updating flag to true for this product + self.async_updating = True + super(Product, self).save(*args, **kwargs) + # set the async updating flag to true for the sla config assigned to this product + sla_config = getattr(self, 'sla_configuration', None) + if sla_config: + sla_config.async_updating = True + super(SLA_Configuration, sla_config).save() + # launch the async task to update all finding sla expiration dates + from dojo.product.helpers import update_sla_expiration_dates_product_async + update_sla_expiration_dates_product_async(self, sla_config) def __str__(self): return self.name @@ -1123,8 +1194,7 @@ def get_absolute_url(self): @property def violates_sla(self): - findings = Finding.objects.filter(test__engagement__product=self, - active=True) + findings = Finding.objects.filter(test__engagement__product=self, active=True) for f in findings: if f.violates_sla: return True @@ -2110,20 +2180,22 @@ def __str__(self): class Finding(models.Model): - title = models.CharField(max_length=511, verbose_name=_('Title'), help_text=_("A short description of the flaw.")) date = models.DateField(default=get_current_date, verbose_name=_('Date'), help_text=_("The date the flaw was discovered.")) - sla_start_date = models.DateField( blank=True, null=True, verbose_name=_('SLA Start Date'), help_text=_("(readonly)The date used as start date for SLA calculation. Set by expiring risk acceptances. Empty by default, causing a fallback to 'date'.")) - + sla_expiration_date = models.DateField( + blank=True, + null=True, + verbose_name=_('SLA Expiration Date'), + help_text=_("(readonly)The date SLA expires for this finding. Empty by default, causing a fallback to 'date'.")) cwe = models.IntegerField(default=0, null=True, blank=True, verbose_name=_("CWE"), help_text=_("The CWE number associated with this flaw.")) @@ -2750,19 +2822,28 @@ def status(self): return ", ".join([str(s) for s in status]) def _age(self, start_date): + from dateutil.parser import parse + if start_date and isinstance(start_date, str): + start_date = parse(start_date).date() + from dojo.utils import get_work_days if settings.SLA_BUSINESS_DAYS: if self.mitigated: - days = get_work_days(self.date, self.mitigated.date()) + mitigated_date = self.mitigated + if isinstance(mitigated_date, datetime): + mitigated_date = self.mitigated.date() + days = get_work_days(self.date, mitigated_date) else: days = get_work_days(self.date, get_current_date()) else: - from datetime import datetime if isinstance(start_date, datetime): start_date = start_date.date() if self.mitigated: - diff = self.mitigated.date() - start_date + mitigated_date = self.mitigated + if isinstance(mitigated_date, datetime): + mitigated_date = self.mitigated.date() + diff = mitigated_date - start_date else: diff = get_current_date() - start_date days = diff.days @@ -2772,9 +2853,9 @@ def _age(self, start_date): def age(self): return self._age(self.date) - def get_sla_periods(self): - sla_configuration = SLA_Configuration.objects.filter(id=self.test.engagement.product.sla_configuration_id).first() - return sla_configuration + @property + def sla_age(self): + return self._age(self.get_sla_start_date()) def get_sla_start_date(self): if self.sla_start_date: @@ -2782,16 +2863,34 @@ def get_sla_start_date(self): else: return self.date - @property - def sla_age(self): - return self._age(self.get_sla_start_date()) + def get_sla_period(self): + sla_configuration = SLA_Configuration.objects.filter(id=self.test.engagement.product.sla_configuration_id).first() + return getattr(sla_configuration, self.severity.lower(), None) + + def set_sla_expiration_date(self): + system_settings = System_Settings.objects.get() + if not system_settings.enable_finding_sla: + return None + + days_remaining = None + sla_period = self.get_sla_period() + if sla_period: + days_remaining = sla_period - self.sla_age + + if days_remaining: + if self.mitigated: + mitigated_date = self.mitigated + if isinstance(mitigated_date, datetime): + mitigated_date = self.mitigated.date() + self.sla_expiration_date = mitigated_date + relativedelta(days=days_remaining) + else: + self.sla_expiration_date = get_current_date() + relativedelta(days=days_remaining) def sla_days_remaining(self): sla_calculation = None - sla_periods = self.get_sla_periods() - sla_age = getattr(sla_periods, self.severity.lower(), None) - if sla_age: - sla_calculation = sla_age - self.sla_age + sla_period = self.get_sla_period() + if sla_period: + sla_calculation = sla_period - self.sla_age return sla_calculation def sla_deadline(self): @@ -2923,6 +3022,9 @@ def save(self, dedupe_option=True, rules_option=True, product_grading_option=Tru elif (self.file_path is not None): self.static_finding = True + # update the SLA expiration date last, after all other finding fields have been updated + self.set_sla_expiration_date() + logger.debug("Saving finding of id " + str(self.id) + " dedupe_option:" + str(dedupe_option) + " (self.pk is %s)", "None" if self.pk is None else "not None") super(Finding, self).save(*args, **kwargs) diff --git a/dojo/product/helpers.py b/dojo/product/helpers.py index c2d3f634ae..74530744cd 100644 --- a/dojo/product/helpers.py +++ b/dojo/product/helpers.py @@ -1,11 +1,31 @@ import contextlib -from celery.utils.log import get_task_logger +import logging from dojo.celery import app -from dojo.models import Product, Engagement, Test, Finding, Endpoint +from dojo.models import SLA_Configuration, Product, Engagement, Test, Finding, Endpoint from dojo.decorators import dojo_async_task -logger = get_task_logger(__name__) +logger = logging.getLogger(__name__) + + +@dojo_async_task +@app.task +def update_sla_expiration_dates_product_async(product, sla_config, *args, **kwargs): + update_sla_expiration_dates_product_sync(product, sla_config) + + +def update_sla_expiration_dates_product_sync(product, sla_config): + logger.info(f"Updating finding SLA expiration dates within product {product}") + # update each finding that is within the SLA configuration that was saved + for f in Finding.objects.filter(test__engagement__product=product): + f.save() + # reset the async updating flag to false for the sla config assigned to this product + if sla_config: + sla_config.async_updating = False + super(SLA_Configuration, sla_config).save() + # set the async updating flag to false for the sla config assigned to this product + product.async_updating = False + super(Product, product).save() @dojo_async_task diff --git a/dojo/product/views.py b/dojo/product/views.py index be1b9afe0c..c2dc16098c 100755 --- a/dojo/product/views.py +++ b/dojo/product/views.py @@ -873,10 +873,15 @@ def edit_product(request, pid): form = ProductForm(request.POST, instance=product) jira_project = jira_helper.get_jira_project(product) if form.is_valid(): + initial_sla_config = Product.objects.get(pk=form.instance.id).sla_configuration form.save() + msg = 'Product updated successfully.' + # check if the SLA config was changed, append additional context to message + if initial_sla_config != form.instance.sla_configuration: + msg += ' All SLA expiration dates for findings within this product will be recalculated asynchronously for the newly assigned SLA configuration.' messages.add_message(request, messages.SUCCESS, - _('Product updated successfully.'), + _(msg), extra_tags='alert-success') success, jform = jira_helper.process_jira_project_form(request, instance=jira_project, product=product) diff --git a/dojo/sla_config/helpers.py b/dojo/sla_config/helpers.py new file mode 100644 index 0000000000..e9665adce4 --- /dev/null +++ b/dojo/sla_config/helpers.py @@ -0,0 +1,26 @@ +import logging +from dojo.models import SLA_Configuration, Product, Finding +from dojo.celery import app +from dojo.decorators import dojo_async_task + +logger = logging.getLogger(__name__) + + +@dojo_async_task +@app.task +def update_sla_expiration_dates_sla_config_async(sla_config, severities, products, *args, **kwargs): + update_sla_expiration_dates_sla_config_sync(sla_config, severities, products) + + +def update_sla_expiration_dates_sla_config_sync(sla_config, severities, products): + logger.info(f"Updating finding SLA expiration dates within the {sla_config} SLA configuration") + # update each finding that is within the SLA configuration that was saved + for f in Finding.objects.filter(test__engagement__product__sla_configuration_id=sla_config.id, severity__in=severities): + f.save() + # reset the async updating flag to false for all products using this sla config + for product in products: + product.async_updating = False + super(Product, product).save() + # reset the async updating flag to false for this sla config + sla_config.async_updating = False + super(SLA_Configuration, sla_config).save() diff --git a/dojo/sla_config/views.py b/dojo/sla_config/views.py index f247cd7725..e85b06ea8f 100644 --- a/dojo/sla_config/views.py +++ b/dojo/sla_config/views.py @@ -8,7 +8,7 @@ from dojo.authorization.authorization import user_has_configuration_permission_or_403 from dojo.authorization.authorization_decorators import user_is_configuration_authorized from dojo.forms import SLAConfigForm -from dojo.models import SLA_Configuration, System_Settings +from dojo.models import SLA_Configuration, System_Settings, Product from dojo.utils import add_breadcrumb logger = logging.getLogger(__name__) @@ -41,13 +41,20 @@ def edit_sla_config(request, slaid): if request.method == 'POST' and request.POST.get('delete'): if sla_config.id != 1: - user_has_configuration_permission_or_403( - request.user, 'dojo.delete_sla_configuration') - sla_config.delete() - messages.add_message(request, - messages.SUCCESS, - 'SLA Configuration Deleted.', - extra_tags='alert-success') + if Product.objects.filter(sla_configuration=sla_config).count(): + msg = f"The \"{sla_config}\" SLA configuration could not be deleted, as it is currently in use by one or more products." + messages.add_message(request, + messages.ERROR, + msg, + extra_tags='alert-warning') + else: + user_has_configuration_permission_or_403( + request.user, 'dojo.delete_sla_configuration') + sla_config.delete() + messages.add_message(request, + messages.SUCCESS, + 'SLA Configuration Deleted.', + extra_tags='alert-success') return HttpResponseRedirect(reverse('sla_config', )) else: messages.add_message(request, @@ -59,12 +66,12 @@ def edit_sla_config(request, slaid): elif request.method == 'POST': form = SLAConfigForm(request.POST, instance=sla_config) if form.is_valid(): - form.save() + form.save(commit=True) messages.add_message(request, messages.SUCCESS, - 'SLA configuration successfully updated.', + 'SLA configuration successfully updated. All SLA expiration dates for findings within this SLA configuration will be recalculated asynchronously.', extra_tags='alert-success') - form.save(commit=True) + return HttpResponseRedirect(reverse('sla_config', )) else: form = SLAConfigForm(instance=sla_config) diff --git a/dojo/templates/dojo/form_fields.html b/dojo/templates/dojo/form_fields.html index fe2b949162..98706ee46d 100644 --- a/dojo/templates/dojo/form_fields.html +++ b/dojo/templates/dojo/form_fields.html @@ -73,8 +73,7 @@ {% endif %}
    {{ field|addcss:"class:form-control" }} - - +

    {{ field.field.widget.attrs.message }}

    {% for error in field.errors %} {{ error }} {% endfor %} diff --git a/dojo/templatetags/display_tags.py b/dojo/templatetags/display_tags.py index d0251eaad4..0095428194 100644 --- a/dojo/templatetags/display_tags.py +++ b/dojo/templatetags/display_tags.py @@ -255,7 +255,7 @@ def finding_sla(finding): title = "" severity = finding.severity find_sla = finding.sla_days_remaining() - sla_age = getattr(finding.get_sla_periods(), severity.lower(), None) + sla_age = finding.get_sla_period() if finding.mitigated: status = "blue" status_text = 'Remediated within SLA for ' + severity.lower() + ' findings (' + str(sla_age) + ' days since ' + finding.get_sla_start_date().strftime("%b %d, %Y") + ')' diff --git a/dojo/utils.py b/dojo/utils.py index eac65b08a4..135d341e54 100644 --- a/dojo/utils.py +++ b/dojo/utils.py @@ -1547,7 +1547,7 @@ def calculate_grade(product, *args, **kwargs): grade_product = "grade_product(%s, %s, %s, %s)" % ( critical, high, medium, low) product.prod_numeric_grade = aeval(grade_product) - product.save() + super(Product, product).save() def get_celery_worker_status(): diff --git a/unittests/dojo_test_case.py b/unittests/dojo_test_case.py index c5165febba..e6f0b19fce 100644 --- a/unittests/dojo_test_case.py +++ b/unittests/dojo_test_case.py @@ -18,7 +18,7 @@ from dojo.models import (SEVERITIES, DojoMeta, Endpoint, Endpoint_Status, Engagement, Finding, JIRA_Issue, JIRA_Project, Notes, Product, Product_Type, System_Settings, Test, - Test_Type, User) + SLA_Configuration, Test_Type, User) logger = logging.getLogger(__name__) @@ -53,6 +53,11 @@ def create_product_type(self, name, *args, description='dummy description', **kw product_type.save() return product_type + def create_sla_configuration(self, name, *args, description='dummy description', critical=7, high=30, medium=60, low=120, **kwargs): + sla_configuration = SLA_Configuration(name=name, description=description, critical=critical, high=high, medium=medium, low=low) + sla_configuration.save() + return sla_configuration + def create_product(self, name, *args, description='dummy description', prod_type=None, tags=[], **kwargs): if not prod_type: prod_type = Product_Type.objects.first() diff --git a/unittests/test_finding_model.py b/unittests/test_finding_model.py index ca7494142e..e6053dcd91 100644 --- a/unittests/test_finding_model.py +++ b/unittests/test_finding_model.py @@ -1,5 +1,7 @@ from .dojo_test_case import DojoTestCase -from dojo.models import Finding, Test, Engagement, DojoMeta +from dojo.models import User, Finding, Test, Engagement, DojoMeta +from datetime import datetime, timedelta +from crum import impersonate class TestFindingModel(DojoTestCase): @@ -262,3 +264,147 @@ def test_get_references_with_links_markdown(self): finding = Finding() finding.references = 'URL: [https://www.example.com](https://www.example.com)' self.assertEqual('URL: [https://www.example.com](https://www.example.com)', finding.get_references_with_links()) + + +class TestFindingSLAExpiration(DojoTestCase): + fixtures = ['dojo_testdata.json'] + + def run(self, result=None): + testuser = User.objects.get(username='admin') + testuser.usercontactinfo.block_execution = True + testuser.save() + + # unit tests are running without any user, which will result in actions like dedupe happening in the celery process + # this doesn't work in unittests as unittests are using an in memory sqlite database and celery can't see the data + # so we're running the test under the admin user context and set block_execution to True + with impersonate(testuser): + super().run(result) + + def test_sla_expiration_date(self): + """ + tests if the SLA expiration date and SLA days remaining are calculated correctly + after a finding's severity is updated + """ + user, _ = User.objects.get_or_create(username='admin') + product_type = self.create_product_type('test_product_type') + sla_config = self.create_sla_configuration(name='test_sla_config') + product = self.create_product(name='test_product', prod_type=product_type) + product.sla_configuration = sla_config + product.save() + engagement = self.create_engagement('test_eng', product) + test = self.create_test(engagement=engagement, scan_type='ZAP Scan', title='test_test') + finding = Finding.objects.create( + test=test, + reporter=user, + title='test_finding', + severity='Critical', + date=datetime.now().date()) + finding.set_sla_expiration_date() + + expected_sla_days = getattr(product.sla_configuration, finding.severity.lower(), None) + self.assertEqual(finding.sla_expiration_date, datetime.now().date() + timedelta(days=expected_sla_days)) + self.assertEqual(finding.sla_days_remaining(), expected_sla_days) + + def test_sla_expiration_date_after_finding_severity_updated(self): + """ + tests if the SLA expiration date and SLA days remaining are calculated correctly + after a finding's severity is updated + """ + user, _ = User.objects.get_or_create(username='admin') + product_type = self.create_product_type('test_product_type') + sla_config = self.create_sla_configuration(name='test_sla_config') + product = self.create_product(name='test_product', prod_type=product_type) + product.sla_configuration = sla_config + product.save() + engagement = self.create_engagement('test_eng', product) + test = self.create_test(engagement=engagement, scan_type='ZAP Scan', title='test_test') + finding = Finding.objects.create( + test=test, + reporter=user, + title='test_finding', + severity='Critical', + date=datetime.now().date()) + finding.set_sla_expiration_date() + + expected_sla_days = getattr(product.sla_configuration, finding.severity.lower(), None) + self.assertEqual(finding.sla_expiration_date, datetime.now().date() + timedelta(days=expected_sla_days)) + self.assertEqual(finding.sla_days_remaining(), expected_sla_days) + + finding.severity = 'Medium' + finding.set_sla_expiration_date() + + expected_sla_days = getattr(product.sla_configuration, finding.severity.lower(), None) + self.assertEqual(finding.sla_expiration_date, datetime.now().date() + timedelta(days=expected_sla_days)) + self.assertEqual(finding.sla_days_remaining(), expected_sla_days) + + def test_sla_expiration_date_after_product_updated(self): + """ + tests if the SLA expiration date and SLA days remaining are calculated correctly + after a product changed from one SLA configuration to another + """ + user, _ = User.objects.get_or_create(username='admin') + product_type = self.create_product_type('test_product_type') + sla_config_1 = self.create_sla_configuration(name='test_sla_config_1') + sla_config_2 = self.create_sla_configuration( + name='test_sla_config_2', + critical=1, + high=2, + medium=3, + low=4) + product = self.create_product(name='test_product', prod_type=product_type) + product.sla_configuration = sla_config_1 + product.save() + engagement = self.create_engagement('test_eng', product) + test = self.create_test(engagement=engagement, scan_type='ZAP Scan', title='test_test') + finding = Finding.objects.create( + test=test, + reporter=user, + title='test_finding', + severity='Critical', + date=datetime.now().date()) + + expected_sla_days = getattr(product.sla_configuration, finding.severity.lower(), None) + self.assertEqual(finding.sla_expiration_date, datetime.now().date() + timedelta(days=expected_sla_days)) + self.assertEqual(finding.sla_days_remaining(), expected_sla_days) + + product.sla_configuration = sla_config_2 + product.save() + + finding.set_sla_expiration_date() + + expected_sla_days = getattr(product.sla_configuration, finding.severity.lower(), None) + self.assertEqual(finding.sla_expiration_date, datetime.now().date() + timedelta(days=expected_sla_days)) + self.assertEqual(finding.sla_days_remaining(), expected_sla_days) + + def test_sla_expiration_date_after_sla_configuration_updated(self): + """ + tests if the SLA expiration date and SLA days remaining are calculated correctly + after the SLA configuration on a product was updated to a different number of SLA days + """ + user, _ = User.objects.get_or_create(username='admin') + product_type = self.create_product_type('test_product_type') + sla_config = self.create_sla_configuration(name='test_sla_config') + product = self.create_product(name='test_product', prod_type=product_type) + product.sla_configuration = sla_config + product.save() + engagement = self.create_engagement('test_eng', product) + test = self.create_test(engagement=engagement, scan_type='ZAP Scan', title='test_test') + finding = Finding.objects.create( + test=test, + reporter=user, + title='test_finding', + severity='Critical', + date=datetime.now().date()) + + expected_sla_days = getattr(product.sla_configuration, finding.severity.lower(), None) + self.assertEqual(finding.sla_expiration_date, datetime.now().date() + timedelta(days=expected_sla_days)) + self.assertEqual(finding.sla_days_remaining(), expected_sla_days) + + sla_config.critical = 10 + sla_config.save() + + finding.set_sla_expiration_date() + + expected_sla_days = getattr(product.sla_configuration, finding.severity.lower(), None) + self.assertEqual(finding.sla_expiration_date, datetime.now().date() + timedelta(days=expected_sla_days)) + self.assertEqual(finding.sla_days_remaining(), expected_sla_days) diff --git a/unittests/tools/test_veracode_parser.py b/unittests/tools/test_veracode_parser.py index 878629e6b7..55799e9cf8 100644 --- a/unittests/tools/test_veracode_parser.py +++ b/unittests/tools/test_veracode_parser.py @@ -156,8 +156,6 @@ def parse_file_with_mitigated_finding(self): self.assertEqual(datetime.datetime(2020, 6, 1, 10, 2, 1), finding.mitigated) self.assertEqual("app-1234_issue-1", finding.unique_id_from_tool) self.assertEqual(0, finding.sla_age) - self.assertEqual(90, finding.sla_days_remaining()) - self.assertEqual((datetime.datetime(2020, 6, 1, 10, 2, 1) + datetime.timedelta(days=90)).date(), finding.sla_deadline()) @override_settings(USE_FIRST_SEEN=True) def test_parse_file_with_mitigated_fixed_finding_first_seen(self): From aaabbabcd4fa0c29f662f4b53fa2f350b05194a4 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 5 Feb 2024 23:05:03 +0000 Subject: [PATCH 9/9] Update dependency ruff from 0.1.15 to v0.2.1 (requirements-lint.txt) --- requirements-lint.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-lint.txt b/requirements-lint.txt index 4c4b9dda19..96ae82b23a 100644 --- a/requirements-lint.txt +++ b/requirements-lint.txt @@ -1 +1 @@ -ruff==0.1.15 \ No newline at end of file +ruff==0.2.1 \ No newline at end of file