From 7dd3f9adbe3fd70673821d17c997f8c63ad8b691 Mon Sep 17 00:00:00 2001 From: hblankenship Date: Fri, 11 Oct 2024 12:27:41 -0500 Subject: [PATCH 1/6] get or create environment --- dojo/api_v2/serializers.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/dojo/api_v2/serializers.py b/dojo/api_v2/serializers.py index 471dfc019b..32c8b2414f 100644 --- a/dojo/api_v2/serializers.py +++ b/dojo/api_v2/serializers.py @@ -2175,9 +2175,12 @@ def set_context( context = dict(data) # update some vars context["scan"] = data.pop("file", None) - context["environment"] = Development_Environment.objects.get( + # The assumption below is that a Development environment will be gotten or created, whether of the one provided or the Development environment + # because the Development environment always exists... + # If this fails, there are bigger problems. + context["environment"] = Development_Environment.objects.get_or_create( name=data.get("environment", "Development"), - ) + )[0] # Set the active/verified status based upon the overrides if "active" in self.initial_data: context["active"] = data.get("active") @@ -2454,9 +2457,12 @@ def set_context( context = dict(data) # update some vars context["scan"] = data.get("file", None) - context["environment"] = Development_Environment.objects.get( + # The assumption below is that a Development environment will be gotten or created, whether of the one provided or the Development environment + # because the Development environment always exists... + # If this fails, there are bigger problems. + context["environment"] = Development_Environment.objects.get_or_create( name=data.get("environment", "Development"), - ) + )[0] # Set the active/verified status based upon the overrides if "active" in self.initial_data: context["active"] = data.get("active") From 42aeb9fb9e7a1a4a915e97ab9812047cb372d852 Mon Sep 17 00:00:00 2001 From: hblankenship Date: Fri, 11 Oct 2024 12:58:31 -0500 Subject: [PATCH 2/6] honor auto_create_context, update docs --- docs/content/en/integrations/importing.md | 2 +- dojo/api_v2/serializers.py | 35 +++++++++++++++-------- 2 files changed, 24 insertions(+), 13 deletions(-) diff --git a/docs/content/en/integrations/importing.md b/docs/content/en/integrations/importing.md index 20590ee1f7..127f642932 100644 --- a/docs/content/en/integrations/importing.md +++ b/docs/content/en/integrations/importing.md @@ -69,7 +69,7 @@ An import can be performed by specifying the names of these entities in the API } ``` -When `auto_create_context` is `True`, the product and engagement will be created if needed. Make sure your user has sufficient [permissions](../usage/permissions) to do this. +When `auto_create_context` is `True`, the product, engagement, and environment will be created if needed. Make sure your user has sufficient [permissions](../usage/permissions) to do this. A classic way of importing a scan is by specifying the ID of the engagement instead: diff --git a/dojo/api_v2/serializers.py b/dojo/api_v2/serializers.py index 32c8b2414f..b937d96cd0 100644 --- a/dojo/api_v2/serializers.py +++ b/dojo/api_v2/serializers.py @@ -2175,12 +2175,17 @@ def set_context( context = dict(data) # update some vars context["scan"] = data.pop("file", None) - # The assumption below is that a Development environment will be gotten or created, whether of the one provided or the Development environment - # because the Development environment always exists... - # If this fails, there are bigger problems. - context["environment"] = Development_Environment.objects.get_or_create( - name=data.get("environment", "Development"), - )[0] + + if context.get("auto_create_context"): + environment = Development_Environment.objects.get_or_create(name=data.get("environment"))[0] + else: + try: + environment = Development_Environment.objects.get(name=data.get("environment", "Development")) + except: + msg = "Environment named " + data.get("environment") + " does not exist." + raise ValidationError(msg) + + context["environment"] = environment # Set the active/verified status based upon the overrides if "active" in self.initial_data: context["active"] = data.get("active") @@ -2457,12 +2462,18 @@ def set_context( context = dict(data) # update some vars context["scan"] = data.get("file", None) - # The assumption below is that a Development environment will be gotten or created, whether of the one provided or the Development environment - # because the Development environment always exists... - # If this fails, there are bigger problems. - context["environment"] = Development_Environment.objects.get_or_create( - name=data.get("environment", "Development"), - )[0] + + if context.get("auto_create_context"): + environment = Development_Environment.objects.get_or_create(name=data.get("environment"))[0] + else: + try: + environment = Development_Environment.objects.get(name=data.get("environment", "Development")) + except: + msg = "Environment named " + data.get("environment") + " does not exist." + raise ValidationError(msg) + + context["environment"] = environment + # Set the active/verified status based upon the overrides if "active" in self.initial_data: context["active"] = data.get("active") From 13c179d39d99b3aff311b5329d8c1b39f29350af Mon Sep 17 00:00:00 2001 From: hblankenship Date: Fri, 11 Oct 2024 13:56:09 -0500 Subject: [PATCH 3/6] case of not providing environment --- dojo/api_v2/serializers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dojo/api_v2/serializers.py b/dojo/api_v2/serializers.py index b937d96cd0..48395f748f 100644 --- a/dojo/api_v2/serializers.py +++ b/dojo/api_v2/serializers.py @@ -2177,7 +2177,7 @@ def set_context( context["scan"] = data.pop("file", None) if context.get("auto_create_context"): - environment = Development_Environment.objects.get_or_create(name=data.get("environment"))[0] + environment = Development_Environment.objects.get_or_create(name=data.get("environment", "Development"))[0] else: try: environment = Development_Environment.objects.get(name=data.get("environment", "Development")) @@ -2464,7 +2464,7 @@ def set_context( context["scan"] = data.get("file", None) if context.get("auto_create_context"): - environment = Development_Environment.objects.get_or_create(name=data.get("environment"))[0] + environment = Development_Environment.objects.get_or_create(name=data.get("environment", "Development"))[0] else: try: environment = Development_Environment.objects.get(name=data.get("environment", "Development")) From 20eee47f4c4ae2ac8ddf0e266316acb56934d673 Mon Sep 17 00:00:00 2001 From: hblankenship Date: Tue, 15 Oct 2024 17:41:20 -0500 Subject: [PATCH 4/6] create base class, re-use code import, reimport --- dojo/api_v2/serializers.py | 319 ++++++++++--------------------------- 1 file changed, 86 insertions(+), 233 deletions(-) diff --git a/dojo/api_v2/serializers.py b/dojo/api_v2/serializers.py index 48395f748f..368a265735 100644 --- a/dojo/api_v2/serializers.py +++ b/dojo/api_v2/serializers.py @@ -2047,7 +2047,7 @@ def get_findings_list(self, obj) -> List[int]: return obj.open_findings_list -class ImportScanSerializer(serializers.Serializer): +class CommonImportScanSerializer(serializers.Serializer): scan_date = serializers.DateField( required=False, help_text="Scan completion date will be used on all findings.", @@ -2064,6 +2064,7 @@ class ImportScanSerializer(serializers.Serializer): verified = serializers.BooleanField( help_text="Override the verified setting from the tool.", ) + scan_type = serializers.ChoiceField(choices=get_choices_sorted()) # TODO: why do we allow only existing endpoints? endpoint_to_add = serializers.PrimaryKeyRelatedField( @@ -2085,9 +2086,7 @@ class ImportScanSerializer(serializers.Serializer): required=False, help_text="Resource link to source code", ) - engagement = serializers.PrimaryKeyRelatedField( - queryset=Engagement.objects.all(), required=False, - ) + test_title = serializers.CharField(required=False) auto_create_context = serializers.BooleanField(required=False) deduplication_on_engagement = serializers.BooleanField(required=False) @@ -2147,9 +2146,6 @@ class ImportScanSerializer(serializers.Serializer): # extra fields populated in response # need to use the _id suffix as without the serializer framework gets # confused - test = serializers.IntegerField( - read_only=True, - ) # left for backwards compatibility test_id = serializers.IntegerField(read_only=True) engagement_id = serializers.IntegerField(read_only=True) product_id = serializers.IntegerField(read_only=True) @@ -2164,6 +2160,88 @@ class ImportScanSerializer(serializers.Serializer): required=False, ) + def get_importer( + self, + **kwargs: dict, + ) -> BaseImporter: + """ + Returns a new instance of an importer that extends + the BaseImporter class + """ + return DefaultImporter(**kwargs) + + def process_scan( + self, + data: dict, + context: dict, + ) -> None: + """ + Process the scan with all of the supplied data fully massaged + into the format we are expecting + + Raises exceptions in the event of an error + """ + try: + importer = self.get_importer(**context) + context["test"], _, _, _, _, _, _ = importer.process_scan( + context.pop("scan", None), + ) + # Update the response body with some new data + if test := context.get("test"): + data["test"] = test.id + data["test_id"] = test.id + data["engagement_id"] = test.engagement.id + data["product_id"] = test.engagement.product.id + data["product_type_id"] = test.engagement.product.prod_type.id + data["statistics"] = {"after": test.statistics} + # convert to exception otherwise django rest framework will swallow them as 400 error + # exceptions are already logged in the importer + except SyntaxError as se: + raise Exception(se) + except ValueError as ve: + raise Exception(ve) + + def validate(self, data: dict) -> dict: + scan_type = data.get("scan_type") + file = data.get("file") + if not file and requires_file(scan_type): + msg = f"Uploading a Report File is required for {scan_type}" + raise serializers.ValidationError(msg) + if file and is_scan_file_too_large(file): + msg = f"Report file is too large. Maximum supported size is {settings.SCAN_FILE_MAX_SIZE} MB" + raise serializers.ValidationError(msg) + tool_type = requires_tool_type(scan_type) + if tool_type: + api_scan_configuration = data.get("api_scan_configuration") + if ( + api_scan_configuration + and tool_type + != api_scan_configuration.tool_configuration.tool_type.name + ): + msg = f"API scan configuration must be of tool type {tool_type}" + raise serializers.ValidationError(msg) + return data + + def validate_scan_date(self, value: str) -> None: + if value and value > timezone.localdate(): + msg = "The scan_date cannot be in the future!" + raise serializers.ValidationError(msg) + return value + + +class ImportScanSerializer(CommonImportScanSerializer): + + engagement = serializers.PrimaryKeyRelatedField( + queryset=Engagement.objects.all(), required=False, + ) + + # extra fields populated in response + # need to use the _id suffix as without the serializer framework gets + # confused + test = serializers.IntegerField( + read_only=True, + ) # left for backwards compatibility + def set_context( self, data: dict, @@ -2242,47 +2320,6 @@ def process_auto_create_create_context( # Raise an explicit drf exception here raise ValidationError(str(e)) - def get_importer( - self, - **kwargs: dict, - ) -> BaseImporter: - """ - Returns a new instance of an importer that extends - the BaseImporter class - """ - return DefaultImporter(**kwargs) - - def process_scan( - self, - data: dict, - context: dict, - ) -> None: - """ - Process the scan with all of the supplied data fully massaged - into the format we are expecting - - Raises exceptions in the event of an error - """ - try: - importer = self.get_importer(**context) - context["test"], _, _, _, _, _, _ = importer.process_scan( - context.pop("scan", None), - ) - # Update the response body with some new data - if test := context.get("test"): - data["test"] = test.id - data["test_id"] = test.id - data["engagement_id"] = test.engagement.id - data["product_id"] = test.engagement.product.id - data["product_type_id"] = test.engagement.product.prod_type.id - data["statistics"] = {"after": test.statistics} - # convert to exception otherwise django rest framework will swallow them as 400 error - # exceptions are already logged in the importer - except SyntaxError as se: - raise Exception(se) - except ValueError as ve: - raise Exception(ve) - def save(self, push_to_jira=False): # Go through the validate method data = self.validated_data @@ -2293,163 +2330,16 @@ def save(self, push_to_jira=False): # Import the scan with all of the supplied data self.process_scan(data, context) - def validate(self, data: dict) -> dict: - scan_type = data.get("scan_type") - file = data.get("file") - if not file and requires_file(scan_type): - msg = f"Uploading a Report File is required for {scan_type}" - raise serializers.ValidationError(msg) - if file and is_scan_file_too_large(file): - msg = f"Report file is too large. Maximum supported size is {settings.SCAN_FILE_MAX_SIZE} MB" - raise serializers.ValidationError(msg) - tool_type = requires_tool_type(scan_type) - if tool_type: - api_scan_configuration = data.get("api_scan_configuration") - if ( - api_scan_configuration - and tool_type - != api_scan_configuration.tool_configuration.tool_type.name - ): - msg = f"API scan configuration must be of tool type {tool_type}" - raise serializers.ValidationError(msg) - return data - - def validate_scan_date(self, value: str) -> None: - if value and value > timezone.localdate(): - msg = "The scan_date cannot be in the future!" - raise serializers.ValidationError(msg) - return value +class ReImportScanSerializer(TaggitSerializer, CommonImportScanSerializer): -class ReImportScanSerializer(TaggitSerializer, serializers.Serializer): - scan_date = serializers.DateField( - required=False, - help_text="Scan completion date will be used on all findings.", - ) - minimum_severity = serializers.ChoiceField( - choices=SEVERITY_CHOICES, - default="Info", - help_text="Minimum severity level to be imported", - ) - active = serializers.BooleanField( - help_text="Override the active setting from the tool.", - ) - verified = serializers.BooleanField( - help_text="Override the verified setting from the tool.", - ) help_do_not_reactivate = "Select if the import should ignore active findings from the report, useful for triage-less scanners. Will keep existing findings closed, without reactivating them. For more information check the docs." do_not_reactivate = serializers.BooleanField( default=False, required=False, help_text=help_do_not_reactivate, ) - scan_type = serializers.ChoiceField( - choices=get_choices_sorted(), required=True, - ) - endpoint_to_add = serializers.PrimaryKeyRelatedField( - queryset=Endpoint.objects.all(), - required=False, - default=None, - help_text="Enter the ID of an Endpoint that is associated with the target Product. New Findings will be added to that Endpoint.", - ) - file = serializers.FileField(allow_empty_file=True, required=False) - product_type_name = serializers.CharField(required=False) - product_name = serializers.CharField(required=False) - engagement_name = serializers.CharField(required=False) - engagement_end_date = serializers.DateField( - required=False, - help_text="End Date for Engagement. Default is current time + 365 days. Required format year-month-day", - ) - source_code_management_uri = serializers.URLField( - max_length=600, - required=False, - help_text="Resource link to source code", - ) test = serializers.PrimaryKeyRelatedField( required=False, queryset=Test.objects.all(), ) - test_title = serializers.CharField(required=False) - auto_create_context = serializers.BooleanField(required=False) - deduplication_on_engagement = serializers.BooleanField(required=False) - - push_to_jira = serializers.BooleanField(default=False) - # Close the old findings if the parameter is not provided. This is to - # mentain the old API behavior after reintroducing the close_old_findings parameter - # also for ReImport. - close_old_findings = serializers.BooleanField( - required=False, - default=True, - help_text="Select if old findings no longer present in the report get closed as mitigated when importing.", - ) - close_old_findings_product_scope = serializers.BooleanField( - required=False, - default=False, - help_text="Select if close_old_findings applies to all findings of the same type in the product. " - "By default, it is false meaning that only old findings of the same type in the engagement are in scope. " - "Note that this only applies on the first call to reimport-scan.", - ) - version = serializers.CharField( - required=False, - help_text="Version that will be set on existing Test object. Leave empty to leave existing value in place.", - ) - build_id = serializers.CharField( - required=False, help_text="ID of the build that was scanned.", - ) - branch_tag = serializers.CharField( - required=False, help_text="Branch or Tag that was scanned.", - ) - commit_hash = serializers.CharField( - required=False, help_text="Commit that was scanned.", - ) - api_scan_configuration = serializers.PrimaryKeyRelatedField( - allow_null=True, - default=None, - queryset=Product_API_Scan_Configuration.objects.all(), - ) - service = serializers.CharField( - required=False, - help_text="A service is a self-contained piece of functionality within a Product. " - "This is an optional field which is used in deduplication and closing of old findings when set. " - "This affects the whole engagement/product depending on your deduplication scope.", - ) - environment = serializers.CharField(required=False) - lead = serializers.PrimaryKeyRelatedField( - allow_null=True, default=None, queryset=User.objects.all(), - ) - tags = TagListSerializerField( - required=False, - allow_empty=True, - help_text="Modify existing tags that help describe this scan. (Existing test tags will be overwritten)", - ) - - group_by = serializers.ChoiceField( - required=False, - choices=Finding_Group.GROUP_BY_OPTIONS, - help_text="Choose an option to automatically group new findings by the chosen option.", - ) - create_finding_groups_for_all_findings = serializers.BooleanField( - help_text="If set to false, finding groups will only be created when there is more than one grouped finding", - required=False, - default=True, - ) - - # extra fields populated in response - # need to use the _id suffix as without the serializer framework gets - # confused - test_id = serializers.IntegerField(read_only=True) - engagement_id = serializers.IntegerField( - read_only=True, - ) # need to use the _id suffix as without the serializer framework gets confused - product_id = serializers.IntegerField(read_only=True) - product_type_id = serializers.IntegerField(read_only=True) - - statistics = ImportStatisticsSerializer(read_only=True, required=False) - apply_tags_to_findings = serializers.BooleanField( - help_text="If set to True, the tags will be applied to the findings", - required=False, - ) - apply_tags_to_endpoints = serializers.BooleanField( - help_text="If set to True, the tags will be applied to the endpoints", - required=False, - ) def set_context( self, @@ -2529,16 +2419,6 @@ def process_auto_create_create_context( # Raise an explicit drf exception here raise ValidationError(str(e)) - def get_importer( - self, - **kwargs: dict, - ) -> BaseImporter: - """ - Returns a new instance of an importer that extends - the BaseImporter class - """ - return DefaultImporter(**kwargs) - def get_reimporter( self, **kwargs: dict, @@ -2617,33 +2497,6 @@ def save(self, push_to_jira=False): # Import the scan with all of the supplied data self.process_scan(auto_create_manager, data, context) - def validate(self, data): - scan_type = data.get("scan_type") - file = data.get("file") - if not file and requires_file(scan_type): - msg = f"Uploading a Report File is required for {scan_type}" - raise serializers.ValidationError(msg) - if file and is_scan_file_too_large(file): - msg = f"Report file is too large. Maximum supported size is {settings.SCAN_FILE_MAX_SIZE} MB" - raise serializers.ValidationError(msg) - tool_type = requires_tool_type(scan_type) - if tool_type: - api_scan_configuration = data.get("api_scan_configuration") - if ( - api_scan_configuration - and tool_type - != api_scan_configuration.tool_configuration.tool_type.name - ): - msg = f"API scan configuration must be of tool type {tool_type}" - raise serializers.ValidationError(msg) - return data - - def validate_scan_date(self, value): - if value and value > timezone.localdate(): - msg = "The scan_date cannot be in the future!" - raise serializers.ValidationError(msg) - return value - class EndpointMetaImporterSerializer(serializers.Serializer): file = serializers.FileField(required=True) From b4af07c7c14fca36b6f2dadd504cba183f43ad69 Mon Sep 17 00:00:00 2001 From: hblankenship Date: Tue, 15 Oct 2024 18:14:59 -0500 Subject: [PATCH 5/6] put common context code in base --- dojo/api_v2/serializers.py | 91 ++++++++++---------------------------- 1 file changed, 23 insertions(+), 68 deletions(-) diff --git a/dojo/api_v2/serializers.py b/dojo/api_v2/serializers.py index 368a265735..82bef0aa8d 100644 --- a/dojo/api_v2/serializers.py +++ b/dojo/api_v2/serializers.py @@ -2228,24 +2228,7 @@ def validate_scan_date(self, value: str) -> None: raise serializers.ValidationError(msg) return value - -class ImportScanSerializer(CommonImportScanSerializer): - - engagement = serializers.PrimaryKeyRelatedField( - queryset=Engagement.objects.all(), required=False, - ) - - # extra fields populated in response - # need to use the _id suffix as without the serializer framework gets - # confused - test = serializers.IntegerField( - read_only=True, - ) # left for backwards compatibility - - def set_context( - self, - data: dict, - ) -> dict: + def setup_common_context(self, data: dict) -> dict: """ Process all of the user supplied inputs to massage them into the correct format the importer is expecting to see @@ -2294,6 +2277,27 @@ def set_context( if context.get("scan_date") else None ) + return context + + +class ImportScanSerializer(CommonImportScanSerializer): + + engagement = serializers.PrimaryKeyRelatedField( + queryset=Engagement.objects.all(), required=False, + ) + + # extra fields populated in response + # need to use the _id suffix as without the serializer framework gets + # confused + test = serializers.IntegerField( + read_only=True, + ) # left for backwards compatibility + + def set_context( + self, + data: dict, + ) -> dict: + context = self.setup_common_context(dict) # Process the auto create context inputs self.process_auto_create_create_context(context) @@ -2345,57 +2349,8 @@ def set_context( self, data: dict, ) -> dict: - """ - Process all of the user supplied inputs to massage them into the correct - format the importer is expecting to see - """ - context = dict(data) - # update some vars - context["scan"] = data.get("file", None) - if context.get("auto_create_context"): - environment = Development_Environment.objects.get_or_create(name=data.get("environment", "Development"))[0] - else: - try: - environment = Development_Environment.objects.get(name=data.get("environment", "Development")) - except: - msg = "Environment named " + data.get("environment") + " does not exist." - raise ValidationError(msg) - - context["environment"] = environment - - # Set the active/verified status based upon the overrides - if "active" in self.initial_data: - context["active"] = data.get("active") - else: - context["active"] = None - if "verified" in self.initial_data: - context["verified"] = data.get("verified") - else: - context["verified"] = None - # Change the way that endpoints are sent to the importer - if endpoints_to_add := data.get("endpoint_to_add"): - context["endpoints_to_add"] = [endpoints_to_add] - else: - context["endpoint_to_add"] = None - # Convert the tags to a list if needed. At this point, the - # TaggitListSerializer has already removed commas supplied - # by the user, so this operation will consistently return - # a list to be used by the importer - if tags := context.get("tags"): - if isinstance(tags, str): - context["tags"] = tags.split(", ") - # have to make the scan_date_time timezone aware otherwise uploads via - # the API would fail (but unit tests for api upload would pass...) - context["scan_date"] = ( - timezone.make_aware( - datetime.combine(context.get("scan_date"), datetime.min.time()), - ) - if context.get("scan_date") - else None - ) - - return context + return self.setup_common_context(data) def process_auto_create_create_context( self, From 4c33fd9fd063b95257e2b707d63db6a101232150 Mon Sep 17 00:00:00 2001 From: hblankenship Date: Tue, 15 Oct 2024 18:20:27 -0500 Subject: [PATCH 6/6] mistyped dict for data --- dojo/api_v2/serializers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dojo/api_v2/serializers.py b/dojo/api_v2/serializers.py index 82bef0aa8d..e769b15b98 100644 --- a/dojo/api_v2/serializers.py +++ b/dojo/api_v2/serializers.py @@ -2297,7 +2297,7 @@ def set_context( self, data: dict, ) -> dict: - context = self.setup_common_context(dict) + context = self.setup_common_context(data) # Process the auto create context inputs self.process_auto_create_create_context(context)