From a9f8ee890a849dc50b82960144e8dc7d950e12bd Mon Sep 17 00:00:00 2001 From: Mithilesh Pawar <61288229+MithileshPawar@users.noreply.github.com> Date: Wed, 12 Aug 2020 10:28:48 -0400 Subject: [PATCH] Bitbucket (Cloud & Server) Support for Config-as-Code and Auto Profiling. (#380) * Added 'Config as code' support for Bitbucket Cloud & Bitbucket Server. * Added support for Auto Profiling within Bitbucket Cloud & Bitbucket Server. --- .../controller/BitbucketCloudController.java | 82 +++-- .../controller/BitbucketServerController.java | 94 +++-- .../checkmarx/flow/dto/bitbucket/Content.java | 67 ++++ .../checkmarx/flow/dto/bitbucket/History.java | 26 ++ .../checkmarx/flow/dto/bitbucket/Links.java | 28 +- .../checkmarx/flow/dto/bitbucket/Meta.java | 26 ++ .../checkmarx/flow/dto/bitbucket/Value.java | 119 ++++++ .../flow/dto/bitbucketserver/Children.java | 80 +++++ .../flow/dto/bitbucketserver/Content.java | 51 +++ .../flow/dto/bitbucketserver/Path.java | 80 +++++ .../flow/dto/bitbucketserver/Value.java | 78 ++++ .../sastscanning/ScanRequestConverter.java | 6 +- .../flow/service/BitBucketService.java | 339 ++++++++++++++---- .../checkmarx/flow/service/HelperService.java | 2 +- .../deletebranch/DeleteBranchSteps.java | 85 ++--- 15 files changed, 967 insertions(+), 196 deletions(-) create mode 100644 src/main/java/com/checkmarx/flow/dto/bitbucket/Content.java create mode 100644 src/main/java/com/checkmarx/flow/dto/bitbucket/History.java create mode 100644 src/main/java/com/checkmarx/flow/dto/bitbucket/Meta.java create mode 100644 src/main/java/com/checkmarx/flow/dto/bitbucket/Value.java create mode 100644 src/main/java/com/checkmarx/flow/dto/bitbucketserver/Children.java create mode 100644 src/main/java/com/checkmarx/flow/dto/bitbucketserver/Content.java create mode 100644 src/main/java/com/checkmarx/flow/dto/bitbucketserver/Path.java create mode 100644 src/main/java/com/checkmarx/flow/dto/bitbucketserver/Value.java diff --git a/src/main/java/com/checkmarx/flow/controller/BitbucketCloudController.java b/src/main/java/com/checkmarx/flow/controller/BitbucketCloudController.java index 534811051..c02fcc8bc 100644 --- a/src/main/java/com/checkmarx/flow/controller/BitbucketCloudController.java +++ b/src/main/java/com/checkmarx/flow/controller/BitbucketCloudController.java @@ -3,20 +3,21 @@ import com.checkmarx.flow.config.BitBucketProperties; import com.checkmarx.flow.config.FlowProperties; import com.checkmarx.flow.config.JiraProperties; -import com.checkmarx.flow.dto.*; +import com.checkmarx.flow.dto.BugTracker; +import com.checkmarx.flow.dto.ControllerRequest; +import com.checkmarx.flow.dto.EventResponse; +import com.checkmarx.flow.dto.ScanRequest; import com.checkmarx.flow.dto.bitbucket.*; import com.checkmarx.flow.exception.InvalidTokenException; -import com.checkmarx.flow.service.ConfigurationOverrider; -import com.checkmarx.flow.service.FilterFactory; -import com.checkmarx.flow.service.FlowService; -import com.checkmarx.flow.service.HelperService; +import com.checkmarx.flow.service.*; import com.checkmarx.flow.utils.HTMLHelper; import com.checkmarx.flow.utils.ScanUtils; import com.checkmarx.sdk.config.Constants; import com.checkmarx.sdk.config.CxProperties; +import com.checkmarx.sdk.dto.CxConfig; import com.checkmarx.sdk.dto.filtering.FilterConfiguration; import lombok.RequiredArgsConstructor; -import org.slf4j.Logger; +import lombok.extern.slf4j.Slf4j; import org.slf4j.MDC; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.*; @@ -24,16 +25,15 @@ import java.util.ArrayList; import java.util.List; import java.util.Locale; - +@Slf4j @RequiredArgsConstructor @RestController -@RequestMapping(value = "/" ) +@RequestMapping(value = "/") public class BitbucketCloudController extends WebhookController { private static final String EVENT = "X-Event-Key"; private static final String PUSH = EVENT + "=repo:push"; private static final String MERGE = EVENT + "=pullrequest:created"; - private static final Logger log = org.slf4j.LoggerFactory.getLogger(BitbucketCloudController.class); private final FlowProperties flowProperties; private final BitBucketProperties properties; @@ -41,6 +41,7 @@ public class BitbucketCloudController extends WebhookController { private final JiraProperties jiraProperties; private final FlowService flowService; private final HelperService helperService; + private final BitBucketService bitbucketService; private final FilterFactory filterFactory; private final ConfigurationOverrider configOverrider; @@ -54,18 +55,17 @@ public ResponseEntity pushRequest( ControllerRequest controllerRequest, @RequestParam(value = "token") String token - ){ + ) { String uid = helperService.getShortUid(); MDC.put("cx", uid); validateBitBucketRequest(token); log.info("Processing BitBucket MERGE request"); - FlowOverride o = ScanUtils.getMachinaOverride(controllerRequest.getOverride()); controllerRequest = ensureNotNull(controllerRequest); try { Repository repository = body.getRepository(); String app = repository.getName(); - if(!ScanUtils.empty(controllerRequest.getApplication())){ + if (!ScanUtils.empty(controllerRequest.getApplication())) { app = controllerRequest.getApplication(); } @@ -74,11 +74,11 @@ public ResponseEntity pushRequest( bugType = ScanUtils.getBugTypeEnum(controllerRequest.getBug(), flowProperties.getBugTrackerImpl()); } - if(controllerRequest.getAppOnly() != null){ + if (controllerRequest.getAppOnly() != null) { flowProperties.setTrackApplicationOnly(controllerRequest.getAppOnly()); } - if(ScanUtils.empty(product)){ + if (ScanUtils.empty(product)) { product = ScanRequest.Product.CX.getProduct(); } ScanRequest.Product p = ScanRequest.Product.valueOf(product.toUpperCase(Locale.ROOT)); @@ -86,6 +86,7 @@ public ResponseEntity pushRequest( String currentBranch = pullRequest.getSource().getBranch().getName(); String targetBranch = pullRequest.getDestination().getBranch().getName(); List branches = getBranches(controllerRequest, flowProperties); + String hash = pullRequest.getSource().getCommit().getHash(); BugTracker bt = ScanUtils.getBugTracker(controllerRequest.getAssignee(), bugType, jiraProperties, controllerRequest.getBug()); @@ -97,7 +98,7 @@ public ResponseEntity pushRequest( String mergeEndpoint = pullRequest.getLinks().getComments().getHref(); String scanPreset = cxProperties.getScanPreset(); - if(!ScanUtils.empty(controllerRequest.getPreset())){ + if (!ScanUtils.empty(controllerRequest.getPreset())) { scanPreset = controllerRequest.getPreset(); } @@ -106,7 +107,7 @@ public ResponseEntity pushRequest( .product(p) .project(controllerRequest.getProject()) .team(controllerRequest.getTeam()) - .namespace(repository.getOwner().getDisplayName().replace(" ","_")) + .namespace(repository.getOwner().getDisplayName().replace(" ", "_")) .repoName(repository.getName()) .repoUrl(gitUrl) .repoUrlWithAuth(gitUrl.replace(Constants.HTTPS, Constants.HTTPS.concat(properties.getToken()).concat("@"))) @@ -122,10 +123,11 @@ public ResponseEntity pushRequest( .excludeFiles(controllerRequest.getExcludeFiles()) .bugTracker(bt) .filter(filter) + .hash(hash) .build(); - request = configOverrider.overrideScanRequestProperties(o, request); - request.putAdditionalMetadata(HTMLHelper.WEB_HOOK_PAYLOAD, body.toString()); + fillRequestWithAdditionalData(request, repository, body.toString()); + checkForConfigAsCode(request); request.setId(uid); if (helperService.isBranch2Scan(request, branches)) { @@ -149,18 +151,16 @@ public ResponseEntity pushRequest( ControllerRequest controllerRequest, @RequestParam(value = "token") String token - ){ + ) { String uid = helperService.getShortUid(); MDC.put("cx", uid); validateBitBucketRequest(token); controllerRequest = ensureNotNull(controllerRequest); - FlowOverride o = ScanUtils.getMachinaOverride(controllerRequest.getOverride()); - try { Repository repository = body.getRepository(); String app = repository.getName(); - if(!ScanUtils.empty(controllerRequest.getApplication())){ + if (!ScanUtils.empty(controllerRequest.getApplication())) { app = controllerRequest.getApplication(); } @@ -168,17 +168,18 @@ public ResponseEntity pushRequest( setBugTracker(flowProperties, controllerRequest); BugTracker.Type bugType = ScanUtils.getBugTypeEnum(controllerRequest.getBug(), flowProperties.getBugTrackerImpl()); - if(controllerRequest.getAppOnly() != null){ + if (controllerRequest.getAppOnly() != null) { flowProperties.setTrackApplicationOnly(controllerRequest.getAppOnly()); } - if(ScanUtils.empty(product)){ + if (ScanUtils.empty(product)) { product = ScanRequest.Product.CX.getProduct(); } ScanRequest.Product p = ScanRequest.Product.valueOf(product.toUpperCase(Locale.ROOT)); - List changeList = body.getPush().getChanges(); + List changeList = body.getPush().getChanges(); String currentBranch = changeList.get(0).getNew().getName(); List branches = getBranches(controllerRequest, flowProperties); + String hash = changeList.get(0).getNew().getTarget().getHash(); BugTracker bt = ScanUtils.getBugTracker(controllerRequest.getAssignee(), bugType, jiraProperties, controllerRequest.getBug()); @@ -189,10 +190,10 @@ public ResponseEntity pushRequest( /*Determine emails*/ List emails = new ArrayList<>(); - for(Change ch: changeList){ - for(Commit c: ch.getCommits()){ + for (Change ch : changeList) { + for (Commit c : ch.getCommits()) { String author = c.getAuthor().getRaw(); - if(!ScanUtils.empty(author)){ + if (!ScanUtils.empty(author)) { emails.add(author); } } @@ -201,7 +202,7 @@ public ResponseEntity pushRequest( String gitUrl = repository.getLinks().getHtml().getHref().concat(".git"); String scanPreset = cxProperties.getScanPreset(); - if(!ScanUtils.empty(controllerRequest.getPreset())){ + if (!ScanUtils.empty(controllerRequest.getPreset())) { scanPreset = controllerRequest.getPreset(); } @@ -210,7 +211,7 @@ public ResponseEntity pushRequest( .product(p) .project(controllerRequest.getProject()) .team(controllerRequest.getTeam()) - .namespace(repository.getOwner().getDisplayName().replace(" ","_")) + .namespace(repository.getOwner().getDisplayName().replace(" ", "_")) .repoName(repository.getName()) .repoUrl(gitUrl) .repoUrlWithAuth(gitUrl.replace(Constants.HTTPS, Constants.HTTPS.concat(properties.getToken()).concat("@"))) @@ -224,10 +225,11 @@ public ResponseEntity pushRequest( .excludeFiles(controllerRequest.getExcludeFiles()) .bugTracker(bt) .filter(filter) + .hash(hash) .build(); - request = configOverrider.overrideScanRequestProperties(o, request); - request.putAdditionalMetadata(HTMLHelper.WEB_HOOK_PAYLOAD, body.toString()); + fillRequestWithAdditionalData(request, repository, body.toString()); + checkForConfigAsCode(request); request.setId(uid); if (helperService.isBranch2Scan(request, branches)) { @@ -242,12 +244,24 @@ public ResponseEntity pushRequest( /** * Token/Credential validation */ - private void validateBitBucketRequest(String token){ + private void validateBitBucketRequest(String token) { log.info("Validating BitBucket request token"); - if(!properties.getWebhookToken().equals(token)){ + if (!properties.getWebhookToken().equals(token)) { log.error("BitBucket request token validation failed"); throw new InvalidTokenException(); } log.info("Validation successful"); } + + private void checkForConfigAsCode(ScanRequest request) { + CxConfig cxConfig = bitbucketService.getCxConfigOverride(request); + configOverrider.overrideScanRequestProperties(cxConfig, request); + } + + private void fillRequestWithAdditionalData(ScanRequest request, Repository repository, String hookPayload) { + String repoSelfUrl = repository.getLinks().getSelf().getHref(); + request.putAdditionalMetadata(BitBucketService.REPO_SELF_URL, repoSelfUrl); + request.putAdditionalMetadata(HTMLHelper.WEB_HOOK_PAYLOAD, hookPayload); + } + } diff --git a/src/main/java/com/checkmarx/flow/controller/BitbucketServerController.java b/src/main/java/com/checkmarx/flow/controller/BitbucketServerController.java index 904b3c974..e646b669b 100644 --- a/src/main/java/com/checkmarx/flow/controller/BitbucketServerController.java +++ b/src/main/java/com/checkmarx/flow/controller/BitbucketServerController.java @@ -3,18 +3,19 @@ import com.checkmarx.flow.config.BitBucketProperties; import com.checkmarx.flow.config.FlowProperties; import com.checkmarx.flow.config.JiraProperties; -import com.checkmarx.flow.dto.*; +import com.checkmarx.flow.dto.BugTracker; +import com.checkmarx.flow.dto.ControllerRequest; +import com.checkmarx.flow.dto.EventResponse; +import com.checkmarx.flow.dto.ScanRequest; import com.checkmarx.flow.dto.bitbucketserver.*; import com.checkmarx.flow.exception.InvalidTokenException; import com.checkmarx.flow.exception.MachinaRuntimeException; -import com.checkmarx.flow.service.ConfigurationOverrider; -import com.checkmarx.flow.service.FilterFactory; -import com.checkmarx.flow.service.FlowService; -import com.checkmarx.flow.service.HelperService; +import com.checkmarx.flow.service.*; import com.checkmarx.flow.utils.HTMLHelper; import com.checkmarx.flow.utils.ScanUtils; import com.checkmarx.sdk.config.Constants; import com.checkmarx.sdk.config.CxProperties; +import com.checkmarx.sdk.dto.CxConfig; import com.checkmarx.sdk.dto.filtering.FilterConfiguration; import com.fasterxml.jackson.databind.ObjectMapper; import lombok.RequiredArgsConstructor; @@ -39,7 +40,7 @@ @RestController -@RequestMapping(value = "/" ) +@RequestMapping(value = "/") @RequiredArgsConstructor public class BitbucketServerController extends WebhookController { @@ -51,10 +52,12 @@ public class BitbucketServerController extends WebhookController { private static final String MERGED = EVENT + "=pr:merged"; private static final String PR_SOURCE_BRANCH_UPDATED = EVENT + "=pr:from_ref_updated"; private static final String HMAC_ALGORITHM = "HMACSha256"; - private static final String MERGE_COMMENT = "/projects/{project}/repos/{repo}/pull-requests/{id}/comments"; - private static final String BLOCKER_COMMENT = "/projects/{project}/repos/{repo}/pull-requests/{id}/blocker-comments"; + private static final String PROJECT_REPO_PATH = "/projects/{project}/repos/{repo}"; + private static final String MERGE_COMMENT = "/pull-requests/{id}/comments"; + private static final String BLOCKER_COMMENT = "/pull-requests/{id}/blocker-comments"; private static final String BUILD_API_PATH = "/rest/build-status/latest/commits/{commit}"; private static final Charset CHARSET = StandardCharsets.UTF_8; + private static final int INDEX_FROM_CHANGES = 0; private static final Logger log = org.slf4j.LoggerFactory.getLogger(BitbucketServerController.class); private final FlowProperties flowProperties; @@ -63,6 +66,7 @@ public class BitbucketServerController extends WebhookController { private final JiraProperties jiraProperties; private final FlowService flowService; private final HelperService helperService; + private final BitBucketService bitbucketService; private final FilterFactory filterFactory; private final ConfigurationOverrider configOverrider; @@ -71,7 +75,7 @@ public class BitbucketServerController extends WebhookController { @PostConstruct public void init() throws NoSuchAlgorithmException, InvalidKeyException { // initialize HMAC with SHA1 algorithm and secret - if(!ScanUtils.empty(properties.getWebhookToken())) { + if (!ScanUtils.empty(properties.getWebhookToken())) { SecretKeySpec secret = new SecretKeySpec(properties.getWebhookToken().getBytes(CHARSET), HMAC_ALGORITHM); hmac = Mac.getInstance(HMAC_ALGORITHM); hmac.init(secret); @@ -80,7 +84,7 @@ public void init() throws NoSuchAlgorithmException, InvalidKeyException { @PostMapping(value = {"/{product}", "/"}, headers = PING) public String pingEvent( - @PathVariable(value = "product", required = false) String product){ + @PathVariable(value = "product", required = false) String product) { log.info("Processing Bitbucket Server PING request"); return "ok"; } @@ -94,7 +98,7 @@ public ResponseEntity mergeRequest( @PathVariable(value = "product", required = false) String product, @RequestHeader(value = SIGNATURE) String signature, ControllerRequest controllerRequest - ){ + ) { return doMergeEvent(body, product, signature, controllerRequest); } @@ -107,7 +111,7 @@ public ResponseEntity mergedRequest( @PathVariable(value = "product", required = false) String product, @RequestHeader(value = SIGNATURE) String signature, ControllerRequest controllerRequest - ){ + ) { return doMergeEvent(body, product, signature, controllerRequest); } @@ -120,8 +124,8 @@ public ResponseEntity prSourceBranchUpdateRequest( @PathVariable(value = "product", required = false) String product, @RequestHeader(value = SIGNATURE) String signature, ControllerRequest controllerRequest - ){ - return doMergeEvent(body, product, signature,controllerRequest); + ) { + return doMergeEvent(body, product, signature, controllerRequest); } private ResponseEntity doMergeEvent(String body, @@ -133,7 +137,6 @@ private ResponseEntity doMergeEvent(String body, verifyHmacSignature(body, signature); controllerRequest = ensureNotNull(controllerRequest); - FlowOverride o = ScanUtils.getMachinaOverride(controllerRequest.getOverride()); ObjectMapper mapper = new ObjectMapper(); PullEvent event; @@ -169,6 +172,7 @@ private ResponseEntity doMergeEvent(String body, String currentBranch = fromRef.getDisplayId(); String targetBranch = toRef.getDisplayId(); List branches = getBranches(controllerRequest, flowProperties); + String fromRefLatestCommit = fromRef.getLatestCommit(); BugTracker bt = ScanUtils.getBugTracker(controllerRequest.getAssignee(), bugType, jiraProperties, controllerRequest.getBug()); @@ -176,24 +180,20 @@ private ResponseEntity doMergeEvent(String body, setExclusionProperties(cxProperties, controllerRequest); - String gitUrl = getGitUrl(fromRefRepository); String gitAuthUrl = getGitAuthUrl(gitUrl); - String mergeEndpoint = properties.getUrl().concat(properties.getApiPath()).concat(MERGE_COMMENT); - mergeEndpoint = mergeEndpoint.replace("{project}", toRefRepository.getProject().getKey()); - mergeEndpoint = mergeEndpoint.replace("{repo}", toRefRepository.getSlug()); + String repoSelfUrl = getRepoSelfUrl(toRefRepository.getProject().getKey(), toRefRepository.getSlug()); + + String mergeEndpoint = repoSelfUrl.concat(MERGE_COMMENT); mergeEndpoint = mergeEndpoint.replace("{id}", pullRequest.getId().toString()); String buildStatusEndpoint = properties.getUrl().concat(BUILD_API_PATH); - buildStatusEndpoint = buildStatusEndpoint.replace("{commit}", fromRef.getLatestCommit()); + buildStatusEndpoint = buildStatusEndpoint.replace("{commit}", fromRefLatestCommit); - String blockerCommentUrl = properties.getUrl().concat(BLOCKER_COMMENT); - blockerCommentUrl = blockerCommentUrl.replace("{project}", toRefRepository.getProject().getKey()); - blockerCommentUrl = blockerCommentUrl.replace("{repo}", toRefRepository.getSlug()); + String blockerCommentUrl = repoSelfUrl.concat(BLOCKER_COMMENT); blockerCommentUrl = blockerCommentUrl.replace("{id}", pullRequest.getId().toString()); - String scanPreset = cxProperties.getScanPreset(); if (!ScanUtils.empty(controllerRequest.getPreset())) { scanPreset = controllerRequest.getPreset(); @@ -220,15 +220,18 @@ private ResponseEntity doMergeEvent(String body, .excludeFiles(controllerRequest.getExcludeFiles()) .bugTracker(bt) .filter(filter) + .hash(fromRefLatestCommit) .build(); - request = configOverrider.overrideScanRequestProperties(o, request); + request.putAdditionalMetadata(BitBucketService.REPO_SELF_URL, repoSelfUrl); + setBrowseUrl(fromRefRepository, request); + checkForConfigAsCode(request); request.putAdditionalMetadata(HTMLHelper.WEB_HOOK_PAYLOAD, body); request.putAdditionalMetadata("buildStatusUrl", buildStatusEndpoint); request.putAdditionalMetadata("cxBaseUrl", cxProperties.getBaseUrl()); request.putAdditionalMetadata("blocker-comment-url", blockerCommentUrl); request.setId(uid); - setBrowseUrl(fromRefRepository, request); + //only initiate scan/automation if target branch is applicable if (helperService.isBranch2Scan(request, branches)) { flowService.initiateAutomation(request); @@ -257,13 +260,12 @@ public ResponseEntity pushRequest( @RequestHeader(value = SIGNATURE) String signature, ControllerRequest controllerRequest - ){ + ) { String uid = helperService.getShortUid(); MDC.put("cx", uid); verifyHmacSignature(body, signature); controllerRequest = ensureNotNull(controllerRequest); - FlowOverride o = ScanUtils.getMachinaOverride(controllerRequest.getOverride()); ObjectMapper mapper = new ObjectMapper(); PushEvent event; @@ -276,7 +278,7 @@ public ResponseEntity pushRequest( try { Repository repository = event.getRepository(); String app = repository.getName(); - if(!ScanUtils.empty(controllerRequest.getApplication())){ + if (!ScanUtils.empty(controllerRequest.getApplication())) { app = controllerRequest.getApplication(); } @@ -286,12 +288,13 @@ public ResponseEntity pushRequest( Optional.ofNullable(controllerRequest.getAppOnly()).ifPresent(flowProperties::setTrackApplicationOnly); - if(ScanUtils.empty(product)){ + if (ScanUtils.empty(product)) { product = ScanRequest.Product.CX.getProduct(); } ScanRequest.Product p = ScanRequest.Product.valueOf(product.toUpperCase(Locale.ROOT)); - String currentBranch = ScanUtils.getBranchFromRef(event.getChanges().get(0).getRefId()); + String currentBranch = ScanUtils.getBranchFromRef(event.getChanges().get(INDEX_FROM_CHANGES).getRefId()); List branches = getBranches(controllerRequest, flowProperties); + String latestCommit = event.getChanges().get(INDEX_FROM_CHANGES).getToHash(); BugTracker bt = ScanUtils.getBugTracker(controllerRequest.getAssignee(), bugType, jiraProperties, controllerRequest.getBug()); FilterConfiguration filter = filterFactory.getFilter(controllerRequest, flowProperties); @@ -305,7 +308,7 @@ public ResponseEntity pushRequest( String gitAuthUrl = getGitAuthUrl(gitUrl); String scanPreset = cxProperties.getScanPreset(); - if(!ScanUtils.empty(controllerRequest.getPreset())){ + if (!ScanUtils.empty(controllerRequest.getPreset())) { scanPreset = controllerRequest.getPreset(); } @@ -328,13 +331,16 @@ public ResponseEntity pushRequest( .excludeFiles(controllerRequest.getExcludeFiles()) .bugTracker(bt) .filter(filter) + .hash(latestCommit) .build(); + + setBrowseUrl(repository, request); - request = configOverrider.overrideScanRequestProperties(o, request); - request.putAdditionalMetadata(HTMLHelper.WEB_HOOK_PAYLOAD, body); + checkForConfigAsCode(request); + fillRequestWithAdditionalData(request, repository, body); request.setId(uid); //only initiate scan/automation if target branch is applicable - if(helperService.isBranch2Scan(request, branches)){ + if (helperService.isBranch2Scan(request, branches)) { flowService.initiateAutomation(request); } } catch (IllegalArgumentException e) { @@ -379,6 +385,24 @@ private String getGitUrl(Repository repository) { .concat(repository.getProject().getKey().concat("/")) .concat(repository.getSlug()).concat(".git"); } + + private void checkForConfigAsCode(ScanRequest request) { + CxConfig cxConfig = bitbucketService.getCxConfigOverride(request); + configOverrider.overrideScanRequestProperties(cxConfig, request); + } + + private void fillRequestWithAdditionalData(ScanRequest request, Repository repository, String hookPayload) { + String repoSelfUrl = getRepoSelfUrl(repository.getProject().getKey(), repository.getSlug()); + request.putAdditionalMetadata(BitBucketService.REPO_SELF_URL, repoSelfUrl); + request.putAdditionalMetadata(HTMLHelper.WEB_HOOK_PAYLOAD, hookPayload); + } + + String getRepoSelfUrl(String projectKey, String repoSlug){ + String repoSelfUrl = properties.getUrl().concat(properties.getApiPath()).concat(PROJECT_REPO_PATH); + repoSelfUrl = repoSelfUrl.replace("{project}", projectKey); + repoSelfUrl = repoSelfUrl.replace("{repo}", repoSlug); + return repoSelfUrl; + } } diff --git a/src/main/java/com/checkmarx/flow/dto/bitbucket/Content.java b/src/main/java/com/checkmarx/flow/dto/bitbucket/Content.java new file mode 100644 index 000000000..0fb5dcf95 --- /dev/null +++ b/src/main/java/com/checkmarx/flow/dto/bitbucket/Content.java @@ -0,0 +1,67 @@ +package com.checkmarx.flow.dto.bitbucket; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; + +import java.util.List; + +@JsonInclude(JsonInclude.Include.NON_NULL) +@JsonPropertyOrder({ + "pagelen", + "values", + "page", + "next" +}) +public class Content { + + @JsonProperty("pagelen") + private Integer pagelen; + @JsonProperty("values") + private List values = null; + @JsonProperty("page") + private Integer page; + @JsonProperty("next") + private String next; + + @JsonProperty("pagelen") + public Integer getPagelen() { + return pagelen; + } + + @JsonProperty("pagelen") + public void setPagelen(Integer pagelen) { + this.pagelen = pagelen; + } + + @JsonProperty("values") + public List getValues() { + return values; + } + + @JsonProperty("values") + public void setValues(List values) { + this.values = values; + } + + @JsonProperty("page") + public Integer getPage() { + return page; + } + + @JsonProperty("page") + public void setPage(Integer page) { + this.page = page; + } + + @JsonProperty("next") + public String getNext() { + return next; + } + + @JsonProperty("next") + public void setNext(String next) { + this.next = next; + } + +} \ No newline at end of file diff --git a/src/main/java/com/checkmarx/flow/dto/bitbucket/History.java b/src/main/java/com/checkmarx/flow/dto/bitbucket/History.java new file mode 100644 index 000000000..783b3732b --- /dev/null +++ b/src/main/java/com/checkmarx/flow/dto/bitbucket/History.java @@ -0,0 +1,26 @@ +package com.checkmarx.flow.dto.bitbucket; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; + +@JsonInclude(JsonInclude.Include.NON_NULL) +@JsonPropertyOrder({ + "href" +}) +public class History { + + @JsonProperty("href") + private String href; + + @JsonProperty("href") + public String getHref() { + return href; + } + + @JsonProperty("href") + public void setHref(String href) { + this.href = href; + } + +} \ No newline at end of file diff --git a/src/main/java/com/checkmarx/flow/dto/bitbucket/Links.java b/src/main/java/com/checkmarx/flow/dto/bitbucket/Links.java index 32b9d7efa..547451eac 100644 --- a/src/main/java/com/checkmarx/flow/dto/bitbucket/Links.java +++ b/src/main/java/com/checkmarx/flow/dto/bitbucket/Links.java @@ -10,7 +10,9 @@ "commits", "self", "comments", - "html" + "html", + "meta", + "history" }) public class Links { @@ -22,6 +24,10 @@ public class Links { private Html html; @JsonProperty("comments") private Comments comments; + @JsonProperty("meta") + private Meta meta; + @JsonProperty("history") + private History history; @JsonProperty("commits") @@ -63,4 +69,24 @@ public void setComments(Comments comments) { public Comments getComments() { return this.comments; } + + @JsonProperty("meta") + public Meta getMeta() { + return meta; + } + + @JsonProperty("meta") + public void setMeta(Meta meta) { + this.meta = meta; + } + + @JsonProperty("history") + public History getHistory() { + return history; + } + + @JsonProperty("history") + public void setHistory(History history) { + this.history = history; + } } diff --git a/src/main/java/com/checkmarx/flow/dto/bitbucket/Meta.java b/src/main/java/com/checkmarx/flow/dto/bitbucket/Meta.java new file mode 100644 index 000000000..8485a2f3a --- /dev/null +++ b/src/main/java/com/checkmarx/flow/dto/bitbucket/Meta.java @@ -0,0 +1,26 @@ +package com.checkmarx.flow.dto.bitbucket; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; + +@JsonInclude(JsonInclude.Include.NON_NULL) +@JsonPropertyOrder({ + "href" +}) +public class Meta { + + @JsonProperty("href") + private String href; + + @JsonProperty("href") + public String getHref() { + return href; + } + + @JsonProperty("href") + public void setHref(String href) { + this.href = href; + } + +} \ No newline at end of file diff --git a/src/main/java/com/checkmarx/flow/dto/bitbucket/Value.java b/src/main/java/com/checkmarx/flow/dto/bitbucket/Value.java new file mode 100644 index 000000000..df7b66694 --- /dev/null +++ b/src/main/java/com/checkmarx/flow/dto/bitbucket/Value.java @@ -0,0 +1,119 @@ +package com.checkmarx.flow.dto.bitbucket; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; + +import java.util.List; + +@JsonInclude(JsonInclude.Include.NON_NULL) +@JsonPropertyOrder({ + "path", + "type", + "links", + "commit", + "mimetype", + "escaped_path", + "attributes", + "size" +}) +public class Value { + + @JsonProperty("path") + private String path; + @JsonProperty("type") + private String type; + @JsonProperty("links") + private Links links; + @JsonProperty("commit") + private Commit commit; + @JsonProperty("mimetype") + private Object mimetype; + @JsonProperty("escaped_path") + private String escapedPath; + @JsonProperty("attributes") + private List attributes = null; + @JsonProperty("size") + private Integer size; + + @JsonProperty("path") + public String getPath() { + return path; + } + + @JsonProperty("path") + public void setPath(String path) { + this.path = path; + } + + @JsonProperty("type") + public String getType() { + return type; + } + + @JsonProperty("type") + public void setType(String type) { + this.type = type; + } + + @JsonProperty("links") + public Links getLinks() { + return links; + } + + @JsonProperty("links") + public void setLinks(Links links) { + this.links = links; + } + + @JsonProperty("commit") + public Commit getCommit() { + return commit; + } + + @JsonProperty("commit") + public void setCommit(Commit commit) { + this.commit = commit; + } + + @JsonProperty("mimetype") + public Object getMimetype() { + return mimetype; + } + + @JsonProperty("mimetype") + public void setMimetype(Object mimetype) { + this.mimetype = mimetype; + } + + @JsonProperty("escaped_path") + public String getEscapedPath() { + return escapedPath; + } + + @JsonProperty("escaped_path") + public void setEscapedPath(String escapedPath) { + this.escapedPath = escapedPath; + } + + @JsonProperty("attributes") + public List getAttributes() { + return attributes; + } + + @JsonProperty("attributes") + public void setAttributes(List attributes) { + this.attributes = attributes; + } + + @JsonProperty("size") + public Integer getSize() { + return size; + } + + @JsonProperty("size") + public void setSize(Integer size) { + this.size = size; + } + +} \ No newline at end of file diff --git a/src/main/java/com/checkmarx/flow/dto/bitbucketserver/Children.java b/src/main/java/com/checkmarx/flow/dto/bitbucketserver/Children.java new file mode 100644 index 000000000..0654d3df4 --- /dev/null +++ b/src/main/java/com/checkmarx/flow/dto/bitbucketserver/Children.java @@ -0,0 +1,80 @@ +package com.checkmarx.flow.dto.bitbucketserver; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; + +import java.util.List; + +@JsonInclude(JsonInclude.Include.NON_NULL) +@JsonPropertyOrder({ + "size", + "limit", + "isLastPage", + "values", + "start" +}) +public class Children { + + @JsonProperty("size") + private Integer size; + @JsonProperty("limit") + private Integer limit; + @JsonProperty("isLastPage") + private Boolean isLastPage; + @JsonProperty("values") + private List values = null; + @JsonProperty("start") + private Integer start; + + @JsonProperty("size") + public Integer getSize() { + return size; + } + + @JsonProperty("size") + public void setSize(Integer size) { + this.size = size; + } + + @JsonProperty("limit") + public Integer getLimit() { + return limit; + } + + @JsonProperty("limit") + public void setLimit(Integer limit) { + this.limit = limit; + } + + @JsonProperty("isLastPage") + public Boolean getIsLastPage() { + return isLastPage; + } + + @JsonProperty("isLastPage") + public void setIsLastPage(Boolean isLastPage) { + this.isLastPage = isLastPage; + } + + @JsonProperty("values") + public List getValues() { + return values; + } + + @JsonProperty("values") + public void setValues(List values) { + this.values = values; + } + + @JsonProperty("start") + public Integer getStart() { + return start; + } + + @JsonProperty("start") + public void setStart(Integer start) { + this.start = start; + } + +} \ No newline at end of file diff --git a/src/main/java/com/checkmarx/flow/dto/bitbucketserver/Content.java b/src/main/java/com/checkmarx/flow/dto/bitbucketserver/Content.java new file mode 100644 index 000000000..c0d8c8d6f --- /dev/null +++ b/src/main/java/com/checkmarx/flow/dto/bitbucketserver/Content.java @@ -0,0 +1,51 @@ +package com.checkmarx.flow.dto.bitbucketserver; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; + +@JsonInclude(JsonInclude.Include.NON_NULL) +@JsonPropertyOrder({ + "path", + "revision", + "children" +}) +public class Content { + + @JsonProperty("path") + private Path path; + @JsonProperty("revision") + private String revision; + @JsonProperty("children") + private Children children; + + @JsonProperty("path") + public Path getPath() { + return path; + } + + @JsonProperty("path") + public void setPath(Path path) { + this.path = path; + } + + @JsonProperty("revision") + public String getRevision() { + return revision; + } + + @JsonProperty("revision") + public void setRevision(String revision) { + this.revision = revision; + } + + @JsonProperty("children") + public Children getChildren() { + return children; + } + + @JsonProperty("children") + public void setChildren(Children children) { + this.children = children; + } +} \ No newline at end of file diff --git a/src/main/java/com/checkmarx/flow/dto/bitbucketserver/Path.java b/src/main/java/com/checkmarx/flow/dto/bitbucketserver/Path.java new file mode 100644 index 000000000..6cea66bb9 --- /dev/null +++ b/src/main/java/com/checkmarx/flow/dto/bitbucketserver/Path.java @@ -0,0 +1,80 @@ +package com.checkmarx.flow.dto.bitbucketserver; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; + +import java.util.List; + +@JsonInclude(JsonInclude.Include.NON_NULL) +@JsonPropertyOrder({ + "components", + "parent", + "name", + "toString", + "extension" +}) +public class Path { + + @JsonProperty("components") + private List components = null; + @JsonProperty("parent") + private String parent; + @JsonProperty("name") + private String name; + @JsonProperty("toString") + private String toString; + @JsonProperty("extension") + private String extension; + + @JsonProperty("components") + public List getComponents() { + return components; + } + + @JsonProperty("components") + public void setComponents(List components) { + this.components = components; + } + + @JsonProperty("parent") + public String getParent() { + return parent; + } + + @JsonProperty("parent") + public void setParent(String parent) { + this.parent = parent; + } + + @JsonProperty("name") + public String getName() { + return name; + } + + @JsonProperty("name") + public void setName(String name) { + this.name = name; + } + + @JsonProperty("toString") + public String getToString() { + return toString; + } + + @JsonProperty("toString") + public void setToString(String toString) { + this.toString = toString; + } + + @JsonProperty("extension") + public String getExtension() { + return extension; + } + + @JsonProperty("extension") + public void setExtension(String extension) { + this.extension = extension; + } + +} \ No newline at end of file diff --git a/src/main/java/com/checkmarx/flow/dto/bitbucketserver/Value.java b/src/main/java/com/checkmarx/flow/dto/bitbucketserver/Value.java new file mode 100644 index 000000000..6b2427d31 --- /dev/null +++ b/src/main/java/com/checkmarx/flow/dto/bitbucketserver/Value.java @@ -0,0 +1,78 @@ +package com.checkmarx.flow.dto.bitbucketserver; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; + +@JsonInclude(JsonInclude.Include.NON_NULL) +@JsonPropertyOrder({ + "path", + "node", + "type", + "contentId", + "size" +}) +public class Value { + + @JsonProperty("path") + private Path path; + @JsonProperty("node") + private String node; + @JsonProperty("type") + private String type; + @JsonProperty("contentId") + private String contentId; + @JsonProperty("size") + private Integer size; + + @JsonProperty("path") + public Path getPath() { + return path; + } + + @JsonProperty("path") + public void setPath(Path path) { + this.path = path; + } + + @JsonProperty("node") + public String getNode() { + return node; + } + + @JsonProperty("node") + public void setNode(String node) { + this.node = node; + } + + @JsonProperty("type") + public String getType() { + return type; + } + + @JsonProperty("type") + public void setType(String type) { + this.type = type; + } + + @JsonProperty("contentId") + public String getContentId() { + return contentId; + } + + @JsonProperty("contentId") + public void setContentId(String contentId) { + this.contentId = contentId; + } + + @JsonProperty("size") + public Integer getSize() { + return size; + } + + @JsonProperty("size") + public void setSize(Integer size) { + this.size = size; + } + +} \ No newline at end of file diff --git a/src/main/java/com/checkmarx/flow/sastscanning/ScanRequestConverter.java b/src/main/java/com/checkmarx/flow/sastscanning/ScanRequestConverter.java index bfd01be05..b3e9c8ffd 100644 --- a/src/main/java/com/checkmarx/flow/sastscanning/ScanRequestConverter.java +++ b/src/main/java/com/checkmarx/flow/sastscanning/ScanRequestConverter.java @@ -3,6 +3,7 @@ import com.checkmarx.flow.config.FlowProperties; import com.checkmarx.flow.dto.ScanRequest; import com.checkmarx.flow.dto.Sources; +import com.checkmarx.flow.service.BitBucketService; import com.checkmarx.flow.service.GitHubService; import com.checkmarx.flow.service.GitLabService; import com.checkmarx.flow.service.HelperService; @@ -34,6 +35,7 @@ public class ScanRequestConverter { private final FlowProperties flowProperties; private final GitHubService gitService; private final GitLabService gitLabService; + private final BitBucketService bitBucketService; public CxScanParams toScanParams(ScanRequest scanRequest) throws CheckmarxException { String ownerId = determineTeamAndOwnerID(scanRequest); @@ -158,10 +160,8 @@ private Sources getRepoContent(ScanRequest request) { sources = gitLabService.getRepoContent(request); break; case BITBUCKET: - log.warn("Profiling is not available for BitBucket Cloud"); - break; case BITBUCKETSERVER: - log.warn("Profiling is not available for BitBucket Server"); + sources = bitBucketService.getRepoContent(request); break; case ADO: log.warn("Profiling is not available for Azure DevOps"); diff --git a/src/main/java/com/checkmarx/flow/service/BitBucketService.java b/src/main/java/com/checkmarx/flow/service/BitBucketService.java index 005cd2780..2b071cf14 100644 --- a/src/main/java/com/checkmarx/flow/service/BitBucketService.java +++ b/src/main/java/com/checkmarx/flow/service/BitBucketService.java @@ -4,15 +4,23 @@ import com.checkmarx.flow.config.FlowProperties; import com.checkmarx.flow.dto.ScanDetails; import com.checkmarx.flow.dto.ScanRequest; +import com.checkmarx.flow.dto.Sources; +import com.checkmarx.flow.dto.bitbucketserver.Content; +import com.checkmarx.flow.dto.bitbucketserver.Value; import com.checkmarx.flow.dto.report.PullRequestReport; import com.checkmarx.flow.exception.BitBucketClientException; import com.checkmarx.flow.utils.HTMLHelper; import com.checkmarx.flow.utils.ScanUtils; +import com.checkmarx.sdk.dto.CxConfig; import com.checkmarx.sdk.dto.ScanResults; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.exception.ExceptionUtils; +import org.apache.logging.log4j.util.Strings; import org.json.JSONArray; import org.json.JSONObject; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.http.*; import org.springframework.stereotype.Service; @@ -22,34 +30,60 @@ import java.beans.ConstructorProperties; import java.util.Base64; import java.util.HashMap; +import java.util.List; import java.util.Map; - +@Slf4j @Service -public class BitBucketService { +public class BitBucketService extends RepoService { - private static final Logger log = LoggerFactory.getLogger(BitBucketService.class); + public static final String REPO_SELF_URL = "repo-self-url"; private static final String LOG_COMMENT = "comment: {}"; private final RestTemplate restTemplate; private final BitBucketProperties properties; - + private final FlowProperties flowProperties; private final ThresholdValidator thresholdValidator; private static final String BUILD_IN_PROGRESS = "INPROGRESS"; private static final String BUILD_SUCCESSFUL = "SUCCESSFUL"; private static final String BUILD_FAILED = "FAILED"; + private static final String BITBUCKET_DIRECTORY = "DIRECTORY"; + private static final String BITBUCKET_FILE = "FILE"; + private static final String BITBUCKET_CLOUD_FILE = "commit_file"; + private static final String FILE_CONTENT_FOR_BB_CLOUD = "/src/{hash}/{config}"; + private static final String FILE_CONTENT_FOR_BB_SERVER = "/raw/{config}?at={hash}"; + private static final String BROWSE_CONTENT_FOR_BB_SERVER = "/browse/{path}?at={branch}"; + private static final String BROWSE_CONTENT_FOR_BB_CLOUD_WITH_DEPTH_PARAM = "/src/{hash}/?pagelen=100&max_depth={depth}"; private static final String BUILD_STATUS_KEY_FOR_CXFLOW = "cxflow"; - public static final String CX_USER_SCAN_QUEUE = "/CxWebClient/UserQueue.aspx"; + private static final String HTTP_BODY_IS_NULL = "Unable to download Config as code file. Response body is null."; + private static final String CONTENT_NOT_FOUND_IN_RESPONSE = "Content not found in JSON response for Config as code"; + public static final String PATH_SEPARATOR = "/"; + private String browseRepoEndpoint = ""; - @ConstructorProperties({"restTemplate", "properties", "thresholdValidator"}) - public BitBucketService(@Qualifier("flowRestTemplate") RestTemplate restTemplate, BitBucketProperties properties, ThresholdValidator thresholdValidator) { + @ConstructorProperties({"restTemplate", "properties","flowProperties", "thresholdValidator"}) + public BitBucketService(@Qualifier("flowRestTemplate") RestTemplate restTemplate, BitBucketProperties properties, FlowProperties flowProperties, ThresholdValidator thresholdValidator) { this.restTemplate = restTemplate; this.properties = properties; + this.flowProperties = flowProperties; this.thresholdValidator = thresholdValidator; } - private HttpHeaders createAuthHeaders(){ + private static JSONObject getJSONComment(String comment) { + JSONObject requestBody = new JSONObject(); + JSONObject content = new JSONObject(); + content.put("raw", comment); + requestBody.put("content", content); + return requestBody; + } + + private static JSONObject getServerJSONComment(String comment) { + JSONObject requestBody = new JSONObject(); + requestBody.put("text", comment); + return requestBody; + } + + private HttpHeaders createAuthHeaders() { String encoding = Base64.getEncoder().encodeToString(properties.getToken().getBytes()); HttpHeaders httpHeaders = new HttpHeaders(); httpHeaders.set(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_VALUE); @@ -58,55 +92,51 @@ private HttpHeaders createAuthHeaders(){ return httpHeaders; } - void processMerge(ScanRequest request,ScanResults results) throws BitBucketClientException { + void processMerge(ScanRequest request, ScanResults results) throws BitBucketClientException { try { String comment = HTMLHelper.getMergeCommentMD(request, results, properties); log.debug(LOG_COMMENT, comment); sendMergeComment(request, comment); - } catch (HttpClientErrorException e){ + } catch (HttpClientErrorException e) { log.error("Error occurred while creating Merge Request comment", e); throw new BitBucketClientException(); } } - void processServerMerge(ScanRequest request,ScanResults results, ScanDetails scanDetails) throws BitBucketClientException { + void processServerMerge(ScanRequest request, ScanResults results, ScanDetails scanDetails) throws BitBucketClientException { try { String comment = HTMLHelper.getMergeCommentMD(request, results, properties); log.debug(LOG_COMMENT, comment); - if(properties.isBlockMerge()) { + if (properties.isBlockMerge()) { if (thresholdValidator.isMergeAllowed(results, properties, new PullRequestReport(scanDetails, request))) { sendServerMergeComment(request, comment); - } - else - { + } else { sendServerMergeTask(request, comment); } - } - else - { + } else { sendServerMergeComment(request, comment); } - } catch (HttpClientErrorException e){ + } catch (HttpClientErrorException e) { log.error("Error occurred while creating Merge Request comment", e); throw new BitBucketClientException(); } } - public void setBuildStartStatus(ScanRequest request){ + public void setBuildStartStatus(ScanRequest request) { - if(properties.isBlockMerge()) { + if (properties.isBlockMerge()) { String cxBaseUrl = request.getAdditionalMetadata("cxBaseUrl"); JSONObject buildStatusBody = createBuildStatusRequestBody(BUILD_IN_PROGRESS,BUILD_STATUS_KEY_FOR_CXFLOW,"Checkmarx Scan Initiated", cxBaseUrl.concat(CX_USER_SCAN_QUEUE) ,"Waiting for scan to complete.."); - sendBuildStatus(request,buildStatusBody.toString()); + sendBuildStatus(request, buildStatusBody.toString()); } } - public void setBuildEndStatus(ScanRequest request,ScanResults results, ScanDetails scanDetails){ + public void setBuildEndStatus(ScanRequest request, ScanResults results, ScanDetails scanDetails) { - if(properties.isBlockMerge()) { + if (properties.isBlockMerge()) { String status = BUILD_SUCCESSFUL; if (!thresholdValidator.isMergeAllowed(results, properties, new PullRequestReport(scanDetails, request))) { @@ -115,7 +145,7 @@ public void setBuildEndStatus(ScanRequest request,ScanResults results, ScanDetai JSONObject buildStatusBody = createBuildStatusRequestBody(status,BUILD_STATUS_KEY_FOR_CXFLOW,"Checkmarx Scan Results", results.getLink(),results.getScanSummary().toString()); - sendBuildStatus(request,buildStatusBody.toString()); + sendBuildStatus(request, buildStatusBody.toString()); } } @@ -134,53 +164,50 @@ private JSONObject createBuildStatusRequestBody(String buildState, String buildK buildJsonBody.put("url", buildUrl); //Following fields are optional - if(!ScanUtils.empty(buildName)) { + if (!ScanUtils.empty(buildName)) { buildJsonBody.put("name", buildName); } - if(!ScanUtils.empty(buildDescription)) { + if (!ScanUtils.empty(buildDescription)) { buildJsonBody.put("description", buildDescription); } return buildJsonBody; } - private void sendBuildStatus(ScanRequest request, String buildStatusRequestBody) - { + private void sendBuildStatus(ScanRequest request, String buildStatusRequestBody) { String buildStatusApiUrl = request.getAdditionalMetadata("buildStatusUrl"); HttpEntity httpEntity = new HttpEntity<>(buildStatusRequestBody, createAuthHeaders()); restTemplate.exchange(buildStatusApiUrl, HttpMethod.POST, httpEntity, String.class); } - public void sendMergeComment(ScanRequest request, String comment){ + public void sendMergeComment(ScanRequest request, String comment) { HttpEntity httpEntity = new HttpEntity<>(getJSONComment(comment).toString(), createAuthHeaders()); restTemplate.exchange(request.getMergeNoteUri(), HttpMethod.POST, httpEntity, String.class); } - public void sendServerMergeComment(ScanRequest request, String comment){ + public void sendServerMergeComment(ScanRequest request, String comment) { HttpEntity httpEntity = new HttpEntity<>(getServerJSONComment(comment).toString(), createAuthHeaders()); restTemplate.exchange(request.getMergeNoteUri(), HttpMethod.POST, httpEntity, String.class); } - private void sendServerMergeTask(ScanRequest request, String comment){ + private void sendServerMergeTask(ScanRequest request, String comment) { ResponseEntity retrievedResult = retrieveExistingOpenTasks(request); - Object cxFlowTask = getCxFlowTask(retrievedResult); + Object cxFlowTask = getCxFlowTask(retrievedResult); - if(cxFlowTask !=null) - { + if (cxFlowTask != null) { Integer taskId = ((JSONObject) cxFlowTask).getInt("id"); Integer taskVersion = ((JSONObject) cxFlowTask).getInt("version"); JSONObject taskBody = new JSONObject(); taskBody.put("id", taskId); taskBody.put("version", taskVersion); - taskBody.put("severity","BLOCKER"); + taskBody.put("severity", "BLOCKER"); taskBody.put("text", comment); HttpEntity httpEntity = new HttpEntity<>(taskBody.toString(), createAuthHeaders()); - restTemplate.exchange(request.getMergeNoteUri().concat("/"+ taskId), HttpMethod.PUT, httpEntity, String.class); + restTemplate.exchange(request.getMergeNoteUri().concat("/" + taskId), HttpMethod.PUT, httpEntity, String.class); - } - else { + } else { JSONObject taskBody = new JSONObject(); taskBody.put("severity", "BLOCKER"); taskBody.put("text", comment); @@ -191,33 +218,27 @@ private void sendServerMergeTask(ScanRequest request, String comment){ private Object getCxFlowTask(ResponseEntity retrievedResult) { - if(retrievedResult.getBody() != null) { + if (retrievedResult.getBody() != null) { JSONObject json = new JSONObject(retrievedResult.getBody()); JSONArray taskList = json.getJSONArray("values"); - if(!taskList.isEmpty()) - { - for ( Object task : taskList) - { - Object author = ((JSONObject)task).get("author"); - String taskAuthor = (String) ((JSONObject)author).get("slug"); - - if(taskAuthor.equals(getCxFlowServiceAccountSlug())) - { - return task; - } - } - } - else - { + if (!taskList.isEmpty()) { + for (Object task : taskList) { + Object author = ((JSONObject) task).get("author"); + String taskAuthor = (String) ((JSONObject) author).get("slug"); + + if (taskAuthor.equals(getCxFlowServiceAccountSlug())) { + return task; + } + } + } else { return null; } } return null; } - private String getCxFlowServiceAccountSlug() - { + private String getCxFlowServiceAccountSlug() { String[] basicAuthCredentials = properties.getToken().split(":"); return basicAuthCredentials[0]; @@ -231,7 +252,7 @@ private ResponseEntity retrieveExistingOpenTasks(ScanRequest request) { params.put("state", "OPEN"); HttpEntity httpEntity = new HttpEntity<>(createAuthHeaders()); - return restTemplate.exchange(blockerCommentUrl.concat("?state={state}"), HttpMethod.GET,httpEntity,String.class,params); + return restTemplate.exchange(blockerCommentUrl.concat("?state={state}"), HttpMethod.GET, httpEntity, String.class, params); } @@ -240,31 +261,205 @@ void processCommit(ScanRequest request, ScanResults results) throws BitBucketCli String comment = HTMLHelper.getMergeCommentMD(request, results, properties); log.debug(LOG_COMMENT, comment); sendCommitComment(request, comment); - } catch (HttpClientErrorException e){ + } catch (HttpClientErrorException e) { log.error("Error occurred while creating Commit comment", e); throw new BitBucketClientException(); } } - private void sendCommitComment(ScanRequest request, String comment){ + private void sendCommitComment(ScanRequest request, String comment) { JSONObject note = new JSONObject(); note.put("note", comment); HttpEntity httpEntity = new HttpEntity<>(note.toString(), createAuthHeaders()); restTemplate.exchange(request.getMergeNoteUri(), HttpMethod.POST, httpEntity, String.class); } - private static JSONObject getJSONComment(String comment) { - JSONObject requestBody = new JSONObject(); - JSONObject content = new JSONObject(); - content.put("raw", comment); - requestBody.put("content", content); - return requestBody; + @Override + public Sources getRepoContent(ScanRequest request) { + + log.debug("Auto profiling is enabled"); + if(ScanUtils.anyEmpty(request.getNamespace(), request.getRepoName(), request.getBranch())){ + return null; + } + Sources sources = new Sources(); + browseRepoEndpoint = getBitbucketEndPoint(request); + if (request.getRepoType().equals(ScanRequest.Repository.BITBUCKETSERVER)) { + scanGitContentFromBitbucketServer(0, browseRepoEndpoint, sources); + } + else + { + scanGitContentFromBBCloud(browseRepoEndpoint, sources); + } + return sources; } - private static JSONObject getServerJSONComment(String comment) { - JSONObject requestBody = new JSONObject(); - requestBody.put("text", comment); - return requestBody; + private String getBitbucketEndPoint(ScanRequest request) { + String repoSelfUrl = request.getAdditionalMetadata(REPO_SELF_URL); + String endpoint; + + if (request.getRepoType().equals(ScanRequest.Repository.BITBUCKETSERVER)) { + endpoint = repoSelfUrl.concat(BROWSE_CONTENT_FOR_BB_SERVER); + endpoint = endpoint.replace("{branch}", request.getBranch()); + } + else { + endpoint = repoSelfUrl.concat(BROWSE_CONTENT_FOR_BB_CLOUD_WITH_DEPTH_PARAM); + endpoint = endpoint.replace("{hash}", request.getHash()); + endpoint = endpoint.replace("{depth}", flowProperties.getProfilingDepth().toString()); + } + return endpoint; + } + + private void scanGitContentFromBBCloud(String endpoint, Sources sources){ + + com.checkmarx.flow.dto.bitbucket.Content content = getRepoContentFromBBCloud(endpoint); + List values = content.getValues(); + + for(com.checkmarx.flow.dto.bitbucket.Value value: values){ + String type = value.getType(); + if (type.equals(BITBUCKET_CLOUD_FILE)){ + String fileName = value.getEscapedPath(); + String filePath = value.getPath(); + sources.addSource(filePath, fileName); + } + } + } + + private com.checkmarx.flow.dto.bitbucket.Content getRepoContentFromBBCloud(String endpoint) { + log.info("Getting repo content from {}", endpoint); + com.checkmarx.flow.dto.bitbucket.Content content = new com.checkmarx.flow.dto.bitbucket.Content(); + HttpHeaders headers = createAuthHeaders(); + try { + ResponseEntity response = restTemplate.exchange( + endpoint, + HttpMethod.GET, + new HttpEntity<>(headers), + String.class + ); + if(response.getBody() == null){ + log.warn(HTTP_BODY_IS_NULL); + } + ObjectMapper objectMapper = new ObjectMapper(); + content = objectMapper.readValue(response.getBody(), com.checkmarx.flow.dto.bitbucket.Content.class); + return content; + } catch (NullPointerException e) { + log.warn(CONTENT_NOT_FOUND_IN_RESPONSE); + } catch (HttpClientErrorException e) { + log.warn("Repo content is unavailable. The reason can be that branch has been deleted."); + } catch (JsonProcessingException e) { + log.error(String.format("Error in processing the JSON response from the repo. Error details : %s", ExceptionUtils.getRootCauseMessage(e))); + } + return content; + } + + private Content getRepoContentFromBitbucketServer(String endpoint) { + log.info("Getting repo content from {}", endpoint); + Content content = new Content(); + HttpHeaders headers = createAuthHeaders(); + try { + ResponseEntity response = restTemplate.exchange( + endpoint, + HttpMethod.GET, + new HttpEntity<>(headers), + String.class + ); + if(response.getBody() == null){ + log.warn(HTTP_BODY_IS_NULL); + } + ObjectMapper objectMapper = new ObjectMapper(); + content = objectMapper.readValue(response.getBody(), Content.class); + return content; + } catch (NullPointerException e) { + log.warn(CONTENT_NOT_FOUND_IN_RESPONSE); + } catch (HttpClientErrorException e) { + log.warn("Repo content is unavailable. The reason can be that branch has been deleted."); + } catch (JsonProcessingException e) { + log.error(String.format("Error in processing the JSON response from the repo. Error details : %s", ExceptionUtils.getRootCauseMessage(e))); + } + return content; + } + + private void scanGitContentFromBitbucketServer(int depth, String endpoint, Sources sources){ + + if(depth >= flowProperties.getProfilingDepth()){ + return; + } + if(depth == 0) { + endpoint = endpoint.replace("{path}", Strings.EMPTY); + } + + Content content = getRepoContentFromBitbucketServer(endpoint); + List values = content.getChildren().getValues(); + + for(Value value: values){ + String type = value.getType(); + if(type.equals(BITBUCKET_DIRECTORY)){ + String directoryName = value.getPath().getToString(); + String fullDirectoryPath = content.getPath().getToString(); + fullDirectoryPath = fullDirectoryPath + PATH_SEPARATOR + directoryName; + String directoryURL = browseRepoEndpoint.replace("{path}",fullDirectoryPath); + scanGitContentFromBitbucketServer(depth + 1, directoryURL, sources); + } + else if (type.equals(BITBUCKET_FILE)){ + String directoryName = content.getPath().getToString(); + String fileName = value.getPath().getToString(); + String fullPath = directoryName.concat("/").concat(fileName); + sources.addSource(fullPath, fileName); + } + } + } + + @Override + public CxConfig getCxConfigOverride(ScanRequest request) { + CxConfig result = null; + if (StringUtils.isNotBlank(properties.getConfigAsCode())) { + try { + result = loadCxConfigFromBitbucket(request); + } catch (NullPointerException e) { + log.warn(CONTENT_NOT_FOUND_IN_RESPONSE); + } catch (HttpClientErrorException.NotFound e) { + log.info(String.format("No Config as code was found with the name: %s", properties.getConfigAsCode())); + } catch (Exception e) { + log.error(String.format("Error in getting config as code from the repo. Error details : %s", ExceptionUtils.getRootCauseMessage(e))); + } + } + return result; + } + + private CxConfig loadCxConfigFromBitbucket(ScanRequest request) { + CxConfig cxConfig; + HttpHeaders headers = createAuthHeaders(); + String repoSelfUrl = request.getAdditionalMetadata(REPO_SELF_URL); + + String urlTemplate; + if (request.getRepoType().equals(ScanRequest.Repository.BITBUCKETSERVER)) { + urlTemplate = repoSelfUrl.concat(FILE_CONTENT_FOR_BB_SERVER); + } else { + urlTemplate = repoSelfUrl.concat(FILE_CONTENT_FOR_BB_CLOUD); + } + + Map uriVariables = new HashMap<>(); + uriVariables.put("hash", request.getHash()); + uriVariables.put("config", properties.getConfigAsCode()); + + ResponseEntity response = restTemplate.exchange( + urlTemplate, + HttpMethod.GET, + new HttpEntity<>(headers), + String.class, uriVariables + ); + if (response.getBody() == null) { + log.warn(HTTP_BODY_IS_NULL); + cxConfig = null; + } else { + JSONObject json = new JSONObject(response.getBody()); + if (ScanUtils.empty(json.toString())) { + log.warn(CONTENT_NOT_FOUND_IN_RESPONSE); + cxConfig = null; + } else { + cxConfig = com.checkmarx.sdk.utils.ScanUtils.getConfigAsCode(json.toString()); + } + } + return cxConfig; } } diff --git a/src/main/java/com/checkmarx/flow/service/HelperService.java b/src/main/java/com/checkmarx/flow/service/HelperService.java index 7d3d2b1a9..615c7cf50 100644 --- a/src/main/java/com/checkmarx/flow/service/HelperService.java +++ b/src/main/java/com/checkmarx/flow/service/HelperService.java @@ -177,7 +177,7 @@ public String getStringFromFile(String path) throws IOException { * Determine what preset to use based on Sources and Profile mappings */ public String getPresetFromSources(Sources sources){ - if(sources == null || profiles == null || sources.getLanguageStats() == null || sources.getSources() == null){ + if(sources == null || profiles == null || sources.getSources() == null){ return cxProperties.getScanPreset(); } diff --git a/src/test/java/com/checkmarx/flow/cucumber/component/deletebranch/DeleteBranchSteps.java b/src/test/java/com/checkmarx/flow/cucumber/component/deletebranch/DeleteBranchSteps.java index 63f4b9dd3..cf7a150ae 100644 --- a/src/test/java/com/checkmarx/flow/cucumber/component/deletebranch/DeleteBranchSteps.java +++ b/src/test/java/com/checkmarx/flow/cucumber/component/deletebranch/DeleteBranchSteps.java @@ -50,7 +50,7 @@ public class DeleteBranchSteps { private final ConfigurationOverrider configOverrider; private String branch; - + private Boolean deleteCalled; private String repoName; @@ -99,7 +99,7 @@ private void initCxClientMock() { ScanResultsAnswerer answerer = new ScanResultsAnswerer(); doAnswer(answerer).when(cxClientMock).deleteProject(any()); try { - when(cxClientMock.getTeamId(anyString(), anyString())).thenReturn(TEAM); + when(cxClientMock.getTeamId(anyString(),anyString())).thenReturn(TEAM); when(cxClientMock.getTeamId(anyString())).thenReturn(TEAM); } catch (CheckmarxException e) { fail(e.getMessage()); @@ -115,7 +115,7 @@ public Object answer(InvocationOnMock invocation) { } @Given("GitHub repo name is {string}") - public void setRepoName(String repoName) { + public void setRepoName(String repoName){ this.repoName = repoName; initGitHubProperties(); } @@ -124,28 +124,28 @@ public void setRepoName(String repoName) { public void validateNoException() { // If we have arrived here, no exception was thrown. } - + @And("GitHub trigger is {string}") - public void setTrigger(String trigger) { + public void setTrigger(String trigger){ this.trigger = trigger; } - + @And("GitHub branch is {string}") public void setBranch(String branch) { this.branch = branch; } - + @And("a project {string} {string} in SAST") - public void setProjectId(String projectName, String exists) { + public void setProjectId(String projectName, String exists){ int projectId; if (exists.equals("exists") || Boolean.parseBoolean(exists)) { projectId = EXISTING_PROJECT_ID; - } else { + }else{ projectId = Constants.UNKNOWN_INT; } when(cxClientMock.getProjectId(anyString(), any())).thenReturn(projectId); } - + @And("the {string} branch is {string} as determined by application.yml") public void theBranchIsSpecifiedAsProtected(String branch, String protectedOrNot) { List protectedBranches = flowProperties.getBranches(); @@ -155,7 +155,7 @@ public void theBranchIsSpecifiedAsProtected(String branch, String protectedOrNot protectedBranches.remove(branch); } } - + @When("GitHub notifies cxFlow that a {string} branch/ref was deleted") public void githubNotifiesCxFlowThatABranchWasDeleted(String deletedBranch) { branch = deletedBranch; @@ -175,11 +175,11 @@ public void cxflowWillNotCallTheSASTDeleteAPI(String willCall, String projectNam private void verifyDeleteApiCall(boolean expectingCall) { if (expectingCall) { assertEquals(Boolean.TRUE, deleteCalled); - } else { + }else{ assertEquals(Boolean.FALSE, deleteCalled); } } - + private void sendDeleteEvent() { DeleteEvent deleteEvent = new DeleteEvent(); Repository repo = new Repository(); @@ -193,56 +193,41 @@ private void sendDeleteEvent() { deleteEvent.setRepository(repo); deleteEvent.setRefType(trigger); deleteEvent.setRef(branch); - + try { String deleteEventStr = mapper.writeValueAsString(deleteEvent); gitHubControllerSpy.deleteBranchRequest( - deleteEventStr, "SIGNATURE", "CX", null, null, TEAM); + deleteEventStr,"SIGNATURE", "CX", null, null, TEAM ); } catch (JsonProcessingException e) { fail("Unable to parse " + deleteEvent.toString()); } } - private void initHelperServiceMock() { - when(helperService.getShortUid()).thenReturn("123456"); - when(helperService.getCxTeam(any())).thenReturn(TEAM); - when(helperService.getCxProject(any())).thenReturn(PROJECT_NAME); - when(helperService.getPresetFromSources(any())).thenReturn(PRESET); + private void initHelperServiceMock() { + when(helperService.getShortUid()).thenReturn("123456"); + when(helperService.getCxTeam(any())).thenReturn(TEAM); + when(helperService.getCxProject(any())).thenReturn(PROJECT_NAME); + when(helperService.getPresetFromSources(any())).thenReturn(PRESET); when(helperService.isBranchProtected(anyString(), anyList(), any())).thenCallRealMethod(); } private void initMockGitHubController() { doNothing().when(gitHubControllerSpy).verifyHmacSignature(any(), any()); } - + private void initServices() { ProjectNameGenerator projectNameGeneratorSpy = spy(new ProjectNameGenerator(helperService, cxProperties, null)); - initProjectNameGeneratorSpy(projectNameGeneratorSpy); - - ScanRequestConverter scanRequestConverter = new ScanRequestConverter(helperService, - cxProperties, - cxClientMock, - flowProperties, - gitHubService, - null); - - SastScanner sastScanner = new SastScanner(null, - cxClientMock, - helperService, - cxProperties, - flowProperties, - null, - null, - scanRequestConverter, - null, - projectNameGeneratorSpy); - List scanners = new LinkedList<>(); + initProjectNameGeneratorSpy(projectNameGeneratorSpy); + + ScanRequestConverter scanRequestConverter = new ScanRequestConverter(helperService, cxProperties, cxClientMock, flowProperties, gitHubService, null,null); + SastScanner sastScanner = new SastScanner(null, cxClientMock, helperService, cxProperties, flowProperties, null, null, scanRequestConverter, null, projectNameGeneratorSpy); + List scanners= new LinkedList<>(); scanners.add(sastScanner); - + FlowService flowServiceSpy = spy(new FlowService(scanners, projectNameGeneratorSpy, null)); - + //gitHubControllerSpy is a spy which will run real methods. //It will connect to a real github repository to read a real cx.config file //And thus it will work with real gitHubService @@ -256,7 +241,7 @@ private void initServices() { sastScanner, filterFactory, configOverrider)); - + } private void initProjectNameGeneratorSpy(ProjectNameGenerator projectNameGenerator) { @@ -264,11 +249,11 @@ private void initProjectNameGeneratorSpy(ProjectNameGenerator projectNameGenerat } public Object interceptProjectName(InvocationOnMock invocation) { - try { - calculatedProjectName = (String) invocation.callRealMethod(); - } catch (Throwable throwable) { - fail(throwable.getMessage()); - } + try { + calculatedProjectName = (String)invocation.callRealMethod(); + } catch (Throwable throwable) { + fail(throwable.getMessage()); + } return null; + } } -}