Skip to content

Commit

Permalink
Merge pull request #48 from CYS4srl/general-refactor
Browse files Browse the repository at this point in the history 
General refactor
  • Loading branch information
@beryxz
beryxz authored Mar 20, 2024
2 parents eea7b88 + fe1126a commit 93b95b1
Show file tree
Hide file tree
Showing 30 changed files with 596 additions and 551 deletions.
2 changes: 1 addition & 1 deletion src/main/java/burp/BurpExtender.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
import burp.api.montoya.BurpExtension;
import burp.api.montoya.MontoyaApi;
import com.cys4.sensitivediscoverer.MainUI;
import com.cys4.sensitivediscoverer.Utils;
import com.cys4.sensitivediscoverer.utils.Utils;

public class BurpExtender implements BurpExtension {
@Override
Expand Down
16 changes: 11 additions & 5 deletions src/main/java/com/cys4/sensitivediscoverer/MainUI.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,16 +7,17 @@
import burp.api.montoya.MontoyaApi;
import com.cys4.sensitivediscoverer.model.RegexEntity;
import com.cys4.sensitivediscoverer.model.ScannerOptions;
import com.cys4.sensitivediscoverer.tab.AboutTab;
import com.cys4.sensitivediscoverer.tab.ApplicationTab;
import com.cys4.sensitivediscoverer.tab.LoggerTab;
import com.cys4.sensitivediscoverer.tab.OptionsTab;
import com.cys4.sensitivediscoverer.ui.tab.AboutTab;
import com.cys4.sensitivediscoverer.ui.tab.ApplicationTab;
import com.cys4.sensitivediscoverer.ui.tab.LoggerTab;
import com.cys4.sensitivediscoverer.ui.tab.OptionsTab;

import javax.swing.*;
import java.awt.*;
import java.util.List;
import java.util.Properties;

import static com.cys4.sensitivediscoverer.Utils.loadConfigFile;
import static com.cys4.sensitivediscoverer.utils.Utils.loadConfigFile;

public class MainUI {
private final MontoyaApi burpApi;
Expand Down Expand Up @@ -100,4 +101,9 @@ public String getExtensionName() {
return configProperties.getProperty("ui.extension_name");
}

public static final class UIOptions {
public static final Font H1_FONT = new Font("SansSerif", Font.BOLD, 16);
public static final Font H2_FONT = new Font("SansSerif", Font.BOLD, 14);
public static final Color ACCENT_COLOR = new Color(255, 102, 51);
}
}
95 changes: 35 additions & 60 deletions src/main/java/com/cys4/sensitivediscoverer/RegexScanner.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,37 +6,34 @@

import burp.api.montoya.MontoyaApi;
import burp.api.montoya.core.ByteArray;
import burp.api.montoya.http.message.HttpHeader;
import burp.api.montoya.http.message.MimeType;
import burp.api.montoya.http.message.requests.HttpRequest;
import burp.api.montoya.http.message.responses.HttpResponse;
import burp.api.montoya.proxy.ProxyHttpRequestResponse;
import com.cys4.sensitivediscoverer.model.HttpRecord;
import com.cys4.sensitivediscoverer.model.LogEntity;
import com.cys4.sensitivediscoverer.model.RegexEntity;
import com.cys4.sensitivediscoverer.model.ScannerOptions;
import com.cys4.sensitivediscoverer.utils.BurpUtils;
import com.cys4.sensitivediscoverer.utils.ScannerUtils;

import java.nio.charset.StandardCharsets;
import java.util.EnumSet;
import java.util.List;
import java.util.Objects;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.function.Consumer;
import java.util.function.Function;
import java.util.regex.MatchResult;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import java.util.stream.Stream;

public class RegexScanner {
private final MontoyaApi burpApi;
private final ScannerOptions scannerOptions;
private final List<RegexEntity> generalRegexList;
private final List<RegexEntity> extensionsRegexList;
/**
* List of MIME types to ignore while scanning when the relevant option is enabled
*/
private final EnumSet<MimeType> blacklistedMimeTypes = EnumSet.of(
public static final EnumSet<MimeType> blacklistedMimeTypes = EnumSet.of(
MimeType.APPLICATION_FLASH,
MimeType.FONT_WOFF,
MimeType.FONT_WOFF2,
Expand All @@ -52,6 +49,10 @@ public class RegexScanner {
MimeType.SOUND,
MimeType.VIDEO
);
private final MontoyaApi burpApi;
private final ScannerOptions scannerOptions;
private final List<RegexEntity> generalRegexList;
private final List<RegexEntity> extensionsRegexList;
/**
* Flag that indicates if the scan must be interrupted.
* Used to interrupt scan before completion.
Expand Down Expand Up @@ -124,78 +125,52 @@ private void analyzeSingleMessage(List<RegexEntity> regexList,
Consumer<LogEntity> logEntriesCallback) {
// The initial checks must be kept ordered based on the amount of information required from Burp APIs.
// API calls (to MontoyaAPI) for specific parts of the request/response are quite slow.

// check if URL is in scope
HttpRequest request = proxyEntry.finalRequest();
if (scannerOptions.isFilterInScopeCheckbox() && (!request.isInScope())) return;

// skip empty responses
if (ScannerUtils.isUrlOutOfScope(scannerOptions, request)) return;
HttpResponse response = proxyEntry.response();
if (Objects.isNull(response)) return;

// check for blacklisted MIME types
if (scannerOptions.isFilterSkipMediaTypeCheckbox() && isMimeTypeBlacklisted(response.statedMimeType(), response.inferredMimeType()))
return;

// check for max request size
if (ScannerUtils.isResponseEmpty(response)) return;
if (ScannerUtils.isMimeTypeBlacklisted(scannerOptions, response)) return;
ByteArray responseBody = response.body();
if (scannerOptions.isFilterSkipMaxSizeCheckbox() && responseBody.length() > scannerOptions.getConfigMaxResponseSize())
return;
if (ScannerUtils.isResponseSizeOverMaxSize(scannerOptions, responseBody)) return;

// Not using request.bodyToString() as it's extremely slow
String requestBodyDecoded = new String(request.body().getBytes(), StandardCharsets.UTF_8);
String requestHeaders = String.join("\r\n", request.headers().stream().map(HttpHeader::toString).toList());
// Not using response.bodyToString() as it's extremely slow
String responseBodyDecoded = new String(responseBody.getBytes(), StandardCharsets.UTF_8);
String responseHeaders = String.join("\r\n", response.headers().stream().map(HttpHeader::toString).toList());
// Not using bodyToString() as it's extremely slow
String requestUrl = request.url();
String requestBodyDecoded = BurpUtils.convertByteArrayToString(request.body());
String requestHeaders = BurpUtils.convertHttpHeaderListToString(request.headers());
String responseBodyDecoded = BurpUtils.convertByteArrayToString(responseBody);
String responseHeaders = BurpUtils.convertHttpHeaderListToString(response.headers());

for (RegexEntity regex : regexList) {
if (this.interruptScan) return;
if (!regex.isActive()) continue;

getRegexMatchers(regex, request.url(), requestHeaders, requestBodyDecoded, responseHeaders, responseBodyDecoded)
.parallelStream()
.forEach(matcher -> {
while (matcher.find()) {
logEntriesCallback.accept(new LogEntity(request, response, regex, matcher.group()));
}
});
Consumer<String> logMatchCallback = match -> logEntriesCallback.accept(new LogEntity(request, response, regex, match));
performMatchingOnMessage(regex, logMatchCallback, new HttpRecord(requestUrl, requestHeaders, requestBodyDecoded, responseHeaders, responseBodyDecoded));
}
}

private List<Matcher> getRegexMatchers(RegexEntity regex,
String requestUrl,
String requestHeaders,
String requestBody,
String responseHeaders,
String responseBody) {
private void performMatchingOnMessage(RegexEntity regex, Consumer<String> logMatchCallback, HttpRecord requestResponse) {
getRegexMatchers(regex, requestResponse)
.flatMap(Matcher::results)
.map(MatchResult::group)
.forEach(logMatchCallback);
}

private Stream<Matcher> getRegexMatchers(RegexEntity regex, HttpRecord requestResponse) {
Pattern regexCompiled = regex.getRegexCompiled();

//TODO keep track of section where regex matched. Show the section in the logger table;
return regex.getSections()
.parallelStream()
.map(proxyItemSection -> switch (proxyItemSection) {
case REQ_URL -> requestUrl;
case REQ_HEADERS -> requestHeaders;
case REQ_BODY -> requestBody;
case RES_HEADERS -> responseHeaders;
case RES_BODY -> responseBody;
case REQ_URL -> requestResponse.requestUrl();
case REQ_HEADERS -> requestResponse.requestHeaders();
case REQ_BODY -> requestResponse.requestBody();
case RES_HEADERS -> requestResponse.responseHeaders();
case RES_BODY -> requestResponse.responseBody();
})
.filter(Objects::nonNull)
.map(regexCompiled::matcher)
.collect(Collectors.toList());
}

/**
* Checks if the MimeType is inside the list of blacklisted mime types "mime_types.json".
* If the stated mime type in the header isBlank, then the inferred mime type is used.
*
* @param statedMimeType Stated mime type from a HttpResponse object
* @param inferredMimeType Inferred mime type from a HttpResponse object
* @return True if the mime type is blacklisted
*/
private boolean isMimeTypeBlacklisted(MimeType statedMimeType, MimeType inferredMimeType) {
return blacklistedMimeTypes.contains(Objects.isNull(statedMimeType) ? inferredMimeType : statedMimeType);
.map(regexCompiled::matcher);
}

/**
Expand Down
1 change: 1 addition & 0 deletions src/main/java/com/cys4/sensitivediscoverer/RegexSeeder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import com.cys4.sensitivediscoverer.model.JsonRegexEntity;
import com.cys4.sensitivediscoverer.model.ProxyItemSection;
import com.cys4.sensitivediscoverer.model.RegexEntity;
import com.cys4.sensitivediscoverer.utils.Utils;
import com.google.gson.Gson;
import com.google.gson.reflect.TypeToken;

Expand Down
Loading

0 comments on commit 93b95b1

Please sign in to comment.