diff --git a/README.md b/README.md index ebb3228..d843f28 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,7 @@ * [Manual install](#manual-install) * [Usage](#usage) * [Importing Lists](#importing-lists) + * [Importing Lists - Advanced](#importing-lists---advanced) * [How to compile from source code](#how-to-compile-from-source-code) * [Using Maven from CLI](#using-maven-from-cli) * [About us](#about-us) @@ -132,6 +133,45 @@ Both `CSV` and `JSON` files with their respective extensions are supported. Regexes must be compliant with the Java's Regexes Style. If in doubt, use [regex101](https://regex101.com/) with the `Java 8` flavour to test regexes. +### Importing Lists - Advanced + +In the lists it's also possible to specify the sections that the regexes should match. If no section is specified, by default `all` sections are used. + +Each HTTP item composed of a request and a response is divided into the following matchable sections: + +- `req_url`: Request URL; +- `req_headers`: Request headers without the request line (first line); +- `req_body`: Request body; +- `res_headers`: Response headers without the status line (first line); +- `res_body`: Response body; + +The following sections groups are also available to make things easier and more readable: + +- `req`: req_url + req_headers + req_body +- `res`: res_headers + res_body +- `all`: req + res + +To specify the sections to match use the following format: + +- For **CSV** files, add a `sections` column and for each row add the list of sections to be matched as a string with sections delimited by the character `|`: + + ```csv + "description","regex","sections" + "Google e-mail","\w+@gmail.com","req|res_body" + ``` + +- For **JSON** files, add a `sections` field containing the array of sections to be matched as strings: + + ```json + [ + { + "description": "Google e-mail", + "regex": "\\w+@gmail.com", + "sections": [ "req", "res_body" ] + } + ] + ``` + ## How to compile from source code ### Using Maven from CLI diff --git a/src/main/java/com/cys4/sensitivediscoverer/RegexListViewer.java b/src/main/java/com/cys4/sensitivediscoverer/RegexListViewer.java index bc0ed78..dbfd048 100644 --- a/src/main/java/com/cys4/sensitivediscoverer/RegexListViewer.java +++ b/src/main/java/com/cys4/sensitivediscoverer/RegexListViewer.java @@ -15,6 +15,7 @@ import javax.swing.*; import javax.swing.border.TitledBorder; import java.awt.*; +import java.util.Arrays; import java.util.List; import java.util.function.Supplier; @@ -251,22 +252,15 @@ private JMenuItem createNewRegexMenuItem(RegexListContext ctx, private JMenuItem createListSaveMenuItem(List regexEntities) { JMenuItem menuItem = new JMenuItem(getLocaleString("options-list-save")); - String[] options = {"JSON", "CSV"}; + List options = Arrays.asList("JSON", "CSV"); menuItem.addActionListener(actionEvent -> { - int dialog = JOptionPane.showOptionDialog( - null, - getLocaleString("options-list-save-formatDialogMessage"), - getLocaleString("options-list-save-formatDialogTitle"), - JOptionPane.DEFAULT_OPTION, - JOptionPane.QUESTION_MESSAGE, - null, - options, - null - ); - if ("JSON".equals(options[dialog])) { - Utils.saveListToJSON(regexEntities); - } else if ("CSV".equals(options[dialog])) { - Utils.saveListToCSV(regexEntities); + + String fileName = Utils.selectFile(options, false); + + if (fileName.toUpperCase().endsWith("JSON")) { + Utils.saveListToJSON(fileName, regexEntities); + } else if (fileName.toUpperCase().endsWith("CSV")) { + Utils.saveListToCSV(fileName, regexEntities); } }); @@ -276,24 +270,18 @@ private JMenuItem createListSaveMenuItem(List regexEntities) { private JMenuItem createListOpenMenuItem(RegexListContext ctx, JPanel tabPaneOptions, RegexListViewerTableModel tableModel) { - String[] options = {"JSON", "CSV"}; + List options = Arrays.asList("JSON", "CSV"); JMenuItem menuItem = new JMenuItem(getLocaleString("options-list-open")); menuItem.addActionListener(actionEvent -> { - int dialog = JOptionPane.showOptionDialog( - null, - getLocaleString("options-list-open-formatDialogMessage"), - getLocaleString("options-list-open-formatDialogTitle"), - JOptionPane.DEFAULT_OPTION, - JOptionPane.QUESTION_MESSAGE, - null, - options, - null - ); - if ("JSON".equals(options[dialog])) { - Utils.openListFromJSON(ctx); - } else if ("CSV".equals(options[dialog])) { - Utils.openListFromCSV(ctx); + + String fileName = Utils.selectFile(options, true); + + if (fileName.toUpperCase().endsWith("JSON")) { + Utils.openListFromJSON(fileName, ctx); + } else if (fileName.toUpperCase().endsWith("CSV")) { + Utils.openListFromCSV(fileName, ctx); } + tableModel.fireTableDataChanged(); tabPaneOptions.validate(); tabPaneOptions.repaint(); diff --git a/src/main/java/com/cys4/sensitivediscoverer/Utils.java b/src/main/java/com/cys4/sensitivediscoverer/Utils.java index e989193..5967c89 100644 --- a/src/main/java/com/cys4/sensitivediscoverer/Utils.java +++ b/src/main/java/com/cys4/sensitivediscoverer/Utils.java @@ -18,6 +18,7 @@ import java.lang.reflect.Type; import java.nio.charset.StandardCharsets; import java.nio.file.Files; +import java.nio.file.Path; import java.util.List; import java.util.*; import java.util.regex.Matcher; @@ -84,28 +85,40 @@ public static void saveToFile(String extensionName, List lines) { } /** - * Open JFileChooser to get lines from a file + * Open JFileChooser to get a file name * - * @param extensionName the extension to filter files - * @return The lines from the file, or null if there was an error + * @param extensionNames the extensions to filter files + * @param openFile Set to true if the file should be opened, false if it should be saved + * @return The filename, or empty string if there was an error */ - public static List linesFromFile(String extensionName) { + public static String selectFile(List extensionNames, boolean openFile) { JFrame parentFrame = new JFrame(); JFileChooser fileChooser = new JFileChooser(); - FileNameExtensionFilter filter = new FileNameExtensionFilter("." + extensionName, extensionName); - fileChooser.setFileFilter(filter); - fileChooser.setDialogTitle(getLocaleString("utils-linesFromFile-importFile")); - int userSelection = fileChooser.showOpenDialog(parentFrame); - if (userSelection != JFileChooser.APPROVE_OPTION) return null; + fileChooser.setAcceptAllFileFilterUsed(false); - File selectedFile = fileChooser.getSelectedFile(); - try { - return Files.readAllLines(selectedFile.toPath()); - } catch (IOException e) { - e.printStackTrace(); - return null; - } + //add supported extensions + extensionNames.stream() + .map(extensionName -> new FileNameExtensionFilter("." + extensionName, extensionName)) + .forEachOrdered(fileChooser::addChoosableFileFilter); + + //set window title to Open or Save + fileChooser.setDialogTitle(getLocaleString(openFile ? + "utils-linesFromFile-importFile" + : "utils-saveToFile-exportFile")); + + //show the Open or Save window + int userSelection = openFile ? + fileChooser.showOpenDialog(parentFrame) : + fileChooser.showSaveDialog(parentFrame); + + if (userSelection != JFileChooser.APPROVE_OPTION) return ""; + + String exportFilePath = fileChooser.getSelectedFile().getAbsolutePath(); + + String selectedExt = fileChooser.getFileFilter().getDescription().toLowerCase(); + + return exportFilePath.toLowerCase().endsWith(selectedExt) ? exportFilePath : exportFilePath + selectedExt; } /** @@ -177,7 +190,26 @@ public static InputStream getResourceAsStream(String name) { return Utils.class.getClassLoader().getResourceAsStream(name); } - public static void saveListToCSV(List regexEntities) { + public static void writeLinesToFile(String fileName, List lines) { + try { + PrintWriter pwt = new PrintWriter(fileName, StandardCharsets.UTF_8); + lines.forEach(pwt::println); + pwt.close(); + } catch (Exception e) { + e.printStackTrace(); + } + } + + private static List readLinesFromFile(String fileName) { + try { + return Files.readAllLines(Path.of(fileName)); + } catch (IOException e) { + e.printStackTrace(); + return null; + } + } + + public static void saveListToCSV(String csvFile, List regexEntities) { List lines = new ArrayList<>(); lines.add("\"description\",\"regex\",\"sections\""); @@ -190,10 +222,10 @@ public static void saveListToCSV(List regexEntities) { lines.add(String.format("\"%s\",\"%s\",\"%s\"", description, regex, sections)); }); - Utils.saveToFile("csv", lines); + writeLinesToFile(csvFile, lines); } - public static void saveListToJSON(List regexEntities) { + public static void saveListToJSON(String jsonFile, List regexEntities) { List lines = new ArrayList<>(); regexEntities.forEach(regexEntity -> { @@ -210,29 +242,38 @@ public static void saveListToJSON(List regexEntities) { Gson gson = builder.create(); Type tListEntries = (new TypeToken>() { }).getType(); - Utils.saveToFile("json", List.of(gson.toJson(lines, tListEntries))); + + writeLinesToFile(jsonFile, List.of(gson.toJson(lines, tListEntries))); } - public static void openListFromCSV(RegexListContext ctx) { + public static void openListFromCSV(String csvFile, RegexListContext ctx) { StringBuilder alreadyAddedMsg = new StringBuilder(); - List lines = Utils.linesFromFile("csv"); + List lines = readLinesFromFile(csvFile); if (Objects.isNull(lines)) return; - // for each line after the first (Headers Line) - lines.subList(1, lines.size()).forEach(line -> { + //Skip header line if present + int startRow = (lines.get(0).contains("\"description\",\"regex\"")) ? 1 : 0; + + // for each line + lines.subList(startRow, lines.size()).forEach(line -> { Matcher matcher = RegexEntity.checkRegexEntityFromCSV(line); - if (!matcher.find()) return; + + if (!matcher.find()) + return; + + //load sections if presents, otherwise set all sections + boolean hasSections = !(matcher.group(3) == null || matcher.group(3).isBlank()); String description = matcher.group(1).replaceAll("\"\"", "\""); String regex = matcher.group(2).replaceAll("\"\"", "\""); - List sections = List.of(matcher.group(3).replaceAll("\"\"", "\"").split("\\|")); + List sections = hasSections ? List.of(matcher.group(3).replaceAll("\"\"", "\"").split("\\|")) : null; RegexEntity newRegexEntity = new RegexEntity( description, regex, true, - ProxyItemSection.deserializeSections(sections) + hasSections ? ProxyItemSection.deserializeSections(sections) : ProxyItemSection.ALL ); if (!ctx.getRegexEntities().contains(newRegexEntity)) { @@ -248,11 +289,11 @@ public static void openListFromCSV(RegexListContext ctx) { alreadyAddedMsg.toString()); } - public static void openListFromJSON(RegexListContext ctx) { + public static void openListFromJSON(String jsonFile, RegexListContext ctx) { Gson gson = new Gson(); StringBuilder alreadyAddedMsg = new StringBuilder(); - List lines = Utils.linesFromFile("json"); + List lines = readLinesFromFile(jsonFile); if (Objects.isNull(lines)) return; Type tArrayListRegexEntity = new TypeToken>() { diff --git a/src/main/java/com/cys4/sensitivediscoverer/model/RegexEntity.java b/src/main/java/com/cys4/sensitivediscoverer/model/RegexEntity.java index c61a4fd..319ead7 100644 --- a/src/main/java/com/cys4/sensitivediscoverer/model/RegexEntity.java +++ b/src/main/java/com/cys4/sensitivediscoverer/model/RegexEntity.java @@ -58,13 +58,13 @@ public RegexEntity(RegexEntity entity) throws IllegalArgumentException { /** * Checks if the input is in the format: `"Description","Regex","Sections"` - * + * Matches also if sections are not present, in this case group(3) is null * @param input Text string to check against the format * @return If the input was in the correct format, a Matcher object where group(1) = description, group(2) = regex, group(3) = sections */ public static Matcher checkRegexEntityFromCSV(String input) { return Pattern - .compile("^\\s*[\"'](.*?)[\"']\\s*,\\s*[\"'](.+?)[\"']\\s*,\\s*[\"'](.+?)[\"']\\s*$") + .compile("^[\t ]*[\"'](.+?)[\"'][\t ]*,[\t ]*[\"'](.+?)[\"'][\t ]*(?:,[\t ]*[\"'](.+?)[\"'][\t ]*)?$") .matcher(input); } diff --git a/src/main/java/com/cys4/sensitivediscoverer/tab/LoggerTab.java b/src/main/java/com/cys4/sensitivediscoverer/tab/LoggerTab.java index 91328ac..cbec59a 100644 --- a/src/main/java/com/cys4/sensitivediscoverer/tab/LoggerTab.java +++ b/src/main/java/com/cys4/sensitivediscoverer/tab/LoggerTab.java @@ -321,6 +321,9 @@ private JToggleButton createExportLogsButton() { JMenuItem itemToCSV = new JMenuItem(getLocaleString("common-toCSV")); itemToCSV.addActionListener(actionEvent -> { + String csvFile = Utils.selectFile(List.of("CSV"), false); + if (csvFile.isBlank()) return; + java.util.List lines = new ArrayList<>(); lines.add(String.format("\"%s\",\"%s\"", @@ -334,12 +337,15 @@ private JToggleButton createExportLogsButton() { lines.add(String.format("\"%s\",\"%s\"", url, matchEscaped)); } - Utils.saveToFile("csv", lines); + Utils.writeLinesToFile(csvFile, lines); }); menu.add(itemToCSV); JMenuItem itemToJSON = new JMenuItem(getLocaleString("common-toJSON")); itemToJSON.addActionListener(actionEvent -> { + String jsonFile = Utils.selectFile(List.of("JSON"), false); + if (jsonFile.isBlank()) return; + java.util.List lines = new ArrayList<>(); String prop1 = LogsTableModel.Column.URL.getNameFormatted(); @@ -357,7 +363,8 @@ private JToggleButton createExportLogsButton() { Gson gson = builder.create(); Type tListEntries = new TypeToken>() { }.getType(); - Utils.saveToFile("json", List.of(gson.toJson(lines, tListEntries))); + + Utils.writeLinesToFile(jsonFile, List.of(gson.toJson(lines, tListEntries))); }); menu.add(itemToJSON); diff --git a/src/main/resources/TextUI_en_US.properties b/src/main/resources/TextUI_en_US.properties index 99cf880..bad9fe8 100644 --- a/src/main/resources/TextUI_en_US.properties +++ b/src/main/resources/TextUI_en_US.properties @@ -44,13 +44,9 @@ options-list-clear=Clear list options-list-clear-message=Delete ALL the regex in the list? options-list-clear-title=Clear list? options-list-open=Open list... -options-list-open-formatDialogMessage=Choose the import format -options-list-open-formatDialogTitle=Import regex list options-list-open-alreadyPresentWarn=These regexes are already present options-list-open-alreadyPresentTitle=Regexes alert options-list-save=Save list... -options-list-save-formatDialogMessage=Choose the export format -options-list-save-formatDialogTitle=Export regex list options-list-new=New regex options-list-new-dialogTitle=Add a regular expression options-list-delete=Delete selected diff --git a/src/test/java/com/cys4/sensitivediscoverer/model/RegexEntityTest.java b/src/test/java/com/cys4/sensitivediscoverer/model/RegexEntityTest.java index 1bc4480..26c8f41 100644 --- a/src/test/java/com/cys4/sensitivediscoverer/model/RegexEntityTest.java +++ b/src/test/java/com/cys4/sensitivediscoverer/model/RegexEntityTest.java @@ -1,11 +1,14 @@ package com.cys4.sensitivediscoverer.model; +import com.cys4.sensitivediscoverer.Utils; +import com.cys4.sensitivediscoverer.tab.LoggerTab; import org.junit.jupiter.api.Test; import java.util.EnumSet; import java.util.regex.Matcher; import static org.assertj.core.api.Assertions.*; +import com.cys4.sensitivediscoverer.tab.LoggerTab; class RegexEntityTest { @@ -51,14 +54,24 @@ void testActiveFlag() { assertThat(entity.isActive()).isTrue(); } + @Test + void checkRegexEntityFromCSVNoSections() { + Matcher csvMatcher = RegexEntity.checkRegexEntityFromCSV("\"description\",\"regex\""); + assertThat(csvMatcher.find()).isTrue(); + assertThat(csvMatcher.groupCount()).isEqualTo(3); + assertThat(csvMatcher.group(1)).isEqualTo("description"); + assertThat(csvMatcher.group(2)).isEqualTo("regex"); + assertThat(csvMatcher.group(3)).isNullOrEmpty(); + } + @Test void checkRegexEntityFromCSV() { - Matcher csvMatcher = RegexEntity.checkRegexEntityFromCSV("\"description\",\"regex\",\"SECTION_1,SECTION_2\""); + Matcher csvMatcher = RegexEntity.checkRegexEntityFromCSV("\"description\",\"regex\",\"SECTION_1|SECTION_2\""); assertThat(csvMatcher.find()).isTrue(); assertThat(csvMatcher.groupCount()).isEqualTo(3); assertThat(csvMatcher.group(1)).isEqualTo("description"); assertThat(csvMatcher.group(2)).isEqualTo("regex"); - assertThat(csvMatcher.group(3)).isEqualTo("SECTION_1,SECTION_2"); + assertThat(csvMatcher.group(3)).isEqualTo("SECTION_1|SECTION_2"); } @Test