Merge pull request #43 from CYS4srl/improve-import-export

Improvement import/export Regex

README.md

@@ -13,6 +13,7 @@
     * [Manual install](#manual-install)
   * [Usage](#usage)
     * [Importing Lists](#importing-lists)
+    * [Importing Lists - Advanced](#importing-lists---advanced)
   * [How to compile from source code](#how-to-compile-from-source-code)
     * [Using Maven from CLI](#using-maven-from-cli)
   * [About us](#about-us)
@@ -132,6 +133,45 @@ Both `CSV` and `JSON` files with their respective extensions are supported.
 
 Regexes must be compliant with the Java's Regexes Style. If in doubt, use [regex101](https://regex101.com/) with the `Java 8` flavour to test regexes.
 
+### Importing Lists - Advanced
+
+In the lists it's also possible to specify the sections that the regexes should match. If no section is specified, by default `all` sections are used.
+
+Each HTTP item composed of a request and a response is divided into the following matchable sections:
+
+- `req_url`: Request URL;
+- `req_headers`: Request headers without the request line (first line);
+- `req_body`: Request body;
+- `res_headers`: Response headers without the status line (first line);
+- `res_body`: Response body;
+
+The following sections groups are also available to make things easier and more readable:
+
+- `req`: req_url + req_headers + req_body
+- `res`: res_headers + res_body
+- `all`: req + res
+
+To specify the sections to match use the following format:
+
+- For **CSV** files, add a `sections` column and for each row add the list of sections to be matched as a string with sections delimited by the character `|`:
+
+  ```csv
+  "description","regex","sections"
+  "Google e-mail","\w+@gmail.com","req|res_body"
+  ```
+
+- For **JSON** files, add a `sections` field containing the array of sections to be matched as strings:
+
+  ```json
+  [
+    {
+      "description": "Google e-mail",
+      "regex": "\\w+@gmail.com",
+      "sections": [ "req", "res_body" ]
+    }
+  ]
+  ```
+
 ## How to compile from source code
 
 ### Using Maven from CLI

src/main/java/com/cys4/sensitivediscoverer/RegexListViewer.java

@@ -15,6 +15,7 @@
 import javax.swing.*;
 import javax.swing.border.TitledBorder;
 import java.awt.*;
+import java.util.Arrays;
 import java.util.List;
 import java.util.function.Supplier;
 
@@ -251,22 +252,15 @@ private JMenuItem createNewRegexMenuItem(RegexListContext ctx,
 
     private JMenuItem createListSaveMenuItem(List<RegexEntity> regexEntities) {
         JMenuItem menuItem = new JMenuItem(getLocaleString("options-list-save"));
-        String[] options = {"JSON", "CSV"};
+        List<String> options = Arrays.asList("JSON", "CSV");
         menuItem.addActionListener(actionEvent -> {
-            int dialog = JOptionPane.showOptionDialog(
-                    null,
-                    getLocaleString("options-list-save-formatDialogMessage"),
-                    getLocaleString("options-list-save-formatDialogTitle"),
-                    JOptionPane.DEFAULT_OPTION,
-                    JOptionPane.QUESTION_MESSAGE,
-                    null,
-                    options,
-                    null
-            );
-            if ("JSON".equals(options[dialog])) {
-                Utils.saveListToJSON(regexEntities);
-            } else if ("CSV".equals(options[dialog])) {
-                Utils.saveListToCSV(regexEntities);
+
+            String fileName = Utils.selectFile(options, false);
+
+            if (fileName.toUpperCase().endsWith("JSON")) {
+                Utils.saveListToJSON(fileName, regexEntities);
+            } else if (fileName.toUpperCase().endsWith("CSV")) {
+                Utils.saveListToCSV(fileName, regexEntities);
             }
         });
 
@@ -276,24 +270,18 @@ private JMenuItem createListSaveMenuItem(List<RegexEntity> regexEntities) {
     private JMenuItem createListOpenMenuItem(RegexListContext ctx,
                                              JPanel tabPaneOptions,
                                              RegexListViewerTableModel tableModel) {
-        String[] options = {"JSON", "CSV"};
+        List<String> options = Arrays.asList("JSON", "CSV");
         JMenuItem menuItem = new JMenuItem(getLocaleString("options-list-open"));
         menuItem.addActionListener(actionEvent -> {
-            int dialog = JOptionPane.showOptionDialog(
-                    null,
-                    getLocaleString("options-list-open-formatDialogMessage"),
-                    getLocaleString("options-list-open-formatDialogTitle"),
-                    JOptionPane.DEFAULT_OPTION,
-                    JOptionPane.QUESTION_MESSAGE,
-                    null,
-                    options,
-                    null
-            );
-            if ("JSON".equals(options[dialog])) {
-                Utils.openListFromJSON(ctx);
-            } else if ("CSV".equals(options[dialog])) {
-                Utils.openListFromCSV(ctx);
+
+            String fileName = Utils.selectFile(options, true);
+
+            if (fileName.toUpperCase().endsWith("JSON")) {
+                Utils.openListFromJSON(fileName, ctx);
+            } else if (fileName.toUpperCase().endsWith("CSV")) {
+                Utils.openListFromCSV(fileName, ctx);
             }
+
             tableModel.fireTableDataChanged();
             tabPaneOptions.validate();
             tabPaneOptions.repaint();

src/main/java/com/cys4/sensitivediscoverer/Utils.java

@@ -18,6 +18,7 @@
 import java.lang.reflect.Type;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
+import java.nio.file.Path;
 import java.util.List;
 import java.util.*;
 import java.util.regex.Matcher;
@@ -84,28 +85,40 @@ public static void saveToFile(String extensionName, List<String> lines) {
     }
 
     /**
-     * Open JFileChooser to get lines from a file
+     * Open JFileChooser to get a file name
      *
-     * @param extensionName the extension to filter files
-     * @return The lines from the file, or null if there was an error
+     * @param extensionNames the extensions to filter files
+     * @param openFile       Set to true if the file should be opened, false if it should be saved
+     * @return The filename, or empty string if there was an error
      */
-    public static List<String> linesFromFile(String extensionName) {
+    public static String selectFile(List<String> extensionNames, boolean openFile) {
         JFrame parentFrame = new JFrame();
         JFileChooser fileChooser = new JFileChooser();
-        FileNameExtensionFilter filter = new FileNameExtensionFilter("." + extensionName, extensionName);
-        fileChooser.setFileFilter(filter);
-        fileChooser.setDialogTitle(getLocaleString("utils-linesFromFile-importFile"));
 
-        int userSelection = fileChooser.showOpenDialog(parentFrame);
-        if (userSelection != JFileChooser.APPROVE_OPTION) return null;
+        fileChooser.setAcceptAllFileFilterUsed(false);
 
-        File selectedFile = fileChooser.getSelectedFile();
-        try {
-            return Files.readAllLines(selectedFile.toPath());
-        } catch (IOException e) {
-            e.printStackTrace();
-            return null;
-        }
+        //add supported extensions
+        extensionNames.stream()
+                .map(extensionName -> new FileNameExtensionFilter("." + extensionName, extensionName))
+                .forEachOrdered(fileChooser::addChoosableFileFilter);
+
+        //set window title to Open or Save
+        fileChooser.setDialogTitle(getLocaleString(openFile ?
+                "utils-linesFromFile-importFile"
+                : "utils-saveToFile-exportFile"));
+
+        //show the Open or Save window
+        int userSelection = openFile ?
+                fileChooser.showOpenDialog(parentFrame) :
+                fileChooser.showSaveDialog(parentFrame);
+
+        if (userSelection != JFileChooser.APPROVE_OPTION) return "";
+
+        String exportFilePath = fileChooser.getSelectedFile().getAbsolutePath();
+
+        String selectedExt = fileChooser.getFileFilter().getDescription().toLowerCase();
+
+        return exportFilePath.toLowerCase().endsWith(selectedExt) ? exportFilePath : exportFilePath + selectedExt;
     }
 
     /**
@@ -177,7 +190,26 @@ public static InputStream getResourceAsStream(String name) {
         return Utils.class.getClassLoader().getResourceAsStream(name);
     }
 
-    public static void saveListToCSV(List<RegexEntity> regexEntities) {
+    public static void writeLinesToFile(String fileName, List<String> lines) {
+        try {
+            PrintWriter pwt = new PrintWriter(fileName, StandardCharsets.UTF_8);
+            lines.forEach(pwt::println);
+            pwt.close();
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+
+    private static List<String> readLinesFromFile(String fileName) {
+        try {
+            return Files.readAllLines(Path.of(fileName));
+        } catch (IOException e) {
+            e.printStackTrace();
+            return null;
+        }
+    }
+
+    public static void saveListToCSV(String csvFile, List<RegexEntity> regexEntities) {
         List<String> lines = new ArrayList<>();
 
         lines.add("\"description\",\"regex\",\"sections\"");
@@ -190,10 +222,10 @@ public static void saveListToCSV(List<RegexEntity> regexEntities) {
             lines.add(String.format("\"%s\",\"%s\",\"%s\"", description, regex, sections));
         });
 
-        Utils.saveToFile("csv", lines);
+        writeLinesToFile(csvFile, lines);
     }
 
-    public static void saveListToJSON(List<RegexEntity> regexEntities) {
+    public static void saveListToJSON(String jsonFile, List<RegexEntity> regexEntities) {
         List<JsonObject> lines = new ArrayList<>();
 
         regexEntities.forEach(regexEntity -> {
@@ -210,29 +242,38 @@ public static void saveListToJSON(List<RegexEntity> regexEntities) {
         Gson gson = builder.create();
         Type tListEntries = (new TypeToken<ArrayList<JsonObject>>() {
         }).getType();
-        Utils.saveToFile("json", List.of(gson.toJson(lines, tListEntries)));
+
+        writeLinesToFile(jsonFile, List.of(gson.toJson(lines, tListEntries)));
     }
 
-    public static void openListFromCSV(RegexListContext ctx) {
+    public static void openListFromCSV(String csvFile, RegexListContext ctx) {
         StringBuilder alreadyAddedMsg = new StringBuilder();
 
-        List<String> lines = Utils.linesFromFile("csv");
+        List<String> lines = readLinesFromFile(csvFile);
         if (Objects.isNull(lines)) return;
 
-        // for each line after the first (Headers Line)
-        lines.subList(1, lines.size()).forEach(line -> {
+        //Skip header line if present
+        int startRow = (lines.get(0).contains("\"description\",\"regex\"")) ? 1 : 0;
+
+        // for each line
+        lines.subList(startRow, lines.size()).forEach(line -> {
             Matcher matcher = RegexEntity.checkRegexEntityFromCSV(line);
-            if (!matcher.find()) return;
+
+            if (!matcher.find())
+                return;
+
+            //load sections if presents, otherwise set all sections
+            boolean hasSections = !(matcher.group(3) == null || matcher.group(3).isBlank());
 
             String description = matcher.group(1).replaceAll("\"\"", "\"");
             String regex = matcher.group(2).replaceAll("\"\"", "\"");
-            List<String> sections = List.of(matcher.group(3).replaceAll("\"\"", "\"").split("\\|"));
+            List<String> sections = hasSections ? List.of(matcher.group(3).replaceAll("\"\"", "\"").split("\\|")) : null;
 
             RegexEntity newRegexEntity = new RegexEntity(
                     description,
                     regex,
                     true,
-                    ProxyItemSection.deserializeSections(sections)
+                    hasSections ? ProxyItemSection.deserializeSections(sections) : ProxyItemSection.ALL
             );
 
             if (!ctx.getRegexEntities().contains(newRegexEntity)) {
@@ -248,11 +289,11 @@ public static void openListFromCSV(RegexListContext ctx) {
                 alreadyAddedMsg.toString());
     }
 
-    public static void openListFromJSON(RegexListContext ctx) {
+    public static void openListFromJSON(String jsonFile, RegexListContext ctx) {
         Gson gson = new Gson();
         StringBuilder alreadyAddedMsg = new StringBuilder();
 
-        List<String> lines = Utils.linesFromFile("json");
+        List<String> lines = readLinesFromFile(jsonFile);
         if (Objects.isNull(lines)) return;
 
         Type tArrayListRegexEntity = new TypeToken<ArrayList<JsonRegexEntity>>() {

src/main/java/com/cys4/sensitivediscoverer/model/RegexEntity.java

@@ -58,13 +58,13 @@ public RegexEntity(RegexEntity entity) throws IllegalArgumentException {
 
     /**
      * Checks if the input is in the format: `"Description","Regex","Sections"`
-     *
+     * Matches also if sections are not present, in this case group(3) is null
      * @param input Text string to check against the format
      * @return If the input was in the correct format, a Matcher object where group(1) = description, group(2) = regex, group(3) = sections
      */
     public static Matcher checkRegexEntityFromCSV(String input) {
         return Pattern
-                .compile("^\\s*[\"'](.*?)[\"']\\s*,\\s*[\"'](.+?)[\"']\\s*,\\s*[\"'](.+?)[\"']\\s*$")
+                .compile("^[\t ]*[\"'](.+?)[\"'][\t ]*,[\t ]*[\"'](.+?)[\"'][\t ]*(?:,[\t ]*[\"'](.+?)[\"'][\t ]*)?$")
                 .matcher(input);
     }
 

src/main/java/com/cys4/sensitivediscoverer/tab/LoggerTab.java

@@ -321,6 +321,9 @@ private JToggleButton createExportLogsButton() {
 
         JMenuItem itemToCSV = new JMenuItem(getLocaleString("common-toCSV"));
         itemToCSV.addActionListener(actionEvent -> {
+            String csvFile = Utils.selectFile(List.of("CSV"), false);
+            if (csvFile.isBlank()) return;
+
             java.util.List<String> lines = new ArrayList<>();
 
             lines.add(String.format("\"%s\",\"%s\"",
@@ -334,12 +337,15 @@ private JToggleButton createExportLogsButton() {
                 lines.add(String.format("\"%s\",\"%s\"", url, matchEscaped));
             }
 
-            Utils.saveToFile("csv", lines);
+            Utils.writeLinesToFile(csvFile, lines);
         });
         menu.add(itemToCSV);
 
         JMenuItem itemToJSON = new JMenuItem(getLocaleString("common-toJSON"));
         itemToJSON.addActionListener(actionEvent -> {
+            String jsonFile = Utils.selectFile(List.of("JSON"), false);
+            if (jsonFile.isBlank()) return;
+
             java.util.List<JsonObject> lines = new ArrayList<>();
 
             String prop1 = LogsTableModel.Column.URL.getNameFormatted();
@@ -357,7 +363,8 @@ private JToggleButton createExportLogsButton() {
             Gson gson = builder.create();
             Type tListEntries = new TypeToken<ArrayList<JsonObject>>() {
             }.getType();
-            Utils.saveToFile("json", List.of(gson.toJson(lines, tListEntries)));
+
+            Utils.writeLinesToFile(jsonFile, List.of(gson.toJson(lines, tListEntries)));
         });
         menu.add(itemToJSON);
 

src/main/resources/TextUI_en_US.properties

@@ -44,13 +44,9 @@ options-list-clear=Clear list
 options-list-clear-message=Delete ALL the regex in the list?
 options-list-clear-title=Clear list?
 options-list-open=Open list...
-options-list-open-formatDialogMessage=Choose the import format
-options-list-open-formatDialogTitle=Import regex list
 options-list-open-alreadyPresentWarn=These regexes are already present
 options-list-open-alreadyPresentTitle=Regexes alert
 options-list-save=Save list...
-options-list-save-formatDialogMessage=Choose the export format
-options-list-save-formatDialogTitle=Export regex list
 options-list-new=New regex
 options-list-new-dialogTitle=Add a regular expression
 options-list-delete=Delete selected