Merge remote-tracking branch 'upstream/master' into fixattachfilerela…

…tive

* upstream/master:
  Update dependencies (#4251)
  Fix author list parser (#4169) (#4248)
  Solved #3823 File annotation (#4246)
  Fix importer vulnerability (#4240)

CHANGELOG.md

@@ -25,8 +25,9 @@ We refer to [GitHub issues](https://github.com/JabRef/jabref/issues) by using `#
 - We changed the default keyboard shortcuts for moving between entries when the entry editor is active to ̀<kbd>alt</kbd> + <kbd>up/down</kbd>.
 - Opening a new file now prompts the directory of the currently selected file, instead of the directory of the last opened file.
 - Window state is saved on close and restored on start.
-- Files without a defined external file type are now directly opened with the default aplication of the operating system
+- Files without a defined external file type are now directly opened with the default application of the operating system
 - We streamlined the process to rename and move files by removing the confirmation dialogs.
+- We removed the redundant new lines of markings and wrapped the summary in the File annotation tab. [#3823](https://github.com/JabRef/jabref/issues/3823)
 
 
 
@@ -60,8 +61,9 @@ We refer to [GitHub issues](https://github.com/JabRef/jabref/issues) by using `#
 - We fixed an issue where the "Convert to BibTeX-Cleanup" moved the content of the `file` field to the `pdf` field [#4120](https://github.com/JabRef/jabref/issues/4120)
 - We fixed an issue where the preview pane in entry preview in preferences wasn't showing the citation style selected [#3849](https://github.com/JabRef/jabref/issues/3849)
 - We fixed an issue where the default entry preview style still contained the field `review`. The field `review` in the style is now replaced with comment to be consistent with the entry editor [#4098](https://github.com/JabRef/jabref/issues/4098)
-- We fixed an issue where filles added via the "Attach file" contextmenu of an entry were not made relative. [#4201](https://github.com/JabRef/jabref/issues/4201)
-
+- We fixed an issue where users were vulnerable to XXE attacks during parsing [#4229](https://github.com/JabRef/jabref/issues/4229)
+- We fixed an issue where files added via the "Attach file" contextmenu of an entry were not made relative. [#4201](https://github.com/JabRef/jabref/issues/4201)
+- We fixed an issue where author list parser can't generate bibtex for Chinese author. [#4169](https://github.com/JabRef/jabref/issues/4169)
 
 
 

build.gradle

@@ -21,7 +21,7 @@ plugins {
     id "de.sebastianboegl.shadow.transformer.log4j" version "2.1.1"
     id "com.simonharrer.modernizer" version '1.6.0-1'
     id 'me.champeau.gradle.jmh' version '0.4.7'
-    id 'net.ltgt.errorprone' version '0.0.15'
+    id 'net.ltgt.errorprone' version '0.0.16'
     id 'com.github.ben-manes.versions' version '0.20.0'
 }
 
@@ -117,7 +117,7 @@ dependencies {
 
     compile 'net.java.dev.glazedlists:glazedlists_java15:1.9.1'
 
-    compile 'com.google.guava:guava:25.1-jre'
+    compile 'com.google.guava:guava:26.0-jre'
 
     // JavaFX stuff
     compile 'de.jensd:fontawesomefx-materialdesignfont:1.7.22-4'
@@ -137,10 +137,10 @@ dependencies {
 
     // >1.8.0-beta is required for java 9 compatibility
     compile 'org.slf4j:slf4j-api:1.8.0-beta2'
-    compile 'org.apache.logging.log4j:log4j-slf4j-impl:2.11.0'
-    compile 'org.apache.logging.log4j:log4j-jcl:2.11.0'
-    compile 'org.apache.logging.log4j:log4j-api:2.11.0'
-    compile 'org.apache.logging.log4j:log4j-core:2.11.0'
+    compile 'org.apache.logging.log4j:log4j-slf4j-impl:2.11.1'
+    compile 'org.apache.logging.log4j:log4j-jcl:2.11.1'
+    compile 'org.apache.logging.log4j:log4j-api:2.11.1'
+    compile 'org.apache.logging.log4j:log4j-core:2.11.1'
 
     // need to use snapshots as the stable version is from 2013 and doesn't support v1.0.1 CitationStyles
     compile 'org.citationstyles:styles:1.0.1-SNAPSHOT'
@@ -149,8 +149,8 @@ dependencies {
 
     compile 'com.github.tomtung:latex2unicode_2.12:0.2.2'
 
-    compile group: 'com.microsoft.azure', name: 'applicationinsights-core', version: '2.1.1'
-    compile group: 'com.microsoft.azure', name: 'applicationinsights-logging-log4j2', version: '2.1.1'
+    compile group: 'com.microsoft.azure', name: 'applicationinsights-core', version: '2.1.2'
+    compile group: 'com.microsoft.azure', name: 'applicationinsights-logging-log4j2', version: '2.1.2'
 
     testImplementation 'org.junit.jupiter:junit-jupiter-api:5.2.0'
     testCompile 'org.junit.jupiter:junit-jupiter-params:5.2.0'
@@ -160,7 +160,7 @@ dependencies {
     testCompile 'org.junit-pioneer:junit-pioneer:0.1.2'
     testRuntime 'org.apache.logging.log4j:log4j-core:2.11.0'
     testRuntime 'org.apache.logging.log4j:log4j-jul:2.11.0'
-    testCompile 'org.mockito:mockito-core:2.20.0'
+    testCompile 'org.mockito:mockito-core:2.21.0'
     testCompile 'com.github.tomakehurst:wiremock:2.18.0'
     testCompile 'org.assertj:assertj-swing-junit:3.8.0'
     testCompile 'org.reflections:reflections:0.9.11'

src/main/java/org/jabref/gui/entryeditor/fileannotationtab/FileAnnotationTabView.java

@@ -15,6 +15,7 @@
 import javafx.scene.control.ListView;
 import javafx.scene.control.SelectionMode;
 import javafx.scene.control.TextArea;
+import javafx.scene.control.Tooltip;
 import javafx.scene.layout.ColumnConstraints;
 import javafx.scene.layout.GridPane;
 import javafx.scene.layout.VBox;
@@ -63,7 +64,6 @@ public void initialize() {
         annotationList.getSelectionModel().setSelectionMode(SelectionMode.SINGLE);
         annotationList.getSelectionModel().selectedItemProperty().addListener((observable, oldValue, newValue) -> viewModel.notifyNewSelectedAnnotation(newValue));
         ViewModelListCellFactory<FileAnnotationViewModel> cellFactory = new ViewModelListCellFactory<FileAnnotationViewModel>()
-                .withTooltip(FileAnnotationViewModel::getMarking)
                 .withGraphic(this::createFileAnnotationNode);
         annotationList.setCellFactory(cellFactory);
         annotationList.setPlaceholder(new Label(Localization.lang("File has no attached annotations")));
@@ -94,12 +94,19 @@ private Node createFileAnnotationNode(FileAnnotationViewModel annotation) {
         Label date = new Label(annotation.getDate());
         Label page = new Label(Localization.lang("Page") + ": " + annotation.getPage());
 
-        marking.setStyle("-fx-font-weight: bold");
+        marking.setStyle("-fx-font-size: 0.75em; -fx-font-weight: bold");
         marking.setMaxHeight(30);
 
+        Tooltip markingTooltip = new Tooltip(annotation.getMarking());
+        markingTooltip.setMaxWidth(800);
+        markingTooltip.setWrapText(true);
+        marking.setTooltip(markingTooltip);
+
         // add alignment for text in the list
         marking.setTextAlignment(TextAlignment.LEFT);
         marking.setAlignment(Pos.TOP_LEFT);
+        marking.setMaxWidth(500);
+        marking.setWrapText(true);
         author.setTextAlignment(TextAlignment.LEFT);
         author.setAlignment(Pos.TOP_LEFT);
         date.setTextAlignment(TextAlignment.RIGHT);

src/main/java/org/jabref/gui/entryeditor/fileannotationtab/FileAnnotationViewModel.java

@@ -11,6 +11,7 @@
 
 public class FileAnnotationViewModel {
 
+    private static final String NEWLINE = String.format("%n");
     private final FileAnnotation annotation;
     private StringProperty author = new SimpleStringProperty();
     private StringProperty page = new SimpleStringProperty();
@@ -31,7 +32,13 @@ private void setupContentProperties(FileAnnotation annotation) {
             this.content.set(annotation.getLinkedFileAnnotation().getContent());
             String annotationContent = annotation.getContent();
             String illegibleTextMessage = Localization.lang("The marked area does not contain any legible text!");
-            this.marking.set(annotationContent.isEmpty() ? illegibleTextMessage : annotationContent);
+            String markingContent = (annotationContent.isEmpty() ? illegibleTextMessage : annotationContent);
+            // remove newlines && hyphens before linebreaks
+            markingContent = markingContent.replaceAll("-" + NEWLINE, "");
+            new RemoveHyphenatedNewlinesFormatter().format(markingContent);
+            // remove new lines not preceded by '.' or ':'
+            markingContent = markingContent.replaceAll("(?<![.|:])" + NEWLINE, " ");
+            this.marking.set(markingContent);
         } else {
             String content = annotation.getContent();
             // remove newlines && hyphens before linebreaks

src/main/java/org/jabref/logic/formatter/bibtexfields/RemoveHyphenatedNewlinesFormatter.java

@@ -10,7 +10,7 @@
  * Removes all hyphenated line breaks in the string.
  */
 public class RemoveHyphenatedNewlinesFormatter extends Formatter {
-    private static final Pattern HYPHENATED_WORDS = Pattern.compile("(-\r\n|-\n|-\r)");
+    private static final Pattern HYPHENATED_WORDS = Pattern.compile("-\\R");
 
     @Override
     public String getName() {

src/main/java/org/jabref/logic/formatter/bibtexfields/RemoveNewlinesFormatter.java

@@ -10,7 +10,7 @@
  * Removes all line breaks in the string.
  */
 public class RemoveNewlinesFormatter extends Formatter {
-    private static final Pattern LINEBREAKS = Pattern.compile("(\r?\n|\r)");
+    private static final Pattern LINEBREAKS = Pattern.compile("\\R");
 
     @Override
     public String getName() {

src/main/java/org/jabref/logic/importer/fileformat/MsBibImporter.java

@@ -6,12 +6,15 @@
 
 import javax.xml.parsers.DocumentBuilder;
 import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.parsers.ParserConfigurationException;
 
 import org.jabref.logic.importer.Importer;
 import org.jabref.logic.importer.ParserResult;
 import org.jabref.logic.msbib.MSBibDatabase;
 import org.jabref.logic.util.StandardFileType;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import org.w3c.dom.Document;
 import org.xml.sax.ErrorHandler;
 import org.xml.sax.InputSource;
@@ -20,24 +23,27 @@
 
 /**
  * Importer for the MS Office 2007 XML bibliography format
- * By S. M. Mahbub Murshed
  *
  * ...
  */
 public class MsBibImporter extends Importer {
 
+    private static final Logger LOGGER = LoggerFactory.getLogger(MsBibImporter.class);
+    private static final String DISABLEDTD = "http://apache.org/xml/features/disallow-doctype-decl";
+    private static final String DISABLEEXTERNALDTD = "http://apache.org/xml/features/nonvalidating/load-external-dtd";
+
     @Override
     public boolean isRecognizedFormat(BufferedReader reader) throws IOException {
         Objects.requireNonNull(reader);
 
         /*
-            The correct behaviour is to return false if it is certain that the file is
+            The correct behavior is to return false if it is certain that the file is
             not of the MsBib type, and true otherwise. Returning true is the safe choice
             if not certain.
          */
         Document docin;
         try {
-            DocumentBuilder dbuild = DocumentBuilderFactory.newInstance().newDocumentBuilder();
+            DocumentBuilder dbuild = makeSafeDocBuilderFactory(DocumentBuilderFactory.newInstance()).newDocumentBuilder();
             dbuild.setErrorHandler(new ErrorHandler() {
 
                 @Override
@@ -55,6 +61,7 @@ public void error(SAXParseException exception) throws SAXException {
                     throw exception;
                 }
             });
+
             docin = dbuild.parse(new InputSource(reader));
         } catch (Exception e) {
             return false;
@@ -85,4 +92,29 @@ public String getDescription() {
         return "Importer for the MS Office 2007 XML bibliography format.";
     }
 
+    /**
+     * DocumentBuilderFactory makes a XXE safe Builder factory from dBuild. If not supported by current
+     * XML then returns original builder given and logs error.
+     * @param dBuild | DocumentBuilderFactory to be made XXE safe.
+     * @return If supported, XXE safe DocumentBuilderFactory. Else, returns original builder given
+     */
+    private DocumentBuilderFactory makeSafeDocBuilderFactory(DocumentBuilderFactory dBuild) {
+        String feature = null;
+
+        try {
+            feature = DISABLEDTD;
+            dBuild.setFeature(feature, true);
+
+            feature = DISABLEEXTERNALDTD;
+            dBuild.setFeature(feature, false);
+
+            dBuild.setXIncludeAware(false);
+            dBuild.setExpandEntityReferences(false);
+
+        } catch (ParserConfigurationException e) {
+            LOGGER.warn("Builder not fully configured. Feature:'{}' is probably not supported by current XML processor. {}", feature, e);
+        }
+
+        return dBuild;
+    }
 }

src/main/java/org/jabref/model/entry/AuthorListParser.java

@@ -264,7 +264,8 @@ private Optional<Author> getAuthor() {
                 false);
         String jrPart = jrPartStart < 0 ? null : concatTokens(tokens, jrPartStart, jrPartEnd, OFFSET_TOKEN, false);
 
-        if ((firstPart != null) && (lastPart != null) && lastPart.equals(lastPart.toUpperCase(Locale.ROOT)) && (lastPart.length() < 5)) {
+        if ((firstPart != null) && (lastPart != null) && lastPart.equals(lastPart.toUpperCase(Locale.ROOT)) && (lastPart.length() < 5)
+            && (Character.UnicodeScript.of(lastPart.charAt(0)) != Character.UnicodeScript.HAN)) {
             // The last part is a small string in complete upper case, so interpret it as initial of the first name
             // This is the case for example in "Smith SH" which we think of as lastname=Smith and firstname=SH
             // The length < 5 constraint should allow for "Smith S.H." as input
@@ -378,7 +379,7 @@ private int getToken() {
             }
             if (!firstLetterIsFound && (currentBackslash < 0) && Character.isLetter(c)) {
                 if (bracesLevel == 0) {
-                    tokenCase = Character.isUpperCase(c);
+                    tokenCase = Character.isUpperCase(c) || (Character.UnicodeScript.of(c) == Character.UnicodeScript.HAN);
                 } else {
                     // If this is a particle in braces, always treat it as if it starts with
                     // an upper case letter. Otherwise a name such as "{van den Bergen}, Hans"

src/main/java/org/jabref/model/strings/StringUtil.java

@@ -197,7 +197,7 @@ public static String wrap(String in, int wrapAmount, String newline) {
         String[] lines = in.split("\n");
         StringBuilder result = new StringBuilder();
         // remove all whitespace at the end of the string, this especially includes \r created when the field content has \r\n as line separator
-        addWrappedLine(result, CharMatcher.WHITESPACE.trimTrailingFrom(lines[0]), wrapAmount, newline); // See
+        addWrappedLine(result, CharMatcher.whitespace().trimTrailingFrom(lines[0]), wrapAmount, newline); // See
         for (int i = 1; i < lines.length; i++) {
 
             if (lines[i].trim().isEmpty()) {
@@ -209,7 +209,7 @@ public static String wrap(String in, int wrapAmount, String newline) {
                 result.append(newline);
                 result.append('\t');
                 // remove all whitespace at the end of the string, this especially includes \r created when the field content has \r\n as line separator
-                String line = CharMatcher.WHITESPACE.trimTrailingFrom(lines[i]);
+                String line = CharMatcher.whitespace().trimTrailingFrom(lines[i]);
                 addWrappedLine(result, line, wrapAmount, newline);
             }
         }

src/main/resources/l10n/JabRef_zh.properties

@@ -1914,3 +1914,4 @@ View\ event\ log=查看事件日志
 Website=网站
 Write\ XMP-metadata\ to\ PDFs=将 XMP 元数据写入到 PDF 中
 
+Removes\ all\ line\ breaks\ in\ the\ field\ content.=删除字段内容中所有的换行。

src/test/java/org/jabref/gui/entryeditor/fileannotationtab/FileAnnotationViewModelTest.java

@@ -0,0 +1,37 @@
+package org.jabref.gui.entryeditor.fileannotationtab;
+
+import java.time.LocalDateTime;
+import java.util.Optional;
+
+import org.jabref.model.pdf.FileAnnotation;
+import org.jabref.model.pdf.FileAnnotationType;
+
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+public class FileAnnotationViewModelTest {
+
+    @Test
+    public void removeOnlyLineBreaksNotPrecededByPeriodOrColon() {
+        String content = "This is content";
+        String marking = String.format("This is paragraph 1.%n" +
+                                       "This is paragr-%naph 2, and it crosses%nseveral lines,%nnow you can see next paragraph:%n"
+                                       + "This is paragraph%n3.");
+
+        FileAnnotation linkedFileAnnotation = new FileAnnotation("John", LocalDateTime.now(), 3, content, FileAnnotationType.FREETEXT, Optional.empty());
+        FileAnnotation annotation = new FileAnnotation("Jaroslav Kucha ˇr", LocalDateTime.parse("2017-07-20T10:11:30"), 1, marking, FileAnnotationType.HIGHLIGHT, Optional.of(linkedFileAnnotation));
+
+        FileAnnotationViewModel annotationViewModel = new FileAnnotationViewModel(annotation);
+
+        assertEquals("Jaroslav Kucha ˇr", annotationViewModel.getAuthor());
+        assertEquals(1, annotation.getPage());
+        assertEquals("2017-07-20 10:11:30", annotationViewModel.getDate());
+        assertEquals("This is content", annotationViewModel.getContent());
+
+        assertEquals(String.format("This is paragraph 1.%n" +
+                                   "This is paragraph 2, and it crosses several lines, now you can see next paragraph:%n"
+                                   + "This is paragraph 3."),
+                     annotationViewModel.getMarking());
+    }
+}

src/test/java/org/jabref/logic/formatter/bibtexfields/RemoveHyphenatedNewlinesFormatterTest.java

@@ -19,4 +19,10 @@ public void removeHyphensBeforeNewlines() {
         assertEquals("water", formatter.format("wa-\r\nter"));
         assertEquals("water", formatter.format("wa-\rter"));
     }
+
+    @Test
+    public void removeHyphensBeforePlatformSpecificNewlines() {
+        String newLine = String.format("%n");
+        assertEquals("water", formatter.format("wa-" + newLine + "ter"));
+    }
 }

src/test/java/org/jabref/logic/formatter/bibtexfields/RemoveNewlinesFormatterTest.java

@@ -29,4 +29,10 @@ public void removeCarriageReturn() {
     public void removeLineFeed() {
         assertEquals("n linebreak", formatter.format("n\nlinebreak"));
     }
+
+    @Test
+    public void removePlatformSpecificNewLine() {
+        String newLine = String.format("%n");
+        assertEquals("linebreak on current platform", formatter.format("linebreak on" + newLine + "current platform"));
+    }
 }

src/test/java/org/jabref/model/entry/AuthorListParameterTest.java

@@ -25,6 +25,7 @@ public class AuthorListParameterTest {
     public static Collection<Object[]> data() {
 
         return Arrays.asList(new Object[][] {
+            {"王, 军", authorList(new Author("军", "军.", null, "王", null))},
             { "Doe, John", authorList(new Author("John", "J.", null, "Doe", null)) },
             { "von Berlichingen zu Hornberg, Johann Gottfried",
                     authorList(new Author("Johann Gottfried", "J. G.", "von", "Berlichingen zu Hornberg", null)) },