Update user agent and log URL (#11852)

* Update user agent and log URL * Remove non-used "setSSLVerification" * Enable automatic redirect-following * Make use of TLS 1.2 * Fix static * Qucik fix to hide API keys * Add CHANGELOG.md entry * Add hints on SSL certificates * Fix Exception catching * Streamline test code * Fix URL for "Volltext" * Extract method * Fix IllegalArgumentException in IEEE
JabRef · Sep 29, 2024 · 8309e65 · 8309e65
1 parent e395853
commit 8309e65
Show file tree

Hide file tree

Showing 11 changed files with 86 additions and 99 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -47,6 +47,7 @@ Note that this project **does not** adhere to [Semantic Versioning](https://semv
 - ⚠️ Renamed command line parameters `embeddBibfileInPdf` to `embedBibFileInPdf`, `writeMetadatatoPdf` to `writeMetadataToPdf`, and `writeXMPtoPdf` to `writeXmpToPdf`. [#11575](https://github.com/JabRef/jabref/pull/11575)
 - The browse button for a Custom theme now opens in the directory of the current used CSS file. [#11597](https://github.com/JabRef/jabref/pull/11597)
 - The browse button for a Custom exporter now opens in the directory of the current used exporter file. [#11717](https://github.com/JabRef/jabref/pull/11717)
+- JabRef now uses TLS 1.2 for all HTTPS connections. [#11852](https://github.com/JabRef/jabref/pull/11852)
 - We improved the display of long messages in the integrity check dialog. [#11619](https://github.com/JabRef/jabref/pull/11619)
 - We improved the undo/redo buttons in the main toolbar and main menu to be disabled when there is nothing to undo/redo. [#8807](https://github.com/JabRef/jabref/issues/8807)
 - We improved the DOI detection in PDF imports. [#11782](https://github.com/JabRef/jabref/pull/11782)
@@ -73,6 +74,7 @@ Note that this project **does not** adhere to [Semantic Versioning](https://semv
 - We fixed an issue where search result highlighting was incorrectly highlighting the boolean operators. [#11595](https://github.com/JabRef/jabref/issues/11595)
 - We fixed an issue where search result highlighting was broken at complex searches. [#8067](https://github.com/JabRef/jabref/issues/8067)
 - We fixed an exception when searching for unlinked files. [#11731](https://github.com/JabRef/jabref/issues/11731)
+- We fixed an issue with the link to the full text at the BVB fetcher. [#11852](https://github.com/JabRef/jabref/pull/11852)
 - We fixed an issue where two contradicting notifications were shown when cutting an entry in the main table. [#11724](https://github.com/JabRef/jabref/pull/11724)
 - We fixed an issue where unescaped braces in the arXiv fetcher were not treated. [#11704](https://github.com/JabRef/jabref/issues/11704)
 - We fixed an issue where HTML instead of the fulltext pdf was downloaded when importing arXiv entries. [#4913](https://github.com/JabRef/jabref/issues/4913)

diff --git a/src/main/java/org/jabref/gui/linkedfile/DownloadLinkedFileAction.java b/src/main/java/org/jabref/gui/linkedfile/DownloadLinkedFileAction.java
@@ -10,10 +10,7 @@
 import java.util.List;
 import java.util.Optional;
 
-import javax.net.ssl.HostnameVerifier;
-import javax.net.ssl.HttpsURLConnection;
 import javax.net.ssl.SSLHandshakeException;
-import javax.net.ssl.SSLSocketFactory;
 
 import javafx.beans.property.DoubleProperty;
 import javafx.beans.property.ReadOnlyDoubleProperty;
@@ -243,8 +240,6 @@ private boolean checkSSLHandshake(URLDownload urlDownload) {
     }
 
     private BackgroundTask<Path> prepareDownloadTask(Path targetDirectory, URLDownload urlDownload) {
-        SSLSocketFactory defaultSSLSocketFactory = HttpsURLConnection.getDefaultSSLSocketFactory();
-        HostnameVerifier defaultHostnameVerifier = HttpsURLConnection.getDefaultHostnameVerifier();
         return BackgroundTask
                 .wrap(() -> {
                     String suggestedName;
@@ -263,7 +258,6 @@ private BackgroundTask<Path> prepareDownloadTask(Path targetDirectory, URLDownlo
                 .then(destination -> new FileDownloadTask(urlDownload.getSource(), destination))
                 .onFailure(ex -> LOGGER.error("Error in download", ex))
                 .onFinished(() -> {
-                    URLDownload.setSSLVerification(defaultSSLSocketFactory, defaultHostnameVerifier);
                     downloadProgress.unbind();
                     downloadProgress.set(1);
                 });

diff --git a/src/main/java/org/jabref/gui/preferences/websearch/WebSearchTabViewModel.java b/src/main/java/org/jabref/gui/preferences/websearch/WebSearchTabViewModel.java
@@ -5,10 +5,6 @@
 import java.util.Optional;
 import java.util.stream.Collectors;
 
-import javax.net.ssl.HostnameVerifier;
-import javax.net.ssl.HttpsURLConnection;
-import javax.net.ssl.SSLSocketFactory;
-
 import javafx.beans.property.BooleanProperty;
 import javafx.beans.property.ListProperty;
 import javafx.beans.property.ObjectProperty;
@@ -217,15 +213,10 @@ public void checkCustomApiKey() {
         if (!apiKey.isEmpty()) {
             URLDownload urlDownload;
             try {
-                SSLSocketFactory defaultSslSocketFactory = HttpsURLConnection.getDefaultSSLSocketFactory();
-                HostnameVerifier defaultHostnameVerifier = HttpsURLConnection.getDefaultHostnameVerifier();
-
                 urlDownload = new URLDownload(testUrlWithoutApiKey + apiKey);
                 // The HEAD request cannot be used because its response is not 200 (maybe 404 or 596...).
                 int statusCode = ((HttpURLConnection) urlDownload.getSource().openConnection()).getResponseCode();
                 keyValid = (statusCode >= 200) && (statusCode < 300);
-
-                URLDownload.setSSLVerification(defaultSslSocketFactory, defaultHostnameVerifier);
             } catch (IOException | UnirestException e) {
                 keyValid = false;
             }

diff --git a/src/main/java/org/jabref/logic/importer/FetcherException.java b/src/main/java/org/jabref/logic/importer/FetcherException.java
@@ -14,10 +14,8 @@
 
 public class FetcherException extends JabRefException {
     private static final Logger LOGGER = LoggerFactory.getLogger(FetcherException.class);
-
-    Pattern API_KEY_PATTERN = Pattern.compile("(?i)(api|key|api[-_]?key)=[^&]*");
-
-    String REDACTED_STRING = "[REDACTED]";
+    private static final Pattern API_KEY_PATTERN = Pattern.compile("(?i)(api|key|api[-_]?key)=[^&]*");
+    private static String REDACTED_STRING = "[REDACTED]";
 
     private final String url;
     private final SimpleHttpResponse httpResponse;
@@ -91,6 +89,10 @@ private String getRedactedUrl() {
         return API_KEY_PATTERN.matcher(url).replaceAll(REDACTED_STRING);
     }
 
+    public static Object getRedactedUrl(URL source) {
+        return API_KEY_PATTERN.matcher(source.toString()).replaceAll(REDACTED_STRING);
+    }
+
     private String getPrefix() {
         String superLocalizedMessage = super.getLocalizedMessage();
         if (!StringUtil.isBlank(superLocalizedMessage)) {

diff --git a/src/main/java/org/jabref/logic/importer/fetcher/BvbFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/BvbFetcher.java
@@ -14,6 +14,9 @@
 import org.apache.hc.core5.net.URIBuilder;
 import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode;
 
+/**
+ * Fetcher for <a href="Bibliotheksverbund Bayern (BVB)">https://www.bib-bvb.de/</a>.
+ */
 public class BvbFetcher implements SearchBasedParserFetcher {
 
     private static final String URL_PATTERN = "http://bvbr.bib-bvb.de:5661/bvb01sru?";

diff --git a/src/main/java/org/jabref/logic/importer/fetcher/IEEE.java b/src/main/java/org/jabref/logic/importer/fetcher/IEEE.java
@@ -30,6 +30,7 @@
 import org.jabref.model.entry.field.StandardField;
 import org.jabref.model.entry.identifier.DOI;
 import org.jabref.model.entry.types.StandardEntryType;
+import org.jabref.model.strings.StringUtil;
 
 import kong.unirest.core.json.JSONArray;
 import kong.unirest.core.json.JSONObject;
@@ -118,10 +119,9 @@ private static BibEntry parseJsonResponse(JSONObject jsonEntry, Character keywor
         entry.setField(StandardField.ISBN, jsonEntry.optString("isbn"));
         entry.setField(StandardField.ISSN, jsonEntry.optString("issn"));
         entry.setField(StandardField.ISSUE, jsonEntry.optString("issue"));
-        try {
-            entry.addFile(new LinkedFile(URI.create(jsonEntry.optString("pdf_url")).toURL(), "PDF"));
-        } catch (MalformedURLException e) {
-            LOGGER.error("Fetched PDF URL String is malformed.");
+        String pdfUrl = jsonEntry.optString("pdf_url");
+        if (!StringUtil.isBlank(pdfUrl)) {
+            entry.addFile(new LinkedFile("", pdfUrl, "PDF"));
         }
         entry.setField(StandardField.JOURNALTITLE, jsonEntry.optString("publication_title"));
         entry.setField(StandardField.DATE, jsonEntry.optString("publication_date"));

diff --git a/src/main/java/org/jabref/logic/importer/fetcher/ResearchGate.java b/src/main/java/org/jabref/logic/importer/fetcher/ResearchGate.java
@@ -69,7 +69,7 @@ public Optional<URL> findFullText(BibEntry entry) throws IOException, FetcherExc
         try {
             html = getHTML(entry);
         } catch (FetcherException | NullPointerException e) {
-            LOGGER.debug("ResearchGate server is not available");
+            LOGGER.debug("ResearchGate server is not available", e);
             return Optional.empty();
         }
         Elements eLink = html.getElementsByTag("section");

diff --git a/src/main/java/org/jabref/logic/importer/fileformat/MarcXmlParser.java b/src/main/java/org/jabref/logic/importer/fileformat/MarcXmlParser.java
@@ -18,10 +18,12 @@
 import org.jabref.logic.importer.AuthorListParser;
 import org.jabref.logic.importer.ParseException;
 import org.jabref.logic.importer.Parser;
+import org.jabref.logic.util.StandardFileType;
 import org.jabref.model.entry.AuthorList;
 import org.jabref.model.entry.BibEntry;
 import org.jabref.model.entry.Date;
 import org.jabref.model.entry.LinkedFile;
+import org.jabref.model.entry.field.Field;
 import org.jabref.model.entry.field.StandardField;
 import org.jabref.model.entry.types.StandardEntryType;
 import org.jabref.model.strings.StringUtil;
@@ -380,17 +382,7 @@ private void putDoi(BibEntry bibEntry, Element datafield) {
 
         if ("e".equals(ind1) && StringUtil.isNotBlank("u") && StringUtil.isNotBlank(resource)) { // DOI
             String fulltext = getSubfield("3", datafield);
-
-            if ("Volltext".equals(fulltext)) {
-                try {
-                    LinkedFile linkedFile = new LinkedFile(URI.create(resource).toURL(), "PDF");
-                    bibEntry.setField(StandardField.FILE, linkedFile.toString());
-                } catch (MalformedURLException e) {
-                    LOGGER.info("Malformed URL: {}", resource);
-                }
-            } else {
-                bibEntry.setField(StandardField.DOI, resource);
-            }
+            handleVolltext(bibEntry, fulltext, resource, StandardField.DOI);
         }
     }
 
@@ -402,17 +394,20 @@ private void putElectronicLocation(BibEntry bibEntry, Element datafield) {
         if ("4".equals(ind1) && "0".equals(ind2)) {
             String fulltext = getSubfield("3", datafield);
             String resource = getSubfield("u", datafield);
+            handleVolltext(bibEntry, fulltext, resource, StandardField.URL);
+        }
+    }
 
-            if ("Volltext".equals(fulltext) && StringUtil.isNotBlank(resource)) {
-                try {
-                    LinkedFile linkedFile = new LinkedFile(URI.create(resource).toURL(), "PDF");
-                    bibEntry.setField(StandardField.FILE, linkedFile.toString());
-                } catch (MalformedURLException e) {
-                    LOGGER.info("Malformed URL: {}", resource);
-                }
-            } else {
-                bibEntry.setField(StandardField.URL, resource);
+    private static void handleVolltext(BibEntry bibEntry, String fieldName, String resource, Field fallBackField) {
+        if ("Volltext".equals(fieldName) && StringUtil.isNotBlank(resource)) {
+            try {
+                LinkedFile linkedFile = new LinkedFile("", URI.create(resource).toURL(), StandardFileType.PDF.getName());
+                bibEntry.setFiles(List.of(linkedFile));
+            } catch (MalformedURLException | IllegalArgumentException e) {
+                LOGGER.info("Malformed URL: {}", resource);
             }
+        } else {
+            bibEntry.setField(fallBackField, resource);
         }
     }
 

diff --git a/src/main/java/org/jabref/logic/net/URLDownload.java b/src/main/java/org/jabref/logic/net/URLDownload.java
@@ -24,6 +24,9 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.StandardCopyOption;
+import java.security.KeyManagementException;
+import java.security.NoSuchAlgorithmException;
+import java.security.SecureRandom;
 import java.time.Duration;
 import java.util.Collections;
 import java.util.HashMap;
@@ -32,9 +35,8 @@
 import java.util.Map.Entry;
 import java.util.Optional;
 
-import javax.net.ssl.HostnameVerifier;
 import javax.net.ssl.HttpsURLConnection;
-import javax.net.ssl.SSLSocketFactory;
+import javax.net.ssl.SSLContext;
 
 import org.jabref.http.dto.SimpleHttpResponse;
 import org.jabref.logic.importer.FetcherClientException;
@@ -65,7 +67,7 @@
  */
 public class URLDownload {
 
-    public static final String USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36";
+    public static final String USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:130.0) Gecko/20100101 Firefox/130.0";
     private static final Logger LOGGER = LoggerFactory.getLogger(URLDownload.class);
     private static final Duration DEFAULT_CONNECT_TIMEOUT = Duration.ofSeconds(30);
     private static final int MAX_RETRIES = 3;
@@ -74,6 +76,7 @@ public class URLDownload {
     private final Map<String, String> parameters = new HashMap<>();
     private String postData = "";
     private Duration connectTimeout = DEFAULT_CONNECT_TIMEOUT;
+    private SSLContext sslContext;
 
     static {
         Unirest.config()
@@ -96,18 +99,14 @@ public URLDownload(String source) throws MalformedURLException {
     public URLDownload(URL source) {
         this.source = source;
         this.addHeader("User-Agent", URLDownload.USER_AGENT);
-    }
 
-    /**
-     * @param socketFactory trust manager
-     * @param verifier      host verifier
-     */
-    public static void setSSLVerification(SSLSocketFactory socketFactory, HostnameVerifier verifier) {
         try {
-            HttpsURLConnection.setDefaultSSLSocketFactory(socketFactory);
-            HttpsURLConnection.setDefaultHostnameVerifier(verifier);
-        } catch (Exception e) {
-            LOGGER.error("A problem occurred when reset SSL verification", e);
+            sslContext = SSLContext.getInstance("TLSv1.2");
+            sslContext.init(null, null, new SecureRandom());
+            // Note: SSL certificates are installed at {@link TrustStoreManager#configureTrustStore(Path)}
+        } catch (NoSuchAlgorithmException | KeyManagementException e) {
+            LOGGER.error("Could not initialize SSL context", e);
+            sslContext = null;
         }
     }
 
@@ -384,7 +383,7 @@ public URLConnection openConnection() throws FetcherException {
             } else if (status >= 400) {
                 // in case of an error, propagate the error message
                 SimpleHttpResponse httpResponse = new SimpleHttpResponse(httpURLConnection);
-                LOGGER.info("{}", httpResponse);
+                LOGGER.info("{}: {}", FetcherException.getRedactedUrl(this.source), httpResponse);
                 if (status < 500) {
                     throw new FetcherClientException(this.source, httpResponse);
                 } else {
@@ -397,6 +396,15 @@ public URLConnection openConnection() throws FetcherException {
 
     private URLConnection getUrlConnection() throws IOException {
         URLConnection connection = this.source.openConnection();
+
+        if (connection instanceof HttpURLConnection httpConnection) {
+            httpConnection.setInstanceFollowRedirects(true);
+        }
+
+        if ((sslContext != null) && (connection instanceof HttpsURLConnection httpsConnection)) {
+            httpsConnection.setSSLSocketFactory(sslContext.getSocketFactory());
+        }
+
         connection.setConnectTimeout((int) connectTimeout.toMillis());
         for (Entry<String, String> entry : this.parameters.entrySet()) {
             connection.setRequestProperty(entry.getKey(), entry.getValue());

diff --git a/src/main/java/org/jabref/logic/net/ssl/TrustStoreManager.java b/src/main/java/org/jabref/logic/net/ssl/TrustStoreManager.java
@@ -27,6 +27,9 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+/**
+ * @implNote SSL certificates are installed at {@link TrustStoreManager#configureTrustStore(Path)}
+ */
 public class TrustStoreManager {
 
     private static final Logger LOGGER = LoggerFactory.getLogger(TrustStoreManager.class);
@@ -163,7 +166,9 @@ public static void createTruststoreFileIfNotExist(Path storePath) {
         }
     }
 
-    // based on https://stackoverflow.com/a/62586564/3450689
+    /**
+     * @implNote based on https://stackoverflow.com/a/62586564/3450689
+     */
     private static void configureTrustStore(Path myStorePath) throws NoSuchAlgorithmException, KeyManagementException, KeyStoreException,
         CertificateException, IOException {
         X509TrustManager jreTrustManager = getJreTrustManager();