From 3521bd7dbf4514a160321da4eda25facb786bedf Mon Sep 17 00:00:00 2001 From: rehan Date: Fri, 5 Jul 2024 11:15:08 +0530 Subject: [PATCH 01/42] Revert "Add missing "" This reverts commit 0c612e7c58e218c3b3eeea122805f531b99f0eb1. --- docs/code-howtos/tools.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/code-howtos/tools.md b/docs/code-howtos/tools.md index 3f57a14ad6b..d7d347a238d 100644 --- a/docs/code-howtos/tools.md +++ b/docs/code-howtos/tools.md @@ -37,7 +37,7 @@ See also: [https://help.github.com/articles/syncing-a-fork/](https://help.github (As Administrator - one time) 1. Install [chocolatey](https://chocolatey.org) -2. `choco install git.install -y --params "/GitAndUnixToolsOnPath /WindowsTerminal"` +2. `choco install git.install -y --params "/GitAndUnixToolsOnPath /WindowsTerminal` 3. `choco install notepadplusplus` 4. If you want to have your JDK also managed via chocolatey: `choco install temurin` From 75f13fcae21752cca34a21982e381b66f77a9c6f Mon Sep 17 00:00:00 2001 From: rehan Date: Sat, 13 Jul 2024 17:57:38 +0530 Subject: [PATCH 02/42] Created SimpleHttpResponse class to simplify HTTP Response handling --- .../jabref/http/dto/SimpleHttpResponse.java | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 src/main/java/org/jabref/http/dto/SimpleHttpResponse.java diff --git a/src/main/java/org/jabref/http/dto/SimpleHttpResponse.java b/src/main/java/org/jabref/http/dto/SimpleHttpResponse.java new file mode 100644 index 00000000000..93dd6fc1ee7 --- /dev/null +++ b/src/main/java/org/jabref/http/dto/SimpleHttpResponse.java @@ -0,0 +1,62 @@ +package org.jabref.http.dto; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.net.HttpURLConnection; +import java.nio.charset.StandardCharsets; + +public record SimpleHttpResponse(int statusCode, String responseBody , String responseMessage) { + + public SimpleHttpResponse(int statusCode,String responseBody,String responseMessage) { + this.statusCode=statusCode; + this.responseBody=truncateResponseBody(responseBody); + this.responseMessage=responseMessage; + } + + public SimpleHttpResponse(HttpURLConnection connection) throws IOException { + this( + connection.getResponseCode(), + getResponseBody(connection), + connection.getResponseMessage() + ); + } + + /** + * Truncates the response body to 1 KB if it exceeds that size. + * Appends "... (truncated)" to indicate truncation. + * + * @param responseBody the original response body + * @return the truncated response body + */ + private static String truncateResponseBody(String responseBody) { + byte[] bytes = responseBody.getBytes(StandardCharsets.UTF_8); + int maxLength = 1024; // 1 KB + if (bytes.length > maxLength) { + // Truncate the response body to 1 KB and append "... (truncated)" + return new String(bytes, 0, maxLength, StandardCharsets.UTF_8) + "... (truncated)"; + } + // Return the original response body if it's within the 1 KB limit + return responseBody; + } + + /** + * Reads the response body from the HttpURLConnection and returns it as a string. + * This method is used to retrieve the response body from the connection, + * which may contain error messages or other information from the server. + * + * @param connection the HttpURLConnection to read the response body from + * @return the response body as a string + * @throws IOException if an I/O error occurs while reading the response body + */ + private static String getResponseBody(HttpURLConnection connection) throws IOException { + try (BufferedReader in = new BufferedReader(new InputStreamReader(connection.getErrorStream()))) { + String inputLine; + StringBuilder content = new StringBuilder(); + while ((inputLine = in.readLine()) != null) { + content.append(inputLine); + } + return truncateResponseBody(content.toString()); + } + } +} From d05fb63c319bf66c1ead060689f4220217eedeb8 Mon Sep 17 00:00:00 2001 From: rehan Date: Sat, 13 Jul 2024 18:00:16 +0530 Subject: [PATCH 03/42] Refactored FetcherClientException to use SimpleHttpResponse, allowing for more detailed error messages and debugging --- .../logic/importer/FetcherClientException.java | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/jabref/logic/importer/FetcherClientException.java b/src/main/java/org/jabref/logic/importer/FetcherClientException.java index ada85e02903..8005f2e5aad 100644 --- a/src/main/java/org/jabref/logic/importer/FetcherClientException.java +++ b/src/main/java/org/jabref/logic/importer/FetcherClientException.java @@ -1,18 +1,20 @@ package org.jabref.logic.importer; +import org.jabref.http.dto.SimpleHttpResponse; + /** * Should be thrown when you encounter an HTTP status code error >= 400 and < 500. */ public class FetcherClientException extends FetcherException { - private int statusCode; + private SimpleHttpResponse httpResponse; public FetcherClientException(String errorMessage, Throwable cause) { super(errorMessage, cause); } - public FetcherClientException(String errorMessage, Throwable cause, int statusCode) { + public FetcherClientException(String errorMessage, Throwable cause, SimpleHttpResponse httpResponse) { super(errorMessage, cause); - this.statusCode = statusCode; + this.httpResponse = httpResponse; } public FetcherClientException(String errorMessage) { @@ -23,7 +25,13 @@ public FetcherClientException(String errorMessage, String localizedMessage, Thro super(errorMessage, localizedMessage, cause); } - public int getStatusCode() { - return statusCode; + public FetcherClientException(String errorMessage,SimpleHttpResponse httpResponse) { + super(errorMessage); + this.httpResponse=httpResponse; } + + public SimpleHttpResponse getHttpResponse() { + return httpResponse; + } + } From fd0cd11e0ab66c2ecf77389b765b0f86ab538dc1 Mon Sep 17 00:00:00 2001 From: rehan Date: Sat, 13 Jul 2024 18:05:43 +0530 Subject: [PATCH 04/42] replaced API keys with [REDACTED] in the error messages --- .../jabref/gui/linkedfile/DownloadLinkedFileAction.java | 2 +- .../jabref/logic/importer/SearchBasedParserFetcher.java | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/jabref/gui/linkedfile/DownloadLinkedFileAction.java b/src/main/java/org/jabref/gui/linkedfile/DownloadLinkedFileAction.java index 8fc11e21d52..2f2b4b1c9ab 100644 --- a/src/main/java/org/jabref/gui/linkedfile/DownloadLinkedFileAction.java +++ b/src/main/java/org/jabref/gui/linkedfile/DownloadLinkedFileAction.java @@ -203,7 +203,7 @@ private void onFailure(URLDownload urlDownload, Exception ex) { String failedTitle = Localization.lang("Failed to download from URL"); int statusCode; if (ex instanceof FetcherClientException clientException) { - statusCode = clientException.getStatusCode(); + statusCode = clientException.getHttpResponse().statusCode(); if (statusCode == 401) { dialogService.showInformationDialogAndWait(failedTitle, Localization.lang("401 Unauthorized: Access Denied. You are not authorized to access this resource. Please check your credentials and try again. If you believe you should have access, please contact the administrator for assistance.\nURL: %0 \n %1", urlDownload.getSource(), fetcherExceptionMessage)); } else if (statusCode == 403) { diff --git a/src/main/java/org/jabref/logic/importer/SearchBasedParserFetcher.java b/src/main/java/org/jabref/logic/importer/SearchBasedParserFetcher.java index 16f8a33598a..0c86d1beaf9 100644 --- a/src/main/java/org/jabref/logic/importer/SearchBasedParserFetcher.java +++ b/src/main/java/org/jabref/logic/importer/SearchBasedParserFetcher.java @@ -59,9 +59,13 @@ private List getBibEntries(URL urlForQuery) throws FetcherException { fetchedEntries.forEach(this::doPostCleanup); return fetchedEntries; } catch (IOException e) { - throw new FetcherException("A network error occurred while fetching from " + urlForQuery, e); + throw new FetcherException(("A network error occurred while fetching from " + urlForQuery).replaceAll +// // Regular expression to redact API keys from the error message + ("(?i)(api[_-]?key)=.*","[REDACTED]"), e); } catch (ParseException e) { - throw new FetcherException("An internal parser error occurred while fetching from " + urlForQuery, e); + throw new FetcherException(("An internal parser error occurred while fetching from " + urlForQuery).replaceAll + // Regular expression to redact API keys from the error message + ("(?i)(api[_-]?key)=.*","[REDACTED]"), e); } } From 6e91ce3929ac8d51b9a0b06f471209ec7eb65f33 Mon Sep 17 00:00:00 2001 From: rehan Date: Sat, 13 Jul 2024 18:13:05 +0530 Subject: [PATCH 05/42] Created SimpleHttpResponse instance from HttpURLConnection and included it in FetcherClientException --- src/main/java/org/jabref/logic/net/URLDownload.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/jabref/logic/net/URLDownload.java b/src/main/java/org/jabref/logic/net/URLDownload.java index 19d4d2e4bc1..482eb80a48d 100644 --- a/src/main/java/org/jabref/logic/net/URLDownload.java +++ b/src/main/java/org/jabref/logic/net/URLDownload.java @@ -40,6 +40,7 @@ import javax.net.ssl.TrustManager; import javax.net.ssl.X509TrustManager; +import org.jabref.http.dto.SimpleHttpResponse; import org.jabref.logic.importer.FetcherClientException; import org.jabref.logic.importer.FetcherServerException; import org.jabref.logic.util.io.FileUtil; @@ -395,8 +396,9 @@ public URLConnection openConnection() throws IOException { connection = new URLDownload(newUrl).openConnection(); } if ((status >= 400) && (status < 500)) { + SimpleHttpResponse httpResponse = new SimpleHttpResponse(lConnection); LOGGER.info("HTTP {}, details: {}, {}", status, lConnection.getResponseMessage(), lConnection.getContentLength() > 0 ? lConnection.getContent() : ""); - throw new IOException(new FetcherClientException("Encountered HTTP %s %s".formatted(status, lConnection.getResponseMessage()))); + throw new IOException(new FetcherClientException("Encountered HTTP %s %s".formatted(status, lConnection.getResponseMessage()),httpResponse)); } if (status >= 500) { LOGGER.info("HTTP {}, details: {}, {}", status, lConnection.getResponseMessage(), lConnection.getContentLength() > 0 ? lConnection.getContent() : ""); From c4de9f544a9d6bef07e7929561592d0a94ebbfda Mon Sep 17 00:00:00 2001 From: rehan Date: Sat, 13 Jul 2024 18:38:13 +0530 Subject: [PATCH 06/42] improved UX when a http error is encountered catching the FetcherException and displaying the errorDialog with HttpResponse Body --- .../fetcher/WebSearchPaneViewModel.java | 22 ++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/jabref/gui/importer/fetcher/WebSearchPaneViewModel.java b/src/main/java/org/jabref/gui/importer/fetcher/WebSearchPaneViewModel.java index 943e3ceddc0..aa10c3b34d1 100644 --- a/src/main/java/org/jabref/gui/importer/fetcher/WebSearchPaneViewModel.java +++ b/src/main/java/org/jabref/gui/importer/fetcher/WebSearchPaneViewModel.java @@ -2,6 +2,7 @@ import java.util.concurrent.Callable; +import javafx.application.Platform; import javafx.beans.property.ListProperty; import javafx.beans.property.ObjectProperty; import javafx.beans.property.SimpleListProperty; @@ -14,11 +15,9 @@ import org.jabref.gui.DialogService; import org.jabref.gui.StateManager; import org.jabref.gui.importer.ImportEntriesDialog; +import org.jabref.gui.linkedfile.DownloadLinkedFileAction; import org.jabref.gui.util.BackgroundTask; -import org.jabref.logic.importer.CompositeIdFetcher; -import org.jabref.logic.importer.ParserResult; -import org.jabref.logic.importer.SearchBasedFetcher; -import org.jabref.logic.importer.WebFetchers; +import org.jabref.logic.importer.*; import org.jabref.logic.l10n.Localization; import org.jabref.model.strings.StringUtil; import org.jabref.model.util.OptionalUtil; @@ -144,7 +143,20 @@ public void search() { } SearchBasedFetcher activeFetcher = getSelectedFetcher(); - Callable parserResultCallable = () -> new ParserResult(activeFetcher.performSearch(query)); + + ParserResult parserResult; + try{ + parserResult = new ParserResult(activeFetcher.performSearch(query)); + } catch (FetcherException e) { + if(e.getCause()!=null && e.getCause().getCause() instanceof FetcherClientException clientException) { + dialogService.showErrorDialogAndWait(e.getMessage(),clientException.getHttpResponse().responseBody(),clientException); + } else{ + dialogService.showErrorDialogAndWait(e.getMessage(),e); + } + return; + } + + Callable parserResultCallable = () -> parserResult; String fetcherName = activeFetcher.getName(); if (CompositeIdFetcher.containsValidId(query)) { From 3c4a722d4b82b295ee77c91f0f2ac4150e712a8c Mon Sep 17 00:00:00 2001 From: Rehan Chalana <139042983+RehanChalana@users.noreply.github.com> Date: Sun, 14 Jul 2024 09:52:57 +0530 Subject: [PATCH 07/42] Update tools.md --- docs/code-howtos/tools.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/code-howtos/tools.md b/docs/code-howtos/tools.md index d7d347a238d..3f57a14ad6b 100644 --- a/docs/code-howtos/tools.md +++ b/docs/code-howtos/tools.md @@ -37,7 +37,7 @@ See also: [https://help.github.com/articles/syncing-a-fork/](https://help.github (As Administrator - one time) 1. Install [chocolatey](https://chocolatey.org) -2. `choco install git.install -y --params "/GitAndUnixToolsOnPath /WindowsTerminal` +2. `choco install git.install -y --params "/GitAndUnixToolsOnPath /WindowsTerminal"` 3. `choco install notepadplusplus` 4. If you want to have your JDK also managed via chocolatey: `choco install temurin` From 2de617aa2149b1f1b22f0a1cd272802566fb8c08 Mon Sep 17 00:00:00 2001 From: rehan Date: Sun, 14 Jul 2024 10:15:03 +0530 Subject: [PATCH 08/42] check-style fixes --- .../fetcher/WebSearchPaneViewModel.java | 19 +++++++++++-------- .../jabref/http/dto/SimpleHttpResponse.java | 11 +++++------ .../importer/SearchBasedParserFetcher.java | 10 ++++------ 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/src/main/java/org/jabref/gui/importer/fetcher/WebSearchPaneViewModel.java b/src/main/java/org/jabref/gui/importer/fetcher/WebSearchPaneViewModel.java index aa10c3b34d1..0c9bf7b6d09 100644 --- a/src/main/java/org/jabref/gui/importer/fetcher/WebSearchPaneViewModel.java +++ b/src/main/java/org/jabref/gui/importer/fetcher/WebSearchPaneViewModel.java @@ -2,7 +2,6 @@ import java.util.concurrent.Callable; -import javafx.application.Platform; import javafx.beans.property.ListProperty; import javafx.beans.property.ObjectProperty; import javafx.beans.property.SimpleListProperty; @@ -15,9 +14,13 @@ import org.jabref.gui.DialogService; import org.jabref.gui.StateManager; import org.jabref.gui.importer.ImportEntriesDialog; -import org.jabref.gui.linkedfile.DownloadLinkedFileAction; import org.jabref.gui.util.BackgroundTask; -import org.jabref.logic.importer.*; +import org.jabref.logic.importer.CompositeIdFetcher; +import org.jabref.logic.importer.FetcherClientException; +import org.jabref.logic.importer.FetcherException; +import org.jabref.logic.importer.ParserResult; +import org.jabref.logic.importer.SearchBasedFetcher; +import org.jabref.logic.importer.WebFetchers; import org.jabref.logic.l10n.Localization; import org.jabref.model.strings.StringUtil; import org.jabref.model.util.OptionalUtil; @@ -145,13 +148,13 @@ public void search() { SearchBasedFetcher activeFetcher = getSelectedFetcher(); ParserResult parserResult; - try{ + try { parserResult = new ParserResult(activeFetcher.performSearch(query)); } catch (FetcherException e) { - if(e.getCause()!=null && e.getCause().getCause() instanceof FetcherClientException clientException) { - dialogService.showErrorDialogAndWait(e.getMessage(),clientException.getHttpResponse().responseBody(),clientException); - } else{ - dialogService.showErrorDialogAndWait(e.getMessage(),e); + if (e.getCause() != null && e.getCause().getCause() instanceof FetcherClientException clientException) { + dialogService.showErrorDialogAndWait(e.getMessage(), clientException.getHttpResponse().responseBody(), clientException); + } else { + dialogService.showErrorDialogAndWait(e.getMessage(), e); } return; } diff --git a/src/main/java/org/jabref/http/dto/SimpleHttpResponse.java b/src/main/java/org/jabref/http/dto/SimpleHttpResponse.java index 93dd6fc1ee7..8d02fe60cae 100644 --- a/src/main/java/org/jabref/http/dto/SimpleHttpResponse.java +++ b/src/main/java/org/jabref/http/dto/SimpleHttpResponse.java @@ -6,12 +6,11 @@ import java.net.HttpURLConnection; import java.nio.charset.StandardCharsets; -public record SimpleHttpResponse(int statusCode, String responseBody , String responseMessage) { - - public SimpleHttpResponse(int statusCode,String responseBody,String responseMessage) { - this.statusCode=statusCode; - this.responseBody=truncateResponseBody(responseBody); - this.responseMessage=responseMessage; +public record SimpleHttpResponse(int statusCode, String responseBody, String responseMessage) { + public SimpleHttpResponse(int statusCode, String responseBody, String responseMessage) { + this.statusCode = statusCode; + this.responseBody = truncateResponseBody(responseBody); + this.responseMessage = responseMessage; } public SimpleHttpResponse(HttpURLConnection connection) throws IOException { diff --git a/src/main/java/org/jabref/logic/importer/SearchBasedParserFetcher.java b/src/main/java/org/jabref/logic/importer/SearchBasedParserFetcher.java index 0c86d1beaf9..40187ea6ef2 100644 --- a/src/main/java/org/jabref/logic/importer/SearchBasedParserFetcher.java +++ b/src/main/java/org/jabref/logic/importer/SearchBasedParserFetcher.java @@ -59,13 +59,11 @@ private List getBibEntries(URL urlForQuery) throws FetcherException { fetchedEntries.forEach(this::doPostCleanup); return fetchedEntries; } catch (IOException e) { - throw new FetcherException(("A network error occurred while fetching from " + urlForQuery).replaceAll -// // Regular expression to redact API keys from the error message - ("(?i)(api[_-]?key)=.*","[REDACTED]"), e); + // Regular expression to redact API keys from the error message + throw new FetcherException(("A network error occurred while fetching from " + urlForQuery).replaceAll("(?i)(api[_-]?key)=.*", "[REDACTED]"), e); } catch (ParseException e) { - throw new FetcherException(("An internal parser error occurred while fetching from " + urlForQuery).replaceAll - // Regular expression to redact API keys from the error message - ("(?i)(api[_-]?key)=.*","[REDACTED]"), e); + // Regular expression to redact API keys from the error message + throw new FetcherException(("An internal parser error occurred while fetching from " + urlForQuery).replaceAll("(?i)(api[_-]?key)=.*", "[REDACTED]"), e); } } From 09191678e2e2b93d908fcaca8dd37dc45022b2d2 Mon Sep 17 00:00:00 2001 From: rehan Date: Sun, 14 Jul 2024 10:23:42 +0530 Subject: [PATCH 09/42] check-style fixes --- .../org/jabref/logic/importer/FetcherClientException.java | 5 ++--- src/main/java/org/jabref/logic/net/URLDownload.java | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/jabref/logic/importer/FetcherClientException.java b/src/main/java/org/jabref/logic/importer/FetcherClientException.java index 8005f2e5aad..e13bcb139c8 100644 --- a/src/main/java/org/jabref/logic/importer/FetcherClientException.java +++ b/src/main/java/org/jabref/logic/importer/FetcherClientException.java @@ -25,13 +25,12 @@ public FetcherClientException(String errorMessage, String localizedMessage, Thro super(errorMessage, localizedMessage, cause); } - public FetcherClientException(String errorMessage,SimpleHttpResponse httpResponse) { + public FetcherClientException(String errorMessage, SimpleHttpResponse httpResponse) { super(errorMessage); - this.httpResponse=httpResponse; + this.httpResponse = httpResponse; } public SimpleHttpResponse getHttpResponse() { return httpResponse; } - } diff --git a/src/main/java/org/jabref/logic/net/URLDownload.java b/src/main/java/org/jabref/logic/net/URLDownload.java index 482eb80a48d..51670cc3641 100644 --- a/src/main/java/org/jabref/logic/net/URLDownload.java +++ b/src/main/java/org/jabref/logic/net/URLDownload.java @@ -398,7 +398,7 @@ public URLConnection openConnection() throws IOException { if ((status >= 400) && (status < 500)) { SimpleHttpResponse httpResponse = new SimpleHttpResponse(lConnection); LOGGER.info("HTTP {}, details: {}, {}", status, lConnection.getResponseMessage(), lConnection.getContentLength() > 0 ? lConnection.getContent() : ""); - throw new IOException(new FetcherClientException("Encountered HTTP %s %s".formatted(status, lConnection.getResponseMessage()),httpResponse)); + throw new IOException(new FetcherClientException("Encountered HTTP %s %s".formatted(status, lConnection.getResponseMessage()), httpResponse)); } if (status >= 500) { LOGGER.info("HTTP {}, details: {}, {}", status, lConnection.getResponseMessage(), lConnection.getContentLength() > 0 ? lConnection.getContent() : ""); From cad3f7e05e2db33df73c4878352e04d4eda4bb28 Mon Sep 17 00:00:00 2001 From: rehan Date: Sun, 14 Jul 2024 10:46:24 +0530 Subject: [PATCH 10/42] moved api_key removal regex to static Pattern with Pattern.compile --- .../logic/importer/SearchBasedParserFetcher.java | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/jabref/logic/importer/SearchBasedParserFetcher.java b/src/main/java/org/jabref/logic/importer/SearchBasedParserFetcher.java index 40187ea6ef2..09eeba090ca 100644 --- a/src/main/java/org/jabref/logic/importer/SearchBasedParserFetcher.java +++ b/src/main/java/org/jabref/logic/importer/SearchBasedParserFetcher.java @@ -6,6 +6,7 @@ import java.net.URISyntaxException; import java.net.URL; import java.util.List; +import java.util.regex.Pattern; import org.jabref.model.entry.BibEntry; @@ -32,8 +33,14 @@ * We need multi inheritance, because a fetcher might implement multiple query types (such as id fetching {@link IdBasedFetcher}), complete entry {@link EntryBasedFetcher}, and search-based fetcher (this class). *

*/ + public interface SearchBasedParserFetcher extends SearchBasedFetcher, ParserFetcher { + /** + * Pattern to redact API keys from error messages. + */ + Pattern API_KEY_PATTERN = Pattern.compile("(?i)(api[_-]?key)=.*"); + /** * This method is used to send queries with advanced URL parameters. * This method is necessary as the performSearch method does not support certain URL parameters that are used for @@ -60,10 +67,10 @@ private List getBibEntries(URL urlForQuery) throws FetcherException { return fetchedEntries; } catch (IOException e) { // Regular expression to redact API keys from the error message - throw new FetcherException(("A network error occurred while fetching from " + urlForQuery).replaceAll("(?i)(api[_-]?key)=.*", "[REDACTED]"), e); + throw new FetcherException(API_KEY_PATTERN.matcher(("A network error occurred while fetching from " + urlForQuery)).replaceAll("[REDACTED]"), e); } catch (ParseException e) { // Regular expression to redact API keys from the error message - throw new FetcherException(("An internal parser error occurred while fetching from " + urlForQuery).replaceAll("(?i)(api[_-]?key)=.*", "[REDACTED]"), e); + throw new FetcherException(API_KEY_PATTERN.matcher(("An internal parser error occurred while fetching from " + urlForQuery)).replaceAll("[REDACTED]"), e); } } From b21a7af50ba3511e64ebba1a4c5d480e094da3f6 Mon Sep 17 00:00:00 2001 From: rehan Date: Tue, 16 Jul 2024 09:58:18 +0530 Subject: [PATCH 11/42] extract max response length to a class constant MAX_RESPONSE_LENGTH --- .../java/org/jabref/http/dto/SimpleHttpResponse.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/jabref/http/dto/SimpleHttpResponse.java b/src/main/java/org/jabref/http/dto/SimpleHttpResponse.java index 8d02fe60cae..f01e74e0d0f 100644 --- a/src/main/java/org/jabref/http/dto/SimpleHttpResponse.java +++ b/src/main/java/org/jabref/http/dto/SimpleHttpResponse.java @@ -7,6 +7,8 @@ import java.nio.charset.StandardCharsets; public record SimpleHttpResponse(int statusCode, String responseBody, String responseMessage) { + private static final int MAX_RESPONSE_LENGTH = 1024; // 1 KB + public SimpleHttpResponse(int statusCode, String responseBody, String responseMessage) { this.statusCode = statusCode; this.responseBody = truncateResponseBody(responseBody); @@ -30,10 +32,10 @@ public SimpleHttpResponse(HttpURLConnection connection) throws IOException { */ private static String truncateResponseBody(String responseBody) { byte[] bytes = responseBody.getBytes(StandardCharsets.UTF_8); - int maxLength = 1024; // 1 KB - if (bytes.length > maxLength) { + if (bytes.length > MAX_RESPONSE_LENGTH) { // Truncate the response body to 1 KB and append "... (truncated)" - return new String(bytes, 0, maxLength, StandardCharsets.UTF_8) + "... (truncated)"; + // Response is in English, thus we append English text - and not localized text + return new String(bytes, 0, MAX_RESPONSE_LENGTH, StandardCharsets.UTF_8) + "... (truncated)"; } // Return the original response body if it's within the 1 KB limit return responseBody; @@ -52,7 +54,7 @@ private static String getResponseBody(HttpURLConnection connection) throws IOExc try (BufferedReader in = new BufferedReader(new InputStreamReader(connection.getErrorStream()))) { String inputLine; StringBuilder content = new StringBuilder(); - while ((inputLine = in.readLine()) != null) { + while ((content.length() < MAX_RESPONSE_LENGTH) && (inputLine = in.readLine()) != null) { content.append(inputLine); } return truncateResponseBody(content.toString()); From 7a30b01bd2f19357c303473a0d56f16f18ab7077 Mon Sep 17 00:00:00 2001 From: rehan Date: Tue, 16 Jul 2024 16:49:52 +0530 Subject: [PATCH 12/42] moved ParserResult to Callable for achieving asynchronous execution --- .../fetcher/WebSearchPaneViewModel.java | 23 ++++++++++--------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/src/main/java/org/jabref/gui/importer/fetcher/WebSearchPaneViewModel.java b/src/main/java/org/jabref/gui/importer/fetcher/WebSearchPaneViewModel.java index 0c9bf7b6d09..0bbadc0e7e7 100644 --- a/src/main/java/org/jabref/gui/importer/fetcher/WebSearchPaneViewModel.java +++ b/src/main/java/org/jabref/gui/importer/fetcher/WebSearchPaneViewModel.java @@ -2,6 +2,7 @@ import java.util.concurrent.Callable; +import javafx.application.Platform; import javafx.beans.property.ListProperty; import javafx.beans.property.ObjectProperty; import javafx.beans.property.SimpleListProperty; @@ -147,19 +148,19 @@ public void search() { SearchBasedFetcher activeFetcher = getSelectedFetcher(); - ParserResult parserResult; - try { - parserResult = new ParserResult(activeFetcher.performSearch(query)); - } catch (FetcherException e) { - if (e.getCause() != null && e.getCause().getCause() instanceof FetcherClientException clientException) { - dialogService.showErrorDialogAndWait(e.getMessage(), clientException.getHttpResponse().responseBody(), clientException); - } else { - dialogService.showErrorDialogAndWait(e.getMessage(), e); + Callable parserResultCallable = () -> { + try { + return new ParserResult(activeFetcher.performSearch(query)); + } catch (FetcherException e) { + if (e.getCause() != null && e.getCause().getCause() instanceof FetcherClientException clientException) { + Platform.runLater(() -> dialogService.showErrorDialogAndWait(e.getMessage(), clientException.getHttpResponse().responseBody(), clientException)); + } else { + Platform.runLater(() -> dialogService.showErrorDialogAndWait(e.getMessage(), e)); + } } - return; - } + return new ParserResult(); + }; - Callable parserResultCallable = () -> parserResult; String fetcherName = activeFetcher.getName(); if (CompositeIdFetcher.containsValidId(query)) { From cf881052c5a57e619cd8b004eb749ec796ec6bc6 Mon Sep 17 00:00:00 2001 From: rehan Date: Wed, 17 Jul 2024 11:01:40 +0530 Subject: [PATCH 13/42] changed the content of dialogue box to include http response and response body --- .../jabref/gui/importer/fetcher/WebSearchPaneViewModel.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/jabref/gui/importer/fetcher/WebSearchPaneViewModel.java b/src/main/java/org/jabref/gui/importer/fetcher/WebSearchPaneViewModel.java index 0bbadc0e7e7..54b618f5c87 100644 --- a/src/main/java/org/jabref/gui/importer/fetcher/WebSearchPaneViewModel.java +++ b/src/main/java/org/jabref/gui/importer/fetcher/WebSearchPaneViewModel.java @@ -152,10 +152,11 @@ public void search() { try { return new ParserResult(activeFetcher.performSearch(query)); } catch (FetcherException e) { + // FetcherException's cause can be both IOException and ParseException if (e.getCause() != null && e.getCause().getCause() instanceof FetcherClientException clientException) { - Platform.runLater(() -> dialogService.showErrorDialogAndWait(e.getMessage(), clientException.getHttpResponse().responseBody(), clientException)); + Platform.runLater(() -> dialogService.showErrorDialogAndWait(e.getMessage(), (clientException.getMessage() + "\n" + clientException.getHttpResponse().responseBody()), clientException)); } else { - Platform.runLater(() -> dialogService.showErrorDialogAndWait(e.getMessage(), e)); + Platform.runLater(() -> dialogService.showErrorDialogAndWait(e.getMessage(), e.getCause().getMessage(), e)); } } return new ParserResult(); From c02816e31b185b1d7c705109897d9c07bb59295b Mon Sep 17 00:00:00 2001 From: rehan Date: Wed, 17 Jul 2024 14:17:40 +0530 Subject: [PATCH 14/42] updated SearchBasedParserFetcher 1. updated API_KEY_PATTERN a. now it matches with "key also" b. it only hides the api_key and does not remove everything after it 2. moved "[REDACTED]" string to a class constant REDACTED_STRING --- .../logic/importer/SearchBasedParserFetcher.java | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/jabref/logic/importer/SearchBasedParserFetcher.java b/src/main/java/org/jabref/logic/importer/SearchBasedParserFetcher.java index 09eeba090ca..5117d1d8d0b 100644 --- a/src/main/java/org/jabref/logic/importer/SearchBasedParserFetcher.java +++ b/src/main/java/org/jabref/logic/importer/SearchBasedParserFetcher.java @@ -39,7 +39,13 @@ public interface SearchBasedParserFetcher extends SearchBasedFetcher, ParserFetc /** * Pattern to redact API keys from error messages. */ - Pattern API_KEY_PATTERN = Pattern.compile("(?i)(api[_-]?key)=.*"); + Pattern API_KEY_PATTERN = Pattern.compile("(?i)(key|api[-_]?key)=[^&]*"); + /** + * A constant string used to replace or mask sensitive information , such as API keys; + */ + String REDACTED_STRING = "[REDACTED]"; + + /** * This method is used to send queries with advanced URL parameters. @@ -67,10 +73,10 @@ private List getBibEntries(URL urlForQuery) throws FetcherException { return fetchedEntries; } catch (IOException e) { // Regular expression to redact API keys from the error message - throw new FetcherException(API_KEY_PATTERN.matcher(("A network error occurred while fetching from " + urlForQuery)).replaceAll("[REDACTED]"), e); + throw new FetcherException(API_KEY_PATTERN.matcher(("A network error occurred while fetching from " + urlForQuery)).replaceAll(REDACTED_STRING), e); } catch (ParseException e) { // Regular expression to redact API keys from the error message - throw new FetcherException(API_KEY_PATTERN.matcher(("An internal parser error occurred while fetching from " + urlForQuery)).replaceAll("[REDACTED]"), e); + throw new FetcherException(API_KEY_PATTERN.matcher(("An internal parser error occurred while fetching from " + urlForQuery)).replaceAll(REDACTED_STRING), e); } } From 0294c11f5593fb7d04f32c7629d003de600a9489 Mon Sep 17 00:00:00 2001 From: rehan Date: Thu, 25 Jul 2024 14:19:07 +0530 Subject: [PATCH 15/42] updated PagedSearchBasedParserFetcher to not expose api keys in the FetcherException --- .../importer/PagedSearchBasedParserFetcher.java | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/jabref/logic/importer/PagedSearchBasedParserFetcher.java b/src/main/java/org/jabref/logic/importer/PagedSearchBasedParserFetcher.java index 5c2fa3d3dbb..94166ec25c9 100644 --- a/src/main/java/org/jabref/logic/importer/PagedSearchBasedParserFetcher.java +++ b/src/main/java/org/jabref/logic/importer/PagedSearchBasedParserFetcher.java @@ -6,6 +6,7 @@ import java.net.URISyntaxException; import java.net.URL; import java.util.List; +import java.util.regex.Pattern; import org.jabref.model.entry.BibEntry; import org.jabref.model.paging.Page; @@ -14,6 +15,15 @@ public interface PagedSearchBasedParserFetcher extends SearchBasedParserFetcher, PagedSearchBasedFetcher, ParserFetcher { + /** + * Pattern to redact API keys from error messages. + */ + Pattern API_KEY_PATTERN = Pattern.compile("(?i)(api|key|api[-_]?key)=[^&]*"); + /** + * A constant string used to replace or mask sensitive information , such as API keys; + */ + String REDACTED_STRING = "[REDACTED]"; + @Override default Page performSearchPaged(QueryNode luceneQuery, int pageNumber) throws FetcherException { // ADR-0014 @@ -32,9 +42,9 @@ private List getBibEntries(URL urlForQuery) throws FetcherException { fetchedEntries.forEach(this::doPostCleanup); return fetchedEntries; } catch (IOException e) { - throw new FetcherException("A network error occurred while fetching from " + urlForQuery, e); + throw new FetcherException(API_KEY_PATTERN.matcher(("A network error occurred while fetching from " + urlForQuery)).replaceAll(REDACTED_STRING), e); } catch (ParseException e) { - throw new FetcherException("An internal parser error occurred while fetching from " + urlForQuery, e); + throw new FetcherException(API_KEY_PATTERN.matcher(("An internal parser error occurred while fetching from " + urlForQuery)).replaceAll(REDACTED_STRING), e); } } From 3228418929cbc2611d467e6e256a1a00451ba9f6 Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Wed, 7 Aug 2024 17:41:37 +0200 Subject: [PATCH 16/42] Add TODO --- .../org/jabref/logic/net/URLDownload.java | 25 ++++++++++--------- 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/src/main/java/org/jabref/logic/net/URLDownload.java b/src/main/java/org/jabref/logic/net/URLDownload.java index 51670cc3641..e1e96204b56 100644 --- a/src/main/java/org/jabref/logic/net/URLDownload.java +++ b/src/main/java/org/jabref/logic/net/URLDownload.java @@ -42,7 +42,6 @@ import org.jabref.http.dto.SimpleHttpResponse; import org.jabref.logic.importer.FetcherClientException; -import org.jabref.logic.importer.FetcherServerException; import org.jabref.logic.util.io.FileUtil; import kong.unirest.core.Unirest; @@ -382,9 +381,16 @@ public URLConnection openConnection() throws IOException { } } - if (connection instanceof HttpURLConnection lConnection) { + if (connection instanceof HttpURLConnection httpURLConnection) { // this does network i/o: GET + read returned headers - int status = lConnection.getResponseCode(); + int status; + try { + status = httpURLConnection.getResponseCode(); + } catch (IOException e) { + LOGGER.error("Error getting response code", e); + // TODO: Convert e to FetcherClientException + throw e; + } // normally, 3xx is redirect if ((status == HttpURLConnection.HTTP_MOVED_TEMP) @@ -394,15 +400,10 @@ public URLConnection openConnection() throws IOException { String newUrl = connection.getHeaderField("location"); // open the new connection again connection = new URLDownload(newUrl).openConnection(); - } - if ((status >= 400) && (status < 500)) { - SimpleHttpResponse httpResponse = new SimpleHttpResponse(lConnection); - LOGGER.info("HTTP {}, details: {}, {}", status, lConnection.getResponseMessage(), lConnection.getContentLength() > 0 ? lConnection.getContent() : ""); - throw new IOException(new FetcherClientException("Encountered HTTP %s %s".formatted(status, lConnection.getResponseMessage()), httpResponse)); - } - if (status >= 500) { - LOGGER.info("HTTP {}, details: {}, {}", status, lConnection.getResponseMessage(), lConnection.getContentLength() > 0 ? lConnection.getContent() : ""); - throw new IOException(new FetcherServerException("Encountered HTTP %s %s".formatted(status, lConnection.getResponseMessage()))); + } else if (status >= 400) { + SimpleHttpResponse httpResponse = new SimpleHttpResponse(httpURLConnection); + LOGGER.info("HTTP {}, details: {}, {}", status, httpURLConnection.getResponseMessage(), httpURLConnection.getContentLength() > 0 ? httpURLConnection.getContent() : ""); + throw new IOException(new FetcherClientException("Encountered HTTP %s %s".formatted(status, httpURLConnection.getResponseMessage()), httpResponse)); } } return connection; From 84351ef5301fdb4108f10db0dcc7f7db56949e90 Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Sat, 10 Aug 2024 15:46:19 +0200 Subject: [PATCH 17/42] More intuitive ordering --- .../java/org/jabref/http/dto/SimpleHttpResponse.java | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/jabref/http/dto/SimpleHttpResponse.java b/src/main/java/org/jabref/http/dto/SimpleHttpResponse.java index f01e74e0d0f..2c612db126a 100644 --- a/src/main/java/org/jabref/http/dto/SimpleHttpResponse.java +++ b/src/main/java/org/jabref/http/dto/SimpleHttpResponse.java @@ -6,21 +6,17 @@ import java.net.HttpURLConnection; import java.nio.charset.StandardCharsets; -public record SimpleHttpResponse(int statusCode, String responseBody, String responseMessage) { +public record SimpleHttpResponse(int statusCode, String responseMessage, String responseBody) { private static final int MAX_RESPONSE_LENGTH = 1024; // 1 KB - public SimpleHttpResponse(int statusCode, String responseBody, String responseMessage) { + public SimpleHttpResponse(int statusCode, String responseMessage, String responseBody) { this.statusCode = statusCode; this.responseBody = truncateResponseBody(responseBody); this.responseMessage = responseMessage; } public SimpleHttpResponse(HttpURLConnection connection) throws IOException { - this( - connection.getResponseCode(), - getResponseBody(connection), - connection.getResponseMessage() - ); + this(connection.getResponseCode(), connection.getResponseMessage(), getResponseBody(connection)); } /** From 95054b151bbd2ec3a41119a0b5a6bbb108c5eeb0 Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Sat, 10 Aug 2024 16:13:53 +0200 Subject: [PATCH 18/42] No error display, but new ParserResult from error --- .../gui/importer/fetcher/WebSearchPaneViewModel.java | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/src/main/java/org/jabref/gui/importer/fetcher/WebSearchPaneViewModel.java b/src/main/java/org/jabref/gui/importer/fetcher/WebSearchPaneViewModel.java index 54b618f5c87..87e9227881a 100644 --- a/src/main/java/org/jabref/gui/importer/fetcher/WebSearchPaneViewModel.java +++ b/src/main/java/org/jabref/gui/importer/fetcher/WebSearchPaneViewModel.java @@ -2,7 +2,6 @@ import java.util.concurrent.Callable; -import javafx.application.Platform; import javafx.beans.property.ListProperty; import javafx.beans.property.ObjectProperty; import javafx.beans.property.SimpleListProperty; @@ -17,7 +16,6 @@ import org.jabref.gui.importer.ImportEntriesDialog; import org.jabref.gui.util.BackgroundTask; import org.jabref.logic.importer.CompositeIdFetcher; -import org.jabref.logic.importer.FetcherClientException; import org.jabref.logic.importer.FetcherException; import org.jabref.logic.importer.ParserResult; import org.jabref.logic.importer.SearchBasedFetcher; @@ -152,14 +150,8 @@ public void search() { try { return new ParserResult(activeFetcher.performSearch(query)); } catch (FetcherException e) { - // FetcherException's cause can be both IOException and ParseException - if (e.getCause() != null && e.getCause().getCause() instanceof FetcherClientException clientException) { - Platform.runLater(() -> dialogService.showErrorDialogAndWait(e.getMessage(), (clientException.getMessage() + "\n" + clientException.getHttpResponse().responseBody()), clientException)); - } else { - Platform.runLater(() -> dialogService.showErrorDialogAndWait(e.getMessage(), e.getCause().getMessage(), e)); - } + return ParserResult.fromError(e); } - return new ParserResult(); }; String fetcherName = activeFetcher.getName(); From 3abb7a26decf88d9649f40a4c0692ba93e7fe665 Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Sat, 10 Aug 2024 16:34:40 +0200 Subject: [PATCH 19/42] MAke use of SimpleHttpResponse Co-authored-by: Carl Christian Snethlage <50491877+calixtus@users.noreply.github.com> --- .../linkedfile/DownloadLinkedFileAction.java | 34 +++++++++++-------- .../org/jabref/logic/JabRefException.java | 5 +-- .../importer/FetcherClientException.java | 21 +++--------- .../logic/importer/FetcherException.java | 27 +++++++++++++++ .../importer/FetcherServerException.java | 19 +++++------ .../org/jabref/logic/net/URLDownload.java | 11 ++++-- src/main/resources/l10n/JabRef_en.properties | 11 +++--- 7 files changed, 77 insertions(+), 51 deletions(-) diff --git a/src/main/java/org/jabref/gui/linkedfile/DownloadLinkedFileAction.java b/src/main/java/org/jabref/gui/linkedfile/DownloadLinkedFileAction.java index 2f2b4b1c9ab..bfcae690366 100644 --- a/src/main/java/org/jabref/gui/linkedfile/DownloadLinkedFileAction.java +++ b/src/main/java/org/jabref/gui/linkedfile/DownloadLinkedFileAction.java @@ -29,6 +29,7 @@ import org.jabref.gui.fieldeditors.URLUtil; import org.jabref.gui.util.BackgroundTask; import org.jabref.gui.util.TaskExecutor; +import org.jabref.http.dto.SimpleHttpResponse; import org.jabref.logic.externalfiles.LinkedFileHandler; import org.jabref.logic.importer.FetcherClientException; import org.jabref.logic.importer.FetcherServerException; @@ -199,24 +200,29 @@ private void onSuccess(Path targetDirectory, Path downloadedFile) { private void onFailure(URLDownload urlDownload, Exception ex) { LOGGER.error("Error downloading from URL: {}", urlDownload, ex); - String fetcherExceptionMessage = ex.getMessage(); + String fetcherExceptionMessage = ex.getLocalizedMessage(); String failedTitle = Localization.lang("Failed to download from URL"); - int statusCode; - if (ex instanceof FetcherClientException clientException) { - statusCode = clientException.getHttpResponse().statusCode(); - if (statusCode == 401) { - dialogService.showInformationDialogAndWait(failedTitle, Localization.lang("401 Unauthorized: Access Denied. You are not authorized to access this resource. Please check your credentials and try again. If you believe you should have access, please contact the administrator for assistance.\nURL: %0 \n %1", urlDownload.getSource(), fetcherExceptionMessage)); - } else if (statusCode == 403) { - dialogService.showInformationDialogAndWait(failedTitle, Localization.lang("403 Forbidden: Access Denied. You do not have permission to access this resource. Please contact the administrator for assistance or try a different action.\nURL: %0 \n %1", urlDownload.getSource(), fetcherExceptionMessage)); - } else if (statusCode == 404) { - dialogService.showInformationDialogAndWait(failedTitle, Localization.lang("404 Not Found Error: The requested resource could not be found. It seems that the file you are trying to download is not available or has been moved. Please verify the URL and try again. If you believe this is an error, please contact the administrator for further assistance.\nURL: %0 \n %1", urlDownload.getSource(), fetcherExceptionMessage)); + if (ex instanceof FetcherClientException fetcherException) { + Optional httpResponse = fetcherException.getHttpResponse(); + if (httpResponse.isPresent()) { + int statusCode = httpResponse.get().statusCode(); + if (statusCode == 401) { + dialogService.showInformationDialogAndWait(failedTitle, Localization.lang("401 Unauthorized: Access Denied. You are not authorized to access this resource. Please check your credentials and try again. If you believe you should have access, please contact the administrator for assistance.\nURL: %0\nDetails: %1", urlDownload.getSource(), fetcherExceptionMessage)); + } else if (statusCode == 403) { + dialogService.showInformationDialogAndWait(failedTitle, Localization.lang("403 Forbidden: Access Denied. You do not have permission to access this resource. Please contact the administrator for assistance or try a different action.\nURL: %0\nDetails: %1", urlDownload.getSource(), fetcherExceptionMessage)); + } else if (statusCode == 404) { + dialogService.showInformationDialogAndWait(failedTitle, Localization.lang("404 Not Found Error: The requested resource could not be found. It seems that the file you are trying to download is not available or has been moved. Please verify the URL and try again. If you believe this is an error, please contact the administrator for further assistance.\nURL: %0\nDetails: %1", urlDownload.getSource(), fetcherExceptionMessage)); + } else { + dialogService.showErrorDialogAndWait(failedTitle, Localization.lang("Something is wrong on JabRef side. Please check the URL and try again.\nURL: %0\nDetails: %1", urlDownload.getSource(), fetcherExceptionMessage)); + } + } else { + dialogService.showErrorDialogAndWait(failedTitle, Localization.lang("Something is wrong on JabRef side. Please check the URL and try again.\nURL: %0\nDetails: %1", urlDownload.getSource(), fetcherExceptionMessage)); } - } else if (ex instanceof FetcherServerException serverException) { - statusCode = serverException.getStatusCode(); + } else if (ex instanceof FetcherServerException) { dialogService.showInformationDialogAndWait(failedTitle, - Localization.lang("Error downloading from URL. Cause is likely the server side. HTTP Error %0 \n %1 \nURL: %2 \nPlease try again later or contact the server administrator.", statusCode, fetcherExceptionMessage, urlDownload.getSource())); + Localization.lang("Error downloading from URL. Cause is likely the server side.\nPlease try again later or contact the server administrator.\nURL: %0\nDetails: %1", urlDownload.getSource(), fetcherExceptionMessage)); } else { - dialogService.showErrorDialogAndWait(failedTitle, Localization.lang("Error message: %0 \nURL: %1 \nPlease check the URL and try again.", fetcherExceptionMessage, urlDownload.getSource())); + dialogService.showErrorDialogAndWait(failedTitle, Localization.lang("Please check the URL and try again.\nURL: %0\nDetails: %1", urlDownload.getSource(), fetcherExceptionMessage)); } } diff --git a/src/main/java/org/jabref/logic/JabRefException.java b/src/main/java/org/jabref/logic/JabRefException.java index 687383f9823..4caad91fa1d 100644 --- a/src/main/java/org/jabref/logic/JabRefException.java +++ b/src/main/java/org/jabref/logic/JabRefException.java @@ -33,8 +33,9 @@ public JabRefException(Throwable cause) { @Override public String getLocalizedMessage() { if (localizedMessage == null) { - LOGGER.debug("No localized exception message defined. Falling back to getMessage()."); - return getMessage(); + String message = getMessage(); + LOGGER.debug("No localized exception message defined. Falling back to getMessage() ({}).", message); + return message; } else { return localizedMessage; } diff --git a/src/main/java/org/jabref/logic/importer/FetcherClientException.java b/src/main/java/org/jabref/logic/importer/FetcherClientException.java index e13bcb139c8..7dbd0b4b3df 100644 --- a/src/main/java/org/jabref/logic/importer/FetcherClientException.java +++ b/src/main/java/org/jabref/logic/importer/FetcherClientException.java @@ -6,31 +6,20 @@ * Should be thrown when you encounter an HTTP status code error >= 400 and < 500. */ public class FetcherClientException extends FetcherException { - private SimpleHttpResponse httpResponse; - public FetcherClientException(String errorMessage, Throwable cause) { - super(errorMessage, cause); + public FetcherClientException(String errorMessage) { + super(errorMessage); } - public FetcherClientException(String errorMessage, Throwable cause, SimpleHttpResponse httpResponse) { + public FetcherClientException(String errorMessage, Throwable cause) { super(errorMessage, cause); - this.httpResponse = httpResponse; - } - - public FetcherClientException(String errorMessage) { - super(errorMessage); } public FetcherClientException(String errorMessage, String localizedMessage, Throwable cause) { super(errorMessage, localizedMessage, cause); } - public FetcherClientException(String errorMessage, SimpleHttpResponse httpResponse) { - super(errorMessage); - this.httpResponse = httpResponse; - } - - public SimpleHttpResponse getHttpResponse() { - return httpResponse; + public FetcherClientException(SimpleHttpResponse simpleHttpResponse) { + super(simpleHttpResponse); } } diff --git a/src/main/java/org/jabref/logic/importer/FetcherException.java b/src/main/java/org/jabref/logic/importer/FetcherException.java index 03557eef2bc..fdbe9d60d72 100644 --- a/src/main/java/org/jabref/logic/importer/FetcherException.java +++ b/src/main/java/org/jabref/logic/importer/FetcherException.java @@ -1,18 +1,45 @@ package org.jabref.logic.importer; +import java.util.Optional; + +import org.jabref.http.dto.SimpleHttpResponse; import org.jabref.logic.JabRefException; public class FetcherException extends JabRefException { + private final SimpleHttpResponse httpResponse; + + public FetcherException(SimpleHttpResponse httpResponse) { + super(httpResponse.responseMessage()); + this.httpResponse = httpResponse; + } + public FetcherException(String errorMessage, Throwable cause) { super(errorMessage, cause); + httpResponse = null; } public FetcherException(String errorMessage) { super(errorMessage); + httpResponse = null; } public FetcherException(String errorMessage, String localizedMessage, Throwable cause) { super(errorMessage, localizedMessage, cause); + httpResponse = null; + } + + @Override + public String getLocalizedMessage() { + // TODO: This should be moved to a separate class converting "any" exception object to a localized message + if (httpResponse != null) { + return "Encountered HTTP %s %s (%s)".formatted(httpResponse.statusCode(), httpResponse.responseMessage(), httpResponse.responseBody()); + } else { + return super.getLocalizedMessage(); + } + } + + public Optional getHttpResponse() { + return Optional.ofNullable(httpResponse); } } diff --git a/src/main/java/org/jabref/logic/importer/FetcherServerException.java b/src/main/java/org/jabref/logic/importer/FetcherServerException.java index 7b657553b5f..ee769aa6105 100644 --- a/src/main/java/org/jabref/logic/importer/FetcherServerException.java +++ b/src/main/java/org/jabref/logic/importer/FetcherServerException.java @@ -1,28 +1,25 @@ package org.jabref.logic.importer; + +import org.jabref.http.dto.SimpleHttpResponse; + /** * Should be thrown when you encounter a http status code error >= 500 */ public class FetcherServerException extends FetcherException { - private int statusCode; - public FetcherServerException(String errorMessage, Throwable cause) { - super(errorMessage, cause); + public FetcherServerException(String errorMessage) { + super(errorMessage); } - public FetcherServerException(String errorMessage, Throwable cause, int statusCode) { + public FetcherServerException(String errorMessage, Throwable cause) { super(errorMessage, cause); - this.statusCode = statusCode; - } - - public FetcherServerException(String errorMessage) { - super(errorMessage); } public FetcherServerException(String errorMessage, String localizedMessage, Throwable cause) { super(errorMessage, localizedMessage, cause); } - public int getStatusCode() { - return statusCode; + public FetcherServerException(SimpleHttpResponse httpResponse) { + super(httpResponse); } } diff --git a/src/main/java/org/jabref/logic/net/URLDownload.java b/src/main/java/org/jabref/logic/net/URLDownload.java index e1e96204b56..7c6e80d3f7e 100644 --- a/src/main/java/org/jabref/logic/net/URLDownload.java +++ b/src/main/java/org/jabref/logic/net/URLDownload.java @@ -42,6 +42,7 @@ import org.jabref.http.dto.SimpleHttpResponse; import org.jabref.logic.importer.FetcherClientException; +import org.jabref.logic.importer.FetcherServerException; import org.jabref.logic.util.io.FileUtil; import kong.unirest.core.Unirest; @@ -400,10 +401,14 @@ public URLConnection openConnection() throws IOException { String newUrl = connection.getHeaderField("location"); // open the new connection again connection = new URLDownload(newUrl).openConnection(); - } else if (status >= 400) { + } else { SimpleHttpResponse httpResponse = new SimpleHttpResponse(httpURLConnection); - LOGGER.info("HTTP {}, details: {}, {}", status, httpURLConnection.getResponseMessage(), httpURLConnection.getContentLength() > 0 ? httpURLConnection.getContent() : ""); - throw new IOException(new FetcherClientException("Encountered HTTP %s %s".formatted(status, httpURLConnection.getResponseMessage()), httpResponse)); + LOGGER.info("HTTP {} {} - body: {}", status, httpURLConnection.getResponseMessage(), httpURLConnection.getContentLength() > 0 ? httpURLConnection.getContent() : ""); + if ((status >= 400) && (status < 500)) { + throw new IOException(new FetcherClientException(httpResponse)); + } else if (status >= 500) { + throw new IOException(new FetcherServerException(httpResponse)); + } } } return connection; diff --git a/src/main/resources/l10n/JabRef_en.properties b/src/main/resources/l10n/JabRef_en.properties index 8bf35d2088f..09b9287864b 100644 --- a/src/main/resources/l10n/JabRef_en.properties +++ b/src/main/resources/l10n/JabRef_en.properties @@ -2609,12 +2609,13 @@ File\ "%0"\ cannot\ be\ added\!=File "%0" cannot be added! Illegal\ characters\ in\ the\ file\ name\ detected.\nFile\ will\ be\ renamed\ to\ "%0"\ and\ added.=Illegal characters in the file name detected.\nFile will be renamed to "%0" and added. Rename\ and\ add=Rename and add -401\ Unauthorized\:\ Access\ Denied.\ You\ are\ not\ authorized\ to\ access\ this\ resource.\ Please\ check\ your\ credentials\ and\ try\ again.\ If\ you\ believe\ you\ should\ have\ access,\ please\ contact\ the\ administrator\ for\ assistance.\nURL\:\ %0\ \n\ %1=401 Unauthorized: Access Denied. You are not authorized to access this resource. Please check your credentials and try again. If you believe you should have access, please contact the administrator for assistance.\nURL: %0 \n %1 -403\ Forbidden\:\ Access\ Denied.\ You\ do\ not\ have\ permission\ to\ access\ this\ resource.\ Please\ contact\ the\ administrator\ for\ assistance\ or\ try\ a\ different\ action.\nURL\:\ %0\ \n\ %1=403 Forbidden: Access Denied. You do not have permission to access this resource. Please contact the administrator for assistance or try a different action.\nURL: %0 \n %1 -404\ Not\ Found\ Error\:\ The\ requested\ resource\ could\ not\ be\ found.\ It\ seems\ that\ the\ file\ you\ are\ trying\ to\ download\ is\ not\ available\ or\ has\ been\ moved.\ Please\ verify\ the\ URL\ and\ try\ again.\ If\ you\ believe\ this\ is\ an\ error,\ please\ contact\ the\ administrator\ for\ further\ assistance.\nURL\:\ %0\ \n\ %1=404 Not Found Error: The requested resource could not be found. It seems that the file you are trying to download is not available or has been moved. Please verify the URL and try again. If you believe this is an error, please contact the administrator for further assistance.\nURL: %0 \n %1 -Error\ downloading\ from\ URL.\ Cause\ is\ likely\ the\ server\ side.\ HTTP\ Error\ %0\ \n\ %1\ \nURL\:\ %2\ \nPlease\ try\ again\ later\ or\ contact\ the\ server\ administrator.=Error downloading from URL. Cause is likely the server side. HTTP Error %0 \n %1 \nURL: %2 \nPlease try again later or contact the server administrator. -Error\ message\:\ %0\ \nURL\:\ %1\ \nPlease\ check\ the\ URL\ and\ try\ again.=Error message: %0 \nURL: %1 \nPlease check the URL and try again. Failed\ to\ download\ from\ URL=Failed to download from URL +401\ Unauthorized\:\ Access\ Denied.\ You\ are\ not\ authorized\ to\ access\ this\ resource.\ Please\ check\ your\ credentials\ and\ try\ again.\ If\ you\ believe\ you\ should\ have\ access,\ please\ contact\ the\ administrator\ for\ assistance.\nURL\:\ %0\nDetails\:\ %1=401 Unauthorized: Access Denied. You are not authorized to access this resource. Please check your credentials and try again. If you believe you should have access, please contact the administrator for assistance.\nURL: %0\nDetails: %1 +403\ Forbidden\:\ Access\ Denied.\ You\ do\ not\ have\ permission\ to\ access\ this\ resource.\ Please\ contact\ the\ administrator\ for\ assistance\ or\ try\ a\ different\ action.\nURL\:\ %0\nDetails\:\ %1=403 Forbidden: Access Denied. You do not have permission to access this resource. Please contact the administrator for assistance or try a different action.\nURL: %0\nDetails: %1 +404\ Not\ Found\ Error\:\ The\ requested\ resource\ could\ not\ be\ found.\ It\ seems\ that\ the\ file\ you\ are\ trying\ to\ download\ is\ not\ available\ or\ has\ been\ moved.\ Please\ verify\ the\ URL\ and\ try\ again.\ If\ you\ believe\ this\ is\ an\ error,\ please\ contact\ the\ administrator\ for\ further\ assistance.\nURL\:\ %0\nDetails\:\ %1=404 Not Found Error: The requested resource could not be found. It seems that the file you are trying to download is not available or has been moved. Please verify the URL and try again. If you believe this is an error, please contact the administrator for further assistance.\nURL: %0\nDetails: %1 +Error\ downloading\ from\ URL.\ Cause\ is\ likely\ the\ server\ side.\nPlease\ try\ again\ later\ or\ contact\ the\ server\ administrator.\nURL\:\ %0\nDetails\:\ %1=Error downloading from URL. Cause is likely the server side.\nPlease try again later or contact the server administrator.\nURL: %0\nDetails: %1 +Please\ check\ the\ URL\ and\ try\ again.\nURL\:\ %0\nDetails\:\ %1=Please check the URL and try again.\nURL: %0\nDetails: %1 +Something\ is\ wrong\ on\ JabRef\ side.\ Please\ check\ the\ URL\ and\ try\ again.\nURL\:\ %0\nDetails\:\ %1=Something is wrong on JabRef side. Please check the URL and try again.\nURL: %0\nDetails: %1 Finished=Finished Finished\ writing\ metadata\ for\ library\ %0\ (%1\ succeeded,\ %2\ skipped,\ %3\ errors).=Finished writing metadata for library %0 (%1 succeeded, %2 skipped, %3 errors). From 91440e5dbf0cc86c36a32374cf60fa6312955463 Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Sat, 10 Aug 2024 16:36:02 +0200 Subject: [PATCH 20/42] Fix NPE --- src/main/java/org/jabref/http/dto/SimpleHttpResponse.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/jabref/http/dto/SimpleHttpResponse.java b/src/main/java/org/jabref/http/dto/SimpleHttpResponse.java index 2c612db126a..fd50d481789 100644 --- a/src/main/java/org/jabref/http/dto/SimpleHttpResponse.java +++ b/src/main/java/org/jabref/http/dto/SimpleHttpResponse.java @@ -2,6 +2,7 @@ import java.io.BufferedReader; import java.io.IOException; +import java.io.InputStream; import java.io.InputStreamReader; import java.net.HttpURLConnection; import java.nio.charset.StandardCharsets; @@ -47,7 +48,11 @@ private static String truncateResponseBody(String responseBody) { * @throws IOException if an I/O error occurs while reading the response body */ private static String getResponseBody(HttpURLConnection connection) throws IOException { - try (BufferedReader in = new BufferedReader(new InputStreamReader(connection.getErrorStream()))) { + InputStream errorStream = connection.getErrorStream(); + if (errorStream == null) { + return ""; + } + try (BufferedReader in = new BufferedReader(new InputStreamReader(errorStream))) { String inputLine; StringBuilder content = new StringBuilder(); while ((content.length() < MAX_RESPONSE_LENGTH) && (inputLine = in.readLine()) != null) { From e6949bd77b170acff1b06d55e87e72330dc73171 Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Sat, 10 Aug 2024 16:43:16 +0200 Subject: [PATCH 21/42] Add missing quotes and dots - and fix Citation Style processing l10n --- .../gui/importer/fetcher/WebSearchPaneViewModel.java | 2 +- .../java/org/jabref/gui/preview/PreviewViewer.java | 2 +- src/main/resources/l10n/JabRef_en.properties | 10 ++++++---- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/jabref/gui/importer/fetcher/WebSearchPaneViewModel.java b/src/main/java/org/jabref/gui/importer/fetcher/WebSearchPaneViewModel.java index 87e9227881a..12d7809b91c 100644 --- a/src/main/java/org/jabref/gui/importer/fetcher/WebSearchPaneViewModel.java +++ b/src/main/java/org/jabref/gui/importer/fetcher/WebSearchPaneViewModel.java @@ -163,7 +163,7 @@ public void search() { } BackgroundTask task = BackgroundTask.wrap(parserResultCallable) - .withInitialMessage(Localization.lang("Processing %0", query)); + .withInitialMessage(Localization.lang("Processing \"%0\"...", query)); task.onFailure(dialogService::showErrorDialogAndWait); ImportEntriesDialog dialog = new ImportEntriesDialog(stateManager.getActiveDatabase().get(), task); diff --git a/src/main/java/org/jabref/gui/preview/PreviewViewer.java b/src/main/java/org/jabref/gui/preview/PreviewViewer.java index d17400cd6e1..b8e260f1d89 100644 --- a/src/main/java/org/jabref/gui/preview/PreviewViewer.java +++ b/src/main/java/org/jabref/gui/preview/PreviewViewer.java @@ -264,7 +264,7 @@ private void update() { final BibEntry theEntry = entry.get(); BackgroundTask .wrap(() -> layout.generatePreview(theEntry, database)) - .onRunning(() -> setPreviewText("" + Localization.lang("Processing %0", Localization.lang("Citation Style")) + ": " + layout.getDisplayName() + " ..." + "")) + .onRunning(() -> setPreviewText("" + Localization.lang("Processing Citation Style \"%0\"...", layout.getDisplayName()) + "")) .onSuccess(this::setPreviewText) .onFailure(exception -> { LOGGER.error("Error while generating citation style", exception); diff --git a/src/main/resources/l10n/JabRef_en.properties b/src/main/resources/l10n/JabRef_en.properties index 09b9287864b..f7db794d270 100644 --- a/src/main/resources/l10n/JabRef_en.properties +++ b/src/main/resources/l10n/JabRef_en.properties @@ -329,8 +329,14 @@ Extract\ references\ from\ file\ (offline)=Extract references from file (offline Extract\ references\ from\ file\ (online)=Extract references from file (online) Extract\ References\ (offline)=Extract References (offline) Extract\ References\ (online)=Extract References (online) + +Processing...=Processing... +Processing\ "%0"...=Processing "%0"... +Processing\ Citation\ Style\ "%0"...=Processing Citation Style "%0"... Processing\ PDF(s)=Processing PDF(s) +Processing\ file\ %0=Processing file %0 Processing\ a\ large\ number\ of\ files=Processing a large number of files + You\ are\ about\ to\ process\ %0\ files.\ Continue?=You are about to process %0 files. Continue? Will\ write\ metadata\ to\ the\ PDFs\ linked\ from\ selected\ entries.=Will write metadata to the PDFs linked from selected entries. @@ -691,7 +697,6 @@ Preferences=Preferences Preferences\ recorded.=Preferences recorded. Preview=Preview -Citation\ Style=Citation Style Current\ Preview=Current Preview Add\ BST\ file=Add BST file @@ -707,7 +712,6 @@ Selected\ Layouts\ can\ not\ be\ empty=Selected Layouts can not be empty Reset\ default\ preview\ style=Reset default preview style Previous\ entry=Previous entry Problem\ with\ parsing\ entry=Problem with parsing entry -Processing\ %0=Processing %0 Pull\ changes\ from\ shared\ database=Pull changes from shared database Pushed\ citations\ to\ %0=Pushed citations to %0 @@ -2378,7 +2382,6 @@ Error\ reading\ PDF\ content\:\ %0=Error reading PDF content\: %0 Bib\ entry\ was\ successfully\ imported=Bib entry was successfully imported File\ was\ successfully\ imported\ as\ a\ new\ entry=File was successfully imported as a new entry No\ BibTeX\ data\ was\ found.\ An\ empty\ entry\ was\ created\ with\ file\ link.=No BibTeX data was found. An empty entry was created with file link. -Processing\ file\ %0=Processing file %0 Export\ selected=Export selected Separate\ merged\ citations=Separate merged citations @@ -2619,7 +2622,6 @@ Something\ is\ wrong\ on\ JabRef\ side.\ Please\ check\ the\ URL\ and\ try\ agai Finished=Finished Finished\ writing\ metadata\ for\ library\ %0\ (%1\ succeeded,\ %2\ skipped,\ %3\ errors).=Finished writing metadata for library %0 (%1 succeeded, %2 skipped, %3 errors). -Processing...=Processing... Writing\ metadata\ to\ %0=Writing metadata to %0 Get\ more\ themes...=Get more themes... From 1d97b76f3bed61212ccf0026d8cf945715a87478 Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Sat, 10 Aug 2024 17:06:19 +0200 Subject: [PATCH 22/42] Fix handling of errors at CiteSeer Co-authored-by: Carl Christian Snethlage <50491877+calixtus@users.noreply.github.com> --- .../logic/importer/fetcher/CiteSeer.java | 42 ++++++++++++------- 1 file changed, 27 insertions(+), 15 deletions(-) diff --git a/src/main/java/org/jabref/logic/importer/fetcher/CiteSeer.java b/src/main/java/org/jabref/logic/importer/fetcher/CiteSeer.java index 4d63622b68f..a0286d0005e 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/CiteSeer.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/CiteSeer.java @@ -6,6 +6,7 @@ import java.util.Objects; import java.util.Optional; +import org.jabref.http.dto.SimpleHttpResponse; import org.jabref.logic.help.HelpFile; import org.jabref.logic.importer.FetcherException; import org.jabref.logic.importer.FulltextFetcher; @@ -16,14 +17,19 @@ import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.field.StandardField; +import kong.unirest.core.HttpResponse; import kong.unirest.core.JsonNode; import kong.unirest.core.Unirest; import kong.unirest.core.json.JSONArray; import kong.unirest.core.json.JSONElement; import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class CiteSeer implements SearchBasedFetcher, FulltextFetcher { + private static final Logger LOGGER = LoggerFactory.getLogger(CiteSeer.class); + private static final String BASE_URL = "citeseerx.ist.psu.edu"; private static final String API_URL = "https://citeseerx.ist.psu.edu/api/search"; @@ -50,21 +56,27 @@ public List performSearch(QueryNode luceneQuery) throws FetcherExcepti // ADR-0014 try { JSONElement payload = getPayloadJSON(luceneQuery); - JsonNode requestResponse = Unirest.post(API_URL) - .header("authority", BASE_URL) - .header("accept", "application/json, text/plain, */*") - .header("content-type", "application/json;charset=UTF-8") - .header("origin", "https://" + BASE_URL) - .body(payload) - .asJson().getBody(); - - Optional jsonResponse = Optional.of(requestResponse) - .map(JsonNode::getObject) - .filter(Objects::nonNull) - .map(response -> response.optJSONArray("response")) - .filter(Objects::nonNull); - - if (!jsonResponse.isPresent()) { + HttpResponse httpResponse = Unirest.post(API_URL) + .header("authority", BASE_URL) + .header("accept", "application/json, text/plain, */*") + .header("content-type", "application/json;charset=UTF-8") + .header("origin", "https://" + BASE_URL) + .body(payload) + .asJson(); + if (!httpResponse.isSuccess()) { + LOGGER.debug("No success"); + // TODO: body needs to be added to the exception, but we currently only have JSON available, but the error is most probably simple text (or HTML) + SimpleHttpResponse simpleHttpResponse = new SimpleHttpResponse(httpResponse.getStatus(), httpResponse.getStatusText(), ""); + throw new FetcherException(simpleHttpResponse); + } + + JsonNode requestResponse = httpResponse.getBody(); + Optional jsonResponse = Optional.ofNullable(requestResponse) + .map(JsonNode::getObject) + .map(response -> response.optJSONArray("response")); + + if (jsonResponse.isEmpty()) { + LOGGER.debug("No entries found for query: {}", luceneQuery); return List.of(); } From 7663f0e4daa2e48724ee596b181ccd41ca40e17b Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Sat, 10 Aug 2024 17:06:40 +0200 Subject: [PATCH 23/42] "Fix" display of error messages --- .../gui/importer/fetcher/WebSearchPaneViewModel.java | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/jabref/gui/importer/fetcher/WebSearchPaneViewModel.java b/src/main/java/org/jabref/gui/importer/fetcher/WebSearchPaneViewModel.java index 12d7809b91c..f83ab6dee36 100644 --- a/src/main/java/org/jabref/gui/importer/fetcher/WebSearchPaneViewModel.java +++ b/src/main/java/org/jabref/gui/importer/fetcher/WebSearchPaneViewModel.java @@ -16,7 +16,6 @@ import org.jabref.gui.importer.ImportEntriesDialog; import org.jabref.gui.util.BackgroundTask; import org.jabref.logic.importer.CompositeIdFetcher; -import org.jabref.logic.importer.FetcherException; import org.jabref.logic.importer.ParserResult; import org.jabref.logic.importer.SearchBasedFetcher; import org.jabref.logic.importer.WebFetchers; @@ -146,13 +145,7 @@ public void search() { SearchBasedFetcher activeFetcher = getSelectedFetcher(); - Callable parserResultCallable = () -> { - try { - return new ParserResult(activeFetcher.performSearch(query)); - } catch (FetcherException e) { - return ParserResult.fromError(e); - } - }; + Callable parserResultCallable; String fetcherName = activeFetcher.getName(); @@ -160,6 +153,9 @@ public void search() { CompositeIdFetcher compositeIdFetcher = new CompositeIdFetcher(preferencesService.getImportFormatPreferences()); parserResultCallable = () -> new ParserResult(OptionalUtil.toList(compositeIdFetcher.performSearchById(query))); fetcherName = Localization.lang("Identifier-based Web Search"); + } else { + // Execptions are handled below at "task.onFailure(dialogService::showErrorDialogAndWait)" + parserResultCallable = () -> new ParserResult(activeFetcher.performSearch(query)); } BackgroundTask task = BackgroundTask.wrap(parserResultCallable) From 1ab6c2a98aa844628e07486dedecbe718f9a3c1a Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Sat, 10 Aug 2024 17:06:47 +0200 Subject: [PATCH 24/42] Add TODO --- src/main/java/org/jabref/gui/DialogService.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/main/java/org/jabref/gui/DialogService.java b/src/main/java/org/jabref/gui/DialogService.java index 3865f5d000e..30c26110e84 100644 --- a/src/main/java/org/jabref/gui/DialogService.java +++ b/src/main/java/org/jabref/gui/DialogService.java @@ -20,6 +20,7 @@ import org.jabref.gui.util.BaseDialog; import org.jabref.gui.util.DirectoryDialogConfiguration; import org.jabref.gui.util.FileDialogConfiguration; +import org.jabref.logic.importer.FetcherException; import org.jabref.logic.l10n.Localization; import org.controlsfx.control.textfield.CustomPasswordField; @@ -101,6 +102,11 @@ default void showErrorDialogAndWait(Exception exception) { showErrorDialogAndWait(Localization.lang("Unhandled exception occurred."), exception); } + default void showErrorDialogAndWait(FetcherException exception) { + // TODO: Move org.jabref.gui.linkedfile.DownloadLinkedFileAction.onFailure to here + // Include urlDownload.getSource() in FetcherException + } + /** * Create and display error dialog displaying the given exception. * From 8ee95cea097b813ff7f86b8cdea6c075b878cbcf Mon Sep 17 00:00:00 2001 From: rehan Date: Sun, 11 Aug 2024 20:56:07 +0530 Subject: [PATCH 25/42] feat: Added toString to SimpleHttpResponse --- .../java/org/jabref/http/dto/SimpleHttpResponse.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/main/java/org/jabref/http/dto/SimpleHttpResponse.java b/src/main/java/org/jabref/http/dto/SimpleHttpResponse.java index fd50d481789..634f6fe1a31 100644 --- a/src/main/java/org/jabref/http/dto/SimpleHttpResponse.java +++ b/src/main/java/org/jabref/http/dto/SimpleHttpResponse.java @@ -20,6 +20,15 @@ public SimpleHttpResponse(HttpURLConnection connection) throws IOException { this(connection.getResponseCode(), connection.getResponseMessage(), getResponseBody(connection)); } + @Override + public String toString() { + return "SimpleHttpResponse{" + + "statusCode=" + statusCode + + ", responseMessage='" + responseMessage + '\'' + + ", responseBody='" + responseBody + '\'' + + '}'; + } + /** * Truncates the response body to 1 KB if it exceeds that size. * Appends "... (truncated)" to indicate truncation. From 2ba9a6e9b7869b375e577df120c9bd28ec18e849 Mon Sep 17 00:00:00 2001 From: rehan Date: Sun, 11 Aug 2024 20:57:35 +0530 Subject: [PATCH 26/42] refactor: update logger to display SimpleHttpResponse --- src/main/java/org/jabref/logic/net/URLDownload.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/jabref/logic/net/URLDownload.java b/src/main/java/org/jabref/logic/net/URLDownload.java index 7c6e80d3f7e..4720ebd7b1c 100644 --- a/src/main/java/org/jabref/logic/net/URLDownload.java +++ b/src/main/java/org/jabref/logic/net/URLDownload.java @@ -403,7 +403,7 @@ public URLConnection openConnection() throws IOException { connection = new URLDownload(newUrl).openConnection(); } else { SimpleHttpResponse httpResponse = new SimpleHttpResponse(httpURLConnection); - LOGGER.info("HTTP {} {} - body: {}", status, httpURLConnection.getResponseMessage(), httpURLConnection.getContentLength() > 0 ? httpURLConnection.getContent() : ""); + LOGGER.info("{}", httpResponse); if ((status >= 400) && (status < 500)) { throw new IOException(new FetcherClientException(httpResponse)); } else if (status >= 500) { From 84bc323cb7a7c1ea9b81e6a32bf65423304f825a Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Mon, 12 Aug 2024 01:24:30 +0200 Subject: [PATCH 27/42] Make FetcherException URL-aware - org.jabref.logic.importer.fetcher.AbstractIsbnFetcher#ensureThatIsbnIsValid throws URISyntaxException - Cleanup Fetcher(Client|Server)Exception - Move REDACTED nearer to output - Remove "FetcherException" from getUrlForQuery interface Signed-off-by: Oliver Kopp --- .../java/org/jabref/gui/DialogService.java | 36 ++++++- .../org/jabref/gui/JabRefDialogService.java | 1 + .../gui/entryeditor/SciteTabViewModel.java | 11 ++- .../SemanticScholarFetcher.java | 19 +++- .../fetcher/WebSearchPaneViewModel.java | 2 +- .../linkedfile/DownloadLinkedFileAction.java | 29 +----- .../jabref/http/dto/SimpleHttpResponse.java | 40 +++++++- .../importer/EntryBasedParserFetcher.java | 14 ++- .../importer/FetcherClientException.java | 19 +--- .../logic/importer/FetcherException.java | 85 +++++++++++++++- .../importer/FetcherServerException.java | 19 +--- .../logic/importer/IdBasedParserFetcher.java | 23 ++--- .../logic/importer/IdParserFetcher.java | 14 ++- .../PagedSearchBasedParserFetcher.java | 18 +--- .../importer/SearchBasedParserFetcher.java | 16 +-- .../importer/fetcher/ACMPortalFetcher.java | 3 +- .../importer/fetcher/AbstractIsbnFetcher.java | 11 ++- .../logic/importer/fetcher/ArXivFetcher.java | 38 ++++--- .../fetcher/AstrophysicsDataSystem.java | 72 +++++++------- .../importer/fetcher/BiodiversityLibrary.java | 3 +- .../logic/importer/fetcher/CiteSeer.java | 4 +- ...fComputerScienceBibliographiesFetcher.java | 3 +- .../logic/importer/fetcher/CrossRef.java | 4 +- .../logic/importer/fetcher/DBLPFetcher.java | 3 +- .../logic/importer/fetcher/DOABFetcher.java | 3 +- .../logic/importer/fetcher/DOAJFetcher.java | 3 +- .../jabref/logic/importer/fetcher/DiVA.java | 3 +- .../logic/importer/fetcher/DoiFetcher.java | 99 ++++++++++--------- .../logic/importer/fetcher/GoogleScholar.java | 75 ++++++++------ .../fetcher/GrobidCitationFetcher.java | 11 +-- .../logic/importer/fetcher/GvkFetcher.java | 5 +- .../jabref/logic/importer/fetcher/IEEE.java | 3 +- .../importer/fetcher/INSPIREFetcher.java | 27 ++--- .../importer/fetcher/ISIDOREFetcher.java | 9 +- .../importer/fetcher/IacrEprintFetcher.java | 4 +- .../logic/importer/fetcher/JstorFetcher.java | 19 ++-- .../logic/importer/fetcher/LOBIDFetcher.java | 7 +- .../importer/fetcher/LibraryOfCongress.java | 3 +- .../logic/importer/fetcher/MathSciNet.java | 4 +- .../importer/fetcher/MedlineFetcher.java | 2 +- .../jabref/logic/importer/fetcher/Medra.java | 3 +- .../logic/importer/fetcher/MrDLibFetcher.java | 56 +++++------ .../logic/importer/fetcher/ResearchGate.java | 39 +++++--- .../logic/importer/fetcher/RfcFetcher.java | 3 +- .../fetcher/ScholarArchiveFetcher.java | 3 +- .../importer/fetcher/SemanticScholar.java | 7 +- .../importer/fetcher/SpringerFetcher.java | 4 +- .../jabref/logic/importer/fetcher/ZbMATH.java | 4 +- .../DoiToBibtexConverterComIsbnFetcher.java | 3 +- .../isbntobibtex/EbookDeIsbnFetcher.java | 3 +- .../isbntobibtex/OpenLibraryIsbnFetcher.java | 3 +- .../importer/fileformat/ACMPortalParser.java | 18 ++-- .../org/jabref/logic/net/URLDownload.java | 31 ++++-- src/main/resources/l10n/JabRef_en.properties | 13 +-- 54 files changed, 548 insertions(+), 406 deletions(-) diff --git a/src/main/java/org/jabref/gui/DialogService.java b/src/main/java/org/jabref/gui/DialogService.java index 30c26110e84..309f227817a 100644 --- a/src/main/java/org/jabref/gui/DialogService.java +++ b/src/main/java/org/jabref/gui/DialogService.java @@ -20,7 +20,10 @@ import org.jabref.gui.util.BaseDialog; import org.jabref.gui.util.DirectoryDialogConfiguration; import org.jabref.gui.util.FileDialogConfiguration; +import org.jabref.http.dto.SimpleHttpResponse; +import org.jabref.logic.importer.FetcherClientException; import org.jabref.logic.importer.FetcherException; +import org.jabref.logic.importer.FetcherServerException; import org.jabref.logic.l10n.Localization; import org.controlsfx.control.textfield.CustomPasswordField; @@ -99,12 +102,37 @@ default Optional showEditableChoiceDialogAndWait(String title, String con * @param exception the exception causing the error */ default void showErrorDialogAndWait(Exception exception) { - showErrorDialogAndWait(Localization.lang("Unhandled exception occurred."), exception); + if (exception instanceof FetcherException) { + // Somehow, Java does not route correctly to the other method + showErrorDialogAndWait((FetcherException) exception); + } else { + showErrorDialogAndWait(Localization.lang("Unhandled exception occurred."), exception); + } } - default void showErrorDialogAndWait(FetcherException exception) { - // TODO: Move org.jabref.gui.linkedfile.DownloadLinkedFileAction.onFailure to here - // Include urlDownload.getSource() in FetcherException + default void showErrorDialogAndWait(FetcherException fetcherException) { + String failedTitle = Localization.lang("Failed to download from URL"); + String localizedMessage = fetcherException.getLocalizedMessage(); + Optional httpResponse = fetcherException.getHttpResponse(); + if (httpResponse.isPresent()) { + int statusCode = httpResponse.get().statusCode(); + if (statusCode == 401) { + this.showInformationDialogAndWait(failedTitle, Localization.lang("401 Unauthorized: Access Denied. You are not authorized to access this resource. Please check your credentials and try again. If you believe you should have access, please contact the administrator for assistance.") + "\n\n" + localizedMessage); + } else if (statusCode == 403) { + this.showInformationDialogAndWait(failedTitle, Localization.lang("403 Forbidden: Access Denied. You do not have permission to access this resource. Please contact the administrator for assistance or try a different action.") + "\n\n" + localizedMessage); + } else if (statusCode == 404) { + this.showInformationDialogAndWait(failedTitle, Localization.lang("404 Not Found Error: The requested resource could not be found. It seems that the file you are trying to download is not available or has been moved. Please verify the URL and try again. If you believe this is an error, please contact the administrator for further assistance.") + "\n\n" + localizedMessage); + } else { + this.showErrorDialogAndWait(failedTitle, Localization.lang("Something is wrong on JabRef side. Please check the URL and try again.") + "\n\n" + localizedMessage); + } + } else if (fetcherException instanceof FetcherClientException) { + this.showErrorDialogAndWait(failedTitle, Localization.lang("Something is wrong on JabRef side. Please check the URL and try again.") + "\n\n" + localizedMessage); + } else if (fetcherException instanceof FetcherServerException) { + this.showInformationDialogAndWait(failedTitle, + Localization.lang("Error downloading from URL. Cause is likely the server side.\nPlease try again later or contact the server administrator.") + "\n\n" + localizedMessage); + } else { + this.showErrorDialogAndWait(failedTitle, localizedMessage); + } } /** diff --git a/src/main/java/org/jabref/gui/JabRefDialogService.java b/src/main/java/org/jabref/gui/JabRefDialogService.java index 0ec2acdddd5..1e572069602 100644 --- a/src/main/java/org/jabref/gui/JabRefDialogService.java +++ b/src/main/java/org/jabref/gui/JabRefDialogService.java @@ -87,6 +87,7 @@ private FXDialog createDialog(AlertType type, String title, String content) { alert.setResizable(true); TextArea area = new TextArea(content); + area.setWrapText(true); alert.getDialogPane().setContent(area); alert.initOwner(mainWindow); diff --git a/src/main/java/org/jabref/gui/entryeditor/SciteTabViewModel.java b/src/main/java/org/jabref/gui/entryeditor/SciteTabViewModel.java index 2ac46e5cc8a..43aa1b41e57 100644 --- a/src/main/java/org/jabref/gui/entryeditor/SciteTabViewModel.java +++ b/src/main/java/org/jabref/gui/entryeditor/SciteTabViewModel.java @@ -96,8 +96,13 @@ private void cancelSearch() { } public SciteTallyModel fetchTallies(DOI doi) throws FetcherException { + URL url; + try { + url = new URI(BASE_URL + "tallies/" + doi.getDOI()).toURL(); + } catch (MalformedURLException | URISyntaxException ex) { + throw new FetcherException("Malformed URL for DOI", ex); + } try { - URL url = new URI(BASE_URL + "tallies/" + doi.getDOI()).toURL(); LOGGER.debug("Fetching tallies from {}", url); URLDownload download = new URLDownload(url); String response = download.asString(); @@ -110,10 +115,8 @@ public SciteTallyModel fetchTallies(DOI doi) throws FetcherException { throw new FetcherException("Unexpected result data."); } return SciteTallyModel.fromJSONObject(tallies); - } catch (MalformedURLException | URISyntaxException ex) { - throw new FetcherException("Malformed URL for DOI", ex); } catch (IOException ioex) { - throw new FetcherException("Failed to retrieve tallies for DOI - I/O Exception", ioex); + throw new FetcherException(url, "Failed to retrieve tallies for DOI", ioex); } } diff --git a/src/main/java/org/jabref/gui/entryeditor/citationrelationtab/semanticscholar/SemanticScholarFetcher.java b/src/main/java/org/jabref/gui/entryeditor/citationrelationtab/semanticscholar/SemanticScholarFetcher.java index 1463d22aa9c..639b045f09c 100644 --- a/src/main/java/org/jabref/gui/entryeditor/citationrelationtab/semanticscholar/SemanticScholarFetcher.java +++ b/src/main/java/org/jabref/gui/entryeditor/citationrelationtab/semanticscholar/SemanticScholarFetcher.java @@ -1,6 +1,7 @@ package org.jabref.gui.entryeditor.citationrelationtab.semanticscholar; import java.io.IOException; +import java.net.MalformedURLException; import java.net.URI; import java.net.URL; import java.util.List; @@ -34,8 +35,13 @@ public String getAPIUrl(String entry_point, BibEntry entry) { @Override public List searchCitedBy(BibEntry entry) throws FetcherException { if (entry.getDOI().isPresent()) { + URL citationsUrl; + try { + citationsUrl = URI.create(getAPIUrl("citations", entry)).toURL(); + } catch (MalformedURLException e) { + throw new FetcherException("Malformed URL", e); + } try { - URL citationsUrl = URI.create(getAPIUrl("citations", entry)).toURL(); URLDownload urlDownload = new URLDownload(citationsUrl); String apiKey = getApiKey(); @@ -49,7 +55,7 @@ public List searchCitedBy(BibEntry entry) throws FetcherException { .stream().filter(citationDataItem -> citationDataItem.getCitingPaper() != null) .map(citationDataItem -> citationDataItem.getCitingPaper().toBibEntry()).toList(); } catch (IOException e) { - throw new FetcherException("Could not fetch", e); + throw new FetcherException(citationsUrl, "Could not fetch", e); } } return List.of(); @@ -58,8 +64,13 @@ public List searchCitedBy(BibEntry entry) throws FetcherException { @Override public List searchCiting(BibEntry entry) throws FetcherException { if (entry.getDOI().isPresent()) { + URL referencesUrl; + try { + referencesUrl = URI.create(getAPIUrl("references", entry)).toURL(); + } catch (MalformedURLException e) { + throw new FetcherException("Malformed URL", e); + } try { - URL referencesUrl = URI.create(getAPIUrl("references", entry)).toURL(); URLDownload urlDownload = new URLDownload(referencesUrl); String apiKey = getApiKey(); if (!apiKey.isEmpty()) { @@ -73,7 +84,7 @@ public List searchCiting(BibEntry entry) throws FetcherException { .filter(citationDataItem -> citationDataItem.getCitedPaper() != null) .map(referenceDataItem -> referenceDataItem.getCitedPaper().toBibEntry()).toList(); } catch (IOException e) { - throw new FetcherException("Could not fetch", e); + throw new FetcherException(referencesUrl, "Could not fetch", e); } } return List.of(); diff --git a/src/main/java/org/jabref/gui/importer/fetcher/WebSearchPaneViewModel.java b/src/main/java/org/jabref/gui/importer/fetcher/WebSearchPaneViewModel.java index f83ab6dee36..e532df7a635 100644 --- a/src/main/java/org/jabref/gui/importer/fetcher/WebSearchPaneViewModel.java +++ b/src/main/java/org/jabref/gui/importer/fetcher/WebSearchPaneViewModel.java @@ -154,7 +154,7 @@ public void search() { parserResultCallable = () -> new ParserResult(OptionalUtil.toList(compositeIdFetcher.performSearchById(query))); fetcherName = Localization.lang("Identifier-based Web Search"); } else { - // Execptions are handled below at "task.onFailure(dialogService::showErrorDialogAndWait)" + // Exceptions are handled below at "task.onFailure(dialogService::showErrorDialogAndWait)" parserResultCallable = () -> new ParserResult(activeFetcher.performSearch(query)); } diff --git a/src/main/java/org/jabref/gui/linkedfile/DownloadLinkedFileAction.java b/src/main/java/org/jabref/gui/linkedfile/DownloadLinkedFileAction.java index bfcae690366..54695f9577b 100644 --- a/src/main/java/org/jabref/gui/linkedfile/DownloadLinkedFileAction.java +++ b/src/main/java/org/jabref/gui/linkedfile/DownloadLinkedFileAction.java @@ -29,10 +29,8 @@ import org.jabref.gui.fieldeditors.URLUtil; import org.jabref.gui.util.BackgroundTask; import org.jabref.gui.util.TaskExecutor; -import org.jabref.http.dto.SimpleHttpResponse; import org.jabref.logic.externalfiles.LinkedFileHandler; -import org.jabref.logic.importer.FetcherClientException; -import org.jabref.logic.importer.FetcherServerException; +import org.jabref.logic.importer.FetcherException; import org.jabref.logic.l10n.Localization; import org.jabref.logic.net.ProgressInputStream; import org.jabref.logic.net.URLDownload; @@ -200,28 +198,11 @@ private void onSuccess(Path targetDirectory, Path downloadedFile) { private void onFailure(URLDownload urlDownload, Exception ex) { LOGGER.error("Error downloading from URL: {}", urlDownload, ex); - String fetcherExceptionMessage = ex.getLocalizedMessage(); - String failedTitle = Localization.lang("Failed to download from URL"); - if (ex instanceof FetcherClientException fetcherException) { - Optional httpResponse = fetcherException.getHttpResponse(); - if (httpResponse.isPresent()) { - int statusCode = httpResponse.get().statusCode(); - if (statusCode == 401) { - dialogService.showInformationDialogAndWait(failedTitle, Localization.lang("401 Unauthorized: Access Denied. You are not authorized to access this resource. Please check your credentials and try again. If you believe you should have access, please contact the administrator for assistance.\nURL: %0\nDetails: %1", urlDownload.getSource(), fetcherExceptionMessage)); - } else if (statusCode == 403) { - dialogService.showInformationDialogAndWait(failedTitle, Localization.lang("403 Forbidden: Access Denied. You do not have permission to access this resource. Please contact the administrator for assistance or try a different action.\nURL: %0\nDetails: %1", urlDownload.getSource(), fetcherExceptionMessage)); - } else if (statusCode == 404) { - dialogService.showInformationDialogAndWait(failedTitle, Localization.lang("404 Not Found Error: The requested resource could not be found. It seems that the file you are trying to download is not available or has been moved. Please verify the URL and try again. If you believe this is an error, please contact the administrator for further assistance.\nURL: %0\nDetails: %1", urlDownload.getSource(), fetcherExceptionMessage)); - } else { - dialogService.showErrorDialogAndWait(failedTitle, Localization.lang("Something is wrong on JabRef side. Please check the URL and try again.\nURL: %0\nDetails: %1", urlDownload.getSource(), fetcherExceptionMessage)); - } - } else { - dialogService.showErrorDialogAndWait(failedTitle, Localization.lang("Something is wrong on JabRef side. Please check the URL and try again.\nURL: %0\nDetails: %1", urlDownload.getSource(), fetcherExceptionMessage)); - } - } else if (ex instanceof FetcherServerException) { - dialogService.showInformationDialogAndWait(failedTitle, - Localization.lang("Error downloading from URL. Cause is likely the server side.\nPlease try again later or contact the server administrator.\nURL: %0\nDetails: %1", urlDownload.getSource(), fetcherExceptionMessage)); + if (ex instanceof FetcherException fetcherException) { + dialogService.showErrorDialogAndWait(fetcherException); } else { + String fetcherExceptionMessage = ex.getLocalizedMessage(); + String failedTitle = Localization.lang("Failed to download from URL"); dialogService.showErrorDialogAndWait(failedTitle, Localization.lang("Please check the URL and try again.\nURL: %0\nDetails: %1", urlDownload.getSource(), fetcherExceptionMessage)); } } diff --git a/src/main/java/org/jabref/http/dto/SimpleHttpResponse.java b/src/main/java/org/jabref/http/dto/SimpleHttpResponse.java index 634f6fe1a31..a07a9b304d9 100644 --- a/src/main/java/org/jabref/http/dto/SimpleHttpResponse.java +++ b/src/main/java/org/jabref/http/dto/SimpleHttpResponse.java @@ -7,6 +7,8 @@ import java.net.HttpURLConnection; import java.nio.charset.StandardCharsets; +import org.jsoup.HttpStatusException; + public record SimpleHttpResponse(int statusCode, String responseMessage, String responseBody) { private static final int MAX_RESPONSE_LENGTH = 1024; // 1 KB @@ -16,8 +18,12 @@ public SimpleHttpResponse(int statusCode, String responseMessage, String respons this.responseMessage = responseMessage; } - public SimpleHttpResponse(HttpURLConnection connection) throws IOException { - this(connection.getResponseCode(), connection.getResponseMessage(), getResponseBody(connection)); + public SimpleHttpResponse(HttpURLConnection connection) { + this(getStatusCode(connection), getResponseMessage(connection), getResponseBodyDefaultEmpty(connection)); + } + + public SimpleHttpResponse(HttpStatusException e) { + this(e.getStatusCode(), "", ""); } @Override @@ -29,6 +35,36 @@ public String toString() { '}'; } + private static int getStatusCode(HttpURLConnection connection) { + int statusCode; + try { + statusCode = connection.getResponseCode(); + } catch (IOException e) { + statusCode = -1; + } + return statusCode; + } + + private static String getResponseMessage(HttpURLConnection connection) { + String responseMessage; + try { + responseMessage = connection.getResponseMessage(); + } catch (IOException e) { + responseMessage = ""; + } + return responseMessage; + } + + private static String getResponseBodyDefaultEmpty(HttpURLConnection connection) { + String responseBody; + try { + responseBody = getResponseBody(connection); + } catch (IOException e) { + responseBody = ""; + } + return responseBody; + } + /** * Truncates the response body to 1 KB if it exceeds that size. * Appends "... (truncated)" to indicate truncation. diff --git a/src/main/java/org/jabref/logic/importer/EntryBasedParserFetcher.java b/src/main/java/org/jabref/logic/importer/EntryBasedParserFetcher.java index 28795882d36..38a010c62ce 100644 --- a/src/main/java/org/jabref/logic/importer/EntryBasedParserFetcher.java +++ b/src/main/java/org/jabref/logic/importer/EntryBasedParserFetcher.java @@ -12,6 +12,8 @@ import org.jabref.model.entry.BibEntry; +import org.slf4j.LoggerFactory; + /** * Provides a convenient interface for entry-based fetcher, which follow the usual three-step procedure: * 1. Open a URL based on the entry @@ -36,16 +38,16 @@ public interface EntryBasedParserFetcher extends EntryBasedFetcher, ParserFetche default List performSearch(BibEntry entry) throws FetcherException { Objects.requireNonNull(entry); - URL UrlForEntry; + URL urlForEntry; try { - if ((UrlForEntry = getURLForEntry(entry)) == null) { + if ((urlForEntry = getURLForEntry(entry)) == null) { return Collections.emptyList(); } } catch (MalformedURLException | URISyntaxException e) { throw new FetcherException("Search URI is malformed", e); } - try (InputStream stream = new BufferedInputStream(UrlForEntry.openStream())) { + try (InputStream stream = new BufferedInputStream(urlForEntry.openStream())) { List fetchedEntries = getParser().parseEntries(stream); // Post-cleanup @@ -54,9 +56,11 @@ default List performSearch(BibEntry entry) throws FetcherException { return fetchedEntries; } catch (IOException e) { // TODO: Catch HTTP Response 401 errors and report that user has no rights to access resource - throw new FetcherException("A network error occurred", e); + // Same TODO as in org.jabref.logic.net.URLDownload.openConnection. Code should be reused. + LoggerFactory.getLogger(EntryBasedParserFetcher.class).error("Could not fetch from URL {}", urlForEntry, e); + throw new FetcherException(urlForEntry, "A network error occurred", e); } catch (ParseException e) { - throw new FetcherException("An internal parser error occurred", e); + throw new FetcherException(urlForEntry, "An internal parser error occurred", e); } } } diff --git a/src/main/java/org/jabref/logic/importer/FetcherClientException.java b/src/main/java/org/jabref/logic/importer/FetcherClientException.java index 7dbd0b4b3df..a5780cc08ff 100644 --- a/src/main/java/org/jabref/logic/importer/FetcherClientException.java +++ b/src/main/java/org/jabref/logic/importer/FetcherClientException.java @@ -1,25 +1,14 @@ package org.jabref.logic.importer; +import java.net.URL; + import org.jabref.http.dto.SimpleHttpResponse; /** * Should be thrown when you encounter an HTTP status code error >= 400 and < 500. */ public class FetcherClientException extends FetcherException { - - public FetcherClientException(String errorMessage) { - super(errorMessage); - } - - public FetcherClientException(String errorMessage, Throwable cause) { - super(errorMessage, cause); - } - - public FetcherClientException(String errorMessage, String localizedMessage, Throwable cause) { - super(errorMessage, localizedMessage, cause); - } - - public FetcherClientException(SimpleHttpResponse simpleHttpResponse) { - super(simpleHttpResponse); + public FetcherClientException(URL url, SimpleHttpResponse httpResponse) { + super(url, httpResponse); } } diff --git a/src/main/java/org/jabref/logic/importer/FetcherException.java b/src/main/java/org/jabref/logic/importer/FetcherException.java index fdbe9d60d72..e330c22a0e1 100644 --- a/src/main/java/org/jabref/logic/importer/FetcherException.java +++ b/src/main/java/org/jabref/logic/importer/FetcherException.java @@ -1,45 +1,124 @@ package org.jabref.logic.importer; +import java.net.URL; import java.util.Optional; +import java.util.regex.Pattern; import org.jabref.http.dto.SimpleHttpResponse; import org.jabref.logic.JabRefException; +import org.jabref.logic.l10n.Localization; public class FetcherException extends JabRefException { + Pattern API_KEY_PATTERN = Pattern.compile("(?i)(api|key|api[-_]?key)=[^&]*"); + + String REDACTED_STRING = "[REDACTED]"; + + private final String url; private final SimpleHttpResponse httpResponse; - public FetcherException(SimpleHttpResponse httpResponse) { - super(httpResponse.responseMessage()); + public FetcherException(String url, SimpleHttpResponse httpResponse) { + super("URL: %s\nHTTP %s %s\n%s".formatted(url, httpResponse.statusCode(), httpResponse.responseMessage(), httpResponse.responseBody())); + this.url = url; this.httpResponse = httpResponse; } + public FetcherException(URL url, SimpleHttpResponse httpResponse) { + this(url.toString(), httpResponse); + } + + public FetcherException(URL url, Throwable cause) { + super(cause); + httpResponse = null; + this.url = url.toString(); + } + + public FetcherException(URL url, String errorMessage, Throwable cause) { + super(errorMessage, cause); + httpResponse = null; + this.url = url.toString(); + } + + /** + * @param dummy Required to distinguish of {@link FetcherException#FetcherException(String, Throwable)}, which is for a FetcherException with an error message + */ + private FetcherException(String url, Throwable cause, Object dummy) { + super(cause); + httpResponse = null; + this.url = url; + } + + /** + * @param dummy Required to distinguish of {@link FetcherException#FetcherException(String, String, Throwable)}, which is for a localized FetcherException + */ + private FetcherException(String url, String errorMessage, Throwable cause, Object dummy) { + super(errorMessage, cause); + httpResponse = null; + this.url = url; + } + public FetcherException(String errorMessage, Throwable cause) { super(errorMessage, cause); httpResponse = null; + url = null; } public FetcherException(String errorMessage) { super(errorMessage); httpResponse = null; + url = null; } public FetcherException(String errorMessage, String localizedMessage, Throwable cause) { super(errorMessage, localizedMessage, cause); httpResponse = null; + url = null; + } + + public static FetcherException ofUrl(String url, Throwable cause) { + return new FetcherException(url, cause, null); + } + + public static FetcherException ofUrl(String url, String errorMessage) { + return new FetcherException(url, errorMessage, null, null); + } + + public static FetcherException ofUrl(String url, String errorMessage, Throwable cause) { + return new FetcherException(url, errorMessage, cause, null); } @Override public String getLocalizedMessage() { // TODO: This should be moved to a separate class converting "any" exception object to a localized message + // TODO: 5% of the "new-ers" pass a "real" localized message. See org.jabref.logic.importer.fetcher.GoogleScholar.addHitsFromQuery. We should maybe make use of this (and rewrite the whole message handling) + // TODO: Try to convert IOException to some more meaningful information here (or at org.jabref.gui.DialogService.showErrorDialogAndWait(org.jabref.logic.importer.FetcherException)). See also org.jabref.logic.net.URLDownload.openConnection if (httpResponse != null) { - return "Encountered HTTP %s %s (%s)".formatted(httpResponse.statusCode(), httpResponse.responseMessage(), httpResponse.responseBody()); + return getPrefix() + Localization.lang("URL: %0\nHTTP %1 %2\n%3", getRedactedUrl(), httpResponse.statusCode(), httpResponse.responseMessage(), httpResponse.responseBody()); + } else if (url != null) { + return getPrefix() + Localization.lang("URL: %0", getRedactedUrl()); } else { return super.getLocalizedMessage(); } } + private String getRedactedUrl() { + return API_KEY_PATTERN.matcher(url).replaceAll(REDACTED_STRING); + } + + private String getPrefix() { + String superLocalizedMessage = super.getLocalizedMessage(); + if (superLocalizedMessage != null) { + return superLocalizedMessage + "\n"; + } else { + return ""; + } + } + public Optional getHttpResponse() { return Optional.ofNullable(httpResponse); } + + public Optional getUrl() { + return Optional.ofNullable(url); + } } diff --git a/src/main/java/org/jabref/logic/importer/FetcherServerException.java b/src/main/java/org/jabref/logic/importer/FetcherServerException.java index ee769aa6105..ed67992b65d 100644 --- a/src/main/java/org/jabref/logic/importer/FetcherServerException.java +++ b/src/main/java/org/jabref/logic/importer/FetcherServerException.java @@ -1,25 +1,14 @@ package org.jabref.logic.importer; +import java.net.URL; + import org.jabref.http.dto.SimpleHttpResponse; /** * Should be thrown when you encounter a http status code error >= 500 */ public class FetcherServerException extends FetcherException { - - public FetcherServerException(String errorMessage) { - super(errorMessage); - } - - public FetcherServerException(String errorMessage, Throwable cause) { - super(errorMessage, cause); - } - - public FetcherServerException(String errorMessage, String localizedMessage, Throwable cause) { - super(errorMessage, localizedMessage, cause); - } - - public FetcherServerException(SimpleHttpResponse httpResponse) { - super(httpResponse); + public FetcherServerException(URL source, SimpleHttpResponse httpResponse) { + super(source, httpResponse); } } diff --git a/src/main/java/org/jabref/logic/importer/IdBasedParserFetcher.java b/src/main/java/org/jabref/logic/importer/IdBasedParserFetcher.java index a938f8ef6f5..55a7e1e615a 100644 --- a/src/main/java/org/jabref/logic/importer/IdBasedParserFetcher.java +++ b/src/main/java/org/jabref/logic/importer/IdBasedParserFetcher.java @@ -29,7 +29,7 @@ public interface IdBasedParserFetcher extends IdBasedFetcher, ParserFetcher { * * @param identifier the ID */ - URL getUrlForIdentifier(String identifier) throws URISyntaxException, MalformedURLException, FetcherException; + URL getUrlForIdentifier(String identifier) throws URISyntaxException, MalformedURLException; /** * Returns the parser used to convert the response to a list of {@link BibEntry}. @@ -41,34 +41,31 @@ default Optional performSearchById(String identifier) throws FetcherEx if (StringUtil.isBlank(identifier)) { return Optional.empty(); } - - try (InputStream stream = getUrlDownload(getUrlForIdentifier(identifier)).asInputStream()) { + URL urlForIdentifier; + try { + urlForIdentifier = getUrlForIdentifier(identifier); + } catch (URISyntaxException | MalformedURLException e) { + throw new FetcherException("Search URI is malformed", e); + } + try (InputStream stream = getUrlDownload(urlForIdentifier).asInputStream()) { List fetchedEntries = getParser().parseEntries(stream); - if (fetchedEntries.isEmpty()) { return Optional.empty(); } - if (fetchedEntries.size() > 1) { LOGGER.info("Fetcher {} found more than one result for identifier {}. We will use the first entry.", getName(), identifier); } - BibEntry entry = fetchedEntries.getFirst(); - - // Post-cleanup doPostCleanup(entry); - return Optional.of(entry); - } catch (URISyntaxException e) { - throw new FetcherException("Search URI is malformed", e); } catch (IOException e) { // check for the case where we already have a FetcherException from UrlDownload if (e.getCause() instanceof FetcherException fe) { throw fe; } - throw new FetcherException("A network error occurred", e); + throw new FetcherException(urlForIdentifier, "A network error occurred", e); } catch (ParseException e) { - throw new FetcherException("An internal parser error occurred", e); + throw new FetcherException(urlForIdentifier, "An internal parser error occurred", e); } } } diff --git a/src/main/java/org/jabref/logic/importer/IdParserFetcher.java b/src/main/java/org/jabref/logic/importer/IdParserFetcher.java index 5907d81d1a3..fe6245455eb 100644 --- a/src/main/java/org/jabref/logic/importer/IdParserFetcher.java +++ b/src/main/java/org/jabref/logic/importer/IdParserFetcher.java @@ -52,7 +52,13 @@ public interface IdParserFetcher extends IdFetcher, Par default Optional findIdentifier(BibEntry entry) throws FetcherException { Objects.requireNonNull(entry); - try (InputStream stream = new BufferedInputStream(getURLForEntry(entry).openStream())) { + URL urlForEntry; + try { + urlForEntry = getURLForEntry(entry); + } catch (URISyntaxException | MalformedURLException e) { + throw new FetcherException("Search URL is malformed", e); + } + try (InputStream stream = new BufferedInputStream(urlForEntry.openStream())) { List fetchedEntries = getParser().parseEntries(stream); if (fetchedEntries.isEmpty()) { @@ -63,8 +69,6 @@ default Optional findIdentifier(BibEntry entry) throws FetcherException { fetchedEntries.forEach(this::doPostCleanup); return extractIdentifier(entry, fetchedEntries); - } catch (URISyntaxException e) { - throw new FetcherException("Search URI is malformed", e); } catch (FileNotFoundException e) { LOGGER.debug("Id not found"); return Optional.empty(); @@ -73,9 +77,9 @@ default Optional findIdentifier(BibEntry entry) throws FetcherException { if (e.getCause() instanceof FetcherException fe) { throw fe; } - throw new FetcherException("An I/O exception occurred", e); + throw new FetcherException(urlForEntry, "An I/O exception occurred", e); } catch (ParseException e) { - throw new FetcherException("An internal parser error occurred", e); + throw new FetcherException(urlForEntry, "An internal parser error occurred", e); } } } diff --git a/src/main/java/org/jabref/logic/importer/PagedSearchBasedParserFetcher.java b/src/main/java/org/jabref/logic/importer/PagedSearchBasedParserFetcher.java index 94166ec25c9..db8bae8ddf0 100644 --- a/src/main/java/org/jabref/logic/importer/PagedSearchBasedParserFetcher.java +++ b/src/main/java/org/jabref/logic/importer/PagedSearchBasedParserFetcher.java @@ -6,7 +6,6 @@ import java.net.URISyntaxException; import java.net.URL; import java.util.List; -import java.util.regex.Pattern; import org.jabref.model.entry.BibEntry; import org.jabref.model.paging.Page; @@ -15,15 +14,6 @@ public interface PagedSearchBasedParserFetcher extends SearchBasedParserFetcher, PagedSearchBasedFetcher, ParserFetcher { - /** - * Pattern to redact API keys from error messages. - */ - Pattern API_KEY_PATTERN = Pattern.compile("(?i)(api|key|api[-_]?key)=[^&]*"); - /** - * A constant string used to replace or mask sensitive information , such as API keys; - */ - String REDACTED_STRING = "[REDACTED]"; - @Override default Page performSearchPaged(QueryNode luceneQuery, int pageNumber) throws FetcherException { // ADR-0014 @@ -42,9 +32,9 @@ private List getBibEntries(URL urlForQuery) throws FetcherException { fetchedEntries.forEach(this::doPostCleanup); return fetchedEntries; } catch (IOException e) { - throw new FetcherException(API_KEY_PATTERN.matcher(("A network error occurred while fetching from " + urlForQuery)).replaceAll(REDACTED_STRING), e); + throw new FetcherException(urlForQuery, "A network error occurred", e); } catch (ParseException e) { - throw new FetcherException(API_KEY_PATTERN.matcher(("An internal parser error occurred while fetching from " + urlForQuery)).replaceAll(REDACTED_STRING), e); + throw new FetcherException(urlForQuery, "An internal parser error occurred", e); } } @@ -54,10 +44,10 @@ private List getBibEntries(URL urlForQuery) throws FetcherException { * @param luceneQuery the search query * @param pageNumber the number of the page indexed from 0 */ - URL getURLForQuery(QueryNode luceneQuery, int pageNumber) throws URISyntaxException, MalformedURLException, FetcherException; + URL getURLForQuery(QueryNode luceneQuery, int pageNumber) throws URISyntaxException, MalformedURLException; @Override - default URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException, FetcherException { + default URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException { return getURLForQuery(luceneQuery, 0); } diff --git a/src/main/java/org/jabref/logic/importer/SearchBasedParserFetcher.java b/src/main/java/org/jabref/logic/importer/SearchBasedParserFetcher.java index 5117d1d8d0b..4df75fb981f 100644 --- a/src/main/java/org/jabref/logic/importer/SearchBasedParserFetcher.java +++ b/src/main/java/org/jabref/logic/importer/SearchBasedParserFetcher.java @@ -6,7 +6,6 @@ import java.net.URISyntaxException; import java.net.URL; import java.util.List; -import java.util.regex.Pattern; import org.jabref.model.entry.BibEntry; @@ -36,17 +35,6 @@ public interface SearchBasedParserFetcher extends SearchBasedFetcher, ParserFetcher { - /** - * Pattern to redact API keys from error messages. - */ - Pattern API_KEY_PATTERN = Pattern.compile("(?i)(key|api[-_]?key)=[^&]*"); - /** - * A constant string used to replace or mask sensitive information , such as API keys; - */ - String REDACTED_STRING = "[REDACTED]"; - - - /** * This method is used to send queries with advanced URL parameters. * This method is necessary as the performSearch method does not support certain URL parameters that are used for @@ -73,10 +61,10 @@ private List getBibEntries(URL urlForQuery) throws FetcherException { return fetchedEntries; } catch (IOException e) { // Regular expression to redact API keys from the error message - throw new FetcherException(API_KEY_PATTERN.matcher(("A network error occurred while fetching from " + urlForQuery)).replaceAll(REDACTED_STRING), e); + throw new FetcherException(urlForQuery, e); } catch (ParseException e) { // Regular expression to redact API keys from the error message - throw new FetcherException(API_KEY_PATTERN.matcher(("An internal parser error occurred while fetching from " + urlForQuery)).replaceAll(REDACTED_STRING), e); + throw new FetcherException(urlForQuery, "An internal parser error occurred while fetching", e); } } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/ACMPortalFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/ACMPortalFetcher.java index 2d36d11e9d6..93572f9b041 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/ACMPortalFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/ACMPortalFetcher.java @@ -8,7 +8,6 @@ import java.util.Optional; import org.jabref.logic.help.HelpFile; -import org.jabref.logic.importer.FetcherException; import org.jabref.logic.importer.Parser; import org.jabref.logic.importer.SearchBasedParserFetcher; import org.jabref.logic.importer.fetcher.transformers.DefaultQueryTransformer; @@ -49,7 +48,7 @@ private static String createQueryString(QueryNode query) { * @return query URL */ @Override - public URL getURLForQuery(QueryNode query) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getURLForQuery(QueryNode query) throws URISyntaxException, MalformedURLException { URIBuilder uriBuilder = new URIBuilder(SEARCH_URL); uriBuilder.addParameter("AllField", createQueryString(query)); return uriBuilder.build().toURL(); diff --git a/src/main/java/org/jabref/logic/importer/fetcher/AbstractIsbnFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/AbstractIsbnFetcher.java index efa37343f95..80899dd511b 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/AbstractIsbnFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/AbstractIsbnFetcher.java @@ -1,14 +1,13 @@ package org.jabref.logic.importer.fetcher; +import java.net.URISyntaxException; import java.util.Optional; import org.jabref.logic.help.HelpFile; -import org.jabref.logic.importer.FetcherException; import org.jabref.logic.importer.IdBasedParserFetcher; import org.jabref.logic.importer.ImportFormatPreferences; import org.jabref.logic.importer.Parser; import org.jabref.logic.importer.fileformat.BibtexParser; -import org.jabref.logic.l10n.Localization; import org.jabref.model.entry.identifier.ISBN; public abstract class AbstractIsbnFetcher implements IdBasedParserFetcher { @@ -24,10 +23,14 @@ public Optional getHelpPage() { return Optional.of(HelpFile.FETCHER_ISBN); } - protected void ensureThatIsbnIsValid(String identifier) throws FetcherException { + /** + * @throws URISyntaxException if the ISBN is invalid + * @implNote We could have created a new exception (which causes much implementation efforts) or we could have used "FetcherException", which is currently more used for I/O errors than syntax errors (thus also more WTF). Moreover, a ISBN is "kind of" an URI (even if the isbn: prefix is missing) + */ + protected void ensureThatIsbnIsValid(String identifier) throws URISyntaxException { ISBN isbn = new ISBN(identifier); if (!isbn.isValid()) { - throw new FetcherException(Localization.lang("Invalid ISBN: '%0'.", identifier)); + throw new URISyntaxException(identifier, "Invalid ISBN"); } } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/ArXivFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/ArXivFetcher.java index 1e2fb5000b6..cf487d4ec6b 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/ArXivFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/ArXivFetcher.java @@ -529,20 +529,30 @@ private Document callApi(String searchQuery, List ids, int star throw new IllegalArgumentException("The arXiv API limits the number of maximal results to be 2000"); } + URIBuilder uriBuilder; try { - URIBuilder uriBuilder = new URIBuilder(API_URL); - // The arXiv API has problems with accents, so we remove them (i.e. Fréchet -> Frechet) - if (StringUtil.isNotBlank(searchQuery)) { - uriBuilder.addParameter("search_query", StringUtil.stripAccents(searchQuery)); - } - if (!ids.isEmpty()) { - uriBuilder.addParameter("id_list", - ids.stream().map(ArXivIdentifier::getNormalized).collect(Collectors.joining(","))); - } - uriBuilder.addParameter("start", String.valueOf(start)); - uriBuilder.addParameter("max_results", String.valueOf(maxResults)); - URL url = uriBuilder.build().toURL(); + uriBuilder = new URIBuilder(API_URL); + } catch (URISyntaxException e) { + throw new FetcherException("Invalid URL", e); + } + // The arXiv API has problems with accents, so we remove them (i.e. Fréchet -> Frechet) + if (StringUtil.isNotBlank(searchQuery)) { + uriBuilder.addParameter("search_query", StringUtil.stripAccents(searchQuery)); + } + if (!ids.isEmpty()) { + uriBuilder.addParameter("id_list", + ids.stream().map(ArXivIdentifier::getNormalized).collect(Collectors.joining(","))); + } + uriBuilder.addParameter("start", String.valueOf(start)); + uriBuilder.addParameter("max_results", String.valueOf(maxResults)); + URL url = null; + try { + url = uriBuilder.build().toURL(); + } catch (MalformedURLException | URISyntaxException e) { + throw new FetcherException("Invalid URL", e); + } + try { DocumentBuilder builder = DOCUMENT_BUILDER_FACTORY.newDocumentBuilder(); HttpURLConnection connection = (HttpURLConnection) url.openConnection(); @@ -552,8 +562,8 @@ private Document callApi(String searchQuery, List ids, int star } else { return builder.parse(connection.getInputStream()); } - } catch (SAXException | ParserConfigurationException | IOException | URISyntaxException exception) { - throw new FetcherException("arXiv API request failed", exception); + } catch (SAXException | ParserConfigurationException | IOException exception) { + throw new FetcherException(url, "arXiv API request failed", exception); } } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystem.java b/src/main/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystem.java index 147fd4866e5..21c06541d2a 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystem.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystem.java @@ -88,7 +88,7 @@ public String getName() { * @return URL which points to a search request for given query */ @Override - public URL getURLForQuery(QueryNode luceneQuery, int pageNumber) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getURLForQuery(QueryNode luceneQuery, int pageNumber) throws URISyntaxException, MalformedURLException { URIBuilder builder = new URIBuilder(API_SEARCH_URL); builder.addParameter("q", new DefaultQueryTransformer().transformLuceneQuery(luceneQuery).orElse("")); builder.addParameter("fl", "bibcode"); @@ -129,7 +129,7 @@ public URL getURLForEntry(BibEntry entry) throws URISyntaxException, MalformedUR * @return URL which points to a search URL for given identifier */ @Override - public URL getUrlForIdentifier(String identifier) throws FetcherException, URISyntaxException, MalformedURLException { + public URL getUrlForIdentifier(String identifier) throws URISyntaxException, MalformedURLException { String query = "doi:\"" + identifier + "\" OR " + "bibcode:\"" + identifier + "\""; URIBuilder builder = new URIBuilder(API_SEARCH_URL); builder.addParameter("q", query); @@ -178,14 +178,15 @@ public List performSearch(BibEntry entry) throws FetcherException { return Collections.emptyList(); } + URL urlForEntry = null; try { - List bibcodes = fetchBibcodes(getURLForEntry(entry)); - return performSearchByIds(bibcodes); - } catch (URISyntaxException e) { + urlForEntry = getURLForEntry(entry); + } catch (URISyntaxException | MalformedURLException e) { throw new FetcherException("Search URI is malformed", e); - } catch (IOException e) { - throw new FetcherException("A network error occurred", e); } + + List bibcodes = fetchBibcodes(urlForEntry); + return performSearchByIds(bibcodes); } /** @@ -204,7 +205,7 @@ private List fetchBibcodes(URL url) throws FetcherException { } return bibcodes; } catch (IOException e) { - throw new FetcherException("A network error occurred", e); + throw new FetcherException(url, "A network error occurred", e); } catch (JSONException e) { return Collections.emptyList(); } @@ -216,24 +217,25 @@ public Optional performSearchById(String identifier) throws FetcherExc return Optional.empty(); } + URL urlForIdentifier; try { - List bibcodes = fetchBibcodes(getUrlForIdentifier(identifier)); - List fetchedEntries = performSearchByIds(bibcodes); - - if (fetchedEntries.isEmpty()) { - return Optional.empty(); - } - if (fetchedEntries.size() > 1) { - LOGGER.info("Fetcher " + getName() + "found more than one result for identifier " + identifier - + ". We will use the first entry."); - } - BibEntry entry = fetchedEntries.getFirst(); - return Optional.of(entry); - } catch (URISyntaxException e) { + urlForIdentifier = getUrlForIdentifier(identifier); + } catch (URISyntaxException | MalformedURLException e) { throw new FetcherException("Search URI is malformed", e); - } catch (IOException e) { - throw new FetcherException("A network error occurred", e); } + + List bibcodes = fetchBibcodes(urlForIdentifier); + List fetchedEntries = performSearchByIds(bibcodes); + + if (fetchedEntries.isEmpty()) { + return Optional.empty(); + } + if (fetchedEntries.size() > 1) { + LOGGER.info("Fetcher " + getName() + "found more than one result for identifier " + identifier + + ". We will use the first entry."); + } + BibEntry entry = fetchedEntries.getFirst(); + return Optional.of(entry); } /** @@ -245,9 +247,17 @@ private List performSearchByIds(Collection identifiers) throws if (ids.isEmpty()) { return Collections.emptyList(); } + + URL urLforExport; + try { + urLforExport = getURLforExport(); + } catch (URISyntaxException | MalformedURLException e) { + throw new FetcherException("Search URI is malformed", e); + } + try { String postData = buildPostData(ids); - URLDownload download = new URLDownload(getURLforExport()); + URLDownload download = new URLDownload(urLforExport); download.addHeader("Authorization", "Bearer " + importerPreferences.getApiKey(getName()).orElse(API_KEY)); download.addHeader("ContentType", "application/json"); download.setPostData(postData); @@ -266,12 +276,10 @@ private List performSearchByIds(Collection identifiers) throws } catch (JSONException e) { return Collections.emptyList(); } - } catch (URISyntaxException e) { - throw new FetcherException("Search URI is malformed", e); } catch (IOException e) { - throw new FetcherException("A network error occurred", e); + throw new FetcherException(urLforExport, "A network error occurred", e); } catch (ParseException e) { - throw new FetcherException("An internal parser error occurred", e); + throw new FetcherException(urLforExport, "An internal parser error occurred", e); } } @@ -280,10 +288,8 @@ public List performSearch(QueryNode luceneQuery) throws FetcherExcepti URL urlForQuery; try { urlForQuery = getURLForQuery(luceneQuery); - } catch (URISyntaxException e) { + } catch (URISyntaxException | MalformedURLException e) { throw new FetcherException("Search URI is malformed", e); - } catch (IOException e) { - throw new FetcherException("A network error occurred", e); } List bibCodes = fetchBibcodes(urlForQuery); return performSearchByIds(bibCodes); @@ -294,10 +300,8 @@ public Page performSearchPaged(QueryNode luceneQuery, int pageNumber) URL urlForQuery; try { urlForQuery = getURLForQuery(luceneQuery, pageNumber); - } catch (URISyntaxException e) { + } catch (URISyntaxException | MalformedURLException e) { throw new FetcherException("Search URI is malformed", e); - } catch (IOException e) { - throw new FetcherException("A network error occurred", e); } // This is currently just interpreting the complex query as a default string query List bibCodes = fetchBibcodes(urlForQuery); diff --git a/src/main/java/org/jabref/logic/importer/fetcher/BiodiversityLibrary.java b/src/main/java/org/jabref/logic/importer/fetcher/BiodiversityLibrary.java index 8b2a0fec78d..039d45c3c07 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/BiodiversityLibrary.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/BiodiversityLibrary.java @@ -9,7 +9,6 @@ import java.util.List; import java.util.stream.IntStream; -import org.jabref.logic.importer.FetcherException; import org.jabref.logic.importer.ImporterPreferences; import org.jabref.logic.importer.ParseException; import org.jabref.logic.importer.Parser; @@ -197,7 +196,7 @@ public Parser getParser() { } @Override - public URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException { URIBuilder uriBuilder = new URIBuilder(getBaseURL().toURI()); BiodiversityLibraryTransformer transformer = new BiodiversityLibraryTransformer(); uriBuilder.addParameter("op", "PublicationSearch"); diff --git a/src/main/java/org/jabref/logic/importer/fetcher/CiteSeer.java b/src/main/java/org/jabref/logic/importer/fetcher/CiteSeer.java index a0286d0005e..20e73b6bb1b 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/CiteSeer.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/CiteSeer.java @@ -67,7 +67,7 @@ public List performSearch(QueryNode luceneQuery) throws FetcherExcepti LOGGER.debug("No success"); // TODO: body needs to be added to the exception, but we currently only have JSON available, but the error is most probably simple text (or HTML) SimpleHttpResponse simpleHttpResponse = new SimpleHttpResponse(httpResponse.getStatus(), httpResponse.getStatusText(), ""); - throw new FetcherException(simpleHttpResponse); + throw new FetcherException(API_URL, simpleHttpResponse); } JsonNode requestResponse = httpResponse.getBody(); @@ -84,7 +84,7 @@ public List performSearch(QueryNode luceneQuery) throws FetcherExcepti List fetchedEntries = parser.parseCiteSeerResponse(jsonResponse.orElse(new JSONArray())); return fetchedEntries; } catch (ParseException ex) { - throw new FetcherException("An internal parser error occurred while parsing CiteSeer entries, ", ex); + throw new FetcherException("An internal parser error occurred while parsing CiteSeer entries", ex); } } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/CollectionOfComputerScienceBibliographiesFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/CollectionOfComputerScienceBibliographiesFetcher.java index 797b2bc1860..018a9b4c9c5 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/CollectionOfComputerScienceBibliographiesFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/CollectionOfComputerScienceBibliographiesFetcher.java @@ -10,7 +10,6 @@ import org.jabref.logic.formatter.bibtexfields.RemoveNewlinesFormatter; import org.jabref.logic.formatter.bibtexfields.RemoveRedundantSpacesFormatter; import org.jabref.logic.formatter.bibtexfields.ReplaceTabsBySpaceFormater; -import org.jabref.logic.importer.FetcherException; import org.jabref.logic.importer.ImportFormatPreferences; import org.jabref.logic.importer.Parser; import org.jabref.logic.importer.SearchBasedParserFetcher; @@ -35,7 +34,7 @@ public CollectionOfComputerScienceBibliographiesFetcher(ImportFormatPreferences } @Override - public URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException { return new URIBuilder(BASIC_SEARCH_URL) .addParameter("query", new CollectionOfComputerScienceBibliographiesQueryTransformer().transformLuceneQuery(luceneQuery).orElse("")) .addParameter("sort", "score") diff --git a/src/main/java/org/jabref/logic/importer/fetcher/CrossRef.java b/src/main/java/org/jabref/logic/importer/fetcher/CrossRef.java index 35f85e651e5..7a3e20f19a7 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/CrossRef.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/CrossRef.java @@ -67,14 +67,14 @@ public URL getURLForEntry(BibEntry entry) throws URISyntaxException, MalformedUR } @Override - public URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException { URIBuilder uriBuilder = new URIBuilder(API_URL); uriBuilder.addParameter("query", new DefaultQueryTransformer().transformLuceneQuery(luceneQuery).orElse("")); return uriBuilder.build().toURL(); } @Override - public URL getUrlForIdentifier(String identifier) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getUrlForIdentifier(String identifier) throws URISyntaxException, MalformedURLException { URIBuilder uriBuilder = new URIBuilder(API_URL + "/" + identifier); return uriBuilder.build().toURL(); } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/DBLPFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/DBLPFetcher.java index cca32ab6b7d..188dc6c9158 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/DBLPFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/DBLPFetcher.java @@ -12,7 +12,6 @@ import org.jabref.logic.cleanup.FieldFormatterCleanups; import org.jabref.logic.formatter.bibtexfields.ClearFormatter; import org.jabref.logic.help.HelpFile; -import org.jabref.logic.importer.FetcherException; import org.jabref.logic.importer.ImportFormatPreferences; import org.jabref.logic.importer.Parser; import org.jabref.logic.importer.SearchBasedParserFetcher; @@ -44,7 +43,7 @@ public DBLPFetcher(ImportFormatPreferences importFormatPreferences) { } @Override - public URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException { URIBuilder uriBuilder = new URIBuilder(BASIC_SEARCH_URL); uriBuilder.addParameter("q", new DBLPQueryTransformer().transformLuceneQuery(luceneQuery).orElse("")); uriBuilder.addParameter("h", String.valueOf(100)); // number of hits diff --git a/src/main/java/org/jabref/logic/importer/fetcher/DOABFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/DOABFetcher.java index 33f70d7dc61..3408c631142 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/DOABFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/DOABFetcher.java @@ -8,7 +8,6 @@ import java.util.List; import java.util.StringJoiner; -import org.jabref.logic.importer.FetcherException; import org.jabref.logic.importer.Parser; import org.jabref.logic.importer.SearchBasedParserFetcher; import org.jabref.logic.importer.fetcher.transformers.DefaultQueryTransformer; @@ -39,7 +38,7 @@ public String getName() { } @Override - public URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException { URIBuilder builder = new URIBuilder(SEARCH_URL); String query = new DefaultQueryTransformer().transformLuceneQuery(luceneQuery).orElse(""); // adding quotations for the query for more specified results diff --git a/src/main/java/org/jabref/logic/importer/fetcher/DOAJFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/DOAJFetcher.java index 81b7e9af2b7..42f8152f387 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/DOAJFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/DOAJFetcher.java @@ -12,7 +12,6 @@ import java.util.stream.Collectors; import org.jabref.logic.help.HelpFile; -import org.jabref.logic.importer.FetcherException; import org.jabref.logic.importer.ImportFormatPreferences; import org.jabref.logic.importer.Parser; import org.jabref.logic.importer.SearchBasedParserFetcher; @@ -185,7 +184,7 @@ public Optional getHelpPage() { } @Override - public URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException { URIBuilder uriBuilder = new URIBuilder(SEARCH_URL); DOAJFetcher.addPath(uriBuilder, new DefaultLuceneQueryTransformer().transformLuceneQuery(luceneQuery).orElse("")); // Number of results diff --git a/src/main/java/org/jabref/logic/importer/fetcher/DiVA.java b/src/main/java/org/jabref/logic/importer/fetcher/DiVA.java index 88dcd6ebf2f..241c58a3d86 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/DiVA.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/DiVA.java @@ -6,7 +6,6 @@ import java.util.Optional; import org.jabref.logic.help.HelpFile; -import org.jabref.logic.importer.FetcherException; import org.jabref.logic.importer.IdBasedParserFetcher; import org.jabref.logic.importer.ImportFormatPreferences; import org.jabref.logic.importer.Parser; @@ -37,7 +36,7 @@ public Optional getHelpPage() { } @Override - public URL getUrlForIdentifier(String identifier) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getUrlForIdentifier(String identifier) throws URISyntaxException, MalformedURLException { URIBuilder uriBuilder = new URIBuilder("http://www.diva-portal.org/smash/getreferences"); uriBuilder.addParameter("referenceFormat", "BibTex"); diff --git a/src/main/java/org/jabref/logic/importer/fetcher/DoiFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/DoiFetcher.java index 9fa428e5ead..005122afa1e 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/DoiFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/DoiFetcher.java @@ -2,6 +2,7 @@ import java.io.IOException; import java.net.HttpURLConnection; +import java.net.MalformedURLException; import java.net.URL; import java.net.URLConnection; import java.util.Collections; @@ -114,64 +115,70 @@ protected CompletableFuture> asyncPerformSearchById(String id public Optional performSearchById(String identifier) throws FetcherException { Optional doi = DOI.parse(identifier); + if (doi.isEmpty()) { + throw new FetcherException(Localization.lang("Invalid DOI: '%0'.", identifier)); + } + + URL doiURL; try { - if (doi.isPresent()) { - Optional fetchedEntry; + doiURL = new URL(doi.get().getURIAsASCIIString()); + } catch (MalformedURLException e) { + throw new FetcherException("Malformed URL", e); + } - // mEDRA does not return a parsable bibtex string - Optional agency = getAgency(doi.get()); - if (agency.isPresent() && "medra".equalsIgnoreCase(agency.get())) { - return new Medra().performSearchById(identifier); - } - URL doiURL = new URL(doi.get().getURIAsASCIIString()); - - // BibTeX data - URLDownload download = getUrlDownload(doiURL); - download.addHeader("Accept", MediaTypes.APPLICATION_BIBTEX); - - String bibtexString; - URLConnection openConnection; - try { - openConnection = download.openConnection(); - bibtexString = URLDownload.asString(openConnection).trim(); - } catch (IOException e) { - // an IOException with a nested FetcherException will be thrown when you encounter a 400x or 500x http status code - if (e.getCause() instanceof FetcherException fe) { - throw fe; - } - throw e; - } + try { + Optional fetchedEntry; - // BibTeX entry - fetchedEntry = BibtexParser.singleFromString(bibtexString, preferences); - fetchedEntry.ifPresent(this::doPostCleanup); + // mEDRA does not return a parsable bibtex string + Optional agency = getAgency(doi.get()); + if (agency.isPresent() && "medra".equalsIgnoreCase(agency.get())) { + return new Medra().performSearchById(identifier); + } - // Crossref has a dynamic API rate limit - if (agency.isPresent() && "crossref".equalsIgnoreCase(agency.get())) { - updateCrossrefAPIRate(openConnection); - } + // BibTeX data + URLDownload download = getUrlDownload(doiURL); + download.addHeader("Accept", MediaTypes.APPLICATION_BIBTEX); - // Check if the entry is an APS journal and add the article id as the page count if page field is missing - if (fetchedEntry.isPresent() && fetchedEntry.get().hasField(StandardField.DOI)) { - BibEntry entry = fetchedEntry.get(); - if (isAPSJournal(entry, entry.getField(StandardField.DOI).get()) && !entry.hasField(StandardField.PAGES)) { - setPageCountToArticleId(entry, entry.getField(StandardField.DOI).get()); - } + String bibtexString; + URLConnection openConnection; + try { + openConnection = download.openConnection(); + bibtexString = URLDownload.asString(openConnection).trim(); + } catch (IOException e) { + // an IOException with a nested FetcherException will be thrown when you encounter a 400x or 500x http status code + if (e.getCause() instanceof FetcherException fe) { + throw fe; } + throw e; + } - if (openConnection instanceof HttpURLConnection connection) { - connection.disconnect(); + // BibTeX entry + fetchedEntry = BibtexParser.singleFromString(bibtexString, preferences); + fetchedEntry.ifPresent(this::doPostCleanup); + + // Crossref has a dynamic API rate limit + if (agency.isPresent() && "crossref".equalsIgnoreCase(agency.get())) { + updateCrossrefAPIRate(openConnection); + } + + // Check if the entry is an APS journal and add the article id as the page count if page field is missing + if (fetchedEntry.isPresent() && fetchedEntry.get().hasField(StandardField.DOI)) { + BibEntry entry = fetchedEntry.get(); + if (isAPSJournal(entry, entry.getField(StandardField.DOI).get()) && !entry.hasField(StandardField.PAGES)) { + setPageCountToArticleId(entry, entry.getField(StandardField.DOI).get()); } - return fetchedEntry; - } else { - throw new FetcherException(Localization.lang("Invalid DOI: '%0'.", identifier)); } + + if (openConnection instanceof HttpURLConnection connection) { + connection.disconnect(); + } + return fetchedEntry; } catch (IOException e) { - throw new FetcherException(Localization.lang("Connection error"), e); + throw new FetcherException(doiURL, Localization.lang("Connection error"), e); } catch (ParseException e) { - throw new FetcherException("Could not parse BibTeX entry", e); + throw new FetcherException(doiURL, "Could not parse BibTeX entry", e); } catch (JSONException e) { - throw new FetcherException("Could not retrieve Registration Agency", e); + throw new FetcherException(doiURL, "Could not retrieve Registration Agency", e); } } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/GoogleScholar.java b/src/main/java/org/jabref/logic/importer/fetcher/GoogleScholar.java index b2de8a817d1..f0461e8338e 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/GoogleScholar.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/GoogleScholar.java @@ -3,6 +3,7 @@ import java.io.IOException; import java.io.StringReader; import java.net.HttpCookie; +import java.net.MalformedURLException; import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; @@ -132,7 +133,8 @@ private void addHitsFromQuery(List entryList, String queryURL) throws String content = new URLDownload(queryURL).asString(); if (needsCaptcha(content)) { - throw new FetcherException("Fetching from Google Scholar failed: Captacha hit at " + queryURL + ".", + // TODO: Remove "null" + throw new FetcherException(queryURL, "Fetching from Google Scholar failed: Captacha hit." + Localization.lang("This might be caused by reaching the traffic limitation of Google Scholar (see 'Help' for details)."), null); } @@ -153,7 +155,7 @@ private BibEntry downloadEntry(String link) throws IOException, FetcherException } else { Collection entries = result.getDatabase().getEntries(); if (entries.size() != 1) { - LOGGER.debug(entries.size() + " entries found! (" + link + ")"); + LOGGER.debug("{} entries found! ({})", entries.size(), link); throw new FetcherException("Parsing entries from Google Scholar bib file failed."); } else { BibEntry entry = entries.iterator().next(); @@ -179,40 +181,49 @@ private void obtainAndModifyCookie() throws FetcherException { public Page performSearchPaged(QueryNode luceneQuery, int pageNumber) throws FetcherException { ScholarQueryTransformer queryTransformer = new ScholarQueryTransformer(); String transformedQuery = queryTransformer.transformLuceneQuery(luceneQuery).orElse(""); + + obtainAndModifyCookie(); + + URIBuilder uriBuilder; try { - obtainAndModifyCookie(); - List foundEntries = new ArrayList<>(10); - URIBuilder uriBuilder = new URIBuilder(BASIC_SEARCH_URL); - uriBuilder.addParameter("hl", "en"); - uriBuilder.addParameter("btnG", "Search"); - uriBuilder.addParameter("q", transformedQuery); - uriBuilder.addParameter("start", String.valueOf(pageNumber * getPageSize())); - uriBuilder.addParameter("num", String.valueOf(getPageSize())); - uriBuilder.addParameter("as_ylo", String.valueOf(queryTransformer.getStartYear())); - uriBuilder.addParameter("as_yhi", String.valueOf(queryTransformer.getEndYear())); - - try { - addHitsFromQuery(foundEntries, uriBuilder.toString()); + uriBuilder = new URIBuilder(BASIC_SEARCH_URL); + } catch (URISyntaxException e) { + throw new FetcherException("Building URI failed.", e); + } + uriBuilder.addParameter("hl", "en"); + uriBuilder.addParameter("btnG", "Search"); + uriBuilder.addParameter("q", transformedQuery); + uriBuilder.addParameter("start", String.valueOf(pageNumber * getPageSize())); + uriBuilder.addParameter("num", String.valueOf(getPageSize())); + uriBuilder.addParameter("as_ylo", String.valueOf(queryTransformer.getStartYear())); + uriBuilder.addParameter("as_yhi", String.valueOf(queryTransformer.getEndYear())); - if (foundEntries.size() == 10) { - uriBuilder.addParameter("start", "10"); - addHitsFromQuery(foundEntries, uriBuilder.toString()); - } - } catch (IOException e) { - LOGGER.info("IOException for URL {}", uriBuilder.toString()); - // if there are too much requests from the same IP adress google is answering with a 503 and redirecting to a captcha challenge - // The caught IOException looks for example like this: - // java.io.IOException: Server returned HTTP response code: 503 for URL: https://ipv4.google.com/sorry/index?continue=https://scholar.google.com/scholar%3Fhl%3Den%26btnG%3DSearch%26q%3Dbpmn&hl=en&q=CGMSBI0NBDkYuqy9wAUiGQDxp4NLQCWbIEY1HjpH5zFJhv4ANPGdWj0 - if (e.getMessage().contains("Server returned HTTP response code: 503 for URL")) { - throw new FetcherException("Fetching from Google Scholar failed.", - Localization.lang("This might be caused by reaching the traffic limitation of Google Scholar (see 'Help' for details)."), e); - } else { - throw new FetcherException("Error while fetching from " + getName(), e); + List foundEntries = new ArrayList<>(10); + + try { + addHitsFromQuery(foundEntries, uriBuilder.toString()); + if (foundEntries.size() == 10) { + uriBuilder.addParameter("start", "10"); + addHitsFromQuery(foundEntries, uriBuilder.toString()); + } + } catch (IOException e) { + LOGGER.info("IOException for URL {}", uriBuilder); + // if there are too much requests from the same IP adress google is answering with a 503 and redirecting to a captcha challenge + // The caught IOException looks for example like this: + // java.io.IOException: Server returned HTTP response code: 503 for URL: https://ipv4.google.com/sorry/index?continue=https://scholar.google.com/scholar%3Fhl%3Den%26btnG%3DSearch%26q%3Dbpmn&hl=en&q=CGMSBI0NBDkYuqy9wAUiGQDxp4NLQCWbIEY1HjpH5zFJhv4ANPGdWj0 + if (e.getMessage().contains("Server returned HTTP response code: 503 for URL")) { + throw new FetcherException("Fetching from Google Scholar failed.", + Localization.lang("This might be caused by reaching the traffic limitation of Google Scholar (see 'Help' for details)."), e); + } else { + URL url; + try { + url = uriBuilder.build().toURL(); + } catch (URISyntaxException | MalformedURLException ex) { + throw new FetcherException("Wrong URL syntax", e); } + throw new FetcherException(url, "Error while fetching from " + getName(), e); } - return new Page<>(transformedQuery, pageNumber, foundEntries); - } catch (URISyntaxException e) { - throw new FetcherException("Error while fetching from " + getName(), e); } + return new Page<>(transformedQuery, pageNumber, foundEntries); } } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/GrobidCitationFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/GrobidCitationFetcher.java index b2479cc90fb..0576c90e6ea 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/GrobidCitationFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/GrobidCitationFetcher.java @@ -8,6 +8,7 @@ import java.util.Optional; import java.util.stream.Collectors; +import org.jabref.http.dto.SimpleHttpResponse; import org.jabref.logic.importer.FetcherException; import org.jabref.logic.importer.ImportFormatPreferences; import org.jabref.logic.importer.ParseException; @@ -49,17 +50,15 @@ private Optional parseUsingGrobid(String plainText) throws FetcherExce try { return grobidService.processCitation(plainText, importFormatPreferences, GrobidService.ConsolidateCitations.WITH_METADATA); } catch (HttpStatusException e) { - String msg = "Connection failure."; - LOGGER.debug(msg, e); - throw new FetcherException(msg, e.getCause()); + LOGGER.debug("Could not connect to Grobid", e); + throw new FetcherException("{grobid}", new SimpleHttpResponse(e)); } catch (SocketTimeoutException e) { String msg = "Connection timed out."; LOGGER.debug(msg, e); throw new FetcherException(msg, e.getCause()); } catch (IOException | ParseException e) { - String msg = "Could not process citation. " + e.getMessage(); - LOGGER.debug(msg, e); - return Optional.empty(); + LOGGER.debug("Could not process citation", e); + throw new FetcherException("Could not process citation", e); } } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/GvkFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/GvkFetcher.java index 24513e0084d..ae463ba3391 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/GvkFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/GvkFetcher.java @@ -11,7 +11,6 @@ import org.jabref.logic.formatter.bibtexfields.NormalizeNamesFormatter; import org.jabref.logic.formatter.bibtexfields.NormalizePagesFormatter; import org.jabref.logic.help.HelpFile; -import org.jabref.logic.importer.FetcherException; import org.jabref.logic.importer.ImportFormatPreferences; import org.jabref.logic.importer.Parser; import org.jabref.logic.importer.SearchBasedParserFetcher; @@ -48,7 +47,7 @@ public Optional getHelpPage() { } @Override - public URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException { URIBuilder uriBuilder = new URIBuilder(URL_PATTERN); uriBuilder.addParameter("version", "1.1"); uriBuilder.addParameter("operation", "searchRetrieve"); @@ -62,7 +61,7 @@ public URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, Malf } @Override - public URL getUrlForIdentifier(String identifier) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getUrlForIdentifier(String identifier) throws URISyntaxException, MalformedURLException { this.ensureThatIsbnIsValid(identifier); URIBuilder uriBuilder = new URIBuilder(URL_PATTERN); uriBuilder.addParameter("version", "1.1"); diff --git a/src/main/java/org/jabref/logic/importer/fetcher/IEEE.java b/src/main/java/org/jabref/logic/importer/fetcher/IEEE.java index e9293e65065..c444dcff212 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/IEEE.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/IEEE.java @@ -15,7 +15,6 @@ import java.util.stream.Collectors; import org.jabref.logic.help.HelpFile; -import org.jabref.logic.importer.FetcherException; import org.jabref.logic.importer.FulltextFetcher; import org.jabref.logic.importer.ImportFormatPreferences; import org.jabref.logic.importer.ImporterPreferences; @@ -258,7 +257,7 @@ public String getTestUrl() { } @Override - public URL getURLForQuery(QueryNode luceneQuery, int pageNumber) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getURLForQuery(QueryNode luceneQuery, int pageNumber) throws URISyntaxException, MalformedURLException { // transformer is stored globally, because we need to filter out the bib entries by the year manually // the transformer stores the min and max year transformer = new IEEEQueryTransformer(); diff --git a/src/main/java/org/jabref/logic/importer/fetcher/INSPIREFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/INSPIREFetcher.java index 71cf9817a3a..696b423f5e3 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/INSPIREFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/INSPIREFetcher.java @@ -5,7 +5,6 @@ import java.net.URI; import java.net.URISyntaxException; import java.net.URL; -import java.util.ArrayList; import java.util.List; import java.util.Optional; @@ -57,7 +56,7 @@ public Optional getHelpPage() { } @Override - public URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException { URIBuilder uriBuilder = new URIBuilder(INSPIRE_HOST); uriBuilder.addParameter("q", new DefaultLuceneQueryTransformer().transformLuceneQuery(luceneQuery).orElse("")); return uriBuilder.build().toURL(); @@ -88,27 +87,33 @@ public Parser getParser() { @Override public List performSearch(BibEntry entry) throws FetcherException { - List results = new ArrayList<>(); Optional doi = entry.getField(StandardField.DOI); Optional archiveprefix = entry.getFieldOrAlias(StandardField.ARCHIVEPREFIX); Optional eprint = entry.getField(StandardField.EPRINT); - String url; + String urlString; if (archiveprefix.filter("arxiv"::equals).isPresent() && eprint.isPresent()) { - url = INSPIRE_ARXIV_HOST + eprint.get(); + urlString = INSPIRE_ARXIV_HOST + eprint.get(); } else if (doi.isPresent()) { - url = INSPIRE_DOI_HOST + doi.get(); + urlString = INSPIRE_DOI_HOST + doi.get(); } else { - return results; + return List.of(); + } + + URL url; + try { + url = new URI(urlString).toURL(); + } catch (MalformedURLException | URISyntaxException e) { + throw new FetcherException("Invalid URL", e); } try { - URLDownload download = getUrlDownload(new URI(url).toURL()); - results = getParser().parseEntries(download.asInputStream()); + URLDownload download = getUrlDownload(url); + List results = getParser().parseEntries(download.asInputStream()); results.forEach(this::doPostCleanup); return results; - } catch (IOException | ParseException | URISyntaxException e) { - throw new FetcherException("Error occurred during fetching", e); + } catch (IOException | ParseException e) { + throw new FetcherException(url, e); } } } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/ISIDOREFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/ISIDOREFetcher.java index d882a05d0f0..208ac577542 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/ISIDOREFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/ISIDOREFetcher.java @@ -7,7 +7,6 @@ import java.net.URL; import java.nio.charset.StandardCharsets; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Optional; import java.util.StringJoiner; @@ -76,15 +75,13 @@ public Parser getParser() { Element entryElement = document.getDocumentElement(); if (entryElement == null) { - return Collections.emptyList(); + return List.of(); } return parseXMl(entryElement); } catch (FetcherException e) { Unchecked.throwChecked(e); - } catch (ParserConfigurationException | - IOException | - SAXException e) { + } catch (ParserConfigurationException | IOException | SAXException e) { Unchecked.throwChecked(new FetcherException("Issue with parsing link", e)); } return null; @@ -99,7 +96,7 @@ public URLDownload getUrlDownload(URL url) { } @Override - public URL getURLForQuery(QueryNode luceneQuery, int pageNumber) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getURLForQuery(QueryNode luceneQuery, int pageNumber) throws URISyntaxException, MalformedURLException { ISIDOREQueryTransformer queryTransformer = new ISIDOREQueryTransformer(); String transformedQuery = queryTransformer.transformLuceneQuery(luceneQuery).orElse(""); URIBuilder uriBuilder = new URIBuilder(SOURCE_WEB_SEARCH); diff --git a/src/main/java/org/jabref/logic/importer/fetcher/IacrEprintFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/IacrEprintFetcher.java index 7233c998ae3..0a58be73d71 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/IacrEprintFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/IacrEprintFetcher.java @@ -64,7 +64,7 @@ private Optional createEntryFromIacrCitation(String validIdentifier) t try { return BibtexParser.singleFromString(actualEntry, prefs); } catch (ParseException e) { - throw new FetcherException(Localization.lang("Entry from %0 could not be parsed.", "IACR"), e); + throw FetcherException.ofUrl(Localization.lang("Entry from %0 could not be parsed.", "IACR"), e); } } @@ -108,7 +108,7 @@ private String getHtml(String url) throws FetcherException { URLDownload download = new URLDownload(url); return download.asString(); } catch (IOException e) { - throw new FetcherException(Localization.lang("Could not retrieve entry data from '%0'.", url), e); + throw FetcherException.ofUrl(url, Localization.lang("Could not retrieve entry data from '%0'.", url), e); } } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/JstorFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/JstorFetcher.java index b4e4ab0b6eb..c381a6ada2f 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/JstorFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/JstorFetcher.java @@ -12,7 +12,6 @@ import java.util.Optional; import java.util.stream.Collectors; -import org.jabref.logic.importer.FetcherException; import org.jabref.logic.importer.FulltextFetcher; import org.jabref.logic.importer.IdBasedParserFetcher; import org.jabref.logic.importer.ImportFormatPreferences; @@ -48,14 +47,14 @@ public JstorFetcher(ImportFormatPreferences importFormatPreferences) { } @Override - public URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException { URIBuilder uriBuilder = new URIBuilder(SEARCH_HOST); uriBuilder.addParameter("Query", new JstorQueryTransformer().transformLuceneQuery(luceneQuery).orElse("")); return uriBuilder.build().toURL(); } @Override - public URL getUrlForIdentifier(String identifier) throws FetcherException { + public URL getUrlForIdentifier(String identifier) throws MalformedURLException { String start = "https://www.jstor.org/citation/text/"; if (identifier.startsWith("http")) { identifier = identifier.replace("https://www.jstor.org/stable", ""); @@ -63,16 +62,12 @@ public URL getUrlForIdentifier(String identifier) throws FetcherException { } identifier = identifier.replaceAll(URL_QUERY_REGEX, ""); - try { - if (identifier.contains("/")) { - // if identifier links to a entry with a valid doi - return new URL(start + identifier); - } - // else use default doi start. - return new URL(start + "10.2307/" + identifier); - } catch (IOException e) { - throw new FetcherException("could not construct url for jstor", e); + if (identifier.contains("/")) { + // if identifier links to a entry with a valid doi + return new URL(start + identifier); } + // else use default doi start. + return new URL(start + "10.2307/" + identifier); } @Override diff --git a/src/main/java/org/jabref/logic/importer/fetcher/LOBIDFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/LOBIDFetcher.java index 5920b605943..c34d7cc4860 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/LOBIDFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/LOBIDFetcher.java @@ -12,7 +12,6 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; -import org.jabref.logic.importer.FetcherException; import org.jabref.logic.importer.ImporterPreferences; import org.jabref.logic.importer.PagedSearchBasedParserFetcher; import org.jabref.logic.importer.Parser; @@ -58,12 +57,12 @@ public LOBIDFetcher(ImporterPreferences importerPreferences) { * @return URL */ @Override - public URL getURLForQuery(QueryNode luceneQuery, int pageNumber) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getURLForQuery(QueryNode luceneQuery, int pageNumber) throws URISyntaxException, MalformedURLException { URIBuilder uriBuilder = new URIBuilder(API_URL); uriBuilder.addParameter("q", new LOBIDQueryTransformer().transformLuceneQuery(luceneQuery).orElse("")); // search query uriBuilder.addParameter("from", String.valueOf(getPageSize() * pageNumber)); // from entry number, starts indexing at 0 - uriBuilder.addParameter("size", String.valueOf(getPageSize())); // page size - uriBuilder.addParameter("format", "json"); // response format + uriBuilder.addParameter("size", String.valueOf(getPageSize())); + uriBuilder.addParameter("format", "json"); return uriBuilder.build().toURL(); } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/LibraryOfCongress.java b/src/main/java/org/jabref/logic/importer/fetcher/LibraryOfCongress.java index 8ffa700df18..6471c36aae6 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/LibraryOfCongress.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/LibraryOfCongress.java @@ -4,7 +4,6 @@ import java.net.URISyntaxException; import java.net.URL; -import org.jabref.logic.importer.FetcherException; import org.jabref.logic.importer.IdBasedParserFetcher; import org.jabref.logic.importer.ImportFormatPreferences; import org.jabref.logic.importer.Parser; @@ -29,7 +28,7 @@ public String getName() { } @Override - public URL getUrlForIdentifier(String identifier) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getUrlForIdentifier(String identifier) throws URISyntaxException, MalformedURLException { URIBuilder uriBuilder = new URIBuilder("https://lccn.loc.gov/" + identifier + "/mods"); return uriBuilder.build().toURL(); } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/MathSciNet.java b/src/main/java/org/jabref/logic/importer/fetcher/MathSciNet.java index f18b74c9fd7..2af1bebbf42 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/MathSciNet.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/MathSciNet.java @@ -97,7 +97,7 @@ public URL getURLForEntry(BibEntry entry) throws URISyntaxException, MalformedUR } @Override - public URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException { URIBuilder uriBuilder = new URIBuilder("https://mathscinet.ams.org/mathscinet/api/publications/search"); uriBuilder.addParameter("query", new DefaultQueryTransformer().transformLuceneQuery(luceneQuery).orElse("")); // query uriBuilder.addParameter("currentPage", "1"); // start index @@ -106,7 +106,7 @@ public URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, Malf } @Override - public URL getUrlForIdentifier(String identifier) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getUrlForIdentifier(String identifier) throws URISyntaxException, MalformedURLException { URIBuilder uriBuilder = new URIBuilder("https://mathscinet.ams.org/mathscinet/api/publications/format"); uriBuilder.addParameter("formats", "bib"); uriBuilder.addParameter("ids", identifier); // identifier diff --git a/src/main/java/org/jabref/logic/importer/fetcher/MedlineFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/MedlineFetcher.java index 25b6116294a..a3444ad3530 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/MedlineFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/MedlineFetcher.java @@ -126,7 +126,7 @@ public Optional getHelpPage() { } @Override - public URL getUrlForIdentifier(String identifier) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getUrlForIdentifier(String identifier) throws URISyntaxException, MalformedURLException { URIBuilder uriBuilder = new URIBuilder(ID_URL); uriBuilder.addParameter("db", "pubmed"); uriBuilder.addParameter("retmode", "xml"); diff --git a/src/main/java/org/jabref/logic/importer/fetcher/Medra.java b/src/main/java/org/jabref/logic/importer/fetcher/Medra.java index e6329656927..2c52645baf7 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/Medra.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/Medra.java @@ -8,7 +8,6 @@ import java.util.stream.IntStream; import org.jabref.logic.cleanup.DoiCleanup; -import org.jabref.logic.importer.FetcherException; import org.jabref.logic.importer.IdBasedParserFetcher; import org.jabref.logic.importer.ParseException; import org.jabref.logic.importer.Parser; @@ -103,7 +102,7 @@ public URLDownload getUrlDownload(URL url) { } @Override - public URL getUrlForIdentifier(String identifier) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getUrlForIdentifier(String identifier) throws URISyntaxException, MalformedURLException { return new URL(API_URL + "/" + identifier); } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/MrDLibFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/MrDLibFetcher.java index e0d7310c52b..2d1205c6692 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/MrDLibFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/MrDLibFetcher.java @@ -3,7 +3,6 @@ import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; -import java.util.ArrayList; import java.util.Calendar; import java.util.List; import java.util.Optional; @@ -54,31 +53,31 @@ public String getName() { @Override public List performSearch(BibEntry entry) throws FetcherException { Optional title = entry.getFieldLatexFree(StandardField.TITLE); - if (title.isPresent()) { - String response = makeServerRequest(title.get()); - MrDLibImporter importer = new MrDLibImporter(); - ParserResult parserResult; - try { - if (importer.isRecognizedFormat(response)) { - parserResult = importer.importDatabase(response); - heading = importer.getRecommendationsHeading(); - description = importer.getRecommendationsDescription(); - recommendationSetId = importer.getRecommendationSetId(); - } else { - // For displaying An ErrorMessage - description = DEFAULT_MRDLIB_ERROR_MESSAGE; - BibDatabase errorBibDataBase = new BibDatabase(); - parserResult = new ParserResult(errorBibDataBase); - } - } catch (IOException e) { - LOGGER.error(e.getMessage(), e); - throw new FetcherException("JSON Parser IOException."); - } - return parserResult.getDatabase().getEntries(); - } else { + if (!title.isPresent()) { // without a title there is no reason to ask MrDLib - return new ArrayList<>(0); + return List.of(); + } + + String response = makeServerRequest(title.get()); + MrDLibImporter importer = new MrDLibImporter(); + ParserResult parserResult; + try { + if (importer.isRecognizedFormat(response)) { + parserResult = importer.importDatabase(response); + heading = importer.getRecommendationsHeading(); + description = importer.getRecommendationsDescription(); + recommendationSetId = importer.getRecommendationSetId(); + } else { + // For displaying An ErrorMessage + description = DEFAULT_MRDLIB_ERROR_MESSAGE; + BibDatabase errorBibDataBase = new BibDatabase(); + parserResult = new ParserResult(errorBibDataBase); + } + } catch (IOException e) { + LOGGER.error("Error while fetching", e); + throw FetcherException.ofUrl(constructQuery(title.get()), e); } + return parserResult.getDatabase().getEntries(); } public String getHeading() { @@ -96,8 +95,9 @@ public String getDescription() { * @return Returns the server response. This is an XML document as a String. */ private String makeServerRequest(String queryByTitle) throws FetcherException { + String url = constructQuery(queryByTitle); try { - URLDownload urlDownload = new URLDownload(constructQuery(queryByTitle)); + URLDownload urlDownload = new URLDownload(url); String response = urlDownload.asString(); // Conversion of < and > @@ -105,7 +105,7 @@ private String makeServerRequest(String queryByTitle) throws FetcherException { response = response.replace("<", "<"); return response; } catch (IOException e) { - throw new FetcherException("Problem downloading", e); + throw FetcherException.ofUrl(url, e); } } @@ -138,10 +138,10 @@ private String constructQuery(String queryWithTitle) { try { URI uri = builder.build(); - LOGGER.trace("Request: " + uri.toString()); + LOGGER.trace("Request: {}", uri.toString()); return uri.toString(); } catch (URISyntaxException e) { - LOGGER.error(e.getMessage(), e); + LOGGER.error("Invalid URI", e); } return ""; } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/ResearchGate.java b/src/main/java/org/jabref/logic/importer/fetcher/ResearchGate.java index e9f584e0e3a..883722d8296 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/ResearchGate.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/ResearchGate.java @@ -169,26 +169,29 @@ Optional getURLByDoi(DOI doi) throws IOException, NullPointerException { return Optional.of(link); } + private Document getPage(String url) throws IOException { + return Jsoup.connect(url) + .userAgent(URLDownload.USER_AGENT) + .referrer("www.google.com") + .ignoreHttpErrors(true) + .get(); + } + /** * Constructs a URL based on the query, size and page number. - *

* Extract the numerical internal ID and add it to the URL to receive a link to a {@link BibEntry} + *

* * @param luceneQuery the search query. * @return A URL that lets us download a .bib file - * @throws URISyntaxException from {@link URIBuilder}'s build() method - * @throws IOException from {@link Connection}'s get() method */ - private Document getPage(QueryNode luceneQuery) throws URISyntaxException, IOException { + private static String getUrlForQuery(QueryNode luceneQuery) throws URISyntaxException { String query = new DefaultQueryTransformer().transformLuceneQuery(luceneQuery).orElse(""); URIBuilder source = new URIBuilder(SEARCH); source.addParameter("type", "publication"); source.addParameter("query", query); - return Jsoup.connect(source.build().toString()) - .userAgent(URLDownload.USER_AGENT) - .referrer("www.google.com") - .ignoreHttpErrors(true) - .get(); + String url = source.build().toString(); + return url; } @Override @@ -206,13 +209,21 @@ public TrustLevel getTrustLevel() { @Override public List performSearch(QueryNode luceneQuery) throws FetcherException { Document html; + + String url; try { - html = getPage(luceneQuery); + url = getUrlForQuery(luceneQuery); + } catch (URISyntaxException e) { + throw new FetcherException("Invalid URL", e); + } + + try { + html = getPage(url); // ResearchGate's server blocks when too many request are made if (!html.getElementsByClass("nova-legacy-v-publication-item__title").hasText()) { - throw new FetcherException("ResearchGate server unavailable"); + throw FetcherException.ofUrl(url, "ResearchGate server unavailable"); } - } catch (URISyntaxException | IOException e) { + } catch (IOException e) { throw new FetcherException("URL is not correct", e); } @@ -234,7 +245,7 @@ public List performSearch(QueryNode luceneQuery) throws FetcherExcepti entry = parser.parseSingleEntry(bib); entry.ifPresent(list::add); } catch (ParseException e) { - LOGGER.debug("Entry is not convertible to Bibtex", e); + LOGGER.debug("Entry is not convertible to BibTeX", e); } } return list; @@ -245,7 +256,7 @@ private BufferedReader getInputStream(String urlString) { URL url = new URL(urlString); return new BufferedReader(new InputStreamReader(url.openStream())); } catch (IOException e) { - LOGGER.debug("Wrong URL:", e); + LOGGER.debug("Wrong URL", e); } return null; } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/RfcFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/RfcFetcher.java index db1df209765..43849ff3319 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/RfcFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/RfcFetcher.java @@ -7,7 +7,6 @@ import java.util.Optional; import org.jabref.logic.help.HelpFile; -import org.jabref.logic.importer.FetcherException; import org.jabref.logic.importer.IdBasedParserFetcher; import org.jabref.logic.importer.ImportFormatPreferences; import org.jabref.logic.importer.Parser; @@ -46,7 +45,7 @@ public Optional getHelpPage() { * @return the URL of the RFC resource */ @Override - public URL getUrlForIdentifier(String identifier) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getUrlForIdentifier(String identifier) throws URISyntaxException, MalformedURLException { String prefixedIdentifier = identifier.toLowerCase(Locale.ENGLISH); // if not a "draft" version if ((!prefixedIdentifier.startsWith(DRAFT_PREFIX)) && (!prefixedIdentifier.startsWith("rfc"))) { diff --git a/src/main/java/org/jabref/logic/importer/fetcher/ScholarArchiveFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/ScholarArchiveFetcher.java index 27a5273f45b..3c1521e1861 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/ScholarArchiveFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/ScholarArchiveFetcher.java @@ -8,7 +8,6 @@ import java.util.Optional; import java.util.stream.IntStream; -import org.jabref.logic.importer.FetcherException; import org.jabref.logic.importer.PagedSearchBasedParserFetcher; import org.jabref.logic.importer.ParseException; import org.jabref.logic.importer.Parser; @@ -46,7 +45,7 @@ public class ScholarArchiveFetcher implements PagedSearchBasedParserFetcher { * @return URL */ @Override - public URL getURLForQuery(QueryNode luceneQuery, int pageNumber) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getURLForQuery(QueryNode luceneQuery, int pageNumber) throws URISyntaxException, MalformedURLException { URIBuilder uriBuilder = new URIBuilder(API_URL); uriBuilder.addParameter("q", new ScholarArchiveQueryTransformer().transformLuceneQuery(luceneQuery).orElse("")); uriBuilder.addParameter("from", String.valueOf(getPageSize() * pageNumber)); diff --git a/src/main/java/org/jabref/logic/importer/fetcher/SemanticScholar.java b/src/main/java/org/jabref/logic/importer/fetcher/SemanticScholar.java index ab993deeeb1..09f8534e6a3 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/SemanticScholar.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/SemanticScholar.java @@ -133,15 +133,16 @@ String getURLBySource(String source) throws IOException, FetcherException { } @Override - public URL getURLForQuery(QueryNode luceneQuery, int pageNumber) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getURLForQuery(QueryNode luceneQuery, int pageNumber) throws URISyntaxException, MalformedURLException { URIBuilder uriBuilder = new URIBuilder(SOURCE_WEB_SEARCH); uriBuilder.addParameter("query", new DefaultQueryTransformer().transformLuceneQuery(luceneQuery).orElse("")); uriBuilder.addParameter("offset", String.valueOf(pageNumber * getPageSize())); uriBuilder.addParameter("limit", String.valueOf(Math.min(getPageSize(), 10000 - pageNumber * getPageSize()))); // All fields need to be specified uriBuilder.addParameter("fields", "paperId,externalIds,url,title,abstract,venue,year,authors"); - LOGGER.debug("URL for query: {}", uriBuilder.build().toURL()); - return uriBuilder.build().toURL(); + URL result = uriBuilder.build().toURL(); + LOGGER.debug("URL for query: {}", result); + return result; } /** diff --git a/src/main/java/org/jabref/logic/importer/fetcher/SpringerFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/SpringerFetcher.java index b271b113c39..33be2ee890e 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/SpringerFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/SpringerFetcher.java @@ -11,7 +11,6 @@ import java.util.stream.Collectors; import org.jabref.logic.help.HelpFile; -import org.jabref.logic.importer.FetcherException; import org.jabref.logic.importer.ImporterPreferences; import org.jabref.logic.importer.PagedSearchBasedParserFetcher; import org.jabref.logic.importer.Parser; @@ -185,8 +184,7 @@ public String getTestUrl() { * @return URL */ @Override - public URL getURLForQuery(QueryNode luceneQuery, int pageNumber) throws URISyntaxException, MalformedURLException, FetcherException { - + public URL getURLForQuery(QueryNode luceneQuery, int pageNumber) throws URISyntaxException, MalformedURLException { URIBuilder uriBuilder = new URIBuilder(API_URL); uriBuilder.addParameter("q", new SpringerQueryTransformer().transformLuceneQuery(luceneQuery).orElse("")); // Search query uriBuilder.addParameter("api_key", importerPreferences.getApiKey(getName()).orElse(API_KEY)); // API key diff --git a/src/main/java/org/jabref/logic/importer/fetcher/ZbMATH.java b/src/main/java/org/jabref/logic/importer/fetcher/ZbMATH.java index 8f88bdf0675..b02c19d9025 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/ZbMATH.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/ZbMATH.java @@ -104,7 +104,7 @@ public URL getURLForEntry(BibEntry entry) throws URISyntaxException, MalformedUR } @Override - public URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException { URIBuilder uriBuilder = new URIBuilder("https://zbmath.org/bibtexoutput/"); uriBuilder.addParameter("q", new ZbMathQueryTransformer().transformLuceneQuery(luceneQuery).orElse("")); // search all fields uriBuilder.addParameter("start", "0"); // start index @@ -113,7 +113,7 @@ public URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, Malf } @Override - public URL getUrlForIdentifier(String identifier) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getUrlForIdentifier(String identifier) throws URISyntaxException, MalformedURLException { URIBuilder uriBuilder = new URIBuilder("https://zbmath.org/bibtexoutput/"); String query = "an:".concat(identifier); // use an: to search for a zbMATH identifier uriBuilder.addParameter("q", query); diff --git a/src/main/java/org/jabref/logic/importer/fetcher/isbntobibtex/DoiToBibtexConverterComIsbnFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/isbntobibtex/DoiToBibtexConverterComIsbnFetcher.java index 1ebc813c7bc..66bd0235ee6 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/isbntobibtex/DoiToBibtexConverterComIsbnFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/isbntobibtex/DoiToBibtexConverterComIsbnFetcher.java @@ -8,7 +8,6 @@ import java.util.stream.IntStream; import java.util.stream.Stream; -import org.jabref.logic.importer.FetcherException; import org.jabref.logic.importer.ImportFormatPreferences; import org.jabref.logic.importer.ParseException; import org.jabref.logic.importer.Parser; @@ -40,7 +39,7 @@ public String getName() { } @Override - public URL getUrlForIdentifier(String identifier) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getUrlForIdentifier(String identifier) throws URISyntaxException, MalformedURLException { this.ensureThatIsbnIsValid(identifier); return new URIBuilder(BASE_URL) .setPathSegments("getInfo.php") diff --git a/src/main/java/org/jabref/logic/importer/fetcher/isbntobibtex/EbookDeIsbnFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/isbntobibtex/EbookDeIsbnFetcher.java index 42effb2bf05..f55ba6eaa76 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/isbntobibtex/EbookDeIsbnFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/isbntobibtex/EbookDeIsbnFetcher.java @@ -7,7 +7,6 @@ import org.jabref.logic.cleanup.FieldFormatterCleanup; import org.jabref.logic.formatter.bibtexfields.NormalizeNamesFormatter; import org.jabref.logic.formatter.bibtexfields.NormalizePagesFormatter; -import org.jabref.logic.importer.FetcherException; import org.jabref.logic.importer.ImportFormatPreferences; import org.jabref.logic.importer.fetcher.AbstractIsbnFetcher; import org.jabref.model.entry.BibEntry; @@ -31,7 +30,7 @@ public String getName() { } @Override - public URL getUrlForIdentifier(String identifier) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getUrlForIdentifier(String identifier) throws URISyntaxException, MalformedURLException { this.ensureThatIsbnIsValid(identifier); return new URIBuilder(BASE_URL) .addParameter("isbn", identifier) diff --git a/src/main/java/org/jabref/logic/importer/fetcher/isbntobibtex/OpenLibraryIsbnFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/isbntobibtex/OpenLibraryIsbnFetcher.java index 3283a169cf8..cb8e7267d77 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/isbntobibtex/OpenLibraryIsbnFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/isbntobibtex/OpenLibraryIsbnFetcher.java @@ -11,7 +11,6 @@ import java.util.stream.Stream; import org.jabref.logic.importer.AuthorListParser; -import org.jabref.logic.importer.FetcherException; import org.jabref.logic.importer.ImportFormatPreferences; import org.jabref.logic.importer.ParseException; import org.jabref.logic.importer.Parser; @@ -53,7 +52,7 @@ public String getName() { } @Override - public URL getUrlForIdentifier(String identifier) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getUrlForIdentifier(String identifier) throws URISyntaxException, MalformedURLException { this.ensureThatIsbnIsValid(identifier); return new URIBuilder(BASE_URL) .setPathSegments("isbn", identifier + ".json") diff --git a/src/main/java/org/jabref/logic/importer/fileformat/ACMPortalParser.java b/src/main/java/org/jabref/logic/importer/fileformat/ACMPortalParser.java index fb60e4a92f4..e1eef008b2c 100644 --- a/src/main/java/org/jabref/logic/importer/fileformat/ACMPortalParser.java +++ b/src/main/java/org/jabref/logic/importer/fileformat/ACMPortalParser.java @@ -83,16 +83,16 @@ public List parseDoiSearchPage(InputStream stream) throws ParseException return doiList; } - /** - * Obtain BibEntry according to DOI - * - * @param doiList DOI List - * @return BibEntry List - */ public List getBibEntriesFromDoiList(List doiList) throws FetcherException { List bibEntries = new ArrayList<>(); CookieHandler.setDefault(new CookieManager()); - try (InputStream stream = new URLDownload(getUrlFromDoiList(doiList)).asInputStream()) { + URL urlFromDoiList; + try { + urlFromDoiList = getUrlFromDoiList(doiList); + } catch (URISyntaxException | MalformedURLException e) { + throw new FetcherException("Wrong URL", e); + } + try (InputStream stream = new URLDownload(urlFromDoiList).asInputStream()) { String jsonString = new String((stream.readAllBytes()), StandardCharsets.UTF_8); JsonElement jsonElement = JsonParser.parseString(jsonString); @@ -104,8 +104,8 @@ public List getBibEntriesFromDoiList(List doiList) throws Fetc } } } - } catch (IOException | URISyntaxException e) { - throw new FetcherException("A network error occurred while fetching from ", e); + } catch (IOException e) { + throw new FetcherException(urlFromDoiList, e); } return bibEntries; diff --git a/src/main/java/org/jabref/logic/net/URLDownload.java b/src/main/java/org/jabref/logic/net/URLDownload.java index 4720ebd7b1c..c5e312279bd 100644 --- a/src/main/java/org/jabref/logic/net/URLDownload.java +++ b/src/main/java/org/jabref/logic/net/URLDownload.java @@ -364,13 +364,14 @@ private static void copy(InputStream in, Writer out, Charset encoding) throws IO } /** - * Open a connection to this object's URL (with specified settings). If accessing an HTTP URL, don't forget - * to close the resulting connection after usage. + * Open a connection to this object's URL (with specified settings). + *

+ * If accessing an HTTP URL, remeber to close the resulting connection after usage. * * @return an open connection */ public URLConnection openConnection() throws IOException { - URLConnection connection = this.source.openConnection(); + URLConnection connection = getUrlConnection(); connection.setConnectTimeout((int) connectTimeout.toMillis()); for (Entry entry : this.parameters.entrySet()) { connection.setRequestProperty(entry.getKey(), entry.getValue()); @@ -383,17 +384,18 @@ public URLConnection openConnection() throws IOException { } if (connection instanceof HttpURLConnection httpURLConnection) { - // this does network i/o: GET + read returned headers int status; try { + // this does network i/o: GET + read returned headers status = httpURLConnection.getResponseCode(); } catch (IOException e) { LOGGER.error("Error getting response code", e); // TODO: Convert e to FetcherClientException + // Same TODO as in org.jabref.logic.importer.EntryBasedParserFetcher.performSearch. + // Maybe do this in org.jabref.logic.importer.FetcherException.getLocalizedMessage throw e; } - // normally, 3xx is redirect if ((status == HttpURLConnection.HTTP_MOVED_TEMP) || (status == HttpURLConnection.HTTP_MOVED_PERM) || (status == HttpURLConnection.HTTP_SEE_OTHER)) { @@ -405,15 +407,30 @@ public URLConnection openConnection() throws IOException { SimpleHttpResponse httpResponse = new SimpleHttpResponse(httpURLConnection); LOGGER.info("{}", httpResponse); if ((status >= 400) && (status < 500)) { - throw new IOException(new FetcherClientException(httpResponse)); + throw new IOException(new FetcherClientException(this.source, httpResponse)); } else if (status >= 500) { - throw new IOException(new FetcherServerException(httpResponse)); + throw new IOException(new FetcherServerException(this.source, httpResponse)); } } } return connection; } + private URLConnection getUrlConnection() throws IOException { + URLConnection connection = this.source.openConnection(); + connection.setConnectTimeout((int) connectTimeout.toMillis()); + for (Entry entry : this.parameters.entrySet()) { + connection.setRequestProperty(entry.getKey(), entry.getValue()); + } + if (!this.postData.isEmpty()) { + connection.setDoOutput(true); + try (DataOutputStream wr = new DataOutputStream(connection.getOutputStream())) { + wr.writeBytes(this.postData); + } + } + return connection; + } + public void setConnectTimeout(Duration connectTimeout) { if (connectTimeout != null) { this.connectTimeout = connectTimeout; diff --git a/src/main/resources/l10n/JabRef_en.properties b/src/main/resources/l10n/JabRef_en.properties index 18afe55150c..5a4ef4ebcdd 100644 --- a/src/main/resources/l10n/JabRef_en.properties +++ b/src/main/resources/l10n/JabRef_en.properties @@ -1842,7 +1842,6 @@ Fetcher\ '%0'\ did\ not\ find\ an\ entry\ for\ id\ '%1'.=Fetcher '%0' did not fi Select\ first\ entry=Select first entry Select\ last\ entry=Select last entry -Invalid\ ISBN\:\ '%0'.=Invalid ISBN: '%0'. should\ be\ an\ integer\ or\ normalized=should be an integer or normalized should\ be\ normalized=should be normalized @@ -2615,12 +2614,14 @@ Illegal\ characters\ in\ the\ file\ name\ detected.\nFile\ will\ be\ renamed\ to Rename\ and\ add=Rename and add Failed\ to\ download\ from\ URL=Failed to download from URL -401\ Unauthorized\:\ Access\ Denied.\ You\ are\ not\ authorized\ to\ access\ this\ resource.\ Please\ check\ your\ credentials\ and\ try\ again.\ If\ you\ believe\ you\ should\ have\ access,\ please\ contact\ the\ administrator\ for\ assistance.\nURL\:\ %0\nDetails\:\ %1=401 Unauthorized: Access Denied. You are not authorized to access this resource. Please check your credentials and try again. If you believe you should have access, please contact the administrator for assistance.\nURL: %0\nDetails: %1 -403\ Forbidden\:\ Access\ Denied.\ You\ do\ not\ have\ permission\ to\ access\ this\ resource.\ Please\ contact\ the\ administrator\ for\ assistance\ or\ try\ a\ different\ action.\nURL\:\ %0\nDetails\:\ %1=403 Forbidden: Access Denied. You do not have permission to access this resource. Please contact the administrator for assistance or try a different action.\nURL: %0\nDetails: %1 -404\ Not\ Found\ Error\:\ The\ requested\ resource\ could\ not\ be\ found.\ It\ seems\ that\ the\ file\ you\ are\ trying\ to\ download\ is\ not\ available\ or\ has\ been\ moved.\ Please\ verify\ the\ URL\ and\ try\ again.\ If\ you\ believe\ this\ is\ an\ error,\ please\ contact\ the\ administrator\ for\ further\ assistance.\nURL\:\ %0\nDetails\:\ %1=404 Not Found Error: The requested resource could not be found. It seems that the file you are trying to download is not available or has been moved. Please verify the URL and try again. If you believe this is an error, please contact the administrator for further assistance.\nURL: %0\nDetails: %1 -Error\ downloading\ from\ URL.\ Cause\ is\ likely\ the\ server\ side.\nPlease\ try\ again\ later\ or\ contact\ the\ server\ administrator.\nURL\:\ %0\nDetails\:\ %1=Error downloading from URL. Cause is likely the server side.\nPlease try again later or contact the server administrator.\nURL: %0\nDetails: %1 +401\ Unauthorized\:\ Access\ Denied.\ You\ are\ not\ authorized\ to\ access\ this\ resource.\ Please\ check\ your\ credentials\ and\ try\ again.\ If\ you\ believe\ you\ should\ have\ access,\ please\ contact\ the\ administrator\ for\ assistance.=401 Unauthorized: Access Denied. You are not authorized to access this resource. Please check your credentials and try again. If you believe you should have access, please contact the administrator for assistance. +403\ Forbidden\:\ Access\ Denied.\ You\ do\ not\ have\ permission\ to\ access\ this\ resource.\ Please\ contact\ the\ administrator\ for\ assistance\ or\ try\ a\ different\ action.=403 Forbidden: Access Denied. You do not have permission to access this resource. Please contact the administrator for assistance or try a different action. +404\ Not\ Found\ Error\:\ The\ requested\ resource\ could\ not\ be\ found.\ It\ seems\ that\ the\ file\ you\ are\ trying\ to\ download\ is\ not\ available\ or\ has\ been\ moved.\ Please\ verify\ the\ URL\ and\ try\ again.\ If\ you\ believe\ this\ is\ an\ error,\ please\ contact\ the\ administrator\ for\ further\ assistance.=404 Not Found Error: The requested resource could not be found. It seems that the file you are trying to download is not available or has been moved. Please verify the URL and try again. If you believe this is an error, please contact the administrator for further assistance. +Error\ downloading\ from\ URL.\ Cause\ is\ likely\ the\ server\ side.\nPlease\ try\ again\ later\ or\ contact\ the\ server\ administrator.=Error downloading from URL. Cause is likely the server side.\nPlease try again later or contact the server administrator. +Something\ is\ wrong\ on\ JabRef\ side.\ Please\ check\ the\ URL\ and\ try\ again.=Something is wrong on JabRef side. Please check the URL and try again. +URL\:\ %0=URL: %0 +URL\:\ %0\nHTTP\ %1\ %2\n%3=URL: %0\nHTTP %1 %2\n%3 Please\ check\ the\ URL\ and\ try\ again.\nURL\:\ %0\nDetails\:\ %1=Please check the URL and try again.\nURL: %0\nDetails: %1 -Something\ is\ wrong\ on\ JabRef\ side.\ Please\ check\ the\ URL\ and\ try\ again.\nURL\:\ %0\nDetails\:\ %1=Something is wrong on JabRef side. Please check the URL and try again.\nURL: %0\nDetails: %1 Finished=Finished Finished\ writing\ metadata\ for\ library\ %0\ (%1\ succeeded,\ %2\ skipped,\ %3\ errors).=Finished writing metadata for library %0 (%1 succeeded, %2 skipped, %3 errors). From 7801226a3faa049aa9a5b280a10218b99632a9f2 Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Mon, 12 Aug 2024 02:03:48 +0200 Subject: [PATCH 28/42] Use FetcherException more often Also improve localization --- .../org/jabref/cli/ArgumentProcessor.java | 3 +- .../java/org/jabref/gui/DialogService.java | 6 +- .../gui/entryeditor/SciteTabViewModel.java | 27 +++--- .../SemanticScholarFetcher.java | 92 +++++++++---------- .../logic/importer/FetcherException.java | 17 +++- .../fetcher/AstrophysicsDataSystem.java | 10 +- .../importer/fetcher/BibsonomyScraper.java | 3 +- .../importer/fetcher/BiodiversityLibrary.java | 26 ++++-- ...OfComputerScienceBibliographiesParser.java | 5 +- .../logic/importer/fetcher/DoiFetcher.java | 17 +--- .../jabref/logic/importer/fetcher/IEEE.java | 26 +++++- .../importer/fetcher/INSPIREFetcher.java | 3 +- .../logic/importer/fetcher/JstorFetcher.java | 8 +- .../logic/importer/fetcher/ResearchGate.java | 2 +- .../logic/importer/util/ShortDOIService.java | 8 +- .../org/jabref/logic/net/URLDownload.java | 78 +++++++++++----- src/main/resources/l10n/JabRef_en.properties | 8 +- .../importer/fetcher/GoogleScholarTest.java | 2 +- .../importer/fetcher/ResearchGateTest.java | 2 +- .../fileformat/ACMPortalParserTest.java | 8 +- .../org/jabref/logic/net/URLDownloadTest.java | 18 ++-- 21 files changed, 209 insertions(+), 160 deletions(-) diff --git a/src/main/java/org/jabref/cli/ArgumentProcessor.java b/src/main/java/org/jabref/cli/ArgumentProcessor.java index 909fc46c6e2..6ce14604a64 100644 --- a/src/main/java/org/jabref/cli/ArgumentProcessor.java +++ b/src/main/java/org/jabref/cli/ArgumentProcessor.java @@ -1,6 +1,7 @@ package org.jabref.cli; import java.io.IOException; +import java.net.MalformedURLException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; @@ -133,7 +134,7 @@ private Optional importFile(String argument) { // Download web resource to temporary file try { file = new URLDownload(address).toTemporaryFile(); - } catch (IOException e) { + } catch (FetcherException | MalformedURLException e) { System.err.println(Localization.lang("Problem downloading from %1", address) + e.getLocalizedMessage()); return Optional.empty(); } diff --git a/src/main/java/org/jabref/gui/DialogService.java b/src/main/java/org/jabref/gui/DialogService.java index 309f227817a..a0a48c5ee87 100644 --- a/src/main/java/org/jabref/gui/DialogService.java +++ b/src/main/java/org/jabref/gui/DialogService.java @@ -117,11 +117,11 @@ default void showErrorDialogAndWait(FetcherException fetcherException) { if (httpResponse.isPresent()) { int statusCode = httpResponse.get().statusCode(); if (statusCode == 401) { - this.showInformationDialogAndWait(failedTitle, Localization.lang("401 Unauthorized: Access Denied. You are not authorized to access this resource. Please check your credentials and try again. If you believe you should have access, please contact the administrator for assistance.") + "\n\n" + localizedMessage); + this.showInformationDialogAndWait(failedTitle, Localization.lang("Access denied. You are not authorized to access this resource. Please check your credentials and try again. If you believe you should have access, please contact the administrator for assistance.") + "\n\n" + localizedMessage); } else if (statusCode == 403) { - this.showInformationDialogAndWait(failedTitle, Localization.lang("403 Forbidden: Access Denied. You do not have permission to access this resource. Please contact the administrator for assistance or try a different action.") + "\n\n" + localizedMessage); + this.showInformationDialogAndWait(failedTitle, Localization.lang("Access denied. You do not have permission to access this resource. Please contact the administrator for assistance or try a different action.") + "\n\n" + localizedMessage); } else if (statusCode == 404) { - this.showInformationDialogAndWait(failedTitle, Localization.lang("404 Not Found Error: The requested resource could not be found. It seems that the file you are trying to download is not available or has been moved. Please verify the URL and try again. If you believe this is an error, please contact the administrator for further assistance.") + "\n\n" + localizedMessage); + this.showInformationDialogAndWait(failedTitle, Localization.lang("The requested resource could not be found. It seems that the file you are trying to download is not available or has been moved. Please verify the URL and try again. If you believe this is an error, please contact the administrator for further assistance.") + "\n\n" + localizedMessage); } else { this.showErrorDialogAndWait(failedTitle, Localization.lang("Something is wrong on JabRef side. Please check the URL and try again.") + "\n\n" + localizedMessage); } diff --git a/src/main/java/org/jabref/gui/entryeditor/SciteTabViewModel.java b/src/main/java/org/jabref/gui/entryeditor/SciteTabViewModel.java index 43aa1b41e57..2a8216ea0c4 100644 --- a/src/main/java/org/jabref/gui/entryeditor/SciteTabViewModel.java +++ b/src/main/java/org/jabref/gui/entryeditor/SciteTabViewModel.java @@ -1,6 +1,5 @@ package org.jabref.gui.entryeditor; -import java.io.IOException; import java.net.MalformedURLException; import java.net.URI; import java.net.URISyntaxException; @@ -102,22 +101,18 @@ public SciteTallyModel fetchTallies(DOI doi) throws FetcherException { } catch (MalformedURLException | URISyntaxException ex) { throw new FetcherException("Malformed URL for DOI", ex); } - try { - LOGGER.debug("Fetching tallies from {}", url); - URLDownload download = new URLDownload(url); - String response = download.asString(); - LOGGER.debug("Response {}", response); - JSONObject tallies = new JSONObject(response); - if (tallies.has("detail")) { - String message = tallies.getString("detail"); - throw new FetcherException(message); - } else if (!tallies.has("total")) { - throw new FetcherException("Unexpected result data."); - } - return SciteTallyModel.fromJSONObject(tallies); - } catch (IOException ioex) { - throw new FetcherException(url, "Failed to retrieve tallies for DOI", ioex); + LOGGER.debug("Fetching tallies from {}", url); + URLDownload download = new URLDownload(url); + String response = download.asString(); + LOGGER.debug("Response {}", response); + JSONObject tallies = new JSONObject(response); + if (tallies.has("detail")) { + String message = tallies.getString("detail"); + throw new FetcherException(message); + } else if (!tallies.has("total")) { + throw new FetcherException("Unexpected result data."); } + return SciteTallyModel.fromJSONObject(tallies); } public ObjectProperty statusProperty() { diff --git a/src/main/java/org/jabref/gui/entryeditor/citationrelationtab/semanticscholar/SemanticScholarFetcher.java b/src/main/java/org/jabref/gui/entryeditor/citationrelationtab/semanticscholar/SemanticScholarFetcher.java index 639b045f09c..82ad2767f6a 100644 --- a/src/main/java/org/jabref/gui/entryeditor/citationrelationtab/semanticscholar/SemanticScholarFetcher.java +++ b/src/main/java/org/jabref/gui/entryeditor/citationrelationtab/semanticscholar/SemanticScholarFetcher.java @@ -1,6 +1,5 @@ package org.jabref.gui.entryeditor.citationrelationtab.semanticscholar; -import java.io.IOException; import java.net.MalformedURLException; import java.net.URI; import java.net.URL; @@ -34,60 +33,55 @@ public String getAPIUrl(String entry_point, BibEntry entry) { @Override public List searchCitedBy(BibEntry entry) throws FetcherException { - if (entry.getDOI().isPresent()) { - URL citationsUrl; - try { - citationsUrl = URI.create(getAPIUrl("citations", entry)).toURL(); - } catch (MalformedURLException e) { - throw new FetcherException("Malformed URL", e); - } - try { - URLDownload urlDownload = new URLDownload(citationsUrl); - - String apiKey = getApiKey(); - if (!apiKey.isEmpty()) { - urlDownload.addHeader("x-api-key", apiKey); - } - CitationsResponse citationsResponse = new Gson() - .fromJson(urlDownload.asString(), CitationsResponse.class); - - return citationsResponse.getData() - .stream().filter(citationDataItem -> citationDataItem.getCitingPaper() != null) - .map(citationDataItem -> citationDataItem.getCitingPaper().toBibEntry()).toList(); - } catch (IOException e) { - throw new FetcherException(citationsUrl, "Could not fetch", e); - } + if (!entry.getDOI().isPresent()) { + return List.of(); } - return List.of(); + + URL citationsUrl; + try { + citationsUrl = URI.create(getAPIUrl("citations", entry)).toURL(); + } catch (MalformedURLException e) { + throw new FetcherException("Malformed URL", e); + } + URLDownload urlDownload = new URLDownload(citationsUrl); + + String apiKey = getApiKey(); + if (!apiKey.isEmpty()) { + urlDownload.addHeader("x-api-key", apiKey); + } + CitationsResponse citationsResponse = new Gson() + .fromJson(urlDownload.asString(), CitationsResponse.class); + + return citationsResponse.getData() + .stream().filter(citationDataItem -> citationDataItem.getCitingPaper() != null) + .map(citationDataItem -> citationDataItem.getCitingPaper().toBibEntry()).toList(); } @Override public List searchCiting(BibEntry entry) throws FetcherException { - if (entry.getDOI().isPresent()) { - URL referencesUrl; - try { - referencesUrl = URI.create(getAPIUrl("references", entry)).toURL(); - } catch (MalformedURLException e) { - throw new FetcherException("Malformed URL", e); - } - try { - URLDownload urlDownload = new URLDownload(referencesUrl); - String apiKey = getApiKey(); - if (!apiKey.isEmpty()) { - urlDownload.addHeader("x-api-key", apiKey); - } - ReferencesResponse referencesResponse = new Gson() - .fromJson(urlDownload.asString(), ReferencesResponse.class); - - return referencesResponse.getData() - .stream() - .filter(citationDataItem -> citationDataItem.getCitedPaper() != null) - .map(referenceDataItem -> referenceDataItem.getCitedPaper().toBibEntry()).toList(); - } catch (IOException e) { - throw new FetcherException(referencesUrl, "Could not fetch", e); - } + if (!entry.getDOI().isPresent()) { + return List.of(); } - return List.of(); + + URL referencesUrl; + try { + referencesUrl = URI.create(getAPIUrl("references", entry)).toURL(); + } catch (MalformedURLException e) { + throw new FetcherException("Malformed URL", e); + } + + URLDownload urlDownload = new URLDownload(referencesUrl); + String apiKey = getApiKey(); + if (!apiKey.isEmpty()) { + urlDownload.addHeader("x-api-key", apiKey); + } + ReferencesResponse referencesResponse = new Gson() + .fromJson(urlDownload.asString(), ReferencesResponse.class); + + return referencesResponse.getData() + .stream() + .filter(citationDataItem -> citationDataItem.getCitedPaper() != null) + .map(referenceDataItem -> referenceDataItem.getCitedPaper().toBibEntry()).toList(); } @Override diff --git a/src/main/java/org/jabref/logic/importer/FetcherException.java b/src/main/java/org/jabref/logic/importer/FetcherException.java index e330c22a0e1..3340258ab79 100644 --- a/src/main/java/org/jabref/logic/importer/FetcherException.java +++ b/src/main/java/org/jabref/logic/importer/FetcherException.java @@ -7,6 +7,7 @@ import org.jabref.http.dto.SimpleHttpResponse; import org.jabref.logic.JabRefException; import org.jabref.logic.l10n.Localization; +import org.jabref.model.strings.StringUtil; public class FetcherException extends JabRefException { @@ -18,7 +19,8 @@ public class FetcherException extends JabRefException { private final SimpleHttpResponse httpResponse; public FetcherException(String url, SimpleHttpResponse httpResponse) { - super("URL: %s\nHTTP %s %s\n%s".formatted(url, httpResponse.statusCode(), httpResponse.responseMessage(), httpResponse.responseBody())); + // Empty string handled at org.jabref.logic.importer.FetcherException.getPrefix. + super(""); this.url = url; this.httpResponse = httpResponse; } @@ -58,6 +60,9 @@ private FetcherException(String url, String errorMessage, Throwable cause, Objec } public FetcherException(String errorMessage, Throwable cause) { + // TODO: Convert IOException e to FetcherClientException + // Same TODO as in org.jabref.logic.importer.EntryBasedParserFetcher.performSearch. + // Maybe do this in org.jabref.logic.importer.FetcherException.getLocalizedMessage super(errorMessage, cause); httpResponse = null; url = null; @@ -87,6 +92,14 @@ public static FetcherException ofUrl(String url, String errorMessage, Throwable return new FetcherException(url, errorMessage, cause, null); } + public static FetcherException of(URL url, SimpleHttpResponse simpleHttpResponse) { + if (simpleHttpResponse.statusCode() >= 500) { + return new FetcherServerException(url, simpleHttpResponse); + } else { + return new FetcherClientException(url, simpleHttpResponse); + } + } + @Override public String getLocalizedMessage() { // TODO: This should be moved to a separate class converting "any" exception object to a localized message @@ -107,7 +120,7 @@ private String getRedactedUrl() { private String getPrefix() { String superLocalizedMessage = super.getLocalizedMessage(); - if (superLocalizedMessage != null) { + if (!StringUtil.isBlank(superLocalizedMessage)) { return superLocalizedMessage + "\n"; } else { return ""; diff --git a/src/main/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystem.java b/src/main/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystem.java index 21c06541d2a..373cd064427 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystem.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystem.java @@ -1,6 +1,5 @@ package org.jabref.logic.importer.fetcher; -import java.io.IOException; import java.net.MalformedURLException; import java.net.URISyntaxException; import java.net.URL; @@ -204,9 +203,8 @@ private List fetchBibcodes(URL url) throws FetcherException { bibcodes.add(codes.getJSONObject(i).getString("bibcode")); } return bibcodes; - } catch (IOException e) { - throw new FetcherException(url, "A network error occurred", e); } catch (JSONException e) { + LOGGER.error("Error while parsing JSON", e); return Collections.emptyList(); } } @@ -231,8 +229,7 @@ public Optional performSearchById(String identifier) throws FetcherExc return Optional.empty(); } if (fetchedEntries.size() > 1) { - LOGGER.info("Fetcher " + getName() + "found more than one result for identifier " + identifier - + ". We will use the first entry."); + LOGGER.info("Fetcher {} found more than one result for identifier {}. We will use the first entry.", getName(), identifier); } BibEntry entry = fetchedEntries.getFirst(); return Optional.of(entry); @@ -274,10 +271,9 @@ private List performSearchByIds(Collection identifiers) throws return fetchedEntries; } catch (JSONException e) { + LOGGER.error("Error while parsing JSON", e); return Collections.emptyList(); } - } catch (IOException e) { - throw new FetcherException(urLforExport, "A network error occurred", e); } catch (ParseException e) { throw new FetcherException(urLforExport, "An internal parser error occurred", e); } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/BibsonomyScraper.java b/src/main/java/org/jabref/logic/importer/fetcher/BibsonomyScraper.java index f506d0128f2..724ac0a6f0a 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/BibsonomyScraper.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/BibsonomyScraper.java @@ -4,6 +4,7 @@ import java.net.URL; import java.util.Optional; +import org.jabref.logic.importer.FetcherException; import org.jabref.logic.importer.ImportFormatPreferences; import org.jabref.logic.importer.ParseException; import org.jabref.logic.importer.fileformat.BibtexParser; @@ -38,7 +39,7 @@ public static Optional getEntry(String entryUrl, ImportFormatPreferenc URL url = new URL(BibsonomyScraper.BIBSONOMY_SCRAPER + cleanURL + BibsonomyScraper.BIBSONOMY_SCRAPER_POST); String bibtex = new URLDownload(url).asString(); return BibtexParser.singleFromString(bibtex, importFormatPreferences); - } catch (IOException ex) { + } catch (IOException | FetcherException ex) { LOGGER.warn("Could not download entry", ex); return Optional.empty(); } catch (ParseException ex) { diff --git a/src/main/java/org/jabref/logic/importer/fetcher/BiodiversityLibrary.java b/src/main/java/org/jabref/logic/importer/fetcher/BiodiversityLibrary.java index 039d45c3c07..99ce61c4ca9 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/BiodiversityLibrary.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/BiodiversityLibrary.java @@ -1,6 +1,5 @@ package org.jabref.logic.importer.fetcher; -import java.io.IOException; import java.net.MalformedURLException; import java.net.URISyntaxException; import java.net.URL; @@ -9,6 +8,7 @@ import java.util.List; import java.util.stream.IntStream; +import org.jabref.logic.importer.FetcherException; import org.jabref.logic.importer.ImporterPreferences; import org.jabref.logic.importer.ParseException; import org.jabref.logic.importer.Parser; @@ -23,6 +23,7 @@ import org.jabref.model.entry.field.StandardField; import org.jabref.model.entry.types.StandardEntryType; +import com.google.common.annotations.VisibleForTesting; import kong.unirest.core.json.JSONArray; import kong.unirest.core.json.JSONException; import kong.unirest.core.json.JSONObject; @@ -79,7 +80,8 @@ public URL getItemMetadataURL(String identifier) throws URISyntaxException, Malf return uriBuilder.build().toURL(); } - public URL getPartMetadataURL(String identifier) throws URISyntaxException, MalformedURLException { + @VisibleForTesting + URL getPartMetadataURL(String identifier) throws URISyntaxException, MalformedURLException { URIBuilder uriBuilder = new URIBuilder(getBaseURL().toURI()); uriBuilder.addParameter("op", "GetPartMetadata"); uriBuilder.addParameter("pages", "f"); @@ -89,17 +91,22 @@ public URL getPartMetadataURL(String identifier) throws URISyntaxException, Malf return uriBuilder.build().toURL(); } - public JSONObject getDetails(URL url) throws IOException { + public JSONObject getDetails(URL url) throws FetcherException { URLDownload download = new URLDownload(url); String response = download.asString(); Logger.debug("Response {}", response); return new JSONObject(response).getJSONArray("Result").getJSONObject(0); } - public BibEntry parseBibJSONtoBibtex(JSONObject item, BibEntry entry) throws IOException, URISyntaxException { + public BibEntry parseBibJSONtoBibtex(JSONObject item, BibEntry entry) throws FetcherException { if (item.has("BHLType")) { if ("Part".equals(item.getString("BHLType"))) { - URL url = getPartMetadataURL(item.getString("PartID")); + URL url; + try { + url = getPartMetadataURL(item.getString("PartID")); + } catch (URISyntaxException | MalformedURLException e) { + throw new FetcherException("Malformed URL", e); + } JSONObject itemsDetails = getDetails(url); entry.setField(StandardField.LANGUAGE, itemsDetails.optString("Language", "")); @@ -112,7 +119,12 @@ public BibEntry parseBibJSONtoBibtex(JSONObject item, BibEntry entry) throws IOE } if ("Item".equals(item.getString("BHLType"))) { - URL url = getItemMetadataURL(item.getString("ItemID")); + URL url = null; + try { + url = getItemMetadataURL(item.getString("ItemID")); + } catch (URISyntaxException | MalformedURLException e) { + throw new FetcherException("Malformed URL", e); + } JSONObject itemsDetails = getDetails(url); entry.setField(StandardField.EDITOR, itemsDetails.optString("Sponsor", "")); entry.setField(StandardField.PUBLISHER, itemsDetails.optString("HoldingInstitution", "")); @@ -185,7 +197,7 @@ public Parser getParser() { BibEntry entry = jsonResultToBibEntry(item); try { entry = parseBibJSONtoBibtex(item, entry); - } catch (JSONException | IOException | URISyntaxException exception) { + } catch (JSONException | FetcherException exception) { throw new ParseException("Error when parsing entry", exception); } entries.add(entry); diff --git a/src/main/java/org/jabref/logic/importer/fetcher/CollectionOfComputerScienceBibliographiesParser.java b/src/main/java/org/jabref/logic/importer/fetcher/CollectionOfComputerScienceBibliographiesParser.java index 4a351e40cfc..c2f8aebe2be 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/CollectionOfComputerScienceBibliographiesParser.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/CollectionOfComputerScienceBibliographiesParser.java @@ -9,6 +9,7 @@ import java.util.stream.Collectors; import org.jabref.logic.formatter.bibtexfields.HtmlToUnicodeFormatter; +import org.jabref.logic.importer.FetcherException; import org.jabref.logic.importer.ImportFormatPreferences; import org.jabref.logic.importer.ParseException; import org.jabref.logic.importer.Parser; @@ -38,7 +39,7 @@ public List parseEntries(InputStream inputStream) throws ParseExceptio .collect(Collectors.joining()); return bibtexParser.parseEntries(bibtexDataString); - } catch (IOException e) { + } catch (IOException | FetcherException e) { throw new ParseException(e); } } @@ -51,7 +52,7 @@ private List matchRegexFromInputStreamHtml(InputStream inputStream, Patt } } - private List parseBibtexStringsFromLinks(List links) throws IOException { + private List parseBibtexStringsFromLinks(List links) throws IOException, FetcherException { List bibtexStringsFromAllLinks = new ArrayList<>(); for (String link : links) { try (InputStream inputStream = new URLDownload(link).asInputStream()) { diff --git a/src/main/java/org/jabref/logic/importer/fetcher/DoiFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/DoiFetcher.java index 005122afa1e..49a28e140ba 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/DoiFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/DoiFetcher.java @@ -95,7 +95,7 @@ private void doAPILimiting(String identifier) { LOGGER.trace("Thread %s, searching for DOI '%s', waited %.2fs because of API rate limiter".formatted( Thread.currentThread().threadId(), identifier, waitingTime)); } - } catch (IOException e) { + } catch (FetcherException | MalformedURLException e) { LOGGER.warn("Could not limit DOI API access rate", e); } } @@ -141,16 +141,9 @@ public Optional performSearchById(String identifier) throws FetcherExc String bibtexString; URLConnection openConnection; - try { - openConnection = download.openConnection(); - bibtexString = URLDownload.asString(openConnection).trim(); - } catch (IOException e) { - // an IOException with a nested FetcherException will be thrown when you encounter a 400x or 500x http status code - if (e.getCause() instanceof FetcherException fe) { - throw fe; - } - throw e; - } + + openConnection = download.openConnection(); + bibtexString = URLDownload.asString(openConnection).trim(); // BibTeX entry fetchedEntry = BibtexParser.singleFromString(bibtexString, preferences); @@ -222,7 +215,7 @@ public List performSearch(BibEntry entry) throws FetcherException { * * @param doi the DOI to be searched */ - public Optional getAgency(DOI doi) throws IOException { + public Optional getAgency(DOI doi) throws FetcherException, MalformedURLException { Optional agency = Optional.empty(); try { URLDownload download = getUrlDownload(new URL(DOI.AGENCY_RESOLVER + "/" + doi.getDOI())); diff --git a/src/main/java/org/jabref/logic/importer/fetcher/IEEE.java b/src/main/java/org/jabref/logic/importer/fetcher/IEEE.java index c444dcff212..185e9e3906b 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/IEEE.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/IEEE.java @@ -1,7 +1,6 @@ package org.jabref.logic.importer.fetcher; import java.io.BufferedReader; -import java.io.IOException; import java.io.InputStreamReader; import java.net.MalformedURLException; import java.net.URISyntaxException; @@ -15,6 +14,7 @@ import java.util.stream.Collectors; import org.jabref.logic.help.HelpFile; +import org.jabref.logic.importer.FetcherException; import org.jabref.logic.importer.FulltextFetcher; import org.jabref.logic.importer.ImportFormatPreferences; import org.jabref.logic.importer.ImporterPreferences; @@ -134,7 +134,7 @@ private static BibEntry parseJsonResponse(JSONObject jsonEntry, Character keywor } @Override - public Optional findFullText(BibEntry entry) throws IOException { + public Optional findFullText(BibEntry entry) throws FetcherException { Objects.requireNonNull(entry); String stampString = ""; @@ -161,7 +161,12 @@ public Optional findFullText(BibEntry entry) throws IOException { Optional doi = entry.getField(StandardField.DOI).flatMap(DOI::parse); if (doi.isPresent() && doi.get().getDOI().startsWith(IEEE_DOI) && doi.get().getExternalURI().isPresent()) { // Download the HTML page from IEEE - URLDownload urlDownload = new URLDownload(doi.get().getExternalURI().get().toURL()); + URLDownload urlDownload = null; + try { + urlDownload = new URLDownload(doi.get().getExternalURI().get().toURL()); + } catch (MalformedURLException e) { + throw new FetcherException("Malformed URL", e); + } // We don't need to modify the cookies, but we need support for them urlDownload.getCookieFromUrl(); @@ -181,7 +186,12 @@ public Optional findFullText(BibEntry entry) throws IOException { } // Download the HTML page containing a frame with the PDF - URLDownload urlDownload = new URLDownload(BASE_URL + stampString); + URLDownload urlDownload; + try { + urlDownload = new URLDownload(BASE_URL + stampString); + } catch (MalformedURLException e) { + throw new FetcherException("Malformed URL", e); + } // We don't need to modify the cookies, but we need support for them urlDownload.getCookieFromUrl(); @@ -191,7 +201,13 @@ public Optional findFullText(BibEntry entry) throws IOException { if (matcher.find()) { // The PDF was found LOGGER.debug("Full text document found on IEEE Xplore"); - return Optional.of(new URL(matcher.group(1))); + URL value; + try { + value = new URL(matcher.group(1)); + } catch (MalformedURLException e) { + throw new FetcherException("Malformed URL", e); + } + return Optional.of(value); } return Optional.empty(); } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/INSPIREFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/INSPIREFetcher.java index 696b423f5e3..8038ea2edd0 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/INSPIREFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/INSPIREFetcher.java @@ -1,6 +1,5 @@ package org.jabref.logic.importer.fetcher; -import java.io.IOException; import java.net.MalformedURLException; import java.net.URI; import java.net.URISyntaxException; @@ -112,7 +111,7 @@ public List performSearch(BibEntry entry) throws FetcherException { List results = getParser().parseEntries(download.asInputStream()); results.forEach(this::doPostCleanup); return results; - } catch (IOException | ParseException e) { + } catch (ParseException e) { throw new FetcherException(url, e); } } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/JstorFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/JstorFetcher.java index c381a6ada2f..eb54b87dff0 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/JstorFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/JstorFetcher.java @@ -12,6 +12,7 @@ import java.util.Optional; import java.util.stream.Collectors; +import org.jabref.logic.importer.FetcherException; import org.jabref.logic.importer.FulltextFetcher; import org.jabref.logic.importer.IdBasedParserFetcher; import org.jabref.logic.importer.ImportFormatPreferences; @@ -81,7 +82,8 @@ public Parser getParser() { if (text.startsWith("@")) { return parser.parseEntries(text); } - // input stream contains html + // otherwise: input stream contains html + List entries; try { Document doc = Jsoup.parse(inputStream, null, HOST); @@ -95,7 +97,7 @@ public Parser getParser() { stringBuilder.append(data); } entries = new ArrayList<>(parser.parseEntries(stringBuilder.toString())); - } catch (IOException e) { + } catch (IOException | FetcherException e) { throw new ParseException("Could not download data from jstor.org", e); } return entries; @@ -108,7 +110,7 @@ public String getName() { } @Override - public Optional findFullText(BibEntry entry) throws IOException { + public Optional findFullText(BibEntry entry) throws FetcherException, IOException { if (entry.getField(StandardField.URL).isEmpty()) { return Optional.empty(); } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/ResearchGate.java b/src/main/java/org/jabref/logic/importer/fetcher/ResearchGate.java index 883722d8296..d9c9011a822 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/ResearchGate.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/ResearchGate.java @@ -116,7 +116,7 @@ private Document getHTML(BibEntry entry) throws FetcherException, IOException { throw new FetcherException("Could not find a pdf"); } - Optional getURLByString(String query) throws IOException, NullPointerException { + Optional getURLByString(String query) throws IOException, FetcherException { URIBuilder source; String link; try { diff --git a/src/main/java/org/jabref/logic/importer/util/ShortDOIService.java b/src/main/java/org/jabref/logic/importer/util/ShortDOIService.java index f10996f9b4f..e9c6446fc19 100644 --- a/src/main/java/org/jabref/logic/importer/util/ShortDOIService.java +++ b/src/main/java/org/jabref/logic/importer/util/ShortDOIService.java @@ -1,11 +1,11 @@ package org.jabref.logic.importer.util; -import java.io.IOException; import java.net.MalformedURLException; import java.net.URI; import java.net.URISyntaxException; import java.net.URL; +import org.jabref.logic.importer.FetcherException; import org.jabref.logic.importer.ParseException; import org.jabref.logic.net.URLDownload; import org.jabref.model.entry.identifier.DOI; @@ -36,8 +36,8 @@ public DOI getShortDOI(DOI doi) throws ShortDOIServiceException { private JSONObject makeRequest(DOI doi) throws ShortDOIServiceException { - URIBuilder uriBuilder = null; - URL url = null; + URIBuilder uriBuilder; + URL url; try { uriBuilder = new URIBuilder(BASIC_URL); @@ -58,7 +58,7 @@ private JSONObject makeRequest(DOI doi) throws ShortDOIServiceException { throw new ShortDOIServiceException("Cannot get short DOI"); } return resultAsJSON; - } catch (ParseException | IOException | JSONException e) { + } catch (ParseException | JSONException | FetcherException e) { throw new ShortDOIServiceException("Cannot get short DOI", e); } } diff --git a/src/main/java/org/jabref/logic/net/URLDownload.java b/src/main/java/org/jabref/logic/net/URLDownload.java index c5e312279bd..0394bda31b1 100644 --- a/src/main/java/org/jabref/logic/net/URLDownload.java +++ b/src/main/java/org/jabref/logic/net/URLDownload.java @@ -2,7 +2,6 @@ import java.io.BufferedInputStream; import java.io.BufferedReader; -import java.io.ByteArrayInputStream; import java.io.DataOutputStream; import java.io.IOException; import java.io.InputStream; @@ -42,6 +41,7 @@ import org.jabref.http.dto.SimpleHttpResponse; import org.jabref.logic.importer.FetcherClientException; +import org.jabref.logic.importer.FetcherException; import org.jabref.logic.importer.FetcherServerException; import org.jabref.logic.util.io.FileUtil; @@ -241,7 +241,7 @@ public void setPostData(String postData) { * * @return the downloaded string */ - public String asString() throws IOException { + public String asString() throws FetcherException { return asString(StandardCharsets.UTF_8, this.openConnection()); } @@ -251,7 +251,7 @@ public String asString() throws IOException { * @param encoding the desired String encoding * @return the downloaded string */ - public String asString(Charset encoding) throws IOException { + public String asString(Charset encoding) throws FetcherException { return asString(encoding, this.openConnection()); } @@ -261,7 +261,7 @@ public String asString(Charset encoding) throws IOException { * @param existingConnection an existing connection * @return the downloaded string */ - public static String asString(URLConnection existingConnection) throws IOException { + public static String asString(URLConnection existingConnection) throws FetcherException { return asString(StandardCharsets.UTF_8, existingConnection); } @@ -272,16 +272,17 @@ public static String asString(URLConnection existingConnection) throws IOExcepti * @param connection an existing connection * @return the downloaded string */ - public static String asString(Charset encoding, URLConnection connection) throws IOException { - + public static String asString(Charset encoding, URLConnection connection) throws FetcherException { try (InputStream input = new BufferedInputStream(connection.getInputStream()); Writer output = new StringWriter()) { copy(input, output, encoding); return output.toString(); + } catch (IOException e) { + throw new FetcherException("Error downloading", e); } } - public List getCookieFromUrl() throws IOException { + public List getCookieFromUrl() throws FetcherException { CookieManager cookieManager = new CookieManager(); CookieHandler.setDefault(cookieManager); cookieManager.setCookiePolicy(CookiePolicy.ACCEPT_ALL); @@ -302,27 +303,41 @@ public List getCookieFromUrl() throws IOException { * * @param destination the destination file path. */ - public void toFile(Path destination) throws IOException { + public void toFile(Path destination) throws FetcherException { try (InputStream input = new BufferedInputStream(this.openConnection().getInputStream())) { Files.copy(input, destination, StandardCopyOption.REPLACE_EXISTING); } catch (IOException e) { LOGGER.warn("Could not copy input", e); - throw e; + throw new FetcherException("Could not copy input", e); } } /** * Takes the web resource as the source for a monitored input stream. */ - public ProgressInputStream asInputStream() throws IOException { + public ProgressInputStream asInputStream() throws FetcherException { HttpURLConnection urlConnection = (HttpURLConnection) this.openConnection(); - if ((urlConnection.getResponseCode() == HttpURLConnection.HTTP_NOT_FOUND) || (urlConnection.getResponseCode() == HttpURLConnection.HTTP_BAD_REQUEST)) { - LOGGER.error("Response message {} returned for url {}", urlConnection.getResponseMessage(), urlConnection.getURL()); - return new ProgressInputStream(new ByteArrayInputStream(new byte[0]), 0); + int responseCode; + try { + responseCode = urlConnection.getResponseCode(); + } catch (IOException e) { + throw new FetcherException("Error getting response code", e); + } + LOGGER.debug("Response code: {}", responseCode); // We could check for != 200, != 204 + if (responseCode >= 300) { + SimpleHttpResponse simpleHttpResponse = new SimpleHttpResponse(urlConnection); + LOGGER.error("Failed to read from url: {}", simpleHttpResponse); + throw FetcherException.of(this.source, simpleHttpResponse); } long fileSize = urlConnection.getContentLengthLong(); - return new ProgressInputStream(new BufferedInputStream(urlConnection.getInputStream()), fileSize); + InputStream inputStream; + try { + inputStream = urlConnection.getInputStream(); + } catch (IOException e) { + throw new FetcherException("Error getting input stream", e); + } + return new ProgressInputStream(new BufferedInputStream(inputStream), fileSize); } /** @@ -330,7 +345,7 @@ public ProgressInputStream asInputStream() throws IOException { * * @return the path of the temporary file. */ - public Path toTemporaryFile() throws IOException { + public Path toTemporaryFile() throws FetcherException { // Determine file name and extension from source url String sourcePath = source.getPath(); @@ -340,7 +355,12 @@ public Path toTemporaryFile() throws IOException { String extension = "." + FileUtil.getFileExtension(fileNameWithExtension).orElse("tmp"); // Create temporary file and download to it - Path file = Files.createTempFile(fileName, extension); + Path file = null; + try { + file = Files.createTempFile(fileName, extension); + } catch (IOException e) { + throw new FetcherException("Could not create temporary file", e); + } file.toFile().deleteOnExit(); toFile(file); @@ -370,8 +390,13 @@ private static void copy(InputStream in, Writer out, Charset encoding) throws IO * * @return an open connection */ - public URLConnection openConnection() throws IOException { - URLConnection connection = getUrlConnection(); + public URLConnection openConnection() throws FetcherException { + URLConnection connection; + try { + connection = getUrlConnection(); + } catch (IOException e) { + throw new FetcherException("Error opening connection", e); + } connection.setConnectTimeout((int) connectTimeout.toMillis()); for (Entry entry : this.parameters.entrySet()) { connection.setRequestProperty(entry.getKey(), entry.getValue()); @@ -380,6 +405,8 @@ public URLConnection openConnection() throws IOException { connection.setDoOutput(true); try (DataOutputStream wr = new DataOutputStream(connection.getOutputStream())) { wr.writeBytes(this.postData); + } catch (IOException e) { + throw new FetcherException("Could not write output", e); } } @@ -390,10 +417,7 @@ public URLConnection openConnection() throws IOException { status = httpURLConnection.getResponseCode(); } catch (IOException e) { LOGGER.error("Error getting response code", e); - // TODO: Convert e to FetcherClientException - // Same TODO as in org.jabref.logic.importer.EntryBasedParserFetcher.performSearch. - // Maybe do this in org.jabref.logic.importer.FetcherException.getLocalizedMessage - throw e; + throw new FetcherException("Error getting response code", e); } if ((status == HttpURLConnection.HTTP_MOVED_TEMP) @@ -402,14 +426,18 @@ public URLConnection openConnection() throws IOException { // get redirect url from "location" header field String newUrl = connection.getHeaderField("location"); // open the new connection again - connection = new URLDownload(newUrl).openConnection(); + try { + connection = new URLDownload(newUrl).openConnection(); + } catch (MalformedURLException e) { + throw new FetcherException("Could not open URL Download", e); + } } else { SimpleHttpResponse httpResponse = new SimpleHttpResponse(httpURLConnection); LOGGER.info("{}", httpResponse); if ((status >= 400) && (status < 500)) { - throw new IOException(new FetcherClientException(this.source, httpResponse)); + throw new FetcherClientException(this.source, httpResponse); } else if (status >= 500) { - throw new IOException(new FetcherServerException(this.source, httpResponse)); + throw new FetcherServerException(this.source, httpResponse); } } } diff --git a/src/main/resources/l10n/JabRef_en.properties b/src/main/resources/l10n/JabRef_en.properties index 5a4ef4ebcdd..8d543167ddd 100644 --- a/src/main/resources/l10n/JabRef_en.properties +++ b/src/main/resources/l10n/JabRef_en.properties @@ -2614,11 +2614,11 @@ Illegal\ characters\ in\ the\ file\ name\ detected.\nFile\ will\ be\ renamed\ to Rename\ and\ add=Rename and add Failed\ to\ download\ from\ URL=Failed to download from URL -401\ Unauthorized\:\ Access\ Denied.\ You\ are\ not\ authorized\ to\ access\ this\ resource.\ Please\ check\ your\ credentials\ and\ try\ again.\ If\ you\ believe\ you\ should\ have\ access,\ please\ contact\ the\ administrator\ for\ assistance.=401 Unauthorized: Access Denied. You are not authorized to access this resource. Please check your credentials and try again. If you believe you should have access, please contact the administrator for assistance. -403\ Forbidden\:\ Access\ Denied.\ You\ do\ not\ have\ permission\ to\ access\ this\ resource.\ Please\ contact\ the\ administrator\ for\ assistance\ or\ try\ a\ different\ action.=403 Forbidden: Access Denied. You do not have permission to access this resource. Please contact the administrator for assistance or try a different action. -404\ Not\ Found\ Error\:\ The\ requested\ resource\ could\ not\ be\ found.\ It\ seems\ that\ the\ file\ you\ are\ trying\ to\ download\ is\ not\ available\ or\ has\ been\ moved.\ Please\ verify\ the\ URL\ and\ try\ again.\ If\ you\ believe\ this\ is\ an\ error,\ please\ contact\ the\ administrator\ for\ further\ assistance.=404 Not Found Error: The requested resource could not be found. It seems that the file you are trying to download is not available or has been moved. Please verify the URL and try again. If you believe this is an error, please contact the administrator for further assistance. -Error\ downloading\ from\ URL.\ Cause\ is\ likely\ the\ server\ side.\nPlease\ try\ again\ later\ or\ contact\ the\ server\ administrator.=Error downloading from URL. Cause is likely the server side.\nPlease try again later or contact the server administrator. +Access\ denied.\ You\ are\ not\ authorized\ to\ access\ this\ resource.\ Please\ check\ your\ credentials\ and\ try\ again.\ If\ you\ believe\ you\ should\ have\ access,\ please\ contact\ the\ administrator\ for\ assistance.=Access denied. You are not authorized to access this resource. Please check your credentials and try again. If you believe you should have access, please contact the administrator for assistance. +Access\ denied.\ You\ do\ not\ have\ permission\ to\ access\ this\ resource.\ Please\ contact\ the\ administrator\ for\ assistance\ or\ try\ a\ different\ action.=Access denied. You do not have permission to access this resource. Please contact the administrator for assistance or try a different action. +The\ requested\ resource\ could\ not\ be\ found.\ It\ seems\ that\ the\ file\ you\ are\ trying\ to\ download\ is\ not\ available\ or\ has\ been\ moved.\ Please\ verify\ the\ URL\ and\ try\ again.\ If\ you\ believe\ this\ is\ an\ error,\ please\ contact\ the\ administrator\ for\ further\ assistance.=The requested resource could not be found. It seems that the file you are trying to download is not available or has been moved. Please verify the URL and try again. If you believe this is an error, please contact the administrator for further assistance. Something\ is\ wrong\ on\ JabRef\ side.\ Please\ check\ the\ URL\ and\ try\ again.=Something is wrong on JabRef side. Please check the URL and try again. +Error\ downloading\ from\ URL.\ Cause\ is\ likely\ the\ server\ side.\nPlease\ try\ again\ later\ or\ contact\ the\ server\ administrator.=Error downloading from URL. Cause is likely the server side.\nPlease try again later or contact the server administrator. URL\:\ %0=URL: %0 URL\:\ %0\nHTTP\ %1\ %2\n%3=URL: %0\nHTTP %1 %2\n%3 Please\ check\ the\ URL\ and\ try\ again.\nURL\:\ %0\nDetails\:\ %1=Please check the URL and try again.\nURL: %0\nDetails: %1 diff --git a/src/test/java/org/jabref/logic/importer/fetcher/GoogleScholarTest.java b/src/test/java/org/jabref/logic/importer/fetcher/GoogleScholarTest.java index 4432e20a19b..a3a96315a11 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/GoogleScholarTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/GoogleScholarTest.java @@ -38,7 +38,7 @@ void setUp() { } @Test - void linkFound() throws IOException, FetcherException { + void linkFound() throws Exception { entry.setField(StandardField.TITLE, "Towards Application Portability in Platform as a Service"); assertEquals( diff --git a/src/test/java/org/jabref/logic/importer/fetcher/ResearchGateTest.java b/src/test/java/org/jabref/logic/importer/fetcher/ResearchGateTest.java index 5fdccc287c6..8ccc6f6f1af 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/ResearchGateTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/ResearchGateTest.java @@ -56,7 +56,7 @@ void fullTextNotFoundByDOI() throws IOException, FetcherException { } @Test - void getDocumentByTitle() throws IOException, NullPointerException { + void getDocumentByTitle() throws Exception { Optional source = fetcher.getURLByString(entry.getTitle().get()); assertTrue(source.isPresent() && source.get().startsWith(URL_PAGE)); } diff --git a/src/test/java/org/jabref/logic/importer/fileformat/ACMPortalParserTest.java b/src/test/java/org/jabref/logic/importer/fileformat/ACMPortalParserTest.java index fc1672af602..d9f641ab855 100644 --- a/src/test/java/org/jabref/logic/importer/fileformat/ACMPortalParserTest.java +++ b/src/test/java/org/jabref/logic/importer/fileformat/ACMPortalParserTest.java @@ -1,6 +1,5 @@ package org.jabref.logic.importer.fileformat; -import java.io.IOException; import java.net.CookieHandler; import java.net.CookieManager; import java.net.MalformedURLException; @@ -11,7 +10,6 @@ import java.util.Optional; import org.jabref.logic.importer.FetcherException; -import org.jabref.logic.importer.ParseException; import org.jabref.logic.net.URLDownload; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.field.StandardField; @@ -69,7 +67,7 @@ void setUp() throws URISyntaxException, MalformedURLException { } @Test - void parseEntries() throws IOException, ParseException { + void parseEntries() throws Exception { CookieHandler.setDefault(new CookieManager()); List bibEntries = parser.parseEntries(new URLDownload(searchUrl).asInputStream()); for (BibEntry bibEntry : bibEntries) { @@ -79,7 +77,7 @@ void parseEntries() throws IOException, ParseException { } @Test - void parseDoiSearchPage() throws ParseException, IOException { + void parseDoiSearchPage() throws Exception { String testDoi = "10.1145/3129790.3129810"; CookieHandler.setDefault(new CookieManager()); List doiList = parser.parseDoiSearchPage(new URLDownload(searchUrl).asInputStream()); @@ -139,7 +137,7 @@ void parseBibEntryWithFamilyAuthorOnly() { } @Test - void noEntryFound() throws URISyntaxException, IOException, ParseException { + void noEntryFound() throws Exception { CookieHandler.setDefault(new CookieManager()); URL url = new URIBuilder("https://dl.acm.org/action/doSearch?AllField=10.1145/3129790.31298").build().toURL(); List bibEntries = parser.parseEntries(new URLDownload(url).asInputStream()); diff --git a/src/test/java/org/jabref/logic/net/URLDownloadTest.java b/src/test/java/org/jabref/logic/net/URLDownloadTest.java index 926d5c67c38..2c9e41b2b11 100644 --- a/src/test/java/org/jabref/logic/net/URLDownloadTest.java +++ b/src/test/java/org/jabref/logic/net/URLDownloadTest.java @@ -28,21 +28,21 @@ public class URLDownloadTest { private static final Logger LOGGER = LoggerFactory.getLogger(URLDownloadTest.class); @Test - public void stringDownloadWithSetEncoding() throws IOException { + public void stringDownloadWithSetEncoding() throws Exception { URLDownload dl = new URLDownload(new URL("http://www.google.com")); assertTrue(dl.asString().contains("Google"), "google.com should contain google"); } @Test - public void stringDownload() throws IOException { + public void stringDownload() throws Exception { URLDownload dl = new URLDownload(new URL("http://www.google.com")); assertTrue(dl.asString(StandardCharsets.UTF_8).contains("Google"), "google.com should contain google"); } @Test - public void fileDownload() throws IOException { + public void fileDownload() throws Exception { File destination = File.createTempFile("jabref-test", ".html"); try { URLDownload dl = new URLDownload(new URL("http://www.google.com")); @@ -57,14 +57,14 @@ public void fileDownload() throws IOException { } @Test - public void determineMimeType() throws IOException { + public void determineMimeType() throws Exception { URLDownload dl = new URLDownload(new URL("http://www.google.com")); assertTrue(dl.getMimeType().startsWith("text/html")); } @Test - public void downloadToTemporaryFilePathWithoutFileSavesAsTmpFile() throws IOException { + public void downloadToTemporaryFilePathWithoutFileSavesAsTmpFile() throws Exception { URLDownload google = new URLDownload(new URL("http://www.google.com")); String path = google.toTemporaryFile().toString(); @@ -72,7 +72,7 @@ public void downloadToTemporaryFilePathWithoutFileSavesAsTmpFile() throws IOExce } @Test - public void downloadToTemporaryFileKeepsName() throws IOException { + public void downloadToTemporaryFileKeepsName() throws Exception { URLDownload google = new URLDownload(new URL("https://github.com/JabRef/jabref/blob/main/LICENSE")); String path = google.toTemporaryFile().toString(); @@ -81,7 +81,7 @@ public void downloadToTemporaryFileKeepsName() throws IOException { @Test @DisabledOnCIServer("CI Server is apparently blocked") - public void downloadOfFTPSucceeds() throws IOException { + public void downloadOfFTPSucceeds() throws Exception { URLDownload ftp = new URLDownload(new URL("ftp://ftp.informatik.uni-stuttgart.de/pub/library/ncstrl.ustuttgart_fi/INPROC-2016-15/INPROC-2016-15.pdf")); Path path = ftp.toTemporaryFile(); @@ -89,7 +89,7 @@ public void downloadOfFTPSucceeds() throws IOException { } @Test - public void downloadOfHttpSucceeds() throws IOException { + public void downloadOfHttpSucceeds() throws Exception { URLDownload ftp = new URLDownload(new URL("http://www.jabref.org")); Path path = ftp.toTemporaryFile(); @@ -97,7 +97,7 @@ public void downloadOfHttpSucceeds() throws IOException { } @Test - public void downloadOfHttpsSucceeds() throws IOException { + public void downloadOfHttpsSucceeds() throws Exception { URLDownload ftp = new URLDownload(new URL("https://www.jabref.org")); Path path = ftp.toTemporaryFile(); From 836b997a0a5efca3d45d687d7e7f8e799b4755b8 Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Mon, 12 Aug 2024 02:16:19 +0200 Subject: [PATCH 29/42] Add some debug --- .../org/jabref/logic/importer/FetcherException.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/main/java/org/jabref/logic/importer/FetcherException.java b/src/main/java/org/jabref/logic/importer/FetcherException.java index 3340258ab79..03b762c2783 100644 --- a/src/main/java/org/jabref/logic/importer/FetcherException.java +++ b/src/main/java/org/jabref/logic/importer/FetcherException.java @@ -1,5 +1,6 @@ package org.jabref.logic.importer; +import java.io.IOException; import java.net.URL; import java.util.Optional; import java.util.regex.Pattern; @@ -9,7 +10,11 @@ import org.jabref.logic.l10n.Localization; import org.jabref.model.strings.StringUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + public class FetcherException extends JabRefException { + private static final Logger LOGGER = LoggerFactory.getLogger(FetcherException.class); Pattern API_KEY_PATTERN = Pattern.compile("(?i)(api|key|api[-_]?key)=[^&]*"); @@ -63,7 +68,13 @@ public FetcherException(String errorMessage, Throwable cause) { // TODO: Convert IOException e to FetcherClientException // Same TODO as in org.jabref.logic.importer.EntryBasedParserFetcher.performSearch. // Maybe do this in org.jabref.logic.importer.FetcherException.getLocalizedMessage + + // Idea (from org.jabref.logic.importer.fetcher.GoogleScholar.performSearchPaged) + // if (e.getMessage().contains("Server returned HTTP response code: 503 for URL")) { super(errorMessage, cause); + if (LOGGER.isDebugEnabled() && (cause instanceof IOException)) { + LOGGER.debug("I/O exception", cause); + } httpResponse = null; url = null; } From 89c845dee5ff7b289b001a35fee26640830cb241 Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Mon, 12 Aug 2024 02:22:44 +0200 Subject: [PATCH 30/42] Keep response body --- src/main/java/org/jabref/logic/net/URLDownload.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/jabref/logic/net/URLDownload.java b/src/main/java/org/jabref/logic/net/URLDownload.java index 0394bda31b1..7dd4503d47a 100644 --- a/src/main/java/org/jabref/logic/net/URLDownload.java +++ b/src/main/java/org/jabref/logic/net/URLDownload.java @@ -431,7 +431,8 @@ public URLConnection openConnection() throws FetcherException { } catch (MalformedURLException e) { throw new FetcherException("Could not open URL Download", e); } - } else { + } else if (status >= 400) { + // in case of an error, propagate the error message SimpleHttpResponse httpResponse = new SimpleHttpResponse(httpURLConnection); LOGGER.info("{}", httpResponse); if ((status >= 400) && (status < 500)) { From ca3b01cb454bf2ed3b93ea376b86f14d37487871 Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Mon, 12 Aug 2024 18:42:06 +0200 Subject: [PATCH 31/42] Move default implementation to "real" class --- .../java/org/jabref/gui/DialogService.java | 38 +----------------- .../org/jabref/gui/JabRefDialogService.java | 40 +++++++++++++++++++ 2 files changed, 42 insertions(+), 36 deletions(-) diff --git a/src/main/java/org/jabref/gui/DialogService.java b/src/main/java/org/jabref/gui/DialogService.java index a0a48c5ee87..5a6cc7d7919 100644 --- a/src/main/java/org/jabref/gui/DialogService.java +++ b/src/main/java/org/jabref/gui/DialogService.java @@ -20,11 +20,7 @@ import org.jabref.gui.util.BaseDialog; import org.jabref.gui.util.DirectoryDialogConfiguration; import org.jabref.gui.util.FileDialogConfiguration; -import org.jabref.http.dto.SimpleHttpResponse; -import org.jabref.logic.importer.FetcherClientException; import org.jabref.logic.importer.FetcherException; -import org.jabref.logic.importer.FetcherServerException; -import org.jabref.logic.l10n.Localization; import org.controlsfx.control.textfield.CustomPasswordField; import org.controlsfx.dialog.ProgressDialog; @@ -101,39 +97,9 @@ default Optional showEditableChoiceDialogAndWait(String title, String con * * @param exception the exception causing the error */ - default void showErrorDialogAndWait(Exception exception) { - if (exception instanceof FetcherException) { - // Somehow, Java does not route correctly to the other method - showErrorDialogAndWait((FetcherException) exception); - } else { - showErrorDialogAndWait(Localization.lang("Unhandled exception occurred."), exception); - } - } + void showErrorDialogAndWait(Exception exception); - default void showErrorDialogAndWait(FetcherException fetcherException) { - String failedTitle = Localization.lang("Failed to download from URL"); - String localizedMessage = fetcherException.getLocalizedMessage(); - Optional httpResponse = fetcherException.getHttpResponse(); - if (httpResponse.isPresent()) { - int statusCode = httpResponse.get().statusCode(); - if (statusCode == 401) { - this.showInformationDialogAndWait(failedTitle, Localization.lang("Access denied. You are not authorized to access this resource. Please check your credentials and try again. If you believe you should have access, please contact the administrator for assistance.") + "\n\n" + localizedMessage); - } else if (statusCode == 403) { - this.showInformationDialogAndWait(failedTitle, Localization.lang("Access denied. You do not have permission to access this resource. Please contact the administrator for assistance or try a different action.") + "\n\n" + localizedMessage); - } else if (statusCode == 404) { - this.showInformationDialogAndWait(failedTitle, Localization.lang("The requested resource could not be found. It seems that the file you are trying to download is not available or has been moved. Please verify the URL and try again. If you believe this is an error, please contact the administrator for further assistance.") + "\n\n" + localizedMessage); - } else { - this.showErrorDialogAndWait(failedTitle, Localization.lang("Something is wrong on JabRef side. Please check the URL and try again.") + "\n\n" + localizedMessage); - } - } else if (fetcherException instanceof FetcherClientException) { - this.showErrorDialogAndWait(failedTitle, Localization.lang("Something is wrong on JabRef side. Please check the URL and try again.") + "\n\n" + localizedMessage); - } else if (fetcherException instanceof FetcherServerException) { - this.showInformationDialogAndWait(failedTitle, - Localization.lang("Error downloading from URL. Cause is likely the server side.\nPlease try again later or contact the server administrator.") + "\n\n" + localizedMessage); - } else { - this.showErrorDialogAndWait(failedTitle, localizedMessage); - } - } + void showErrorDialogAndWait(FetcherException fetcherException); /** * Create and display error dialog displaying the given exception. diff --git a/src/main/java/org/jabref/gui/JabRefDialogService.java b/src/main/java/org/jabref/gui/JabRefDialogService.java index 1e572069602..f50820c1c8e 100644 --- a/src/main/java/org/jabref/gui/JabRefDialogService.java +++ b/src/main/java/org/jabref/gui/JabRefDialogService.java @@ -47,6 +47,10 @@ import org.jabref.gui.util.FileDialogConfiguration; import org.jabref.gui.util.UiTaskExecutor; import org.jabref.gui.util.ZipFileChooser; +import org.jabref.http.dto.SimpleHttpResponse; +import org.jabref.logic.importer.FetcherClientException; +import org.jabref.logic.importer.FetcherException; +import org.jabref.logic.importer.FetcherServerException; import org.jabref.logic.l10n.Localization; import com.tobiasdiez.easybind.EasyBind; @@ -205,6 +209,42 @@ public void showErrorDialogAndWait(String message, Throwable exception) { exceptionDialog.showAndWait(); } + @Override + public void showErrorDialogAndWait(Exception exception) { + if (exception instanceof FetcherException) { + // Somehow, Java does not route correctly to the other method + showErrorDialogAndWait((FetcherException) exception); + } else { + showErrorDialogAndWait(Localization.lang("Unhandled exception occurred."), exception); + } + } + + @Override + public void showErrorDialogAndWait(FetcherException fetcherException) { + String failedTitle = Localization.lang("Failed to download from URL"); + String localizedMessage = fetcherException.getLocalizedMessage(); + Optional httpResponse = fetcherException.getHttpResponse(); + if (httpResponse.isPresent()) { + int statusCode = httpResponse.get().statusCode(); + if (statusCode == 401) { + this.showInformationDialogAndWait(failedTitle, Localization.lang("Access denied. You are not authorized to access this resource. Please check your credentials and try again. If you believe you should have access, please contact the administrator for assistance.") + "\n\n" + localizedMessage); + } else if (statusCode == 403) { + this.showInformationDialogAndWait(failedTitle, Localization.lang("Access denied. You do not have permission to access this resource. Please contact the administrator for assistance or try a different action.") + "\n\n" + localizedMessage); + } else if (statusCode == 404) { + this.showInformationDialogAndWait(failedTitle, Localization.lang("The requested resource could not be found. It seems that the file you are trying to download is not available or has been moved. Please verify the URL and try again. If you believe this is an error, please contact the administrator for further assistance.") + "\n\n" + localizedMessage); + } else { + this.showErrorDialogAndWait(failedTitle, Localization.lang("Something is wrong on JabRef side. Please check the URL and try again.") + "\n\n" + localizedMessage); + } + } else if (fetcherException instanceof FetcherClientException) { + this.showErrorDialogAndWait(failedTitle, Localization.lang("Something is wrong on JabRef side. Please check the URL and try again.") + "\n\n" + localizedMessage); + } else if (fetcherException instanceof FetcherServerException) { + this.showInformationDialogAndWait(failedTitle, + Localization.lang("Error downloading from URL. Cause is likely the server side.\nPlease try again later or contact the server administrator.") + "\n\n" + localizedMessage); + } else { + this.showErrorDialogAndWait(failedTitle, localizedMessage); + } + } + @Override public void showErrorDialogAndWait(String title, String content, Throwable exception) { ExceptionDialog exceptionDialog = new ExceptionDialog(exception); From ded5c56331710c94775a3f8e7366abb4866fcee9 Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Mon, 12 Aug 2024 19:01:32 +0200 Subject: [PATCH 32/42] Keep URL and HTTP untranslated --- .../java/org/jabref/logic/importer/FetcherException.java | 6 +++--- src/main/resources/l10n/JabRef_en.properties | 2 -- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/jabref/logic/importer/FetcherException.java b/src/main/java/org/jabref/logic/importer/FetcherException.java index 03b762c2783..005795e928e 100644 --- a/src/main/java/org/jabref/logic/importer/FetcherException.java +++ b/src/main/java/org/jabref/logic/importer/FetcherException.java @@ -7,7 +7,6 @@ import org.jabref.http.dto.SimpleHttpResponse; import org.jabref.logic.JabRefException; -import org.jabref.logic.l10n.Localization; import org.jabref.model.strings.StringUtil; import org.slf4j.Logger; @@ -117,9 +116,10 @@ public String getLocalizedMessage() { // TODO: 5% of the "new-ers" pass a "real" localized message. See org.jabref.logic.importer.fetcher.GoogleScholar.addHitsFromQuery. We should maybe make use of this (and rewrite the whole message handling) // TODO: Try to convert IOException to some more meaningful information here (or at org.jabref.gui.DialogService.showErrorDialogAndWait(org.jabref.logic.importer.FetcherException)). See also org.jabref.logic.net.URLDownload.openConnection if (httpResponse != null) { - return getPrefix() + Localization.lang("URL: %0\nHTTP %1 %2\n%3", getRedactedUrl(), httpResponse.statusCode(), httpResponse.responseMessage(), httpResponse.responseBody()); + // We decided to not "translate" technical terms (URL, HTTP) + return getPrefix() + String.format("URL: %s\nHTTP %d %s\n%s", getRedactedUrl(), httpResponse.statusCode(), httpResponse.responseMessage(), httpResponse.responseBody()); } else if (url != null) { - return getPrefix() + Localization.lang("URL: %0", getRedactedUrl()); + return getPrefix() + "URL: %s".format(getRedactedUrl()); } else { return super.getLocalizedMessage(); } diff --git a/src/main/resources/l10n/JabRef_en.properties b/src/main/resources/l10n/JabRef_en.properties index 8d543167ddd..59ec8e76926 100644 --- a/src/main/resources/l10n/JabRef_en.properties +++ b/src/main/resources/l10n/JabRef_en.properties @@ -2619,8 +2619,6 @@ Access\ denied.\ You\ do\ not\ have\ permission\ to\ access\ this\ resource.\ Pl The\ requested\ resource\ could\ not\ be\ found.\ It\ seems\ that\ the\ file\ you\ are\ trying\ to\ download\ is\ not\ available\ or\ has\ been\ moved.\ Please\ verify\ the\ URL\ and\ try\ again.\ If\ you\ believe\ this\ is\ an\ error,\ please\ contact\ the\ administrator\ for\ further\ assistance.=The requested resource could not be found. It seems that the file you are trying to download is not available or has been moved. Please verify the URL and try again. If you believe this is an error, please contact the administrator for further assistance. Something\ is\ wrong\ on\ JabRef\ side.\ Please\ check\ the\ URL\ and\ try\ again.=Something is wrong on JabRef side. Please check the URL and try again. Error\ downloading\ from\ URL.\ Cause\ is\ likely\ the\ server\ side.\nPlease\ try\ again\ later\ or\ contact\ the\ server\ administrator.=Error downloading from URL. Cause is likely the server side.\nPlease try again later or contact the server administrator. -URL\:\ %0=URL: %0 -URL\:\ %0\nHTTP\ %1\ %2\n%3=URL: %0\nHTTP %1 %2\n%3 Please\ check\ the\ URL\ and\ try\ again.\nURL\:\ %0\nDetails\:\ %1=Please check the URL and try again.\nURL: %0\nDetails: %1 Finished=Finished From cb350b0a6237f1c5f78c53a5254b6b26281399b9 Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Mon, 12 Aug 2024 19:18:56 +0200 Subject: [PATCH 33/42] Delete 2x "ofUrl" --- .../logic/importer/FetcherException.java | 8 ---- .../importer/fetcher/IacrEprintFetcher.java | 2 +- .../logic/importer/fetcher/MrDLibFetcher.java | 48 +++++++++---------- .../logic/importer/fetcher/ResearchGate.java | 18 +++---- 4 files changed, 33 insertions(+), 43 deletions(-) diff --git a/src/main/java/org/jabref/logic/importer/FetcherException.java b/src/main/java/org/jabref/logic/importer/FetcherException.java index 005795e928e..dfe847bd432 100644 --- a/src/main/java/org/jabref/logic/importer/FetcherException.java +++ b/src/main/java/org/jabref/logic/importer/FetcherException.java @@ -90,14 +90,6 @@ public FetcherException(String errorMessage, String localizedMessage, Throwable url = null; } - public static FetcherException ofUrl(String url, Throwable cause) { - return new FetcherException(url, cause, null); - } - - public static FetcherException ofUrl(String url, String errorMessage) { - return new FetcherException(url, errorMessage, null, null); - } - public static FetcherException ofUrl(String url, String errorMessage, Throwable cause) { return new FetcherException(url, errorMessage, cause, null); } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/IacrEprintFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/IacrEprintFetcher.java index 0a58be73d71..35c0ba1bbe9 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/IacrEprintFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/IacrEprintFetcher.java @@ -64,7 +64,7 @@ private Optional createEntryFromIacrCitation(String validIdentifier) t try { return BibtexParser.singleFromString(actualEntry, prefs); } catch (ParseException e) { - throw FetcherException.ofUrl(Localization.lang("Entry from %0 could not be parsed.", "IACR"), e); + throw new FetcherException(Localization.lang("Entry from %0 could not be parsed.", "IACR"), e); } } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/MrDLibFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/MrDLibFetcher.java index 2d1205c6692..ba12ac0d9aa 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/MrDLibFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/MrDLibFetcher.java @@ -1,8 +1,10 @@ package org.jabref.logic.importer.fetcher; import java.io.IOException; +import java.net.MalformedURLException; import java.net.URI; import java.net.URISyntaxException; +import java.net.URL; import java.util.Calendar; import java.util.List; import java.util.Optional; @@ -58,7 +60,14 @@ public List performSearch(BibEntry entry) throws FetcherException { return List.of(); } - String response = makeServerRequest(title.get()); + URL url; + try { + url = constructQuery(title.get()); + } catch (URISyntaxException | MalformedURLException e) { + throw new FetcherException("Invalid URL", e); + } + + String response = makeServerRequest(url); MrDLibImporter importer = new MrDLibImporter(); ParserResult parserResult; try { @@ -75,7 +84,7 @@ public List performSearch(BibEntry entry) throws FetcherException { } } catch (IOException e) { LOGGER.error("Error while fetching", e); - throw FetcherException.ofUrl(constructQuery(title.get()), e); + throw new FetcherException(url, e); } return parserResult.getDatabase().getEntries(); } @@ -91,31 +100,25 @@ public String getDescription() { /** * Contact the server with the title of the selected item * - * @param queryByTitle the query holds the title of the selected entry. Used to make a query to the MDL Server * @return Returns the server response. This is an XML document as a String. */ - private String makeServerRequest(String queryByTitle) throws FetcherException { - String url = constructQuery(queryByTitle); - try { - URLDownload urlDownload = new URLDownload(url); - String response = urlDownload.asString(); + private String makeServerRequest(URL url) throws FetcherException { + URLDownload urlDownload = new URLDownload(url); + String response = urlDownload.asString(); - // Conversion of < and > - response = response.replace(">", ">"); - response = response.replace("<", "<"); - return response; - } catch (IOException e) { - throw FetcherException.ofUrl(url, e); - } + // Conversion of < and > + response = response.replace(">", ">"); + response = response.replace("<", "<"); + return response; } /** * Constructs the query based on title of the BibEntry. Adds statistical stuff to the url. * - * @param queryWithTitle the title of the bib entry. + * @param queryWithTitle the query holds the title of the selected entry. Used to make a query to the MDL Server * @return the string used to make the query at mdl server */ - private String constructQuery(String queryWithTitle) { + private URL constructQuery(String queryWithTitle) throws URISyntaxException, MalformedURLException { // The encoding does not work for / so we convert them by our own queryWithTitle = queryWithTitle.replace("/", " "); URIBuilder builder = new URIBuilder(); @@ -136,13 +139,8 @@ private String constructQuery(String queryWithTitle) { builder.addParameter("timezone", Calendar.getInstance().getTimeZone().getID()); } - try { - URI uri = builder.build(); - LOGGER.trace("Request: {}", uri.toString()); - return uri.toString(); - } catch (URISyntaxException e) { - LOGGER.error("Invalid URI", e); - } - return ""; + URI uri = builder.build(); + LOGGER.trace("Request: {}", uri.toString()); + return uri.toURL(); } } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/ResearchGate.java b/src/main/java/org/jabref/logic/importer/fetcher/ResearchGate.java index d9c9011a822..1f1f0196ed1 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/ResearchGate.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/ResearchGate.java @@ -3,6 +3,7 @@ import java.io.BufferedReader; import java.io.IOException; import java.io.InputStreamReader; +import java.net.MalformedURLException; import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; @@ -169,8 +170,8 @@ Optional getURLByDoi(DOI doi) throws IOException, NullPointerException { return Optional.of(link); } - private Document getPage(String url) throws IOException { - return Jsoup.connect(url) + private Document getPage(URL url) throws IOException { + return Jsoup.connect(url.toString()) .userAgent(URLDownload.USER_AGENT) .referrer("www.google.com") .ignoreHttpErrors(true) @@ -185,13 +186,12 @@ private Document getPage(String url) throws IOException { * @param luceneQuery the search query. * @return A URL that lets us download a .bib file */ - private static String getUrlForQuery(QueryNode luceneQuery) throws URISyntaxException { + private static URL getUrlForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException { String query = new DefaultQueryTransformer().transformLuceneQuery(luceneQuery).orElse(""); URIBuilder source = new URIBuilder(SEARCH); source.addParameter("type", "publication"); source.addParameter("query", query); - String url = source.build().toString(); - return url; + return source.build().toURL(); } @Override @@ -210,10 +210,10 @@ public TrustLevel getTrustLevel() { public List performSearch(QueryNode luceneQuery) throws FetcherException { Document html; - String url; + URL url; try { url = getUrlForQuery(luceneQuery); - } catch (URISyntaxException e) { + } catch (URISyntaxException | MalformedURLException e) { throw new FetcherException("Invalid URL", e); } @@ -221,10 +221,10 @@ public List performSearch(QueryNode luceneQuery) throws FetcherExcepti html = getPage(url); // ResearchGate's server blocks when too many request are made if (!html.getElementsByClass("nova-legacy-v-publication-item__title").hasText()) { - throw FetcherException.ofUrl(url, "ResearchGate server unavailable"); + throw new FetcherException(url, "Required HTML element not found", null); } } catch (IOException e) { - throw new FetcherException("URL is not correct", e); + throw new FetcherException(url, e); } Elements sol = html.getElementsByClass("nova-legacy-v-publication-item__title"); From 7687bbd2382bb73b6aaf3bd322987fd480cb38ca Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Mon, 12 Aug 2024 19:24:05 +0200 Subject: [PATCH 34/42] Remove "ofUrl" and dummy constructors --- .../logic/importer/FetcherException.java | 22 -------- .../importer/fetcher/IacrEprintFetcher.java | 52 ++++++++++++++----- 2 files changed, 39 insertions(+), 35 deletions(-) diff --git a/src/main/java/org/jabref/logic/importer/FetcherException.java b/src/main/java/org/jabref/logic/importer/FetcherException.java index dfe847bd432..2a7f0bdc63f 100644 --- a/src/main/java/org/jabref/logic/importer/FetcherException.java +++ b/src/main/java/org/jabref/logic/importer/FetcherException.java @@ -45,24 +45,6 @@ public FetcherException(URL url, String errorMessage, Throwable cause) { this.url = url.toString(); } - /** - * @param dummy Required to distinguish of {@link FetcherException#FetcherException(String, Throwable)}, which is for a FetcherException with an error message - */ - private FetcherException(String url, Throwable cause, Object dummy) { - super(cause); - httpResponse = null; - this.url = url; - } - - /** - * @param dummy Required to distinguish of {@link FetcherException#FetcherException(String, String, Throwable)}, which is for a localized FetcherException - */ - private FetcherException(String url, String errorMessage, Throwable cause, Object dummy) { - super(errorMessage, cause); - httpResponse = null; - this.url = url; - } - public FetcherException(String errorMessage, Throwable cause) { // TODO: Convert IOException e to FetcherClientException // Same TODO as in org.jabref.logic.importer.EntryBasedParserFetcher.performSearch. @@ -90,10 +72,6 @@ public FetcherException(String errorMessage, String localizedMessage, Throwable url = null; } - public static FetcherException ofUrl(String url, String errorMessage, Throwable cause) { - return new FetcherException(url, errorMessage, cause, null); - } - public static FetcherException of(URL url, SimpleHttpResponse simpleHttpResponse) { if (simpleHttpResponse.statusCode() >= 500) { return new FetcherServerException(url, simpleHttpResponse); diff --git a/src/main/java/org/jabref/logic/importer/fetcher/IacrEprintFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/IacrEprintFetcher.java index 35c0ba1bbe9..d1d630ffe21 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/IacrEprintFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/IacrEprintFetcher.java @@ -1,6 +1,8 @@ package org.jabref.logic.importer.fetcher; import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URI; import java.net.URL; import java.util.Objects; import java.util.Optional; @@ -19,10 +21,14 @@ import org.jabref.model.entry.field.StandardField; import org.jabref.model.strings.StringUtil; -public class IacrEprintFetcher implements FulltextFetcher, IdBasedFetcher { +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +public class IacrEprintFetcher implements FulltextFetcher, IdBasedFetcher { public static final String NAME = "IACR eprints"; + private static final Logger LOGGER = LoggerFactory.getLogger(IacrEprintFetcher.class); + private static final Pattern WITHOUT_LETTERS_SPACE = Pattern.compile("[^0-9/]"); private static final Predicate IDENTIFIER_PREDICATE = Pattern.compile("\\d{4}/\\d{3,5}").asPredicate(); @@ -55,7 +61,13 @@ public Optional performSearchById(String identifier) throws FetcherExc } private Optional createEntryFromIacrCitation(String validIdentifier) throws FetcherException { - String bibtexCitationHtml = getHtml(CITATION_URL_PREFIX + validIdentifier); + URL url; + try { + url = URI.create(CITATION_URL_PREFIX + validIdentifier).toURL(); + } catch (MalformedURLException e) { + throw new FetcherException("Invalid URL", e); + } + String bibtexCitationHtml = getHtml(url); if (bibtexCitationHtml.contains("No such report found")) { throw new FetcherException(Localization.lang("No results found.")); } @@ -69,7 +81,13 @@ private Optional createEntryFromIacrCitation(String validIdentifier) t } private void setAdditionalFields(BibEntry entry, String identifier) throws FetcherException { - String entryUrl = DESCRIPTION_URL_PREFIX + identifier; + URL entryUrl; + try { + entryUrl = URI.create(DESCRIPTION_URL_PREFIX + identifier).toURL(); + } catch (MalformedURLException e) { + throw new FetcherException("Invalid URL", e); + } + String descriptiveHtml = getHtml(entryUrl); entry.setField(StandardField.ABSTRACT, getAbstract(descriptiveHtml)); @@ -77,8 +95,12 @@ private void setAdditionalFields(BibEntry entry, String identifier) throws Fetch // Version information for entries after year 2000 if (isFromOrAfterYear2000(entry)) { - String entryVersion = VERSION_URL_PREFIX + identifier; - String versionHtml = getHtml(entryVersion); + try { + entryUrl = URI.create(VERSION_URL_PREFIX + identifier).toURL(); + } catch (MalformedURLException e) { + throw new FetcherException("Invalid URL", e); + } + String versionHtml = getHtml(entryUrl); String version = getVersion(identifier, versionHtml); entry.setField(StandardField.VERSION, version); entry.setField(StandardField.URL, entryUrl + "/" + version); @@ -103,13 +125,9 @@ private String getDate(String descriptiveHtml) throws FetcherException { return dateStringAsInHtml; } - private String getHtml(String url) throws FetcherException { - try { - URLDownload download = new URLDownload(url); - return download.asString(); - } catch (IOException e) { - throw FetcherException.ofUrl(url, Localization.lang("Could not retrieve entry data from '%0'.", url), e); - } + private String getHtml(URL url) throws FetcherException { + URLDownload download = new URLDownload(url); + return download.asString(); } private String getRequiredValueBetween(String from, String to, String haystack) throws FetcherException { @@ -140,7 +158,15 @@ public Optional findFullText(BibEntry entry) throws IOException, FetcherExc Optional urlField = entry.getField(StandardField.URL); if (urlField.isPresent()) { - String descriptiveHtml = getHtml(urlField.get()); + URL url; + try { + url = URI.create(urlField.get()).toURL(); + } catch (MalformedURLException e) { + LOGGER.warn("Invalid URL {}", urlField.get(), e); + return Optional.empty(); + } + + String descriptiveHtml = getHtml(url); String startOfFulltextLink = " Date: Mon, 12 Aug 2024 19:27:01 +0200 Subject: [PATCH 35/42] Make FetcherException(URL url, SimpleHttpResponse httpResponse) protected --- .../logic/importer/FetcherException.java | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/main/java/org/jabref/logic/importer/FetcherException.java b/src/main/java/org/jabref/logic/importer/FetcherException.java index 2a7f0bdc63f..77e6715d050 100644 --- a/src/main/java/org/jabref/logic/importer/FetcherException.java +++ b/src/main/java/org/jabref/logic/importer/FetcherException.java @@ -29,7 +29,7 @@ public FetcherException(String url, SimpleHttpResponse httpResponse) { this.httpResponse = httpResponse; } - public FetcherException(URL url, SimpleHttpResponse httpResponse) { + protected FetcherException(URL url, SimpleHttpResponse httpResponse) { this(url.toString(), httpResponse); } @@ -72,14 +72,6 @@ public FetcherException(String errorMessage, String localizedMessage, Throwable url = null; } - public static FetcherException of(URL url, SimpleHttpResponse simpleHttpResponse) { - if (simpleHttpResponse.statusCode() >= 500) { - return new FetcherServerException(url, simpleHttpResponse); - } else { - return new FetcherClientException(url, simpleHttpResponse); - } - } - @Override public String getLocalizedMessage() { // TODO: This should be moved to a separate class converting "any" exception object to a localized message @@ -115,4 +107,12 @@ public Optional getHttpResponse() { public Optional getUrl() { return Optional.ofNullable(url); } + + public static FetcherException of(URL url, SimpleHttpResponse simpleHttpResponse) { + if (simpleHttpResponse.statusCode() >= 500) { + return new FetcherServerException(url, simpleHttpResponse); + } else { + return new FetcherClientException(url, simpleHttpResponse); + } + } } From d151e2bfaf21f39df7f488e47c1e793869411a9d Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Mon, 12 Aug 2024 19:43:53 +0200 Subject: [PATCH 36/42] Adapt test --- src/test/java/org/jabref/logic/net/URLDownloadTest.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/test/java/org/jabref/logic/net/URLDownloadTest.java b/src/test/java/org/jabref/logic/net/URLDownloadTest.java index 2c9e41b2b11..0b4771d2fb9 100644 --- a/src/test/java/org/jabref/logic/net/URLDownloadTest.java +++ b/src/test/java/org/jabref/logic/net/URLDownloadTest.java @@ -128,11 +128,9 @@ public void connectTimeoutIsNeverNull() throws MalformedURLException { } @Test - public void test503ErrorThrowsNestedIOExceptionWithFetcherServerException() throws Exception { + public void test503ErrorThrowsFetcherServerException() throws Exception { URLDownload urlDownload = new URLDownload(new URL("http://httpstat.us/503")); - - Exception exception = assertThrows(IOException.class, urlDownload::asString); - assertInstanceOf(FetcherServerException.class, exception.getCause()); + assertThrows(FetcherServerException.class, urlDownload::asString); } @Test From 847c3bf693ee66cfed7c0be37130f89d28ff88d5 Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Mon, 12 Aug 2024 19:46:37 +0200 Subject: [PATCH 37/42] Fix connection opened twice --- src/main/java/org/jabref/logic/net/URLDownload.java | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/src/main/java/org/jabref/logic/net/URLDownload.java b/src/main/java/org/jabref/logic/net/URLDownload.java index 7dd4503d47a..db9e3b4d9f7 100644 --- a/src/main/java/org/jabref/logic/net/URLDownload.java +++ b/src/main/java/org/jabref/logic/net/URLDownload.java @@ -397,18 +397,6 @@ public URLConnection openConnection() throws FetcherException { } catch (IOException e) { throw new FetcherException("Error opening connection", e); } - connection.setConnectTimeout((int) connectTimeout.toMillis()); - for (Entry entry : this.parameters.entrySet()) { - connection.setRequestProperty(entry.getKey(), entry.getValue()); - } - if (!this.postData.isEmpty()) { - connection.setDoOutput(true); - try (DataOutputStream wr = new DataOutputStream(connection.getOutputStream())) { - wr.writeBytes(this.postData); - } catch (IOException e) { - throw new FetcherException("Could not write output", e); - } - } if (connection instanceof HttpURLConnection httpURLConnection) { int status; From 1a3da174943c6ee6feea2d59afae9eebe4fa5556 Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Mon, 12 Aug 2024 20:00:43 +0200 Subject: [PATCH 38/42] Adapt test --- src/test/java/org/jabref/logic/net/URLDownloadTest.java | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/test/java/org/jabref/logic/net/URLDownloadTest.java b/src/test/java/org/jabref/logic/net/URLDownloadTest.java index 0b4771d2fb9..38b0846d8a5 100644 --- a/src/test/java/org/jabref/logic/net/URLDownloadTest.java +++ b/src/test/java/org/jabref/logic/net/URLDownloadTest.java @@ -1,7 +1,6 @@ package org.jabref.logic.net; import java.io.File; -import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; import java.nio.charset.StandardCharsets; @@ -17,7 +16,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -134,10 +132,8 @@ public void test503ErrorThrowsFetcherServerException() throws Exception { } @Test - public void test429ErrorThrowsNestedIOExceptionWithFetcherServerException() throws Exception { + public void test429ErrorThrowsFetcherClientException() throws Exception { URLDownload urlDownload = new URLDownload(new URL("http://httpstat.us/429")); - - Exception exception = assertThrows(IOException.class, urlDownload::asString); - assertInstanceOf(FetcherClientException.class, exception.getCause()); + Exception exception = assertThrows(FetcherClientException.class, urlDownload::asString); } } From ca979c98e4ff3bf315976154e812804dab88e2f7 Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Mon, 12 Aug 2024 23:11:42 +0200 Subject: [PATCH 39/42] Remove obsolete localization --- src/main/resources/l10n/JabRef_en.properties | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/resources/l10n/JabRef_en.properties b/src/main/resources/l10n/JabRef_en.properties index 59ec8e76926..9cec9d1f4f2 100644 --- a/src/main/resources/l10n/JabRef_en.properties +++ b/src/main/resources/l10n/JabRef_en.properties @@ -1917,7 +1917,6 @@ Removes\ all\ line\ breaks\ in\ the\ field\ content.=Removes all line breaks in Remove\ hyphenated\ line\ breaks=Remove hyphenated line breaks Removes\ all\ hyphenated\ line\ breaks\ in\ the\ field\ content.=Removes all hyphenated line breaks in the field content. -Could\ not\ retrieve\ entry\ data\ from\ '%0'.=Could not retrieve entry data from '%0'. Entry\ from\ %0\ could\ not\ be\ parsed.=Entry from %0 could not be parsed. Invalid\ identifier\:\ '%0'.=Invalid identifier: '%0'. empty\ citation\ key=empty citation key From d6d16ac148c39a83b270543d1a421226188ff971 Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Mon, 12 Aug 2024 23:13:11 +0200 Subject: [PATCH 40/42] Fix instanceOf Pattern --- src/main/java/org/jabref/gui/JabRefDialogService.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/jabref/gui/JabRefDialogService.java b/src/main/java/org/jabref/gui/JabRefDialogService.java index f50820c1c8e..b92954bff36 100644 --- a/src/main/java/org/jabref/gui/JabRefDialogService.java +++ b/src/main/java/org/jabref/gui/JabRefDialogService.java @@ -211,9 +211,9 @@ public void showErrorDialogAndWait(String message, Throwable exception) { @Override public void showErrorDialogAndWait(Exception exception) { - if (exception instanceof FetcherException) { + if (exception instanceof FetcherException fetcherException) { // Somehow, Java does not route correctly to the other method - showErrorDialogAndWait((FetcherException) exception); + showErrorDialogAndWait(fetcherException); } else { showErrorDialogAndWait(Localization.lang("Unhandled exception occurred."), exception); } From 3405e211f08855a3c37db10f499391e9ee972b0b Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Mon, 12 Aug 2024 23:17:44 +0200 Subject: [PATCH 41/42] .formatted works. --- src/main/java/org/jabref/logic/importer/FetcherException.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/jabref/logic/importer/FetcherException.java b/src/main/java/org/jabref/logic/importer/FetcherException.java index 77e6715d050..47f672a8c51 100644 --- a/src/main/java/org/jabref/logic/importer/FetcherException.java +++ b/src/main/java/org/jabref/logic/importer/FetcherException.java @@ -79,9 +79,9 @@ public String getLocalizedMessage() { // TODO: Try to convert IOException to some more meaningful information here (or at org.jabref.gui.DialogService.showErrorDialogAndWait(org.jabref.logic.importer.FetcherException)). See also org.jabref.logic.net.URLDownload.openConnection if (httpResponse != null) { // We decided to not "translate" technical terms (URL, HTTP) - return getPrefix() + String.format("URL: %s\nHTTP %d %s\n%s", getRedactedUrl(), httpResponse.statusCode(), httpResponse.responseMessage(), httpResponse.responseBody()); + return getPrefix() + "URL: %s\nHTTP %d %s\n%s".formatted(getRedactedUrl(), httpResponse.statusCode(), httpResponse.responseMessage(), httpResponse.responseBody()); } else if (url != null) { - return getPrefix() + "URL: %s".format(getRedactedUrl()); + return getPrefix() + "URL: %s".formatted(getRedactedUrl()); } else { return super.getLocalizedMessage(); } From d92e2a5737cc6e15984fb3346993e7cc986d2fc7 Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Mon, 12 Aug 2024 23:19:59 +0200 Subject: [PATCH 42/42] Add CHANGELOG.md entry --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index bcc7ea5fe45..1191882234c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,7 @@ Note that this project **does not** adhere to [Semantic Versioning](https://semv ### Changed +- When a communication error with an [online service](https://docs.jabref.org/collect/import-using-online-bibliographic-database) occurs, JabRef displays the HTTP error. [#11223](https://github.com/JabRef/jabref/issues/11223) - The Pubmed/Medline Plain importer now imports the PMID field as well [#11488](https://github.com/JabRef/jabref/issues/11488) - The 'Check for updates' menu bar button is now always enabled. [#11485](https://github.com/JabRef/jabref/pull/11485) - JabRef respects the [configuration for storing files relative to the .bib file](https://docs.jabref.org/finding-sorting-and-cleaning-entries/filelinks#directories-for-files) in more cases. [#11492](https://github.com/JabRef/jabref/pull/11492)