Download linked files for id fetchers

CHANGELOG.md

@@ -18,7 +18,7 @@ Note that this project **does not** adhere to [Semantic Versioning](http://semve
 - We enabled updating an existing entry with data from InspireHEP. [#9351](https://github.com/JabRef/jabref/issues/9351)
 - We added a fetcher for the Bibliotheksverbund Bayern (experimental). [#9641](https://github.com/JabRef/jabref/pull/9641)
 - We enabled scrolling in the groups list when dragging a group on another group. [#2869](https://github.com/JabRef/jabref/pull/2869)
-
+- We added the option to automatically download online files when a new entry is created from an existing ID (e.g. DOI). The option can be disabled in the preferences under "Import and Export" [#9756](https://github.com/JabRef/jabref/issues/9756)
 
 
 

src/main/java/org/jabref/gui/EntryTypeView.java

@@ -20,6 +20,7 @@
 import org.jabref.gui.util.BaseDialog;
 import org.jabref.gui.util.ControlHelper;
 import org.jabref.gui.util.IconValidationDecorator;
+import org.jabref.gui.util.TaskExecutor;
 import org.jabref.gui.util.ViewModelListCellFactory;
 import org.jabref.logic.importer.IdBasedFetcher;
 import org.jabref.logic.importer.ImportFormatReader;
@@ -49,6 +50,7 @@ public class EntryTypeView extends BaseDialog<EntryType> {
 
     @Inject StateManager stateManager;
     @Inject ImportFormatReader importFormatReader;
+    @Inject TaskExecutor taskExecutor;
 
     @FXML private ButtonType generateButton;
     @FXML private TextField idTextField;
@@ -120,7 +122,7 @@ private void addEntriesToPane(FlowPane pane, Collection<? extends BibEntryType>
     @FXML
     public void initialize() {
         visualizer.setDecoration(new IconValidationDecorator());
-        viewModel = new EntryTypeViewModel(preferencesService, libraryTab, dialogService, stateManager, importFormatReader);
+        viewModel = new EntryTypeViewModel(preferencesService, libraryTab, dialogService, stateManager, importFormatReader, taskExecutor);
 
         idBasedFetchers.itemsProperty().bind(viewModel.fetcherItemsProperty());
         idTextField.textProperty().bindBidirectional(viewModel.idTextProperty());

src/main/java/org/jabref/gui/EntryTypeViewModel.java

@@ -16,6 +16,7 @@
 
 import org.jabref.gui.externalfiles.ImportHandler;
 import org.jabref.gui.importer.NewEntryAction;
+import org.jabref.gui.util.TaskExecutor;
 import org.jabref.logic.importer.FetcherClientException;
 import org.jabref.logic.importer.FetcherException;
 import org.jabref.logic.importer.FetcherServerException;
@@ -53,17 +54,20 @@ public class EntryTypeViewModel {
     private final Validator idFieldValidator;
     private final StateManager stateManager;
     private final ImportFormatReader importFormatReader;
+    private final TaskExecutor taskExecutor;
 
     public EntryTypeViewModel(PreferencesService preferences,
                               LibraryTab libraryTab,
                               DialogService dialogService,
                               StateManager stateManager,
-                              ImportFormatReader importFormatReader) {
+                              ImportFormatReader importFormatReader,
+                              TaskExecutor taskExecutor) {
         this.libraryTab = libraryTab;
         this.preferencesService = preferences;
         this.dialogService = dialogService;
         this.stateManager = stateManager;
         this.importFormatReader = importFormatReader;
+        this.taskExecutor = taskExecutor;
 
         fetchers.addAll(WebFetchers.getIdBasedFetchers(
                 preferences.getImportFormatPreferences(),
@@ -173,7 +177,8 @@ public void runFetcherWorker() {
                         libraryTab.getUndoManager(),
                         stateManager,
                         dialogService,
-                        importFormatReader);
+                        importFormatReader,
+                        taskExecutor);
                 handler.importEntryWithDuplicateCheck(libraryTab.getBibDatabaseContext(), entry);
 
                 searchSuccesfulProperty.set(true);

src/main/java/org/jabref/gui/LibraryTab.java

@@ -505,7 +505,8 @@ private void createMainTable() {
                 stateManager,
                 Globals.getKeyPrefs(),
                 Globals.getClipboardManager(),
-                Globals.IMPORT_FORMAT_READER);
+                Globals.IMPORT_FORMAT_READER,
+                Globals.TASK_EXECUTOR);
 
         // Add the listener that binds selection to state manager (TODO: should be replaced by proper JavaFX binding as soon as table is implemented in JavaFX)
         mainTable.addSelectionListener(listEvent -> stateManager.setSelectedEntries(mainTable.getSelectedEntries()));

src/main/java/org/jabref/gui/bibtexextractor/BibtexExtractorViewModel.java

@@ -56,7 +56,8 @@ public BibtexExtractorViewModel(BibDatabaseContext bibdatabaseContext,
                 undoManager,
                 stateManager,
                 dialogService,
-                importFormatReader);
+                importFormatReader,
+                taskExecutor);
     }
 
     public StringProperty inputTextProperty() {

src/main/java/org/jabref/gui/externalfiles/ImportHandler.java

@@ -17,9 +17,11 @@
 import org.jabref.gui.Globals;
 import org.jabref.gui.StateManager;
 import org.jabref.gui.duplicationFinder.DuplicateResolverDialog;
+import org.jabref.gui.fieldeditors.LinkedFileViewModel;
 import org.jabref.gui.undo.UndoableInsertEntries;
 import org.jabref.gui.util.BackgroundTask;
 import org.jabref.gui.util.DefaultTaskExecutor;
+import org.jabref.gui.util.TaskExecutor;
 import org.jabref.logic.citationkeypattern.CitationKeyGenerator;
 import org.jabref.logic.database.DuplicateCheck;
 import org.jabref.logic.externalfiles.ExternalFilesContentImporter;
@@ -38,6 +40,7 @@
 import org.jabref.model.FieldChange;
 import org.jabref.model.database.BibDatabaseContext;
 import org.jabref.model.entry.BibEntry;
+import org.jabref.model.entry.LinkedFile;
 import org.jabref.model.entry.field.StandardField;
 import org.jabref.model.entry.identifier.ArXivIdentifier;
 import org.jabref.model.entry.identifier.DOI;
@@ -62,21 +65,24 @@ public class ImportHandler {
     private final StateManager stateManager;
     private final DialogService dialogService;
     private final ImportFormatReader importFormatReader;
+    private final TaskExecutor taskExecutor;
 
     public ImportHandler(BibDatabaseContext database,
                          PreferencesService preferencesService,
                          FileUpdateMonitor fileupdateMonitor,
                          UndoManager undoManager,
                          StateManager stateManager,
                          DialogService dialogService,
-                         ImportFormatReader importFormatReader) {
+                         ImportFormatReader importFormatReader,
+                         TaskExecutor taskExecutor) {
 
         this.bibDatabaseContext = database;
         this.preferencesService = preferencesService;
         this.fileUpdateMonitor = fileupdateMonitor;
         this.stateManager = stateManager;
         this.dialogService = dialogService;
         this.importFormatReader = importFormatReader;
+        this.taskExecutor = taskExecutor;
 
         this.linker = new ExternalFilesEntryLinker(preferencesService.getFilePreferences(), database);
         this.contentImporter = new ExternalFilesContentImporter(preferencesService.getImportFormatPreferences());
@@ -225,6 +231,17 @@ public void importEntryWithDuplicateCheck(BibDatabaseContext bibDatabaseContext,
                                        preferencesService.getTimestampPreferences());
 
         addToGroups(List.of(entry), stateManager.getSelectedGroup(this.bibDatabaseContext));
+
+        if (preferencesService.getFilePreferences().shouldDownloadLinkedFiles()) {
+                entry.getFiles().stream().filter(LinkedFile::isOnlineLink).forEach(linkedFile ->
+                        new LinkedFileViewModel(
+                                linkedFile,
+                                entry,
+                                bibDatabaseContext,
+                                taskExecutor,
+                                dialogService,
+                                preferencesService).download());
+        }
     }
 
     private void addToGroups(List<BibEntry> entries, Collection<GroupTreeNode> groups) {

src/main/java/org/jabref/gui/externalfiles/UnlinkedFilesDialogViewModel.java

@@ -98,7 +98,8 @@ public UnlinkedFilesDialogViewModel(DialogService dialogService,
                 undoManager,
                 stateManager,
                 dialogService,
-                importFormatReader);
+                importFormatReader,
+                taskExecutor);
 
         this.fileFilterList = FXCollections.observableArrayList(
                 new FileExtensionViewModel(StandardFileType.ANY_FILE, preferences.getFilePreferences()),

src/main/java/org/jabref/gui/importer/GenerateEntryFromIdAction.java

@@ -80,7 +80,7 @@ public void execute() {
             Optional<BibEntry> result = bibEntry;
             if (result.isPresent()) {
                 final BibEntry entry = result.get();
-                ImportHandler handler = new ImportHandler(libraryTab.getBibDatabaseContext(), preferencesService, Globals.getFileUpdateMonitor(), libraryTab.getUndoManager(), stateManager, dialogService, null);
+                ImportHandler handler = new ImportHandler(libraryTab.getBibDatabaseContext(), preferencesService, Globals.getFileUpdateMonitor(), libraryTab.getUndoManager(), stateManager, dialogService, null, taskExecutor);
                 handler.importEntryWithDuplicateCheck(libraryTab.getBibDatabaseContext(), entry);
             } else {
                 dialogService.notify("No entry found or import canceled");

src/main/java/org/jabref/gui/importer/ImportEntriesViewModel.java

@@ -186,7 +186,8 @@ private void buildImportHandlerThenImportEntries(List<BibEntry> entriesToImport)
                 undoManager,
                 stateManager,
                 dialogService,
-                Globals.IMPORT_FORMAT_READER);
+                Globals.IMPORT_FORMAT_READER,
+                taskExecutor);
         importHandler.importEntries(entriesToImport);
         dialogService.notify(Localization.lang("Number of entries successfully imported") + ": " + entriesToImport.size());
     }

src/main/java/org/jabref/gui/maintable/MainTable.java

@@ -41,6 +41,7 @@
 import org.jabref.gui.util.ControlHelper;
 import org.jabref.gui.util.CustomLocalDragboard;
 import org.jabref.gui.util.DefaultTaskExecutor;
+import org.jabref.gui.util.TaskExecutor;
 import org.jabref.gui.util.ViewModelTableRowFactory;
 import org.jabref.logic.importer.FetcherClientException;
 import org.jabref.logic.importer.FetcherException;
@@ -80,7 +81,8 @@ public MainTable(MainTableDataModel model,
                      StateManager stateManager,
                      KeyBindingRepository keyBindingRepository,
                      ClipBoardManager clipBoardManager,
-                     ImportFormatReader importFormatReader) {
+                     ImportFormatReader importFormatReader,
+                     TaskExecutor taskExecutor) {
         super();
 
         this.libraryTab = libraryTab;
@@ -99,7 +101,8 @@ public MainTable(MainTableDataModel model,
                 undoManager,
                 stateManager,
                 dialogService,
-                importFormatReader);
+                importFormatReader,
+                taskExecutor);
 
         localDragboard = stateManager.getLocalDragboard();
 

src/main/java/org/jabref/gui/preferences/importexport/ImportExportTab.fxml

@@ -17,6 +17,7 @@
     <Label styleClass="sectionHeader" text="%General"/>
     <CheckBox fx:id="generateNewKeyOnImport" text="%Generate a new key for imported entries (overwriting their default)"/>
     <CheckBox fx:id="warnAboutDuplicatesOnImport" text="%Warn about duplicates on import"/>
+    <CheckBox fx:id="downloadLinkedOnlineFiles" text="%Download linked online files"/>
 
     <Label styleClass="sectionHeader" text="%Custom DOI URI"/>
     <HBox alignment="CENTER_LEFT" spacing="10.0">

src/main/java/org/jabref/gui/preferences/importexport/ImportExportTab.java

@@ -33,6 +33,7 @@ public class ImportExportTab extends AbstractPreferenceTabView<ImportExportTabVi
     @FXML private TextField grobidURL;
 
     @FXML private CheckBox warnAboutDuplicatesOnImport;
+    @FXML private CheckBox downloadLinkedOnlineFiles;
 
     public ImportExportTab() {
         ViewLoader.view(this)
@@ -65,6 +66,8 @@ public void initialize() {
         grobidURL.textProperty().bindBidirectional(viewModel.grobidURLProperty());
         grobidURL.disableProperty().bind(grobidEnabled.selectedProperty().not());
 
+        downloadLinkedOnlineFiles.selectedProperty().bindBidirectional(viewModel.shouldDownloadLinkedOnlineFiles());
+
         new ViewModelListCellFactory<FetcherApiKey>()
                 .withText(FetcherApiKey::getName)
                 .install(apiKeySelector);

src/main/java/org/jabref/gui/preferences/importexport/ImportExportTabViewModel.java

@@ -35,6 +35,7 @@
 import org.jabref.model.entry.field.Field;
 import org.jabref.model.entry.field.FieldFactory;
 import org.jabref.model.metadata.SaveOrderConfig;
+import org.jabref.preferences.FilePreferences;
 import org.jabref.preferences.ImportExportPreferences;
 import org.jabref.preferences.PreferencesService;
 
@@ -58,6 +59,7 @@ public class ImportExportTabViewModel implements PreferenceTabViewModel {
     private final BooleanProperty grobidEnabledProperty = new SimpleBooleanProperty();
     private final StringProperty grobidURLProperty = new SimpleStringProperty("");
     private final BooleanProperty warnAboutDuplicatesOnImportProperty = new SimpleBooleanProperty();
+    private final BooleanProperty shouldDownloadLinkedOnlineFiles = new SimpleBooleanProperty();
 
     private final DialogService dialogService;
     private final PreferencesService preferencesService;
@@ -66,6 +68,7 @@ public class ImportExportTabViewModel implements PreferenceTabViewModel {
     private final ImporterPreferences importerPreferences;
     private final SaveOrderConfig initialExportOrder;
     private final ImportExportPreferences importExportPreferences;
+    private final FilePreferences filePreferences;
 
     public ImportExportTabViewModel(PreferencesService preferencesService, DialogService dialogService) {
         this.dialogService = dialogService;
@@ -75,6 +78,7 @@ public ImportExportTabViewModel(PreferencesService preferencesService, DialogSer
         this.doiPreferences = preferencesService.getDOIPreferences();
         this.initialExportOrder = preferencesService.getExportSaveOrder();
         this.importExportPreferences = preferencesService.getImportExportPreferences();
+        this.filePreferences = preferencesService.getFilePreferences();
     }
 
     @Override
@@ -84,6 +88,8 @@ public void setValues() {
         useCustomDOINameProperty.setValue(doiPreferences.getDefaultBaseURI());
         warnAboutDuplicatesOnImportProperty.setValue(importExportPreferences.shouldWarnAboutDuplicatesOnImport());
 
+        shouldDownloadLinkedOnlineFiles.setValue(filePreferences.shouldDownloadLinkedFiles());
+
         switch (initialExportOrder.getOrderType()) {
             case SPECIFIED -> exportInSpecifiedOrderProperty.setValue(true);
             case ORIGINAL -> exportInOriginalProperty.setValue(true);
@@ -111,6 +117,8 @@ public void storeSettings() {
         grobidPreferences.setGrobidOptOut(grobidPreferences.isGrobidOptOut());
         grobidPreferences.setGrobidURL(grobidURLProperty.getValue());
 
+        filePreferences.setDownloadLinkedFiles(shouldDownloadLinkedOnlineFiles.getValue());
+
         doiPreferences.setUseCustom(useCustomDOIProperty.get());
         doiPreferences.setDefaultBaseURI(useCustomDOINameProperty.getValue().trim());
 
@@ -180,6 +188,10 @@ public BooleanProperty warnAboutDuplicatesOnImportProperty() {
         return warnAboutDuplicatesOnImportProperty;
     }
 
+    public BooleanProperty shouldDownloadLinkedOnlineFiles() {
+        return shouldDownloadLinkedOnlineFiles;
+    }
+
     public void checkCustomApiKey() {
         final String apiKeyName = selectedApiKeyProperty.get().getName();
 
@@ -218,7 +230,7 @@ public void checkCustomApiKey() {
                 urlDownload = new URLDownload(testUrlWithoutApiKey + apiKey);
                 // The HEAD request cannot be used because its response is not 200 (maybe 404 or 596...).
                 int statusCode = ((HttpURLConnection) urlDownload.getSource().openConnection()).getResponseCode();
-                keyValid = statusCode >= 200 && statusCode < 300;
+                keyValid = (statusCode >= 200) && (statusCode < 300);
 
                 URLDownload.setSSLVerification(defaultSslSocketFactory, defaultHostnameVerifier);
             } catch (IOException | kong.unirest.UnirestException e) {

src/main/java/org/jabref/gui/preferences/linkedfiles/LinkedFilesTabViewModel.java

@@ -105,7 +105,6 @@ public void storeSettings() {
         filePreferences.setStoreFilesRelativeToBibFile(useBibLocationAsPrimaryProperty.getValue());
         filePreferences.setFileNamePattern(fileNamePatternProperty.getValue());
         filePreferences.setFileDirectoryPattern(fileDirectoryPatternProperty.getValue());
-        filePreferences.setDownloadLinkedFiles(filePreferences.shouldDownloadLinkedFiles()); // set in ImportEntriesViewModel
         filePreferences.setFulltextIndexLinkedFiles(fulltextIndex.getValue());
 
         // Autolink preferences