Download many files in one go (#1869)

* Download many files in one go

* Added warnings dialog if 10 or more entries are selected for full text download

* Extracted magic number and fixed file name generation fallback

* Some code cleanups

CHANGELOG.md

@@ -13,6 +13,8 @@ We refer to [GitHub issues](https://github.com/JabRef/jabref/issues) by using `#
 
 ### Changed
 - Added integrity check for fields with BibTeX keys, e.g., `crossref` and `related`, to check that the key exists
+- [#1496](https://github.com/JabRef/jabref/issues/1496) Keep track of which entry a downloaded file belongs to
+- Made it possible to download multiple entries in one action
 
 ### Fixed
 - Fixed NullPointerException when opening search result window for an untitled database 

src/main/java/net/sf/jabref/cli/ArgumentProcessor.java

@@ -237,7 +237,7 @@ private List<ParserResult> importAndOpenFiles() {
                 // BIB files to open. Other files, and files that could not be opened
                 // as bib, we try to import instead.
                 boolean bibExtension = aLeftOver.toLowerCase(Locale.ENGLISH).endsWith("bib");
-                ParserResult pr = null;
+                ParserResult pr = ParserResult.getNullResult();
                 if (bibExtension) {
                     pr = OpenDatabase.loadDatabaseOrAutoSave(aLeftOver, false,
                             Globals.prefs.getImportFormatPreferences());

src/main/java/net/sf/jabref/external/DownloadExternalFile.java

@@ -21,6 +21,7 @@
 import net.sf.jabref.logic.net.URLDownload;
 import net.sf.jabref.logic.util.OS;
 import net.sf.jabref.logic.util.io.FileUtil;
+import net.sf.jabref.model.entry.BibEntry;
 import net.sf.jabref.preferences.JabRefPreferences;
 
 import org.apache.commons.logging.Log;
@@ -43,15 +44,16 @@ public class DownloadExternalFile {
 
     private final JabRefFrame frame;
     private final BibDatabaseContext databaseContext;
-    private final String bibtexKey;
+    private final BibEntry entry;
     private FileListEntryEditor editor;
     private boolean downloadFinished;
     private boolean dontShowDialog;
 
-    public DownloadExternalFile(JabRefFrame frame, BibDatabaseContext databaseContext, String bibtexKey) {
+
+    public DownloadExternalFile(JabRefFrame frame, BibDatabaseContext databaseContext, BibEntry entry) {
         this.frame = frame;
         this.databaseContext = databaseContext;
-        this.bibtexKey = bibtexKey;
+        this.entry = entry;
     }
 
     /**
@@ -155,8 +157,8 @@ public void download(URL url, final DownloadCallback callback) throws IOExceptio
         }
         final String suggestDir = directory == null ? System.getProperty("user.home") : directory;
         File file = new File(new File(suggestDir), suggestedName);
-        FileListEntry entry = new FileListEntry("", file.getCanonicalPath(), suggestedType);
-        editor = new FileListEntryEditor(frame, entry, true, false, databaseContext);
+        FileListEntry fileListEntry = new FileListEntry("", file.getCanonicalPath(), suggestedType);
+        editor = new FileListEntryEditor(frame, fileListEntry, true, false, databaseContext);
         editor.getProgressBar().setIndeterminate(true);
         editor.setOkEnabled(false);
         editor.setExternalConfirm(closeEntry -> {
@@ -181,7 +183,8 @@ public void download(URL url, final DownloadCallback callback) throws IOExceptio
         }
         // Editor closed. Go on:
         if (editor.okPressed()) {
-            File toFile = directory == null ? new File(entry.link) : expandFilename(directory, entry.link);
+            File toFile = directory == null ? new File(fileListEntry.link) : expandFilename(directory,
+                    fileListEntry.link);
             String dirPrefix;
             if (directory == null) {
                 dirPrefix = null;
@@ -202,12 +205,13 @@ public void download(URL url, final DownloadCallback callback) throws IOExceptio
 
                 // If the local file is in or below the main file directory, change the
                 // path to relative:
-                if ((directory != null) && entry.link.startsWith(directory) &&
-                        (entry.link.length() > dirPrefix.length())) {
-                    entry = new FileListEntry(entry.description, entry.link.substring(dirPrefix.length()), entry.type);
+                if ((dirPrefix != null) && fileListEntry.link.startsWith(directory)
+                        && (fileListEntry.link.length() > dirPrefix.length())) {
+                    fileListEntry = new FileListEntry(fileListEntry.description,
+                            fileListEntry.link.substring(dirPrefix.length()), fileListEntry.type);
                 }
 
-                callback.downloadComplete(entry);
+                callback.downloadComplete(fileListEntry);
             } catch (IOException ex) {
                 LOGGER.warn("Problem downloading file", ex);
             }
@@ -253,10 +257,8 @@ private void downloadFinished() {
         editor.getProgressBar().setValue(editor.getProgressBar().getMaximum());
     }
 
-    // FIXME: will break download if no bibtexkey is present!
     private String getSuggestedFileName(String suffix) {
-        String plannedName = FileUtil.createFileNameFromPattern(databaseContext.getDatabase(),
-                frame.getCurrentBasePanel().getSelectedEntries().get(0),
+        String plannedName = FileUtil.createFileNameFromPattern(databaseContext.getDatabase(), entry,
                 Globals.prefs.get(JabRefPreferences.IMPORT_FILENAMEPATTERN),
                 Globals.prefs.getLayoutFormatterPreferences(Globals.journalAbbreviationLoader));
 

src/main/java/net/sf/jabref/external/DroppedFileHandler.java

@@ -273,7 +273,7 @@ private boolean tryXmpImport(String fileName, ExternalFileType fileType, NamedCo
                             panel.getBibDatabaseContext().getFileDirectory(Globals.prefs.getFileDirectoryPreferences()))
                     .toString();
         } else {
-            if (renameCheckBox.isSelected()) {
+            if (renameCheckBox.isSelected() || (single == null)) {
                 destFilename = fileName;
             } else {
                 destFilename = single.getCiteKey() + "." + fileType.getExtension();

src/main/java/net/sf/jabref/external/FindFullTextAction.java

@@ -2,8 +2,12 @@
 
 import java.io.IOException;
 import java.net.URL;
+import java.util.ArrayList;
 import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
 import java.util.Optional;
+import java.util.concurrent.ConcurrentHashMap;
 
 import javax.swing.JOptionPane;
 
@@ -26,9 +30,10 @@
 public class FindFullTextAction extends AbstractWorker {
     private static final Log LOGGER = LogFactory.getLog(FindFullTextAction.class);
 
+    private static final int warningLimit = 10; // The minimum number of selected entries to ask the user for confirmation
+
     private final BasePanel basePanel;
-    private BibEntry entry;
-    private Optional<URL> result;
+    private final Map<Optional<URL>, BibEntry> downloads = new ConcurrentHashMap<>();
 
     public FindFullTextAction(BasePanel basePanel) {
         this.basePanel = basePanel;
@@ -41,52 +46,73 @@ public void init() throws Throwable {
 
     @Override
     public void run() {
-        if (basePanel.getSelectedEntries().size() != 1) {
-            basePanel.output(Localization.lang("This operation requires exactly one item to be selected."));
-            result = Optional.empty();
-        } else {
-            entry = basePanel.getSelectedEntries().get(0);
+        if (basePanel.getSelectedEntries().size() >= warningLimit) {
+            String[] options = new String[]{Localization.lang("Look up full text documents"), Localization.lang("Cancel")};
+            int answer = JOptionPane.showOptionDialog(basePanel.frame(),
+                    Localization.lang(
+                            "You are about to look up full text documents for %0 entries.",
+                            String.valueOf(basePanel.getSelectedEntries().size())) + "\n"
+                            + Localization.lang("Do you still want to continue?"),
+                    Localization.lang("Look up full text documents"), JOptionPane.OK_CANCEL_OPTION,
+                    JOptionPane.QUESTION_MESSAGE, null, options, options[0]);
+            if (answer != JOptionPane.OK_OPTION) {
+                basePanel.output(Localization.lang("Operation canceled."));
+                return;
+            }
+        }
+        for (BibEntry entry : basePanel.getSelectedEntries()) {
             FulltextFetchers fft = new FulltextFetchers();
-            result = fft.findFullTextPDF(entry);
+            downloads.put(fft.findFullTextPDF(entry), entry);
         }
     }
 
     @Override
     public void update() {
-        if (result.isPresent()) {
-            List<String> dirs = basePanel.getBibDatabaseContext()
-                    .getFileDirectory(Globals.prefs.getFileDirectoryPreferences());
-            if (dirs.isEmpty()) {
-                JOptionPane.showMessageDialog(basePanel.frame(),
-                        Localization.lang("Main file directory not set!") + " " + Localization.lang("Preferences")
-                                + " -> " + Localization.lang("External programs"),
-                        Localization.lang("Directory not found"), JOptionPane.ERROR_MESSAGE);
-                return;
-            }
-            String bibtexKey = entry.getCiteKey();
-            // TODO: this needs its own thread as it blocks the UI!
-            DownloadExternalFile def = new DownloadExternalFile(basePanel.frame(), basePanel.getBibDatabaseContext(), bibtexKey);
-            try {
-                def.download(result.get(), file -> {
-                    FileListTableModel tm = new FileListTableModel();
-                    entry.getField(FieldName.FILE).ifPresent(tm::setContent);
-                    tm.addEntry(tm.getRowCount(), file);
-                    String newValue = tm.getStringRepresentation();
-                    UndoableFieldChange edit = new UndoableFieldChange(entry, FieldName.FILE,
-                            entry.getField(FieldName.FILE).orElse(null), newValue);
-                    entry.setField(FieldName.FILE, newValue);
-                    basePanel.getUndoManager().addEdit(edit);
-                    basePanel.markBaseChanged();
-                });
-            } catch (IOException e) {
-                LOGGER.warn("Problem downloading file", e);
+        List<Optional<URL>> remove = new ArrayList<>();
+        for (Entry<Optional<URL>, BibEntry> download : downloads.entrySet()) {
+            BibEntry entry = download.getValue();
+            Optional<URL> result = download.getKey();
+            if (result.isPresent()) {
+                List<String> dirs = basePanel.getBibDatabaseContext()
+                        .getFileDirectory(Globals.prefs.getFileDirectoryPreferences());
+                if (dirs.isEmpty()) {
+                    JOptionPane.showMessageDialog(basePanel.frame(),
+                            Localization.lang("Main file directory not set!") + " " + Localization.lang("Preferences")
+                                    + " -> " + Localization.lang("External programs"),
+                            Localization.lang("Directory not found"), JOptionPane.ERROR_MESSAGE);
+                    return;
+                }
+                DownloadExternalFile def = new DownloadExternalFile(basePanel.frame(),
+                        basePanel.getBibDatabaseContext(), entry);
+                try {
+                    def.download(result.get(), file -> {
+                        FileListTableModel tm = new FileListTableModel();
+                        entry.getField(FieldName.FILE).ifPresent(tm::setContent);
+                        tm.addEntry(tm.getRowCount(), file);
+                        String newValue = tm.getStringRepresentation();
+                        UndoableFieldChange edit = new UndoableFieldChange(entry, FieldName.FILE,
+                                entry.getField(FieldName.FILE).orElse(null), newValue);
+                        entry.setField(FieldName.FILE, newValue);
+                        basePanel.getUndoManager().addEdit(edit);
+                        basePanel.markBaseChanged();
+                    });
+                } catch (IOException e) {
+                    LOGGER.warn("Problem downloading file", e);
+                }
+                basePanel.output(Localization.lang("Finished downloading full text document for entry %0.",
+                        entry.getCiteKeyOptional().orElse(Localization.lang("undefined"))));
+            } else {
+                String title = Localization.lang("Full text document download failed");
+                String message = Localization.lang("Full text document download failed for entry %0.",
+                        entry.getCiteKeyOptional().orElse(Localization.lang("undefined")));
+
+                basePanel.output(message);
+                JOptionPane.showMessageDialog(basePanel.frame(), message, title, JOptionPane.ERROR_MESSAGE);
             }
-            basePanel.output(Localization.lang("Finished downloading full text document"));
+            remove.add(result);
         }
-        else {
-            String message = Localization.lang("Full text document download failed");
-            basePanel.output(message);
-            JOptionPane.showMessageDialog(basePanel.frame(), message, message, JOptionPane.ERROR_MESSAGE);
+        for (Optional<URL> result : remove) {
+            downloads.remove(result);
         }
     }
 }

src/main/java/net/sf/jabref/external/MoveFileAction.java

@@ -140,7 +140,7 @@ public void actionPerformed(ActionEvent event) {
 
             }
 
-            if (!newFile.equals(file)) {
+            if ((newFile != null) && !newFile.equals(file)) {
                 try {
                     boolean success = file.renameTo(newFile);
                     if (!success) {

src/main/java/net/sf/jabref/gui/fieldeditors/FileListEditor.java

@@ -460,7 +460,7 @@ private void downloadFile() {
             }
         }
         DownloadExternalFile def = new DownloadExternalFile(frame,
-                frame.getCurrentBasePanel().getBibDatabaseContext(), bibtexKey.orElse(null));
+                frame.getCurrentBasePanel().getBibDatabaseContext(), entryEditor.getEntry());
         try {
             def.download(this);
         } catch (IOException ex) {

src/main/java/net/sf/jabref/gui/groups/GroupsTree.java

@@ -168,7 +168,7 @@ public void dragOver(DropTargetDragEvent dtde) {
         } else if (dtde
                 .isDataFlavorSupported(TransferableEntrySelection.FLAVOR_INTERNAL)) {
             // check if node accepts explicit assignment
-            if (path == null) {
+            if (target == null) {
                 dtde.rejectDrag();
             } else {
                 // this would be the place to check if the dragging entries

src/main/java/net/sf/jabref/gui/importer/ImportInspectionDialog.java

@@ -438,13 +438,10 @@ public void entryListComplete() {
     }
 
     /**
-     * Generate key for the selected entry only.
+     * Generate key for an entry.
      */
-    private void generateKeySelectedEntry() {
-        if (selectionModel.getSelected().size() != 1) {
-            return;
-        }
-        BibEntry entry = selectionModel.getSelected().get(0);
+    private void generateKeyForEntry(BibEntry entry) {
+
         entries.getReadWriteLock().writeLock().lock();
         try {
             BibDatabase database;
@@ -1166,21 +1163,19 @@ public void actionPerformed(ActionEvent actionEvent) {
                 return;
             }
             entry = selectionModel.getSelected().get(0);
-            Optional<String> bibtexKey = entry.getCiteKeyOptional();
-            if (!bibtexKey.isPresent()) {
+            if (!entry.getCiteKeyOptional().isPresent()) {
                 int answer = JOptionPane.showConfirmDialog(frame,
                         Localization.lang("This entry has no BibTeX key. Generate key now?"),
                         Localization.lang("Download file"), JOptionPane.OK_CANCEL_OPTION, JOptionPane.QUESTION_MESSAGE);
                 if (answer == JOptionPane.OK_OPTION) {
-                    generateKeySelectedEntry();
-                    bibtexKey = entry.getCiteKeyOptional();
+                    generateKeyForEntry(entry);
                 }
             }
-            DownloadExternalFile def = new DownloadExternalFile(frame, bibDatabaseContext, bibtexKey.get());
+            DownloadExternalFile def = new DownloadExternalFile(frame, bibDatabaseContext, entry);
             try {
                 def.download(this);
             } catch (IOException ex) {
-                LOGGER.warn("Could not downlod file", ex);
+                LOGGER.warn("Could not download file", ex);
             }
         }
 
@@ -1218,7 +1213,7 @@ public void actionPerformed(ActionEvent actionEvent) {
                         Localization.lang("This entry has no BibTeX key. Generate key now?"),
                         Localization.lang("Download file"), JOptionPane.OK_CANCEL_OPTION, JOptionPane.QUESTION_MESSAGE);
                 if (answer == JOptionPane.OK_OPTION) {
-                    generateKeySelectedEntry();
+                    generateKeyForEntry(entry);
                 } else {
                     return; // Can't go on without the bibtex key.
                 }

src/main/java/net/sf/jabref/logic/bibtex/comparator/EntryComparator.java

@@ -87,12 +87,14 @@ public int compare(BibEntry e1, BibEntry e2) {
             }
         }
 
-        if ((f1 == null) && (f2 == null)) {
-            return next == null ? idCompare(e1, e2) : next.compare(e1, e2);
-        }
-        if ((f1 != null) && (f2 == null)) {
-            return -1;
+        if (f2 == null) {
+            if (f1 == null) {
+                return next == null ? idCompare(e1, e2) : next.compare(e1, e2);
+            } else {
+                return -1;
+            }
         }
+
         if (f1 == null) { // f2 != null here automatically
             return 1;
         }

src/main/java/net/sf/jabref/logic/util/io/FileUtil.java

@@ -379,7 +379,8 @@ public static List<File> getListOfLinkedFiles(List<BibEntry> bes, List<String> f
      */
     public static String createFileNameFromPattern(BibDatabase database, BibEntry entry,
             String fileNamePattern, LayoutFormatterPreferences prefs) {
-        String targetName = entry.getCiteKeyOptional().orElse("default");
+        String targetName = null;
+
         StringReader sr = new StringReader(fileNamePattern);
         Layout layout = null;
         try {
@@ -390,6 +391,10 @@ public static String createFileNameFromPattern(BibDatabase database, BibEntry en
         if (layout != null) {
             targetName = layout.doLayout(entry, database);
         }
+
+        if ((targetName == null) || targetName.isEmpty()) {
+            targetName = entry.getCiteKeyOptional().orElse("default");
+        }
         //Removes illegal characters from filename
         targetName = FileNameCleaner.cleanFileName(targetName);
         return targetName;

src/main/resources/l10n/JabRef_da.properties

@@ -869,7 +869,6 @@ Sort_the_following_fields_as_numeric_fields=Sorter_følgende_felter_som_numerisk
 Line_%0\:_Found_corrupted_BibTeX_key.=Linje_%0\:_Fandt_ødelagt_BibTeX-nøgle.
 Line_%0\:_Found_corrupted_BibTeX_key_(contains_whitespaces).=Linje_%0\:_Fandt_ødelagt_BibTeX-nøgle_(indeholder_blanktegn).
 Line_%0\:_Found_corrupted_BibTeX_key_(comma_missing).=Linje_%0\:_Fandt_ødelagt_BibTeX-nøgle_(manglende_komma).
-Finished_downloading_full_text_document=Download_af_fuldtekst-dokument_afsluttet
 Full_text_document_download_failed=Download_af_fuldtekst-dokument_mislykkedes
 Update_to_current_column_order=Brug_nuværende_kolonnerækkefølge
 
@@ -1772,3 +1771,9 @@ Opens_JabRef's_website=
 Opens_a_link_where_the_current_development_version_can_be_downloaded=
 See_what_has_been_changed_in_the_JabRef_versions=
 Referenced_BibTeX_key_does_not_exist=
+
+
+Finished_downloading_full_text_document_for_entry_%0.=
+Full_text_document_download_failed_for_entry_%0.=
+Look_up_full_text_documents=
+You_are_about_to_look_up_full_text_documents_for_%0_entries.=