Skip to content

Commit

Permalink
Implement an interface to import PDF metadata from multiple sources (…
Browse files Browse the repository at this point in the history 
…XMP, Grobid, ...) (#7929)

Co-authored-by: Christoph <siedlerkiller@gmail.com>
Co-authored-by: Carl Christian Snethlage <50491877+calixtus@users.noreply.github.com>
  • Loading branch information
@btut @Siedlerchr @calixtus
3 people authored Aug 21, 2021
1 parent 8720fd2 commit fd1cab0
Show file tree
Hide file tree
Showing 17 changed files with 924 additions and 69 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ Note that this project **does not** adhere to [Semantic Versioning](http://semve
- We added a fulltext search feature. [#2838](https://github.com/JabRef/jabref/pull/2838)
- We improved the deduction of bib-entries from imported fulltext pdfs. [#7947](https://github.com/JabRef/jabref/pull/7947)
- We added unprotect_terms to the list of bracketed pattern modifiers [#7826](https://github.com/JabRef/jabref/pull/7960)
- We added a dialog that allows to parse metadata from linked pdfs. [#7929](https://github.com/JabRef/jabref/pull/7929)
- We added an icon picker in group edit dialog. [#6142](https://github.com/JabRef/jabref/issues/6142)
- We added a preference to Opt-In to JabRef's online metadata extraction service (Grobid) usage. [8002](https://github.com/JabRef/jabref/pull/8002)

Expand Down
3 changes: 1 addition & 2 deletions src/main/java/org/jabref/gui/externalfiles/DownloadFullTextAction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -144,8 +144,7 @@ private void addLinkedFileFromURL(BibDatabaseContext databaseContext, URL url, B
databaseContext,
Globals.TASK_EXECUTOR,
dialogService,
preferences.getXmpPreferences(),
preferences.getFilePreferences(),
preferences,
ExternalFileTypes.getInstance());

onlineFile.download();
Expand Down
87 changes: 61 additions & 26 deletions src/main/java/org/jabref/gui/fieldeditors/LinkedFileViewModel.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
import java.util.List;
import java.util.Optional;
import java.util.function.BiPredicate;
import java.util.function.Supplier;

import javax.net.ssl.HostnameVerifier;
import javax.net.ssl.HttpsURLConnection;
Expand All @@ -35,23 +36,30 @@
import org.jabref.gui.icon.IconTheme;
import org.jabref.gui.icon.JabRefIcon;
import org.jabref.gui.linkedfile.LinkedFileEditDialogView;
import org.jabref.gui.mergeentries.MultiMergeEntriesView;
import org.jabref.gui.util.BackgroundTask;
import org.jabref.gui.util.ControlHelper;
import org.jabref.gui.util.TaskExecutor;
import org.jabref.logic.externalfiles.LinkedFileHandler;
import org.jabref.logic.importer.Importer;
import org.jabref.logic.importer.ParserResult;
import org.jabref.logic.importer.fileformat.PdfContentImporter;
import org.jabref.logic.importer.fileformat.PdfEmbeddedBibFileImporter;
import org.jabref.logic.importer.fileformat.PdfGrobidImporter;
import org.jabref.logic.importer.fileformat.PdfVerbatimBibTextImporter;
import org.jabref.logic.importer.fileformat.PdfXmpImporter;
import org.jabref.logic.l10n.Localization;
import org.jabref.logic.net.URLDownload;
import org.jabref.logic.util.io.FileNameUniqueness;
import org.jabref.logic.util.io.FileUtil;
import org.jabref.logic.xmp.XmpPreferences;
import org.jabref.logic.xmp.XmpUtilWriter;
import org.jabref.model.database.BibDatabaseContext;
import org.jabref.model.entry.BibEntry;
import org.jabref.model.entry.LinkedFile;
import org.jabref.model.strings.StringUtil;
import org.jabref.model.util.FileHelper;
import org.jabref.model.util.OptionalUtil;
import org.jabref.preferences.FilePreferences;
import org.jabref.preferences.PreferencesService;

import de.saxsys.mvvmfx.utils.validation.FunctionBasedValidator;
import de.saxsys.mvvmfx.utils.validation.ValidationMessage;
Expand All @@ -69,12 +77,11 @@ public class LinkedFileViewModel extends AbstractViewModel {
private final DoubleProperty downloadProgress = new SimpleDoubleProperty(-1);
private final BooleanProperty downloadOngoing = new SimpleBooleanProperty(false);
private final BooleanProperty isAutomaticallyFound = new SimpleBooleanProperty(false);
private final BooleanProperty canWriteXMPMetadata = new SimpleBooleanProperty(false);
private final BooleanProperty isOfflinePdf = new SimpleBooleanProperty(false);
private final DialogService dialogService;
private final BibEntry entry;
private final TaskExecutor taskExecutor;
private final FilePreferences filePreferences;
private final XmpPreferences xmpPreferences;
private final PreferencesService preferences;
private final LinkedFileHandler linkedFileHandler;
private final ExternalFileTypes externalFileTypes;

Expand All @@ -85,38 +92,36 @@ public LinkedFileViewModel(LinkedFile linkedFile,
BibDatabaseContext databaseContext,
TaskExecutor taskExecutor,
DialogService dialogService,
XmpPreferences xmpPreferences,
FilePreferences filePreferences,
PreferencesService preferences,
ExternalFileTypes externalFileTypes) {

this.linkedFile = linkedFile;
this.filePreferences = filePreferences;
this.linkedFileHandler = new LinkedFileHandler(linkedFile, entry, databaseContext, filePreferences);
this.preferences = preferences;
this.linkedFileHandler = new LinkedFileHandler(linkedFile, entry, databaseContext, preferences.getFilePreferences());
this.databaseContext = databaseContext;
this.entry = entry;
this.dialogService = dialogService;
this.taskExecutor = taskExecutor;
this.externalFileTypes = externalFileTypes;
this.xmpPreferences = xmpPreferences;

fileExistsValidator = new FunctionBasedValidator<>(
linkedFile.linkProperty(),
link -> {
if (linkedFile.isOnlineLink()) {
return true;
} else {
Optional<Path> path = FileHelper.find(databaseContext, link, filePreferences);
Optional<Path> path = FileHelper.find(databaseContext, link, preferences.getFilePreferences());
return path.isPresent() && Files.exists(path.get());
}
},
ValidationMessage.warning(Localization.lang("Could not find file '%0'.", linkedFile.getLink())));

downloadOngoing.bind(downloadProgress.greaterThanOrEqualTo(0).and(downloadProgress.lessThan(1)));
canWriteXMPMetadata.setValue(!linkedFile.isOnlineLink() && linkedFile.getFileType().equalsIgnoreCase("pdf"));
isOfflinePdf.setValue(!linkedFile.isOnlineLink() && linkedFile.getFileType().equalsIgnoreCase("pdf"));
}

public BooleanProperty canWriteXMPMetadataProperty() {
return canWriteXMPMetadata;
public BooleanProperty isOfflinePdfProperty() {
return isOfflinePdf;
}

public boolean isAutomaticallyFound() {
Expand Down Expand Up @@ -211,7 +216,7 @@ public void openFolder() {
Optional<Path> resolvedPath = FileHelper.find(
databaseContext,
linkedFile.getLink(),
filePreferences);
preferences.getFilePreferences());

if (resolvedPath.isPresent()) {
JabRefDesktop.openFolderAndSelectFile(resolvedPath.get());
Expand Down Expand Up @@ -246,7 +251,7 @@ public void renameFileToName(String targetFileName) {
return;
}

Optional<Path> file = linkedFile.findIn(databaseContext, filePreferences);
Optional<Path> file = linkedFile.findIn(databaseContext, preferences.getFilePreferences());
if (file.isPresent()) {
performRenameWithConflictCheck(targetFileName);
} else {
Expand Down Expand Up @@ -283,13 +288,13 @@ public void moveToDefaultDirectory() {
}

// Get target folder
Optional<Path> fileDir = databaseContext.getFirstExistingFileDir(filePreferences);
Optional<Path> fileDir = databaseContext.getFirstExistingFileDir(preferences.getFilePreferences());
if (fileDir.isEmpty()) {
dialogService.showErrorDialogAndWait(Localization.lang("Move file"), Localization.lang("File directory is not set or does not exist!"));
return;
}

Optional<Path> file = linkedFile.findIn(databaseContext, filePreferences);
Optional<Path> file = linkedFile.findIn(databaseContext, preferences.getFilePreferences());
if ((file.isPresent())) {
// Found the linked file, so move it
try {
Expand Down Expand Up @@ -325,9 +330,9 @@ public boolean isGeneratedNameSameAsOriginal() {
* @return true if suggested filepath is same as existing filepath.
*/
public boolean isGeneratedPathSameAsOriginal() {
Optional<Path> newDir = databaseContext.getFirstExistingFileDir(filePreferences);
Optional<Path> newDir = databaseContext.getFirstExistingFileDir(preferences.getFilePreferences());

Optional<Path> currentDir = linkedFile.findIn(databaseContext, filePreferences).map(Path::getParent);
Optional<Path> currentDir = linkedFile.findIn(databaseContext, preferences.getFilePreferences()).map(Path::getParent);

BiPredicate<Path, Path> equality = (fileA, fileB) -> {
try {
Expand All @@ -351,7 +356,7 @@ public void moveToDefaultDirectoryAndRename() {
* successfully, does not exist in the first place or the user choose to remove it)
*/
public boolean delete() {
Optional<Path> file = linkedFile.findIn(databaseContext, filePreferences);
Optional<Path> file = linkedFile.findIn(databaseContext, preferences.getFilePreferences());

if (file.isEmpty()) {
LOGGER.warn("Could not find file " + linkedFile.getLink());
Expand Down Expand Up @@ -395,13 +400,13 @@ public void edit() {
public void writeXMPMetadata() {
// Localization.lang("Writing XMP metadata...")
BackgroundTask<Void> writeTask = BackgroundTask.wrap(() -> {
Optional<Path> file = linkedFile.findIn(databaseContext, filePreferences);
Optional<Path> file = linkedFile.findIn(databaseContext, preferences.getFilePreferences());
if (file.isEmpty()) {
// TODO: Print error message
// Localization.lang("PDF does not exist");
} else {
try {
XmpUtilWriter.writeXmp(file.get(), entry, databaseContext.getDatabase(), xmpPreferences);
XmpUtilWriter.writeXmp(file.get(), entry, databaseContext.getDatabase(), preferences.getXmpPreferences());
} catch (IOException | TransformerException ex) {
// TODO: Print error message
// Localization.lang("Error while writing") + " '" + file.toString() + "': " + ex;
Expand All @@ -421,7 +426,7 @@ public void download() {
throw new UnsupportedOperationException("In order to download the file it has to be an online link");
}
try {
Optional<Path> targetDirectory = databaseContext.getFirstExistingFileDir(filePreferences);
Optional<Path> targetDirectory = databaseContext.getFirstExistingFileDir(preferences.getFilePreferences());
if (targetDirectory.isEmpty()) {
dialogService.showErrorDialogAndWait(Localization.lang("Download file"), Localization.lang("File directory is not set or does not exist!"));
return;
Expand All @@ -443,7 +448,7 @@ public void download() {
}

if (!isDuplicate) {
LinkedFile newLinkedFile = LinkedFilesEditorViewModel.fromFile(destination, databaseContext.getFileDirectories(filePreferences), externalFileTypes);
LinkedFile newLinkedFile = LinkedFilesEditorViewModel.fromFile(destination, databaseContext.getFileDirectories(preferences.getFilePreferences()), externalFileTypes);
List<LinkedFile> linkedFiles = entry.getFiles();

entry.addLinkedFile(entry, linkedFile, newLinkedFile, linkedFiles);
Expand Down Expand Up @@ -495,7 +500,7 @@ public BackgroundTask<Path> prepareDownloadTask(Path targetDirectory, URLDownloa
String suggestedTypeName = externalFileType.getName();
linkedFile.setFileType(suggestedTypeName);
String suggestedName = linkedFileHandler.getSuggestedFileName(externalFileType.getExtension());
String fulltextDir = FileUtil.createDirNameFromPattern(databaseContext.getDatabase(), entry, filePreferences.getFileDirectoryPattern());
String fulltextDir = FileUtil.createDirNameFromPattern(databaseContext.getDatabase(), entry, preferences.getFilePreferences().getFileDirectoryPattern());
suggestedName = FileNameUniqueness.getNonOverWritingFileName(targetDirectory.resolve(fulltextDir), suggestedName);
return targetDirectory.resolve(fulltextDir).resolve(suggestedName);
})
Expand Down Expand Up @@ -538,4 +543,34 @@ public LinkedFile getFile() {
public ValidationStatus fileExistsValidationStatus() {
return fileExistsValidator.getValidationStatus();
}

public void parsePdfMetadataAndShowMergeDialog() {
linkedFile.findIn(databaseContext, preferences.getFilePreferences()).ifPresent(filePath -> {
MultiMergeEntriesView dialog = new MultiMergeEntriesView(preferences, taskExecutor);
dialog.addSource(Localization.lang("Entry"), entry);
dialog.addSource(Localization.lang("Verbatim"), wrapImporterToSupplier(new PdfVerbatimBibTextImporter(preferences.getImportFormatPreferences()), filePath));
dialog.addSource(Localization.lang("Embedded"), wrapImporterToSupplier(new PdfEmbeddedBibFileImporter(preferences.getImportFormatPreferences()), filePath));
dialog.addSource("Grobid", wrapImporterToSupplier(new PdfGrobidImporter(preferences.getImportSettingsPreferences(), preferences.getImportFormatPreferences()), filePath));
dialog.addSource(Localization.lang("XMP metadata"), wrapImporterToSupplier(new PdfXmpImporter(preferences.getXmpPreferences()), filePath));
dialog.addSource(Localization.lang("Content"), wrapImporterToSupplier(new PdfContentImporter(preferences.getImportFormatPreferences()), filePath));
dialog.showAndWait().ifPresent(newEntry -> {
databaseContext.getDatabase().removeEntry(entry);
databaseContext.getDatabase().insertEntry(newEntry);
});
});
}

private Supplier<BibEntry> wrapImporterToSupplier(Importer importer, Path filePath) {
return () -> {
try {
ParserResult parserResult = importer.importDatabase(filePath, preferences.getDefaultEncoding());
if (parserResult.isInvalid() || parserResult.isEmpty() || !parserResult.getDatabase().hasEntries()) {
return null;
}
return parserResult.getDatabase().getEntries().get(0);
} catch (IOException e) {
return null;
}
};
}
}
20 changes: 16 additions & 4 deletions src/main/java/org/jabref/gui/fieldeditors/LinkedFilesEditor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import javafx.scene.input.MouseEvent;
import javafx.scene.input.TransferMode;
import javafx.scene.layout.HBox;
import javafx.scene.layout.Priority;
import javafx.scene.text.Text;

import org.jabref.gui.DialogService;
Expand All @@ -34,6 +35,7 @@
import org.jabref.gui.autocompleter.SuggestionProvider;
import org.jabref.gui.copyfiles.CopySingleFileAction;
import org.jabref.gui.icon.IconTheme;
import org.jabref.gui.importer.GrobidOptInDialogHelper;
import org.jabref.gui.keyboard.KeyBinding;
import org.jabref.gui.util.BindingsHelper;
import org.jabref.gui.util.TaskExecutor;
Expand Down Expand Up @@ -81,7 +83,7 @@ public LinkedFilesEditor(Field field,

ViewModelListCellFactory<LinkedFileViewModel> cellFactory = new ViewModelListCellFactory<LinkedFileViewModel>()
.withStringTooltip(LinkedFileViewModel::getDescription)
.withGraphic(LinkedFilesEditor::createFileDisplay)
.withGraphic(this::createFileDisplay)
.withContextMenu(this::createContextMenuForFile)
.withOnMouseClickedEvent(this::handleItemMouseClick)
.setOnDragDetected(this::handleOnDragDetected)
Expand Down Expand Up @@ -142,7 +144,7 @@ private void handleOnDragDropped(LinkedFileViewModel originalItem, DragEvent eve
event.consume();
}

private static Node createFileDisplay(LinkedFileViewModel linkedFile) {
private Node createFileDisplay(LinkedFileViewModel linkedFile) {
PseudoClass opacity = PseudoClass.getPseudoClass("opacity");

Node icon = linkedFile.getTypeIcon().getGraphicNode();
Expand All @@ -162,6 +164,7 @@ private static Node createFileDisplay(LinkedFileViewModel linkedFile) {
progressIndicator.visibleProperty().bind(linkedFile.downloadOngoingProperty());

HBox info = new HBox(8);
HBox.setHgrow(info, Priority.ALWAYS);
info.setStyle("-fx-padding: 0.5em 0 0.5em 0;"); // To align with buttons below which also have 0.5em padding
info.getChildren().setAll(icon, link, desc, progressIndicator);

Expand All @@ -174,14 +177,23 @@ private static Node createFileDisplay(LinkedFileViewModel linkedFile) {

Button writeXMPMetadata = IconTheme.JabRefIcons.IMPORT.asButton();
writeXMPMetadata.setTooltip(new Tooltip(Localization.lang("Write BibTeXEntry as XMP metadata to PDF.")));
writeXMPMetadata.visibleProperty().bind(linkedFile.canWriteXMPMetadataProperty());
writeXMPMetadata.visibleProperty().bind(linkedFile.isOfflinePdfProperty());
writeXMPMetadata.setOnAction(event -> linkedFile.writeXMPMetadata());
writeXMPMetadata.getStyleClass().setAll("icon-button");

Button parsePdfMetadata = IconTheme.JabRefIcons.FILE_SEARCH.asButton();
parsePdfMetadata.setTooltip(new Tooltip(Localization.lang("Parse Metadata from PDF.")));
parsePdfMetadata.visibleProperty().bind(linkedFile.isOfflinePdfProperty());
parsePdfMetadata.setOnAction(event -> {
GrobidOptInDialogHelper.showAndWaitIfUserIsUndecided(dialogService);
linkedFile.parsePdfMetadataAndShowMergeDialog();
});
parsePdfMetadata.getStyleClass().setAll("icon-button");

HBox container = new HBox(10);
container.setPrefHeight(Double.NEGATIVE_INFINITY);

container.getChildren().addAll(acceptAutoLinkedFile, info, writeXMPMetadata);
container.getChildren().addAll(acceptAutoLinkedFile, info, writeXMPMetadata, parsePdfMetadata);

return container;
}
Expand Down
15 changes: 5 additions & 10 deletions src/main/java/org/jabref/gui/fieldeditors/LinkedFilesEditorViewModel.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -106,8 +106,7 @@ public LinkedFileViewModel fromFile(Path file) {
databaseContext,
taskExecutor,
dialogService,
preferences.getXmpPreferences(),
preferences.getFilePreferences(),
preferences,
externalFileTypes);
}

Expand All @@ -123,8 +122,7 @@ private List<LinkedFileViewModel> parseToFileViewModel(String stringValue) {
databaseContext,
taskExecutor,
dialogService,
preferences.getXmpPreferences(),
preferences.getFilePreferences(),
preferences,
externalFileTypes))
.collect(Collectors.toList());
}
Expand Down Expand Up @@ -154,8 +152,7 @@ public void addNewFile() {
databaseContext,
taskExecutor,
dialogService,
preferences.getXmpPreferences(),
preferences.getFilePreferences(),
preferences,
externalFileTypes));
});
}
Expand Down Expand Up @@ -192,8 +189,7 @@ private List<LinkedFileViewModel> findAssociatedNotLinkedFiles(BibEntry entry) {
databaseContext,
taskExecutor,
dialogService,
preferences.getXmpPreferences(),
preferences.getFilePreferences(),
preferences,
externalFileTypes);
newLinkedFile.markAsAutomaticallyFound();
result.add(newLinkedFile);
Expand Down Expand Up @@ -243,8 +239,7 @@ private void addFromURL(URL url) {
databaseContext,
taskExecutor,
dialogService,
preferences.getXmpPreferences(),
preferences.getFilePreferences(),
preferences,
externalFileTypes);
files.add(onlineFile);
onlineFile.download();
Expand Down
Loading

0 comments on commit fd1cab0

Please sign in to comment.