Lucene search backend

.devcontainer/devcontainer.json

@@ -31,7 +31,7 @@
         // Install java.
         // See https://github.com/devcontainers/features/tree/main/src/java#options for details.
         "ghcr.io/devcontainers/features/java:1": {
-            "version": "20.0.2-tem",
+            "version": "21",
             "installGradle": false,
             "jdkDistro": "tem"
         }

CHANGELOG.md

@@ -11,6 +11,7 @@ Note that this project **does not** adhere to [Semantic Versioning](https://semv
 
 ### Added
 
+- We reintroduced the floating search in the main table. [#8963](https://github.com/JabRef/jabref/pull/8963)
 - We added the possibility to find (and add) papers that cite or are cited by a given paper. [#6187](https://github.com/JabRef/jabref/issues/6187)
 - We added an error-specific message for when a download from a URL fails. [#9826](https://github.com/JabRef/jabref/issues/9826)
 - We added support for customizing the citation command (e.g., `[@key1,@key2]`) when [pushing to external applications](https://docs.jabref.org/cite/pushtoapplications). [#10133](https://github.com/JabRef/jabref/issues/10133)
@@ -27,6 +28,7 @@ Note that this project **does not** adhere to [Semantic Versioning](https://semv
 
 ### Fixed
 
+- The search in the library now displays probable search hits instead of exact matches. Sorting by hit score can be done by the new score table column. [#8963](https://github.com/JabRef/jabref/pull/8963)
 - It is possible again to use "current table sort order" for the order of entries when saving. [#9869](https://github.com/JabRef/jabref/issues/9869)
 - Passwords can be stored in GNOME key ring. [#10274](https://github.com/JabRef/jabref/issues/10274)
 - We fixed an issue where groups based on an aux file could not be created due to an exception [#10350](https://github.com/JabRef/jabref/issues/10350)

build.gradle

@@ -41,15 +41,15 @@ group = "org.jabref"
 version = project.findProperty('projVersion') ?: '100.0.0'
 
 java {
-    sourceCompatibility = JavaVersion.VERSION_19
-    targetCompatibility = JavaVersion.VERSION_19
+    sourceCompatibility = JavaVersion.VERSION_21
+    targetCompatibility = JavaVersion.VERSION_21
     // Workaround needed for Eclipse, probably because of https://github.com/gradle/gradle/issues/16922
     // Should be removed as soon as Gradle 7.0.1 is released ( https://github.com/gradle/gradle/issues/16922#issuecomment-828217060 )
     modularity.inferModulePath.set(false)
 
     toolchain {
         // If this is updated, also update .devcontainer/devcontainer.json#L34
-        languageVersion = JavaLanguageVersion.of(20)
+        languageVersion = JavaLanguageVersion.of(21)
     }
 }
 

src/jmh/java/org/jabref/benchmarks/Benchmarks.java

@@ -3,10 +3,8 @@
 import java.io.IOException;
 import java.io.StringReader;
 import java.io.StringWriter;
-import java.util.EnumSet;
 import java.util.List;
 import java.util.Random;
-import java.util.stream.Collectors;
 
 import org.jabref.gui.Globals;
 import org.jabref.logic.bibtex.FieldPreferences;
@@ -19,7 +17,6 @@
 import org.jabref.logic.importer.fileformat.BibtexParser;
 import org.jabref.logic.layout.format.HTMLChars;
 import org.jabref.logic.layout.format.LatexToUnicodeFormatter;
-import org.jabref.logic.search.SearchQuery;
 import org.jabref.logic.util.OS;
 import org.jabref.model.database.BibDatabase;
 import org.jabref.model.database.BibDatabaseContext;
@@ -33,7 +30,6 @@
 import org.jabref.model.groups.KeywordGroup;
 import org.jabref.model.groups.WordKeywordGroup;
 import org.jabref.model.metadata.MetaData;
-import org.jabref.model.search.rules.SearchRules.SearchFlags;
 import org.jabref.preferences.JabRefPreferences;
 
 import org.openjdk.jmh.Main;
@@ -103,16 +99,14 @@ public String write() throws Exception {
 
     @Benchmark
     public List<BibEntry> search() {
-        // FIXME: Reuse SearchWorker here
-        SearchQuery searchQuery = new SearchQuery("Journal Title 500", EnumSet.noneOf(SearchFlags.class));
-        return database.getEntries().stream().filter(searchQuery::isMatch).collect(Collectors.toList());
+        // FixMe: Create Benchmark for LuceneSearch
+        return null;
     }
 
     @Benchmark
-    public List<BibEntry> parallelSearch() {
-        // FIXME: Reuse SearchWorker here
-        SearchQuery searchQuery = new SearchQuery("Journal Title 500", EnumSet.noneOf(SearchFlags.class));
-        return database.getEntries().parallelStream().filter(searchQuery::isMatch).collect(Collectors.toList());
+    public List<BibEntry> index() {
+        // FixMe: Create Benchmark for LuceneIndexer
+        return null;
     }
 
     @Benchmark

src/main/java/org/jabref/gui/LibraryTab.java

@@ -50,9 +50,9 @@
 import org.jabref.logic.importer.util.FileFieldParser;
 import org.jabref.logic.l10n.Localization;
 import org.jabref.logic.pdf.FileAnnotationCache;
-import org.jabref.logic.pdf.search.indexing.IndexingTaskManager;
-import org.jabref.logic.pdf.search.indexing.PdfIndexer;
 import org.jabref.logic.search.SearchQuery;
+import org.jabref.logic.search.indexing.IndexingTaskManager;
+import org.jabref.logic.search.indexing.LuceneIndexer;
 import org.jabref.logic.shared.DatabaseLocation;
 import org.jabref.logic.util.UpdateField;
 import org.jabref.logic.util.io.FileUtil;
@@ -231,12 +231,11 @@ public void onDatabaseLoadingSucceed(ParserResult result) {
 
         feedData(context);
 
-        if (preferencesService.getFilePreferences().shouldFulltextIndexLinkedFiles()) {
-            try {
-                indexingTaskManager.updateIndex(PdfIndexer.of(bibDatabaseContext, preferencesService.getFilePreferences()), bibDatabaseContext);
-            } catch (IOException e) {
-                LOGGER.error("Cannot access lucene index", e);
-            }
+        try {
+            indexingTaskManager.manageFulltextIndexAccordingToPrefs(LuceneIndexer.of(bibDatabaseContext, preferencesService));
+            indexingTaskManager.updateIndex(LuceneIndexer.of(bibDatabaseContext, preferencesService));
+        } catch (IOException e) {
+            LOGGER.error("Cannot access lucene index", e);
         }
     }
 
@@ -270,6 +269,9 @@ public void feedData(BibDatabaseContext bibDatabaseContextFromParserResult) {
         bibDatabaseContextFromParserResult.getDatabase().registerListener(this);
         bibDatabaseContextFromParserResult.getMetaData().registerListener(this);
 
+        if (this.tableModel != null) {
+            this.tableModel.removeBindings();
+        }
         this.tableModel = new MainTableDataModel(getBibDatabaseContext(), preferencesService, stateManager);
         citationStyleCache = new CitationStyleCache(bibDatabaseContextFromParserResult);
         annotationCache = new FileAnnotationCache(bibDatabaseContextFromParserResult, preferencesService.getFilePreferences());
@@ -389,9 +391,7 @@ public void updateTabTitle(boolean isChanged) {
             setTooltip(new Tooltip(toolTipText.toString()));
         });
 
-        if (preferencesService.getFilePreferences().shouldFulltextIndexLinkedFiles()) {
-            indexingTaskManager.updateDatabaseName(tabTitle.toString());
-        }
+        indexingTaskManager.updateDatabaseName(tabTitle.toString());
     }
 
     @Subscribe
@@ -904,7 +904,7 @@ public void listen(EntriesAddedEvent addedEntriesEvent) {
 
             // Automatically add new entries to the selected group (or set of groups)
             if (preferencesService.getGroupsPreferences().shouldAutoAssignGroup()) {
-                stateManager.getSelectedGroup(bibDatabaseContext).forEach(
+                stateManager.getSelectedGroups(bibDatabaseContext).forEach(
                         selectedGroup -> selectedGroup.addEntriesToGroup(addedEntriesEvent.getBibEntries()));
             }
         }
@@ -944,50 +944,40 @@ private class IndexUpdateListener {
 
         @Subscribe
         public void listen(EntriesAddedEvent addedEntryEvent) {
-            if (preferencesService.getFilePreferences().shouldFulltextIndexLinkedFiles()) {
-                try {
-                    PdfIndexer pdfIndexer = PdfIndexer.of(bibDatabaseContext, preferencesService.getFilePreferences());
-                    for (BibEntry addedEntry : addedEntryEvent.getBibEntries()) {
-                        indexingTaskManager.addToIndex(pdfIndexer, addedEntry, bibDatabaseContext);
-                    }
-                } catch (IOException e) {
-                    LOGGER.error("Cannot access lucene index", e);
+            try {
+                LuceneIndexer luceneIndexer = LuceneIndexer.of(bibDatabaseContext, preferencesService);
+                for (BibEntry addedEntry : addedEntryEvent.getBibEntries()) {
+                    indexingTaskManager.addToIndex(luceneIndexer, addedEntry);
                 }
+            } catch (IOException e) {
+                LOGGER.error("Cannot access lucene index", e);
             }
         }
 
         @Subscribe
         public void listen(EntriesRemovedEvent removedEntriesEvent) {
-            if (preferencesService.getFilePreferences().shouldFulltextIndexLinkedFiles()) {
-                try {
-                    PdfIndexer pdfIndexer = PdfIndexer.of(bibDatabaseContext, preferencesService.getFilePreferences());
-                    for (BibEntry removedEntry : removedEntriesEvent.getBibEntries()) {
-                        indexingTaskManager.removeFromIndex(pdfIndexer, removedEntry);
-                    }
-                } catch (IOException e) {
-                    LOGGER.error("Cannot access lucene index", e);
+            try {
+                LuceneIndexer luceneIndexer = LuceneIndexer.of(bibDatabaseContext, preferencesService);
+                for (BibEntry removedEntry : removedEntriesEvent.getBibEntries()) {
+                    indexingTaskManager.removeFromIndex(luceneIndexer, removedEntry);
                 }
+            } catch (IOException e) {
+                LOGGER.error("Cannot access lucene index", e);
             }
         }
 
         @Subscribe
         public void listen(FieldChangedEvent fieldChangedEvent) {
-            if (preferencesService.getFilePreferences().shouldFulltextIndexLinkedFiles()) {
-                if (fieldChangedEvent.getField().equals(StandardField.FILE)) {
-                    List<LinkedFile> oldFileList = FileFieldParser.parse(fieldChangedEvent.getOldValue());
-                    List<LinkedFile> newFileList = FileFieldParser.parse(fieldChangedEvent.getNewValue());
-
-                    List<LinkedFile> addedFiles = new ArrayList<>(newFileList);
-                    addedFiles.remove(oldFileList);
-                    List<LinkedFile> removedFiles = new ArrayList<>(oldFileList);
-                    removedFiles.remove(newFileList);
-
-                    try {
-                        indexingTaskManager.addToIndex(PdfIndexer.of(bibDatabaseContext, preferencesService.getFilePreferences()), fieldChangedEvent.getBibEntry(), addedFiles, bibDatabaseContext);
-                        indexingTaskManager.removeFromIndex(PdfIndexer.of(bibDatabaseContext, preferencesService.getFilePreferences()), fieldChangedEvent.getBibEntry(), removedFiles);
-                    } catch (IOException e) {
-                        LOGGER.warn("I/O error when writing lucene index", e);
+            for (BibEntry bibEntry : fieldChangedEvent.getBibEntries()) {
+                try {
+                    List<LinkedFile> toRemoveList = new ArrayList<>();
+                    if (fieldChangedEvent.getField().equals(StandardField.FILE)) {
+                        toRemoveList.addAll(FileFieldParser.parse(fieldChangedEvent.getOldValue()));
+                        toRemoveList.removeAll(FileFieldParser.parse(fieldChangedEvent.getNewValue()));
                     }
+                    indexingTaskManager.updateIndex(LuceneIndexer.of(bibDatabaseContext, preferencesService), bibEntry, toRemoveList);
+                } catch (IOException e) {
+                    LOGGER.warn("I/O error when writing lucene index", e);
                 }
             }
         }

src/main/java/org/jabref/gui/MainMenu.java

@@ -283,7 +283,7 @@ private void createMenu() {
 
                 new SeparatorMenuItem(),
 
-                factory.createMenuItem(StandardActions.REBUILD_FULLTEXT_SEARCH_INDEX, new RebuildFulltextSearchIndexAction(stateManager, frame::getCurrentLibraryTab, dialogService, preferencesService.getFilePreferences(), taskExecutor))
+                factory.createMenuItem(StandardActions.REBUILD_FULLTEXT_SEARCH_INDEX, new RebuildFulltextSearchIndexAction(stateManager, frame::getCurrentLibraryTab, dialogService, preferencesService, preferencesService.getFilePreferences(), taskExecutor))
         );
         SidePaneType webSearchPane = SidePaneType.WEB_SEARCH;
         SidePaneType groupsPane = SidePaneType.GROUPS;

src/main/java/org/jabref/gui/StateManager.java

@@ -2,17 +2,16 @@
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.stream.Collectors;
 
 import javafx.beans.Observable;
 import javafx.beans.binding.Bindings;
-import javafx.beans.property.IntegerProperty;
 import javafx.beans.property.ObjectProperty;
 import javafx.beans.property.ReadOnlyListProperty;
 import javafx.beans.property.ReadOnlyListWrapper;
-import javafx.beans.property.SimpleIntegerProperty;
 import javafx.beans.property.SimpleObjectProperty;
 import javafx.collections.FXCollections;
 import javafx.collections.ObservableList;
@@ -31,6 +30,7 @@
 import org.jabref.model.database.BibDatabaseContext;
 import org.jabref.model.entry.BibEntry;
 import org.jabref.model.groups.GroupTreeNode;
+import org.jabref.model.pdf.search.LuceneSearchResults;
 import org.jabref.model.util.OptionalUtil;
 
 import com.tobiasdiez.easybind.EasyBind;
@@ -60,7 +60,7 @@ public class StateManager {
     private final ObservableList<BibEntry> selectedEntries = FXCollections.observableArrayList();
     private final ObservableMap<BibDatabaseContext, ObservableList<GroupTreeNode>> selectedGroups = FXCollections.observableHashMap();
     private final OptionalObjectProperty<SearchQuery> activeSearchQuery = OptionalObjectProperty.empty();
-    private final ObservableMap<BibDatabaseContext, IntegerProperty> searchResultMap = FXCollections.observableHashMap();
+    private final ObservableMap<BibDatabaseContext, Map<BibEntry, LuceneSearchResults>> searchResults = FXCollections.observableHashMap();
     private final OptionalObjectProperty<Node> focusOwner = OptionalObjectProperty.empty();
     private final ObservableList<Pair<BackgroundTask<?>, Task<?>>> backgroundTasks = FXCollections.observableArrayList(task -> new Observable[]{task.getValue().progressProperty(), task.getValue().runningProperty()});
     private final EasyBinding<Boolean> anyTaskRunning = EasyBind.reduce(backgroundTasks, tasks -> tasks.map(Pair::getValue).anyMatch(Task::isRunning));
@@ -97,14 +97,6 @@ public OptionalObjectProperty<SearchQuery> activeSearchQueryProperty() {
         return activeSearchQuery;
     }
 
-    public void setActiveSearchResultSize(BibDatabaseContext database, IntegerProperty resultSize) {
-        searchResultMap.put(database, resultSize);
-    }
-
-    public IntegerProperty getSearchResultSize() {
-        return searchResultMap.getOrDefault(activeDatabase.getValue().orElse(new BibDatabaseContext()), new SimpleIntegerProperty(0));
-    }
-
     public ReadOnlyListProperty<GroupTreeNode> activeGroupProperty() {
         return activeGroups.getReadOnlyProperty();
     }
@@ -122,7 +114,7 @@ public void setSelectedGroups(BibDatabaseContext database, List<GroupTreeNode> n
         selectedGroups.put(database, FXCollections.observableArrayList(newSelectedGroups));
     }
 
-    public ObservableList<GroupTreeNode> getSelectedGroup(BibDatabaseContext database) {
+    public ObservableList<GroupTreeNode> getSelectedGroups(BibDatabaseContext database) {
         ObservableList<GroupTreeNode> selectedGroupsForDatabase = selectedGroups.get(database);
         return selectedGroupsForDatabase != null ? selectedGroupsForDatabase : FXCollections.observableArrayList();
     }
@@ -205,6 +197,10 @@ public void setLastAutomaticFieldEditorEdit(LastAutomaticFieldEditorEdit automat
         lastAutomaticFieldEditorEditProperty().set(automaticFieldEditorEdit);
     }
 
+    public ObservableMap<BibDatabaseContext, Map<BibEntry, LuceneSearchResults>> getSearchResults() {
+        return searchResults;
+    }
+
     public List<String> collectAllDatabasePaths() {
         List<String> list = new ArrayList<>();
         getOpenDatabases().stream()

src/main/java/org/jabref/gui/entryeditor/CommentsTab.java

@@ -16,7 +16,7 @@
 import org.jabref.gui.util.TaskExecutor;
 import org.jabref.logic.journals.JournalAbbreviationRepository;
 import org.jabref.logic.l10n.Localization;
-import org.jabref.logic.pdf.search.indexing.IndexingTaskManager;
+import org.jabref.logic.search.indexing.IndexingTaskManager;
 import org.jabref.model.database.BibDatabaseContext;
 import org.jabref.model.entry.BibEntry;
 import org.jabref.model.entry.field.Field;
@@ -35,9 +35,9 @@ public CommentsTab(PreferencesService preferences,
                        DialogService dialogService,
                        StateManager stateManager,
                        ThemeManager themeManager,
-                       IndexingTaskManager indexingTaskManager,
                        TaskExecutor taskExecutor,
-                       JournalAbbreviationRepository journalAbbreviationRepository) {
+                       JournalAbbreviationRepository journalAbbreviationRepository,
+                       IndexingTaskManager indexingTaskManager) {
         super(
                 false,
                 databaseContext,
@@ -49,8 +49,7 @@ public CommentsTab(PreferencesService preferences,
                 themeManager,
                 taskExecutor,
                 journalAbbreviationRepository,
-                indexingTaskManager
-        );
+                indexingTaskManager);
         this.defaultOwner = preferences.getOwnerPreferences().getDefaultOwner();
         setText(Localization.lang("Comments"));
         setGraphic(IconTheme.JabRefIcons.COMMENT.getGraphicNode());

src/main/java/org/jabref/gui/entryeditor/DeprecatedFieldsTab.java

@@ -17,7 +17,7 @@
 import org.jabref.gui.util.TaskExecutor;
 import org.jabref.logic.journals.JournalAbbreviationRepository;
 import org.jabref.logic.l10n.Localization;
-import org.jabref.logic.pdf.search.indexing.IndexingTaskManager;
+import org.jabref.logic.search.indexing.IndexingTaskManager;
 import org.jabref.model.database.BibDatabaseContext;
 import org.jabref.model.database.BibDatabaseMode;
 import org.jabref.model.entry.BibEntry;