diff --git a/CHANGELOG.md b/CHANGELOG.md index 9bdaedf8512..40f8f00ee26 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,7 @@ Note that this project **does not** adhere to [Semantic Versioning](http://semve - We fixed the bug when strike the delete key in the text field. [#6421](https://github.com/JabRef/jabref/issues/6421) - We added a BibTex key modifier for truncating strings. [#3915](https://github.com/JabRef/jabref/issues/3915) - We added support for jumping to target entry when typing letter/digit after sorting a column in maintable [#6146](https://github.com/JabRef/jabref/issues/6146) +- We added a new fetcher to enable users to search all available E-Libraries simultaneously. [koppor#369](https://github.com/koppor/jabref/issues/369) - We added the field "entrytype" to the export sort criteria [#6531](https://github.com/JabRef/jabref/pull/6531) ### Changed diff --git a/src/main/java/org/jabref/logic/importer/WebFetchers.java b/src/main/java/org/jabref/logic/importer/WebFetchers.java index f21521fa6a8..bc2efc8b90d 100644 --- a/src/main/java/org/jabref/logic/importer/WebFetchers.java +++ b/src/main/java/org/jabref/logic/importer/WebFetchers.java @@ -12,6 +12,7 @@ import org.jabref.logic.importer.fetcher.ArXiv; import org.jabref.logic.importer.fetcher.AstrophysicsDataSystem; import org.jabref.logic.importer.fetcher.CiteSeer; +import org.jabref.logic.importer.fetcher.CompositeSearchBasedFetcher; import org.jabref.logic.importer.fetcher.CrossRef; import org.jabref.logic.importer.fetcher.DBLPFetcher; import org.jabref.logic.importer.fetcher.DOAJFetcher; @@ -99,6 +100,7 @@ public static SortedSet getSearchBasedFetchers(ImportFormatP set.add(new CiteSeer()); set.add(new DOAJFetcher(importFormatPreferences)); set.add(new IEEE(importFormatPreferences)); + set.add(new CompositeSearchBasedFetcher(set, 30)); return set; } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/CompositeSearchBasedFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/CompositeSearchBasedFetcher.java new file mode 100644 index 00000000000..95ea76e66eb --- /dev/null +++ b/src/main/java/org/jabref/logic/importer/fetcher/CompositeSearchBasedFetcher.java @@ -0,0 +1,59 @@ +package org.jabref.logic.importer.fetcher; + +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import org.jabref.logic.help.HelpFile; +import org.jabref.logic.importer.FetcherException; +import org.jabref.logic.importer.SearchBasedFetcher; +import org.jabref.model.entry.BibEntry; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class CompositeSearchBasedFetcher implements SearchBasedFetcher { + + private static final Logger LOGGER = LoggerFactory.getLogger(CompositeSearchBasedFetcher.class); + + private final Set fetchers; + private final int maximumNumberOfReturnedResults; + + public CompositeSearchBasedFetcher(Set searchBasedFetchers, int maximumNumberOfReturnedResults) + throws IllegalArgumentException { + if (searchBasedFetchers == null) { + throw new IllegalArgumentException("The set of searchBasedFetchers must not be null!"); + } + // Remove the Composite Fetcher instance from its own fetcher set to prevent a StackOverflow + this.fetchers = searchBasedFetchers.stream() + .filter(searchBasedFetcher -> searchBasedFetcher != this) + .collect(Collectors.toSet()); + this.maximumNumberOfReturnedResults = maximumNumberOfReturnedResults; + } + + @Override + public List performSearch(String query) { + return fetchers.stream().flatMap(searchBasedFetcher -> { + try { + return searchBasedFetcher.performSearch(query).stream(); + } catch (FetcherException e) { + LOGGER.warn(String.format("%s API request failed", searchBasedFetcher.getName()), e); + return Stream.empty(); + } + }).parallel() + .limit(maximumNumberOfReturnedResults) + .collect(Collectors.toList()); + } + + @Override + public String getName() { + return "SearchAll"; + } + + @Override + public Optional getHelpPage() { + return Optional.empty(); + } +} diff --git a/src/test/java/org/jabref/logic/importer/fetcher/CompositeSearchBasedFetcherTest.java b/src/test/java/org/jabref/logic/importer/fetcher/CompositeSearchBasedFetcherTest.java new file mode 100644 index 00000000000..50866933d9d --- /dev/null +++ b/src/test/java/org/jabref/logic/importer/fetcher/CompositeSearchBasedFetcherTest.java @@ -0,0 +1,125 @@ +package org.jabref.logic.importer.fetcher; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.stream.Stream; + +import org.jabref.logic.bibtex.FieldContentFormatterPreferences; +import org.jabref.logic.importer.FetcherException; +import org.jabref.logic.importer.ImportFormatPreferences; +import org.jabref.logic.importer.SearchBasedFetcher; +import org.jabref.model.entry.BibEntry; +import org.jabref.testutils.category.FetcherTest; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +@FetcherTest +public class CompositeSearchBasedFetcherTest { + + private static final Logger LOGGER = LoggerFactory.getLogger(CompositeSearchBasedFetcherTest.class); + + @Test + public void createCompositeFetcherWithNullSet() { + Assertions.assertThrows(IllegalArgumentException.class, + () -> new CompositeSearchBasedFetcher(null, 0)); + } + + @Test + public void performSearchWithoutFetchers() { + Set empty = new HashSet<>(); + CompositeSearchBasedFetcher fetcher = new CompositeSearchBasedFetcher(empty, Integer.MAX_VALUE); + + List result = fetcher.performSearch("quantum"); + + Assertions.assertEquals(result, Collections.EMPTY_LIST); + } + + @ParameterizedTest(name = "Perform Search on empty query.") + @MethodSource("performSearchParameters") + public void performSearchOnEmptyQuery(Set fetchers) { + CompositeSearchBasedFetcher compositeFetcher = new CompositeSearchBasedFetcher(fetchers, Integer.MAX_VALUE); + + List queryResult = compositeFetcher.performSearch(""); + + Assertions.assertEquals(queryResult, Collections.EMPTY_LIST); + } + + @ParameterizedTest(name = "Perform search on query \"quantum\". Using the CompositeFetcher of the following " + + "Fetchers: {arguments}") + @MethodSource("performSearchParameters") + public void performSearchOnNonEmptyQuery(Set fetchers) { + CompositeSearchBasedFetcher compositeFetcher = new CompositeSearchBasedFetcher(fetchers, Integer.MAX_VALUE); + + List compositeResult = compositeFetcher.performSearch("quantum"); + for (SearchBasedFetcher fetcher : fetchers) { + try { + Assertions.assertTrue(compositeResult.containsAll(fetcher.performSearch("quantum"))); + } catch (FetcherException e) { + /* We catch the Fetcher exception here, since the failing fetcher also fails in the CompositeFetcher + * and just leads to no additional results in the returned list. Therefore the test should not fail + * due to the fetcher exception + */ + LOGGER.debug(String.format("Fetcher %s failed ", fetcher.getName()), e); + } + } + } + + /** + * This method provides other methods with different sized sets of search-based fetchers wrapped in arguments. + * + * @return A stream of Arguments wrapping set of fetchers. + */ + static Stream performSearchParameters() { + ImportFormatPreferences importFormatPreferences = mock(ImportFormatPreferences.class); + when(importFormatPreferences.getFieldContentFormatterPreferences()) + .thenReturn(mock(FieldContentFormatterPreferences.class)); + List> fetcherParameters = new ArrayList<>(); + List list = new ArrayList<>(); + + list.add(new ArXiv(importFormatPreferences)); + list.add(new INSPIREFetcher(importFormatPreferences)); + list.add(new GvkFetcher()); + list.add(new AstrophysicsDataSystem(importFormatPreferences)); + list.add(new MathSciNet(importFormatPreferences)); + list.add(new ZbMATH(importFormatPreferences)); + list.add(new GoogleScholar(importFormatPreferences)); + list.add(new DBLPFetcher(importFormatPreferences)); + list.add(new SpringerFetcher()); + list.add(new CrossRef()); + list.add(new CiteSeer()); + list.add(new DOAJFetcher(importFormatPreferences)); + list.add(new IEEE(importFormatPreferences)); + /* Disabled due to an issue regarding comparison: Title fields of the entries that otherwise are equivalent differ + * due to different JAXBElements. + */ + // list.add(new MedlineFetcher()); + + // Create different sized sets of fetchers to use in the composite fetcher. + // Selected 273 to have differencing sets + for (int i = 1; i < Math.pow(2, list.size()); i += 273) { + Set fetchers = new HashSet<>(); + // Only shift i at maximum to its MSB to the right + for (int j = 0; Math.pow(2, j) <= i; j++) { + // Add fetcher j to the list if the j-th bit of i is 1 + if ((i >> j) % 2 == 1) { + fetchers.add(list.get(j)); + } + } + fetcherParameters.add(fetchers); + } + + return fetcherParameters.stream().map(Arguments::of); + } +} diff --git a/src/test/java/org/jabref/logic/importer/fetcher/IEEETest.java b/src/test/java/org/jabref/logic/importer/fetcher/IEEETest.java index c5ac3f885d0..4d6e7e21657 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/IEEETest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/IEEETest.java @@ -11,7 +11,6 @@ import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.field.StandardField; import org.jabref.model.entry.types.StandardEntryType; -import org.jabref.support.DisabledOnCIServer; import org.jabref.testutils.category.FetcherTest; import org.junit.jupiter.api.BeforeEach; @@ -77,7 +76,6 @@ void findByDOIButNotURL() throws IOException { } @Test - @DisabledOnCIServer("CI server is unreliable") void notFoundByURL() throws IOException { entry.setField(StandardField.URL, "http://dx.doi.org/10.1109/ACCESS.2016.2535486");