Merge pull request #6236 from JabRef/pagedFetcher

src/main/java/org/jabref/logic/importer/PagedSearchBasedFetcher.java

@@ -0,0 +1,21 @@
+package org.jabref.logic.importer;
+
+import org.jabref.model.entry.BibEntry;
+import org.jabref.model.paging.Page;
+
+public interface PagedSearchBasedFetcher extends SearchBasedFetcher {
+
+    /**
+     * @param query      search query send to endpoint
+     * @param pageNumber requested site number
+     * @return Page with search results
+     */
+    Page<BibEntry> performSearchPaged(String query, int pageNumber) throws FetcherException;
+
+    /**
+     * @return default pageSize
+     */
+    default int getPageSize() {
+        return 20;
+    }
+}

src/main/java/org/jabref/logic/importer/PagedSearchBasedParserFetcher.java

@@ -0,0 +1,16 @@
+package org.jabref.logic.importer;
+
+import java.net.MalformedURLException;
+import java.net.URISyntaxException;
+import java.net.URL;
+
+public interface PagedSearchBasedParserFetcher extends SearchBasedParserFetcher, PagedSearchBasedFetcher {
+
+    /**
+     * Constructs a URL based on the query, size and page number.
+     * @param query the search query
+     * @param size the size of the page
+     * @param pageNumber the number of the page
+     * */
+    URL getURLForQuery(String query, int size, int pageNumber) throws URISyntaxException, MalformedURLException, FetcherException;
+}

src/main/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystem.java

@@ -24,15 +24,16 @@
 import org.jabref.logic.importer.FetcherException;
 import org.jabref.logic.importer.IdBasedParserFetcher;
 import org.jabref.logic.importer.ImportFormatPreferences;
+import org.jabref.logic.importer.PagedSearchBasedParserFetcher;
 import org.jabref.logic.importer.ParseException;
 import org.jabref.logic.importer.Parser;
-import org.jabref.logic.importer.SearchBasedParserFetcher;
 import org.jabref.logic.importer.fileformat.BibtexParser;
 import org.jabref.logic.net.URLDownload;
 import org.jabref.logic.util.BuildInfo;
 import org.jabref.model.entry.BibEntry;
 import org.jabref.model.entry.field.StandardField;
 import org.jabref.model.entry.field.UnknownField;
+import org.jabref.model.paging.Page;
 import org.jabref.model.strings.StringUtil;
 import org.jabref.model.util.DummyFileUpdateMonitor;
 
@@ -44,7 +45,7 @@
 /**
  * Fetches data from the SAO/NASA Astrophysics Data System (https://ui.adsabs.harvard.edu/)
  */
-public class AstrophysicsDataSystem implements IdBasedParserFetcher, SearchBasedParserFetcher, EntryBasedParserFetcher {
+public class AstrophysicsDataSystem implements IdBasedParserFetcher, PagedSearchBasedParserFetcher, EntryBasedParserFetcher {
 
     private static final String API_SEARCH_URL = "https://api.adsabs.harvard.edu/v1/search/query";
     private static final String API_EXPORT_URL = "https://api.adsabs.harvard.edu/v1/export/bibtexabs";
@@ -89,6 +90,16 @@ public URL getURLForQuery(String query) throws URISyntaxException, MalformedURLE
         return builder.build().toURL();
     }
 
+    @Override
+    public URL getURLForQuery(String query, int size, int pageNumber) throws URISyntaxException, MalformedURLException, FetcherException {
+        URIBuilder builder = new URIBuilder(API_SEARCH_URL);
+        builder.addParameter("q", query);
+        builder.addParameter("fl", "bibcode");
+        builder.addParameter("rows", String.valueOf(size));
+        builder.addParameter("start", String.valueOf(size * pageNumber));
+        return builder.build().toURL();
+    }
+
     /**
      * @param entry BibEntry for which a search URL is created
      * @return URL which points to a search request for given entry
@@ -286,4 +297,21 @@ private List<BibEntry> performSearchByIds(Collection<String> identifiers) throws
             throw new FetcherException("An internal parser error occurred", e);
         }
     }
+
+    @Override
+    public Page<BibEntry> performSearchPaged(String query, int pageNumber) throws FetcherException {
+
+        if (StringUtil.isBlank(query)) {
+            return new Page<>(query, pageNumber);
+        }
+        try {
+            List<String> bibcodes = fetchBibcodes(getURLForQuery(query, getPageSize(), pageNumber));
+            Collection<BibEntry> results = performSearchByIds(bibcodes);
+            return new Page<>(query, pageNumber, results);
+        } catch (URISyntaxException e) {
+            throw new FetcherException("Search URI is malformed", e);
+        } catch (IOException e) {
+            throw new FetcherException("A network error occurred", e);
+        }
+    }
 }

src/main/java/org/jabref/model/paging/Page.java

@@ -0,0 +1,37 @@
+package org.jabref.model.paging;
+
+import java.util.Collection;
+import java.util.Collections;
+
+public class Page<T> {
+
+    private int pageNumber;
+    private String query;
+    private Collection<T> content;
+
+    public Page(String query, int pageNumber, Collection<T> content) {
+        this.query = query;
+        this.pageNumber = pageNumber;
+        this.content = Collections.unmodifiableCollection(content);
+    }
+
+    public Page(String query, int pageNumber) {
+        this(query, pageNumber, Collections.emptyList());
+    }
+
+    public Collection<T> getContent() {
+        return content;
+    }
+
+    public int getPageNumber() {
+        return pageNumber;
+    }
+
+    public String getQuery() {
+        return query;
+    }
+
+    public int getSize() {
+        return content.size();
+    }
+}

src/test/java/org/jabref/logic/importer/WebFetchersTest.java

@@ -85,6 +85,9 @@ void getSearchBasedFetchersReturnsAllFetcherDerivingFromSearchBasedFetcher() thr
             // Remove ACM, because it doesn't work currently
             expected.remove(ACMPortalFetcher.class);
 
+            expected.remove(PagedSearchBasedParserFetcher.class);
+            expected.remove(PagedSearchBasedFetcher.class);
+
             // Remove GROBID, because we don't want to show this to the user
             expected.remove(GrobidCitationFetcher.class);
 

src/test/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystemTest.java

@@ -8,13 +8,15 @@
 import org.jabref.model.entry.BibEntry;
 import org.jabref.model.entry.field.StandardField;
 import org.jabref.model.entry.types.StandardEntryType;
+import org.jabref.model.paging.Page;
 import org.jabref.testutils.category.FetcherTest;
 
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
@@ -198,4 +200,26 @@ public void testPerformSearchByLuceyPaulEntry() throws Exception {
         Optional<BibEntry> fetchedEntry = fetcher.performSearchById("2000JGR...10520297L");
         assertEquals(Optional.of(luceyPaulEntry), fetchedEntry);
     }
+
+    @Test
+    public void performSearchByQueryPaged_searchLimitsSize() throws Exception {
+        Page<BibEntry> page = fetcher.performSearchPaged("author:\"A\"", 0);
+        assertEquals(fetcher.getPageSize(), page.getSize(), "fetcher return wrong page size");
+    }
+
+    @Test
+    public void performSearchByQueryPaged_invalidAuthorsReturnEmptyPages() throws Exception {
+        Page<BibEntry> page = fetcher.performSearchPaged("author:\"ThisAuthorWillNotBeFound\"", 0);
+        Page<BibEntry> page5 = fetcher.performSearchPaged("author:\"ThisAuthorWillNotBeFound\"", 5);
+        assertEquals(0, page.getSize(), "fetcher doesnt return empty pages for invalid author");
+        assertEquals(0, page5.getSize(), "fetcher doesnt return empty pages for invalid author");
+    }
+
+    @Test
+    public void performSearchByQueryPaged_twoPagesNotEqual() throws Exception {
+        Page<BibEntry> page = fetcher.performSearchPaged("author:\"A\"", 0);
+        Page<BibEntry> page2 = fetcher.performSearchPaged("author:\"A\"", 1);
+        // This tests if the fetcher actually performs paging
+        assertNotEquals(page.getContent(), page2.getContent(), "Two consecutive pages shouldn't be equal");
+    }
 }