Merge pull request #184 from nzdev/v1/skip-take-select

skip take & select fields

src/Examine.Test/Search/FluentApiTests.cs

@@ -2073,5 +2073,45 @@ public void Select_Fields_Native_Query()
 
         }
 
+        [Test]
+        public void Lucene_Document_Skip_Results_Returns_Different_Results()
+        {
+            var analyzer = new StandardAnalyzer(Version.LUCENE_30);
+            using (var luceneDir = new RandomIdRAMDirectory())
+            using (var indexer = new TestIndex(luceneDir, analyzer))
+
+
+            {
+
+
+                indexer.IndexItems(new[] {
+                    ValueSet.FromObject(1.ToString(), "content",
+                        new { nodeName = "umbraco", headerText = "world", writerName = "administrator" }),
+                    ValueSet.FromObject(2.ToString(), "content",
+                        new { nodeName = "umbraco", headerText = "umbraco", writerName = "administrator" }),
+                    ValueSet.FromObject(3.ToString(), "content",
+                        new { nodeName = "umbraco", headerText = "umbraco", writerName = "administrator" }),
+                    ValueSet.FromObject(4.ToString(), "content",
+                        new { nodeName = "hello", headerText = "world", writerName = "blah" })
+                    });
+
+                var searcher = indexer.GetSearcher();
+
+                //Arrange
+                var sc = searcher.CreateQuery("content").Field("writerName", "administrator");
+
+                //Act
+                var results = sc.Execute();
+                var first = results.First();
+                var third = results.Skip(2).First();
+                Assert.AreNotEqual(first, third, "Third result should be different");
+                var resultsLuceneSkip = sc.ExecuteWithSkip(2);
+                //Assert
+                Assert.IsTrue(resultsLuceneSkip.Count() == 1,"More results fetched than expected");
+                Assert.AreEqual(resultsLuceneSkip.First(), third, "Third result should be same as Execute().Skip(2).First()");
+            }
+
+
+        }
     }
 }

src/Examine/Examine.csproj

@@ -184,6 +184,8 @@
     <Compile Include="Search\INestedBooleanOperation.cs" />
     <Compile Include="Search\INestedQuery.cs" />
     <Compile Include="Search\IOrdering.cs" />
+    <Compile Include="Search\IQueryExecutor2.cs" />
+    <Compile Include="Search\QueryExecutorExtensions.cs" />
     <Compile Include="Search\QueryExtensions.cs" />
     <Compile Include="Search\SortableField.cs" />
     <Compile Include="Search\SortType.cs" />

src/Examine/LuceneEngine/LuceneSearchResults.cs

@@ -47,6 +47,78 @@ internal LuceneSearchResults(Query query, IEnumerable<SortField> sortField, Sear
 
             DoSearch(query, sortField, maxResults);
         }
+        internal LuceneSearchResults(Query query, IEnumerable<SortField> sortField, Searcher searcher, int skip, int? take = null, FieldSelector fieldSelector = null)
+        {
+            LuceneQuery = query;
+
+            LuceneSearcher = searcher;
+            DoSearch(query, sortField, skip, take);
+            FieldSelector = fieldSelector;
+        }
+
+        private void DoSearch(Query query, IEnumerable<SortField> sortField, int skip, int? take = null)
+        {
+            int maxResults = take != null ? take.Value + skip : int.MaxValue;
+            //This try catch is because analyzers strip out stop words and sometimes leave the query
+            //with null values. This simply tries to extract terms, if it fails with a null
+            //reference then its an invalid null query, NotSupporteException occurs when the query is
+            //valid but the type of query can't extract terms.
+            //This IS a work-around, theoretically Lucene itself should check for null query parameters
+            //before throwing exceptions.
+            try
+            {
+                var set = new HashSet<Term>();
+                query.ExtractTerms(set);
+            }
+            catch (NullReferenceException)
+            {
+                //this means that an analyzer has stipped out stop words and now there are
+                //no words left to search on
+
+                //it could also mean that potentially a IIndexFieldValueType is throwing a null ref
+                TotalItemCount = 0;
+                return;
+            }
+            catch (NotSupportedException)
+            {
+                //swallow this exception, we should continue if this occurs.
+            }
+
+            maxResults = maxResults >= 1 ? Math.Min(maxResults, LuceneSearcher.MaxDoc) : LuceneSearcher.MaxDoc;
+
+            Collector topDocsCollector;
+            var sortFields = sortField as SortField[] ?? sortField.ToArray();
+            if (sortFields.Length > 0)
+            {
+                topDocsCollector = TopFieldCollector.Create(
+                    new Sort(sortFields), maxResults, false, false, false, false);
+            }
+            else
+            {
+                topDocsCollector = TopScoreDocCollector.Create(maxResults, true);
+            }
+
+            LuceneSearcher.Search(query, topDocsCollector);
+
+            if (sortFields.Length > 0 && take != null && take.Value >= 0)
+            {
+                TopDocs = ((TopFieldCollector)topDocsCollector).TopDocs(skip,take.Value);
+            }
+            else if (sortFields.Length > 0 && (take == null || take.Value < 0))
+            {
+                TopDocs = ((TopFieldCollector)topDocsCollector).TopDocs(skip);
+            }
+            else if ( take != null && take.Value >= 0)
+            {
+                TopDocs = ((TopScoreDocCollector)topDocsCollector).TopDocs(skip,take.Value);
+            }
+            else
+            {
+                TopDocs = ((TopScoreDocCollector)topDocsCollector).TopDocs(skip);
+            }
+
+            TotalItemCount = TopDocs.TotalHits;
+        }
 
         private void DoSearch(Query query, IEnumerable<SortField> sortField, int maxResults)
         {

src/Examine/LuceneEngine/Search/LuceneBooleanOperation.cs

@@ -10,7 +10,7 @@ namespace Examine.LuceneEngine.Search
     /// An implementation of the fluent API boolean operations
     /// </summary>
     [DebuggerDisplay("{_search}")]
-    public class LuceneBooleanOperation : LuceneBooleanOperationBase
+    public class LuceneBooleanOperation : LuceneBooleanOperationBase, IQueryExecutor2
     {
         private readonly LuceneSearchQuery _search;
 
@@ -66,5 +66,7 @@ public LuceneBooleanOperation(LuceneSearchQuery search)
         #endregion
 
         public override string ToString() => _search.ToString();
+
+        public override ISearchResults ExecuteWithSkip(int skip, int? take = null) => _search.ExecuteWithSkip(skip, take);
     }
 }

src/Examine/LuceneEngine/Search/LuceneBooleanOperationBase.cs

@@ -95,5 +95,10 @@ public virtual IOrdering SelectAllFields()
         {
             throw new NotImplementedException();
         }
+
+        public virtual ISearchResults ExecuteWithSkip(int skip, int? take = null)
+        {
+            throw new NotImplementedException();
+        }
     }
 }

src/Examine/LuceneEngine/Search/LuceneSearchQuery.cs

@@ -15,7 +15,7 @@ namespace Examine.LuceneEngine.Search
     /// This class is used to query against Lucene.Net
     /// </summary>
     [DebuggerDisplay("Category: {Category}, LuceneQuery: {Query}")]
-    public class LuceneSearchQuery : LuceneSearchQueryBase, IQueryExecutor
+    public class LuceneSearchQuery : LuceneSearchQueryBase, IQueryExecutor, IQueryExecutor2
     {
         private readonly ISearchContext _searchContext;
 
@@ -158,10 +158,40 @@ private ISearchResults Search(int maxResults = 500)
                 }
             }
 
-            var pagesResults = new LuceneSearchResults(query, SortFields, searcher, maxResults, Selector);
+            var pagesResults = new LuceneSearchResults(query, SortFields, searcher, maxResults,Selector);
             return pagesResults;
         }
 
+        /// <summary>
+        /// Performs a search using skip and take
+        /// </summary>
+        private ISearchResults SearchWithSkip(int skip,int? take)
+        {
+            var searcher = _searchContext.Searcher;
+            if (searcher == null) return EmptySearchResults.Instance;
+
+            // capture local
+            var query = Query;
+
+            if (!string.IsNullOrEmpty(Category))
+            {
+                // if category is supplied then wrap the query (if there's other queries to wrap!)
+                if (query.Clauses.Count > 0)
+                {
+                    query = new BooleanQuery
+                    {
+                        { query, Occur.MUST }
+                    };
+                }
+
+                // and then add the category field query as a must
+                var categoryQuery = GetFieldInternalQuery(Providers.LuceneIndex.CategoryFieldName, new ExamineValue(Examineness.Explicit, Category), false);
+                query.Add(categoryQuery, Occur.MUST);
+            }
+
+            var pagesResults = new LuceneSearchResults(query, SortFields, searcher, skip,take, Selector);
+            return pagesResults;
+        }
 
         /// <summary>
         /// Internal operation for adding the ordered results
@@ -226,7 +256,7 @@ private LuceneBooleanOperation OrderByInternal(bool descending, params SortableF
         internal IBooleanOperation SelectFieldsInternal(ISet<string> loadedFieldNames)
         {
             Selector = new SetBasedFieldSelector(loadedFieldNames, new HashSet<string>());
-            return new LuceneBooleanOperation(this);
+            return CreateOp();
         }
 
         internal IBooleanOperation SelectFieldsInternal(Hashtable loadedFieldNames)
@@ -237,37 +267,36 @@ internal IBooleanOperation SelectFieldsInternal(Hashtable loadedFieldNames)
                 hs.Add(item);
             }
             Selector = new SetBasedFieldSelector(hs, new HashSet<string>());
-            return new LuceneBooleanOperation(this);
+            return CreateOp();
         }
 
         internal IBooleanOperation SelectFieldsInternal(params string[] loadedFieldNames)
         {
             ISet<string> loaded = new HashSet<string>(loadedFieldNames);
             Selector = new SetBasedFieldSelector(loaded, new HashSet<string>());
-            return new LuceneBooleanOperation(this);
+            return CreateOp();
         }
 
         internal IBooleanOperation SelectFieldInternal(string fieldName)
         {
             ISet<string> loaded = new HashSet<string>(new string[] { fieldName });
             Selector = new SetBasedFieldSelector(loaded, new HashSet<string>());
-            return new LuceneBooleanOperation(this);
+            return CreateOp();
         }
 
         public IBooleanOperation SelectFirstFieldOnlyInternal()
         {
             Selector = new LoadFirstFieldSelector();
-            return new LuceneBooleanOperation(this);
+            return CreateOp();
         }
         public IBooleanOperation SelectAllFieldsInternal()
         {
             Selector = null;
-            return new LuceneBooleanOperation(this);
+            return CreateOp();
         }
 
         protected override LuceneBooleanOperationBase CreateOp() => new LuceneBooleanOperation(this);
 
-
-
+        public ISearchResults ExecuteWithSkip(int skip, int? take = null) => SearchWithSkip(skip,take);
     }
 }

src/Examine/Search/IQueryExecutor.cs

@@ -12,7 +12,13 @@ public interface IQueryExecutor
         /// <returns></returns>
         ISearchResults Execute(int maxResults = 500);
 
-
 
+        /// <summary>
+        /// Executes the query with skip already applied. Use when you only need a single page of results to save on memory. To obtain additional pages you will need to execute the query again.
+        /// </summary>
+        /// <param name="skip">Number of results to skip</param>
+        /// <param name="take">Number of results to take</param>
+        /// <returns></returns>
+        ISearchResults ExecuteWithSkip(int skip, int? take = null);
     }
 }

src/Examine/Search/IQueryExecutor2.cs

@@ -0,0 +1,19 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Examine.Search
+{
+    public  interface IQueryExecutor2 : IQueryExecutor
+    {
+        /// <summary>
+        /// Executes the query with skip already applied. Use when you only need a single page of results to save on memory. To obtain additional pages you will need to execute the query again.
+        /// </summary>
+        /// <param name="skip">Number of results to skip</param>
+        /// <param name="take">Number of results to take</param>
+        /// <returns></returns>
+        ISearchResults ExecuteWithSkip(int skip, int? take = null);
+    }
+}

src/Examine/Search/QueryExecutorExtensions.cs

@@ -0,0 +1,26 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Examine.Search
+{
+    public static class QueryExecutorExtensions
+    {
+        /// <summary>
+        /// Executes the query with skip already applied. Use when you only need a single page of results to save on memory. To obtain additional pages you will need to execute the query again.
+        /// </summary>
+        /// <param name="skip">Number of results to skip</param>
+        /// <param name="take">Number of results to take</param>
+        /// <returns></returns>
+        public static ISearchResults ExecuteWithSkip(this IQueryExecutor query,int skip, int? take = null)
+        {
+            if(!(query is IQueryExecutor2 queryExecutor2))
+            {
+                throw new NotSupportedException("IQueryExecutor2 is not implemented");
+            }
+            return queryExecutor2.ExecuteWithSkip(skip, take);
+        }
+    }
+}