Merge pull request #98 from aodn/features/6030-text-search-include-param

Features/6030 text search include param

server/src/main/java/au/org/aodn/ogcapi/server/common/RestApi.java

@@ -154,7 +154,7 @@ public ResponseEntity<?> getCollections(
         // TODO: Support other CRS.
         if (CQLFilterType.convert(filterLang) == CQLFilterType.CQL && CQLCrsType.convertFromUrl(crs) == CQLCrsType.EPSG4326) {
             if (datetime != null) {
-                filter = commonService.processDatetimeParameter(datetime, filter);
+                filter = OGCApiService.processDatetimeParameter(datetime, filter);
             }
             return commonService.getCollectionList(
                     q,

server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/CQLFields.java

@@ -179,9 +179,10 @@ public enum CQLFields implements CQLFieldsInterface {
             (literal) -> MatchQuery.of(m -> m
                     .fuzziness("AUTO")
                     .field(StacBasicField.Title.searchField)
-                    .prefixLength(2)
+                    .prefixLength(3)
                     // Increase the relevance of matches in title
                     .boost(2.0F)
+                    .operator(Operator.And) // ensure all terms are matched with fuzziness
                     .query(literal))._toQuery(),
             null
     ),
@@ -191,7 +192,8 @@ public enum CQLFields implements CQLFieldsInterface {
             (literal) -> MatchQuery.of(m -> m
                     .fuzziness("AUTO")
                     .field(StacBasicField.Description.searchField)
-                    .prefixLength(2)
+                    .prefixLength(3)
+                    .operator(Operator.And) // ensure all terms are matched with fuzziness
                     .query(literal))._toQuery(),
             null
     ),

server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java

@@ -254,6 +254,9 @@ public ElasticSearchBase.SearchResult searchByParameters(List<String> keywords,
                 for (String t : keywords) {
                     should.add(CQLFields.fuzzy_title.getPropertyEqualToQuery(t));
                     should.add(CQLFields.fuzzy_desc.getPropertyEqualToQuery(t));
+                    should.add(CQLFields.parameter_vocabs.getPropertyEqualToQuery(t));
+                    should.add(CQLFields.organisation_vocabs.getPropertyEqualToQuery(t));
+                    should.add(CQLFields.platform_vocabs.getPropertyEqualToQuery(t));
                 }
             }
 

server/src/main/java/au/org/aodn/ogcapi/server/core/service/OGCApiService.java

@@ -93,7 +93,7 @@ public <R> ResponseEntity<R> getCollectionList(List<String> keywords,
      * @param filter - Any existing filter
      * @return - A combined filter with datetime rewrite.
      */
-    public String processDatetimeParameter(String datetime, String filter) {
+    public static String processDatetimeParameter(String datetime, String filter) {
 
         // TODO: How to handle this? e.g how to know if it is before or after if ?datetime=<timestamp instant>
 

server/src/test/java/au/org/aodn/ogcapi/server/common/RestApiTest.java

@@ -59,11 +59,11 @@ public void verifyClusterIsHealthy() throws IOException {
         super.assertClusterHealthResponse();
     }
     /**
-     * The search is a fuzzy search based on title and description. So you expect 1 hit only
+     * The search is a fuzzy search based on title and description and few param field. So you expect 1 hit only
      * @throws IOException - IO Exception
      */
     @Test
-    public void verifyApiCollectionsQueryOnText() throws IOException {
+    public void verifyApiCollectionsQueryOnText1() throws IOException {
         super.insertJsonToElasticIndex(
                 "516811d7-cd1e-207a-e0440003ba8c79dd.json",
                 "7709f541-fc0c-4318-b5b9-9053aa474e0e.json"
@@ -96,6 +96,36 @@ public void verifyApiCollectionsQueryOnText() throws IOException {
                 collections.getBody().getCollections().get(1).getId(),
                 "Correct UUID - 7709f541-fc0c-4318-b5b9-9053aa474e0e");
     }
+    /**
+     * The search is a fuzzy search based on title and description and few param field. This test add one more text
+     * which should hit the organization, paramter or vocab field
+     * 516811d7-cd1e-207a-e0440003ba8c79dd - Have repoduction
+     * 073fde5a-bff3-1c1f-e053-08114f8c5588 - Nothing match (although the word 'and' will match, but we use AND operator in fuzzy match so it will not count)
+     * 9fdb1eee-bc28-43a9-88c5-972324784837 - Contains 'precipitation and evaporation' in parameter_vocabs
+     *
+     * @throws IOException - IO Exception
+     */
+    @Test
+    public void verifyApiCollectionsQueryOnText2() throws IOException {
+        super.insertJsonToElasticIndex(
+                "516811d7-cd1e-207a-e0440003ba8c79dd.json",
+                "073fde5a-bff3-1c1f-e053-08114f8c5588.json",
+                "9fdb1eee-bc28-43a9-88c5-972324784837.json"
+        );
+
+        // Call rest api directly and get query result
+        ResponseEntity<Collections> collections = testRestTemplate.getForEntity(getBasePath() + "/collections?q=reproduction,precipitation and evaporation", Collections.class);
+        assertEquals(2, Objects.requireNonNull(collections.getBody()).getCollections().size(), "Only 2 hit");
+        assertEquals(
+                "516811d7-cd1e-207a-e0440003ba8c79dd",
+                collections.getBody().getCollections().get(0).getId(),
+                "Correct UUID - 516811d7-cd1e-207a-e0440003ba8c79dd");
+
+        assertEquals(
+                "9fdb1eee-bc28-43a9-88c5-972324784837",
+                collections.getBody().getCollections().get(1).getId(),
+                "Correct UUID - 9fdb1eee-bc28-43a9-88c5-972324784837");
+    }
     /**
      * The datetime field after xxx/.. xxx/ etc. It uses CQL internally so no need to test Before After During in CQL
      */
@@ -466,11 +496,10 @@ public void verifyCQLPropertyScore() throws IOException {
         assertEquals(error.getStatusCode(), HttpStatus.INTERNAL_SERVER_ERROR);
         assertEquals(Objects.requireNonNull(error.getBody()).getMessage(), "Or combine with query setting do not make sense", "correct error");
 
-        // Lower score rate make more match
+        // Lower score but the fuzzy is now with operator AND, therefore it will try to match all words 'dataset' and 'includes' with fuzzy
         collections = testRestTemplate.getForEntity(getBasePath() + "/collections?q='dataset includes'&filter=score>=1", Collections.class);
-        assertEquals(2, Objects.requireNonNull(collections.getBody()).getCollections().size(), "hit 2, with score 1");
+        assertEquals(1, Objects.requireNonNull(collections.getBody()).getCollections().size(), "hit 1, with score 3");
         assertEquals("bf287dfe-9ce4-4969-9c59-51c39ea4d011", Objects.requireNonNull(collections.getBody()).getCollections().get(0).getId(), "bf287dfe-9ce4-4969-9c59-51c39ea4d011");
-        assertEquals("7709f541-fc0c-4318-b5b9-9053aa474e0e", Objects.requireNonNull(collections.getBody()).getCollections().get(1).getId(), "7709f541-fc0c-4318-b5b9-9053aa474e0e");
 
         // Increase score will drop one record
         collections = testRestTemplate.getForEntity(getBasePath() + "/collections?q='dataset includes'&filter=score>=3", Collections.class);

server/src/test/java/au/org/aodn/ogcapi/server/core/service/OGCApiServiceTest.java

@@ -1,44 +1,33 @@
 package au.org.aodn.ogcapi.server.core.service;
 
 import org.junit.jupiter.api.Test;
-
-import java.util.List;
-
 import static org.junit.jupiter.api.Assertions.assertEquals;
 
 public class OGCApiServiceTest {
 
-    // The class is abstract, so need to implement it before we can test it.
-    class OGCApiServiceImpl extends OGCApiService {
-        @Override
-        public List<String> getConformanceDeclaration() {
-            return null;
-        }
-    };
     /**
      * Verify process function correct, it converts datetime field to CQL filter, here the time isn't important
      * as parser will handle it and error out if date time format is incorrect.
      */
     @Test
     public void verifyProcessDatetimeParameter() {
-        OGCApiServiceImpl impl = new OGCApiServiceImpl();
 
-        String o = impl.processDatetimeParameter("../2021-10-10", "");
+        String o = OGCApiService.processDatetimeParameter("../2021-10-10", "");
         assertEquals( "temporal before 2021-10-10", o, "Before incorrect1");
 
-        o = impl.processDatetimeParameter("/2021-10-10", "");
+        o = OGCApiService.processDatetimeParameter("/2021-10-10", "");
         assertEquals( "temporal before 2021-10-10", o, "Before incorrect2");
 
-        o = impl.processDatetimeParameter("2021-10-10/", "");
+        o = OGCApiService.processDatetimeParameter("2021-10-10/", "");
         assertEquals( "temporal after 2021-10-10", o, "After incorrect1");
 
-        o = impl.processDatetimeParameter("2021-10-10/..", "");
+        o = OGCApiService.processDatetimeParameter("2021-10-10/..", "");
         assertEquals( "temporal after 2021-10-10", o, "After incorrect1");
 
-        o = impl.processDatetimeParameter("2021-10-10/2022-10-10", "");
+        o = OGCApiService.processDatetimeParameter("2021-10-10/2022-10-10", "");
         assertEquals( "temporal during 2021-10-10/2022-10-10", o, "During incorrect1");
 
-        o = impl.processDatetimeParameter("/2021-10-10", "geometry is null");
+        o = OGCApiService.processDatetimeParameter("/2021-10-10", "geometry is null");
         assertEquals( "geometry is null AND temporal before 2021-10-10", o, "Before plus filter incorrect1");
     }
 }