feat: hybrid search improvements for v1.8.x

meilisearch · Apr 23, 2024 · 94b1875 · 94b1875
1 parent 0dc6d90
commit 94b1875
Show file tree

Hide file tree

Showing 6 changed files with 419 additions and 4 deletions.
diff --git a/src/types/types.ts b/src/types/types.ts
@@ -215,7 +215,6 @@ export type SearchResponse<
   query: string
   facetDistribution?: FacetDistribution
   facetStats?: FacetStats
-  vector?: number[]
 } & (undefined extends S
   ? Partial<FinitePagination & InfinitePagination>
   : true extends IsFinitePagination<NonNullable<S>>
@@ -335,30 +334,63 @@ export type NonSeparatorTokens = string[] | null
 export type Dictionary = string[] | null
 export type ProximityPrecision = 'byWord' | 'byAttribute'
 
+export type Distribution = {
+  mean: number
+  sigma: number
+}
+
 export type OpenAiEmbedder = {
   source: 'openAi'
   model?: string
   apiKey?: string
   documentTemplate?: string
   dimensions?: number
+  distribution?: Distribution
 }
 
 export type HuggingFaceEmbedder = {
   source: 'huggingFace'
   model?: string
   revision?: string
   documentTemplate?: string
+  distribution?: Distribution
 }
 
 export type UserProvidedEmbedder = {
   source: 'userProvided'
   dimensions: number
+  distribution?: Distribution
+}
+
+export type RestEmbedder = {
+  source: 'rest'
+  url: string
+  apiKey?: string
+  dimensions?: number
+  documentTemplate?: string
+  inputField?: string[] | null
+  inputType?: 'text' | 'textArray'
+  query?: Record<string, any> | null
+  pathToEmbeddings?: string[] | null
+  embeddingObject?: string[] | null
+  distribution?: Distribution
+}
+
+export type OllamaEmbedder = {
+  source: 'ollama'
+  url?: string
+  apiKey?: string
+  model?: string
+  documentTemplate?: string
+  distribution?: Distribution
 }
 
 export type Embedder =
   | OpenAiEmbedder
   | HuggingFaceEmbedder
   | UserProvidedEmbedder
+  | RestEmbedder
+  | OllamaEmbedder
   | null
 
 export type Embedders = Record<string, Embedder> | null

diff --git a/tests/__snapshots__/settings.test.ts.snap b/tests/__snapshots__/settings.test.ts.snap
@@ -94,6 +94,53 @@ exports[`Test on settings Admin key: Get default settings of empty index with pr
 }
 `;
 
+exports[`Test on settings Admin key: Reset embedders settings  1`] = `
+{
+  "dictionary": [],
+  "displayedAttributes": [
+    "*",
+  ],
+  "distinctAttribute": null,
+  "faceting": {
+    "maxValuesPerFacet": 100,
+    "sortFacetValuesBy": {
+      "*": "alpha",
+    },
+  },
+  "filterableAttributes": [],
+  "nonSeparatorTokens": [],
+  "pagination": {
+    "maxTotalHits": 1000,
+  },
+  "proximityPrecision": "byWord",
+  "rankingRules": [
+    "words",
+    "typo",
+    "proximity",
+    "attribute",
+    "sort",
+    "exactness",
+  ],
+  "searchCutoffMs": null,
+  "searchableAttributes": [
+    "*",
+  ],
+  "separatorTokens": [],
+  "sortableAttributes": [],
+  "stopWords": [],
+  "synonyms": {},
+  "typoTolerance": {
+    "disableOnAttributes": [],
+    "disableOnWords": [],
+    "enabled": true,
+    "minWordSizeForTypos": {
+      "oneTypo": 5,
+      "twoTypos": 9,
+    },
+  },
+}
+`;
+
 exports[`Test on settings Admin key: Reset settings 1`] = `
 {
   "dictionary": [],
@@ -188,6 +235,66 @@ exports[`Test on settings Admin key: Reset settings of empty index 1`] = `
 }
 `;
 
+exports[`Test on settings Admin key: Update embedders settings  1`] = `
+{
+  "dictionary": [],
+  "displayedAttributes": [
+    "*",
+  ],
+  "distinctAttribute": null,
+  "embedders": {
+    "default": {
+      "apiKey": "<yoXXXXX...",
+      "dimensions": 1536,
+      "distribution": {
+        "mean": 0.7,
+        "sigma": 0.3,
+      },
+      "documentTemplate": "A document template",
+      "model": "text-embedding-3-small",
+      "source": "openAi",
+    },
+  },
+  "faceting": {
+    "maxValuesPerFacet": 100,
+    "sortFacetValuesBy": {
+      "*": "alpha",
+    },
+  },
+  "filterableAttributes": [],
+  "nonSeparatorTokens": [],
+  "pagination": {
+    "maxTotalHits": 1000,
+  },
+  "proximityPrecision": "byWord",
+  "rankingRules": [
+    "words",
+    "typo",
+    "proximity",
+    "attribute",
+    "sort",
+    "exactness",
+  ],
+  "searchCutoffMs": null,
+  "searchableAttributes": [
+    "*",
+  ],
+  "separatorTokens": [],
+  "sortableAttributes": [],
+  "stopWords": [],
+  "synonyms": {},
+  "typoTolerance": {
+    "disableOnAttributes": [],
+    "disableOnWords": [],
+    "enabled": true,
+    "minWordSizeForTypos": {
+      "oneTypo": 5,
+      "twoTypos": 9,
+    },
+  },
+}
+`;
+
 exports[`Test on settings Admin key: Update searchableAttributes settings on empty index 1`] = `
 {
   "dictionary": [],
@@ -536,6 +643,53 @@ exports[`Test on settings Master key: Get default settings of empty index with p
 }
 `;
 
+exports[`Test on settings Master key: Reset embedders settings  1`] = `
+{
+  "dictionary": [],
+  "displayedAttributes": [
+    "*",
+  ],
+  "distinctAttribute": null,
+  "faceting": {
+    "maxValuesPerFacet": 100,
+    "sortFacetValuesBy": {
+      "*": "alpha",
+    },
+  },
+  "filterableAttributes": [],
+  "nonSeparatorTokens": [],
+  "pagination": {
+    "maxTotalHits": 1000,
+  },
+  "proximityPrecision": "byWord",
+  "rankingRules": [
+    "words",
+    "typo",
+    "proximity",
+    "attribute",
+    "sort",
+    "exactness",
+  ],
+  "searchCutoffMs": null,
+  "searchableAttributes": [
+    "*",
+  ],
+  "separatorTokens": [],
+  "sortableAttributes": [],
+  "stopWords": [],
+  "synonyms": {},
+  "typoTolerance": {
+    "disableOnAttributes": [],
+    "disableOnWords": [],
+    "enabled": true,
+    "minWordSizeForTypos": {
+      "oneTypo": 5,
+      "twoTypos": 9,
+    },
+  },
+}
+`;
+
 exports[`Test on settings Master key: Reset settings 1`] = `
 {
   "dictionary": [],
@@ -630,6 +784,66 @@ exports[`Test on settings Master key: Reset settings of empty index 1`] = `
 }
 `;
 
+exports[`Test on settings Master key: Update embedders settings  1`] = `
+{
+  "dictionary": [],
+  "displayedAttributes": [
+    "*",
+  ],
+  "distinctAttribute": null,
+  "embedders": {
+    "default": {
+      "apiKey": "<yoXXXXX...",
+      "dimensions": 1536,
+      "distribution": {
+        "mean": 0.7,
+        "sigma": 0.3,
+      },
+      "documentTemplate": "A document template",
+      "model": "text-embedding-3-small",
+      "source": "openAi",
+    },
+  },
+  "faceting": {
+    "maxValuesPerFacet": 100,
+    "sortFacetValuesBy": {
+      "*": "alpha",
+    },
+  },
+  "filterableAttributes": [],
+  "nonSeparatorTokens": [],
+  "pagination": {
+    "maxTotalHits": 1000,
+  },
+  "proximityPrecision": "byWord",
+  "rankingRules": [
+    "words",
+    "typo",
+    "proximity",
+    "attribute",
+    "sort",
+    "exactness",
+  ],
+  "searchCutoffMs": null,
+  "searchableAttributes": [
+    "*",
+  ],
+  "separatorTokens": [],
+  "sortableAttributes": [],
+  "stopWords": [],
+  "synonyms": {},
+  "typoTolerance": {
+    "disableOnAttributes": [],
+    "disableOnWords": [],
+    "enabled": true,
+    "minWordSizeForTypos": {
+      "oneTypo": 5,
+      "twoTypos": 9,
+    },
+  },
+}
+`;
+
 exports[`Test on settings Master key: Update searchableAttributes settings on empty index 1`] = `
 {
   "dictionary": [],