proto-beta: Tokenizer customization

src/indexes.ts

@@ -47,6 +47,9 @@ import {
   DocumentsDeletionQuery,
   SearchForFacetValuesParams,
   SearchForFacetValuesResponse,
+  SeparatorTokens,
+  NonSeparatorTokens,
+  Dictionary,
 } from './types'
 import { removeUndefinedFromObject } from './utils'
 import { HttpRequests } from './http-requests'
@@ -1117,6 +1120,133 @@ class Index<T extends Record<string, any> = Record<string, any>> {
 
     return new EnqueuedTask(task)
   }
+
+  ///
+  /// SEPARATOR TOKENS
+  ///
+
+  /**
+   * Get the list of all separator tokens.
+   *
+   * @returns Promise containing array of separator tokens
+   */
+  async getSeparatorTokens(): Promise<string[]> {
+    const url = `indexes/${this.uid}/settings/separator-tokens`
+    return await this.httpRequest.get<string[]>(url)
+  }
+
+  /**
+   * Update the list of separator tokens. Overwrite the old list.
+   *
+   * @param separatorTokens - Array that contains separator tokens.
+   * @returns Promise containing an EnqueuedTask or null
+   */
+  async updateSeparatorTokens(
+    separatorTokens: SeparatorTokens
+  ): Promise<EnqueuedTask> {
+    const url = `indexes/${this.uid}/settings/separator-tokens`
+    const task = await this.httpRequest.put(url, separatorTokens)
+
+    return new EnqueuedTask(task)
+  }
+
+  /**
+   * Reset the separator tokens list to its default value
+   *
+   * @returns Promise containing an EnqueuedTask
+   */
+  async resetSeparatorTokens(): Promise<EnqueuedTask> {
+    const url = `indexes/${this.uid}/settings/separator-tokens`
+    const task = await this.httpRequest.delete<EnqueuedTask>(url)
+
+    task.enqueuedAt = new Date(task.enqueuedAt)
+
+    return task
+  }
+
+  ///
+  /// NON-SEPARATOR TOKENS
+  ///
+
+  /**
+   * Get the list of all non-separator tokens.
+   *
+   * @returns Promise containing array of non-separator tokens
+   */
+  async getNonSeparatorTokens(): Promise<string[]> {
+    const url = `indexes/${this.uid}/settings/non-separator-tokens`
+    return await this.httpRequest.get<string[]>(url)
+  }
+
+  /**
+   * Update the list of non-separator tokens. Overwrite the old list.
+   *
+   * @param nonSeparatorTokens - Array that contains non-separator tokens.
+   * @returns Promise containing an EnqueuedTask or null
+   */
+  async updateNonSeparatorTokens(
+    nonSeparatorTokens: NonSeparatorTokens
+  ): Promise<EnqueuedTask> {
+    const url = `indexes/${this.uid}/settings/non-separator-tokens`
+    const task = await this.httpRequest.put(url, nonSeparatorTokens)
+
+    return new EnqueuedTask(task)
+  }
+
+  /**
+   * Reset the non-separator tokens list to its default value
+   *
+   * @returns Promise containing an EnqueuedTask
+   */
+  async resetNonSeparatorTokens(): Promise<EnqueuedTask> {
+    const url = `indexes/${this.uid}/settings/non-separator-tokens`
+    const task = await this.httpRequest.delete<EnqueuedTask>(url)
+
+    task.enqueuedAt = new Date(task.enqueuedAt)
+
+    return task
+  }
+
+  ///
+  /// DICTIONARY
+  ///
+
+  /**
+   * Get the dictionary settings of a Meilisearch index.
+   *
+   * @returns Promise containing the dictionary settings
+   */
+  async getDictionary(): Promise<string[]> {
+    const url = `indexes/${this.uid}/settings/dictionary`
+    return await this.httpRequest.get<string[]>(url)
+  }
+
+  /**
+   * Update the the dictionary settings. Overwrite the old settings.
+   *
+   * @param dictionary - Array that contains the new dictionary settings.
+   * @returns Promise containing an EnqueuedTask or null
+   */
+  async updateDictionary(dictionary: Dictionary): Promise<EnqueuedTask> {
+    const url = `indexes/${this.uid}/settings/dictionary`
+    const task = await this.httpRequest.put(url, dictionary)
+
+    return new EnqueuedTask(task)
+  }
+
+  /**
+   * Reset the dictionary settings to its default value
+   *
+   * @returns Promise containing an EnqueuedTask
+   */
+  async resetDictionary(): Promise<EnqueuedTask> {
+    const url = `indexes/${this.uid}/settings/dictionary`
+    const task = await this.httpRequest.delete<EnqueuedTask>(url)
+
+    task.enqueuedAt = new Date(task.enqueuedAt)
+
+    return task
+  }
 }
 
 export { Index }

src/types/types.ts

@@ -312,6 +312,9 @@ export type TypoTolerance = {
     twoTypos?: number | null
   }
 } | null
+export type SeparatorTokens = string[] | null
+export type NonSeparatorTokens = string[] | null
+export type Dictionary = string[] | null
 
 export type FacetOrder = 'alpha' | 'count'
 
@@ -336,6 +339,9 @@ export type Settings = {
   typoTolerance?: TypoTolerance
   faceting?: Faceting
   pagination?: PaginationSettings
+  separatorTokens?: SeparatorTokens
+  nonSeparatorTokens?: NonSeparatorTokens
+  dictionary?: Dictionary
 }
 
 /*

tests/dictionary.test.ts

@@ -0,0 +1,124 @@
+import { EnqueuedTask } from '../src/enqueued-task'
+import {
+  clearAllIndexes,
+  config,
+  BAD_HOST,
+  MeiliSearch,
+  getClient,
+  dataset,
+} from './utils/meilisearch-test-utils'
+
+const index = {
+  uid: 'movies_test',
+}
+
+jest.setTimeout(100 * 1000)
+
+afterAll(() => {
+  return clearAllIndexes(config)
+})
+
+describe.each([{ permission: 'Master' }, { permission: 'Admin' }])(
+  'Test on dictionary',
+  ({ permission }) => {
+    beforeEach(async () => {
+      const client = await getClient('Master')
+      const { taskUid } = await client.index(index.uid).addDocuments(dataset)
+      await client.waitForTask(taskUid)
+    })
+
+    test(`${permission} key: Get default dictionary`, async () => {
+      const client = await getClient(permission)
+      const response: string[] = await client.index(index.uid).getDictionary()
+
+      expect(response).toEqual([])
+    })
+
+    test(`${permission} key: Update dictionary`, async () => {
+      const client = await getClient(permission)
+      const newDictionary = ['J. K.', 'J. R. R.']
+      const task: EnqueuedTask = await client
+        .index(index.uid)
+        .updateDictionary(newDictionary)
+      await client.index(index.uid).waitForTask(task.taskUid)
+
+      const response: string[] = await client.index(index.uid).getDictionary()
+
+      expect(response).toEqual(newDictionary)
+    })
+
+    test(`${permission} key: Update dictionary with null value`, async () => {
+      const client = await getClient(permission)
+      const newDictionary = null
+      const task: EnqueuedTask = await client
+        .index(index.uid)
+        .updateDictionary(newDictionary)
+      await client.index(index.uid).waitForTask(task.taskUid)
+
+      const response: string[] = await client.index(index.uid).getDictionary()
+
+      expect(response).toEqual([])
+    })
+
+    test(`${permission} key: Reset dictionary`, async () => {
+      const client = await getClient(permission)
+      const task: EnqueuedTask = await client.index(index.uid).resetDictionary()
+      await client.index(index.uid).waitForTask(task.taskUid)
+
+      const response: string[] = await client.index(index.uid).getDictionary()
+
+      expect(response).toEqual([])
+    })
+  }
+)
+
+describe.each([
+  { host: BAD_HOST, trailing: false },
+  { host: `${BAD_HOST}/api`, trailing: false },
+  { host: `${BAD_HOST}/trailing/`, trailing: true },
+])('Tests on url construction', ({ host, trailing }) => {
+  test(`Test getDictionary route`, async () => {
+    const route = `indexes/${index.uid}/settings/dictionary`
+    const client = new MeiliSearch({ host })
+    const strippedHost = trailing ? host.slice(0, -1) : host
+    await expect(
+      client.index(index.uid).getDictionary()
+    ).rejects.toHaveProperty(
+      'message',
+      `request to ${strippedHost}/${route} failed, reason: connect ECONNREFUSED ${BAD_HOST.replace(
+        'http://',
+        ''
+      )}`
+    )
+  })
+
+  test(`Test updateDictionary route`, async () => {
+    const route = `indexes/${index.uid}/settings/dictionary`
+    const client = new MeiliSearch({ host })
+    const strippedHost = trailing ? host.slice(0, -1) : host
+    await expect(
+      client.index(index.uid).updateDictionary([])
+    ).rejects.toHaveProperty(
+      'message',
+      `request to ${strippedHost}/${route} failed, reason: connect ECONNREFUSED ${BAD_HOST.replace(
+        'http://',
+        ''
+      )}`
+    )
+  })
+
+  test(`Test resetDictionary route`, async () => {
+    const route = `indexes/${index.uid}/settings/dictionary`
+    const client = new MeiliSearch({ host })
+    const strippedHost = trailing ? host.slice(0, -1) : host
+    await expect(
+      client.index(index.uid).resetDictionary()
+    ).rejects.toHaveProperty(
+      'message',
+      `request to ${strippedHost}/${route} failed, reason: connect ECONNREFUSED ${BAD_HOST.replace(
+        'http://',
+        ''
+      )}`
+    )
+  })
+})