[JS] Anno crud

js/package.json

@@ -1,6 +1,6 @@
 {
   "name": "langsmith",
-  "version": "0.1.58",
+  "version": "0.1.59",
   "description": "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform.",
   "packageManager": "yarn@1.22.19",
   "files": [

js/src/client.ts

@@ -32,6 +32,8 @@
   TracerSession,
   TracerSessionResult,
   ValueType,
+  AnnotationQueue,
+  RunWithAnnotationQueueInfo,
 } from "./schemas.js";
 import {
   convertLangChainMessageToExample,
@@ -1289,7 +1291,7 @@
     treeFilter?: string;
     isRoot?: boolean;
     dataSourceType?: string;
   }): Promise<any> {
     let projectIds_ = projectIds || [];
     if (projectNames) {
       projectIds_ = [
@@ -1577,7 +1579,7 @@
         `Failed to list shared examples: ${response.status} ${response.statusText}`
       );
     }
     return result.map((example: any) => ({
       ...example,
       _hostUrl: this.getHostUrl(),
     }));
@@ -2720,7 +2722,7 @@
     }
 
     const feedbackResult = await evaluator.evaluateRun(run_, referenceExample);
     const [_, feedbacks] = await this._logEvaluationFeedback(
       feedbackResult,
       run_,
       sourceInfo
@@ -3054,7 +3056,7 @@
   async _logEvaluationFeedback(
     evaluatorResponse: EvaluationResult | EvaluationResults,
     run?: Run,
     sourceInfo?: { [key: string]: any }
   ): Promise<[results: EvaluationResult[], feedbacks: Feedback[]]> {
     const evalResults: Array<EvaluationResult> =
       this._selectEvalResults(evaluatorResponse);
@@ -3093,7 +3095,7 @@
   public async logEvaluationFeedback(
     evaluatorResponse: EvaluationResult | EvaluationResults,
     run?: Run,
     sourceInfo?: { [key: string]: any }
   ): Promise<EvaluationResult[]> {
     const [results] = await this._logEvaluationFeedback(
       evaluatorResponse,
@@ -3103,6 +3105,208 @@
     return results;
   }
 
+  /**
+   * API for managing annotation queues
+   */
+
+  /**
+   * List the annotation queues on the LangSmith API.
+   * @param options - The options for listing annotation queues
+   * @param options.queueIds - The IDs of the queues to filter by
+   * @param options.name - The name of the queue to filter by
+   * @param options.nameContains - The substring that the queue name should contain
+   * @param options.limit - The maximum number of queues to return
+   * @returns An iterator of AnnotationQueue objects
+   */
+  public async *listAnnotationQueues(
+    options: {
+      queueIds?: string[];
+      name?: string;
+      nameContains?: string;
+      limit?: number;
+    } = {}
+  ): AsyncIterableIterator<AnnotationQueue> {
+    const { queueIds, name, nameContains, limit } = options;
+    const params = new URLSearchParams();
+    if (queueIds) {
+      queueIds.forEach((id, i) => {
+        assertUuid(id, `queueIds[${i}]`);
+        params.append("ids", id);
+      });
+    }
+    if (name) params.append("name", name);
+    if (nameContains) params.append("name_contains", nameContains);
+    params.append(
+      "limit",
+      (limit !== undefined ? Math.min(limit, 100) : 100).toString()
+    );
+
+    let count = 0;
+    for await (const queues of this._getPaginated<AnnotationQueue>(
+      "/annotation-queues",
+      params
+    )) {
+      yield* queues;
+      count++;
+      if (limit !== undefined && count >= limit) break;
+    }
+  }
+
+  /**
+   * Create an annotation queue on the LangSmith API.
+   * @param options - The options for creating an annotation queue
+   * @param options.name - The name of the annotation queue
+   * @param options.description - The description of the annotation queue
+   * @param options.queueId - The ID of the annotation queue
+   * @returns The created AnnotationQueue object
+   */
+  public async createAnnotationQueue(options: {
+    name: string;
+    description?: string;
+    queueId?: string;
+  }): Promise<AnnotationQueue> {
+    const { name, description, queueId } = options;
+    const body = {
+      name,
+      description,
+      id: queueId || uuid.v4(),
+    };
+
+    const response = await this.caller.call(
+      _getFetchImplementation(),
+      `${this.apiUrl}/annotation-queues`,
+      {
+        method: "POST",
+        headers: { ...this.headers, "Content-Type": "application/json" },
+        body: JSON.stringify(
+          Object.fromEntries(
+            Object.entries(body).filter(([_, v]) => v !== undefined)
+          )
+        ),
+        signal: AbortSignal.timeout(this.timeout_ms),
+        ...this.fetchOptions,
+      }
+    );
+    await raiseForStatus(response, "create annotation queue");
+    const data = await response.json();
+    return data as AnnotationQueue;
+  }
+
+  /**
+   * Read an annotation queue with the specified queue ID.
+   * @param queueId - The ID of the annotation queue to read
+   * @returns The AnnotationQueue object
+   */
+  public async readAnnotationQueue(queueId: string): Promise<AnnotationQueue> {
+    // TODO: Replace when actual endpoint is added
+    const queueIteratorResult = await this.listAnnotationQueues({
+      queueIds: [queueId],
+    }).next();
+    if (queueIteratorResult.done) {
+      throw new Error(`Annotation queue with ID ${queueId} not found`);
+    }
+    return queueIteratorResult.value;
+  }
+
+  /**
+   * Update an annotation queue with the specified queue ID.
+   * @param queueId - The ID of the annotation queue to update
+   * @param options - The options for updating the annotation queue
+   * @param options.name - The new name for the annotation queue
+   * @param options.description - The new description for the annotation queue
+   */
+  public async updateAnnotationQueue(
+    queueId: string,
+    options: {
+      name: string;
+      description?: string;
+    }
+  ): Promise<void> {
+    const { name, description } = options;
+    const response = await this.caller.call(
+      _getFetchImplementation(),
+      `${this.apiUrl}/annotation-queues/${assertUuid(queueId, "queueId")}`,
+      {
+        method: "PATCH",
+        headers: { ...this.headers, "Content-Type": "application/json" },
+        body: JSON.stringify({ name, description }),
+        signal: AbortSignal.timeout(this.timeout_ms),
+        ...this.fetchOptions,
+      }
+    );
+    await raiseForStatus(response, "update annotation queue");
+  }
+
+  /**
+   * Delete an annotation queue with the specified queue ID.
+   * @param queueId - The ID of the annotation queue to delete
+   */
+  public async deleteAnnotationQueue(queueId: string): Promise<void> {
+    const response = await this.caller.call(
+      _getFetchImplementation(),
+      `${this.apiUrl}/annotation-queues/${assertUuid(queueId, "queueId")}`,
+      {
+        method: "DELETE",
+        headers: { ...this.headers, Accept: "application/json" },
+        signal: AbortSignal.timeout(this.timeout_ms),
+        ...this.fetchOptions,
+      }
+    );
+    await raiseForStatus(response, "delete annotation queue");
+  }
+
+  /**
+   * Add runs to an annotation queue with the specified queue ID.
+   * @param queueId - The ID of the annotation queue
+   * @param runIds - The IDs of the runs to be added to the annotation queue
+   */
+  public async addRunsToAnnotationQueue(
+    queueId: string,
+    runIds: string[]
+  ): Promise<void> {
+    const response = await this.caller.call(
+      _getFetchImplementation(),
+      `${this.apiUrl}/annotation-queues/${assertUuid(queueId, "queueId")}/runs`,
+      {
+        method: "POST",
+        headers: { ...this.headers, "Content-Type": "application/json" },
+        body: JSON.stringify(
+          runIds.map((id, i) => assertUuid(id, `runIds[${i}]`).toString())
+        ),
+        signal: AbortSignal.timeout(this.timeout_ms),
+        ...this.fetchOptions,
+      }
+    );
+    await raiseForStatus(response, "add runs to annotation queue");
+  }
+
+  /**
+   * Get a run from an annotation queue at the specified index.
+   * @param queueId - The ID of the annotation queue
+   * @param index - The index of the run to retrieve
+   * @returns A Promise that resolves to a RunWithAnnotationQueueInfo object
+   * @throws {Error} If the run is not found at the given index or for other API-related errors
+   */
+  public async getRunFromAnnotationQueue(
+    queueId: string,
+    index: number
+  ): Promise<RunWithAnnotationQueueInfo> {
+    const baseUrl = `/annotation-queues/${assertUuid(queueId, "queueId")}/run`;
+    const response = await this.caller.call(
+      _getFetchImplementation(),
+      `${this.apiUrl}${baseUrl}/${index}`,
+      {
+        method: "GET",
+        headers: this.headers,
+        signal: AbortSignal.timeout(this.timeout_ms),
+        ...this.fetchOptions,
+      }
+    );
+
+    await raiseForStatus(response, "get run from annotation queue");
+    return await response.json();
+  }
+
   protected async _currentTenantIsOwner(owner: string): Promise<boolean> {
     const settings = await this._getSettings();
     return owner == "-" || settings.tenant_handle === owner;
@@ -3159,7 +3363,7 @@
     promptIdentifier: string,
     like: boolean
   ): Promise<LikePromptResponse> {
     const [owner, promptName, _] = parsePromptIdentifier(promptIdentifier);
     const response = await this.caller.call(
       _getFetchImplementation(),
       `${this.apiUrl}/likes/${owner}/${promptName}`,
@@ -3228,7 +3432,7 @@
       ListCommitsResponse
     >(
       `/commits/${promptOwnerAndName}/`,
-      {} as URLSearchParams,
+      new URLSearchParams(),
       (res) => res.commits
     )) {
       yield* commits;
@@ -3264,7 +3468,7 @@
   }
 
   public async getPrompt(promptIdentifier: string): Promise<Prompt | null> {
     const [owner, promptName, _] = parsePromptIdentifier(promptIdentifier);
     const response = await this.caller.call(
       _getFetchImplementation(),
       `${this.apiUrl}/repos/${owner}/${promptName}`,
@@ -3308,7 +3512,7 @@
       );
     }
 
     const [owner, promptName, _] = parsePromptIdentifier(promptIdentifier);
     if (!(await this._currentTenantIsOwner(owner))) {
       throw await this._ownerConflictError("create a prompt", owner);
     }
@@ -3341,7 +3545,7 @@
 
   public async createCommit(
     promptIdentifier: string,
     object: any,
     options?: {
       parentCommitHash?: string;
     }
@@ -3350,7 +3554,7 @@
       throw new Error("Prompt does not exist, you must create it first.");
     }
 
     const [owner, promptName, _] = parsePromptIdentifier(promptIdentifier);
     const resolvedParentCommitHash =
       options?.parentCommitHash === "latest" || !options?.parentCommitHash
         ? await this._getLatestCommitHash(`${owner}/${promptName}`)

js/src/index.ts

@@ -14,4 +14,4 @@ export { RunTree, type RunTreeConfig } from "./run_trees.js";
 export { overrideFetchImplementation } from "./singletons/fetch.js";
 
 // Update using yarn bump-version
-export const __version__ = "0.1.58";
+export const __version__ = "0.1.59";

js/src/schemas.ts

@@ -474,3 +474,31 @@ export interface LangSmithSettings {
   created_at: string;
   tenant_handle?: string;
 }
+
+export interface AnnotationQueue {
+  /** The unique identifier of the annotation queue. */
+  id: string;
+
+  /** The name of the annotation queue. */
+  name: string;
+
+  /** An optional description of the annotation queue. */
+  description?: string;
+
+  /** The timestamp when the annotation queue was created. */
+  created_at: string;
+
+  /** The timestamp when the annotation queue was last updated. */
+  updated_at: string;
+
+  /** The ID of the tenant associated with the annotation queue. */
+  tenant_id: string;
+}
+
+export interface RunWithAnnotationQueueInfo extends BaseRun {
+  /** The last time this run was reviewed. */
+  last_reviewed_time?: string;
+
+  /** The time this run was added to the queue. */
+  added_at?: string;
+}

js/src/tests/client.int.test.ts

@@ -1147,3 +1147,96 @@ test("clonePublicDataset method can clone a dataset", async () => {
     }
   }
 });
+
+test("annotationqueue crud", async () => {
+  const client = new Client();
+  const queueName = `test-queue-${uuidv4().substring(0, 8)}`;
+  const projectName = `test-project-${uuidv4().substring(0, 8)}`;
+  const queueId = uuidv4();
+
+  try {
+    // 1. Create an annotation queue
+    const queue = await client.createAnnotationQueue({
+      name: queueName,
+      description: "Initial description",
+      queueId,
+    });
+    expect(queue).toBeDefined();
+    expect(queue.name).toBe(queueName);
+
+    // 1a. Get the annotation queue
+    const fetchedQueue = await client.readAnnotationQueue(queue.id);
+    expect(fetchedQueue).toBeDefined();
+    expect(fetchedQueue.name).toBe(queueName);
+
+    // 1b. List annotation queues and check nameContains
+    const listedQueues = await toArray(
+      client.listAnnotationQueues({ nameContains: queueName })
+    );
+    expect(listedQueues.length).toBeGreaterThan(0);
+    expect(listedQueues.some((q) => q.id === queue.id)).toBe(true);
+
+    // 2. Create a run in a random project
+    await client.createProject({ projectName });
+    const runId = uuidv4();
+    await client.createRun({
+      id: runId,
+      name: "Test Run",
+      run_type: "chain",
+      inputs: { foo: "bar" },
+      outputs: { baz: "qux" },
+      project_name: projectName,
+    });
+
+    // Wait for run to be found in the db
+    const maxWaitTime = 30000; // 30 seconds
+    const startTime = Date.now();
+    let foundRun = null;
+
+    while (Date.now() - startTime < maxWaitTime) {
+      try {
+        foundRun = await client.readRun(runId);
+        if (foundRun) break;
+      } catch (error) {
+        // If run is not found, getRun might throw an error
+        // We'll ignore it and keep trying
+      }
+      await new Promise((resolve) => setTimeout(resolve, 1000)); // Wait for 1 second before trying again
+    }
+
+    if (!foundRun) {
+      throw new Error(
+        `Run with ID ${runId} not found after ${maxWaitTime / 1000} seconds`
+      );
+    }
+
+    // 3. Add the run to the annotation queue
+    await client.addRunsToAnnotationQueue(fetchedQueue.id, [runId]);
+
+    // 4. Update the annotation queue description and check that it is updated
+    const newDescription = "Updated description";
+    await client.updateAnnotationQueue(queue.id, {
+      name: queueName,
+      description: newDescription,
+    });
+    const updatedQueue = await client.readAnnotationQueue(queue.id);
+    expect(updatedQueue.description).toBe(newDescription);
+
+    // Get the run from the annotation queue
+    const run = await client.getRunFromAnnotationQueue(queueId, 0);
+    expect(run).toBeDefined();
+    expect(run.id).toBe(runId);
+    expect(run.name).toBe("Test Run");
+    expect(run.run_type).toBe("chain");
+    expect(run.inputs).toEqual({ foo: "bar" });
+    expect(run.outputs).toEqual({ baz: "qux" });
+  } finally {
+    // 6. Delete the annotation queue
+    await client.deleteAnnotationQueue(queueId);
+
+    // Clean up the project
+    if (await client.hasProject({ projectName })) {
+      await client.deleteProject({ projectName });
+    }
+  }
+});

js/src/utils/_uuid.ts

@@ -1,7 +1,12 @@
 import * as uuid from "uuid";
 
-export function assertUuid(str: string): void {
+export function assertUuid(str: string, which?: string): string {
   if (!uuid.validate(str)) {
-    throw new Error(`Invalid UUID: ${str}`);
+    const msg =
+      which !== undefined
+        ? `Invalid UUID for ${which}: ${str}`
+        : `Invalid UUID: ${str}`;
+    throw new Error(msg);
   }
+  return str;
 }