From 9a53a5da90f6313c81fd221f5beeecbad5c479f4 Mon Sep 17 00:00:00 2001
From: William Fu-Hinthorn <13333726+hinthornw@users.noreply.github.com>
Date: Tue, 17 Sep 2024 10:12:06 -0700
Subject: [PATCH 1/7] [JS] Anno crud

---
 js/src/client.ts      | 213 ++++++++++++++++++++++++++++++++++++++++++
 js/src/schemas.ts     |  28 ++++++
 js/src/utils/_uuid.ts |   9 +-
 3 files changed, 248 insertions(+), 2 deletions(-)

diff --git a/js/src/client.ts b/js/src/client.ts
index 5a57c3651..c362af893 100644
--- a/js/src/client.ts
+++ b/js/src/client.ts
@@ -32,6 +32,8 @@ import {
   TracerSession,
   TracerSessionResult,
   ValueType,
+  AnnotationQueue,
+  RunWithAnnotationQueueInfo,
 } from "./schemas.js";
 import {
   convertLangChainMessageToExample,
@@ -3103,6 +3105,217 @@ export class Client {
     return results;
   }
 
+  /**
+   * API for managing annotation queues
+   */
+
+  /**
+   * List the annotation queues on the LangSmith API.
+   * @param options - The options for listing annotation queues
+   * @param options.queueIds - The IDs of the queues to filter by
+   * @param options.name - The name of the queue to filter by
+   * @param options.nameContains - The substring that the queue name should contain
+   * @param options.limit - The maximum number of queues to return
+   * @returns An iterator of AnnotationQueue objects
+   */
+  public async *listAnnotationQueues(
+    options: {
+      queueIds?: string[];
+      name?: string;
+      nameContains?: string;
+      limit?: number;
+    } = {}
+  ): AsyncIterableIterator<AnnotationQueue> {
+    const { queueIds, name, nameContains, limit } = options;
+    const params: Record<string, any> = {
+      ids: queueIds?.map((id, i) => assertUuid(id, `queueIds[${i}]`)),
+      name,
+      name_contains: nameContains,
+      limit: limit !== undefined ? Math.min(limit, 100) : 100,
+    };
+
+    let count = 0;
+    for await (const queue of this._getPaginated(
+      "/annotation-queues",
+      params as URLSearchParams
+    )) {
+      yield queue as unknown as AnnotationQueue;
+      count++;
+      if (limit !== undefined && count >= limit) break;
+    }
+  }
+
+  /**
+   * Create an annotation queue on the LangSmith API.
+   * @param options - The options for creating an annotation queue
+   * @param options.name - The name of the annotation queue
+   * @param options.description - The description of the annotation queue
+   * @param options.queueId - The ID of the annotation queue
+   * @returns The created AnnotationQueue object
+   */
+  public async createAnnotationQueue(options: {
+    name: string;
+    description?: string;
+    queueId?: string;
+  }): Promise<AnnotationQueue> {
+    const { name, description, queueId } = options;
+    const body = {
+      name,
+      description,
+      id: queueId || uuid.v4(),
+    };
+
+    const response = await this.caller.call(
+      _getFetchImplementation(),
+      `${this.apiUrl}/annotation-queues`,
+      {
+        method: "POST",
+        headers: { ...this.headers, "Content-Type": "application/json" },
+        body: JSON.stringify(
+          Object.fromEntries(
+            Object.entries(body).filter(([_, v]) => v !== undefined)
+          )
+        ),
+        signal: AbortSignal.timeout(this.timeout_ms),
+        ...this.fetchOptions,
+      }
+    );
+    await raiseForStatus(response, "create annotation queue");
+    const data = await response.json();
+    return data as AnnotationQueue;
+  }
+
+  /**
+   * Read an annotation queue with the specified queue ID.
+   * @param queueId - The ID of the annotation queue to read
+   * @returns The AnnotationQueue object
+   */
+  public async readAnnotationQueue(queueId: string): Promise<AnnotationQueue> {
+    // TODO: Replace when actual endpoint is added
+    const queue = await this.listAnnotationQueues({
+      queueIds: [queueId],
+    }).next();
+    if (queue.done) {
+      throw new Error(`Annotation queue with ID ${queueId} not found`);
+    }
+    return queue.value;
+  }
+
+  /**
+   * Update an annotation queue with the specified queue ID.
+   * @param queueId - The ID of the annotation queue to update
+   * @param options - The options for updating the annotation queue
+   * @param options.name - The new name for the annotation queue
+   * @param options.description - The new description for the annotation queue
+   */
+  public async updateAnnotationQueue(
+    queueId: string,
+    options: {
+      name: string;
+      description?: string;
+    }
+  ): Promise<void> {
+    const { name, description } = options;
+    const response = await this.caller.call(
+      _getFetchImplementation(),
+      `${this.apiUrl}/annotation-queues/${assertUuid(queueId, "queueId")}`,
+      {
+        method: "PATCH",
+        headers: { ...this.headers, "Content-Type": "application/json" },
+        body: JSON.stringify({ name, description }),
+        signal: AbortSignal.timeout(this.timeout_ms),
+        ...this.fetchOptions,
+      }
+    );
+    await raiseForStatus(response, "update annotation queue");
+  }
+
+  /**
+   * Delete an annotation queue with the specified queue ID.
+   * @param queueId - The ID of the annotation queue to delete
+   */
+  public async deleteAnnotationQueue(queueId: string): Promise<void> {
+    const response = await this.caller.call(
+      _getFetchImplementation(),
+      `${this.apiUrl}/annotation-queues/${assertUuid(queueId, "queueId")}`,
+      {
+        method: "DELETE",
+        headers: { ...this.headers, Accept: "application/json" },
+        signal: AbortSignal.timeout(this.timeout_ms),
+        ...this.fetchOptions,
+      }
+    );
+    await raiseForStatus(response, "delete annotation queue");
+  }
+
+  /**
+   * Add runs to an annotation queue with the specified queue ID.
+   * @param queueId - The ID of the annotation queue
+   * @param runIds - The IDs of the runs to be added to the annotation queue
+   */
+  public async addRunsToAnnotationQueue(
+    queueId: string,
+    runIds: string[]
+  ): Promise<void> {
+    const response = await this.caller.call(
+      _getFetchImplementation(),
+      `${this.apiUrl}/annotation-queues/${assertUuid(queueId, "queueId")}/runs`,
+      {
+        method: "POST",
+        headers: { ...this.headers, "Content-Type": "application/json" },
+        body: JSON.stringify(
+          runIds.map((id, i) => assertUuid(id, `runIds[${i}]`).toString())
+        ),
+        signal: AbortSignal.timeout(this.timeout_ms),
+        ...this.fetchOptions,
+      }
+    );
+    await raiseForStatus(response, "add runs to annotation queue");
+  }
+
+  /**
+   * List runs from an annotation queue with the specified queue ID.
+   * @param queueId - The ID of the annotation queue
+   * @param limit - The maximum number of runs to return
+   * @returns An iterator of RunWithAnnotationQueueInfo objects
+   */
+  public async *listRunsFromAnnotationQueue(
+    queueId: string,
+    limit?: number
+  ): AsyncIterableIterator<RunWithAnnotationQueueInfo> {
+    const path = `/annotation-queues/${assertUuid(queueId, "queueId")}/runs`;
+    const limit_ = limit !== undefined ? Math.min(limit, 100) : 100;
+
+    let offset = 0;
+    let count = 0;
+    while (true) {
+      const url = `${this.apiUrl}${path}?offset=${offset}&limit=${limit_}`;
+      const response = await this.caller.call(_getFetchImplementation(), url, {
+        method: "GET",
+        headers: this.headers,
+        signal: AbortSignal.timeout(this.timeout_ms),
+        ...this.fetchOptions,
+      });
+      await raiseForStatus(response, `Failed to fetch ${path}`);
+      const data = await response.json();
+      const runs = data.runs as RunWithAnnotationQueueInfo[];
+
+      if (runs.length === 0) {
+        break;
+      }
+
+      for (const run of runs) {
+        yield run;
+        count++;
+        if (limit !== undefined && count >= limit) {
+          return;
+        }
+      }
+
+      offset += runs.length;
+    }
+  }
+
   protected async _currentTenantIsOwner(owner: string): Promise<boolean> {
     const settings = await this._getSettings();
     return owner == "-" || settings.tenant_handle === owner;
diff --git a/js/src/schemas.ts b/js/src/schemas.ts
index 4d73f29aa..7275f6d39 100644
--- a/js/src/schemas.ts
+++ b/js/src/schemas.ts
@@ -474,3 +474,31 @@ export interface LangSmithSettings {
   created_at: string;
   tenant_handle?: string;
 }
+
+export interface AnnotationQueue {
+  /** The unique identifier of the annotation queue. */
+  id: string;
+
+  /** The name of the annotation queue. */
+  name: string;
+
+  /** An optional description of the annotation queue. */
+  description?: string;
+
+  /** The timestamp when the annotation queue was created. */
+  created_at: string;
+
+  /** The timestamp when the annotation queue was last updated. */
+  updated_at: string;
+
+  /** The ID of the tenant associated with the annotation queue. */
+  tenant_id: string;
+}
+
+export interface RunWithAnnotationQueueInfo extends BaseRun {
+  /** The last time this run was reviewed. */
+  last_reviewed_time?: string;
+
+  /** The time this run was added to the queue. */
+  added_at?: string;
+}
diff --git a/js/src/utils/_uuid.ts b/js/src/utils/_uuid.ts
index 714235131..51d71f020 100644
--- a/js/src/utils/_uuid.ts
+++ b/js/src/utils/_uuid.ts
@@ -1,7 +1,12 @@
 import * as uuid from "uuid";
 
-export function assertUuid(str: string): void {
+export function assertUuid(str: string, which?: string): string {
   if (!uuid.validate(str)) {
-    throw new Error(`Invalid UUID: ${str}`);
+    const msg =
+      which !== undefined
+        ? `Invalid UUID for ${which}: ${str}`
+        : `Invalid UUID: ${str}`;
+    throw new Error(msg);
   }
+  return str;
 }

From 5718627ae3df1b6566a5d87880823a63a2203be7 Mon Sep 17 00:00:00 2001
From: nhuang-lc <nick@langchain.dev>
Date: Tue, 17 Sep 2024 17:11:18 -0700
Subject: [PATCH 2/7] Add methods in slots attribute (#1012)

Add methods that are in the slots attribute to the functions list

---------

Co-authored-by: Nick Huang <nickhuang@Nicks-MacBook-Pro.local>
---
 python/docs/create_api_rst.py | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/python/docs/create_api_rst.py b/python/docs/create_api_rst.py
index 3f51da948..253352767 100644
--- a/python/docs/create_api_rst.py
+++ b/python/docs/create_api_rst.py
@@ -108,6 +108,18 @@ def _load_module_members(module_path: str, namespace: str) -> ModuleMembers:
                     else "Pydantic" if issubclass(type_, BaseModel) else "Regular"
                 )
             )
+            if hasattr(type_, "__slots__"):
+                for func_name, func_type in inspect.getmembers(type_):
+                    if inspect.isfunction(func_type):
+                        functions.append(
+                            FunctionInfo(
+                                name=func_name,
+                                qualified_name=f"{namespace}.{name}.{func_name}",
+                                is_public=not func_name.startswith("_"),
+                                is_deprecated=".. deprecated::"
+                                in (func_type.__doc__ or ""),
+                            )
+                        )
             classes_.append(
                 ClassInfo(
                     name=name,

From c1a95046d592f0aa39384edd3c50e65379c239ce Mon Sep 17 00:00:00 2001
From: William FH <13333726+hinthornw@users.noreply.github.com>
Date: Tue, 17 Sep 2024 17:51:29 -0700
Subject: [PATCH 3/7] [Python] Filter beta warning in prompt deserialiazation
 (#1013)

---
 python/langsmith/client.py | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/python/langsmith/client.py b/python/langsmith/client.py
index 6377bd2d0..cb0e863f0 100644
--- a/python/langsmith/client.py
+++ b/python/langsmith/client.py
@@ -15,6 +15,7 @@
 import atexit
 import collections
 import concurrent.futures as cf
+import contextlib
 import datetime
 import functools
 import importlib
@@ -5294,11 +5295,19 @@ def pull_prompt(
                 "The client.pull_prompt function requires the langchain_core"
                 "package to run.\nInstall with `pip install langchain_core`"
             )
+        try:
+            from langchain_core._api import suppress_langchain_beta_warning
+        except ImportError:
+
+            @contextlib.contextmanager
+            def suppress_langchain_beta_warning():
+                yield
 
         prompt_object = self.pull_prompt_commit(
             prompt_identifier, include_model=include_model
         )
-        prompt = loads(json.dumps(prompt_object.manifest))
+        with suppress_langchain_beta_warning():
+            prompt = loads(json.dumps(prompt_object.manifest))
 
         if (
             isinstance(prompt, BasePromptTemplate)

From 69c1ae0c67af6b2d111af335cb53031655799092 Mon Sep 17 00:00:00 2001
From: William FH <13333726+hinthornw@users.noreply.github.com>
Date: Wed, 18 Sep 2024 06:06:27 -0700
Subject: [PATCH 4/7] Always trace evaluators (#1014)

---
 js/package.json                    | 2 +-
 js/src/evaluation/_runner.ts       | 1 +
 js/src/index.ts                    | 2 +-
 js/src/tests/run_trees.int.test.ts | 4 ++--
 4 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/js/package.json b/js/package.json
index eec53685e..a941749c0 100644
--- a/js/package.json
+++ b/js/package.json
@@ -1,6 +1,6 @@
 {
   "name": "langsmith",
-  "version": "0.1.57",
+  "version": "0.1.58",
   "description": "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform.",
   "packageManager": "yarn@1.22.19",
   "files": [
diff --git a/js/src/evaluation/_runner.ts b/js/src/evaluation/_runner.ts
index f6a946bf2..cdd5b3ccf 100644
--- a/js/src/evaluation/_runner.ts
+++ b/js/src/evaluation/_runner.ts
@@ -566,6 +566,7 @@ export class _ExperimentManager {
               : new Date(example.created_at).toISOString(),
           },
           client: fields.client,
+          tracingEnabled: true,
         };
         const evaluatorResponse = await evaluator.evaluateRun(
           run,
diff --git a/js/src/index.ts b/js/src/index.ts
index 42fa4669f..54ead368d 100644
--- a/js/src/index.ts
+++ b/js/src/index.ts
@@ -14,4 +14,4 @@ export { RunTree, type RunTreeConfig } from "./run_trees.js";
 export { overrideFetchImplementation } from "./singletons/fetch.js";
 
 // Update using yarn bump-version
-export const __version__ = "0.1.57";
+export const __version__ = "0.1.58";
diff --git a/js/src/tests/run_trees.int.test.ts b/js/src/tests/run_trees.int.test.ts
index ecd40976f..15199efda 100644
--- a/js/src/tests/run_trees.int.test.ts
+++ b/js/src/tests/run_trees.int.test.ts
@@ -16,7 +16,7 @@ test.concurrent(
   "Test post and patch run",
   async () => {
     const projectName = `__test_run_tree_js ${uuid.v4()}`;
-    const langchainClient = new Client({ timeout_ms: 30000 });
+    const langchainClient = new Client({ timeout_ms: 30_000 });
     const parentRunConfig: RunTreeConfig = {
       name: "parent_run",
       run_type: "chain",
@@ -33,7 +33,7 @@ test.concurrent(
     );
     await parent_run.postRun();
 
-    const child_llm_run = await parent_run.createChild({
+    const child_llm_run = parent_run.createChild({
       name: "child_run",
       run_type: "llm",
       inputs: { text: "hello world" },

From 2f9789317139bfc066d0832398c397abd3fc2658 Mon Sep 17 00:00:00 2001
From: William Fu-Hinthorn <13333726+hinthornw@users.noreply.github.com>
Date: Wed, 18 Sep 2024 09:43:36 -0700
Subject: [PATCH 5/7] Add tests

---
 js/src/client.ts                | 23 ++++++---
 js/src/tests/client.int.test.ts | 90 +++++++++++++++++++++++++++++++++
 2 files changed, 105 insertions(+), 8 deletions(-)

diff --git a/js/src/client.ts b/js/src/client.ts
index 79c5e23f0..7d3a4fe22 100644
--- a/js/src/client.ts
+++ b/js/src/client.ts
@@ -3127,17 +3127,24 @@ export class Client {
     } = {}
   ): AsyncIterableIterator<AnnotationQueue> {
     const { queueIds, name, nameContains, limit } = options;
-    const params: Record<string, any> = {
-      ids: queueIds?.map((id, i) => assertUuid(id, `queueIds[${i}]`)),
-      name,
-      name_contains: nameContains,
-      limit: limit !== undefined ? Math.min(limit, 100) : 100,
-    };
+    const params = new URLSearchParams();
+    if (queueIds) {
+      queueIds.forEach((id, i) => {
+        assertUuid(id, `queueIds[${i}]`);
+        params.append("ids", id);
+      });
+    }
+    if (name) params.append("name", name);
+    if (nameContains) params.append("name_contains", nameContains);
+    params.append(
+      "limit",
+      (limit !== undefined ? Math.min(limit, 100) : 100).toString()
+    );
 
     let count = 0;
     for await (const queue of this._getPaginated(
       "/annotation-queues",
-      params as URLSearchParams
+      params
     )) {
       yield queue as unknown as AnnotationQueue;
       count++;
@@ -3441,7 +3448,7 @@ export class Client {
       ListCommitsResponse
     >(
       `/commits/${promptOwnerAndName}/`,
-      {} as URLSearchParams,
+      new URLSearchParams(),
       (res) => res.commits
     )) {
       yield* commits;
diff --git a/js/src/tests/client.int.test.ts b/js/src/tests/client.int.test.ts
index 1846538f5..bb7af37c2 100644
--- a/js/src/tests/client.int.test.ts
+++ b/js/src/tests/client.int.test.ts
@@ -1147,3 +1147,93 @@ test("clonePublicDataset method can clone a dataset", async () => {
     }
   }
 });
+
+test("annotationqueue crud", async () => {
+  const client = new Client();
+  const queueName = `test-queue-${uuidv4().substring(0, 8)}`;
+  const projectName = `test-project-${uuidv4().substring(0, 8)}`;
+  const queueId = uuidv4();
+
+  try {
+    // 1. Create an annotation queue
+    const queue = await client.createAnnotationQueue({
+      name: queueName,
+      description: "Initial description",
+      queueId,
+    });
+    expect(queue).toBeDefined();
+    expect(queue.name).toBe(queueName);
+
+    // 1a. Get the annotation queue
+    const fetchedQueue = await client.readAnnotationQueue(queue.id);
+    expect(fetchedQueue).toBeDefined();
+    expect(fetchedQueue.name).toBe(queueName);
+
+    // 1b. List annotation queues and check nameContains
+    const listedQueues = await toArray(
+      client.listAnnotationQueues({ nameContains: queueName })
+    );
+    expect(listedQueues.length).toBeGreaterThan(0);
+    expect(listedQueues.some((q) => q.id === queue.id)).toBe(true);
+
+    // 2. Create a run in a random project
+    await client.createProject({ projectName });
+    const runId = uuidv4();
+    await client.createRun({
+      id: runId,
+      name: "Test Run",
+      run_type: "chain",
+      inputs: { foo: "bar" },
+      outputs: { baz: "qux" },
+      project_name: projectName,
+    });
+
+    // Wait for run to be found in the db
+    const maxWaitTime = 30000; // 30 seconds
+    const startTime = Date.now();
+    let foundRun = null;
+
+    while (Date.now() - startTime < maxWaitTime) {
+      try {
+        foundRun = await client.readRun(runId);
+        if (foundRun) break;
+      } catch (error) {
+        // If run is not found, getRun might throw an error
+        // We'll ignore it and keep trying
+      }
+      await new Promise((resolve) => setTimeout(resolve, 1000)); // Wait for 1 second before trying again
+    }
+
+    if (!foundRun) {
+      throw new Error(
+        `Run with ID ${runId} not found after ${maxWaitTime / 1000} seconds`
+      );
+    }
+
+    // 3. Add the run to the annotation queue
+    await client.addRunsToAnnotationQueue(fetchedQueue.id, [runId]);
+
+    // 4. Check that the run is in the annotation queue
+    const queuedRuns = await toArray(
+      client.listRunsFromAnnotationQueue(queue.id)
+    );
+    expect(queuedRuns.some((r) => r.id === runId)).toBe(true);
+
+    // 5. Update the annotation queue description and check that it is updated
+    const newDescription = "Updated description";
+    await client.updateAnnotationQueue(queue.id, {
+      name: queueName,
+      description: newDescription,
+    });
+    const updatedQueue = await client.readAnnotationQueue(queue.id);
+    expect(updatedQueue.description).toBe(newDescription);
+  } finally {
+    // 6. Delete the annotation queue
+    await client.deleteAnnotationQueue(queueId);
+
+    // Clean up the project
+    if (await client.hasProject({ projectName })) {
+      await client.deleteProject({ projectName });
+    }
+  }
+});

From edad4f4308202399c3f94f9be4bf31e42fbc046b Mon Sep 17 00:00:00 2001
From: William Fu-Hinthorn <13333726+hinthornw@users.noreply.github.com>
Date: Wed, 18 Sep 2024 11:12:41 -0700
Subject: [PATCH 6/7] rm method

---
 js/src/client.ts                | 69 +++++++++------------------------
 js/src/tests/client.int.test.ts | 17 ++++----
 python/langsmith/client.py      | 51 +++++++++---------------
 3 files changed, 47 insertions(+), 90 deletions(-)

diff --git a/js/src/client.ts b/js/src/client.ts
index b21fd5597..b7ef0beef 100644
--- a/js/src/client.ts
+++ b/js/src/client.ts
@@ -3281,61 +3281,30 @@ export class Client {
   }
 
   /**
-   * List runs from an annotation queue with the specified queue ID.
+   * Get a run from an annotation queue at the specified index.
    * @param queueId - The ID of the annotation queue
-   * @param limit - The maximum number of runs to return
-   * @returns An async iterable of RunWithAnnotationQueueInfo objects
+   * @param index - The index of the run to retrieve
+   * @returns A Promise that resolves to a RunWithAnnotationQueueInfo object
+   * @throws {Error} If the run is not found at the given index or for other API-related errors
    */
-  public async *listRunsFromAnnotationQueue(
+  public async getRunFromAnnotationQueue(
     queueId: string,
-    limit?: number
-  ): AsyncIterable<RunWithAnnotationQueueInfo> {
+    index: number
+  ): Promise<RunWithAnnotationQueueInfo> {
     const baseUrl = `/annotation-queues/${assertUuid(queueId, "queueId")}/run`;
-    let index = 0;
-    let i = 0;
-    while (true) {
-      try {
-        console.log("GETTT", `${this.apiUrl}${baseUrl}/${index}`);
-        const response = await this.caller.call(
-          _getFetchImplementation(),
-          `${this.apiUrl}${baseUrl}/${index}`,
-          {
-            method: "GET",
-            headers: this.headers,
-            signal: AbortSignal.timeout(this.timeout_ms),
-            ...this.fetchOptions,
-          }
-        );
-
-        if (!response.ok) {
-          if (response.status === 404) {
-            break;
-          }
-          await raiseForStatus(response, "list runs from annotation queue");
-        }
-
-        const run: RunWithAnnotationQueueInfo = await response.json();
-        yield run;
-
-        i++;
-        if (limit !== undefined && i >= limit) {
-          return;
-        }
-
-        index++;
-      } catch (error) {
-        if (
-          error &&
-          typeof error === "object" &&
-          "message" in error &&
-          typeof error.message === "string" &&
-          error.message.includes("404")
-        ) {
-          break;
-        }
-        throw error;
+    const response = await this.caller.call(
+      _getFetchImplementation(),
+      `${this.apiUrl}${baseUrl}/${index}`,
+      {
+        method: "GET",
+        headers: this.headers,
+        signal: AbortSignal.timeout(this.timeout_ms),
+        ...this.fetchOptions,
       }
-    }
+    );
+
+    await raiseForStatus(response, "get run from annotation queue");
+    return await response.json();
   }
 
   protected async _currentTenantIsOwner(owner: string): Promise<boolean> {
diff --git a/js/src/tests/client.int.test.ts b/js/src/tests/client.int.test.ts
index bb7af37c2..ddf160fa3 100644
--- a/js/src/tests/client.int.test.ts
+++ b/js/src/tests/client.int.test.ts
@@ -1213,13 +1213,7 @@ test("annotationqueue crud", async () => {
     // 3. Add the run to the annotation queue
     await client.addRunsToAnnotationQueue(fetchedQueue.id, [runId]);
 
-    // 4. Check that the run is in the annotation queue
-    const queuedRuns = await toArray(
-      client.listRunsFromAnnotationQueue(queue.id)
-    );
-    expect(queuedRuns.some((r) => r.id === runId)).toBe(true);
-
-    // 5. Update the annotation queue description and check that it is updated
+    // 4. Update the annotation queue description and check that it is updated
     const newDescription = "Updated description";
     await client.updateAnnotationQueue(queue.id, {
       name: queueName,
@@ -1227,6 +1221,15 @@ test("annotationqueue crud", async () => {
     });
     const updatedQueue = await client.readAnnotationQueue(queue.id);
     expect(updatedQueue.description).toBe(newDescription);
+
+    // Get the run from the annotation queue
+    const run = await client.getRunFromAnnotationQueue(queueId, 0);
+    expect(run).toBeDefined();
+    expect(run.id).toBe(runId);
+    expect(run.name).toBe("Test Run");
+    expect(run.run_type).toBe("chain");
+    expect(run.inputs).toEqual({ foo: "bar" });
+    expect(run.outputs).toEqual({ baz: "qux" });
   } finally {
     // 6. Delete the annotation queue
     await client.deleteAnnotationQueue(queueId);
diff --git a/python/langsmith/client.py b/python/langsmith/client.py
index 4cc2ef91a..123f869f6 100644
--- a/python/langsmith/client.py
+++ b/python/langsmith/client.py
@@ -4681,45 +4681,30 @@ def add_runs_to_annotation_queue(
         )
         ls_utils.raise_for_status_with_text(response)
 
-    def list_runs_from_annotation_queue(
-        self, queue_id: ID_TYPE, *, limit: Optional[int] = None
-    ) -> Iterator[ls_schemas.RunWithAnnotationQueueInfo]:
-        """List runs from an annotation queue with the specified queue ID.
+    def get_run_from_annotation_queue(
+        self, queue_id: ID_TYPE, *, index: int
+    ) -> ls_schemas.RunWithAnnotationQueueInfo:
+        """Get a run from an annotation queue at the specified index.
 
         Args:
             queue_id (ID_TYPE): The ID of the annotation queue.
-            limit (Optional[int]): The maximum number of runs to return.
+            index (int): The index of the run to retrieve.
 
-        Yields:
-            ls_schemas.RunWithAnnotationQueueInfo: An iterator of runs from the
-                annotation queue.
+        Returns:
+            ls_schemas.RunWithAnnotationQueueInfo: The run at the specified index.
+
+        Raises:
+            ls_utils.LangSmithNotFoundError: If the run is not found at the given index.
+            ls_utils.LangSmithError: For other API-related errors.
         """
         base_url = f"/annotation-queues/{_as_uuid(queue_id, 'queue_id')}/run"
-        index = 0
-        i = 0
-        while True:
-            try:
-                response = self.request_with_retries(
-                    "GET",
-                    f"{base_url}/{index}",
-                    headers=self._headers,
-                )
-                if response.status_code == 404:
-                    break
-                ls_utils.raise_for_status_with_text(response)
-
-                run = ls_schemas.RunWithAnnotationQueueInfo(**response.json())
-                yield run
-
-                i += 1
-                if limit is not None and i >= limit:
-                    return
-
-                index += 1
-            except ls_utils.LangSmithNotFoundError:
-                break
-            except ls_utils.LangSmithError as e:
-                raise e
+        response = self.request_with_retries(
+            "GET",
+            f"{base_url}/{index}",
+            headers=self._headers,
+        )
+        ls_utils.raise_for_status_with_text(response)
+        return ls_schemas.RunWithAnnotationQueueInfo(**response.json())
 
     def create_comparative_experiment(
         self,

From e848c5927bb2045b2b3d11476f1aba9a1fe1ec51 Mon Sep 17 00:00:00 2001
From: William Fu-Hinthorn <13333726+hinthornw@users.noreply.github.com>
Date: Wed, 18 Sep 2024 11:52:53 -0700
Subject: [PATCH 7/7] Bump

---
 js/package.json       | 2 +-
 js/src/index.ts       | 2 +-
 python/pyproject.toml | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/js/package.json b/js/package.json
index a941749c0..d149b61d2 100644
--- a/js/package.json
+++ b/js/package.json
@@ -1,6 +1,6 @@
 {
   "name": "langsmith",
-  "version": "0.1.58",
+  "version": "0.1.59",
   "description": "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform.",
   "packageManager": "yarn@1.22.19",
   "files": [
diff --git a/js/src/index.ts b/js/src/index.ts
index 54ead368d..a43ebda35 100644
--- a/js/src/index.ts
+++ b/js/src/index.ts
@@ -14,4 +14,4 @@ export { RunTree, type RunTreeConfig } from "./run_trees.js";
 export { overrideFetchImplementation } from "./singletons/fetch.js";
 
 // Update using yarn bump-version
-export const __version__ = "0.1.58";
+export const __version__ = "0.1.59";
diff --git a/python/pyproject.toml b/python/pyproject.toml
index ff7ff3f4f..46582df93 100644
--- a/python/pyproject.toml
+++ b/python/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "langsmith"
-version = "0.1.121"
+version = "0.1.122"
 description = "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform."
 authors = ["LangChain <support@langchain.dev>"]
 license = "MIT"