allow filtering datasets by metadata

js/package.json

@@ -1,6 +1,6 @@
 {
   "name": "langsmith",
-  "version": "0.1.44",
+  "version": "0.1.45",
   "description": "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform.",
   "packageManager": "yarn@1.22.19",
   "files": [

js/src/client.ts

@@ -1265,7 +1265,7 @@
     treeFilter?: string;
     isRoot?: boolean;
     dataSourceType?: string;
   }): Promise<any> {
     let projectIds_ = projectIds || [];
     if (projectNames) {
       projectIds_ = [
@@ -1553,7 +1553,7 @@
         `Failed to list shared examples: ${response.status} ${response.statusText}`
       );
     }
     return result.map((example: any) => ({
       ...example,
       _hostUrl: this.getHostUrl(),
     }));
@@ -1910,16 +1910,19 @@
       dataType,
       inputsSchema,
       outputsSchema,
+      metadata,
     }: {
       description?: string;
       dataType?: DataType;
       inputsSchema?: KVMap;
       outputsSchema?: KVMap;
+      metadata?: RecordStringAny;
     } = {}
   ): Promise<Dataset> {
     const body: KVMap = {
       name,
       description,
+      extra: metadata ? { metadata } : undefined,
     };
     if (dataType) {
       body.data_type = dataType;
@@ -2065,12 +2068,14 @@
     datasetIds,
     datasetName,
     datasetNameContains,
+    metadata,
   }: {
     limit?: number;
     offset?: number;
     datasetIds?: string[];
     datasetName?: string;
     datasetNameContains?: string;
+    metadata?: RecordStringAny;
   } = {}): AsyncIterable<Dataset> {
     const path = "/datasets";
     const params = new URLSearchParams({
@@ -2088,6 +2093,9 @@
     if (datasetNameContains !== undefined) {
       params.append("name_contains", datasetNameContains);
     }
+    if (metadata !== undefined) {
+      params.append("metadata", JSON.stringify(metadata));
+    }
     for await (const datasets of this._getPaginated<Dataset>(path, params)) {
       yield* datasets;
     }
@@ -2646,7 +2654,7 @@
     }
 
     const feedbackResult = await evaluator.evaluateRun(run_, referenceExample);
     const [_, feedbacks] = await this._logEvaluationFeedback(
       feedbackResult,
       run_,
       sourceInfo
@@ -2976,7 +2984,7 @@
   async _logEvaluationFeedback(
     evaluatorResponse: EvaluationResult | EvaluationResults,
     run?: Run,
     sourceInfo?: { [key: string]: any }
   ): Promise<[results: EvaluationResult[], feedbacks: Feedback[]]> {
     const evalResults: Array<EvaluationResult> =
       this._selectEvalResults(evaluatorResponse);
@@ -3015,7 +3023,7 @@
   public async logEvaluationFeedback(
     evaluatorResponse: EvaluationResult | EvaluationResults,
     run?: Run,
     sourceInfo?: { [key: string]: any }
   ): Promise<EvaluationResult[]> {
     const [results] = await this._logEvaluationFeedback(
       evaluatorResponse,
@@ -3081,7 +3089,7 @@
     promptIdentifier: string,
     like: boolean
   ): Promise<LikePromptResponse> {
     const [owner, promptName, _] = parsePromptIdentifier(promptIdentifier);
     const response = await this.caller.call(
       fetch,
       `${this.apiUrl}/likes/${owner}/${promptName}`,
@@ -3186,7 +3194,7 @@
   }
 
   public async getPrompt(promptIdentifier: string): Promise<Prompt | null> {
     const [owner, promptName, _] = parsePromptIdentifier(promptIdentifier);
     const response = await this.caller.call(
       fetch,
       `${this.apiUrl}/repos/${owner}/${promptName}`,
@@ -3230,7 +3238,7 @@
       );
     }
 
     const [owner, promptName, _] = parsePromptIdentifier(promptIdentifier);
     if (!(await this._currentTenantIsOwner(owner))) {
       throw await this._ownerConflictError("create a prompt", owner);
     }
@@ -3259,7 +3267,7 @@
 
   public async createCommit(
     promptIdentifier: string,
     object: any,
     options?: {
       parentCommitHash?: string;
     }
@@ -3268,7 +3276,7 @@
       throw new Error("Prompt does not exist, you must create it first.");
     }
 
     const [owner, promptName, _] = parsePromptIdentifier(promptIdentifier);
     const resolvedParentCommitHash =
       options?.parentCommitHash === "latest" || !options?.parentCommitHash
         ? await this._getLatestCommitHash(`${owner}/${promptName}`)

js/src/index.ts

@@ -12,4 +12,4 @@ export type {
 export { RunTree, type RunTreeConfig } from "./run_trees.js";
 
 // Update using yarn bump-version
-export const __version__ = "0.1.44";
+export const __version__ = "0.1.45";

js/src/tests/client.int.test.ts

@@ -183,6 +183,7 @@ test.concurrent(
     });
     const dataset = await langchainClient.createDataset(datasetName, {
       dataType: "llm",
+      metadata: { key: "valuefoo" },
     });
     await langchainClient.createExample(
       { input: "hello world" },
@@ -193,6 +194,12 @@ test.concurrent(
     );
     const loadedDataset = await langchainClient.readDataset({ datasetName });
     expect(loadedDataset.data_type).toEqual("llm");
+
+    const datasetsByMetadata = await toArray(
+      langchainClient.listDatasets({ metadata: { key: "valuefoo" } })
+    );
+    expect(datasetsByMetadata.length).toEqual(1);
+    expect(datasetsByMetadata.map((d) => d.id)).toContain(dataset.id);
     await langchainClient.deleteDataset({ datasetName });
   },
   180_000

python/langsmith/client.py

@@ -2504,6 +2504,7 @@ def create_dataset(
         data_type: ls_schemas.DataType = ls_schemas.DataType.kv,
         inputs_schema: Optional[Dict[str, Any]] = None,
         outputs_schema: Optional[Dict[str, Any]] = None,
+        metadata: Optional[dict] = None,
     ) -> ls_schemas.Dataset:
         """Create a dataset in the LangSmith API.
 
@@ -2515,6 +2516,8 @@ def create_dataset(
             The description of the dataset.
         data_type : DataType or None, default=DataType.kv
             The data type of the dataset.
+        metadata: dict or None, default=None
+            Additional metadata to associate with the dataset.
 
         Returns:
         -------
@@ -2525,6 +2528,7 @@ def create_dataset(
             "name": dataset_name,
             "data_type": data_type.value,
             "created_at": datetime.datetime.now().isoformat(),
+            "extra": {"metadata": metadata} if metadata else None,
         }
         if description is not None:
             dataset["description"] = description
@@ -2737,6 +2741,7 @@ def list_datasets(
         data_type: Optional[str] = None,
         dataset_name: Optional[str] = None,
         dataset_name_contains: Optional[str] = None,
+        metadata: Optional[Dict[str, Any]] = None,
         limit: Optional[int] = None,
     ) -> Iterator[ls_schemas.Dataset]:
         """List the datasets on the LangSmith API.
@@ -2757,6 +2762,8 @@ def list_datasets(
             params["name"] = dataset_name
         if dataset_name_contains is not None:
             params["name_contains"] = dataset_name_contains
+        if metadata is not None:
+            params["metadata"] = json.dumps(metadata)
         for i, dataset in enumerate(
             self._get_paginated_list("/datasets", params=params)
         ):

python/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "langsmith"
-version = "0.1.102"
+version = "0.1.103"
 description = "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform."
 authors = ["LangChain <support@langchain.dev>"]
 license = "MIT"

python/tests/integration_tests/test_client.py

@@ -461,7 +461,9 @@ def test_list_datasets(langchain_client: Client) -> None:
     ds1n = "__test_list_datasets1" + uuid4().hex[:4]
     ds2n = "__test_list_datasets2" + uuid4().hex[:4]
     try:
-        dataset1 = langchain_client.create_dataset(ds1n, data_type=DataType.llm)
+        dataset1 = langchain_client.create_dataset(
+            ds1n, data_type=DataType.llm, metadata={"foo": "barqux"}
+        )
         dataset2 = langchain_client.create_dataset(ds2n, data_type=DataType.kv)
         assert dataset1.url is not None
         assert dataset2.url is not None
@@ -484,6 +486,13 @@ def test_list_datasets(langchain_client: Client) -> None:
             )
         )
         assert len(datasets) == 1
+        # Sub-filter on metadata
+        datasets = list(
+            langchain_client.list_datasets(
+                dataset_ids=[dataset1.id, dataset2.id], metadata={"foo": "barqux"}
+            )
+        )
+        assert len(datasets) == 1
     finally:
         # Delete datasets
         for name in [ds1n, ds2n]: