From e5eb3238ad47f5d2956d4f43ac5aca4427591207 Mon Sep 17 00:00:00 2001 From: Jianghong Date: Sun, 12 May 2024 15:58:03 -0400 Subject: [PATCH 1/3] Revert "Make sse buffer size configurable" This reverts commit eb7937e65838bb8116e06780c0ad06e2c4f10638. --- src/wkok/openai_clojure/sse.clj | 48 +++++++-------------------------- 1 file changed, 10 insertions(+), 38 deletions(-) diff --git a/src/wkok/openai_clojure/sse.clj b/src/wkok/openai_clojure/sse.clj index 440400c..a654781 100644 --- a/src/wkok/openai_clojure/sse.clj +++ b/src/wkok/openai_clojure/sse.clj @@ -5,11 +5,8 @@ [hato.middleware :as hm] [clojure.core.async :as a] [clojure.string :as string] - [cheshire.core :as json] - [clojure.core.async.impl.protocols :as impl]) - (:import (java.io InputStream) - (clojure.lang Counted) - (java.util LinkedList))) + [cheshire.core :as json]) + (:import (java.io InputStream))) (def event-mask (re-pattern (str "(?s).+?\n\n"))) @@ -31,39 +28,13 @@ (-> (subs raw-event data-idx) (json/parse-string true))))) -(deftype InfiniteBuffer [^LinkedList buf] - impl/UnblockingBuffer - impl/Buffer - (full? [_this] - false) - (remove! [_this] - (.removeLast buf)) - (add!* [this itm] - (.addFirst buf itm) - this) - (close-buf! [_this]) - Counted - (count [_this] - (.size buf))) - -(defn infinite-buffer [] - (InfiniteBuffer. (LinkedList.))) - (defn calc-buffer-size - "- Use stream_buffer_len if provided. - - Otherwise, buffer size should be at least equal to max_tokens - plus the [DONE] terminator if it is provided. - - Else fallbacks on ##Inf and use an infinite-buffer instead" - [{:keys [stream_buffer_len max_tokens]}] - (or stream_buffer_len - (when max_tokens (inc max_tokens)) - ##Inf)) - -(defn make-buffer [params] - (let [size (calc-buffer-size params)] - (if (= size ##Inf) - (infinite-buffer) - (a/sliding-buffer size)))) + "Buffer size should be at least equal to max_tokens + or 16 (the default in openai as of 2023-02-19) + plus the [DONE] terminator" + [{:keys [max_tokens] + :or {max_tokens 16}}] + (inc max_tokens)) (defn sse-events "Returns a core.async channel with events as clojure data structures. @@ -72,7 +43,8 @@ (let [event-stream ^InputStream (:body (http/request (merge request params {:as :stream}))) - events (a/chan (make-buffer params) (map parse-event))] + buffer-size (calc-buffer-size params) + events (a/chan (a/sliding-buffer buffer-size) (map parse-event))] (a/thread (loop [byte-coll []] (let [byte-arr (byte-array (max 1 (.available event-stream))) From 26e28ef3f9f20871edb275df3413c7b6d090083c Mon Sep 17 00:00:00 2001 From: Jianghong Date: Sun, 12 May 2024 17:59:12 -0400 Subject: [PATCH 2/3] Use buffer instead of sliding buffer with a bigger size --- src/wkok/openai_clojure/sse.clj | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/wkok/openai_clojure/sse.clj b/src/wkok/openai_clojure/sse.clj index a654781..b3a443d 100644 --- a/src/wkok/openai_clojure/sse.clj +++ b/src/wkok/openai_clojure/sse.clj @@ -28,12 +28,16 @@ (-> (subs raw-event data-idx) (json/parse-string true))))) +; Per this discussion: https://community.openai.com/t/clarification-for-max-tokens/19576 +; if the max_tokens is not provided, the response will try to use all the available +; tokens to generate response, hence DEFAULT_BUFFER_SIZE should be large enough +(def ^:private DEFAULT_BUFFER_SIZE 100000) + (defn calc-buffer-size "Buffer size should be at least equal to max_tokens - or 16 (the default in openai as of 2023-02-19) plus the [DONE] terminator" [{:keys [max_tokens] - :or {max_tokens 16}}] + :or {max_tokens DEFAULT_BUFFER_SIZE}}] (inc max_tokens)) (defn sse-events @@ -44,7 +48,7 @@ params {:as :stream}))) buffer-size (calc-buffer-size params) - events (a/chan (a/sliding-buffer buffer-size) (map parse-event))] + events (a/chan (a/buffer buffer-size) (map parse-event))] (a/thread (loop [byte-coll []] (let [byte-arr (byte-array (max 1 (.available event-stream))) @@ -153,3 +157,4 @@ (assoc ctx :response (if (:stream params) (sse-request ctx') (http/request request'))))))}) + From a87ad65d77546c1ac37a13009a71d1139880b84d Mon Sep 17 00:00:00 2001 From: Jianghong Date: Sat, 18 May 2024 17:38:40 -0400 Subject: [PATCH 3/3] Update assistants HTTP header --- src/wkok/openai_clojure/api.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/wkok/openai_clojure/api.clj b/src/wkok/openai_clojure/api.clj index 303f240..d7a2786 100644 --- a/src/wkok/openai_clojure/api.clj +++ b/src/wkok/openai_clojure/api.clj @@ -448,7 +448,7 @@ ([params] (create-assistant params nil)) ([params options] - (let [opts (assoc-in options [:openai-beta] "assistants=v1")] + (let [opts (assoc-in options [:openai-beta] "assistants=v2")] (core/response-for :create-assistant params opts))))