feat: add support fastchat http bindings

* feat: add support fastchat http bindings

Signed-off-by: Lei Wen <wenlei03@qiyi.com>

crates/http-api-bindings/src/fastchat.rs

@@ -0,0 +1,93 @@
+use async_trait::async_trait;
+use reqwest::header;
+use serde::{Deserialize, Serialize};
+use serde_json::Value;
+use tabby_inference::{TextGeneration, TextGenerationOptions};
+
+#[derive(Serialize)]
+struct Request {
+    model: String,
+    prompt: Vec<String>,
+    max_tokens: usize,
+    temperature: f32,
+}
+
+#[derive(Deserialize)]
+struct Response {
+    choices: Vec<Prediction>,
+}
+
+#[derive(Deserialize)]
+struct Prediction {
+    text: Vec<String>,
+}
+
+pub struct FastChatEngine {
+    client: reqwest::Client,
+    api_endpoint: String,
+    model_name: String,
+}
+
+impl FastChatEngine {
+    pub fn create(api_endpoint: &str, model_name: &str, authorization: &str) -> Self {
+        let mut headers = reqwest::header::HeaderMap::new();
+        if authorization.len() > 0 {
+            headers.insert(
+                "Authorization",
+                header::HeaderValue::from_str(authorization)
+                .expect("Failed to create authorization header"),
+                );
+        }
+        let client = reqwest::Client::builder()
+            .default_headers(headers)
+            .build()
+            .expect("Failed to construct HTTP client");
+        Self {
+            api_endpoint: api_endpoint.to_owned(),
+            model_name: model_name.to_owned(),
+            client,
+        }
+    }
+
+    pub fn prompt_template() -> String {
+        "{prefix}<MID>{suffix}".to_owned()
+    }
+}
+
+#[async_trait]
+impl TextGeneration for FastChatEngine {
+    async fn generate(&self, prompt: &str, options: TextGenerationOptions) -> String {
+        let _stop_sequences: Vec<String> = options
+            .stop_words
+            .iter()
+            .map(|x| x.to_string())
+            .collect();
+
+        let tokens: Vec<&str> = prompt.split("<MID>").collect();
+        let request = Request {
+            model: self.model_name.to_owned(),
+            prompt: vec![tokens[0].to_owned()],
+            max_tokens: options.max_decoding_length,
+            temperature: options.sampling_temperature,
+        };
+
+        // API Documentation: https://github.com/lm-sys/FastChat/blob/main/docs/openai_api.md
+        let resp = self
+            .client
+            .post(&self.api_endpoint)
+            .json(&request)
+            .send()
+            .await
+            .expect("Failed to making completion request");
+
+        if resp.status() != 200 {
+            let err: Value = resp.json().await.expect("Failed to parse response");
+            println!("Request failed: {}", err);
+            std::process::exit(1);
+        }
+
+        let resp: Response = resp.json().await.expect("Failed to parse response");
+
+        resp.choices[0].text[0].clone()
+    }
+}

crates/http-api-bindings/src/lib.rs

@@ -1 +1,2 @@
 pub mod vertex_ai;
+pub mod fastchat;

crates/tabby/src/serve/completions.rs

@@ -6,6 +6,7 @@ use std::{path::Path, sync::Arc};
 use axum::{extract::State, Json};
 use ctranslate2_bindings::{CTranslate2Engine, CTranslate2EngineOptionsBuilder};
 use http_api_bindings::vertex_ai::VertexAIEngine;
+use http_api_bindings::fastchat::FastChatEngine;
 use hyper::StatusCode;
 use serde::{Deserialize, Serialize};
 use serde_json::Value;
@@ -158,10 +159,6 @@ fn create_engine(args: &crate::serve::ServeArgs) -> (Box<dyn TextGeneration>, Op
             .as_str()
             .expect("Type unmatched");
 
-        if kind != "vertex-ai" {
-            fatal!("Only vertex_ai is supported for http backend");
-        }
-
         let api_endpoint = params
             .get("api_endpoint")
             .expect("Missing api_endpoint field")
@@ -172,8 +169,21 @@ fn create_engine(args: &crate::serve::ServeArgs) -> (Box<dyn TextGeneration>, Op
             .expect("Missing authorization field")
             .as_str()
             .expect("Type unmatched");
-        let engine = Box::new(VertexAIEngine::create(api_endpoint, authorization));
-        (engine, Some(VertexAIEngine::prompt_template()))
+
+        if kind == "vertex-ai" {
+            let engine = Box::new(VertexAIEngine::create(api_endpoint, authorization));
+            (engine, Some(VertexAIEngine::prompt_template()))
+        } else if kind == "fastchat" {
+            let model_name = params
+                .get("model_name")
+                .expect("Missing model_name field")
+                .as_str()
+                .expect("Type unmatched");
+            let engine = Box::new(FastChatEngine::create(api_endpoint, model_name, authorization));
+            (engine, Some(FastChatEngine::prompt_template()))
+        } else {
+            fatal!("Only vertex_ai and fastchat are supported for http backend");
+        }
     }
 }
 

crates/tabby/src/serve/mod.rs

@@ -120,7 +120,7 @@ pub struct ServeArgs {
 }
 
 #[cfg(not(all(target_os = "macos", target_arch = "aarch64")))]
-fn should_download_ggml_files(device: &Device) -> bool {
+fn should_download_ggml_files(_device: &Device) -> bool {
     false
 }