Add test on language model settings page (#584)

app/controllers/settings/api_services_controller.rb

@@ -30,6 +30,16 @@ def update
     end
   end
 
+  def test
+    @api_service = Current.user.api_services.find_by(id: params[:api_service_id])
+    @answer = @api_service.test_api_service(params[:url], params[:token])
+
+    respond_to do |format|
+      format.html { redirect_to settings_api_services_path, notice: "Tested: #{@answer}", status: :see_other }
+      format.turbo_stream
+    end
+  end
+
   def destroy
     @api_service.deleted!
     redirect_to settings_api_services_path, notice: "Deleted", status: :see_other

app/controllers/settings/language_models_controller.rb

@@ -34,6 +34,16 @@ def update
     end
   end
 
+  def test
+    @language_model = Current.user.language_models.find_by(id: params[:language_model_id])
+    @answer = @language_model.test(params[:model])
+
+    respond_to do |format|
+      format.html { redirect_to settings_language_models_path, notice: "Tested: #{@answer}", status: :see_other }
+      format.turbo_stream
+    end
+  end
+
   def destroy
     @language_model.deleted!
     redirect_to settings_language_models_path, notice: "Deleted", status: :see_other

app/javascript/stimulus/testing_controller.js

@@ -0,0 +1,25 @@
+import { Controller } from "@hotwired/stimulus"
+
+export default class extends Controller {
+  static targets = ["model", "test", "url","token"]
+
+  update_link_language_model(event) {
+    const link = event.currentTarget;
+    const href = link.href.split('?')[0];
+    link.href = href + "?model=" + this.modelTarget.value
+  }
+
+  update_link_api_service(event) {
+    const link = event.currentTarget;
+    const href = link.href.split('?')[0];
+    link.href = href + "?url=" + this.urlTarget.value + "&token=" + this.tokenTarget.value
+    console.log("Link: ", link)
+  }
+
+  disable_test_link() {
+    const link = this.testTarget;
+    link.addEventListener('click', function(event) {
+      event.preventDefault();
+    });
+  }
+}

app/models/api_service.rb

@@ -39,6 +39,10 @@ def effective_token
     token.presence || default_llm_key
   end
 
+  def test_api_service(url = nil, token = nil)
+    ai_backend.test_api_service(self, url, token)
+  end
+
   private
 
   def default_llm_key

app/models/language_model.rb

@@ -31,6 +31,10 @@ def supports_tools?
       api_service.name != "Groq" # TODO: Remove this short circuit once I can debug tool use with Groq
   end
 
+  def test(api_name = nil)
+    ai_backend.test_language_model(self, api_name)
+  end
+
   private
 
   def populate_position

app/services/ai_backend.rb

@@ -48,6 +48,27 @@ def stream_next_conversation_message(&chunk_handler)
     end
   end
 
+  def self.test_language_model(language_model, api_name = nil)
+    api_name ||= language_model.api_name
+    url = language_model.api_service.url
+    token = language_model.api_service.effective_token
+    return "Error: API key (token) is blank" if language_model.api_service.requires_token? && token.blank?
+
+    test_execute(url, token, api_name)
+  end
+
+  def self.test_api_service(api_service, url = nil, token = nil)
+    url ||= api_service.url
+    token ||= api_service.effective_token
+    language_model = LanguageModel.where(best: true, api_service: api_service).first
+    api_name = language_model.api_name unless language_model.nil?
+
+    return "Error: API key (token) is blank" if api_service.requires_token? && token.blank?
+    return "Error: API name is blank. Define a best Language Model for this API service." if api_name.blank?
+
+    test_execute(url, token, api_name)
+  end
+
   private
 
   def client_method_name

app/services/ai_backend/anthropic.rb

@@ -14,6 +14,26 @@ def self.client
     end
   end
 
+  def self.test_execute(url, token, api_name)
+    Rails.logger.info "Connecting to Anthropic API server at #{url} with access token of length #{token.to_s.length}"
+    client = ::Anthropic::Client.new(
+      uri_base: url,
+      access_token: token
+    )
+
+    Rails.logger.info "Testing using model #{api_name}"
+    client.messages(
+      model: api_name,
+      messages: [
+        { "role": "user", "content": "Hello!" }
+      ],
+      system: "You are a helpful assistant.",
+      parameters: { max_tokens: 1000 }
+    ).dig("content", 0, "text")
+  rescue => e
+    "Error: #{e.message}"
+  end
+
   def initialize(user, assistant, conversation = nil, message = nil)
     super(user, assistant, conversation, message)
     begin

app/services/ai_backend/gemini.rb

@@ -15,6 +15,27 @@ def self.client
     end
   end
 
+  def self.test_execute(url, token, api_name)
+    Rails.logger.info "Connecting to Gemini API server at #{url} with access token of length #{token.to_s.length}"
+    client = ::Gemini.new(
+      credentials: {
+        service: "generative-language-api",
+        api_key: token,
+        version: "v1beta"
+      },
+      options: {
+        model: api_name,
+        server_sent_events: true
+      }
+    )
+
+    client.generate_content({
+      contents: { role: "user", parts: { text: "Hello!" }}
+    }).dig("candidates", 0, "content", "parts", 0, "text")
+  rescue ::Faraday::Error => e
+    "Error: #{e.message}"
+  end
+
   def initialize(user, assistant, conversation = nil, message = nil)
     super(user, assistant, conversation, message)
     begin

app/services/ai_backend/open_ai.rb

@@ -14,6 +14,29 @@ def self.client
     end
   end
 
+  def self.test_execute(url, token, api_name)
+    if Rails.env.test?
+      client = ::TestClient::OpenAI.new(
+        access_token: token,
+        uri_base: url
+      )
+      response = client.send(:chat, ** {parameters: {model: api_name, messages: [{ role: "user", content: "Hello!" }]}})
+    else
+      Rails.logger.info "Connecting to OpenAI API server at #{url} with access token of length #{token.to_s.length}"
+      client = ::OpenAI::Client.new(
+        access_token: token,
+        uri_base: url
+      )
+
+      Rails.logger.info "Testing using model #{api_name}"
+      response = client.chat(parameters: {model: api_name, messages: [{ role: "user", content: "Hello!" }]})
+    end
+
+    response.dig("choices", 0, "message", "content")
+  rescue ::Faraday::Error => e
+    "Error: #{e.message}"
+  end
+
   def initialize(user, assistant, conversation = nil, message = nil)
     super(user, assistant, conversation, message)
     begin

app/views/settings/api_services/_form.html.erb

@@ -1,4 +1,4 @@
-<%= form_with(model: [:settings, api_service], class: "contents") do |form| %>
+<%= form_with(model: [:settings, api_service], class: "contents", data: {controller: "testing"}) do |form| %>
   <% if api_service.errors.any? %>
     <div id="error_explanation" class="bg-red-50 text-red-500 px-3 py-2 font-medium rounded-lg mt-3">
       <h2><%= pluralize(api_service.errors.count, "error") %> prohibited this API service from being saved:</h2>
@@ -21,6 +21,7 @@
     <%= form.select :driver,
       APIService.drivers.keys,
       {},
+      data: { action: "change->testing#disable_test_link" },
       class: %|
         block
         border border-gray-200 outline-none
@@ -32,9 +33,22 @@
     %>
   </div>
 
-  <div class="my-5">
+  <div class="my-5 inline-block w-full">
     <%= form.label :url %>
-    <%= form.text_field :url, class: "block shadow rounded-md border border-gray-200 outline-none px-3 py-2 mt-2 w-full dark:text-black" %>
+     <span>
+      <% if api_service.persisted? %>
+        <% link_url = settings_api_services_path + "/" + api_service.id.to_s + "/test" %>
+        <%= link_to "Test", link_url,
+          data: { testing_target: "test", turbo_stream: true, action: "click->testing#update_link_api_service" },
+          class: "underline text-blue-600"
+        %>
+      <% end %>
+      <span id="test_result"></span>
+    </span>
+    <%= form.text_field :url,
+      data: { testing_target: "url" },
+      class: "block shadow rounded-md border border-gray-200 outline-none px-3 py-2 mt-2 w-full dark:text-black"
+    %>
   </div>
 
   <div class="my-5" data-controller="transition" data-transition-toggle-class="hidden">
@@ -167,7 +181,16 @@
         %>.
       </li>
     </ol>
-    <%= form.text_field :token, class: "block shadow rounded-md border border-gray-200 outline-none px-3 py-2 mt-2 w-full dark:text-black", autocomplete: "off" %>
+    <%= form.text_field :token,
+      class: %|
+        block shadow
+        rounded-md border border-gray-200 outline-none
+        px-3 py-2 mt-2 w-full dark:text-black
+      |,
+      autocomplete: "off",
+      id: "input_token",
+      data: { testing_target: "token" }
+    %>
   </div>
 
   <%= form.submit "Save",

app/views/settings/api_services/test.turbo_stream.erb

@@ -0,0 +1,11 @@
+<turbo-stream action="replace" target="test_result">
+  <template>
+    <div id="test_result" class="inline-block">
+      <% if @answer.start_with?("Error:") %>
+        <span class="text-red-700"><%= @answer %></span>
+      <% else %>
+        <span class="text-green-700"><%= @answer %></span>
+      <% end %>
+    </div>
+  </template>
+</turbo-stream>

app/views/settings/language_models/_form.html.erb

@@ -1,4 +1,4 @@
-<%= form_with(model: [:settings, language_model], class: "contents") do |form| %>
+<%= form_with(model: [:settings, language_model], class: "contents", data: { controller: "testing" }) do |form| %>
   <% if language_model.errors.any? %>
     <div id="error_explanation" class="bg-red-50 text-red-500 px-3 py-2 font-medium rounded-lg mt-3">
       <h2><%= pluralize(language_model.errors.count, "error") %> prohibited this language model from being saved:</h2>
@@ -16,9 +16,22 @@
     <%= form.text_field :name, class: "block shadow rounded-md border border-gray-200 outline-none px-3 py-2 mt-2 w-full dark:text-black" %>
   </div>
 
-  <div class="my-5">
+  <div class="my-5 inline-block w-full">
     <%= form.label :api_name %>
-    <%= form.text_field :api_name, class: "block shadow rounded-md border border-gray-200 outline-none px-3 py-2 mt-2 w-full dark:text-black" %>
+    <span>
+      <% if language_model.persisted? %>
+        <% link_url = settings_language_models_path + "/" + language_model.id.to_s + "/test" %>
+        <%= link_to "Test", link_url,
+          data: { testing_target: "test", turbo_stream: true, action: "click->testing#update_link_language_model" },
+          class: "underline text-blue-600"
+        %>
+        <span id="test_result"></span>
+      <% end %>
+    </span>
+    <%= form.text_field :api_name,
+      data: { testing_target: "model" },
+      class: "block shadow rounded-md border border-gray-200 outline-none px-3 py-2 mt-2 w-full dark:text-black"
+      %>
     <span>As specified in the API docs</span>
   </div>
 
@@ -28,6 +41,7 @@
       <%= form.select :api_service_id,
         Current.user.api_services.ordered.pluck(:name, :id).reverse,
         {},
+        data: { action: "change->testing#disable_test_link" },
         class: %|
           block
           border border-gray-200 outline-none

app/views/settings/language_models/test.turbo_stream.erb

@@ -0,0 +1,11 @@
+<turbo-stream action="replace" target="test_result">
+  <template>
+    <div id="test_result" class="inline-block">
+      <% if @answer.start_with?("Error:") %>
+        <span class="text-red-700"><%= @answer %></span>
+      <% else %>
+        <span class="text-green-700"><%= @answer %></span>
+      <% end %>
+    </div>
+  </template>
+</turbo-stream>

config/routes.rb

@@ -16,8 +16,12 @@
   namespace :settings do
     resources :assistants, except: [:index, :show]
     resource :person, only: [:edit, :update]
-    resources :language_models
-    resources :api_services, except: [:show]
+    resources :language_models do
+      get :test, to: "language_models#test"
+    end
+    resources :api_services, except: [:show]  do
+      get :test, to: "api_services#test"
+    end
     resources :memories, only: [:index, :destroy] do
       delete :destroy, to: "memories#destroy_all", on: :collection
     end

test/controllers/settings/api_services_controller_test.rb

@@ -108,4 +108,12 @@ class Settings::APIServicesControllerTest < ActionDispatch::IntegrationTest
     get edit_settings_api_service_url(api_services(:keith_other_service))
     assert_select "button#instructions.hidden"
   end
+
+  test "test should return success and a message" do
+    TestClient::OpenAI.stub :text, "Success." do
+      get settings_api_service_test_url(format: :turbo_stream, api_service_id: api_services(:keith_openai_service).id, url: "https://api.openai.com/v1", token: "abc-secret123")
+      assert_response :success
+      assert_contains_text "div#test_result", "Success."
+    end
+  end
 end

test/controllers/settings/language_models_controller_test.rb

@@ -199,4 +199,12 @@ class Settings::LanguageModelsControllerTest < ActionDispatch::IntegrationTest
       assert_select 'input[name="language_model[supports_system_message]"][checked="checked"]', false
     end
   end
+
+  test "test should return success and a message" do
+    TestClient::OpenAI.stub :text, "Success." do
+      get settings_language_model_test_url(format: :turbo_stream, language_model_id: language_models(:guanaco).id, model: "gpt-4o")
+      assert_response :success
+      assert_contains_text "div#test_result", "Success."
+    end
+  end
 end