register_models() plugin hook

.github/workflows/test.yml

@@ -21,11 +21,19 @@ jobs:
         cache-dependency-path: setup.py
     - name: Install dependencies
       run: |
-        pip install '.[test]'
+        pip install -e '.[test]'
     - name: Run tests
       run: |
         pytest
     - name: Check if cog needs to be run
       run: |
-        pip install -r docs/requirements.txt
         cog --check docs/*.md
+    - name: Run Black
+      run: |
+        black --check .
+    - name: Run mypy
+      run: |
+        mypy llm
+    - name: Run ruff
+      run: |
+        ruff .

Justfile

@@ -1,6 +1,10 @@
 # Run tests and linters
 @default: test lint
 
+# Install dependencies and test dependencies
+@init:
+  pipenv run pip install -e '.[test]'
+
 # Run pytest with supplied options
 @test *options:
   pipenv run pytest {{options}}
@@ -9,6 +13,8 @@
 @lint:
   pipenv run black . --check
   pipenv run cog --check README.md docs/*.md
+  pipenv run mypy llm
+  pipenv run ruff .
 
 # Rebuild docs with cog
 @cog:
@@ -21,3 +27,8 @@
 # Apply Black
 @black:
   pipenv run black .
+
+# Run automatic fixes
+@fix: cog
+  pipenv run ruff . --fix
+  pipenv run black .

MANIFEST.in

@@ -0,0 +1 @@
+global-exclude tests/*

docs/help.md

@@ -60,6 +60,8 @@ Commands:
   install    Install packages from PyPI into the same environment as LLM
   keys       Manage stored API keys for different models
   logs       Tools for exploring logged prompts and responses
+  models     Manage available models
+  openai     Commands for working directly with the OpenAI API
   plugins    List installed plugins
   templates  Manage stored prompt templates
   uninstall  Uninstall Python packages from the LLM environment
@@ -73,17 +75,18 @@ Usage: llm prompt [OPTIONS] [PROMPT]
   Documentation: https://llm.datasette.io/en/stable/usage.html
 
 Options:
-  --system TEXT               System prompt to use
-  -m, --model TEXT            Model to use
-  -t, --template TEXT         Template to use
-  -p, --param <TEXT TEXT>...  Parameters for template
-  --no-stream                 Do not stream output
-  -n, --no-log                Don't log to database
-  -c, --continue              Continue the most recent conversation.
-  --chat INTEGER              Continue the conversation with the given chat ID.
-  --key TEXT                  API key to use
-  --save TEXT                 Save prompt with this template name
-  --help                      Show this message and exit.
+  -s, --system TEXT            System prompt to use
+  -m, --model TEXT             Model to use
+  -o, --option <TEXT TEXT>...  key/value options for the model
+  -t, --template TEXT          Template to use
+  -p, --param <TEXT TEXT>...   Parameters for template
+  --no-stream                  Do not stream output
+  -n, --no-log                 Don't log to database
+  -c, --continue               Continue the most recent conversation.
+  --chat INTEGER               Continue the conversation with the given chat ID.
+  --key TEXT                   API key to use
+  --save TEXT                  Save prompt with this template name
+  --help                       Show this message and exit.
 ```
 ### llm init-db --help
 ```
@@ -167,6 +170,37 @@ Options:
   -t, --truncate       Truncate long strings in output
   --help               Show this message and exit.
 ```
+### llm models --help
+```
+Usage: llm models [OPTIONS] COMMAND [ARGS]...
+
+  Manage available models
+
+Options:
+  --help  Show this message and exit.
+
+Commands:
+  default  Show or set the default model
+  list     List available models
+```
+#### llm models list --help
+```
+Usage: llm models list [OPTIONS]
+
+  List available models
+
+Options:
+  --help  Show this message and exit.
+```
+#### llm models default --help
+```
+Usage: llm models default [OPTIONS] [MODEL]
+
+  Show or set the default model
+
+Options:
+  --help  Show this message and exit.
+```
 ### llm templates --help
 ```
 Usage: llm templates [OPTIONS] COMMAND [ARGS]...
@@ -229,13 +263,14 @@ Options:
 ```
 ### llm install --help
 ```
-Usage: llm install [OPTIONS] PACKAGES...
+Usage: llm install [OPTIONS] [PACKAGES]...
 
   Install packages from PyPI into the same environment as LLM
 
 Options:
-  -U, --upgrade  Upgrade packages to latest version
-  --help         Show this message and exit.
+  -U, --upgrade             Upgrade packages to latest version
+  -e, --editable DIRECTORY  Install a project in editable mode from this path
+  --help                    Show this message and exit.
 ```
 ### llm uninstall --help
 ```
@@ -247,4 +282,27 @@ Options:
   -y, --yes  Don't ask for confirmation
   --help     Show this message and exit.
 ```
+### llm openai --help
+```
+Usage: llm openai [OPTIONS] COMMAND [ARGS]...
+
+  Commands for working directly with the OpenAI API
+
+Options:
+  --help  Show this message and exit.
+
+Commands:
+  models  List models available to you from the OpenAI API
+```
+#### llm openai models --help
+```
+Usage: llm openai models [OPTIONS]
+
+  List models available to you from the OpenAI API
+
+Options:
+  --json      Output as JSON
+  --key TEXT  OpenAI API key
+  --help      Show this message and exit.
+```
 <!-- [[[end]]] -->

docs/index.md

@@ -29,9 +29,11 @@ maxdepth: 3
 ---
 setup
 usage
+python-api
 templates
 logging
 plugins
+tutorial-model-plugin
 help
 contributing
 changelog

docs/logging.md

@@ -18,8 +18,6 @@ On my Mac that outputs:
 ```
 This will differ for other operating systems.
 
-(You can customize the location of this file by setting a path in the `LLM_LOG_PATH` environment variable.)
-
 Once that SQLite database has been created any prompts you run will be logged to that database.
 
 To avoid logging a prompt, pass `--no-log` or `-n` to the command:
@@ -64,7 +62,7 @@ import sqlite_utils
 import re
 db = sqlite_utils.Database(memory=True)
 migrate(db)
-schema = db["log"].schema
+schema = db["logs"].schema
 
 def cleanup_sql(sql):
     first_line = sql.split('(')[0]
@@ -77,16 +75,19 @@ cog.out(
 )
 ]]] -->
 ```sql
-CREATE TABLE "log" (
+CREATE TABLE "logs" (
   [id] INTEGER PRIMARY KEY,
   [model] TEXT,
-  [timestamp] TEXT,
   [prompt] TEXT,
   [system] TEXT,
+  [prompt_json] TEXT,
+  [options_json] TEXT,
   [response] TEXT,
-  [chat_id] INTEGER REFERENCES [log]([id]),
-  [debug] TEXT,
-  [duration_ms] INTEGER
+  [response_json] TEXT,
+  [reply_to_id] INTEGER REFERENCES [logs]([id]),
+  [chat_id] INTEGER REFERENCES [logs]([id]),
+  [duration_ms] INTEGER,
+  [datetime_utc] TEXT
 );
 ```
 <!-- [[[end]]] -->

docs/python-api.md

@@ -0,0 +1,37 @@
+# Python API
+
+LLM provides a Python API for executing prompts, in addition to the command-line interface.
+
+Understanding this API is also important for writing plugins.
+
+The API consists of the following key classes:
+
+- `Model` - represents a language model against which prompts can be executed
+- `Prompt` - a prompt that can be prepared and then executed against a model
+- `Response` - the response executing a prompt against a model
+- `Template` - a reusable template for generating prompts
+
+## Prompt
+
+A prompt object represents all of the information needed to be passed to the LLM. This could be a single prompt string, but it might also include a separate system prompt, various settings (for temperature etc) or even a JSON array of previous messages.
+
+## Model
+
+The `Model` class is an abstract base class that needs to be subclassed to provide a concrete implementation. Different LLMs will use different implementations of this class.
+
+Model instances provide the following methods:
+
+- `prompt(prompt: str, stream: bool, ...options) -> Response` - a convenience wrapper which creates a `Prompt` instance and then executes it. This is the most common way to use LLM models.
+- `response(prompt: Prompt, stream: bool) -> Response` - execute a prepared Prompt instance against the model and return a `Response`.
+
+Models usually return subclasses of `Response` that are specific to that model.
+
+## Response
+
+The response from an LLM. This could encapusulate a string of text, but for streaming APIs this class will be iterable, with each iteration yielding a short string of text as it is generated.
+
+Calling `.text()` will return the full text of the response, waiting for the stream to stop executing if necessary.
+
+## Template
+
+Templates are reusable objects that can be used to generate prompts. They are used  by the {ref}`prompt-templates` feature.

docs/setup.md

@@ -89,3 +89,17 @@ The environment variable will be used only if no `--key` option is passed to the
 If no environment variable is found, the tool will fall back to checking `keys.json`.
 
 You can force the tool to use the key from `keys.json` even if an environment variable has also been set using `llm "prompt" --key openai`.
+
+## Custom directory location
+
+This tool stores various files - prompt templates, stored keys, preferences, a database of logs - in a directory on your computer.
+
+On macOS this is `~/Library/Application Support/io.datasette.llm/`.
+
+On Linux it may be something like `~/.config/io.datasette.llm/`.
+
+You can set a custom location for this directory by setting the `LLM_USER_PATH` environment variable:
+
+```bash
+export LLM_USER_PATH=/path/to/my/custom/directory
+```