Skip to content
Tests

fix: Avoid thread starvation on many concurrent requests by making use of asyncio to lock llama_proxy context #2856

Sign in to view logs

fix: Avoid thread starvation on many concurrent requests by making use of asyncio to lock llama_proxy context

fix: Avoid thread starvation on many concurrent requests by making use of asyncio to lock llama_proxy context #2856

Workflow file for this run

.github/workflows/test.yaml at ad35fc1
name: Tests
on:
pull_request:
branches:
- main
push:
branches:
- main
env:
REPO_ID: Qwen/Qwen2-0.5B-Instruct-GGUF
MODEL_FILE: qwen2-0_5b-instruct-q8_0.gguf
jobs:
download-model:
runs-on: ubuntu-latest
steps:
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: "3.9"
- name: Install huggingface-hub
run: pip install huggingface-hub
- name: Download model
run: huggingface-cli download ${{ env.REPO_ID }} ${{ env.MODEL_FILE }}
- name: Cache model
uses: actions/cache@v4
with:
path: ~/.cache/huggingface/hub
key: ${{ runner.os }}-model-${{ env.REPO_ID }}-${{ env.MODEL_FILE }}
build-linux:
needs: download-model
runs-on: ubuntu-latest
strategy:
matrix:
python-version: ["3.9", "3.10", "3.11", "3.12"]
steps:
- uses: actions/checkout@v4
with:
submodules: "recursive"
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
cache: 'pip'
- name: Restore model cache
uses: actions/cache@v4
with:
path: ~/.cache/huggingface/hub
key: ${{ runner.os }}-model-${{ env.REPO_ID }}-${{ env.MODEL_FILE }}
- name: Install dependencies (Linux/MacOS)
run: |
python -m pip install --upgrade pip
python -m pip install uv
python -m uv pip install -e .[all] --verbose
shell: bash
- name: Test with pytest
run: |
python -m pytest
build-windows:
needs: download-model
runs-on: windows-latest
strategy:
matrix:
python-version: ["3.9", "3.10", "3.11", "3.12"]
steps:
- uses: actions/checkout@v4
with:
submodules: "recursive"
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
cache: 'pip'
- name: Restore model cache
uses: actions/cache@v4
with:
path: ~/.cache/huggingface/hub
key: ${{ runner.os }}-model-${{ env.REPO_ID }}-${{ env.MODEL_FILE }}
- name: Install dependencies (Windows)
run: |
python -m pip install --upgrade pip
python -m pip install uv
python -m uv pip install -e .[all] --verbose
shell: cmd
- name: Test with pytest
run: |
python -m pytest
build-macos:
needs: download-model
runs-on: macos-13
strategy:
matrix:
python-version: ["3.9", "3.10", "3.11", "3.12"]
steps:
- uses: actions/checkout@v4
with:
submodules: "recursive"
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
cache: 'pip'
- name: System Info
run: |
uname -a
sysctl -n machdep.cpu.brand_string
python3 -c "import platform; print(platform.machine(), platform.architecture())"
- name: Restore model cache
uses: actions/cache@v4
with:
path: ~/.cache/huggingface/hub
key: ${{ runner.os }}-model-${{ env.REPO_ID }}-${{ env.MODEL_FILE }}
- name: Install dependencies (Linux/MacOS)
run: |
python3 -m pip install --upgrade pip
python3 -m pip install uv
python3 -m uv pip install -e .[all] --verbose
CMAKE_ARGS="-DLLAMA_METAL=off" python3 -m uv pip install .[all] --verbose
shell: bash
- name: Test with pytest
run: |
python3 -m pytest
build-macos-metal:
needs: download-model
runs-on: macos-13
steps:
- uses: actions/checkout@v4
with:
submodules: "recursive"
- name: Set up Python 3.9
uses: actions/setup-python@v5
with:
python-version: "3.9"
- name: System Info
run: |
uname -a
sysctl -n machdep.cpu.brand_string
python3 -c "import platform; print(platform.machine(), platform.architecture())"
- name: Restore model cache
uses: actions/cache@v4
with:
path: ~/.cache/huggingface/hub
key: ${{ runner.os }}-model-${{ env.REPO_ID }}-${{ env.MODEL_FILE }}
- name: Install dependencies
run: |
python3 -m pip install --upgrade pip
CMAKE_ARGS="-DLLAMA_METAL=on" python3 -m pip install .[all] --verbose
shell: bash
- name: Test with pytest
run: |
python3 -m pytest