Migrate AutoGPT agent to poetry (#5219)

Inspired by #1102

* Migrate AutoGPT agent to poetry

  Co-authored-by: rickythefox <richard@ginzburg.se>

* Rewrite automatic dependency check (check_requirements.py) for poetry

* Sort dependencies

* Add instructions for poetry to README

.github/workflows/autogpt-ci.yml

@@ -57,13 +57,13 @@ jobs:
       - name: Set up Python dependency cache
         uses: actions/cache@v3
         with:
-          path: ~/.cache/pip
-          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}-${{ steps.get_date.outputs.date }}
+          path: ~/.cache/pypoetry
+          key: ${{ runner.os }}-poetry-${{ hashFiles('autogpts/autogpt/pyproject.toml') }}-${{ steps.get_date.outputs.date }}
 
-      - name: Install dependencies
+      - name: Install Python dependencies
         run: |
-          python -m pip install --upgrade pip
-          pip install -r requirements.txt
+          curl -sSL https://install.python-poetry.org | python3 -
+          poetry install
 
       - name: Lint with flake8
         run: flake8
@@ -156,20 +156,21 @@ jobs:
       - name: Set up Python dependency cache
         uses: actions/cache@v3
         with:
-          path: ~/.cache/pip
-          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}-${{ steps.get_date.outputs.date }}
+          path: ~/.cache/pypoetry
+          key: ${{ runner.os }}-poetry-${{ hashFiles('autogpts/autogpt/pyproject.toml') }}-${{ steps.get_date.outputs.date }}
 
       - name: Install Python dependencies
         run: |
-          python -m pip install --upgrade pip
-          pip install -r requirements.txt
+          curl -sSL https://install.python-poetry.org | python3 -
+          poetry install
 
       - name: Run pytest with coverage
         run: |
-          pytest -vv --cov=autogpt --cov-branch --cov-report term-missing --cov-report xml \
+          poetry run pytest -vv \
+            --cov=autogpt --cov-branch --cov-report term-missing --cov-report xml \
             --numprocesses=logical --durations=10 \
             tests/unit tests/integration tests/challenges
-          python tests/challenges/utils/build_current_score.py
+          poetry run python tests/challenges/utils/build_current_score.py
         env:
           CI: true
           PROXY: ${{ github.event_name == 'pull_request_target' && secrets.PROXY || '' }}

.github/workflows/autogpt-docker-ci.yml

@@ -117,7 +117,7 @@ jobs:
         run: |
           set +e
           test_output=$(
-            docker run --env CI --env OPENAI_API_KEY --entrypoint python ${{ env.IMAGE_NAME }} -m \
+            docker run --env CI --env OPENAI_API_KEY ${{ env.IMAGE_NAME }} run \
             pytest -v --cov=autogpt --cov-branch --cov-report term-missing \
               --numprocesses=4 --durations=10 \
               tests/unit tests/integration 2>&1

.github/workflows/benchmark-ci.yml

@@ -162,8 +162,8 @@ jobs:
           elif [ "$AGENT_NAME" == "Auto-GPT" ]; then
             python -m venv venv
             source venv/bin/activate
-            pip install -r requirements.txt
-            pip uninstall agbenchmark -y
+            curl -sSL https://install.python-poetry.org | python3 -
+            poetry install --without benchmark
           elif [ "$AGENT_NAME" == "mini-agi" ]; then
             python -m venv venv
             source venv/bin/activate

.github/workflows/benchmarks.yml

@@ -39,13 +39,13 @@ jobs:
       - name: Set up Python dependency cache
         uses: actions/cache@v3
         with:
-          path: ~/.cache/pip
-          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}-${{ steps.get_date.outputs.date }}
+          path: ~/.cache/pypoetry
+          key: ${{ runner.os }}-poetry-${{ hashFiles('autogpts/autogpt/pyproject.toml') }}-${{ steps.get_date.outputs.date }}
 
       - name: Install Python dependencies
         run: |
-          python -m pip install --upgrade pip
-          pip install -r requirements.txt
+          curl -sSL https://install.python-poetry.org | python3 -
+          poetry install
 
       - name: Run pytest with coverage
         run: |

autogpts/autogpt/.devcontainer/devcontainer.json

@@ -46,11 +46,11 @@
   // "forwardPorts": [],
 
   // Use 'postCreateCommand' to run commands after the container is created.
-  // "postCreateCommand": "pip3 install --user -r requirements.txt",
+  // "postCreateCommand": "poetry install",
 
   // Set `remoteUser` to `root` to connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root.
   "remoteUser": "vscode",
 
   // Add the freshly containerized repo to the list of safe repositories
-  "postCreateCommand": "git config --global --add safe.directory /workspace/Auto-GPT && pip3 install --user -r requirements.txt"
-}
+  "postCreateCommand": "git config --global --add safe.directory /workspace/Auto-GPT && poetry install"
+}

autogpts/autogpt/.dockerignore

@@ -9,6 +9,4 @@ logs/*
 agbenchmark_config/logs/*
 agbenchmark_config/reports/*
 
-*.md
 *.png
-!BULLETIN.md

autogpts/autogpt/Dockerfile

@@ -17,30 +17,37 @@ RUN apt-get update && apt-get install -y \
 # Set environment variables
 ENV PIP_NO_CACHE_DIR=yes \
     PYTHONUNBUFFERED=1 \
-    PYTHONDONTWRITEBYTECODE=1
+    PYTHONDONTWRITEBYTECODE=1 \
+    POETRY_HOME="/opt/poetry" \
+    POETRY_VIRTUALENVS_IN_PROJECT=0 \
+    POETRY_NO_INTERACTION=1
 
-# Install the required python packages globally
-ENV PATH="$PATH:/root/.local/bin"
-COPY requirements.txt .
+# Install and configure Poetry
+RUN curl -sSL https://install.python-poetry.org | python3 -
+ENV PATH="$POETRY_HOME/bin:$PATH"
+RUN poetry config installer.max-workers 10
+
+WORKDIR /app
+COPY pyproject.toml poetry.lock ./
 
 # Set the entrypoint
-ENTRYPOINT ["python", "-m", "autogpt", "--install-plugin-deps"]
+ENTRYPOINT ["poetry"]
+CMD ["run", "autogpt", "--install-plugin-deps"]
 
 # dev build -> include everything
 FROM autogpt-base as autogpt-dev
-RUN pip install --no-cache-dir -r requirements.txt
-WORKDIR /app
+RUN poetry install --no-root
 ONBUILD COPY . ./
 
 # release build -> include bare minimum
 FROM autogpt-base as autogpt-release
-RUN sed -i '/Items below this point will not be included in the Docker Image/,$d' requirements.txt && \
-	pip install --no-cache-dir -r requirements.txt
-WORKDIR /app
+RUN poetry install --no-root --without dev,benchmark
 ONBUILD COPY autogpt/ ./autogpt
 ONBUILD COPY scripts/ ./scripts
 ONBUILD COPY plugins/ ./plugins
 ONBUILD COPY prompt_settings.yaml ./prompt_settings.yaml
+ONBUILD COPY README.md ./README.md
 ONBUILD RUN mkdir ./data
 
-FROM autogpt-${BUILD_TYPE} AS auto-gpt
+FROM autogpt-${BUILD_TYPE} AS autogpt
+RUN poetry install --only-root

autogpts/autogpt/README.md

@@ -60,6 +60,11 @@ Please see the [documentation][docs] for full setup instructions and configurati
 [docs/usage]: https://docs.agpt.co/usage/
 [docs/plugins]: https://docs.agpt.co/plugins/
 
+## 🏗️ Setting up for development
+1. Make sure `poetry` is installed: `python3 -m pip install poetry`
+2. Install all dependencies: `poetry install`
+
+
 <h2 align="center"> 💖 Help Fund Auto-GPT's Development 💖</h2>
 <p align="center">
 If you can spare a coffee, you can help to cover the costs of developing Auto-GPT and help to push the boundaries of fully autonomous AI!