Feature/tighten up ci pipeline

.github/workflows/ci.yml

@@ -72,6 +72,8 @@ jobs:
       - name: Run unittest tests with coverage
         run: |
           pytest --cov=autogpt --cov-report term-missing --cov-branch --cov-report xml --cov-report term
+        env:
+          CI: true
 
       - name: Upload coverage reports to Codecov
         uses: codecov/codecov-action@v3

tests/integration/challenges/memory/test_memory_challenge_b.py

@@ -6,7 +6,7 @@
 from tests.integration.challenges.utils import generate_noise, get_level_to_run
 from tests.utils import requires_api_key
 
-LEVEL_CURRENTLY_BEATEN = 2
+LEVEL_CURRENTLY_BEATEN = None
 MAX_LEVEL = 5
 NOISE = 1000
 
@@ -24,15 +24,14 @@ def test_memory_challenge_b(
         memory_management_agent (Agent)
         user_selected_level (int)
     """
-
     current_level = get_level_to_run(
         user_selected_level, LEVEL_CURRENTLY_BEATEN, MAX_LEVEL
     )
     task_ids = [str(i * 1111) for i in range(1, current_level + 1)]
     create_instructions_files(memory_management_agent, current_level, task_ids)
 
     try:
-        run_interaction_loop(memory_management_agent, 40)
+        run_interaction_loop(memory_management_agent, 60)
     except SystemExit:
         file_path = str(memory_management_agent.workspace.get_path("output.txt"))
         content = read_file(file_path)

tests/integration/conftest.py

@@ -1,6 +1,8 @@
+import os
+
 import pytest
 
-from tests.vcr.openai_filter import before_record_request, before_record_response
+from tests.vcr.vcr_filter import before_record_request, before_record_response
 
 
 @pytest.fixture(scope="session")
@@ -15,4 +17,5 @@ def vcr_config():
             "X-OpenAI-Client-User-Agent",
             "User-Agent",
         ],
+        "match_on": ["method", "uri", "body"],
     }

tests/vcr/vcr_filter.py

@@ -0,0 +1,73 @@
+import json
+import re
+from typing import Any, Dict, List
+
+REPLACEMENTS: List[Dict[str, str]] = [
+    {
+        "regex": r"\w{3} \w{3} {1,2}\d{1,2} \d{2}:\d{2}:\d{2} \d{4}",
+        "replacement": "Tue Jan  1 00:00:00 2000",
+    },
+    {
+        "regex": r"<selenium.webdriver.chrome.webdriver.WebDriver[^>]*>",
+        "replacement": "",
+    },
+]
+
+
+def replace_message_content(content: str, replacements: List[Dict[str, str]]) -> str:
+    for replacement in replacements:
+        pattern = re.compile(replacement["regex"])
+        content = pattern.sub(replacement["replacement"], content)
+
+    return content
+
+
+def replace_timestamp_in_request(request: Any) -> Any:
+    try:
+        if not request or not request.body:
+            return request
+        body = json.loads(request.body)
+    except ValueError:
+        return request
+
+    if "messages" not in body:
+        return request
+    body[
+        "max_tokens"
+    ] = 0  # this field is incosistent between requests and not used at the moment.
+    for message in body["messages"]:
+        if "content" in message and "role" in message:
+            if message["role"] == "system":
+                message["content"] = replace_message_content(
+                    message["content"], REPLACEMENTS
+                )
+
+    request.body = json.dumps(body)
+    return request
+
+
+def before_record_response(response: Dict[str, Any]) -> Dict[str, Any]:
+    if "Transfer-Encoding" in response["headers"]:
+        del response["headers"]["Transfer-Encoding"]
+    return response
+
+
+def before_record_request(request: Any) -> Any:
+    filtered_request = filter_hostnames(request)
+    filtered_request_without_dynamic_data = replace_timestamp_in_request(
+        filtered_request
+    )
+    return filtered_request_without_dynamic_data
+
+
+def filter_hostnames(request: Any) -> Any:
+    allowed_hostnames: List[str] = [
+        "api.openai.com",
+        "localhost:50337",
+    ]
+
+    # Add your implementation here for filtering hostnames
+    if any(hostname in request.url for hostname in allowed_hostnames):
+        return request
+    else:
+        return None