test: Use temporary directories, instead of deleting directories. Use…

… terminate() instead of kill(), so that pytest-cov can measure subprocesses.

tests/__init__.py

@@ -1,7 +1,5 @@
 import io
-import os.path
 import pkgutil
-import shutil
 
 
 def get_egg_data(basename):
@@ -10,12 +8,3 @@ def get_egg_data(basename):
 
 def root_add_version(root, project, version, basename):
     root.eggstorage.put(io.BytesIO(get_egg_data(basename)), project, version)
-
-
-def clean(config, setting):
-    directory = os.path.realpath(config.get(setting))
-    basedir = os.path.realpath(os.path.dirname(os.path.dirname(__file__)))
-    # Avoid accidentally deleting directories outside the project.
-    assert os.path.commonprefix((directory, basedir)) == basedir
-    if os.path.exists(directory):
-        shutil.rmtree(directory)

tests/conftest.py

@@ -6,7 +6,7 @@
 from scrapyd import Config
 from scrapyd.app import application
 from scrapyd.website import Root
-from tests import clean, root_add_version
+from tests import root_add_version
 
 
 @pytest.fixture()
@@ -16,18 +16,19 @@ def txrequest():
     return Request(http_channel)
 
 
-@pytest.fixture(params=[None, ("scrapyd", "items_dir", "items")], ids=["default", "items_dir"])
-def root(request):
+# Use this fixture when testing the Scrapyd web UI or API or writing configuration files.
+@pytest.fixture()
+def chdir(monkeypatch, tmpdir):
+    return monkeypatch.chdir(tmpdir)
+
+
+@pytest.fixture(params=[None, (Config.SECTION, "items_dir", "items")], ids=["default", "items_dir"])
+def root(request, chdir):
     config = Config()
     if request.param:
         config.cp.set(*request.param)
 
-    app = application(config)
-
-    yield Root(config, app)
-
-    for setting in ("dbs_dir", "eggs_dir"):
-        clean(config, setting)
+    return Root(config, application(config))
 
 
 @pytest.fixture()

tests/mockserver.py

@@ -1,11 +1,12 @@
-import os
+import os.path
 import re
-import shutil
 import socket
 import sys
 from subprocess import PIPE, Popen
 from urllib.parse import urljoin
 
+BASEDIR = os.path.abspath(os.path.dirname(__file__))
+
 
 def get_ephemeral_port():
     # Somehow getting random high port doesn't work on pypy
@@ -22,25 +23,27 @@ def __init__(self, authentication=None):
 
     def __enter__(self, authentication=None):
         """Launch Scrapyd application object with ephemeral port"""
-        command = [sys.executable, "-m", "tests.start_mock_app", get_ephemeral_port()]
+        command = [sys.executable, os.path.join(BASEDIR, "start_mock_app.py"), get_ephemeral_port()]
         if self.authentication is not None:
             command.append("--auth=" + self.authentication)
 
-        self.proc = Popen(command, stdout=PIPE)
+        self.process = Popen(command, stdout=PIPE)
+
+        # The loop is expected to run 3 times.
+        # 2001-02-03 04:05:06-0000 [-] Log opened.
+        # 2001-02-03 04:05:06-0000 [-] Basic authentication disabled as either `username` or `password` is unset
+        # 2001-02-03 04:05:06-0000 [-] Scrapyd web console available at http://127.0.0.1:53532/
         for _ in range(10):
-            msg = self.proc.stdout.readline().strip().decode("ascii")
-            addr_line = re.search("available at (.+/)", msg)
-            if addr_line:
-                self.url = addr_line.group(1)
+            line = self.process.stdout.readline().strip().decode("ascii")
+            if address := re.search("available at (.+/)", line):
+                self.url = address.group(1)
                 break
 
         return self
 
     def __exit__(self, exc_type, exc_value, traceback):
-        self.proc.kill()
-        self.proc.communicate()
-        if os.path.isdir("eggs") and os.listdir("eggs") != []:
-            shutil.rmtree("eggs")
+        self.process.terminate()
+        self.process.communicate()
 
     def urljoin(self, path):
         return urljoin(self.url, path)

tests/test_eggstorage.py

@@ -51,7 +51,7 @@ def test_sorted_versions(versions, expected):
     assert sorted_versions(versions) == expected
 
 
-def test_config():
+def test_config(chdir):
     config = Config()
     config.cp.set("scrapyd", "eggstorage", "tests.test_eggstorage.FakeEggStorage")
 

tests/test_endpoints.py

@@ -10,7 +10,7 @@
 
 
 @pytest.fixture()
-def mock_scrapyd():
+def mock_scrapyd(chdir):
     with MockScrapydServer() as server:
         yield server
 

tests/test_webservice.py

@@ -5,20 +5,17 @@
 import pytest
 from twisted.web import error
 
-from scrapyd.config import Config
 from scrapyd.exceptions import DirectoryTraversalError, RunnerError
 from scrapyd.interfaces import IEggStorage, IJobStorage
 from scrapyd.jobstorage import Job
 from scrapyd.txapp import application
 from scrapyd.webservice import UtilsCache, get_spider_list
-from tests import clean, get_egg_data, root_add_version
+from tests import get_egg_data, root_add_version
 
 
 @pytest.fixture()
-def app():
-    yield application
-
-    clean(Config(), "eggs_dir")
+def app(chdir):
+    return application
 
 
 def add_test_version(app, project, version, basename):