Windows CI fixes (#523)

test(windows): Get tests passing

- Use terminate() instead of kill(), so that pytest-cov can measure subprocesses
- Use temporary directories, instead of deleting directories
- Fix call to cancel_button from 848fc18
- Set Job end_time like 833e354
- Remove mock and pythonpath

scrapyd/website.py

@@ -314,7 +314,7 @@ def prep_tab_pending(self):
                     "Project": escape(project),
                     "Spider": escape(m["name"]),
                     "Job": escape(m["_job"]),
-                    "Cancel": self.cancel_button(project=project, jobid=m["_job"], base_path=self.base_path),
+                    "Cancel": cancel_button(project=project, jobid=m["_job"], base_path=self.base_path),
                 }
             )
             for project, queue in self.root.scheduler.queues.items()
@@ -333,7 +333,7 @@ def prep_tab_running(self):
                     "Runtime": microsec_trunc(datetime.now() - p.start_time),
                     "Log": f'<a href="{self.base_path}{job_log_url(p)}">Log</a>',
                     "Items": f'<a href="{self.base_path}{job_items_url(p)}">Items</a>',
-                    "Cancel": self.cancel_button(project=p.project, jobid=p.job, base_path=self.base_path),
+                    "Cancel": cancel_button(project=p.project, jobid=p.job, base_path=self.base_path),
                 }
             )
             for p in self.root.launcher.processes.values()

tests/conftest.py

@@ -1,6 +1,3 @@
-import os.path
-import shutil
-
 import pytest
 from twisted.web import http
 from twisted.web.http import Request
@@ -19,23 +16,19 @@ def txrequest():
     return Request(http_channel)
 
 
-@pytest.fixture(params=[None, ("scrapyd", "items_dir", "items")], ids=["default", "items_dir"])
-def root(request):
+# Use this fixture when testing the Scrapyd web UI or API or writing configuration files.
+@pytest.fixture()
+def chdir(monkeypatch, tmpdir):
+    return monkeypatch.chdir(tmpdir)
+
+
+@pytest.fixture(params=[None, (Config.SECTION, "items_dir", "items")], ids=["default", "items_dir"])
+def root(request, chdir):
     config = Config()
     if request.param:
         config.cp.set(*request.param)
 
-    app = application(config)
-
-    yield Root(config, app)
-
-    for setting in ("dbs_dir", "eggs_dir"):
-        directory = os.path.realpath(config.get(setting))
-        basedir = os.path.realpath(os.path.dirname(os.path.dirname(__file__)))
-        # Avoid accidentally deleting directories outside the project.
-        assert os.path.commonprefix((directory, basedir)) == basedir
-        if os.path.exists(directory):
-            shutil.rmtree(directory)
+    return Root(config, application(config))
 
 
 @pytest.fixture()

tests/mockserver.py

@@ -1,11 +1,12 @@
-import os
+import os.path
 import re
-import shutil
 import socket
 import sys
 from subprocess import PIPE, Popen
 from urllib.parse import urljoin
 
+BASEDIR = os.path.abspath(os.path.dirname(__file__))
+
 
 def get_ephemeral_port():
     # Somehow getting random high port doesn't work on pypy
@@ -22,25 +23,27 @@ def __init__(self, authentication=None):
 
     def __enter__(self, authentication=None):
         """Launch Scrapyd application object with ephemeral port"""
-        command = [sys.executable, "-m", "tests.start_mock_app", get_ephemeral_port()]
+        command = [sys.executable, os.path.join(BASEDIR, "start_mock_app.py"), get_ephemeral_port()]
         if self.authentication is not None:
             command.append("--auth=" + self.authentication)
 
-        self.proc = Popen(command, stdout=PIPE)
+        self.process = Popen(command, stdout=PIPE)
+
+        # The loop is expected to run 3 times.
+        # 2001-02-03 04:05:06-0000 [-] Log opened.
+        # 2001-02-03 04:05:06-0000 [-] Basic authentication disabled as either `username` or `password` is unset
+        # 2001-02-03 04:05:06-0000 [-] Scrapyd web console available at http://127.0.0.1:53532/
         for _ in range(10):
-            msg = self.proc.stdout.readline().strip().decode("ascii")
-            addr_line = re.search("available at (.+/)", msg)
-            if addr_line:
-                self.url = addr_line.group(1)
+            line = self.process.stdout.readline().strip().decode("ascii")
+            if address := re.search("available at (.+/)", line):
+                self.url = address.group(1)
                 break
 
         return self
 
     def __exit__(self, exc_type, exc_value, traceback):
-        self.proc.kill()
-        self.proc.communicate()
-        if os.path.isdir("eggs") and os.listdir("eggs") != []:
-            shutil.rmtree("eggs")
+        self.process.terminate()
+        self.process.communicate()
 
     def urljoin(self, path):
         return urljoin(self.url, path)

tests/test_eggstorage.py

@@ -51,7 +51,7 @@ def test_sorted_versions(versions, expected):
     assert sorted_versions(versions) == expected
 
 
-def test_config():
+def test_config(chdir):
     config = Config()
     config.cp.set("scrapyd", "eggstorage", "tests.test_eggstorage.FakeEggStorage")
 

tests/test_endpoints.py

@@ -10,7 +10,7 @@
 
 
 @pytest.fixture()
-def mock_scrapyd():
+def mock_scrapyd(chdir):
     with MockScrapydServer() as server:
         yield server
 

tests/test_jobstorage.py

@@ -1,12 +1,14 @@
+import datetime
+
 from zope.interface.verify import verifyObject
 
 from scrapyd.config import Config
 from scrapyd.interfaces import IJobStorage
 from scrapyd.jobstorage import Job, MemoryJobStorage, SqliteJobStorage
 
-j1 = Job("p1", "s1")
-j2 = Job("p2", "s2")
-j3 = Job("p3", "s3")
+j1 = Job("p1", "s1", end_time=datetime.datetime(2001, 2, 3, 4, 5, 6, 7))
+j2 = Job("p2", "s2", end_time=datetime.datetime(2001, 2, 3, 4, 5, 6, 8))
+j3 = Job("p3", "s3", end_time=datetime.datetime(2001, 2, 3, 4, 5, 6, 9))
 
 
 def pytest_generate_tests(metafunc):

tests/test_webservice.py

@@ -1,40 +1,21 @@
 import io
 import os
 import re
-from unittest import mock
 
 import pytest
-from scrapy.utils.test import get_pythonpath
 from twisted.web import error
 
-from scrapyd import get_application
 from scrapyd.exceptions import DirectoryTraversalError, RunnerError
-from scrapyd.interfaces import IEggStorage
+from scrapyd.interfaces import IEggStorage, IJobStorage
 from scrapyd.jobstorage import Job
+from scrapyd.txapp import application
 from scrapyd.webservice import UtilsCache, get_spider_list
 from tests import get_egg_data, root_add_version
 
 
-def fake_list_jobs(*args, **kwargs):
-    yield Job("proj1", "spider-a", "id1234")
-
-
-def fake_list_spiders(*args, **kwargs):
-    return []
-
-
-def fake_list_spiders_other(*args, **kwargs):
-    return ["quotesbot", "toscrape-css"]
-
-
-def get_pythonpath_scrapyd():
-    scrapyd_path = __import__("scrapyd").__path__[0]
-    return os.path.join(os.path.dirname(scrapyd_path), get_pythonpath(), os.environ.get("PYTHONPATH", ""))
-
-
 @pytest.fixture()
-def app():
-    return get_application()
+def app(chdir):
+    return application
 
 
 def add_test_version(app, project, version, basename):
@@ -43,7 +24,7 @@ def add_test_version(app, project, version, basename):
 
 def test_get_spider_list_log_stdout(app):
     add_test_version(app, "logstdout", "logstdout", "logstdout")
-    spiders = get_spider_list("logstdout", pythonpath=get_pythonpath_scrapyd())
+    spiders = get_spider_list("logstdout")
 
     # If LOG_STDOUT were respected, the output would be [].
     assert sorted(spiders) == ["spider1", "spider2"]
@@ -52,50 +33,50 @@ def test_get_spider_list_log_stdout(app):
 def test_get_spider_list(app):
     # mybot.egg has two spiders, spider1 and spider2
     add_test_version(app, "mybot", "r1", "mybot")
-    spiders = get_spider_list("mybot", pythonpath=get_pythonpath_scrapyd())
+    spiders = get_spider_list("mybot")
     assert sorted(spiders) == ["spider1", "spider2"]
 
     # mybot2.egg has three spiders, spider1, spider2 and spider3...
     # BUT you won't see it here because it's cached.
     # Effectivelly it's like if version was never added
     add_test_version(app, "mybot", "r2", "mybot2")
-    spiders = get_spider_list("mybot", pythonpath=get_pythonpath_scrapyd())
+    spiders = get_spider_list("mybot")
     assert sorted(spiders) == ["spider1", "spider2"]
 
     # Let's invalidate the cache for this project...
     UtilsCache.invalid_cache("mybot")
 
     # Now you get the updated list
-    spiders = get_spider_list("mybot", pythonpath=get_pythonpath_scrapyd())
+    spiders = get_spider_list("mybot")
     assert sorted(spiders) == ["spider1", "spider2", "spider3"]
 
     # Let's re-deploy mybot.egg and clear cache. It now sees 2 spiders
     add_test_version(app, "mybot", "r3", "mybot")
     UtilsCache.invalid_cache("mybot")
-    spiders = get_spider_list("mybot", pythonpath=get_pythonpath_scrapyd())
+    spiders = get_spider_list("mybot")
     assert sorted(spiders) == ["spider1", "spider2"]
 
     # And re-deploying the one with three (mybot2.egg) with a version that
     # isn't the higher, won't change what get_spider_list() returns.
     add_test_version(app, "mybot", "r1a", "mybot2")
     UtilsCache.invalid_cache("mybot")
-    spiders = get_spider_list("mybot", pythonpath=get_pythonpath_scrapyd())
+    spiders = get_spider_list("mybot")
     assert sorted(spiders) == ["spider1", "spider2"]
 
 
 @pytest.mark.skipif(os.name == "nt", reason="get_spider_list() unicode fails on windows")
 def test_get_spider_list_unicode(app):
     # mybotunicode.egg has two spiders, araña1 and araña2
     add_test_version(app, "mybotunicode", "r1", "mybotunicode")
-    spiders = get_spider_list("mybotunicode", pythonpath=get_pythonpath_scrapyd())
+    spiders = get_spider_list("mybotunicode")
 
     assert sorted(spiders) == ["araña1", "araña2"]
 
 
 def test_failed_spider_list(app):
     add_test_version(app, "mybot3", "r1", "mybot3")
     with pytest.raises(RunnerError) as exc:
-        get_spider_list("mybot3", pythonpath=get_pythonpath_scrapyd())
+        get_spider_list("mybot3")
 
     assert re.search(f"Exception: This should break the `scrapy list` command{os.linesep}$", str(exc.value))
 
@@ -195,8 +176,10 @@ def test_list_jobs(txrequest, root_with_egg):
     assert set(content) == {"node_name", "status", "pending", "running", "finished"}
 
 
-@mock.patch("scrapyd.jobstorage.MemoryJobStorage.__iter__", new=fake_list_jobs)
 def test_list_jobs_finished(txrequest, root_with_egg):
+    jobstorage = root_with_egg.app.getComponent(IJobStorage)
+    jobstorage.add(Job("proj1", "spider-a", "id1234"))
+
     txrequest.args = {}
     endpoint = b"listjobs.json"
     content = root_with_egg.children[endpoint].render_GET(txrequest)