diff --git a/pyproject.toml b/pyproject.toml index 15bc96c5..b0ef8704 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -67,9 +67,6 @@ version = {attr = "setuptools_scm.get_version"} check_untyped_defs = true disallow_incomplete_defs = true disallow_untyped_calls = true - -[[tool.mypy.overrides]] -module = ["tldextract.*"] disallow_untyped_defs = true [tool.pytest.ini_options] diff --git a/tests/cli_test.py b/tests/cli_test.py index a83ff155..10fe976c 100644 --- a/tests/cli_test.py +++ b/tests/cli_test.py @@ -8,7 +8,7 @@ from tldextract.tldextract import PUBLIC_SUFFIX_LIST_URLS -def test_cli_no_input(monkeypatch): +def test_cli_no_input(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(sys, "argv", ["tldextract"]) with pytest.raises(SystemExit) as ex: main() @@ -16,7 +16,7 @@ def test_cli_no_input(monkeypatch): assert ex.value.code == 1 -def test_cli_parses_args(monkeypatch): +def test_cli_parses_args(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(sys, "argv", ["tldextract", "--some", "nonsense"]) with pytest.raises(SystemExit) as ex: main() @@ -24,7 +24,9 @@ def test_cli_parses_args(monkeypatch): assert ex.value.code == 2 -def test_cli_posargs(capsys, monkeypatch): +def test_cli_posargs( + capsys: pytest.CaptureFixture, monkeypatch: pytest.MonkeyPatch +) -> None: monkeypatch.setattr( sys, "argv", ["tldextract", "example.com", "bbc.co.uk", "forums.bbc.co.uk"] ) @@ -36,7 +38,9 @@ def test_cli_posargs(capsys, monkeypatch): assert stdout == " example com\n bbc co.uk\nforums bbc co.uk\n" -def test_cli_namedargs(capsys, monkeypatch): +def test_cli_namedargs( + capsys: pytest.CaptureFixture, monkeypatch: pytest.MonkeyPatch +) -> None: monkeypatch.setattr( sys, "argv", diff --git a/tests/conftest.py b/tests/conftest.py index 909411e2..c1642ca0 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -8,7 +8,7 @@ @pytest.fixture(autouse=True) -def reset_log_level(): +def reset_log_level() -> None: """Automatically reset log level verbosity between tests. Generally want test output the Unix way: silence is golden. diff --git a/tests/custom_suffix_test.py b/tests/custom_suffix_test.py index 9958ae33..976ba83e 100644 --- a/tests/custom_suffix_test.py +++ b/tests/custom_suffix_test.py @@ -23,7 +23,7 @@ ) -def test_private_extraction(): +def test_private_extraction() -> None: tld = tldextract.TLDExtract(cache_dir=tempfile.mkdtemp(), suffix_list_urls=[]) assert tld("foo.blogspot.com") == ("foo", "blogspot", "com", False) @@ -35,7 +35,7 @@ def test_private_extraction(): ) -def test_suffix_which_is_not_in_custom_list(): +def test_suffix_which_is_not_in_custom_list() -> None: for fun in ( extract_using_fake_suffix_list, extract_using_fake_suffix_list_no_cache, @@ -44,7 +44,7 @@ def test_suffix_which_is_not_in_custom_list(): assert result.suffix == "" -def test_custom_suffixes(): +def test_custom_suffixes() -> None: for fun in ( extract_using_fake_suffix_list, extract_using_fake_suffix_list_no_cache, @@ -54,12 +54,12 @@ def test_custom_suffixes(): assert result.suffix == custom_suffix -def test_suffix_which_is_not_in_extra_list(): +def test_suffix_which_is_not_in_extra_list() -> None: result = extract_using_extra_suffixes("www.google.com") assert result.suffix == "" -def test_extra_suffixes(): +def test_extra_suffixes() -> None: for custom_suffix in EXTRA_SUFFIXES: netloc = "www.foo.bar.baz.quux" + "." + custom_suffix result = extract_using_extra_suffixes(netloc) diff --git a/tests/integration_test.py b/tests/integration_test.py index 432d7b2e..8cce9e1d 100644 --- a/tests/integration_test.py +++ b/tests/integration_test.py @@ -5,7 +5,7 @@ import tldextract -def test_bad_kwargs(): +def test_bad_kwargs() -> None: with pytest.raises(ValueError): tldextract.TLDExtract( cache_dir=None, suffix_list_urls=(), fallback_to_snapshot=False diff --git a/tests/main_test.py b/tests/main_test.py index f95ef6ba..9c26dc72 100644 --- a/tests/main_test.py +++ b/tests/main_test.py @@ -6,8 +6,11 @@ import os import tempfile from collections.abc import Sequence +from pathlib import Path +from typing import Any import pytest +import pytest_mock import responses import tldextract @@ -62,34 +65,34 @@ def assert_extract( assert expected_ipv6_data == ext.ipv6 -def test_american(): +def test_american() -> None: assert_extract("http://www.google.com", ("www.google.com", "www", "google", "com")) -def test_british(): +def test_british() -> None: assert_extract( "http://www.theregister.co.uk", ("www.theregister.co.uk", "www", "theregister", "co.uk"), ) -def test_no_subdomain(): +def test_no_subdomain() -> None: assert_extract("http://gmail.com", ("gmail.com", "", "gmail", "com")) -def test_nested_subdomain(): +def test_nested_subdomain() -> None: assert_extract( "http://media.forums.theregister.co.uk", ("media.forums.theregister.co.uk", "media.forums", "theregister", "co.uk"), ) -def test_odd_but_possible(): +def test_odd_but_possible() -> None: assert_extract("http://www.www.com", ("www.www.com", "www", "www", "com")) assert_extract("http://www.com", ("www.com", "", "www", "com")) -def test_suffix(): +def test_suffix() -> None: assert_extract("com", ("", "", "", "com")) assert_extract("co.uk", ("", "", "", "co.uk")) assert_extract("example.ck", ("", "", "", "example.ck")) @@ -102,7 +105,7 @@ def test_suffix(): assert_extract("buskerud.no", ("buskerud.no", "", "buskerud", "no")) -def test_local_host(): +def test_local_host() -> None: assert_extract( "http://internalunlikelyhostname/", ("", "", "internalunlikelyhostname", "") ) @@ -112,7 +115,7 @@ def test_local_host(): ) -def test_qualified_local_host(): +def test_qualified_local_host() -> None: assert_extract( "http://internalunlikelyhostname.info/", ("internalunlikelyhostname.info", "", "internalunlikelyhostname", "info"), @@ -123,7 +126,7 @@ def test_qualified_local_host(): ) -def test_ip(): +def test_ip() -> None: assert_extract( "http://216.22.0.192/", ("", "", "216.22.0.192", ""), @@ -135,7 +138,7 @@ def test_ip(): ) -def test_lenient_netloc(): +def test_lenient_netloc() -> None: assert lenient_netloc("https://example.com.ca") == "example.com.ca" assert lenient_netloc("https://[example.com.ca]") == "[example.com.ca]" assert lenient_netloc("https://[example.com.ca]:5000") == "[example.com.ca]" @@ -156,27 +159,27 @@ def test_lenient_netloc(): @pytest.mark.skipif(not inet_pton, reason="inet_pton unavailable") -def test_looks_like_ip_with_inet_pton(): +def test_looks_like_ip_with_inet_pton() -> None: assert looks_like_ip("1.1.1.1", inet_pton) is True assert looks_like_ip("a.1.1.1", inet_pton) is False assert looks_like_ip("1.1.1.1\n", inet_pton) is False assert looks_like_ip("256.256.256.256", inet_pton) is False -def test_looks_like_ip_without_inet_pton(): +def test_looks_like_ip_without_inet_pton() -> None: assert looks_like_ip("1.1.1.1", None) is True assert looks_like_ip("a.1.1.1", None) is False assert looks_like_ip("1.1.1.1\n", None) is False assert looks_like_ip("256.256.256.256", None) is False -def test_similar_to_ip(): +def test_similar_to_ip() -> None: assert_extract("1\xe9", ("", "", "1\xe9", "")) assert_extract("1.1.1.1\ncom", ("", "1.1.1", "1\ncom", "")) assert_extract("1.1.1.1\rcom", ("", "1.1.1", "1\rcom", "")) -def test_punycode(): +def test_punycode() -> None: assert_extract( "http://xn--h1alffa9f.xn--p1ai", ("xn--h1alffa9f.xn--p1ai", "", "xn--h1alffa9f", "xn--p1ai"), @@ -212,7 +215,7 @@ def test_punycode(): ) -def test_invalid_puny_with_puny(): +def test_invalid_puny_with_puny() -> None: assert_extract( "http://xn--zckzap6140b352by.blog.so-net.xn--wcvs22d.hk", ( @@ -227,11 +230,11 @@ def test_invalid_puny_with_puny(): ) -def test_invalid_puny_with_nonpuny(): +def test_invalid_puny_with_nonpuny() -> None: assert_extract("xn--ß‌꫶ᢥ.com", ("xn--ß‌꫶ᢥ.com", "", "xn--ß‌꫶ᢥ", "com")) -def test_puny_with_non_puny(): +def test_puny_with_non_puny() -> None: assert_extract( "http://xn--zckzap6140b352by.blog.so-net.教育.hk", ( @@ -243,7 +246,7 @@ def test_puny_with_non_puny(): ) -def test_idna_2008(): +def test_idna_2008() -> None: """Python supports IDNA 2003. The IDNA library adds 2008 support for characters like ß.""" assert_extract( "xn--gieen46ers-73a.de", @@ -255,11 +258,11 @@ def test_idna_2008(): ) -def test_empty(): +def test_empty() -> None: assert_extract("http://", ("", "", "", "")) -def test_scheme(): +def test_scheme() -> None: assert_extract("//", ("", "", "", "")) assert_extract("://", ("", "", "", "")) assert_extract("://example.com", ("", "", "", "")) @@ -283,13 +286,13 @@ def test_scheme(): ) -def test_port(): +def test_port() -> None: assert_extract( "git+ssh://www.github.com:8443/", ("www.github.com", "www", "github", "com") ) -def test_username(): +def test_username() -> None: assert_extract( "ftp://johndoe:5cr1p7k1dd13@1337.warez.com:2501", ("1337.warez.com", "1337", "warez", "com"), @@ -311,7 +314,7 @@ def test_username(): ) -def test_query_fragment(): +def test_query_fragment() -> None: assert_extract("http://google.com?q=cats", ("google.com", "", "google", "com")) assert_extract("http://google.com#Welcome", ("google.com", "", "google", "com")) assert_extract("http://google.com/#Welcome", ("google.com", "", "google", "com")) @@ -321,7 +324,7 @@ def test_query_fragment(): ) -def test_regex_order(): +def test_regex_order() -> None: assert_extract( "http://www.parliament.uk", ("www.parliament.uk", "www", "parliament", "uk") ) @@ -331,7 +334,7 @@ def test_regex_order(): ) -def test_unhandled_by_iana(): +def test_unhandled_by_iana() -> None: assert_extract( "http://www.cgs.act.edu.au/", ("www.cgs.act.edu.au", "www", "cgs", "act.edu.au") ) @@ -340,7 +343,7 @@ def test_unhandled_by_iana(): ) -def test_tld_is_a_website_too(): +def test_tld_is_a_website_too() -> None: assert_extract( "http://www.metp.net.cn", ("www.metp.net.cn", "www", "metp", "net.cn") ) @@ -349,7 +352,7 @@ def test_tld_is_a_website_too(): # ('www.net.cn', 'www', 'net', 'cn')) -def test_no_1st_level_tld(): +def test_no_1st_level_tld() -> None: assert_extract("za", ("", "", "za", "")) assert_extract("example.za", ("", "example", "za", "")) assert_extract("co.za", ("", "", "", "co.za")) @@ -359,7 +362,7 @@ def test_no_1st_level_tld(): ) -def test_dns_root_label(): +def test_dns_root_label() -> None: assert_extract( "http://www.example.com./", ("www.example.com", "www", "example", "com") ) @@ -374,14 +377,14 @@ def test_dns_root_label(): ) -def test_private_domains(): +def test_private_domains() -> None: assert_extract( "http://waiterrant.blogspot.com", ("waiterrant.blogspot.com", "waiterrant", "blogspot", "com"), ) -def test_ipv4(): +def test_ipv4() -> None: assert_extract( "http://127.0.0.1/foo/bar", ("", "", "127.0.0.1", ""), @@ -394,7 +397,7 @@ def test_ipv4(): ) -def test_ipv4_bad(): +def test_ipv4_bad() -> None: assert_extract( "http://256.256.256.256/foo/bar", ("", "256.256.256", "256", ""), @@ -402,7 +405,7 @@ def test_ipv4_bad(): ) -def test_ipv4_lookalike(): +def test_ipv4_lookalike() -> None: assert_extract( "http://127.0.0/foo/bar", ("", "127.0", "0", ""), expected_ip_data="" ) @@ -414,7 +417,7 @@ def test_ipv4_lookalike(): ) -def test_result_as_dict(): +def test_result_as_dict() -> None: result = extract( "http://admin:password1@www.google.com:666/secret/admin/interface?param1=42" ) @@ -427,11 +430,13 @@ def test_result_as_dict(): assert result._asdict() == expected_dict -def test_cache_permission(mocker, monkeypatch, tmpdir): +def test_cache_permission( + mocker: pytest_mock.MockerFixture, monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: """Emit a warning once that this can't cache the latest PSL.""" warning = mocker.patch.object(logging.getLogger("tldextract.cache"), "warning") - def no_permission_makedirs(*args, **kwargs): + def no_permission_makedirs(*args: Any, **kwargs: Any) -> None: raise PermissionError( """[Errno 13] Permission denied: '/usr/local/lib/python3.7/site-packages/tldextract/.suffix_cache""" @@ -440,7 +445,7 @@ def no_permission_makedirs(*args, **kwargs): monkeypatch.setattr(os, "makedirs", no_permission_makedirs) for _ in range(0, 2): - my_extract = tldextract.TLDExtract(cache_dir=tmpdir) + my_extract = tldextract.TLDExtract(cache_dir=str(tmp_path)) assert_extract( "http://www.google.com", ("www.google.com", "www", "google", "com"), @@ -452,16 +457,16 @@ def no_permission_makedirs(*args, **kwargs): @responses.activate -def test_cache_timeouts(tmpdir): +def test_cache_timeouts(tmp_path: Path) -> None: server = "http://some-server.com" responses.add(responses.GET, server, status=408) - cache = DiskCache(tmpdir) + cache = DiskCache(str(tmp_path)) with pytest.raises(SuffixListNotFound): tldextract.suffix_list.find_first_response(cache, [server], 5) -def test_include_psl_private_domain_attr(): +def test_include_psl_private_domain_attr() -> None: extract_private = tldextract.TLDExtract(include_psl_private_domains=True) extract_public = tldextract.TLDExtract(include_psl_private_domains=False) assert extract_private("foo.uk.com") == ExtractResult( @@ -472,7 +477,7 @@ def test_include_psl_private_domain_attr(): ) -def test_tlds_property(): +def test_tlds_property() -> None: extract_private = tldextract.TLDExtract( cache_dir=None, suffix_list_urls=(), include_psl_private_domains=True ) @@ -482,7 +487,7 @@ def test_tlds_property(): assert len(extract_private.tlds) > len(extract_public.tlds) -def test_global_extract(): +def test_global_extract() -> None: assert tldextract.extract( "blogspot.com", include_psl_private_domains=True ) == ExtractResult(subdomain="", domain="", suffix="blogspot.com", is_private=True) diff --git a/tests/test_cache.py b/tests/test_cache.py index 073ce231..0bf20abb 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -5,6 +5,7 @@ import sys import types from collections.abc import Hashable +from pathlib import Path from typing import Any, cast from unittest.mock import Mock @@ -14,8 +15,8 @@ from tldextract.cache import DiskCache, get_cache_dir, get_pkg_unique_identifier -def test_disk_cache(tmpdir): - cache = DiskCache(tmpdir) +def test_disk_cache(tmp_path: Path) -> None: + cache = DiskCache(str(tmp_path)) cache.set("testing", "foo", "bar") assert cache.get("testing", "foo") == "bar" @@ -28,7 +29,7 @@ def test_disk_cache(tmpdir): assert cache.get("testing", "foo") == "baz" -def test_get_pkg_unique_identifier(monkeypatch): +def test_get_pkg_unique_identifier(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(sys, "version_info", (3, 8, 1, "final", 0)) monkeypatch.setattr(sys, "prefix", "/home/john/.pyenv/versions/myvirtualenv") @@ -42,7 +43,7 @@ def test_get_pkg_unique_identifier(monkeypatch): ) -def test_get_cache_dir(monkeypatch): +def test_get_cache_dir(monkeypatch: pytest.MonkeyPatch) -> None: pkg_identifier = "3.8.1.final__myvirtualenv__f01a7b__tldextract-1.2.3" monkeypatch.setattr( tldextract.cache, "get_pkg_unique_identifier", lambda: pkg_identifier @@ -79,8 +80,8 @@ def test_get_cache_dir(monkeypatch): assert get_cache_dir() == "/alt-tld-cache" -def test_run_and_cache(tmpdir): - cache = DiskCache(tmpdir) +def test_run_and_cache(tmp_path: Path) -> None: + cache = DiskCache(str(tmp_path)) return_value1 = "unique return value" some_fn = Mock(return_value=return_value1) diff --git a/tests/test_parallel.py b/tests/test_parallel.py index 46f2798c..59be47fa 100644 --- a/tests/test_parallel.py +++ b/tests/test_parallel.py @@ -2,42 +2,46 @@ import os import os.path from multiprocessing import Pool +from pathlib import Path +import pytest import responses from tldextract import TLDExtract from tldextract.tldextract import PUBLIC_SUFFIX_LIST_URLS -def test_multiprocessing_makes_one_request(tmpdir): +def test_multiprocessing_makes_one_request(tmp_path: Path) -> None: """Ensure there aren't duplicate download requests.""" process_count = 3 with Pool(processes=process_count) as pool: - http_request_counts = pool.map(_run_extractor, [str(tmpdir)] * process_count) + http_request_counts = pool.map(_run_extractor, [tmp_path] * process_count) assert sum(http_request_counts) == 1 @responses.activate -def _run_extractor(cache_dir): +def _run_extractor(cache_dir: Path) -> int: """Run the extractor.""" responses.add(responses.GET, PUBLIC_SUFFIX_LIST_URLS[0], status=208, body="uk.co") - extract = TLDExtract(cache_dir=cache_dir) + extract = TLDExtract(cache_dir=str(cache_dir)) extract("bar.uk.com", include_psl_private_domains=True) return len(responses.calls) @responses.activate -def test_cache_cleared_by_other_process(tmpdir, monkeypatch): +def test_cache_cleared_by_other_process( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: """Simulate a file being deleted after we check for existence but before we try to delete it.""" responses.add(responses.GET, PUBLIC_SUFFIX_LIST_URLS[0], status=208, body="uk.com") - cache_dir = str(tmpdir) + cache_dir = str(tmp_path) extract = TLDExtract(cache_dir=cache_dir) extract("google.com") orig_unlink = os.unlink - def evil_unlink(filename): + def evil_unlink(filename: str) -> None: """Simulate someone deletes the file right before we try to.""" if filename.startswith(cache_dir): orig_unlink(filename) diff --git a/tox.ini b/tox.ini index 437295f1..8153f715 100644 --- a/tox.ini +++ b/tox.ini @@ -28,6 +28,7 @@ deps = mypy pytest pytest-gitignore + pytest-mock responses types-filelock types-requests