diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0638e39..60dbfef 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -23,9 +23,7 @@ jobs: - id: setup-python uses: actions/setup-python@v4 with: - # A lower version is used because of a dependency issue in Python - # versions 3.8-3.11 - python-version: "3.7" + python-version: "3.10" # We need the Go version and Go cache location for the actions/cache step, # so the Go installation must happen before that. - id: setup-go @@ -112,24 +110,12 @@ jobs: matrix: os: - ubuntu-latest - # The versions of nassl pinned by our sslyze version constraint only - # have bdists available for Python 3.6 and 3.7, so we can only support - # those versions of Python. The error seen when trying to install on - # Python 3.8+ is: - # ERROR: Cannot install pshtt because these package versions have - # conflicting dependencies. - # The conflict is caused by: - # sslyze 2.1.4 depends on nassl<2.3.0 and >=2.2.0 - # sslyze 2.1.3 depends on nassl<2.3.0 and >=2.2.0 python-version: - "3.7" - # - "3.8" - # - "3.9" - # - "3.10" + - "3.8" + - "3.9" + - "3.10" # - "3.11" - include: - - os: ubuntu-20.04 - python-version: "3.6" steps: - uses: actions/checkout@v3 - id: setup-python @@ -179,9 +165,7 @@ jobs: - id: setup-python uses: actions/setup-python@v4 with: - # A lower version is used because of a dependency issue in Python - # versions 3.8-3.11 - python-version: "3.7" + python-version: "3.10" - uses: actions/cache@v3 env: BASE_CACHE_KEY: "${{ github.job }}-${{ runner.os }}-\ @@ -221,20 +205,10 @@ jobs: - ubuntu-latest python-version: - "3.7" - # Disabled due to an unresolvable dependency issue between sslyze and - # nassl: - # ERROR: Cannot install pshtt because these package versions have - # conflicting dependencies. - # The conflict is caused by: - # sslyze 2.1.4 depends on nassl<2.3.0 and >=2.2.0 - # sslyze 2.1.3 depends on nassl<2.3.0 and >=2.2.0 - # - "3.8" - # - "3.9" - # - "3.10" + - "3.8" + - "3.9" + - "3.10" # - "3.11" - include: - - os: ubuntu-20.04 - python-version: "3.6" steps: - uses: actions/checkout@v3 - id: setup-python @@ -281,20 +255,10 @@ jobs: - ubuntu-latest python-version: - "3.7" - # Disabled due to an unresolvable dependency issue between sslyze and - # nassl: - # ERROR: Cannot install pshtt because these package versions have - # conflicting dependencies. - # The conflict is caused by: - # sslyze 2.1.4 depends on nassl<2.3.0 and >=2.2.0 - # sslyze 2.1.3 depends on nassl<2.3.0 and >=2.2.0 - # - "3.8" - # - "3.9" - # - "3.10" + - "3.8" + - "3.9" + - "3.10" # - "3.11" - include: - - os: ubuntu-20.04 - python-version: "3.6" steps: - uses: actions/checkout@v3 - id: setup-python diff --git a/setup.py b/setup.py index ce25344..4bb7498 100644 --- a/setup.py +++ b/setup.py @@ -75,18 +75,14 @@ def get_version(version_file): # that you indicate whether you support Python 2, Python 3 or both. "Programming Language :: Python :: 3", "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", - # "Programming Language :: Python :: 3.8", - # "Programming Language :: Python :: 3.9", - # "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", # "Programming Language :: Python :: 3.11", "Programming Language :: Python :: Implementation :: CPython", ], - # The versions of nassl pinned by our sslyze version constraint only have - # bdists available for cp36 and cp37 on PyPI so we can only support Python - # 3.6 and 3.7 at this time. - python_requires=">=3.6, <3.8", + python_requires=">=3.7", # What does your project relate to? keywords="https best practices", packages=find_packages(where="src"), @@ -103,7 +99,7 @@ def get_version(version_file): "requests>=2.18.4", # This is necessary to support the python_requires kwarg "setuptools >= 24.2.0", - "sslyze>=2.1.3,<3.0.0", + "sslyze>=3.0.0,<5.0.0", "wget>=3.2", ], extras_require={ diff --git a/src/pshtt/pshtt.py b/src/pshtt/pshtt.py index fff7fed..2031a2f 100644 --- a/src/pshtt/pshtt.py +++ b/src/pshtt/pshtt.py @@ -3,9 +3,11 @@ # Standard Python Libraries import base64 import codecs +import datetime import json import logging import os +from pathlib import Path # Python3 import re import sys from urllib import parse as urlparse @@ -15,14 +17,17 @@ from publicsuffixlist.compat import PublicSuffixList # type: ignore from publicsuffixlist.update import updatePSL # type: ignore import requests - -# Unable to find type stubs for the sslyze package. -import sslyze # type: ignore -from sslyze.server_connectivity_tester import ( # type: ignore - ServerConnectivityError, +from sslyze import ( # type: ignore + Scanner, ServerConnectivityTester, + ServerNetworkLocationViaDirectConnection, + ServerScanRequest, +) +from sslyze.errors import ConnectionToServerFailed # type: ignore +from sslyze.plugins.certificate_info.implementation import ( # type: ignore + CertificateInfoExtraArguments, ) -import sslyze.synchronous_scanner # type: ignore +from sslyze.plugins.scan_commands import ScanCommand # type: ignore import urllib3 from . import utils @@ -643,10 +648,15 @@ def https_check(endpoint): # remove the https:// from prefix for sslyze try: hostname = endpoint.url[8:] - server_tester = ServerConnectivityTester(hostname=hostname, port=443) - server_info = server_tester.perform() + server_location = ( + ServerNetworkLocationViaDirectConnection.with_ip_address_lookup( + hostname=hostname, port=443 + ) + ) + server_tester = ServerConnectivityTester() + server_info = server_tester.perform(server_location) endpoint.live = True - ip = server_info.ip_address + ip = server_location.ip_address if endpoint.ip is None: endpoint.ip = ip else: @@ -657,16 +667,16 @@ def https_check(endpoint): endpoint.ip, ip, ) - if server_info.client_auth_requirement.name == "REQUIRED": + if server_info.tls_probing_result.client_auth_requirement.name == "REQUIRED": endpoint.https_client_auth_required = True logging.warning("%s: Client Authentication REQUIRED", endpoint.url) - except ServerConnectivityError as err: + except ConnectionToServerFailed as err: endpoint.live = False endpoint.https_valid = False logging.exception( "%s: Error in sslyze server connectivity check when connecting to %s", endpoint.url, - err.server_info.hostname, + err.server_location.hostname, ) utils.debug("%s: %s", endpoint.url, err) return @@ -680,18 +690,39 @@ def https_check(endpoint): try: cert_plugin_result = None - command = sslyze.plugins.certificate_info_plugin.CertificateInfoScanCommand( - ca_file=CA_FILE - ) - scanner = sslyze.synchronous_scanner.SynchronousScanner() - cert_plugin_result = scanner.run_scan_command(server_info, command) + scanner = Scanner() + command = ScanCommand.CERTIFICATE_INFO + if CA_FILE is not None: + command_extra_args = { + command: CertificateInfoExtraArguments(custom_ca_file=Path(CA_FILE)) + } + scan_request = ServerScanRequest( + server_info=server_info, + scan_commands_extra_arguments=command_extra_args, + scan_commands=[command], + ) + else: + scan_request = ServerScanRequest( + server_info=server_info, scan_commands=[command] + ) + scanner.queue_scan(scan_request) + # Retrieve results from generator object + scan_result = [x for x in scanner.get_results()][0] + cert_plugin_result = scan_result.scan_commands_results[ + ScanCommand.CERTIFICATE_INFO + ] except Exception as err: try: if "timed out" in str(err): logging.exception( "%s: Retrying sslyze scanner certificate plugin.", endpoint.url ) - cert_plugin_result = scanner.run_scan_command(server_info, command) + scanner.queue_scan(scan_request) + # Consume the generator object and retrieve the first result + scan_result = [x for x in scanner.get_results()][0] + cert_plugin_result = scan_result.scan_commands_results[ + ScanCommand.CERTIFICATE_INFO + ] else: logging.exception( "%s: Unknown exception in sslyze scanner certificate plugin.", @@ -716,20 +747,68 @@ def https_check(endpoint): return try: + # Default endpoint assessments to False until proven True. + endpoint.https_expired_cert = False + endpoint.https_self_signed_cert = False + endpoint.https_bad_chain = False + endpoint.https_bad_hostname = False + + # Default trust to Fase until proven True public_trust = True custom_trust = True public_not_trusted_names = [] - validation_results = cert_plugin_result.path_validation_result_list - for result in validation_results: - if result.was_validation_successful: - # We're assuming that it is trusted to start with - pass - else: - if "Custom" in result.trust_store.name: - custom_trust = False + for certificate_deployment in cert_plugin_result.certificate_deployments: + validation_results = certificate_deployment.path_validation_results + for result in validation_results: + if result.was_validation_successful: + # We're assuming that it is trusted to start with + pass else: - public_trust = False - public_not_trusted_names.append(result.trust_store.name) + if "Custom" in result.trust_store.name: + custom_trust = False + else: + public_trust = False + public_not_trusted_names.append(result.trust_store.name) + + if STORE in result.trust_store.name: + cert_chain = result.verified_certificate_chain + leaf_cert = cert_chain[0] + + # Check for leaf certificate expiration/self-signature. + if leaf_cert.not_valid_after < datetime.datetime.now(): + endpoint.https_expired_cert = True + + # Check to see if the cert is self-signed + if leaf_cert.issuer == leaf_cert.subject: + endpoint.https_self_signed_cert = True + + # Check certificate chain till the second last element + # The last cert being the root cert is self signed and + # hence the self signed check is not valid + # NOTE: If this is the only flag that's set, it's probably + # an incomplete chain + # If this isn't the only flag that is set, it might be + # because there is another error. More debugging would + # need to be done at this point, but not through sslyze + # because sslyze doesn't have enough granularity + for cert in cert_chain[:-1]: + # Check for certificate expiration + if cert.not_valid_after < datetime.datetime.now(): + endpoint.https_bad_chain = True + + # Check to see if the cert is self-signed + if cert.issuer == cert.subject or not cert.issuer: + endpoint.https_bad_chain = True + + # If leaf certificate subject does NOT match hostname, bad hostname + # NOTE: Since sslyze 3.0.0, ever since JSON output for certinfo, + # SAN(s) are checked as part of _certificate_matches_hostname which + # called as part of leaf_certificate_subject_matches_hostname + if ( + not certificate_deployment.leaf_certificate_subject_matches_hostname + ): + endpoint.https_bad_hostname = True + if public_trust: logging.warning( "%s: Publicly trusted by common trust stores.", endpoint.url @@ -751,116 +830,73 @@ def https_check(endpoint): endpoint.https_custom_trusted = custom_trust except Exception as err: # Ignore exception - logging.exception("%s: Unknown exception examining trust.", endpoint.url) - utils.debug("%s: Unknown exception examining trust: %s", endpoint.url, err) - - try: - cert_response = cert_plugin_result.as_text() - except AttributeError: logging.exception( - "%s: Known error in sslyze 1.X with EC public keys. See https://github.com/nabla-c0d3/sslyze/issues/215", + "%s: Unknown exception examining certificate deployment.", endpoint.url + ) + utils.debug( + "%s: Unknown exception examining certificate deployment: %s", endpoint.url, + err, ) - return - except Exception as err: - endpoint.unknown_error = True - logging.exception("%s: Unknown exception in cert plugin.", endpoint.url) - utils.debug("%s: %s", endpoint.url, err) - return - - # Debugging - # for msg in cert_response: - # print(msg) - - # Default endpoint assessments to False until proven True. - endpoint.https_expired_cert = False - endpoint.https_self_signed_cert = False - endpoint.https_bad_chain = False - endpoint.https_bad_hostname = False - - # STORE will be either "Mozilla" or "Custom" - # depending on what the user chose. - - # A certificate can have multiple issues. - for msg in cert_response: - # Check for missing SAN. - if (("DNS Subject Alternative Names") in msg) and (("[]") in msg): - endpoint.https_bad_hostname = True - - # Check for certificate expiration. - if ( - (STORE in msg) - and (("FAILED") in msg) - and (("certificate has expired") in msg) - ): - endpoint.https_expired_cert = True - - # Check to see if the cert is self-signed - if ( - (STORE in msg) - and (("FAILED") in msg) - and (("self signed certificate") in msg) - ): - endpoint.https_self_signed_cert = True - - # Check to see if there is a bad chain - - # NOTE: If this is the only flag that's set, it's probably - # an incomplete chain - # If this isnt the only flag that is set, it's might be - # because there is another error. More debugging would - # need to be done at this point, but not through sslyze - # because sslyze doesn't have enough granularity - - if ( - (STORE in msg) - and (("FAILED") in msg) - and ( - (("unable to get local issuer certificate") in msg) - or (("self signed certificate") in msg) - ) - ): - endpoint.https_bad_chain = True - - # Check for whether the hostname validates. - if ( - (("Hostname Validation") in msg) - and (("FAILED") in msg) - and (("Certificate does NOT match") in msg) - ): - endpoint.https_bad_hostname = True try: - endpoint.https_cert_chain_len = len( - cert_plugin_result.received_certificate_chain - ) + endpoint.https_cert_chain_len = 0 + for certificate_deployment in cert_plugin_result.certificate_deployments: + endpoint.https_cert_chain_len += len( + certificate_deployment.received_certificate_chain + ) if endpoint.https_self_signed_cert is False and ( endpoint.https_cert_chain_len < 2 ): # *** TODO check that it is not a bad hostname and that the root cert is trusted before suggesting that it is an intermediate cert issue. endpoint.https_missing_intermediate_cert = True - if cert_plugin_result.verified_certificate_chain is None: + has_verfied_cert_chain = True + for certificate_deployment in cert_plugin_result.certificate_deployments: + if certificate_deployment.verified_certificate_chain is None: + has_verfied_cert_chain = False + if not has_verfied_cert_chain: logging.warning( "%s: Untrusted certificate chain, probably due to missing intermediate certificate.", endpoint.url, ) utils.debug( - "%s: Only %d certificates in certificate chain received.", + "%s: Only %s certificates in certificate chain received.", endpoint.url, - cert_plugin_result.received_certificate_chain.__len__(), + endpoint.https_cert_chain_len, ) elif custom_trust is True and public_trust is False: # recheck public trust using custom public trust store with manually added intermediate certificates if PT_INT_CA_FILE is not None: try: cert_plugin_result = None - command = sslyze.plugins.certificate_info_plugin.CertificateInfoScanCommand( - ca_file=PT_INT_CA_FILE - ) - cert_plugin_result = scanner.run_scan_command( - server_info, command + scanner = Scanner() + command = ScanCommand.CERTIFICATE_INFO + command_extra_args = { + command: CertificateInfoExtraArguments( + custom_ca_file=Path(PT_INT_CA_FILE) + ) + } + scan_request = ServerScanRequest( + server_info=server_info, + scan_commands_extra_arguments=command_extra_args, + scan_commands=[command], ) - if cert_plugin_result.verified_certificate_chain is not None: + scanner.queue_scan(scan_request) + # Consume the generator object and retrieve the first result + scan_result = [x for x in scanner.get_results()][0] + cert_plugin_result = scan_result.scan_commands_results[ + ScanCommand.CERTIFICATE_INFO + ] + has_verfied_cert_chain = True + for ( + certificate_deployment + ) in cert_plugin_result.certificate_deployments: + if ( + certificate_deployment.verified_certificate_chain + is None + ): + has_verfied_cert_chain = False + if has_verfied_cert_chain: public_trust = True endpoint.https_public_trusted = public_trust logging.warning(