diff --git a/vulnerabilities/helpers.py b/vulnerabilities/helpers.py index 6767f1f3b..90ab9a1f8 100644 --- a/vulnerabilities/helpers.py +++ b/vulnerabilities/helpers.py @@ -27,6 +27,7 @@ from typing import List from typing import Optional from typing import Tuple +from unittest.mock import MagicMock import requests import saneyaml @@ -67,32 +68,8 @@ def fetch_yaml(url): return saneyaml.load(response.content) -# FIXME: this is NOT how etags work . -# We should instead send the proper HTTP header -# https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/If-None-Match -# and integrate this finely in the processing as this typically needs to use -# streaming=True requests, and proper handling of the HTTP return code -# In all cases this ends up being a single request, not a HEADD followed -# by another real request -def create_etag(data_src, url, etag_key): - """ - Etags are like hashes of web responses. For a data source `data_src`, - we maintain (url, etag) mappings in the DB. `create_etag` creates - (`url`, etag) pair. If a (`url`, etag) already exists then the code - skips processing the response further to avoid duplicate work. - - `etag_key` is the name of header which contains the etag for the url. - """ - etag = requests.head(url).headers.get(etag_key) - if not etag: - return True - - elif url in data_src.config.etags: - if data_src.config.etags[url] == etag: - return False - - data_src.config.etags[url] = etag - return True +# FIXME: Remove this entirely after complete importer-improver migration +create_etag = MagicMock() def contains_alpha(string): diff --git a/vulnerabilities/tests/test_helpers.py b/vulnerabilities/tests/test_helpers.py index e17d6ee2b..0ce847e76 100644 --- a/vulnerabilities/tests/test_helpers.py +++ b/vulnerabilities/tests/test_helpers.py @@ -20,43 +20,10 @@ # VulnerableCode is a free software tool from nexB Inc. and others. # Visit https://github.com/nexB/vulnerablecode/ for support and download. -import dataclasses from unittest import TestCase from unittest.mock import patch from unittest.mock import MagicMock -from vulnerabilities.data_source import DataSource -from vulnerabilities.helpers import create_etag - - -@dataclasses.dataclass -class DummyDataSourceConfiguration: - etags: dict - - -class DummyDataSource(DataSource): - CONFIG_CLASS = DummyDataSourceConfiguration - class TestHelpers(TestCase): - @classmethod - def setUpClass(cls): - data_source_cfg = {"etags": {}} - cls.data_source = DummyDataSource(config=data_source_cfg) - - def test_create_etag(self): - assert self.data_source.config.etags == {} - - mock_response = MagicMock() - mock_response.headers = {"ETag": "0x1234"} - - with patch("vulnerabilities.helpers.requests.head", return_value=mock_response): - assert ( - create_etag(data_src=self.data_source, url="https://example.org", etag_key="ETag") - is True - ) - assert self.data_source.config.etags == {"https://example.org": "0x1234"} - assert ( - create_etag(data_src=self.data_source, url="https://example.org", etag_key="ETag") - is False - ) + ...