1.2 rc3 (#42)

* Bump all deps, drop support for py36, added support for py37,py38,py39

.github/workflows/main.yml

@@ -22,7 +22,7 @@ jobs:
 
     strategy:
       matrix:
-        python-version: [3.7]
+        python-version: [3.7, 3.8, 3.9]
       fail-fast: false
 
     steps:

VERSION.txt

@@ -1 +1 @@
-1.1-RC-2
+1.2rc-3

bin/vmcctl

@@ -11,21 +11,17 @@ start () {
     case $1 in
        worker)
             echo "Starting VMC Worker"
-            celery worker -A vmc.config.celery  -O fair
+            celery --app vmc.config worker -O fair
             ;;
        scheduler)
             echo "Starting VMC Scheduler"
-            celery -A vmc.config.celery beat --scheduler django_celery_beat.schedulers:DatabaseScheduler  --pidfile=/tmp/scheduler.pid
-            ;;
-       monitor)
-            echo "Starting VMC Monitor"
-            celery flower -A vmc.config.celery --address=0.0.0.0 --port=8080
+            celery --app vmc.config beat --scheduler django_celery_beat.schedulers:DatabaseScheduler  --pidfile=/tmp/scheduler.pid
             ;;
        admin)
             echo "Starting VMC Admin"
             vmc migrate
             vmc create_index
-            gunicorn vmc.config.wsgi:application --bind localhost:8001 --access-logfile '-'
+            gunicorn vmc.config.wsgi:application --bind 0.0.0.0:8001 --access-logfile '-'
             ;;
        *)
             echo "Invalid Option, try: worker scheduler monitor admin"

requirements.txt

@@ -1,19 +1,19 @@
-django==2.2.27
-psycopg2-binary==2.7.7
-celery==4.4.7
-requests==2.21.0
-beautifulsoup4==4.7.1
-flower==0.9.5
-django_celery_beat==1.5.0
-django-redis==4.10.0
-lxml==4.6.5
-defusedxml==0.6.0
-django-celery-results==2.0.0
-pyyaml==5.4
-gunicorn==20.0.4
+django==3.2.12
+psycopg2-binary==2.9.3
+celery==5.2.3
+requests==2.27.1
+beautifulsoup4==4.10.0
+django_celery_beat==2.2.1
+django-redis==5.2.0
+lxml==4.7.1
+defusedxml==0.7.1
+django-celery-results==2.2.0
+pyyaml==6.0
+gunicorn==20.1.0
 awesome-slugify==1.6.5
-elasticsearch_dsl==7.3.0
-python-gvm==20.11.0
+elasticsearch_dsl==7.4.0
+python-gvm==21.11.0
 bootstrap-admin==0.4.4
-netaddr==0.7.19
-djangorestframework==3.11.2
+netaddr==0.8.0
+djangorestframework==3.13.1
+tzdata==2021.5

setup.py

@@ -21,7 +21,7 @@
 import os
 from setuptools import setup, find_packages
 
-assert sys.version_info >= (3, 3), 'Python 3.3+ required.'
+assert sys.version_info >= (3, 7), 'Python 3.7+ required.'
 
 THIS_DIRECTORY = os.path.abspath(os.path.dirname(__file__))
 

src/vmc/common/templates/common/admin/change_list.html

@@ -19,7 +19,7 @@
 -->
 
 {% extends "admin/change_list.html" %}
-{% load i18n admin_static admin_urls %}
+{% load i18n static admin_urls %}
 
 {% block object-tools-items %}
 {{ block.super }}

src/vmc/common/xml.py

@@ -18,11 +18,11 @@
  *
 """
 
-from defusedxml import cElementTree
+from defusedxml import ElementTree
 
 
 def iter_elements_by_name(handle, name: str):
-    events = cElementTree.iterparse(handle, events=("start", "end",))
+    events = ElementTree.iterparse(handle, events=("start", "end",))
     _, root = next(events)  # pylint: disable=stop-iteration-return
     for event, elem in events:
         if event == "end" and elem.tag == name:
@@ -31,5 +31,5 @@ def iter_elements_by_name(handle, name: str):
 
 
 def get_root_element(file):
-    return cElementTree.parse(file).getroot()
+    return ElementTree.parse(file).getroot()
 

src/vmc/config/settings.py

@@ -139,6 +139,7 @@ def get_config(key, default):
 else:
     DATABASES['default']['OPTIONS'] = {'unix_socket': get_config('database.unix_socket', '')}
 
+DEFAULT_AUTO_FIELD = 'django.db.models.AutoField'
 
 AUTH_PASSWORD_VALIDATORS = [
     {

src/vmc/config/test_settings.py

@@ -127,6 +127,8 @@ def elastic_configured():
     }
 }
 
+DEFAULT_AUTO_FIELD = 'django.db.models.AutoField'
+
 # Password validation
 # https://docs.djangoproject.com/en/2.1/ref/settings/#auth-password-validators
 

src/vmc/elasticsearch/__init__.py

@@ -31,5 +31,4 @@ def autodiscover():
 __all__ = ['Document', 'TupleValueField', 'EnumField', 'Keyword', 'InnerDoc', 'Nested',
            'Q', 'Date', 'Float', 'Object', 'Search', 'Integer', 'ListField']
 
-
 default_app_config = 'vmc.elasticsearch.apps.ElasticSearchConfig'

src/vmc/elasticsearch/management/commands/create_index.py

@@ -17,8 +17,9 @@
  * under the License.
  *
 """
-
+from django.conf import settings
 from django.core.management.base import BaseCommand
+from elasticsearch_dsl.connections import connections
 
 from vmc.elasticsearch.registries import registry
 
@@ -27,6 +28,7 @@ class Command(BaseCommand):
     help = 'Creates indexes in ElasticSearch'
 
     def handle(self, *args, **options):
+        connections.configure(**settings.ELASTICSEARCH_DSL)
         documents = registry.get_documents()
         for index in documents:
             documents[index].init(index=index)

src/vmc/elasticsearch/signals.py

@@ -19,4 +19,4 @@
 """
 from django.dispatch import Signal
 
-post_save = Signal(providing_args=['new_version', 'old_version', 'created'])
+post_save = Signal()

src/vmc/knowledge_base/tests.py

@@ -79,10 +79,10 @@ def test_should_create_cwe_entries(self):
         uut = result.hits[0]
 
         self.assertEqual(uut.id, 'CWE-1004')
-        self.assertEquals(uut.name, "Sensitive Cookie Without 'HttpOnly' Flag")
-        self.assertEquals(uut.status, "Incomplete")
-        self.assertEquals(uut.weakness_abstraction, "Variant")
-        self.assertEquals(uut.description, "The software uses a cookie to store sensitive information, "
+        self.assertEqual(uut.name, "Sensitive Cookie Without 'HttpOnly' Flag")
+        self.assertEqual(uut.status, "Incomplete")
+        self.assertEqual(uut.weakness_abstraction, "Variant")
+        self.assertEqual(uut.description, "The software uses a cookie to store sensitive information, "
                                            "but the cookie is not marked with the HttpOnly flag.")
         self.assertEqual(uut.extended_description,
                          "The HttpOnly flag directs compatible browsers to prevent client-side script from accessing "
@@ -144,8 +144,8 @@ def test_call_create(self):
                          'information from process memory via a crafted web site, aka "Internet Explorer Information '
                          'Disclosure Vulnerability." This vulnerability is different from those described in '
                          'CVE-2017-0009 and CVE-2017-0059.')
-        self.assertEquals(str(cve.published_date), '2017-03-17 00:59:00+00:00')
-        self.assertEquals(str(cve.last_modified_date), '2017-07-12 01:29:00+00:00')
+        self.assertEqual(str(cve.published_date), '2017-03-17 00:59:00+00:00')
+        self.assertEqual(str(cve.last_modified_date), '2017-07-12 01:29:00+00:00')
 
         self.assertEqual(cve.cwe.id, 'CWE-200')
         self.assertEqual(len(cve.cpe), 3)

src/vmc/processing/__init__.py

@@ -15,5 +15,5 @@
  * KIND, either express or implied.  See the License for the
  * specific language governing permissions and limitations
  * under the License.
- */
+ *
 """

src/vmc/ralph/__init__.py

@@ -17,4 +17,3 @@
  * under the License.
  *
 """
-default_app_config = 'vmc.ralph.apps.RalphConfig'

src/vmc/scanners/nessus/__init__.py

@@ -17,4 +17,5 @@
  * under the License.
  *
 """
+
 default_app_config = 'vmc.scanners.nessus.apps.NessusConfig'

src/vmc/scanners/nessus/tests.py

@@ -254,31 +254,31 @@ def test_get_scans_ids_with_filter(self):
     def test_parse_call(self):
         parsed, scanned_hosts = self.uut.parse(self.internal_xml, "internal.xml")
         vuln_id = str(uuid.uuid3(uuid.NAMESPACE_OID, '10.31.2.30-tcp-22-70658-CVE-2008-5161'))
-        self.assertEquals(len(parsed), 2)
+        self.assertEqual(len(parsed), 2)
         self.assertIsInstance(parsed[vuln_id], VulnerabilityDocument)
-        self.assertEquals(str(parsed[vuln_id].scan_date), '2020-07-19 11:49:32')
-        self.assertEquals(parsed[vuln_id].asset.ip_address, '10.31.2.30')
-        self.assertEquals(parsed[vuln_id].cve.id, 'CVE-2008-5161')
-        self.assertEquals(parsed[vuln_id].port, '22')
-        self.assertEquals(parsed[vuln_id].svc_name, 'ssh')
-        self.assertEquals(parsed[vuln_id].protocol, 'tcp')
-        self.assertEquals(parsed[vuln_id].tenant, None)
-        self.assertEquals(parsed[vuln_id].name, 'SSH Server CBC Mode Ciphers Enabled')
-        self.assertEquals(parsed[vuln_id].solution, 'Contact the vendor or consult product documentation to disable CBC mode '
+        self.assertEqual(str(parsed[vuln_id].scan_date), '2020-07-19 11:49:32')
+        self.assertEqual(parsed[vuln_id].asset.ip_address, '10.31.2.30')
+        self.assertEqual(parsed[vuln_id].cve.id, 'CVE-2008-5161')
+        self.assertEqual(parsed[vuln_id].port, '22')
+        self.assertEqual(parsed[vuln_id].svc_name, 'ssh')
+        self.assertEqual(parsed[vuln_id].protocol, 'tcp')
+        self.assertEqual(parsed[vuln_id].tenant, None)
+        self.assertEqual(parsed[vuln_id].name, 'SSH Server CBC Mode Ciphers Enabled')
+        self.assertEqual(parsed[vuln_id].solution, 'Contact the vendor or consult product documentation to disable CBC mode '
                                         'cipher encryption, and enable CTR or GCM cipher mode encryption.')
-        self.assertEquals(parsed[vuln_id].scan_file_url, "internal.xml")
+        self.assertEqual(parsed[vuln_id].scan_file_url, "internal.xml")
         self.assertIn('The SSH server is configured to support Cipher Block Chaining (CBC)', parsed[vuln_id].description)
 
-        self.assertEquals(1, len(scanned_hosts))
-        self.assertEquals(scanned_hosts[0].ip_address, '10.31.2.30')
+        self.assertEqual(1, len(scanned_hosts))
+        self.assertEqual(scanned_hosts[0].ip_address, '10.31.2.30')
         self.assertEqual(str(scanned_hosts[0].last_scan_date), '2020-07-19 11:49:32')
 
 
         vuln_id = str(uuid.uuid3(uuid.NAMESPACE_OID, '10.31.2.30-tcp-23-42263-NESSUS-42263'))
         self.assertIsInstance(parsed[vuln_id], VulnerabilityDocument)
-        self.assertEquals(parsed[vuln_id].asset.ip_address, '10.31.2.30')
-        self.assertEquals(parsed[vuln_id].asset.mac_address, '3E:CE:D5:62:DF:E2')
-        self.assertEquals(parsed[vuln_id].cve.id, 'NESSUS-42263')
+        self.assertEqual(parsed[vuln_id].asset.ip_address, '10.31.2.30')
+        self.assertEqual(parsed[vuln_id].asset.mac_address, '3E:CE:D5:62:DF:E2')
+        self.assertEqual(parsed[vuln_id].cve.id, 'NESSUS-42263')
         self.assertEqual(parsed[vuln_id].cve.base_score_v3, 6.5)
 
         self.assertEqual(parsed[vuln_id].cve.attack_vector_v3, metrics.AttackVectorV3.NETWORK)

src/vmc/scanners/openvas/__init__.py

@@ -17,4 +17,5 @@
  * under the License.
  *
 """
+
 default_app_config = 'vmc.scanners.openvas.apps.OpenVasConfig'