enable CI on python 3.10 (#2367)

* update github actions

* update pillow

* update websockets

* fix cache

* bump pillow

* fix media test

* disable links versioning

* fix pillow

* disble versioning

* fix flake8

* remove print

* remove unused import

* unlock mypy requirements versions

* fix black

* set debug=False in testing

* disable DEBUG when testing

it raisers an error when reusing existing app instance

* revert spike changes

* fix mypy

* set black python version to py38

* update pytest config

to get all tests

* fix failing test

* clean up app factory

.github/workflows/nose-tests.yml

@@ -3,147 +3,85 @@ name: "Test"
 on: [push, pull_request]
 
 jobs:
-  nose:
+  pytest:
     runs-on: ubuntu-latest
 
     strategy:
       matrix:
-        python-version: [3.8]
+        python-version: ['3.8', '3.10']
 
     steps:
-    - uses: actions/checkout@v2
-
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
+    - uses: actions/checkout@v3
+    - uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python-version }}
 
-    - name: Install packages
-      run: ./scripts/tests_setup
-
-    - name: Cache pip
-      uses: actions/cache@v2
-      with:
-        path: ~/.cache/pip
-        key: ${{ runner.os }}-pip-${{ hashFiles('dev-requirements.txt') }}
-        restore-keys: |
-          ${{ runner.os }}-pip
-          ${{ runner.os }}-
-
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade 'pip<21.3' wheel setuptools
-        pip install -r dev-requirements.txt
+    - run: ./scripts/tests_setup
+    - run: pip install -U pip wheel setuptools
+    - run: pip install -r dev-requirements.txt
 
-    - name: Nosetests
-      run: nosetests --logging-level=ERROR
+    - run: pytest --log-level=ERROR --disable-warnings
 
   behave:
     runs-on: ubuntu-latest
 
     strategy:
       matrix:
-        python-version: [3.8]
+        python-version: ['3.8', '3.10']
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
 
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
+    - uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python-version }}
 
-    - name: Install packages
-      run: ./scripts/tests_setup
-
-    - name: Cache pip
-      uses: actions/cache@v2
-      with:
-        path: ~/.cache/pip
-        key: ${{ runner.os }}-pip-${{ hashFiles('dev-requirements.txt') }}
-        restore-keys: |
-          ${{ runner.os }}-pip
-          ${{ runner.os }}-
-
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade 'pip<21.3' wheel setuptools
-        pip install -r dev-requirements.txt
+    - run: ./scripts/tests_setup
+    - run: pip install -U pip wheel setuptools
+    - run: pip install -r dev-requirements.txt
 
-    - name: Behave
-      run: behave --tags=-sams --format progress2 --logging-level=ERROR
+    - run: behave --tags=-sams --format progress2 --logging-level=ERROR
 
   behave-sams:
 
     runs-on: ubuntu-latest
 
     strategy:
       matrix:
-        python-version: [3.8]
+        python-version: ['3.8', '3.10']
 
     steps:
-      - uses: actions/checkout@v2
-
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
-        with:
-          python-version: ${{ matrix.python-version }}
-
-      - name: Install packages
-        run: ./scripts/tests_setup
+    - uses: actions/checkout@v3
 
-      - name: Start SAMS
-        run: ./scripts/install-start-sams
+    - uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.python-version }}
 
-      - name: Cache pip
-        uses: actions/cache@v2
-        with:
-          path: ~/.cache/pip
-          key: ${{ runner.os }}-pip-${{ hashFiles('dev-requirements.txt') }}
-          restore-keys: |
-            ${{ runner.os }}-pip
-            ${{ runner.os }}-
+    - run: ./scripts/tests_setup
+    - run: ./scripts/install-start-sams
 
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade 'pip<21.3' wheel setuptools
-          pip install -r dev-requirements.txt
+    - run: pip install -U pip wheel setuptools
+    - run: pip install -r dev-requirements.txt
 
-      - name: Behave
-        run: behave --tags=sams --format progress2 --logging-level=ERROR
+    - run: behave --tags=sams --format progress2 --logging-level=ERROR
 
   prodapi:
 
     runs-on: ubuntu-latest
 
     strategy:
       matrix:
-        python-version: [3.8]
+        python-version: ['3.8', '3.10']
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
 
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
+    - uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python-version }}
 
-    - name: Install packages
-      run: ./scripts/tests_setup
+    - run: ./scripts/tests_setup
+    - run: pip install -U pip wheel setuptools
+    - run: pip install -r dev-requirements.txt
 
-    - name: Cache pip
-      uses: actions/cache@v2
-      with:
-        path: ~/.cache/pip
-        key: ${{ runner.os }}-pip-${{ hashFiles('dev-requirements.txt') }}
-        restore-keys: |
-          ${{ runner.os }}-pip
-          ${{ runner.os }}-
-
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade 'pip<21.3' wheel setuptools
-        pip install -r dev-requirements.txt
-
-    - name: prodapi-pytest
-      run: cd prod_api && time pytest
+    - run: pytest prod_api

apps/archive/archive.py

@@ -268,6 +268,7 @@ class ArchiveVersionsResource(Resource):
     internal_resource = True
     privileges = {"PATCH": "archive"}
     collation = False
+    versioning = False
     mongo_indexes = {
         "guid": ([("guid", 1)], {"background": True}),
         "_id_document_1": ([("_id_document", 1)], {"background": True}),

apps/auth/session_purge.py

@@ -9,11 +9,11 @@
 # at https://www.sourcefabric.org/superdesk/license
 
 import superdesk
-from superdesk import app
 from datetime import timedelta
 from superdesk.utc import utcnow
 from eve.utils import date_to_str
 from superdesk import get_resource_service
+from flask import current_app as app
 import logging
 
 logger = logging.getLogger(__name__)

apps/content_filters/content_filter/content_filter_test.py

@@ -9,10 +9,11 @@
 # at https://www.sourcefabric.org/superdesk/license
 import json
 
+from flask import current_app as app
 from eve.utils import ParsedRequest
 from flask_babel import _
 
-from superdesk import get_resource_service, app
+from superdesk import get_resource_service
 from superdesk.resource import Resource
 from superdesk.services import BaseService
 from superdesk.errors import SuperdeskApiError

apps/content_filters/filter_condition/filter_condition_parameters.py

@@ -15,8 +15,9 @@
 from superdesk.resource import Resource
 from superdesk.services import BaseService
 from superdesk.utils import ListCursor
-from superdesk import get_resource_service, config, app
+from superdesk import get_resource_service, config
 from superdesk.io.subjectcodes import get_subjectcodeitems
+from flask import current_app as app
 from eve.utils import ParsedRequest
 
 

apps/content_filters/filter_condition/filter_condition_resource.py

@@ -9,8 +9,9 @@
 # at https://www.sourcefabric.org/superdesk/license
 
 import copy
-from superdesk import get_resource_service, config, app
+from superdesk import get_resource_service, config
 from superdesk.resource import Resource
+from flask import current_app as app
 
 
 default_allowed_filters = [

apps/io/feeding_services/reuters.py

@@ -23,8 +23,6 @@
 from urllib.parse import urlparse, urlunparse
 from flask_babel import _
 
-requests.packages.urllib3.disable_warnings()
-
 
 class ReutersHTTPFeedingService(HTTPFeedingService):
     """

apps/links.py

@@ -45,6 +45,7 @@ def elastic_filter(req=None):
 
 class LinksResource(ArchiveResource):
     item_methods = []
+    versioning = False
     resource_methods = ["GET"]
     datasource = ArchiveResource.datasource.copy()
     datasource.update(

apps/prepopulate/__init__.py

@@ -17,7 +17,7 @@
 
 
 def init_app(app) -> None:
-    if superdesk.app.config.get("SUPERDESK_TESTING", False):
+    if app.config.get("SUPERDESK_TESTING", False):
         endpoint_name = "prepopulate"
         service = PrepopulateService(endpoint_name, backend=superdesk.get_backend())
         PrepopulateResource(endpoint_name, app=app, service=service)

apps/prepopulate/app_prepopulate.py

@@ -25,7 +25,7 @@
 from superdesk.metadata.item import ITEM_STATE, CONTENT_STATE
 from superdesk.resource import Resource
 from superdesk.services import BaseService
-from superdesk.tests import clean_dbs, use_snapshot
+from superdesk.tests import clean_dbs
 from superdesk.utc import utcnow
 from superdesk.timer import timer
 from apps.search_providers import allowed_search_providers, register_search_provider

apps/publish/content/resend.py

@@ -35,6 +35,8 @@ class ResendResource(ArchiveResource):
     resource_methods = ["POST"]
     privileges = {"POST": "resend"}
 
+    versioning = False
+
 
 class ResendService(Service):
 

content_api/behave_setup.py

@@ -26,7 +26,6 @@ def get_test_settings():
     test_settings["CONTENTAPI_ELASTICSEARCH_INDEX"] = TEST_DBNAME
     test_settings["MONGO_DBNAME"] = TEST_DBNAME
     test_settings["MONGO_URI"] = "mongodb://localhost/%s" % TEST_DBNAME
-    test_settings["DEBUG"] = True
     test_settings["TESTING"] = True
     test_settings["SUPERDESK_TESTING"] = True
     test_settings["CONTENT_EXPIRY_MINUTES"] = 99

dev-requirements.txt

@@ -1,8 +1,7 @@
 flake8
+flake8-bugbear
 flake8-docstrings
-flake8-bugbear==19.8.0
 nose
-nose-exclude
 pep8
 pyflakes
 pydocstyle<4.0
@@ -11,8 +10,8 @@ watchdog==0.9.0
 wooper==0.4.4
 requests-mock==1.8.0
 responses
-pytest==5.2.2
-pytest-env==0.6.2
+pytest
+pytest-env
 python3-saml>=1.9,<1.10
 typing_extensions>=3.7.4
 moto[sqs]

mypy-requirements.txt

@@ -1,18 +1,18 @@
-mypy==0.950
-mypy-extensions==0.4.3
-toml==0.10.2
-types-chardet==0.1.1
-types-click==0.1.14
-types-croniter==0.1.2
-types-Flask==0.1.2
-types-Jinja2==0.1.3
-types-MarkupSafe==0.1.5
-types-python-dateutil==0.1.2
-types-pytz==0.1.0
-types-PyYAML==0.1.5
-types-redis==0.1.17
-types-requests==0.1.8
-types-tzlocal==0.1.0
-types-Werkzeug==0.1.8
-typing-extensions==3.10.0.0
+mypy
+mypy-extensions
+toml
+types-chardet
+types-click
+types-croniter
+types-Flask
+types-Jinja2
+types-MarkupSafe
+types-python-dateutil
+types-pytz
+types-PyYAML
+types-redis
+types-requests
+types-tzlocal
+types-Werkzeug
+typing-extensions
 boto3-stubs[s3,sqs]

prod_api/conftest.py

@@ -27,7 +27,6 @@ def get_test_prodapi_app(extra_config=None):
     :return: eve.flaskapp.Eve
     """
     test_config = {
-        "DEBUG": True,
         "TESTING": True,
         "SUPERDESK_TESTING": True,
         "MONGO_CONNECT": False,

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.black]
 line-length = 120
-target-version = ['py36']
+target-version = ['py38']
 include = '\.py$'
 exclude = '''
 (
@@ -15,3 +15,7 @@ exclude = '''
     | env
 )
 '''
+
+[tool.pytest.ini_options]
+testpaths = ["tests", "superdesk", "apps"]
+python_files = "*_test.py *_tests.py test_*.py tests_*.py tests.py test.py"

setup.cfg

@@ -4,6 +4,7 @@ extend-exclude = env,bin,lib,include,src,docs,dist,build,node_modules
 ignore = F811,D200,D202,D205,D400,D401,D100,D101,D102,D103,D104,D105,D107,W503,W504,W605,F401,E261,F841,
          B010,B009,B007,B305,B011,
          E203,E501  # black compatibility
+         B023,B020,B017,B019,  # update to flake8 5
 # W504, W605, F401, E261 and F841 are temporarly ignored, due to recent changes in flake8
 
 [extract_messages]