Merge remote-tracking branch 'origin/release/2.5-fidelity' into develop

content_api/api_audit/service.py

@@ -9,7 +9,7 @@
 # at https://www.sourcefabric.org/superdesk/license
 
 from superdesk.services import BaseService
-from flask import g, request
+from flask import g, request, current_app as app
 from eve.utils import config
 
 
@@ -25,17 +25,31 @@ def audit_item(self, item, id):
         :param id: id of the item
         :return:
         """
+        doc = item.copy()
+        doc["_id"] = id
+        self._audit_docs([doc])
+
+    def audit_items(self, items):
+        self._audit_docs(items)
+
+    def _audit_docs(self, docs):
+        if not len(docs):
+            return
+        if not app.config.get("CONTENTAPI_AUDIT", True):
+            return
         subscriber = getattr(g, "user", None)
         # in behave testing we get user (dict)
         if isinstance(subscriber, dict):
             subscriber = subscriber.get(config.ID_FIELD)
-
-        audit = {
-            "type": item.get("type", ""),
-            "subscriber": subscriber,
-            "uri": item.get("uri", None),
-            "items_id": id,
-            "version": item.get("version", ""),
-            "remote_addr": request.remote_addr,
-        }
-        self.post([audit])
+        audit_docs = [
+            {
+                "type": item.get("type", ""),
+                "subscriber": subscriber,
+                "uri": item.get("uri", None),
+                "items_id": item["_id"],
+                "version": item.get("version", ""),
+                "remote_addr": request.remote_addr,
+            }
+            for item in docs
+        ]
+        self.post(audit_docs)

content_api/app/settings.py

@@ -15,7 +15,7 @@
 `documentation <http://python-eve.org/config.html#global-configuration>`_.
 """
 
-from superdesk.default_settings import env, urlparse
+from superdesk.default_settings import env, urlparse, strtobool
 
 from superdesk.default_settings import (  # noqa
     DEBUG,
@@ -64,3 +64,9 @@
 DATE_FORMAT = "%Y-%m-%dT%H:%M:%S+0000"
 ELASTIC_DATE_FORMAT = "%Y-%m-%d"
 BCRYPT_GENSALT_WORK_FACTOR = 12
+
+#: audit subscriber activity
+#:
+#: .. versionadded:: 2.5
+#:
+CONTENTAPI_AUDIT = strtobool(env("CONTENTAPI_AUDIT", "true"))

content_api/items/service.py

@@ -170,8 +170,11 @@ def on_fetched(self, result):
         :param dict result: dictionary contaning the list of MongoDB documents
             (the fetched items) and some metadata, e.g. pagination info
         """
+        # we have to audit first while we have the item id
+        get_resource_service("api_audit").audit_items(result["_items"])
+
         for document in result["_items"]:
-            self._process_fetched_object(document)
+            self._process_fetched_object(document, audit=False)
 
         if "_links" in result:  # might not be present if HATEOAS disabled
             url_parts = urlparse(request.url)
@@ -241,7 +244,7 @@ def _is_internal_api(self):
         """
         return self.datasource == "items" or self.datasource == "packages"
 
-    def _process_fetched_object(self, document):
+    def _process_fetched_object(self, document, audit=True):
         """Does some processing on the raw document fetched from database.
 
         It sets the item's `uri` field and removes all the fields added by the
@@ -260,7 +263,9 @@ def _process_fetched_object(self, document):
 
         self._process_item_renditions(document)
         self._process_item_associations(document)
-        get_resource_service("api_audit").audit_item(document, _id)
+
+        if audit:
+            get_resource_service("api_audit").audit_item(document, _id)
 
     def _process_item_renditions(self, item):
         hrefs = {}

content_api/search/service.py

@@ -121,7 +121,7 @@ def get(self, req, lookup):
             return self._map_response(response)
         return response
 
-    def _process_fetched_object(self, document):
+    def _process_fetched_object(self, document, **kwargs):
         """Does some processing on the raw document fetched from database.
 
         It sets the item's `uri` field and removes all the fields added by the

content_api/tests/items_service_test.py

@@ -20,12 +20,16 @@
 from superdesk.tests import TestCase
 from superdesk.utc import utcnow
 from superdesk import get_resource_service
+from content_api.api_audit import ApiAuditService
 
 
-class FakeAuditService:
+class FakeAuditService(ApiAuditService):
     def audit_item(self, doc, id):
         return
 
+    def audit_items(self, items):
+        return
+
 
 class FakeAuditResource:
     service = None

superdesk/default_settings.py

@@ -519,6 +519,13 @@ def local_to_utc_hour(hour):
 #: media storage provider
 MEDIA_STORAGE_PROVIDER = env("MEDIA_STORAGE_PROVIDER")
 
+#: avoid checking if media exists
+#:
+#: it should only check during migration, otherwise this generates unnecessary load on the s3
+#:
+#: .. versionadded:: 2.5
+PROXY_MEDIA_STORAGE_CHECK_EXISTS = strtobool(env("PROXY_MEDIA_STORAGE_CHECK_EXISTS", "false"))
+
 #: uses for generation of media url ``(<media_prefix>/<media_id>)``::
 MEDIA_PREFIX = env("MEDIA_PREFIX", "%s/upload-raw" % SERVER_URL.rstrip("/"))
 MEDIA_PREFIXES_TO_FIX = None

superdesk/storage/proxy.py

@@ -37,10 +37,12 @@ def __init__(self, app):
             # make amazon first if configured, so it will be the default
             self._storage.insert(0, AmazonMediaStorage(app))
 
+        self.check_exists = app.config.get("PROXY_MEDIA_STORAGE_CHECK_EXISTS", False)
+
     def storage(self, id_or_filename=None, resource=None, fallback=False) -> SuperdeskMediaStorage:
         if id_or_filename:
             for storage in self._storage:
-                if storage.exists(id_or_filename, resource):
+                if not self.check_exists or storage.exists(id_or_filename, resource):
                     logger.debug("got media from storage id=%s storage=%s", id_or_filename, storage)
                     return storage
             if not fallback:

tests/storage/proxy_test.py

@@ -20,6 +20,7 @@ def setUp(self):
             )
         )
         self.storage = ProxyMediaStorage(self.app)
+        self.storage.check_exists = True
         self.storage._storage = [
             create_autospec(SuperdeskGridFSMediaStorage),
             create_autospec(AmazonMediaStorage),
@@ -61,3 +62,12 @@ def test_amazon_get_by_filename_mongo(self):
         with patch.object(amazon, "get") as get_mock:
             amazon.get_by_filename("{}.jpg".format(str(_id)))
             get_mock.assert_called_once_with(str(_id))
+
+    def test_url_for_media(self):
+        self.storage.check_exists = False
+        self.storage._storage[0].exists.return_value = False
+        self.storage._storage[0].url_for_media.return_value = "first"
+        self.storage._storage[1].exists.return_value = True
+        self.storage._storage[1].url_for_media.return_value = "second"
+        href = self.storage.url_for_media("medid")
+        self.assertEqual(href, "first")