Merge pull request #34704 from dimagi/gh/mobile-reports/limit-row-count

Set max limit on mobile report size during restores

corehq/apps/app_manager/exceptions.py

@@ -1,5 +1,3 @@
-import couchdbkit
-
 from corehq.apps.app_manager.const import APP_V2
 
 
@@ -79,7 +77,8 @@ def format_v1(self, msg):
             return msg
         # Don't display the first two lines which say "Parsing form..." and 'Title: "{form_name}"'
         #
-        # ... and if possible split the third line that looks like e.g. "org.javarosa.xform.parse.XFormParseException: Select question has no choices"
+        # ... and if possible split the third line that looks like
+        # e.g. "org.javarosa.xform.parse.XFormParseException: Select question has no choices"
         # and just return the undecorated string
         #
         # ... unless the first line says
@@ -194,3 +193,14 @@ class DangerousXmlException(Exception):
 
 class AppMisconfigurationError(AppManagerException):
     """Errors in app configuration that are the user's responsibility"""
+
+
+class CannotRestoreException(Exception):
+    """Errors that inherit from this exception will always fail hard in restores"""
+
+
+class MobileUCRTooLargeException(CannotRestoreException):
+
+    def __init__(self, message, row_count):
+        super().__init__(message)
+        self.row_count = row_count

corehq/apps/app_manager/fixtures/mobile_ucr.py

@@ -21,6 +21,7 @@
     MOBILE_UCR_VERSION_2,
 )
 from corehq.apps.app_manager.dbaccessors import get_apps_in_domain
+from corehq.apps.app_manager.exceptions import CannotRestoreException, MobileUCRTooLargeException
 from corehq.apps.app_manager.suite_xml.features.mobile_ucr import (
     is_valid_mobile_select_filter_type,
 )
@@ -108,8 +109,12 @@ def __call__(self, restore_state):
         ]
 
         for provider in providers:
-            fixtures.extend(provider(restore_state, restore_user, needed_versions, report_configs))
-            self.report_ucr_row_count(provider.row_count, provider.version, restore_user.domain)
+            try:
+                fixtures.extend(provider(restore_state, restore_user, needed_versions, report_configs))
+                self.report_ucr_row_count(provider.row_count, provider.version, restore_user.domain)
+            except MobileUCRTooLargeException as err:
+                self.report_ucr_row_count(err.row_count, provider.version, restore_user.domain)
+                raise
 
         return fixtures
 
@@ -270,6 +275,9 @@ def _v1_fixture(self, restore_user, report_configs, fail_hard=False):
             except UserReportsError:
                 if settings.UNIT_TESTING or settings.DEBUG or fail_hard:
                     raise
+            except CannotRestoreException:
+                # raise regardless of fail_hard
+                raise
             except Exception as err:
                 logging.exception('Error generating report fixture: {}'.format(err))
                 if settings.UNIT_TESTING or settings.DEBUG or fail_hard:
@@ -294,7 +302,7 @@ def _row_to_row_elem(
             return row_elem
 
         row_elements, filters_elem = generate_rows_and_filters(
-            self.report_data_cache, report_config, restore_user, _row_to_row_elem
+            self.report_data_cache, report_config, restore_user, _row_to_row_elem, self.row_count
         )
         # the v1 provider writes all reports to one fixture, so the "effective" row_count is the sum of every
         # report's row_count
@@ -426,6 +434,9 @@ def _v2_fixtures(self, restore_user, report_configs, fail_hard=False):
             except UserReportsError:
                 if settings.UNIT_TESTING or settings.DEBUG or fail_hard:
                     raise
+            except CannotRestoreException:
+                # raise regardless of fail_hard
+                raise
             except Exception as err:
                 logging.exception('Error generating report fixture: {}'.format(err))
                 if settings.UNIT_TESTING or settings.DEBUG or fail_hard:
@@ -482,11 +493,14 @@ def _format_last_sync_time(restore_user, sync_time=None):
     return ServerTime(sync_time).user_time(timezone).done().isoformat()
 
 
-def generate_rows_and_filters(report_data_cache, report_config, restore_user, row_to_element):
+def generate_rows_and_filters(
+    report_data_cache, report_config, restore_user, row_to_element, current_row_count=0
+):
     """Generate restore row and filter elements
     :param row_to_element: function (
                 deferred_fields, filter_options_by_field, row, index, is_total_row
             ) -> row_element
+    :param current_row_count: optional int used by v1 reports provider to accumulate row count across all reports
     """
     report, data_source = report_data_cache.get_report_and_datasource(report_config.report_id)
 
@@ -520,18 +534,27 @@ def generate_rows_and_filters(report_data_cache, report_config, restore_user, ro
         {f.field for f in defer_filters},
         filter_options_by_field,
         row_to_element,
+        current_row_count,
     )
     filters_elem = _get_filters_elem(defer_filters, filter_options_by_field, restore_user._couch_user)
 
     return row_elements, filters_elem
 
 
 def get_report_element(
-        report_data_cache, report_config, data_source, deferred_fields, filter_options_by_field, row_to_element):
+    report_data_cache,
+    report_config,
+    data_source,
+    deferred_fields,
+    filter_options_by_field,
+    row_to_element,
+    current_row_count=0,
+):
     """
     :param row_to_element: function (
                 deferred_fields, filter_options_by_field, row, index, is_total_row
             ) -> row_element
+    :param current_row_count: optional int used by v1 reports provider to accumulate row count across all reports
     """
     if data_source.has_total_row:
         total_row_calculator = IterativeTotalRowCalculator(data_source)
@@ -541,6 +564,18 @@ def get_report_element(
     row_elements = []
     row_index = 0
     rows = report_data_cache.get_data(report_config.uuid, data_source)
+    if len(rows) > settings.MAX_MOBILE_UCR_SIZE:
+        raise MobileUCRTooLargeException(
+            f"Report {report_config.report_id} row count {len(rows)} exceeds max allowed row count "
+            f"{settings.MAX_MOBILE_UCR_SIZE}",
+            row_count=len(rows),
+        )
+    if len(rows) + current_row_count > settings.MAX_MOBILE_UCR_SIZE * 2:
+        raise MobileUCRTooLargeException(
+            "You are attempting to restore too many mobile reports. Your Mobile UCR Restore Version is set to 1.0."
+            " Try upgrading to 2.0.",
+            row_count=len(rows) + current_row_count,
+        )
     for row_index, row in enumerate(rows):
         row_elements.append(row_to_element(deferred_fields, filter_options_by_field, row, row_index))
         total_row_calculator.update_totals(row)

corehq/ex-submodules/casexml/apps/phone/restore.py

@@ -26,6 +26,7 @@
     get_simple_response_xml,
 )
 
+from corehq.apps.app_manager.exceptions import CannotRestoreException
 from corehq.apps.domain.models import Domain
 from corehq.blobs import CODES, get_blob_db
 from corehq.blobs.exceptions import NotFound
@@ -595,6 +596,10 @@ def get_response(self):
             )
             response = HttpResponse(response, content_type="text/xml; charset=utf-8",
                                     status=412)  # precondition failed
+        except CannotRestoreException as e:
+            response = get_simple_response_xml(str(e), ResponseNature.OTA_RESTORE_ERROR)
+            response = HttpResponse(response, content_type="text/xml; charset=utf-8", status=400)
+
         if not is_async:
             self._record_timing(response.status_code)
         return response