Admin report for domains which should have data source restriction FF enabled

corehq/apps/hqadmin/reports.py

@@ -23,6 +23,11 @@
 from corehq.apps.users.dbaccessors import get_all_user_search_query
 from corehq.const import SERVER_DATETIME_FORMAT
 from corehq.apps.hqadmin.models import HqDeploy
+from corehq.apps.es.cases import CaseES
+from corehq.apps.es.forms import FormES
+from corehq.toggles import USER_CONFIGURABLE_REPORTS, RESTRICT_DATA_SOURCE_REBUILD
+from corehq.motech.repeaters.const import UCRRestrictionFFStatus
+from corehq.apps.es.aggregations import TermsAggregation
 
 
 class AdminReport(GenericTabularReport):
@@ -333,3 +338,160 @@ def _shorten_and_hyperlink_commit(self, commit_sha):
                 abbrev_sha=commit_sha[:7]
             )
         return None
+
+
+class UCRRebuildRestrictionTable:
+    UCR_RESTRICTION_THRESHOLD = 1_000_000
+
+    restriction_ff_status: str
+
+    def __init__(self, *args, **kwargs):
+        self.restriction_ff_status = kwargs.get('restriction_ff_status')
+
+    @property
+    def headers(self):
+        return DataTablesHeader(
+            DataTablesColumn(gettext_lazy("Domain")),
+            DataTablesColumn(gettext_lazy("Case count")),
+            DataTablesColumn(gettext_lazy("Form count")),
+            DataTablesColumn(gettext_lazy("UCR rebuild restriction status")),
+        )
+
+    @property
+    def rows(self):
+        rows = []
+
+        ucr_domains = self.ucr_domains
+        if not ucr_domains:
+            return []
+
+        case_count_by_domain = self._case_count_by_domain(ucr_domains)
+        form_count_by_domain = self._forms_count_by_domain(ucr_domains)
+
+        for domain in ucr_domains:
+            case_count = getattr(case_count_by_domain.get(domain), 'doc_count', 0)
+            form_count = getattr(form_count_by_domain.get(domain), 'doc_count', 0)
+
+            if self.should_show_domain(domain, case_count, form_count):
+                rows.append(
+                    self._row_data(domain, case_count, form_count)
+                )
+
+        return rows
+
+    @property
+    @memoized
+    def ucr_domains(self):
+        return USER_CONFIGURABLE_REPORTS.get_enabled_domains()
+
+    def should_show_domain(self, domain, total_cases, total_forms):
+        if self._show_all_domains:
+            return True
+
+        should_restrict_rebuild = self._should_restrict_rebuild(total_cases, total_forms)
+        restriction_ff_enabled = self._rebuild_restricted_ff_enabled(domain)
+
+        if self._show_ff_enabled_domains:
+            return restriction_ff_enabled
+        if self._show_ff_disabled_domains:
+            return not restriction_ff_enabled
+        if self._show_should_enable_ff_domains:
+            return should_restrict_rebuild and not restriction_ff_enabled
+        if self._show_should_disable_ff_domains:
+            return not should_restrict_rebuild and restriction_ff_enabled
+
+    @staticmethod
+    def _case_count_by_domain(domains):
+        return CaseES().domain(domains).aggregation(
+            TermsAggregation('domain', 'domain.exact')
+        ).run().aggregations.domain.buckets_dict
+
+    @staticmethod
+    def _forms_count_by_domain(domains):
+        return FormES().domain(domains).aggregation(
+            TermsAggregation('domain', 'domain.exact')
+        ).run().aggregations.domain.buckets_dict
+
+    def _row_data(self, domain, case_count, form_count):
+        return [
+            domain,
+            case_count,
+            form_count,
+            self._ucr_rebuild_restriction_status_column_data(domain, case_count, form_count),
+        ]
+
+    def _should_restrict_rebuild(self, case_count, form_count):
+        return case_count >= self.UCR_RESTRICTION_THRESHOLD or form_count >= self.UCR_RESTRICTION_THRESHOLD
+
+    @staticmethod
+    @memoized
+    def _rebuild_restricted_ff_enabled(domain):
+        return RESTRICT_DATA_SOURCE_REBUILD.enabled(domain)
+
+    @property
+    def _show_ff_enabled_domains(self):
+        return self.restriction_ff_status == UCRRestrictionFFStatus.Enabled.name
+
+    @property
+    def _show_ff_disabled_domains(self):
+        return self.restriction_ff_status == UCRRestrictionFFStatus.NotEnabled.name
+
+    @property
+    def _show_should_enable_ff_domains(self):
+        return self.restriction_ff_status == UCRRestrictionFFStatus.ShouldEnable.name
+
+    @property
+    def _show_should_disable_ff_domains(self):
+        return self.restriction_ff_status == UCRRestrictionFFStatus.CanDisable.name
+
+    @property
+    def _show_all_domains(self):
+        return not self.restriction_ff_status
+
+    def _ucr_rebuild_restriction_status_column_data(self, domain, case_count, form_count):
+        from django.utils.safestring import mark_safe
+        from corehq.apps.toggle_ui.views import ToggleEditView
+
+        restriction_ff_enabled = self._rebuild_restricted_ff_enabled(domain)
+        toggle_edit_url = reverse(ToggleEditView.urlname, args=(RESTRICT_DATA_SOURCE_REBUILD.slug,))
+
+        if self._should_restrict_rebuild(case_count, form_count):
+            if not restriction_ff_enabled:
+                return mark_safe(f"""
+                    <a href={toggle_edit_url}>{gettext_lazy("Rebuild restriction required")}</a>
+                """)
+            return gettext_lazy("Rebuild restricted")
+
+        if restriction_ff_enabled:
+            return mark_safe(f"""
+                <a href={toggle_edit_url}>{gettext_lazy("Rebuild restriction not required")}</a>
+            """)
+        return gettext_lazy("No rebuild restriction required")
+
+
+class UCRDataLoadReport(AdminReport):
+    slug = 'ucr_data_load'
+    name = gettext_lazy("UCR Domains Data Report")
+
+    fields = [
+        'corehq.apps.reports.filters.select.UCRRebuildStatusFilter',
+    ]
+    emailable = False
+    exportable = False
+    disable_pagination = True
+    ajax_pagination = False
+    use_datatables = False
+
+    def __init__(self, request, *args, **kwargs):
+        self.table_data = UCRRebuildRestrictionTable(
+            restriction_ff_status=request.GET.get('ucr_rebuild_restriction')
+        )
+        super().__init__(request, *args, **kwargs)
+
+    @property
+    def headers(self):
+        return self.table_data.headers
+
+    @property
+    def rows(self):
+        return self.table_data.rows

corehq/apps/hqadmin/tests/test_reports.py

@@ -0,0 +1,78 @@
+from django.test import TestCase
+from unittest.mock import patch
+
+from corehq.apps.hqadmin.reports import UCRRebuildRestrictionTable
+from corehq.motech.repeaters.const import UCRRestrictionFFStatus
+
+
+class TestUCRRebuildRestrictionTable(TestCase):
+
+    @patch('corehq.apps.hqadmin.reports.USER_CONFIGURABLE_REPORTS.get_enabled_domains')
+    def test_all_ucr_domains(self, get_enabled_domains_mock):
+        ucr_enabled_domains = ['domain1', 'domain2']
+        get_enabled_domains_mock.return_value = ucr_enabled_domains
+
+        table_data = UCRRebuildRestrictionTable()
+
+        self.assertEqual(
+            table_data.ucr_domains,
+            ucr_enabled_domains
+        )
+
+    def test_should_show_domain_default_show_all(self):
+        table_data = UCRRebuildRestrictionTable()
+        self.assertTrue(table_data.should_show_domain)
+
+    @patch.object(UCRRebuildRestrictionTable, '_rebuild_restricted_ff_enabled')
+    def test_should_show_domain_show_ff_enabled_domains(self, restriction_ff_enabled_mock):
+        """ Test domains which does have the FF enabled """
+        restriction_ff_enabled_mock.return_value = True
+
+        table_data = UCRRebuildRestrictionTable(
+            restriction_ff_status=UCRRestrictionFFStatus.Enabled.name,
+        )
+        self.assertTrue(table_data.should_show_domain(
+            domain='domain', total_cases=100_000_000, total_forms=0)
+        )
+
+    @patch.object(UCRRebuildRestrictionTable, '_rebuild_restricted_ff_enabled')
+    def test_should_show_domain_show_ff_disabled_domains(self, restriction_ff_enabled_mock):
+        """ Test domains which does not have the FF enabled """
+        restriction_ff_enabled_mock.return_value = False
+
+        table_data = UCRRebuildRestrictionTable(
+            restriction_ff_status=UCRRestrictionFFStatus.NotEnabled.name,
+        )
+        self.assertTrue(table_data.should_show_domain(
+            domain='domain', total_cases=10, total_forms=0)
+        )
+
+    @patch.object(UCRRebuildRestrictionTable, '_rebuild_restricted_ff_enabled')
+    def test_should_show_domain_show_should_enable_ff_domains(self, restriction_ff_enabled_mock):
+        """ Test domains which does not have the FF enabled but should have it enabled """
+        restriction_ff_enabled_mock.return_value = False
+
+        table_data = UCRRebuildRestrictionTable(
+            restriction_ff_status=UCRRestrictionFFStatus.ShouldEnable.name,
+        )
+        self.assertTrue(table_data.should_show_domain(
+            domain='domain', total_cases=100_000_000, total_forms=0)
+        )
+        self.assertFalse(table_data.should_show_domain(
+            domain='domain', total_cases=10, total_forms=0)
+        )
+
+    @patch.object(UCRRebuildRestrictionTable, '_rebuild_restricted_ff_enabled')
+    def test_should_show_domain_show_should_disable_ff_domains(self, restriction_ff_enabled_mock):
+        """ Test domains which does have the FF enabled but should not have it enabled """
+        restriction_ff_enabled_mock.return_value = True
+
+        table_data = UCRRebuildRestrictionTable(
+            restriction_ff_status=UCRRestrictionFFStatus.CanDisable.name,
+        )
+        self.assertTrue(table_data.should_show_domain(
+            domain='domain', total_cases=10, total_forms=0)
+        )
+        self.assertFalse(table_data.should_show_domain(
+            domain='domain', total_cases=100_000_000, total_forms=0)
+        )

corehq/apps/hqwebapp/tasks.py

@@ -6,6 +6,8 @@
 from django.core.management import call_command
 from django.utils.translation import gettext as _
 from dimagi.utils.django.email import get_email_configuration
+from django.urls import reverse
+from dimagi.utils.web import get_url_base
 
 from celery.exceptions import MaxRetriesExceededError
 from celery.schedules import crontab
@@ -23,6 +25,8 @@
 from corehq.util.metrics import metrics_track_errors
 from corehq.util.models import TransientBounceEmail
 
+from corehq.motech.repeaters.const import UCRRestrictionFFStatus
+
 
 def mark_subevent_gateway_error(messaging_event_id, error, retrying=False):
     from corehq.apps.sms.models import MessagingEvent, MessagingSubEvent
@@ -253,3 +257,31 @@ def clean_expired_transient_emails():
 def clear_expired_oauth_tokens():
     # https://django-oauth-toolkit.readthedocs.io/en/latest/management_commands.html#cleartokens
     call_command('cleartokens')
+
+
+@periodic_task(run_every=crontab(minute=0, hour=1, day_of_week='mon'))
+def send_domain_ucr_data_info_to_admins():
+    from corehq.apps.reports.dispatcher import AdminReportDispatcher
+    from corehq.apps.reports.filters.select import UCRRebuildStatusFilter
+    from corehq.apps.hqadmin.reports import UCRRebuildRestrictionTable, UCRDataLoadReport
+
+    table = UCRRebuildRestrictionTable(
+        restriction_ff_status=UCRRestrictionFFStatus.ShouldEnable.name,
+    )
+    subject = "Weekly report: projects for ucr restriction FF"
+    recipient = "solutions-tech-app-engineers@dimagi.com"
+
+    endpoint = reverse(AdminReportDispatcher.name(), args=(UCRDataLoadReport.slug,))
+
+    filter_name = UCRRebuildStatusFilter.slug
+    filter_value = UCRRestrictionFFStatus.ShouldEnable.name
+    report_url = f"{get_url_base()}{endpoint}?{filter_name}={filter_value}"
+
+    message = f"""
+        We have identified {len(table.rows)} projects that require the RESTRICT_DATA_SOURCE_REBUILD
+        feature flag to be enabled. Please see the detailed report: {report_url}
+    """
+
+    send_mail_async.delay(
+        subject, message, [recipient]
+    )

corehq/apps/reports/filters/checkbox.py

@@ -0,0 +1,5 @@
+from corehq.apps.reports.filters.base import CheckboxFilter
+
+
+class RestrictedDomainsCheckbox(CheckboxFilter):
+    label = "Show only restricted domains"

corehq/apps/reports/filters/select.py

@@ -15,7 +15,7 @@
     BaseMultipleOptionFilter,
     BaseSingleOptionFilter,
 )
-from corehq.motech.repeaters.const import State
+from corehq.motech.repeaters.const import State, UCRRestrictionFFStatus
 from corehq.motech.repeaters.models import Repeater
 
 
@@ -145,3 +145,18 @@ def options(self):
             State.Cancelled,
             State.Fail,
         ]]
+
+
+class UCRRebuildStatusFilter(BaseSingleOptionFilter):
+    slug = "ucr_rebuild_restriction"
+    label = gettext_lazy("Rebuild restriction feature flag status")
+    default_text = gettext_lazy("Show All")
+
+    @property
+    def options(self):
+        return [(s.name, s.label) for s in [
+            UCRRestrictionFFStatus.Enabled,
+            UCRRestrictionFFStatus.NotEnabled,
+            UCRRestrictionFFStatus.ShouldEnable,
+            UCRRestrictionFFStatus.CanDisable,
+        ]]

corehq/apps/reports/tests/test_dispatcher.py

@@ -52,7 +52,8 @@ def test_get_reports_returns_all_admin_reports(self):
             'user_audit_report',
             'device_log_soft_asserts',
             'deploy_history_report',
-            'phone_number_report'
+            'phone_number_report',
+            'ucr_data_load',
         })
 
 

corehq/motech/repeaters/const.py

@@ -32,3 +32,10 @@ class State(IntegerChoices):
 RECORD_FAILURE_STATE = State.Fail
 RECORD_CANCELLED_STATE = State.Cancelled
 RECORD_EMPTY_STATE = State.Empty
+
+
+class UCRRestrictionFFStatus(IntegerChoices):
+    Enabled = 1, _('Is enabled')
+    NotEnabled = 2, _('Is not enabled')
+    ShouldEnable = 3, _('Should be enabled')
+    CanDisable = 4, _('Can be disabled')

corehq/reports.py

@@ -38,6 +38,7 @@
     DeviceLogSoftAssertReport,
     UserAuditReport,
     UserListReport,
+    UCRDataLoadReport,
 )
 from corehq.apps.linked_domain.views import DomainLinkHistoryReport
 from corehq.apps.reports import commtrack
@@ -330,6 +331,7 @@ def EDIT_DATA_INTERFACES(domain_obj):
         AdminPhoneNumberReport,
         UserAuditReport,
         DeployHistoryReport,
+        UCRDataLoadReport,
     )),
 )
 

corehq/tabs/tabclasses.py

@@ -52,6 +52,7 @@
     DeviceLogSoftAssertReport,
     UserAuditReport,
     UserListReport,
+    UCRDataLoadReport,
 )
 from corehq.apps.hqadmin.views.system import GlobalThresholds
 from corehq.apps.hqwebapp.models import GaTracker
@@ -2601,7 +2602,7 @@ def sidebar_items(self):
                     url=reverse('admin_report_dispatcher', args=(report.slug,)),
                     params="?{}".format(urlencode(report.default_params)) if report.default_params else ""
                 )
-            } for report in [DeviceLogSoftAssertReport, UserAuditReport]
+            } for report in [DeviceLogSoftAssertReport, UserAuditReport, UCRDataLoadReport]
         ]))
         return sections