Fix memory issue on org results indicators excel report

akvo/rsr/dataclasses.py

@@ -575,6 +575,7 @@ class IndicatorData(object):
     type: int = QUANTITATIVE
     measure: str = ''
     cumulative: bool = False
+    ascending: bool = False
     description: str = ''
     baseline_year: Optional[int] = None
     baseline_value: Optional[Decimal] = None
@@ -591,6 +592,7 @@ def make(cls, data, prefix=''):
             type=data.get(f"{prefix}type", QUANTITATIVE),
             measure=data.get(f"{prefix}measure", ''),
             cumulative=data.get(f"{prefix}cumulative", False),
+            ascending=data.get(f"{prefix}ascending", False),
             description=data.get(f"{prefix}description", ''),
             baseline_year=data.get(f"{prefix}baseline_year", None),
             baseline_value=maybe_decimal(data.get(f"{prefix}baseline_value", None)),
@@ -667,6 +669,44 @@ def get_codelist_name(code, version=settings.IATI_VERSION):
             return ''
 
 
+@dataclass(frozen=True)
+class ProjectData(object):
+    id: int
+    title: str
+    subtitle: str
+    iati_activity_id: str
+    date_start_planned: Optional[date]
+    date_end_planned: Optional[date]
+    date_start_actual: Optional[date]
+    date_end_actual: Optional[date]
+    targets_at: str
+    recipient_countries: List[str] = field(default_factory=list)
+    partners: List[str] = field(default_factory=list)
+    results: List[ResultData] = field(default_factory=list)
+
+    @classmethod
+    def make(cls, data, prefix=''):
+        return cls(
+            id=data[f"{prefix}id"],
+            title=data.get(f"{prefix}title", ''),
+            subtitle=data.get(f"{prefix}subtitle", ''),
+            iati_activity_id=data.get(f"{prefix}iati_activity_id", ''),
+            date_start_planned=data.get(f"{prefix}date_start_planned", None),
+            date_end_planned=data.get(f"{prefix}date_end_planned", None),
+            date_start_actual=data.get(f"{prefix}date_start_actual", None),
+            date_end_actual=data.get(f"{prefix}date_end_actual", None),
+            targets_at=data.get(f"{prefix}targets_at", 'period'),
+        )
+
+    @property
+    def country_codes(self):
+        return ', '.join({it.lower() for it in self.recipient_countries})
+
+    @property
+    def partner_names(self):
+        return ', '.join({it for it in self.partners})
+
+
 class IndicatorType(Enum):
     Quantitative = QUANTITATIVE
     Qualitative = QUALITATIVE

akvo/rsr/tests/views/py_reports/test_cumulative_updates.py

@@ -7,7 +7,6 @@
 from akvo.rsr.usecases.period_update_aggregation import aggregate
 from akvo.rsr.views.py_reports import (
     results_indicators_with_map_pdf_reports,
-    results_indicators_excel_report,
     kickstart_word_report,
     eutf_org_results_table_excel_report,
     program_period_labels_overview_pdf_report,
@@ -149,11 +148,6 @@ def make_project_view(self, project: Project) -> utils.ProjectProxy:
         return results_indicators_with_map_pdf_reports.build_view_object(project, self.PERIOD_2_START, self.PERIOD_3_END)
 
 
-class ResultsIndicatorsExcelReportTestCase(ObjectReaderCumulativeUpdateBaseTestCase, BaseTestCase):
-    def make_project_view(self, project: Project) -> utils.ProjectProxy:
-        return results_indicators_excel_report.build_view_object(project.reporting_org)[0]
-
-
 class KickstartWordReportTestCase(ObjectReaderCumulativeUpdateBaseTestCase, BaseTestCase):
     def make_project_view(self, project: Project) -> utils.ProjectProxy:
         return kickstart_word_report.build_view_object(project, self.PERIOD_2_START, self.PERIOD_3_END)

akvo/rsr/views/py_reports/results_indicators_excel_report.py

@@ -5,25 +5,49 @@
 Akvo RSR module. For additional details on the GNU license please
 see < http://www.gnu.org/licenses/agpl.html >.
 """
+from django.conf import settings
 from django.utils import timezone
+from akvo.rsr.dataclasses import IndicatorData, PeriodData, ProjectData, ResultData
 
-from akvo.rsr.models import Organisation, IndicatorPeriod
+from akvo.rsr.models import Organisation, IndicatorPeriod, RecipientCountry, Partnership, Project
 from akvo.rsr.models.result.utils import PERCENTAGE_MEASURE
 from django.contrib.auth.decorators import login_required
 from django.db.models import Count
 from django.shortcuts import get_object_or_404
 from pyexcelerate import Workbook, Style, Font, Color, Fill, Alignment
 
+from akvo.utils import ensure_decimal
+
 from . import utils
 
+REPORT_NAME = 'organisation_results_indicators_table'
+
+
+@login_required
+def add_email_report_job(request, org_id):
+    organisation = get_object_or_404(Organisation, pk=org_id)
+    payload = {
+        'org_id': organisation.id,
+    }
+    recipient = request.user.email
+    return utils.make_async_email_report_task(handle_email_report, payload, recipient, REPORT_NAME)
+
 
-def build_view_object(organisation):
+def handle_email_report(params, recipient):
+    organisation = Organisation.objects.get(pk=params['org_id'])
+    wb = generate_workbook(organisation)
+    filename = '{}-{}-results-and-indicators-simple-table.xlsx'.format(
+        timezone.now().strftime('%Y%m%d'), organisation.id)
+    utils.send_excel_report(wb, recipient, filename)
+
+
+def fetch_periods(organisation: Organisation):
     project_ids = organisation.all_projects()\
         .annotate(results_count=Count('results'))\
         .filter(results_count__gt=0)\
         .values_list('pk', flat=True)
 
-    periods = IndicatorPeriod.objects\
+    queryset = IndicatorPeriod.objects\
         .select_related('indicator', 'indicator__result', 'indicator__result__project')\
         .filter(indicator__result__project__in=project_ids)\
         .exclude(indicator__result__type__iexact='')\
@@ -36,40 +60,128 @@ def build_view_object(organisation):
             'period_start',
             'period_end'
         )
+    return queryset.values(
+        'id', 'period_start', 'period_end', 'target_value', 'target_comment', 'actual_value', 'actual_comment', 'narrative',
+        'indicator__id', 'indicator__title', 'indicator__type', 'indicator__measure', 'indicator__cumulative',
+        'indicator__ascending', 'indicator__description', 'indicator__baseline_year', 'indicator__baseline_value',
+        'indicator__baseline_comment', 'indicator__target_value', 'indicator__target_comment',
+        'indicator__result__id', 'indicator__result__title', 'indicator__result__description',
+        'indicator__result__type', 'indicator__result__aggregation_status',
+        'indicator__result__project__id', 'indicator__result__project__title', 'indicator__result__project__subtitle',
+        'indicator__result__project__iati_activity_id', 'indicator__result__project__date_start_planned',
+        'indicator__result__project__date_end_planned', 'indicator__result__project__date_start_actual',
+        'indicator__result__project__date_end_actual', 'indicator__result__project__targets_at',
+    )
 
-    return utils.make_project_proxies(periods)
 
+def get_view_objects(organisation):
+    raw_periods = fetch_periods(organisation)
+    lookup = {
+        'projects': {},
+        'results': {},
+        'indicators': {},
+        'periods': {},
+    }
+    for r in raw_periods:
+        project_id = r['indicator__result__project__id']
+        result_id = r['indicator__result__id']
+        indicator_id = r['indicator__id']
+        period_id = r['id']
+        if project_id not in lookup['projects']:
+            lookup['projects'][project_id] = ProjectData.make(r, 'indicator__result__project__')
+        project = lookup['projects'][project_id]
+        if result_id not in lookup['results']:
+            result = ResultData.make(r, 'indicator__result__')
+            project.results.append(result)
+            lookup['results'][result_id] = result
+        else:
+            result = lookup['results'][result_id]
+        result = lookup['results'][result_id]
+        if indicator_id not in lookup['indicators']:
+            indicator = IndicatorData.make(r, 'indicator__')
+            result.indicators.append(indicator)
+            lookup['indicators'][indicator_id] = indicator
+        else:
+            indicator = lookup['indicators'][indicator_id]
+        if period_id not in lookup['periods']:
+            period = PeriodData.make(r)
+            indicator.periods.append(period)
+            lookup['periods'][period_id] = period
+    project_ids = {id for id in lookup['projects'].keys()}
+    recipient_countries = RecipientCountry.objects.filter(project__in=project_ids).values('project', 'country')
+    for recipient in recipient_countries:
+        project = lookup['projects'][recipient['project']]
+        project.recipient_countries.append(recipient['country'])
+    partners = Partnership.objects.filter(project__in=project_ids)\
+        .exclude(organisation__isnull=True)\
+        .values('project', 'organisation__name')
+    for partner in partners:
+        project = lookup['projects'][partner['project']]
+        project.partners.append(partner['organisation__name'])
+    return [it for it in lookup['projects'].values()]
 
-REPORT_NAME = 'organisation_results_indicators_table'
 
+def is_using_indicator_target(projects):
+    for project in projects:
+        program = Project.objects.get(id=project.id).get_program()
+        targets_at = program.targets_at if program else project.targets_at
+        if targets_at == 'indicator':
+            return True
+    return False
 
-@login_required
-def add_email_report_job(request, org_id):
-    organisation = get_object_or_404(Organisation, pk=org_id)
-    payload = {
-        'org_id': organisation.id,
-    }
-    recipient = request.user.email
-    return utils.make_async_email_report_task(handle_email_report, payload, recipient, REPORT_NAME)
 
+def get_eutf_members():
+    try:
+        root = Project.objects.get(id=settings.EUTF_ROOT_PROJECT)
+        return root.descendants().values_list('id', flat=True)
+    except Project.DoesNotExist:
+        return []
 
-def handle_email_report(params, recipient):
-    organisation = Organisation.objects.get(pk=params['org_id'])
-    wb = generate_workbook(organisation)
-    filename = '{}-{}-results-and-indicators-simple-table.xlsx'.format(
-        timezone.now().strftime('%Y%m%d'), organisation.id)
-    utils.send_excel_report(wb, recipient, filename)
+
+def get_period_start(period, project, in_eutf_hierarchy):
+    if not in_eutf_hierarchy:
+        return period.period_start
+
+    if project.id == settings.EUTF_ROOT_PROJECT:
+        return period.period_start
+
+    if project.date_start_actual:
+        return project.date_start_actual
+
+    return project.date_start_planned
+
+
+def get_period_end(period, project, in_eutf_hierarchy):
+    if not in_eutf_hierarchy:
+        return period.period_end
+
+    if project.id == settings.EUTF_ROOT_PROJECT:
+        return period.period_end
+
+    if project.date_end_actual:
+        return project.date_end_actual
+
+    return project.date_end_planned
+
+
+def get_total_period_targets(indicator):
+    total = 0
+    for period in indicator.periods:
+        total += ensure_decimal(period.target_value)
+    return total
+
+
+def get_indicator_target(indicator, targets_at):
+    return ensure_decimal(indicator.target_value) if targets_at == 'indicator' else get_total_period_targets(indicator)
 
 
 def generate_workbook(organisation):
-    projects = build_view_object(organisation)
+    projects = get_view_objects(organisation)
+    use_indicator_target = is_using_indicator_target(projects)
+    eutf_members = get_eutf_members()
 
-    use_indicator_target = False
-    for project in projects:
-        if project.use_indicator_target:
-            print(project.id, project.use_indicator_target)
-            use_indicator_target = True
-            break
+    def in_eutf_hierarchy(project):
+        return project.id in eutf_members
 
     wb = Workbook()
     ws = wb.new_sheet('ProjectList')
@@ -191,19 +303,19 @@ def generate_workbook(organisation):
                     col = 17
                     if use_indicator_target:
                         col += 1
-                        ws.set_cell_value(row, col, indicator.target_value or ' ')
+                        ws.set_cell_value(row, col, get_indicator_target(indicator, project.targets_at) or ' ')
                         col += 1
                         ws.set_cell_value(row, col, indicator.target_comment or ' ')
                     col += 1
-                    ws.set_cell_value(row, col, utils.get_period_start(period, project.in_eutf_hierarchy) or ' ')
+                    ws.set_cell_value(row, col, get_period_start(period, project, in_eutf_hierarchy(project)) or ' ')
                     col += 1
-                    ws.set_cell_value(row, col, utils.get_period_end(period, project.in_eutf_hierarchy) or ' ')
+                    ws.set_cell_value(row, col, get_period_end(period, project, in_eutf_hierarchy(project)) or ' ')
                     if not use_indicator_target:
                         col += 1
                         ws.set_cell_value(row, col, period.target_value or ' ')
                         col += 1
                         ws.set_cell_value(row, col, period.target_comment or ' ')
-                    ws.set_cell_value(row, 22, period.actual_value or ' ')
+                    ws.set_cell_value(row, 22, period.period_actual_value or ' ')
                     ws.set_cell_value(row, 23, period.actual_comment or ' ')
                     ws.set_cell_value(row, 24, project.country_codes or ' ')
                     ws.set_cell_value(row, 25, result.iati_type_name or ' ')