Set dataset status to 'Error' if any active endpoints have 'Error' status

src/middleware/overview.middleware.js

@@ -36,6 +36,19 @@ const fetchLatestResources = fetchMany({
   dataset: FetchOptions.performanceDb
 })
 
+/**
+ * Middleware. Updates req with 'datasetErrorStatus'.
+ *
+ * Fetches datasets which have active endpoints in error state.
+ */
+const fetchDatasetErrorStatus = fetchMany({
+  query: ({ params }) => {
+    return performanceDbApi.datasetErrorStatusQuery(params.lpa, { datasetsFilter: Object.keys(config.datasetsConfig) })
+  },
+  result: 'datasetErrorStatus',
+  dataset: FetchOptions.performanceDb
+})
+
 const fetchProvisions = fetchMany({
   query: ({ params }) => {
     const excludeDatasets = Object.keys(config.datasetsConfig).map(dataset => `'${dataset}'`).join(',')
@@ -247,7 +260,7 @@ export function aggregateOverviewData (req, res, next) {
  * @param next
  */
 export function prepareOverviewTemplateParams (req, res, next) {
-  const { orgInfo: organisation, provisions, datasets } = req
+  const { orgInfo: organisation, provisions, datasets, datasetErrorStatus } = req
   // add in any of the missing key 8 datasets
   const keys = new Set(datasets.map(d => d.dataset))
   availableDatasets.forEach((dataset) => {
@@ -263,19 +276,24 @@ export function prepareOverviewTemplateParams (req, res, next) {
     }
   })
 
-  const totalDatasets = datasets.length
-  const [datasetsWithEndpoints, datasetsWithIssues, datasetsWithErrors] =
-      datasets.reduce(orgStatsReducer, [0, 0, 0])
-
   const provisionData = new Map()
   for (const provision of provisions ?? []) {
     provisionData.set(provision.dataset, provision)
   }
 
+  // we patch the datasets' project (based on provision data) and status (based on its endpoints error status)
+  const datasetsWithEndpointErrors = new Set(datasetErrorStatus.map(item => item.dataset))
   for (const dataset of datasets) {
     dataset.project = provisionData.get(dataset.dataset)?.project
+    if (dataset.status !== 'Error' && datasetsWithEndpointErrors.has(dataset.dataset)) {
+      dataset.status = 'Error'
+    }
   }
 
+  const totalDatasets = datasets.length
+  const [datasetsWithEndpoints, datasetsWithIssues, datasetsWithErrors] =
+      datasets.reduce(orgStatsReducer, [0, 0, 0])
+
   const datasetsByReason = _.groupBy(datasets, (ds) => {
     const reason = provisionData.get(ds.dataset)?.provision_reason
     switch (reason) {
@@ -314,6 +332,7 @@ export const getOverview = renderTemplate({
 export default [
   fetchOrgInfo,
   fetchLatestResources,
+  fetchDatasetErrorStatus,
   handleRejections(fetchEntityCounts),
   fetchLpaOverview,
   aggregateOverviewData,

src/services/performanceDbApi.js

@@ -264,6 +264,25 @@ export default {
       REPLACE(organisation, '-eng', '') = '${lpa}'
       ${datasetClause}`
   },
+  /**
+   * Query for datasets with active endpoints with error status.
+   *
+   * @param {string} lpa
+   * @param {{datasetsFilter: string[]}} params
+   * @returns {string} SQL
+   */
+  datasetErrorStatusQuery: (lpa, params) => {
+    return /* sql */ `
+    select 
+      dataset
+    from 
+      provision_summary 
+    where 
+      "active_endpoint_count" > 1 and "error_endpoint_count" >= 1
+      and "organisation" = '${lpa}'
+      and "dataset" in (${params.datasetsFilter.map(dataset => `'${dataset}'`).join(',')})
+    order by dataset asc`
+  },
 
   /**
     *

test/unit/middleware/overview.middleware.test.js

@@ -18,9 +18,52 @@ vi.mock('../../../src/utils/utils.js', async (importOriginal) => {
   }
 })
 
-describe('overview.middleware', () => {
-  const exampleLpa = { name: 'Example LPA', organisation: 'LPA' }
+const exampleLpa = { name: 'Example LPA', organisation: 'LPA' }
+
+const reqTemplate = {
+  params: { lpa: 'LPA' },
+  orgInfo: exampleLpa,
+  datasets: [
+    {
+      dataset: 'dataset1',
+      issue_count: 0,
+      endpoint: 'https://example.com',
+      error: undefined,
+      status: 'Live'
+    },
+    {
+      dataset: 'dataset2',
+      issue_count: 0,
+      endpoint: null,
+      error: undefined,
+      status: 'Needs fixing'
+    },
+    {
+      dataset: 'dataset3',
+      issue_count: 0,
+      endpoint: 'https://example.com',
+      error: undefined,
+      status: 'Error'
+    },
+    {
+      dataset: 'dataset4',
+      issue_count: 0,
+      endpoint: null,
+      error: 'There was a 404 error',
+      status: 'Error'
+    }
+
+  ],
+  provisions: [
+    { dataset: 'dataset1', provision_reason: 'statutory', project: 'open-digital-planning' },
+    { dataset: 'dataset2', provision_reason: 'expected', project: 'open-digital-planning' },
+    { dataset: 'dataset3', provision_reason: 'statutory', project: 'open-digital-planning' },
+    { dataset: 'dataset4', provision_reason: 'expected', project: 'open-digital-planning' }
+  ],
+  datasetErrorStatus: []
+}
 
+describe('overview.middleware', () => {
   const getRenderedErrorCards = (templateParams) => {
     const html = nunjucks.render('organisations/overview.html', templateParams)
     const doc = new jsdom.JSDOM(html).window.document
@@ -30,47 +73,7 @@ describe('overview.middleware', () => {
 
   describe('prepareOverviewTemplateParams', () => {
     it('should render the overview page', async () => {
-      const req = {
-        params: { lpa: 'LPA' },
-        orgInfo: exampleLpa,
-        datasets: [
-          {
-            dataset: 'dataset1',
-            issue_count: 0,
-            endpoint: 'https://example.com',
-            error: undefined,
-            status: 'Live'
-          },
-          {
-            dataset: 'dataset2',
-            issue_count: 0,
-            endpoint: null,
-            error: undefined,
-            status: 'Needs fixing'
-          },
-          {
-            dataset: 'dataset3',
-            issue_count: 0,
-            endpoint: 'https://example.com',
-            error: undefined,
-            status: 'Error'
-          },
-          {
-            dataset: 'dataset4',
-            issue_count: 0,
-            endpoint: null,
-            error: 'There was a 404 error',
-            status: 'Error'
-          }
-
-        ],
-        provisions: [
-          { dataset: 'dataset1', provision_reason: 'statutory', project: 'open-digital-planning' },
-          { dataset: 'dataset2', provision_reason: 'expected', project: 'open-digital-planning' },
-          { dataset: 'dataset3', provision_reason: 'statutory', project: 'open-digital-planning' },
-          { dataset: 'dataset4', provision_reason: 'expected', project: 'open-digital-planning' }
-        ]
-      }
+      const req = structuredClone(reqTemplate)
       const res = { render: vi.fn() }
 
       prepareOverviewTemplateParams(req, res, () => {})
@@ -99,6 +102,23 @@ describe('overview.middleware', () => {
       expect(errorCardNodes[0].querySelector('.govuk-task-list__hint').textContent.trim()).toBe('There was an error accessing the data URL')
       expect(errorCardNodes[1].querySelector('.govuk-task-list__hint').textContent.trim()).toBe('There was a 404 error')
     })
+
+    it('should patch dataset status based on the provision_summary info', () => {
+      const req = structuredClone(reqTemplate)
+      console.assert(req.datasets[0].status === 'Live')
+      req.datasetErrorStatus = [{ dataset: 'dataset1' }]
+      const res = { render: vi.fn() }
+
+      prepareOverviewTemplateParams(req, res, () => {})
+
+      const ds1 = req.templateParams.datasets.statutory[0]
+      expect(ds1.status).toBe('Error')
+      expect(ds1.error).toBeUndefined()
+
+      const ds4 = req.templateParams.datasets.other[1]
+      expect(ds4.status).toBe('Error')
+      expect(ds4.error).toBe(req.datasets[3].error) // Error message should be left untouched
+    })
   })
 
   describe('aggregateOverviewData middleware', () => {
@@ -111,7 +131,7 @@ describe('overview.middleware', () => {
     })
 
     it('should set req.datasets to just the required datasets when input is empty', async () => {
-      await aggregateOverviewData(req, res, next)
+      aggregateOverviewData(req, res, next)
       expect(req.datasets).toEqual([
         { status: 'Not submitted', dataset: 'brownfield-land' }
       ])
@@ -127,7 +147,7 @@ describe('overview.middleware', () => {
 
       req.lpaOverview = exampleData
 
-      await aggregateOverviewData(req, res, next)
+      aggregateOverviewData(req, res, next)
 
       expect(req.datasets).toEqual([
         { endpoint: 'https://example.com', status: 'Live', dataset: 'dataset1', error: undefined, issue_count: 0 },
@@ -147,7 +167,7 @@ describe('overview.middleware', () => {
 
       req.lpaOverview = exampleData
 
-      await aggregateOverviewData(req, res, next)
+      aggregateOverviewData(req, res, next)
 
       expect(req.datasets[0].status).toBe('Error')
       expect(req.datasets[1].status).toBe('Needs fixing')
@@ -161,7 +181,7 @@ describe('overview.middleware', () => {
 
       req.lpaOverview = exampleData
 
-      await aggregateOverviewData(req, res, next)
+      aggregateOverviewData(req, res, next)
 
       expect(req.datasets[0].status).toBe('Needs fixing')
       expect(req.datasets[0].issue_count).toBe(2) // 2 columns affected
@@ -174,7 +194,7 @@ describe('overview.middleware', () => {
         { endpoint: 'https://example.com/2', status: 'Needs fixing', dataset: 'brownfield-land', entity_count: 5, issue_count: 5, fields: 'foo,bar' }
       ]
       req.lpaOverview = exampleData
-      await aggregateOverviewData(req, res, next)
+      aggregateOverviewData(req, res, next)
       expect(req.datasets.length).toEqual(1)
     })
   })