diff --git a/src/middleware/overview.middleware.js b/src/middleware/overview.middleware.js index 13ed4653..932f3eda 100644 --- a/src/middleware/overview.middleware.js +++ b/src/middleware/overview.middleware.js @@ -36,6 +36,19 @@ const fetchLatestResources = fetchMany({ dataset: FetchOptions.performanceDb }) +/** + * Middleware. Updates req with 'datasetErrorStatus'. + * + * Fetches datasets which have active endpoints in error state. + */ +const fetchDatasetErrorStatus = fetchMany({ + query: ({ params }) => { + return performanceDbApi.datasetErrorStatusQuery(params.lpa, { datasetsFilter: Object.keys(config.datasetsConfig) }) + }, + result: 'datasetErrorStatus', + dataset: FetchOptions.performanceDb +}) + const fetchProvisions = fetchMany({ query: ({ params }) => { const excludeDatasets = Object.keys(config.datasetsConfig).map(dataset => `'${dataset}'`).join(',') @@ -247,7 +260,7 @@ export function aggregateOverviewData (req, res, next) { * @param next */ export function prepareOverviewTemplateParams (req, res, next) { - const { orgInfo: organisation, provisions, datasets } = req + const { orgInfo: organisation, provisions, datasets, datasetErrorStatus } = req // add in any of the missing key 8 datasets const keys = new Set(datasets.map(d => d.dataset)) availableDatasets.forEach((dataset) => { @@ -263,19 +276,24 @@ export function prepareOverviewTemplateParams (req, res, next) { } }) - const totalDatasets = datasets.length - const [datasetsWithEndpoints, datasetsWithIssues, datasetsWithErrors] = - datasets.reduce(orgStatsReducer, [0, 0, 0]) - const provisionData = new Map() for (const provision of provisions ?? []) { provisionData.set(provision.dataset, provision) } + // we patch the datasets' project (based on provision data) and status (based on its endpoints error status) + const datasetsWithEndpointErrors = new Set(datasetErrorStatus.map(item => item.dataset)) for (const dataset of datasets) { dataset.project = provisionData.get(dataset.dataset)?.project + if (dataset.status !== 'Error' && datasetsWithEndpointErrors.has(dataset.dataset)) { + dataset.status = 'Error' + } } + const totalDatasets = datasets.length + const [datasetsWithEndpoints, datasetsWithIssues, datasetsWithErrors] = + datasets.reduce(orgStatsReducer, [0, 0, 0]) + const datasetsByReason = _.groupBy(datasets, (ds) => { const reason = provisionData.get(ds.dataset)?.provision_reason switch (reason) { @@ -314,6 +332,7 @@ export const getOverview = renderTemplate({ export default [ fetchOrgInfo, fetchLatestResources, + fetchDatasetErrorStatus, handleRejections(fetchEntityCounts), fetchLpaOverview, aggregateOverviewData, diff --git a/src/services/performanceDbApi.js b/src/services/performanceDbApi.js index 442ee179..525ed2af 100644 --- a/src/services/performanceDbApi.js +++ b/src/services/performanceDbApi.js @@ -264,6 +264,25 @@ export default { REPLACE(organisation, '-eng', '') = '${lpa}' ${datasetClause}` }, + /** + * Query for datasets with active endpoints with error status. + * + * @param {string} lpa + * @param {{datasetsFilter: string[]}} params + * @returns {string} SQL + */ + datasetErrorStatusQuery: (lpa, params) => { + return /* sql */ ` + select + dataset + from + provision_summary + where + coalesce("active_endpoint_count", 0) > 1 and coalesce("error_endpoint_count", 0) >= 1 + and "organisation" = '${lpa}' + and "dataset" in (${params.datasetsFilter.map(dataset => `'${dataset}'`).join(',')}) + order by dataset asc` + }, /** * diff --git a/test/unit/middleware/overview.middleware.test.js b/test/unit/middleware/overview.middleware.test.js index efd5f94a..2bcf178a 100644 --- a/test/unit/middleware/overview.middleware.test.js +++ b/test/unit/middleware/overview.middleware.test.js @@ -18,9 +18,52 @@ vi.mock('../../../src/utils/utils.js', async (importOriginal) => { } }) -describe('overview.middleware', () => { - const exampleLpa = { name: 'Example LPA', organisation: 'LPA' } +const exampleLpa = { name: 'Example LPA', organisation: 'LPA' } + +const reqTemplate = { + params: { lpa: 'LPA' }, + orgInfo: exampleLpa, + datasets: [ + { + dataset: 'dataset1', + issue_count: 0, + endpoint: 'https://example.com', + error: undefined, + status: 'Live' + }, + { + dataset: 'dataset2', + issue_count: 0, + endpoint: null, + error: undefined, + status: 'Needs fixing' + }, + { + dataset: 'dataset3', + issue_count: 0, + endpoint: 'https://example.com', + error: undefined, + status: 'Error' + }, + { + dataset: 'dataset4', + issue_count: 0, + endpoint: null, + error: 'There was a 404 error', + status: 'Error' + } + + ], + provisions: [ + { dataset: 'dataset1', provision_reason: 'statutory', project: 'open-digital-planning' }, + { dataset: 'dataset2', provision_reason: 'expected', project: 'open-digital-planning' }, + { dataset: 'dataset3', provision_reason: 'statutory', project: 'open-digital-planning' }, + { dataset: 'dataset4', provision_reason: 'expected', project: 'open-digital-planning' } + ], + datasetErrorStatus: [] +} +describe('overview.middleware', () => { const getRenderedErrorCards = (templateParams) => { const html = nunjucks.render('organisations/overview.html', templateParams) const doc = new jsdom.JSDOM(html).window.document @@ -30,47 +73,7 @@ describe('overview.middleware', () => { describe('prepareOverviewTemplateParams', () => { it('should render the overview page', async () => { - const req = { - params: { lpa: 'LPA' }, - orgInfo: exampleLpa, - datasets: [ - { - dataset: 'dataset1', - issue_count: 0, - endpoint: 'https://example.com', - error: undefined, - status: 'Live' - }, - { - dataset: 'dataset2', - issue_count: 0, - endpoint: null, - error: undefined, - status: 'Needs fixing' - }, - { - dataset: 'dataset3', - issue_count: 0, - endpoint: 'https://example.com', - error: undefined, - status: 'Error' - }, - { - dataset: 'dataset4', - issue_count: 0, - endpoint: null, - error: 'There was a 404 error', - status: 'Error' - } - - ], - provisions: [ - { dataset: 'dataset1', provision_reason: 'statutory', project: 'open-digital-planning' }, - { dataset: 'dataset2', provision_reason: 'expected', project: 'open-digital-planning' }, - { dataset: 'dataset3', provision_reason: 'statutory', project: 'open-digital-planning' }, - { dataset: 'dataset4', provision_reason: 'expected', project: 'open-digital-planning' } - ] - } + const req = structuredClone(reqTemplate) const res = { render: vi.fn() } prepareOverviewTemplateParams(req, res, () => {}) @@ -99,6 +102,23 @@ describe('overview.middleware', () => { expect(errorCardNodes[0].querySelector('.govuk-task-list__hint').textContent.trim()).toBe('There was an error accessing the data URL') expect(errorCardNodes[1].querySelector('.govuk-task-list__hint').textContent.trim()).toBe('There was a 404 error') }) + + it('should patch dataset status based on the provision_summary info', () => { + const req = structuredClone(reqTemplate) + console.assert(req.datasets[0].status === 'Live') + req.datasetErrorStatus = [{ dataset: 'dataset1' }] + const res = { render: vi.fn() } + + prepareOverviewTemplateParams(req, res, () => {}) + + const ds1 = req.templateParams.datasets.statutory[0] + expect(ds1.status).toBe('Error') + expect(ds1.error).toBeUndefined() + + const ds4 = req.templateParams.datasets.other[1] + expect(ds4.status).toBe('Error') + expect(ds4.error).toBe(req.datasets[3].error) // Error message should be left untouched + }) }) describe('aggregateOverviewData middleware', () => { @@ -111,7 +131,7 @@ describe('overview.middleware', () => { }) it('should set req.datasets to just the required datasets when input is empty', async () => { - await aggregateOverviewData(req, res, next) + aggregateOverviewData(req, res, next) expect(req.datasets).toEqual([ { status: 'Not submitted', dataset: 'brownfield-land' } ]) @@ -127,7 +147,7 @@ describe('overview.middleware', () => { req.lpaOverview = exampleData - await aggregateOverviewData(req, res, next) + aggregateOverviewData(req, res, next) expect(req.datasets).toEqual([ { endpoint: 'https://example.com', status: 'Live', dataset: 'dataset1', error: undefined, issue_count: 0 }, @@ -147,7 +167,7 @@ describe('overview.middleware', () => { req.lpaOverview = exampleData - await aggregateOverviewData(req, res, next) + aggregateOverviewData(req, res, next) expect(req.datasets[0].status).toBe('Error') expect(req.datasets[1].status).toBe('Needs fixing') @@ -161,7 +181,7 @@ describe('overview.middleware', () => { req.lpaOverview = exampleData - await aggregateOverviewData(req, res, next) + aggregateOverviewData(req, res, next) expect(req.datasets[0].status).toBe('Needs fixing') expect(req.datasets[0].issue_count).toBe(2) // 2 columns affected @@ -174,7 +194,7 @@ describe('overview.middleware', () => { { endpoint: 'https://example.com/2', status: 'Needs fixing', dataset: 'brownfield-land', entity_count: 5, issue_count: 5, fields: 'foo,bar' } ] req.lpaOverview = exampleData - await aggregateOverviewData(req, res, next) + aggregateOverviewData(req, res, next) expect(req.datasets.length).toEqual(1) }) })