684 issue details fetch data from all active resources

src/controllers/OrganisationsController.js

@@ -1,6 +1,7 @@
 import datasetTaskListMiddleware from '../middleware/datasetTaskList.middleware.js'
 import datasetOverviewMiddleware from '../middleware/datasetOverview.middleware.js'
-import issueDetailsMiddleware from '../middleware/issueDetails.middleware.js'
+import entityIssueDetailsMiddleware from '../middleware/entityIssueDetails.middleware.js'
+import entryIssueDetailsMiddleware from '../middleware/entryIssueDetails.middleware.js'
 import issueTableMiddleware from '../middleware/issueTable.middleware.js'
 import organisationsMiddleware from '../middleware/organisations.middleware.js'
 import getStartedMiddleware from '../middleware/getStarted.middleware.js'
@@ -11,7 +12,8 @@ const organisationsController = {
   organisationsMiddleware,
   datasetTaskListMiddleware,
   datasetOverviewMiddleware,
-  issueDetailsMiddleware,
+  entityIssueDetailsMiddleware,
+  entryIssueDetailsMiddleware,
   issueTableMiddleware,
   getStartedMiddleware,
   overviewMiddleware,

src/middleware/common.middleware.js

@@ -187,9 +187,10 @@ export const createPaginationTemplateParams = (req, res, next) => {
 
 export const fetchResources = fetchMany({
   query: ({ req }) => `
-    select * from resource r
+    SELECT r.end_date, r.entry_date, r.mime_type, r.resource, r.start_date, rle.endpoint_url, rle.licence, rle.status, rle.latest_log_entry_date, rle.endpoint_entry_date from resource r
     LEFT JOIN resource_organisation ro ON ro.resource = r.resource
     LEFT JOIN resource_dataset rd ON rd.resource = r.resource
+    LEFT JOIN reporting_latest_endpoints rle ON r.resource = rle.resource
     WHERE ro.organisation = '${req.params.lpa}'
     AND rd.dataset = '${req.params.dataset}'
     AND r.end_date = ''
@@ -353,6 +354,16 @@ export const processEntitiesMiddlewares = [
   replaceUnderscoreInEntities
 ]
 
+export const filterOutEntitiesWithoutIssues = (req, res, next) => {
+  const { entities, issues } = req
+
+  req.issueEntities = entities.filter(entity => {
+    return issues.findIndex(issue => issue.entity === entity.entity) >= 0
+  })
+
+  next()
+}
+
 // entity issues
 
 const fetchEntityIssuesForFieldAndType = fetchMany({
@@ -492,6 +503,18 @@ export const addFieldMappingsToIssue = (req, res, next) => {
   next()
 }
 
+export const addReferencesToIssues = (req, res, next) => {
+  const { issues, entities } = req
+
+  req.issues = issues.map(issue => {
+    const reference = entities.find(entity => entity.entity === issue.entity)?.reference
+
+    return { ...issue, reference }
+  })
+
+  next()
+}
+
 /**
  * This middleware chain is responsible for retrieving all entities for the given organisation, their latest issues,
  * filtering out issues that have been fixed, and constructing the table params.
@@ -509,7 +532,8 @@ export const processRelevantIssuesMiddlewares = [
   FilterOutIssuesToMostRecent,
   removeIssuesThatHaveBeenFixed,
   fetchFieldMappings,
-  addFieldMappingsToIssue
+  addFieldMappingsToIssue,
+  addReferencesToIssues
 ]
 
 // Other
@@ -525,3 +549,111 @@ export const setDefaultParams = (req, res, next) => {
 
   next()
 }
+
+export const getSetBaseSubPath = (additionalParts = []) => (req, res, next) => {
+  const params = [
+    req.params.lpa,
+    req.params.dataset,
+    req.params.issue_type,
+    req.params.issue_field,
+    ...additionalParts.map(encodeURIComponent)
+  ]
+
+  req.baseSubpath = params.reduce(
+    (path, param) => (param ? `${path}/${param}` : path),
+    '/organisations'
+  )
+  next()
+}
+
+export const getSetDataRange = (pageLength) => (req, res, next) => {
+  const { recordCount } = req
+  const { pageNumber } = req.parsedParams
+
+  if (typeof recordCount !== 'number' || recordCount < 0) {
+    return next(new Error('Invalid record count'))
+  }
+  if (typeof pageNumber !== 'number' || pageNumber < 1) {
+    return next(new Error('Invalid page number'))
+  }
+  if (typeof pageLength !== 'number' || pageLength < 1) {
+    return next(new Error('Invalid page length'))
+  }
+
+  req.dataRange = {
+    minRow: (pageNumber - 1) * pageLength,
+    maxRow: Math.min((pageNumber - 1) * pageLength + pageLength, recordCount),
+    totalRows: recordCount,
+    maxPageNumber: Math.ceil(recordCount / pageLength),
+    pageLength,
+    offset: (pageNumber - 1) * pageLength
+  }
+  next()
+}
+
+export function getErrorSummaryItems (req, res, next) {
+  const { issue_type: issueType, issue_field: issueField } = req.params
+  const { baseSubpath, issues, issueCount, entities, resources } = req
+
+  let errorHeading = ''
+  let issueItems
+
+  const totalRecordCount = entities ? entities.length : resources[0].entry_count
+  const totalIssues = issueCount?.count || issues.length
+
+  if (issues.length <= 0) {
+    // currently the task list page is getting its issues incorrectly, not factoring in the fact that an issue might have been fixed.
+    logger.warn(`entry issue details was accessed from ${req.headers.referer} but there was no issues`)
+    const error = new Error('issue count must be larger than 0')
+    return next(error)
+  } else if (issues.length < totalRecordCount) {
+    errorHeading = performanceDbApi.getTaskMessage({ issue_type: issueType, num_issues: totalIssues, entityCount: totalRecordCount, field: issueField }, true)
+    issueItems = issues.map((issue, i) => {
+      const pageNum = i + 1
+      let inString = ''
+
+      // we have to hard code this in because though the entitiy has beeb assigned, it has been assigned incorrectly
+      const specialIssueTypeCases = ['reference values are not unique']
+
+      if (issue.reference && !specialIssueTypeCases.includes(issue.issue_type)) {
+        inString = ` in entity with reference ${issue.reference}`
+      } else if (issue.line_number) {
+        inString = ` on row ${issue.line_number}`
+      }
+      return {
+        html: performanceDbApi.getTaskMessage({ issue_type: issueType, num_issues: 1, field: issueField }) + inString,
+        href: `${baseSubpath}/${pageNum}`
+      }
+    })
+  } else {
+    issueItems = [{
+      html: performanceDbApi.getTaskMessage({ issue_type: issueType, num_issues: totalIssues, entityCount: totalRecordCount, field: issueField }, true)
+    }]
+  }
+
+  req.errorSummary = {
+    heading: errorHeading,
+    items: issueItems
+  }
+
+  next()
+}
+
+export const prepareIssueDetailsTemplateParams = (req, res, next) => {
+  const { entry, pagination, dataRange, errorSummary, dataset, orgInfo } = req
+  const { issue_type: issueType, pageNumber } = req.parsedParams
+
+  // schema: OrgIssueDetails
+  req.templateParams = {
+    organisation: orgInfo,
+    dataset,
+    errorSummary,
+    entry,
+    issueType,
+    pagination,
+    pageNumber,
+    dataRange
+  }
+
+  next()
+}

src/middleware/dataview.middleware.js

@@ -1,4 +1,5 @@
-import { createPaginationTemplateParams, extractJsonFieldFromEntities, fetchDatasetInfo, fetchLatestResource, fetchLpaDatasetIssues, fetchOrgInfo, isResourceAccessible, isResourceIdInParams, processSpecificationMiddlewares, replaceUnderscoreInEntities, setDefaultParams, takeResourceIdFromParams, validateQueryParams, show404IfPageNumberNotInRange } from './common.middleware.js'
+import config from '../../config/index.js'
+import { createPaginationTemplateParams, extractJsonFieldFromEntities, fetchDatasetInfo, fetchLatestResource, fetchLpaDatasetIssues, fetchOrgInfo, isResourceAccessible, isResourceIdInParams, processSpecificationMiddlewares, replaceUnderscoreInEntities, setDefaultParams, takeResourceIdFromParams, validateQueryParams, show404IfPageNumberNotInRange, getSetBaseSubPath, getSetDataRange } from './common.middleware.js'
 import { fetchResourceStatus } from './datasetTaskList.middleware.js'
 import { fetchIf, fetchMany, fetchOne, FetchOptions, renderTemplate } from './middleware.builders.js'
 import * as v from 'valibot'
@@ -26,25 +27,8 @@ export const fetchEntities = fetchMany({
   result: 'entities'
 })
 
-export const setBaseSubpath = (req, res, next) => {
-  const { lpa, dataset } = req.params
-  req.baseSubpath = `/organisations/${encodeURIComponent(lpa)}/${encodeURIComponent(dataset)}/data`
-  next()
-}
-
-export const getDataRange = (req, res, next) => {
-  const { entityCount } = req
-  const { pageNumber } = req.parsedParams
-  const pageLength = 50
-  const recordCount = entityCount.count
-  req.dataRange = {
-    minRow: (pageNumber - 1) * pageLength,
-    maxRow: Math.min((pageNumber - 1) * pageLength + pageLength, recordCount),
-    totalRows: recordCount,
-    maxPageNumber: Math.ceil(recordCount / pageLength),
-    pageLength,
-    offset: (pageNumber - 1) * pageLength
-  }
+export const setRecordCount = (req, res, next) => {
+  req.recordCount = req?.entityCount?.count || 0
   next()
 }
 
@@ -118,12 +102,13 @@ export default [
   validatedataviewQueryParams,
   setDefaultParams,
 
-  setBaseSubpath,
+  getSetBaseSubPath(['data']),
   fetchOrgInfo,
   fetchDatasetInfo,
   fetchResourceStatus,
   fetchEntitiesCount,
-  getDataRange,
+  setRecordCount,
+  getSetDataRange(config.tablePageLength),
   show404IfPageNumberNotInRange,
 
   fetchIf(isResourceIdInParams, fetchLatestResource, takeResourceIdFromParams),

src/middleware/entityIssueDetails.middleware.js

@@ -0,0 +1,135 @@
+import { issueErrorMessageHtml } from '../utils/utils.js'
+import {
+  fetchDatasetInfo,
+  fetchOrgInfo,
+  logPageError,
+  validateQueryParams,
+  createPaginationTemplateParams,
+  show404IfPageNumberNotInRange,
+  fetchResources,
+  processRelevantIssuesMiddlewares,
+  processEntitiesMiddlewares,
+  processSpecificationMiddlewares,
+  getSetBaseSubPath,
+  getSetDataRange,
+  getErrorSummaryItems,
+  prepareIssueDetailsTemplateParams,
+  filterOutEntitiesWithoutIssues
+} from './common.middleware.js'
+import { renderTemplate } from './middleware.builders.js'
+import * as v from 'valibot'
+
+export const IssueDetailsQueryParams = v.object({
+  lpa: v.string(),
+  dataset: v.string(),
+  issue_type: v.string(),
+  issue_field: v.string(),
+  pageNumber: v.optional(v.pipe(v.string(), v.transform(s => parseInt(s, 10)), v.minValue(1)), '1'),
+  resourceId: v.optional(v.string())
+})
+
+const validateIssueDetailsQueryParams = validateQueryParams({
+  schema: IssueDetailsQueryParams
+})
+
+/**
+ *
+ * @param {*} text
+ * @param {*} html
+ * @param {*} classes
+ * @returns {{key: {text: string}, value: { html: string}, classes: string}}
+ */
+export const getIssueField = (text, html, classes) => {
+  classes = classes || ''
+  return {
+    key: {
+      text
+    },
+    value: {
+      html: html ? html.toString() : ''
+    },
+    classes
+  }
+}
+
+export const setRecordCount = (req, res, next) => {
+  req.recordCount = req?.issueEntities?.length || 0
+  next()
+}
+
+export function prepareEntity (req, res, next) {
+  const { issueEntities, issues, specification } = req
+  const { pageNumber, issue_type: issueType } = req.parsedParams
+
+  const entityData = issueEntities[pageNumber - 1]
+  const entityIssues = issues.filter(issue => issue.entity === entityData.entity)
+
+  const fields = specification.fields.map(({ field, datasetField }) => {
+    const value = entityData[field] || entityData[datasetField]
+
+    return getIssueField(field, value)
+  })
+
+  entityIssues.forEach(issue => {
+    const field = fields.find(field => field.key.text === issue.field)
+    if (field) {
+      const message = issue.message || issue.type
+      field.value.html = issueErrorMessageHtml(message, null) + field.value.html
+      field.classes += 'dl-summary-card-list__row--error govuk-form-group--error'
+    }
+  })
+
+  for (const issue of entityIssues) {
+    if (!fields.find((field) => field.key.text === issue.field)) {
+      const errorMessage = issue.message || issueType
+      // TODO: pull the html out of here and into the template
+      const valueHtml = issueErrorMessageHtml(errorMessage, issue.value)
+      const classes = 'dl-summary-card-list__row--error govuk-form-group--error'
+
+      fields.push(getIssueField(issue.field, valueHtml, classes))
+    }
+  }
+
+  const geometries = fields
+    .filter((row) => row.field === 'geometry')
+    .map((row) => row.value)
+
+  req.entry = {
+    title: entityData.name || `entity: ${entityData.entity}`,
+    fields,
+    geometries
+  }
+
+  next()
+}
+
+/**
+ * Middleware. Renders the issue details page with the list of issues, entry data,
+ * and organisation and dataset details.
+ */
+export const getIssueDetails = renderTemplate({
+  templateParams: (req) => req.templateParams,
+  template: 'organisations/issueDetails.html',
+  handlerName: 'getIssueDetails'
+})
+
+export default [
+  validateIssueDetailsQueryParams,
+  fetchOrgInfo,
+  fetchDatasetInfo,
+  fetchResources,
+  ...processEntitiesMiddlewares,
+  ...processRelevantIssuesMiddlewares,
+  ...processSpecificationMiddlewares,
+  filterOutEntitiesWithoutIssues,
+  setRecordCount,
+  getSetDataRange(1),
+  show404IfPageNumberNotInRange,
+  getSetBaseSubPath(['entity']),
+  createPaginationTemplateParams,
+  getErrorSummaryItems,
+  prepareEntity,
+  prepareIssueDetailsTemplateParams,
+  getIssueDetails,
+  logPageError
+]