remove unneeded middleware for getting entity issues

got the tasklist page working with all active endpoints
digital-land · Dec 11, 2024 · 381095a · 381095a
1 parent 8c89de9
commit 381095a
Show file tree

Hide file tree

Showing 5 changed files with 95 additions and 137 deletions.
diff --git a/src/middleware/common.middleware.js b/src/middleware/common.middleware.js
@@ -1,5 +1,6 @@
 import logger from '../utils/logger.js'
 import { types } from '../utils/logging.js'
+import { entryIssueGroups } from '../utils/utils.js'
 import performanceDbApi from '../services/performanceDbApi.js'
 import { fetchOne, FetchOptions, FetchOneFallbackPolicy, fetchMany, renderTemplate } from './middleware.builders.js'
 import * as v from 'valibot'
@@ -187,7 +188,7 @@ export const createPaginationTemplateParams = (req, res, next) => {
 
 export const fetchResources = fetchMany({
   query: ({ req }) => `
-    SELECT r.end_date, r.entry_date, r.mime_type, r.resource, r.start_date, rle.endpoint_url, rle.licence, rle.status, rle.latest_log_entry_date, rle.endpoint_entry_date from resource r
+    SELECT DISTINCT r.end_date, r.entry_date, r.mime_type, r.resource, r.start_date, rle.endpoint_url, rle.licence, rle.status, rle.latest_log_entry_date, rle.endpoint_entry_date from resource r
     LEFT JOIN resource_organisation ro ON ro.resource = r.resource
     LEFT JOIN resource_dataset rd ON rd.resource = r.resource
     LEFT JOIN reporting_latest_endpoints rle ON r.resource = rle.resource
@@ -368,91 +369,24 @@ export const filterOutEntitiesWithoutIssues = (req, res, next) => {
 
 const fetchEntityIssuesForFieldAndType = fetchMany({
   query: ({ req, params }) => {
-    const issueTypeFilter = params.issue_type ? `AND issue_type = '${params.issue_type}'` : ''
-    const issueFieldFilter = params.issue_field ? `AND field = '${params.issue_field}'` : ''
-
+    const issueTypeClause = params.issue_type ? `AND i.issue_type = '${params.issue_type}'` : ''
+    const issueFieldClause = params.issue_field ? `AND field = '${params.issue_field}'` : ''
     return `
-      SELECT e.entity, i.* FROM entity e
-      INNER JOIN issue i ON e.entity = i.entity
-      WHERE e.organisation_entity = ${req.orgInfo.entity}
-      ${issueTypeFilter}
-      ${issueFieldFilter}`
+        select * 
+        from issue i
+        LEFT JOIN issue_type it ON i.issue_type = it.issue_type
+        WHERE resource = '${req.resources[0].resource}'
+        ${issueTypeClause}
+        AND it.responsibility = 'external'
+        AND it.severity = 'error'
+        ${issueFieldClause}
+        AND entity != ''
+        `
+    // LIMIT ${req.dataRange.pageLength} OFFSET ${req.dataRange.offset}
   },
-  dataset: FetchOptions.fromParams,
   result: 'issues'
 })
 
-export const FilterOutIssuesToMostRecent = (req, res, next) => {
-  const { resources, issues } = req
-
-  const issuesWithResources = issues.filter(issue => {
-    if (!issue.resource || !resources.find(resource => resource.resource === issue.resource)) {
-      logger.warn(`Missing resource on issue: ${JSON.stringify(issue)}`)
-      return false
-    }
-    return true
-  })
-
-  const groupedIssues = issuesWithResources.reduce((acc, current) => {
-    current.start_date = new Date(resources.find(resource => resource.resource === current.resource)?.start_date)
-    const { entity, field } = current
-    if (!acc[entity]) {
-      acc[entity] = {}
-    }
-    if (!acc[entity][field]) {
-      acc[entity][field] = []
-    }
-    acc[entity][field].push(current)
-
-    return acc
-  }, {})
-
-  const recentIssues = Object.fromEntries(Object.entries(groupedIssues).map(([entityName, issuesByEntity]) =>
-    [
-      entityName,
-      Object.fromEntries(Object.entries(issuesByEntity).map(([field, issues]) => [
-        field,
-        issues.sort((a, b) => b.start_date.getTime() - a.start_date.getTime())[0]
-      ]))
-    ]
-  ))
-
-  const issuesFlattened = []
-
-  Object.values(recentIssues).forEach(issueByEntry => {
-    Object.values(issueByEntry).forEach(issueByField => {
-      issuesFlattened.push(issueByField)
-    })
-  })
-
-  req.issues = issuesFlattened
-  next()
-}
-
-export const fetchIssueTypes = fetchMany({
-  query: () => 'SELECT * FROM issue_type',
-  result: 'issueTypes'
-})
-
-export const addIssueSeverityToIssues = (req, res, next) => {
-  const { issues, issueTypes } = req
-
-  req.issues = issues.map(issue => {
-    const issueTypeData = issueTypes.find(issueType => issueType.issue_type === issue.issue_type)
-    return { ...issue, severity: issueTypeData.severity }
-  })
-
-  next()
-}
-
-const filterOutNonErrorSeverityIssues = (req, res, next) => {
-  const { issues } = req
-
-  req.issues = issues.filter(issue => issue.severity === 'error')
-
-  next()
-}
-
 export const removeIssuesThatHaveBeenFixed = async (req, res, next) => {
   const { issues, resources } = req
 
@@ -527,17 +461,50 @@ export const addFieldMappingsToIssue = (req, res, next) => {
   next()
 }
 
-export const addReferencesToIssues = (req, res, next) => {
-  const { issues, entities } = req
-
-  req.issues = issues.map(issue => {
-    const reference = entities.find(entity => entity.entity === issue.entity)?.reference
+// We can only get the issues without entity from the latest resource as we have no way of knowing if those in previous resources have been fixed?
+export const fetchEntryIssues = fetchMany({
+  query: ({ req, params }) => {
+    const issueTypeClause = params.issue_type ? `AND i.issue_type = '${params.issue_type}'` : ''
+    const issueFieldClause = params.issue_field ? `AND field = '${params.issue_field}'` : ''
+    return `
+      select * 
+      from issue i
+      LEFT JOIN issue_type it ON i.issue_type = it.issue_type
+      WHERE resource = '${req.resources[0].resource}'
+      ${issueTypeClause}
+      AND it.responsibility = 'external'
+      AND it.severity = 'error'
+      ${issueFieldClause}
+      AND (entity = '' OR i.issue_type in ('${entryIssueGroups.map(issue => issue.type).join("', '")}'))
+      LIMIT ${req.dataRange.pageLength} OFFSET ${req.dataRange.offset}
+    `
+  },
+  result: 'entryIssues'
+})
 
-    return { ...issue, reference }
-  })
+export const fetchEntityIssueCounts = fetchMany({
+  query: ({ req }) => `
+    select field, i.issue_type, COUNT(resource+line_number) as count
+    from issue i
+    LEFT JOIN issue_type it ON i.issue_type = it.issue_type
+    WHERE resource in ('${req.resources.map(resource => resource.resource).join("', '")}')
+    AND entity != ''
+    GROUP BY field, i.issue_type
+  `,
+  result: 'entityIssueCounts'
+})
 
-  next()
-}
+export const fetchEntryIssueCounts = fetchMany({
+  query: ({ req }) => `
+    select field, i.issue_type, COUNT(resource+line_number) as count
+    from issue i
+    LEFT JOIN issue_type it ON i.issue_type = it.issue_type
+    WHERE resource =  '${req.resources[0].resource}'
+    AND entity = ''
+    GROUP BY field, i.issue_type
+  `,
+  result: 'entryIssueCounts'
+})
 
 /**
  * This middleware chain is responsible for retrieving all entities for the given organisation, their latest issues,
@@ -553,14 +520,11 @@ export const addReferencesToIssues = (req, res, next) => {
  */
 export const processRelevantIssuesMiddlewares = [
   fetchEntityIssuesForFieldAndType,
-  FilterOutIssuesToMostRecent,
-  fetchIssueTypes,
-  addIssueSeverityToIssues,
-  filterOutNonErrorSeverityIssues,
-  removeIssuesThatHaveBeenFixed,
+  // arguably removeIssuesThatHaveBeenFixed should be s step however we have only currently found one organisation,
+  // however this step is very time consuming, so in order to progress im commenting it out for now
+  // removeIssuesThatHaveBeenFixed,
   fetchFieldMappings,
-  addFieldMappingsToIssue,
-  addReferencesToIssues
+  addFieldMappingsToIssue
 ]
 
 // Other
@@ -620,7 +584,9 @@ export const getSetDataRange = (pageLength) => (req, res, next) => {
 
 export function getErrorSummaryItems (req, res, next) {
   const { issue_type: issueType, issue_field: issueField } = req.params
-  const { issues, issueCount, entities, resources } = req
+  const { entryIssues, issues: entityIssues, issueCount, entities, resources } = req
+
+  const issues = entityIssues || entryIssues
 
   const totalRecordCount = entities ? entities.length : resources[0].entry_count
   const totalIssues = issueCount?.count || issues.length

diff --git a/src/middleware/datasetTaskList.middleware.js b/src/middleware/datasetTaskList.middleware.js
@@ -3,8 +3,9 @@ import {
   validateQueryParams,
   fetchResources,
   processEntitiesMiddlewares,
-  processRelevantIssuesMiddlewares,
-  fetchOrgInfo
+  fetchOrgInfo,
+  fetchEntityIssueCounts,
+  fetchEntryIssueCounts
 } from './common.middleware.js'
 import { fetchOne, renderTemplate } from './middleware.builders.js'
 import performanceDbApi from '../services/performanceDbApi.js'
@@ -36,12 +37,15 @@ function getStatusTag (status) {
 
 export const prepareTasks = (req, res, next) => {
   const { lpa, dataset } = req.parsedParams
-  const { issues, entities, resources } = req
+  const { entities, resources } = req
+  const { entryIssueCounts, entityIssueCounts } = req
 
   const entityCount = entities.length
 
   const specialIssueTypeCases = ['reference values are not unique']
 
+  const issues = [...entryIssueCounts, ...entityIssueCounts]
+
   const groupedIssues = issues.reduce((acc, issue) => {
     const { field, issue_type: type } = issue
     const key = `${field}_${type}`
@@ -148,7 +152,8 @@ export default [
   fetchDatasetInfo,
   fetchResources,
   ...processEntitiesMiddlewares,
-  ...processRelevantIssuesMiddlewares,
+  fetchEntityIssueCounts,
+  fetchEntryIssueCounts,
   prepareTasks,
   prepareDatasetTaskListTemplateParams,
   getDatasetTaskList

diff --git a/src/middleware/entryIssueDetails.middleware.js b/src/middleware/entryIssueDetails.middleware.js
@@ -1,5 +1,5 @@
 import * as v from 'valibot'
-import { createPaginationTemplateParams, fetchDatasetInfo, fetchOrgInfo, fetchResources, getErrorSummaryItems, getSetBaseSubPath, getSetDataRange, prepareIssueDetailsTemplateParams, show404IfPageNumberNotInRange, validateQueryParams } from './common.middleware.js'
+import { createPaginationTemplateParams, fetchDatasetInfo, fetchEntryIssues, fetchOrgInfo, fetchResources, getErrorSummaryItems, getSetBaseSubPath, getSetDataRange, prepareIssueDetailsTemplateParams, show404IfPageNumberNotInRange, validateQueryParams } from './common.middleware.js'
 import { fetchMany, fetchOne, FetchOptions, renderTemplate } from './middleware.builders.js'
 import { issueErrorMessageHtml } from '../utils/utils.js'
 
@@ -40,20 +40,6 @@ export const addResourceMetaDataToResources = (req, res, next) => {
   next()
 }
 
-// We can only get the issues without entity from the latest resource as we have no way of knowing if those in previous resources have been fixed?
-const fetchEntryIssues = fetchMany({
-  query: ({ req, params }) => `
-    select * 
-    from issue i
-    LEFT JOIN issue_type it ON i.issue_type = it.issue_type
-    WHERE resource = '${req.resources[0].resource}'
-    AND i.issue_type = '${params.issue_type}'
-    AND it.responsibility = 'external'
-    AND field = '${params.issue_field}'
-  `,
-  result: 'issues'
-})
-
 const fetchIssueCount = fetchOne({
   query: ({ req, params }) => `
     select count(*) as count
@@ -73,16 +59,15 @@ export const setRecordCount = (req, res, next) => {
 }
 
 export const prepareEntry = (req, res, next) => {
-  const { resources, issues } = req
-  const { pageNumber } = req.parsedParams
+  const { resources, entryIssues } = req
 
-  if (!issues[pageNumber - 1] || !resources) {
+  if (!entryIssues[0] || !resources) {
     const error = new Error('Missing required values on request object')
     error.status = 404
     return next(error)
   }
 
-  const issue = issues[pageNumber - 1]
+  const issue = entryIssues[0]
 
   const errorMessage = issue.message || issue.issue_type
 
@@ -134,10 +119,10 @@ export default [
   fetchResources,
   fetchResourceMetaData,
   addResourceMetaDataToResources,
-  fetchEntryIssues,
   fetchIssueCount,
   setRecordCount,
   getSetDataRange(1),
+  fetchEntryIssues,
   show404IfPageNumberNotInRange,
   getSetBaseSubPath(['entry']),
   createPaginationTemplateParams,

diff --git a/src/middleware/issueTable.middleware.js b/src/middleware/issueTable.middleware.js
@@ -10,6 +10,7 @@ import config from '../../config/index.js'
 import { createPaginationTemplateParams, fetchDatasetInfo, fetchOrgInfo, fetchResources, filterOutEntitiesWithoutIssues, getErrorSummaryItems, getSetBaseSubPath, getSetDataRange, processEntitiesMiddlewares, processRelevantIssuesMiddlewares, processSpecificationMiddlewares, show404IfPageNumberNotInRange, validateQueryParams } from './common.middleware.js'
 import { onlyIf, renderTemplate } from './middleware.builders.js'
 import * as v from 'valibot'
+import { entryIssueGroups } from '../utils/utils.js'
 
 export const IssueTableQueryParams = v.object({
   lpa: v.string(),
@@ -90,22 +91,8 @@ export const prepareTemplateParams = (req, res, next) => {
 
 export const notIssueHasEntity = (req, res, next) => req.issues.length <= 0
 
-const redirectIssueGroups = [
-  {
-    type: 'missing value',
-    field: 'reference'
-  },
-  {
-    type: 'reference values are not unique',
-    field: 'reference'
-  },
-  {
-    type: 'unknown entity - missing reference',
-    field: 'entity'
-  }
-]
 export const issueTypeAndFieldShouldRedirect = (req, res, next) =>
-  redirectIssueGroups.findIndex(({ type, field }) => (type === req.params.issue_type && field === req.params.issue_field)) >= 0
+  entryIssueGroups.findIndex(({ type, field }) => (type === req.params.issue_type && field === req.params.issue_field)) >= 0
 
 export const redirectToEntityView = (req, res, next) => {
   const { lpa, dataset, issue_type: issueType, issue_field: issueField } = req.params

diff --git a/src/utils/utils.js b/src/utils/utils.js
@@ -94,6 +94,21 @@ export const dataSubjects = {
   }
 }
 
+export const entryIssueGroups = [
+  {
+    type: 'missing value',
+    field: 'reference'
+  },
+  {
+    type: 'reference values are not unique',
+    field: 'reference'
+  },
+  {
+    type: 'unknown entity - missing reference',
+    field: 'entity'
+  }
+]
+
 export function makeDatasetsLookup (dataSubjects) {
   const lookup = new Map()
   for (const [key, dataSubject] of Object.entries(dataSubjects)) {