Merge pull request #240 from digital-land/feat/datasetTaskList/query

Feat/dataset task list/query

src/controllers/OrganisationsController.js

@@ -1,7 +1,7 @@
 import datasette from '../services/datasette.js'
 import performanceDbApi from '../services/performanceDbApi.js' // Assume you have an API service module
-import getTaskList from '../utils/issueMessages/getDatasetTaskList.js'
 import logger from '../utils/logger.js'
+import { types } from '../utils/logging.js'
 import { dataSubjects } from '../utils/utils.js'
 
 // get a list of available datasets
@@ -128,44 +128,31 @@ const organisationsController = {
   },
 
   async getDatasetTaskList (req, res, next) {
-    const issues = [
-      {
-        num_issues: 2,
-        issue_type: 'future entry date',
-        resource: 'resource1',
-        status: 'Error'
-      },
-      {
-        num_issues: 3,
-        issue_type: 'invalid coordinates',
-        resource: 'resource2',
-        status: 'Issue'
-      },
-      {
-        num_issues: 1,
-        issue_type: 'invalid decimal',
-        resource: 'resource3',
-        status: 'Warning'
-      }
-    ]
+    const lpa = req.params.lpa
+    const datasetId = req.params.dataset
 
-    const taskList = getTaskList(issues)
+    try {
+      const organisationResult = await datasette.runQuery(`SELECT name FROM organisation WHERE organisation = '${lpa}'`)
+      const organisation = organisationResult.formattedData[0]
 
-    const organisation = {
-      name: "George's fake organisation"
-    }
+      const datasetResult = await datasette.runQuery(`SELECT name FROM dataset WHERE dataset = '${datasetId}'`)
+      const dataset = datasetResult.formattedData[0]
 
-    const dataset = {
-      name: 'Article 4 direction area'
-    }
+      const issues = await performanceDbApi.getLpaDatasetIssues(lpa, datasetId)
 
-    const params = {
-      taskList,
-      organisation,
-      dataset
-    }
+      const taskList = performanceDbApi.getTaskList(issues)
 
-    res.render('organisations/datasetTaskList.html', params)
+      const params = {
+        taskList,
+        organisation,
+        dataset
+      }
+
+      res.render('organisations/datasetTaskList.html', params)
+    } catch (e) {
+      logger.warn(`getDAtasetTaskList() failed for lpa='${lpa}', datasetId='${datasetId}'`, { type: types.App })
+      next(e)
+    }
   }
 
 }

src/routes/organisations.js

@@ -3,12 +3,9 @@ import OrganisationsController from '../controllers/OrganisationsController.js'
 
 const router = express.Router()
 
+router.get('/:lpa/:dataset/get-started', OrganisationsController.getGetStarted)
+router.get('/:lpa/:dataset', OrganisationsController.getDatasetTaskList)
+router.get('/:lpa', OrganisationsController.getOverview)
 router.get('/', OrganisationsController.getOrganisations)
 
-router.get('/:lpa/overview', OrganisationsController.getOverview)
-
-router.get('/:lpa/dataset/:dataset', OrganisationsController.getDatasetTaskList)
-
-router.get('/:lpa/dataset/:dataset/get-started', OrganisationsController.getGetStarted)
-
 export default router

src/services/performanceDbApi.js

@@ -3,6 +3,54 @@
  */
 import datasette from './datasette.js'
 
+// ===========================================
+
+// for now we are using a csv for these messages but we will probably end up moving to a table, so for now this can sit in the fake performance db api
+
+import csv from 'csv-parser' // ToDo: remember to remove this from package.json when we move away from csv
+import fs from 'fs'
+
+const messages = {}
+
+fs.createReadStream('src/content/issueMessages.csv')
+  .pipe(csv())
+  .on('data', (row) => {
+    messages[row.issue_type] = {
+      singular: row.singular_message,
+      plural: row.plural_message.replace('{num_issues}', '{}')
+    }
+  })
+  .on('end', () => {
+    // Messages object is now populated
+  })
+
+function getTaskMessage (issueType, issueCount) {
+  if (!messages[issueType]) {
+    throw new Error(`Unknown issue type: ${issueType}`)
+  }
+
+  const message = issueCount === 1 ? messages[issueType].singular : messages[issueType].plural
+  return message.replace('{}', issueCount)
+}
+
+function getStatusTag (status) {
+  const statusToTagClass = {
+    Error: 'govuk-tag--red',
+    'Needs fixing': 'govuk-tag--yellow',
+    Warning: 'govuk-tag--blue',
+    Issue: 'govuk-tag--blue'
+  }
+
+  return {
+    tag: {
+      text: status,
+      classes: statusToTagClass[status]
+    }
+  }
+}
+
+// ===========================================
+
 /**
  * @typedef {object} Dataset
  * @property {string} endpoint
@@ -110,5 +158,67 @@ ORDER BY
       organisation: result.formattedData[0].organisation,
       datasets
     }
+  },
+
+  getLpaDatasetIssues: async (lpa, datasetId) => {
+    const sql = `
+      SELECT
+        rle.endpoint,
+        rle.resource,
+        rle.exception,
+
+        i.field,
+        i.issue_type,
+        i.line_number,
+        i.value,
+        i.message,
+
+        CASE
+          WHEN COUNT(
+            CASE
+              WHEN it.severity == 'error' THEN 1
+              ELSE null
+            END
+          ) > 0 THEN 'Needs fixing'
+          ELSE 'Live'
+        END AS status,
+        COUNT(i.issue_type) as num_issues
+      FROM
+          provision p
+      LEFT JOIN
+          reporting_latest_endpoints rle
+          ON REPLACE(rle.organisation, '-eng', '') = p.organisation
+          AND rle.pipeline = p.dataset
+      LEFT JOIN
+          issue i ON rle.resource = i.resource AND rle.pipeline = i.dataset
+      LEFT JOIN
+          issue_type it ON i.issue_type = it.issue_type
+      WHERE
+          p.organisation = '${lpa}' AND p.dataset = '${datasetId}'
+          AND (it.severity == 'error' OR it.severity == 'warning')
+      GROUP BY i.issue_type
+      ORDER BY it.severity`
+
+    const result = await datasette.runQuery(sql)
+    return result.formattedData.map((row) => {
+      return {
+        num_issues: row.num_issues,
+        issue_type: row.issue_type,
+        resource: row.resource,
+        status: row.status
+      }
+    })
+  },
+
+  getTaskList: (issues) => {
+    return issues.map((issue) => {
+      return {
+        title: {
+          text: getTaskMessage(issue.issue_type, issue.num_issues)
+        },
+        href: 'toDo',
+        status: getStatusTag(issue.status)
+      }
+    })
   }
 }

src/views/organisations/datasetTaskList.html

@@ -24,8 +24,8 @@ <h2 class="govuk-heading-l">
       {{ organisation.name }}’s task list
     </h2>
 
-    {% if tasks.length == 0 %}
-    <p>There are no issues with {{ organisation.name }}’s {{ dataset }} dataset.</p>
+    {% if taskList.length == 0 %}
+    <p>There are no issues with {{ organisation.name }}'s {{ dataset.name }} dataset.</p>
 
     <p><a
         href="https://www.planning.data.gov.uk/entity/?dataset=brownfield-land&geometry_curie=statistical-geography:E07000091">View

src/views/organisations/find.html

@@ -38,7 +38,7 @@ <h2 class="blockHeading govuk-heading-xl">{{ letter }}</h2>
             <ul class="govuk-list govuk-!-margin-bottom-0">
               {% for org in orgs %}
               <li class="govuk-!-margin-bottom-1 js-filter-item" data-filter="item" data-filter-terms="{{org.name}}">
-                <a href="/organisations/{{ org.organisation }}/overview" class="govuk-link">{{ org.name }}</a>
+                <a href="/organisations/{{ org.organisation }}" class="govuk-link">{{ org.name }}</a>
               </li>
               {% endfor %}
             </ul>

src/views/organisations/overview.html

@@ -67,7 +67,7 @@ <h2 class="govuk-heading-m">Datasets</h2>
           <div class="govuk-task-list__name-and-hint">
             <h2 class="govuk-heading-m">
               <a class="govuk-link govuk-task-list__link"
-                href="/organisations/{{ organisation.organisation }}/dataset/article-4-direction">
+                href="/organisations/{{ organisation.organisation }}/{{dataset.slug}}">
                 {{dataset.slug | datasetSlugToReadableName}}
               </a>
             </h2>

test/unit/datasetTaskListPage.test.js

@@ -81,7 +81,6 @@ describe('Dataset Task List Page', () => {
   it('Renders the correct headings', () => {
     expect(document.querySelector('span.govuk-caption-xl').textContent).toEqual(params.organisation.name)
     expect(document.querySelector('h1').textContent).toContain(params.dataset.name)
-    // expect(document.querySelector('h2').textContent).toContain('How to prepare and submit your World heritage site buffer zone data')
   })
 
   const taskListItems = document.querySelectorAll('.govuk-task-list__item')
@@ -101,4 +100,20 @@ describe('Dataset Task List Page', () => {
       }
     })
   })
+
+  it('renders correctly when no taskList items are passed in', () => {
+    const organisation = { name: 'Test Organisation' }
+    const dataset = { name: 'Test Dataset' }
+    const taskList = []
+
+    const html = nunjucks.render('organisations/datasetTaskList.html', {
+      organisation,
+      dataset,
+      taskList
+    })
+
+    const paragraphText = `There are no issues with ${organisation.name}'s ${dataset.name} dataset.`
+
+    expect(html).toContain(paragraphText)
+  })
 })