Merge pull request #120 from digital-land/feat/getRealData

Feat/get real data from datasette for the dashboard

src/controllers/LpaOverviewController.js

@@ -1,17 +1,49 @@
 import performanceDbApi from '../services/performanceDbApi.js' // Assume you have an API service module
 import logger from '../utils/logger.js'
+import { dataSubjects } from '../utils/utils.js'
+
+// get a list of available datasets
+const availableDatasets = Object.values(dataSubjects)
+  .flatMap(dataSubject =>
+    dataSubject.dataSets
+      .filter(dataset => dataset.available)
+      .map(dataset => dataset.value)
+  )
 
 const LpaOverviewController = {
+  /**
+   * Get LPA overview data and render the overview page
+   * @param {Request} req - Express request object
+   * @param {Response} res - Express response object
+   * @param {NextFunction} next - Express next function
+   * @returns {Promise<void>} - Returns a promise that resolves when the overview page is rendered
+   */
   async getOverview (req, res, next) {
     try {
       const lpa = req.params.lpa
 
-      const response = await performanceDbApi.getLpaOverview(lpa) // Make API request
-      const data = response.data
+      // Make API request
+      const lpaOverview = await performanceDbApi.getLpaOverview(lpa)
+
+      // restructure datasets to usable format
+      const datasets = Object.entries(lpaOverview.datasets).map(([key, value]) => {
+        return {
+          slug: key,
+          ...value
+        }
+      })
 
-      const datasets = Object.entries(data.datasets).map(([key, value]) => {
-        return { ...value, slug: key }
+      // add in any of the missing key 8 datasets
+      const keys = Object.keys(lpaOverview.datasets)
+      availableDatasets.forEach(dataset => {
+        if (!keys.includes(dataset)) {
+          datasets.push({
+            slug: dataset,
+            endpoint: null
+          })
+        }
       })
+
       const totalDatasets = datasets.length
       const [datasetsWithEndpoints, datasetsWithIssues, datasetsWithErrors] = datasets.reduce((accumulator, dataset) => {
         if (dataset.endpoint !== null) accumulator[0]++
@@ -22,7 +54,7 @@ const LpaOverviewController = {
 
       const params = {
         organisation: {
-          name: data.name
+          name: lpaOverview.name
         },
         datasets,
         totalDatasets,

src/filters/makeDatasetSlugToReadableNameFilter.js

@@ -1,3 +1,5 @@
+import logger from '../utils/logger.js'
+
 /**
  * Creates a filter function that takes a dataset slug as input and returns its corresponding readable name.
  * The filter function uses a provided dataset name mapping to look up the readable name.
@@ -16,7 +18,10 @@ export const makeDatasetSlugToReadableNameFilter = (datasetNameMapping) => {
   return (slug) => {
     const name = datasetNameMapping.get(slug)
     if (!name) {
-      throw new Error(`Can't find a name for ${slug}`)
+      // throw new Error(`Can't find a name for ${slug}`)
+      // ToDo: work out what to do here? potentially update it with data from datasette
+      logger.error(`can't find a name for ${slug}`)
+      return slug
     }
     return name
   }

src/services/datasette.js

@@ -0,0 +1,19 @@
+import axios from 'axios'
+import logger from '../utils/logger.js'
+
+const datasetteUrl = 'https://datasette.planning.data.gov.uk'
+const database = 'digital-land'
+
+export default {
+  runQuery: async (query) => {
+    const encodedQuery = encodeURIComponent(query)
+    const url = `${datasetteUrl}/${database}.json?sql=${encodedQuery}`
+    try {
+      const response = await axios.get(url)
+      return response.data
+    } catch (error) {
+      logger.error(error)
+      throw error
+    }
+  }
+}

src/services/performanceDbApi.js

@@ -1,47 +1,122 @@
+/**
+ * Performance DB API service
+ */
+import datasette from './datasette.js'
+
+/**
+ * @typedef {object} Dataset
+ * @property {string} endpoint
+ * @property {?string} error
+ * @property {?string} issue
+ */
+
+/**
+ * @typedef {object} LpaOverview
+ * @property {string} name
+ * @property {{ [dataset: string]: Dataset }} datasets
+ */
+
+/**
+ * Performance DB API service
+ * @export
+ * @default
+ */
 export default {
+  /**
+   * Get LPA overview
+   * @param {string} lpa - LPA ID
+   * @returns {Promise<LpaOverview>} LPA overview
+   */
   getLpaOverview: async (lpa) => {
-    return {
-      data: {
-        name: 'Borechester City Council',
-        datasets: {
-          'article-4-direction': {
-            endpoint: null
-          },
-          'article-4-direction-area': {
-            endpoint: null
-          },
-          'conservation-area': {
-            endpoint: 'http://conservation-area.json',
-            error: null,
-            issue: 'Endpoint has not been updated since 21 May 2023'
-          },
-          'conservation-area-document': {
-            endpoint: 'http://conservation-area-document.json',
-            error: null,
-            issue: null
-          },
-          'listed-building-outline': {
-            endpoint: 'http://listed-building-outline.json',
-            error: null,
-            issue: null
-          },
-          tree: {
-            endpoint: 'http://tree.json',
-            error: null,
-            issue: 'There are 20 issues in this dataset'
-          },
-          'tree-preservation-order': {
-            endpoint: 'http://tree-preservation-order.json',
-            error: 'Error connecting to endpoint',
-            issue: null
-          },
-          'tree-preservation-zone': {
-            endpoint: 'http://tree-preservation-zone.json',
-            error: 'Error connecting to endpoint',
-            issue: null
-          }
-        }
+    const query = `
+    SELECT
+    p.organisation,
+    o.name,
+    p.dataset,
+    rle.pipeline,
+    rle.endpoint,
+    rle.resource,
+    rle.exception,
+    rle.status as http_status,
+    case 
+           when (rle.status != '200') then 'Error'
+           when (it.severity = 'error') then 'Issue'
+           when (it.severity = 'warning') then 'Warning'
+           else 'No issues'
+           end as status,
+    case
+            when (it.severity = 'info') then ''
+            else i.issue_type
+        end as issue_type,
+        case
+            when (it.severity = 'info') then ''
+            else it.severity
+        end as severity,
+        it.responsibility,
+        COUNT(
+            case
+            when it.severity != 'info' then 1
+            else null
+            end
+        ) as issue_count
+FROM
+    provision p
+LEFT JOIN
+    organisation o ON o.organisation = p.organisation
+LEFT JOIN
+    reporting_latest_endpoints rle
+    ON REPLACE(rle.organisation, '-eng', '') = p.organisation
+    AND rle.pipeline = p.dataset
+LEFT JOIN
+    issue i ON rle.resource = i.resource AND rle.pipeline = i.dataset
+LEFT JOIN
+    issue_type it ON i.issue_type = it.issue_type AND it.severity != 'info'
+WHERE
+    p.organisation = '${lpa}'
+GROUP BY
+    p.organisation,
+    p.dataset,
+    o.name,
+    rle.pipeline,
+    rle.endpoint
+ORDER BY
+    p.organisation,
+    o.name;
+`
+
+    const result = await datasette.runQuery(query)
+
+    // convert the rows into an easier to access format
+    const columns = result.columns
+    const rows = result.rows.map((row) => {
+      return row.reduce((acc, val, index) => {
+        acc[columns[index]] = val
+        return acc
+      }, {})
+    })
+
+    const datasets = rows.reduce((accumulator, row) => {
+      let error
+      if (row.http_status !== '200' || row.exception !== '') {
+        error = row.exception !== '' ? row.exception : `endpoint returned with a status of ${row.http_status}`
+      }
+
+      let issue
+      if (row.issue_count > 0) {
+        issue = `There are ${row.issue_count} issues in this dataset`
       }
+
+      accumulator[row.dataset] = {
+        endpoint: row.endpoint,
+        error,
+        issue
+      }
+      return accumulator
+    }, {})
+
+    return {
+      name: result.rows[0][1],
+      datasets
     }
   }
 }

test/unit/lpaOverviewController.test.js

@@ -3,6 +3,11 @@ import LpaOverviewController from '../../src/controllers/LpaOverviewController.j
 import performanceDbApi from '../../src/services/performanceDbApi.js'
 
 vi.mock('../../src/services/performanceDbApi.js')
+vi.mock('../../src/utils/utils.js', () => {
+  return {
+    dataSubjects: {}
+  }
+})
 
 describe('LpaOverviewController', () => {
   beforeEach(() => {
@@ -15,13 +20,11 @@ describe('LpaOverviewController', () => {
     const next = vi.fn()
 
     const expectedResponse = {
-      data: {
-        name: 'Test LPA',
-        datasets: {
-          dataset1: { endpoint: 'https://example.com', issue: false, error: false },
-          dataset2: { endpoint: null, issue: true, error: false },
-          dataset3: { endpoint: 'https://example.com', issue: false, error: true }
-        }
+      name: 'Test LPA',
+      datasets: {
+        dataset1: { endpoint: 'https://example.com', issue: false, error: false },
+        dataset2: { endpoint: null, issue: true, error: false },
+        dataset3: { endpoint: 'https://example.com', issue: false, error: true }
       }
     }