From 9f28f4ee2a999526ec45cb7d6b5c79c33a548f9a Mon Sep 17 00:00:00 2001 From: George Goodall Date: Fri, 12 Jul 2024 14:45:27 +0100 Subject: [PATCH 01/10] get data from datasette --- src/controllers/LpaOverviewController.js | 7 +- .../makeDatasetSlugToReadableNameFilter.js | 7 +- src/services/datasette.js | 19 +++ src/services/performanceDbApi.js | 119 ++++++++++++------ 4 files changed, 107 insertions(+), 45 deletions(-) create mode 100644 src/services/datasette.js diff --git a/src/controllers/LpaOverviewController.js b/src/controllers/LpaOverviewController.js index 0a9e0c0c..e216c7e1 100644 --- a/src/controllers/LpaOverviewController.js +++ b/src/controllers/LpaOverviewController.js @@ -6,10 +6,9 @@ const LpaOverviewController = { try { const lpa = req.params.lpa - const response = await performanceDbApi.getLpaOverview(lpa) // Make API request - const data = response.data + const lpaOverview = await performanceDbApi.getLpaOverview(lpa) // Make API request - const datasets = Object.entries(data.datasets).map(([key, value]) => { + const datasets = Object.entries(lpaOverview.datasets).map(([key, value]) => { return { ...value, slug: key } }) const totalDatasets = datasets.length @@ -22,7 +21,7 @@ const LpaOverviewController = { const params = { organisation: { - name: data.name + name: lpaOverview.name }, datasets, totalDatasets, diff --git a/src/filters/makeDatasetSlugToReadableNameFilter.js b/src/filters/makeDatasetSlugToReadableNameFilter.js index d0358669..07c6d752 100644 --- a/src/filters/makeDatasetSlugToReadableNameFilter.js +++ b/src/filters/makeDatasetSlugToReadableNameFilter.js @@ -1,3 +1,5 @@ +import logger from '../utils/logger.js' + /** * Creates a filter function that takes a dataset slug as input and returns its corresponding readable name. * The filter function uses a provided dataset name mapping to look up the readable name. @@ -16,7 +18,10 @@ export const makeDatasetSlugToReadableNameFilter = (datasetNameMapping) => { return (slug) => { const name = datasetNameMapping.get(slug) if (!name) { - throw new Error(`Can't find a name for ${slug}`) + // throw new Error(`Can't find a name for ${slug}`) + // ToDo: work out what to do here? potentially update it with data from datasette + logger.error(`can't find a name for ${slug}`) + return slug } return name } diff --git a/src/services/datasette.js b/src/services/datasette.js new file mode 100644 index 00000000..137146e7 --- /dev/null +++ b/src/services/datasette.js @@ -0,0 +1,19 @@ +import axios from 'axios' +import logger from '../utils/logger.js' + +const datasetteUrl = 'https://datasette.planning.data.gov.uk' +const database = 'digital-land' + +export default { + runQuery: async (query) => { + const encodedQuery = encodeURIComponent(query) + const url = `${datasetteUrl}/${database}.json?sql=${encodedQuery}` + try { + const response = await axios.get(url) + return response.data + } catch (error) { + logger.error(error) + throw error + } + } +} diff --git a/src/services/performanceDbApi.js b/src/services/performanceDbApi.js index c111a15b..58db6b5c 100644 --- a/src/services/performanceDbApi.js +++ b/src/services/performanceDbApi.js @@ -1,47 +1,86 @@ +import datasette from './datasette.js' + +/* + +*/ + export default { - getLpaOverview: async (lpa) => { + getLpaOverviewOld: async (lpa) => { return { - data: { - name: 'Borechester City Council', - datasets: { - 'article-4-direction': { - endpoint: null - }, - 'article-4-direction-area': { - endpoint: null - }, - 'conservation-area': { - endpoint: 'http://conservation-area.json', - error: null, - issue: 'Endpoint has not been updated since 21 May 2023' - }, - 'conservation-area-document': { - endpoint: 'http://conservation-area-document.json', - error: null, - issue: null - }, - 'listed-building-outline': { - endpoint: 'http://listed-building-outline.json', - error: null, - issue: null - }, - tree: { - endpoint: 'http://tree.json', - error: null, - issue: 'There are 20 issues in this dataset' - }, - 'tree-preservation-order': { - endpoint: 'http://tree-preservation-order.json', - error: 'Error connecting to endpoint', - issue: null - }, - 'tree-preservation-zone': { - endpoint: 'http://tree-preservation-zone.json', - error: 'Error connecting to endpoint', - issue: null - } + name: 'Borechester City Council', + datasets: { + 'article-4-direction': { + endpoint: null + }, + 'article-4-direction-area': { + endpoint: null + }, + 'conservation-area': { + endpoint: 'http://conservation-area.json', + error: null, + issue: 'Endpoint has not been updated since 21 May 2023' + }, + 'conservation-area-document': { + endpoint: 'http://conservation-area-document.json', + error: null, + issue: null + }, + 'listed-building-outline': { + endpoint: 'http://listed-building-outline.json', + error: null, + issue: null + }, + tree: { + endpoint: 'http://tree.json', + error: null, + issue: 'There are 20 issues in this dataset' + }, + 'tree-preservation-order': { + endpoint: 'http://tree-preservation-order.json', + error: 'Error connecting to endpoint', + issue: null + }, + 'tree-preservation-zone': { + endpoint: 'http://tree-preservation-zone.json', + error: 'Error connecting to endpoint', + issue: null } } } + }, + + getLpaOverview: async (lpa) => { + const query = + `SELECT + p.organisation, + o.name, + p.dataset, + rle.endpoint + FROM + provision p + INNER JOIN + organisation o ON o.organisation = p.organisation + LEFT JOIN + reporting_latest_endpoints rle + ON REPLACE(rle.organisation, '-eng', '') = p.organisation + AND rle.pipeline = p.dataset + WHERE p.organisation = '${lpa}' + ORDER BY + p.organisation, + o.name` + + const result = await datasette.runQuery(query) + + const datasets = result.rows.reduce((accumulator, row) => { + accumulator[row[2]] = { + endpoint: row[3] + } + return accumulator + }, {}) + + return { + name: result.rows[0][1], + datasets + } } } From 4605bcab7c671ceee41a06b4dd086bd566ed816d Mon Sep 17 00:00:00 2001 From: George Goodall Date: Fri, 12 Jul 2024 14:59:28 +0100 Subject: [PATCH 02/10] now get errors with endpoints --- src/services/performanceDbApi.js | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/services/performanceDbApi.js b/src/services/performanceDbApi.js index 58db6b5c..d7ffa4ca 100644 --- a/src/services/performanceDbApi.js +++ b/src/services/performanceDbApi.js @@ -55,7 +55,9 @@ export default { p.organisation, o.name, p.dataset, - rle.endpoint + rle.endpoint, + rle.status, + rle.exception FROM provision p INNER JOIN @@ -72,8 +74,13 @@ export default { const result = await datasette.runQuery(query) const datasets = result.rows.reduce((accumulator, row) => { + let error + if (row[4] !== 200 || row[5] !== '') { + error = row[5] !== '' ? row[5] : `endpoint returned with a status of ${row[4]}` + } accumulator[row[2]] = { - endpoint: row[3] + endpoint: row[3], + error } return accumulator }, {}) From 85100c1347afb290b76ed34b0d2537736bef4e6d Mon Sep 17 00:00:00 2001 From: George Goodall Date: Fri, 12 Jul 2024 15:40:01 +0100 Subject: [PATCH 03/10] updated with samritis new query: now showing issues --- src/services/performanceDbApi.js | 103 +++++++++++++++++++++++-------- 1 file changed, 77 insertions(+), 26 deletions(-) diff --git a/src/services/performanceDbApi.js b/src/services/performanceDbApi.js index d7ffa4ca..33cfac2f 100644 --- a/src/services/performanceDbApi.js +++ b/src/services/performanceDbApi.js @@ -50,37 +50,88 @@ export default { }, getLpaOverview: async (lpa) => { - const query = - `SELECT - p.organisation, - o.name, - p.dataset, - rle.endpoint, - rle.status, - rle.exception - FROM - provision p - INNER JOIN - organisation o ON o.organisation = p.organisation - LEFT JOIN - reporting_latest_endpoints rle - ON REPLACE(rle.organisation, '-eng', '') = p.organisation - AND rle.pipeline = p.dataset - WHERE p.organisation = '${lpa}' - ORDER BY - p.organisation, - o.name` + const query = ` + SELECT + p.organisation, + o.name, + p.dataset, + rle.pipeline, + rle.endpoint, + rle.resource, + rle.exception, + rle.status as http_status, + case + when (rle.status != '200') then 'Error' + when (it.severity = 'error') then 'Issue' + when (it.severity = 'warning') then 'Warning' + else 'No issues' + end as status, + case + when (it.severity = 'info') then '' + else i.issue_type + end as issue_type, + case + when (it.severity = 'info') then '' + else it.severity + end as severity, + it.responsibility, + COUNT( + case + when it.severity != 'info' then 1 + else null + end + ) as issue_count +FROM + provision p +LEFT JOIN + organisation o ON o.organisation = p.organisation +LEFT JOIN + reporting_latest_endpoints rle + ON REPLACE(rle.organisation, '-eng', '') = p.organisation + AND rle.pipeline = p.dataset +LEFT JOIN + issue i ON rle.resource = i.resource AND rle.pipeline = i.dataset +LEFT JOIN + issue_type it ON i.issue_type = it.issue_type AND it.severity != 'info' +WHERE + p.organisation = '${lpa}' +GROUP BY + p.organisation, + p.dataset, + o.name, + rle.pipeline, + rle.endpoint +ORDER BY + p.organisation, + o.name; +` const result = await datasette.runQuery(query) - const datasets = result.rows.reduce((accumulator, row) => { + // convert the rows into an easier to access format + const columns = result.columns + const rows = result.rows.map((row) => { + return row.reduce((acc, val, index) => { + acc[columns[index]] = val; + return acc; + }, {}); + }) + + const datasets = rows.reduce((accumulator, row) => { let error - if (row[4] !== 200 || row[5] !== '') { - error = row[5] !== '' ? row[5] : `endpoint returned with a status of ${row[4]}` + if (row.http_status !== '200' || row.exception !== '') { + error = row.exception !== '' ? row.exception : `endpoint returned with a status of ${row.http_status}` + } + + let issue + if (row.issue_count > 0){ + issue = `There are ${row.issue_count} issues in this dataset` } - accumulator[row[2]] = { - endpoint: row[3], - error + + accumulator[row.dataset] = { + endpoint: row.endpoint, + error, + issue } return accumulator }, {}) From f7ede6c3ad02395eb71222d0d2978c595238d5c2 Mon Sep 17 00:00:00 2001 From: George Goodall Date: Fri, 12 Jul 2024 15:40:21 +0100 Subject: [PATCH 04/10] linting --- src/services/performanceDbApi.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/services/performanceDbApi.js b/src/services/performanceDbApi.js index 33cfac2f..747b56d1 100644 --- a/src/services/performanceDbApi.js +++ b/src/services/performanceDbApi.js @@ -112,9 +112,9 @@ ORDER BY const columns = result.columns const rows = result.rows.map((row) => { return row.reduce((acc, val, index) => { - acc[columns[index]] = val; - return acc; - }, {}); + acc[columns[index]] = val + return acc + }, {}) }) const datasets = rows.reduce((accumulator, row) => { @@ -124,7 +124,7 @@ ORDER BY } let issue - if (row.issue_count > 0){ + if (row.issue_count > 0) { issue = `There are ${row.issue_count} issues in this dataset` } From dbb045189e9cd2343eeab692febec8e5316a2f70 Mon Sep 17 00:00:00 2001 From: George Goodall Date: Fri, 12 Jul 2024 16:27:46 +0100 Subject: [PATCH 05/10] refactor lpa overview controller and add js doc --- src/controllers/LpaOverviewController.js | 27 ++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/src/controllers/LpaOverviewController.js b/src/controllers/LpaOverviewController.js index e216c7e1..b665493f 100644 --- a/src/controllers/LpaOverviewController.js +++ b/src/controllers/LpaOverviewController.js @@ -1,16 +1,35 @@ import performanceDbApi from '../services/performanceDbApi.js' // Assume you have an API service module import logger from '../utils/logger.js' +import { dataSubjects } from '../utils/utils.js' + +// get a list of available datasets +const availableDatasets = Object.values(dataSubjects) + .flatMap(dataSubject => + dataSubject.dataSets + .filter(dataset => dataset.available) + .map(dataset => dataset.value) + ) const LpaOverviewController = { + /** + * Get LPA overview data and render the overview page + * @param {Request} req - Express request object + * @param {Response} res - Express response object + * @param {NextFunction} next - Express next function + * @returns {Promise} - Returns a promise that resolves when the overview page is rendered + */ async getOverview (req, res, next) { try { const lpa = req.params.lpa - const lpaOverview = await performanceDbApi.getLpaOverview(lpa) // Make API request + // Make API request + const lpaOverview = await performanceDbApi.getLpaOverview(lpa) + + const datasets = availableDatasets.map((dataset) => ({ + slug: dataset, + ...(lpaOverview.datasets[dataset] || { endpoint: null }) + })) - const datasets = Object.entries(lpaOverview.datasets).map(([key, value]) => { - return { ...value, slug: key } - }) const totalDatasets = datasets.length const [datasetsWithEndpoints, datasetsWithIssues, datasetsWithErrors] = datasets.reduce((accumulator, dataset) => { if (dataset.endpoint !== null) accumulator[0]++ From 7609f6e1a174b6fbcccd7c5389f864597eedf577 Mon Sep 17 00:00:00 2001 From: George Goodall Date: Fri, 12 Jul 2024 16:27:57 +0100 Subject: [PATCH 06/10] remove old method --- src/services/performanceDbApi.js | 44 -------------------------------- 1 file changed, 44 deletions(-) diff --git a/src/services/performanceDbApi.js b/src/services/performanceDbApi.js index 747b56d1..a299fa5f 100644 --- a/src/services/performanceDbApi.js +++ b/src/services/performanceDbApi.js @@ -5,50 +5,6 @@ import datasette from './datasette.js' */ export default { - getLpaOverviewOld: async (lpa) => { - return { - name: 'Borechester City Council', - datasets: { - 'article-4-direction': { - endpoint: null - }, - 'article-4-direction-area': { - endpoint: null - }, - 'conservation-area': { - endpoint: 'http://conservation-area.json', - error: null, - issue: 'Endpoint has not been updated since 21 May 2023' - }, - 'conservation-area-document': { - endpoint: 'http://conservation-area-document.json', - error: null, - issue: null - }, - 'listed-building-outline': { - endpoint: 'http://listed-building-outline.json', - error: null, - issue: null - }, - tree: { - endpoint: 'http://tree.json', - error: null, - issue: 'There are 20 issues in this dataset' - }, - 'tree-preservation-order': { - endpoint: 'http://tree-preservation-order.json', - error: 'Error connecting to endpoint', - issue: null - }, - 'tree-preservation-zone': { - endpoint: 'http://tree-preservation-zone.json', - error: 'Error connecting to endpoint', - issue: null - } - } - } - }, - getLpaOverview: async (lpa) => { const query = ` SELECT From 8511c19a54c6b45ad89aa3b5049feef75e25843e Mon Sep 17 00:00:00 2001 From: George Goodall Date: Fri, 12 Jul 2024 16:29:46 +0100 Subject: [PATCH 07/10] add jsdoc to performanceDbApi --- src/services/performanceDbApi.js | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/src/services/performanceDbApi.js b/src/services/performanceDbApi.js index a299fa5f..17fe893a 100644 --- a/src/services/performanceDbApi.js +++ b/src/services/performanceDbApi.js @@ -1,10 +1,32 @@ +/** + * Performance DB API service + */ import datasette from './datasette.js' -/* +/** + * @typedef {object} Dataset + * @property {string} endpoint + * @property {?string} error + * @property {?string} issue + */ -*/ +/** + * @typedef {object} LpaOverview + * @property {string} name + * @property {{ [dataset: string]: Dataset }} datasets + */ +/** + * Performance DB API service + * @export + * @default + */ export default { + /** + * Get LPA overview + * @param {string} lpa - LPA ID + * @returns {Promise} LPA overview + */ getLpaOverview: async (lpa) => { const query = ` SELECT From 3434d835290311bc207fd17ba27153a7537810d4 Mon Sep 17 00:00:00 2001 From: George Goodall Date: Fri, 12 Jul 2024 16:55:11 +0100 Subject: [PATCH 08/10] update to show all available datasets and include 8 essential ones --- src/controllers/LpaOverviewController.js | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/src/controllers/LpaOverviewController.js b/src/controllers/LpaOverviewController.js index b665493f..497624e4 100644 --- a/src/controllers/LpaOverviewController.js +++ b/src/controllers/LpaOverviewController.js @@ -25,10 +25,24 @@ const LpaOverviewController = { // Make API request const lpaOverview = await performanceDbApi.getLpaOverview(lpa) - const datasets = availableDatasets.map((dataset) => ({ - slug: dataset, - ...(lpaOverview.datasets[dataset] || { endpoint: null }) - })) + // restructure datasets to usable format + const datasets = Object.entries(lpaOverview.datasets).map(([key, value]) => { + return { + slug: key, + ...value + } + }) + + // add in any of the missing key 8 datasets + const keys = Object.keys(lpaOverview.datasets) + availableDatasets.forEach(dataset => { + if (!keys.includes(dataset)) { + datasets.push({ + slug: dataset, + endpoint: null + }) + } + }) const totalDatasets = datasets.length const [datasetsWithEndpoints, datasetsWithIssues, datasetsWithErrors] = datasets.reduce((accumulator, dataset) => { From 05447ab0ba4f93ad94981ef8fe30576b9311fccb Mon Sep 17 00:00:00 2001 From: George Goodall Date: Mon, 15 Jul 2024 11:25:30 +0100 Subject: [PATCH 09/10] started work on fixing tests --- test/unit/lpaOverviewController.test.js | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/test/unit/lpaOverviewController.test.js b/test/unit/lpaOverviewController.test.js index d55223d1..4af280b2 100644 --- a/test/unit/lpaOverviewController.test.js +++ b/test/unit/lpaOverviewController.test.js @@ -15,13 +15,11 @@ describe('LpaOverviewController', () => { const next = vi.fn() const expectedResponse = { - data: { - name: 'Test LPA', - datasets: { - dataset1: { endpoint: 'https://example.com', issue: false, error: false }, - dataset2: { endpoint: null, issue: true, error: false }, - dataset3: { endpoint: 'https://example.com', issue: false, error: true } - } + name: 'Test LPA', + datasets: { + dataset1: { endpoint: 'https://example.com', issue: false, error: false }, + dataset2: { endpoint: null, issue: true, error: false }, + dataset3: { endpoint: 'https://example.com', issue: false, error: true } } } From fd96dd45d060359f3b3111e9308368698029ae5a Mon Sep 17 00:00:00 2001 From: George Goodall Date: Mon, 15 Jul 2024 11:55:29 +0100 Subject: [PATCH 10/10] fix test --- test/unit/lpaOverviewController.test.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/unit/lpaOverviewController.test.js b/test/unit/lpaOverviewController.test.js index 4af280b2..073b5c6d 100644 --- a/test/unit/lpaOverviewController.test.js +++ b/test/unit/lpaOverviewController.test.js @@ -3,6 +3,11 @@ import LpaOverviewController from '../../src/controllers/LpaOverviewController.j import performanceDbApi from '../../src/services/performanceDbApi.js' vi.mock('../../src/services/performanceDbApi.js') +vi.mock('../../src/utils/utils.js', () => { + return { + dataSubjects: {} + } +}) describe('LpaOverviewController', () => { beforeEach(() => {