Merge pull request #124 from digital-land/feat/datasetSlugNameMap

Feat/dataset slug name map

index.js

@@ -11,16 +11,18 @@ import { setupErrorHandlers } from './src/serverSetup/errorHandlers.js'
 import { setupSession } from './src/serverSetup/session.js'
 import { setupNunjucks } from './src/serverSetup/nunjucks.js'
 import { setupSentry } from './src/serverSetup/sentry.js'
-
-import { dataSubjects } from './src/utils/utils.js'
+import { getDatasetSlugNameMapping } from './src/utils/datasetteQueries/getDatasetSlugNameMapping.js'
 
 dotenv.config()
 
 const app = express()
 
 setupMiddlewares(app)
 setupSession(app)
-setupNunjucks({ app, dataSubjects })
+setupNunjucks({
+  app,
+  datasetNameMapping: await getDatasetSlugNameMapping()
+})
 setupRoutes(app)
 setupSentry(app)
 setupErrorHandlers(app)

src/controllers/lpaDetailsController.js

@@ -1,5 +1,5 @@
 import PageController from './pageController.js'
-import { fetchLocalAuthorities } from '../services/fetchLocalAuthorities.js'
+import { fetchLocalAuthorities } from '../utils/datasetteQueries/fetchLocalAuthorities.js'
 
 class LpaDetailsController extends PageController {
   async locals (req, res, next) {

src/filters/filters.js

@@ -4,12 +4,11 @@ import validationMessageLookup from './validationMessageLookup.js'
 import toErrorList from './toErrorList.js'
 import prettifyColumnName from './prettifyColumnName.js'
 import getFullServiceName from './getFullServiceName.js'
-import { makeDatasetSlugToReadableNameFilter, createDatasetMapping } from './makeDatasetSlugToReadableNameFilter.js'
+import { makeDatasetSlugToReadableNameFilter } from './makeDatasetSlugToReadableNameFilter.js'
 
 const { govukMarkdown } = xGovFilters
 
-const addFilters = (nunjucksEnv, { dataSubjects }) => {
-  const datasetNameMapping = createDatasetMapping(dataSubjects)
+const addFilters = (nunjucksEnv, { datasetNameMapping }) => {
   const datasetSlugToReadableName = makeDatasetSlugToReadableNameFilter(datasetNameMapping)
   nunjucksEnv.addFilter('datasetSlugToReadableName', datasetSlugToReadableName)
 

src/filters/getFullServiceName.js

@@ -1,7 +1,10 @@
 import config from '../../config/index.js'
 
-export default (service) => {
-  const serviceName = config.serviceName
-
-  return serviceName.replace('Provide', service)
+const getFullServiceName = (service) => {
+  if (!service || typeof service !== 'string') {
+    throw new Error('Service name must be a non-empty string')
+  }
+  return config.serviceName.replace('Provide', service)
 }
+
+export default getFullServiceName

src/filters/makeDatasetSlugToReadableNameFilter.js

@@ -20,24 +20,9 @@ export const makeDatasetSlugToReadableNameFilter = (datasetNameMapping) => {
     if (!name) {
       // throw new Error(`Can't find a name for ${slug}`)
       // ToDo: work out what to do here? potentially update it with data from datasette
-      logger.warning(`can't find a name for ${slug}`)
+      logger.warn(`can't find a name for ${slug}`)
       return slug
     }
     return name
   }
 }
-
-/**
- *
- * @param {*} dataSubjects
- * @returns {Map<string,string>}
- */
-export const createDatasetMapping = (dataSubjects) => {
-  const mapping = new Map()
-  for (const data of Object.values(dataSubjects)) {
-    for (const dataset of data.dataSets) {
-      mapping.set(dataset.value, dataset.text)
-    }
-  }
-  return mapping
-}

src/serverSetup/nunjucks.js

@@ -2,7 +2,7 @@ import nunjucks from 'nunjucks'
 import config from '../../config/index.js'
 import addFilters from '../filters/filters.js'
 
-export function setupNunjucks ({ app, dataSubjects }) {
+export function setupNunjucks ({ app, datasetNameMapping }) {
   if (app) {
     app.set('view engine', 'html')
   }
@@ -33,7 +33,7 @@ export function setupNunjucks ({ app, dataSubjects }) {
   Object.keys(globalValues).forEach((key) => {
     nunjucksEnv.addGlobal(key, globalValues[key])
   })
-  addFilters(nunjucksEnv, { dataSubjects })
+  addFilters(nunjucksEnv, { datasetNameMapping })
 
   return nunjucks
 }

src/services/datasette.js

@@ -5,15 +5,44 @@ const datasetteUrl = 'https://datasette.planning.data.gov.uk'
 const database = 'digital-land'
 
 export default {
+  /**
+ * Executes a SQL query on the Datasette instance and returns the results.
+ *
+ * @param {string} query - The SQL query to execute.
+ * @returns {Promise<{data: object, formattedData: object}>} - A promise that resolves to an object with the following properties:
+ *   - `data`: The raw data returned by Datasette.
+ *   - `formattedData`: The formatted data, with columns and rows parsed into a usable format.
+ * @throws {Error} If the query fails or there is an error communicating with Datasette.
+ */
   runQuery: async (query) => {
     const encodedQuery = encodeURIComponent(query)
     const url = `${datasetteUrl}/${database}.json?sql=${encodedQuery}`
     try {
       const response = await axios.get(url)
-      return response.data
+      return {
+        ...response.data,
+        formattedData: formatData(response.data.columns, response.data.rows)
+      }
     } catch (error) {
       logger.warn(error)
       throw error
     }
   }
 }
+
+/**
+ * Formats an array of rows into an easier to access format, where each row is an object with column names as keys.
+ *
+ * @param {string[]} columns - An array of column names
+ * @param {any[][]} rows - A 2D array of row data, where each inner array represents a row
+ * @returns {object[]} - An array of objects, where each object represents a row with column names as keys
+ */
+export function formatData (columns, rows) {
+  // convert the rows into an easier to access format
+  return rows.map((row) => {
+    return row.reduce((acc, val, index) => {
+      acc[columns[index]] = val
+      return acc
+    }, {})
+  })
+}

src/services/performanceDbApi.js

@@ -86,19 +86,10 @@ ORDER BY
 
     const result = await datasette.runQuery(query)
 
-    // convert the rows into an easier to access format
-    const columns = result.columns
-    const rows = result.rows.map((row) => {
-      return row.reduce((acc, val, index) => {
-        acc[columns[index]] = val
-        return acc
-      }, {})
-    })
-
-    const datasets = rows.reduce((accumulator, row) => {
+    const datasets = result.formattedData.reduce((accumulator, row) => {
       let error
-      if (row.http_status !== '200' || row.exception !== '') {
-        error = row.exception !== '' ? row.exception : `endpoint returned with a status of ${row.http_status}`
+      if (row.http_status !== '200' || row.exception) {
+        error = row.exception ? row.exception : `endpoint returned with a status of ${row.http_status}`
       }
 
       let issue
@@ -115,7 +106,7 @@ ORDER BY
     }, {})
 
     return {
-      name: result.rows[0][1],
+      name: result.formattedData[0].name,
       datasets
     }
   }

src/services/fetchLocalAuthorities.js → src/utils/datasetteQueries/fetchLocalAuthorities.js

@@ -1,5 +1,5 @@
-import axios from 'axios'
-import logger from '../../src/utils/logger.js'
+import datasette from '../../services/datasette.js'
+import logger from '../logger.js'
 
 /**
  * Fetches a list of local authority names from a specified dataset.
@@ -24,15 +24,14 @@ export const fetchLocalAuthorities = async () => {
     order by
       provision.organisation`
 
-  const url = `https://datasette.planning.data.gov.uk/digital-land.json?sql=${encodeURIComponent(sql)}`
   try {
-    const response = await axios.get(url)
-    const names = response.data.rows.map(row => {
-      if (row[1] === null) {
+    const response = await datasette.runQuery(sql)
+    const names = response.formattedData.map(row => {
+      if (row.name == null) {
         logger.debug('Null value found in response:', row)
         return null
       } else {
-        return row[1]
+        return row.name
       }
     }).filter(name => name !== null) // Filter out null values
     return names

src/utils/datasetteQueries/getDatasetSlugNameMapping.js

@@ -0,0 +1,11 @@
+import datasette from '../../services/datasette.js'
+
+export const getDatasetSlugNameMapping = async () => {
+  const datasetSlugNameTable = await datasette.runQuery('select dataset, name from dataset')
+
+  const datasetMapping = new Map()
+  datasetSlugNameTable.rows.forEach(([slug, name]) => {
+    datasetMapping.set(slug, name)
+  })
+  return datasetMapping
+}

test/unit/check-answers.test.js

@@ -5,7 +5,6 @@ import { setupNunjucks } from '../../src/serverSetup/nunjucks.js'
 import { runGenericPageTests } from './generic-page.js'
 import config from '../../config/index.js'
 import { stripWhitespace } from '../utils/stripWhiteSpace.js'
-import { mockDataSubjects } from './data.js'
 
 describe('check-answers View', async () => {
   const params = {
@@ -19,7 +18,7 @@ describe('check-answers View', async () => {
       hasLicence: 'true'
     }
   }
-  const nunjucks = setupNunjucks({ dataSubjects: mockDataSubjects })
+  const nunjucks = setupNunjucks({ datasetNameMapping: new Map() })
   const html = stripWhitespace(nunjucks.render('check-answers.html', params))
 
   runGenericPageTests(html, {
@@ -43,7 +42,7 @@ describe('check-answers View', async () => {
   })
 
   it('should render the dataset entered', () => {
-    const datasetRegex = new RegExp('<div class="govuk-summary-list__row">.*Dataset.*A Mock dataset.*Change.*</div>', 'g')
+    const datasetRegex = new RegExp('<div class="govuk-summary-list__row">.*Dataset.*mockDataset.*Change.*</div>', 'g')
     expect(html).toMatch(datasetRegex)
   })
 

test/unit/check/confirmationPage.test.js

@@ -5,9 +5,8 @@ import { setupNunjucks } from '../../../src/serverSetup/nunjucks.js'
 import { runGenericPageTests } from '../generic-page.js'
 import config from '../../../config/index.js'
 import { stripWhitespace } from '../../utils/stripWhiteSpace.js'
-import { mockDataSubjects } from '../data.js'
 
-const nunjucks = setupNunjucks({ dataSubjects: mockDataSubjects })
+const nunjucks = setupNunjucks({ datasetNameMapping: new Map() })
 
 describe('Check confirmation View', () => {
   const params = {
@@ -18,12 +17,12 @@ describe('Check confirmation View', () => {
   const html = stripWhitespace(nunjucks.render('submit/confirmation.html', params))
 
   runGenericPageTests(html, {
-    pageTitle: 'A Mock dataset submitted - Submit planning and housing data for England',
+    pageTitle: 'mockDataset submitted - Submit planning and housing data for England',
     serviceName: config.serviceName
   })
 
   it('should render the gov uk panel', () => {
-    const regex = new RegExp('<h1 class="govuk-panel__title".*A Mock dataset submitted.*</h1>', 'g')
+    const regex = new RegExp('<h1 class="govuk-panel__title".*mockDataset submitted.*</h1>', 'g')
     expect(html).toMatch(regex)
   })
 })

test/unit/choose-datasetPage.test.js

@@ -4,7 +4,7 @@ import { runGenericPageTests } from './generic-page.js'
 import config from '../../config/index.js'
 import { testValidationErrorMessage } from './validation-tests.js'
 
-const nunjucks = setupNunjucks({ dataSubjects: {} })
+const nunjucks = setupNunjucks({ datasetNameMapping: new Map() })
 
 describe('choose dataset View', () => {
   const params = {

test/unit/dataset-details.test.js

@@ -8,7 +8,7 @@ import { stripWhitespace } from '../utils/stripWhiteSpace.js'
 import { testValidationErrorMessage } from './validation-tests.js'
 import { mockDataSubjects } from './data.js'
 
-const nunjucks = setupNunjucks({ dataSubjects: mockDataSubjects })
+const nunjucks = setupNunjucks({ datasetNameMapping: new Map() })
 
 function errorTestFn ({
   params,
@@ -41,7 +41,7 @@ describe('dataset details View', () => {
     errors: {}
   }
   const html = stripWhitespace(nunjucks.render('dataset-details.html', params))
-  const datasetName = mockDataSubjects.mockDataset.dataSets[0].text
+  const datasetName = mockDataSubjects.mockDataset.dataSets[0].value
   runGenericPageTests(html, {
     pageTitle: `Enter ${datasetName.toLowerCase()} details - Submit planning and housing data for England`,
     serviceName: config.serviceName

test/unit/datasette.test.js

@@ -0,0 +1,55 @@
+import datasette, { formatData } from '../../src/services/datasette.js'
+import axios from 'axios'
+import { vi, describe, beforeEach, afterEach, it, expect } from 'vitest'
+
+describe('datasette', () => {
+  beforeEach(() => {
+    vi.spyOn(axios, 'get').mockResolvedValue({
+      data: {
+        columns: ['column1', 'column2'],
+        rows: [
+          ['value1', 'value2'],
+          ['value3', 'value4']
+        ]
+      }
+    })
+  })
+
+  afterEach(() => {
+    vi.mocked(axios.get).mockReset()
+  })
+
+  it('runs a SQL query and returns the results', async () => {
+    const query = 'SELECT * FROM table_name'
+    const result = await datasette.runQuery(query)
+
+    expect(result).toEqual({
+      columns: ['column1', 'column2'],
+      formattedData: [
+        { column1: 'value1', column2: 'value2' },
+        { column1: 'value3', column2: 'value4' }
+      ],
+      rows: [
+        ['value1', 'value2'],
+        ['value3', 'value4']
+      ]
+    })
+  })
+
+  it('throws an error if the query fails', async () => {
+    vi.spyOn(axios, 'get').mockRejectedValue(new Error('Query failed'))
+
+    await expect(datasette.runQuery('SELECT * FROM table_name')).rejects.toThrowError('Query failed')
+  })
+
+  it('formats data correctly', () => {
+    const columns = ['column1', 'column2']
+    const rows = [['value1', 'value2'], ['value3', 'value4']]
+    const formattedData = formatData(columns, rows)
+
+    expect(formattedData).toEqual([
+      { column1: 'value1', column2: 'value2' },
+      { column1: 'value3', column2: 'value4' }
+    ])
+  })
+})

test/unit/endpointSubmissionForm/confirmationPage.test.js

@@ -5,9 +5,8 @@ import { setupNunjucks } from '../../../src/serverSetup/nunjucks.js'
 import { runGenericPageTests } from '../generic-page.js'
 import config from '../../../config/index.js'
 import { stripWhitespace } from '../../utils/stripWhiteSpace.js'
-import { mockDataSubjects } from '../data.js'
 
-const nunjucks = setupNunjucks({ dataSubjects: mockDataSubjects })
+const nunjucks = setupNunjucks({ datasetNameMapping: new Map() })
 
 describe('Submit confirmation View', () => {
   const params = {
@@ -18,12 +17,12 @@ describe('Submit confirmation View', () => {
   const html = stripWhitespace(nunjucks.render('submit/confirmation.html', params))
 
   runGenericPageTests(html, {
-    pageTitle: 'A Mock dataset submitted - Submit planning and housing data for England',
+    pageTitle: 'mockDataset submitted - Submit planning and housing data for England',
     serviceName: config.serviceName
   })
 
   it('should render the gov uk panel', () => {
-    const regex = new RegExp('<h1 class="govuk-panel__title".*A Mock dataset submitted.*</h1>', 'g')
+    const regex = new RegExp('<h1 class="govuk-panel__title".*mockDataset submitted.*</h1>', 'g')
     expect(html).toMatch(regex)
   })
 })