get entity counts for resources

src/middleware/common.middleware.js

@@ -2,7 +2,7 @@ import logger from '../utils/logger.js'
 import { types } from '../utils/logging.js'
 import { entryIssueGroups } from '../utils/utils.js'
 import performanceDbApi from '../services/performanceDbApi.js'
-import { fetchOne, FetchOptions, FetchOneFallbackPolicy, fetchMany, renderTemplate } from './middleware.builders.js'
+import { fetchOne, FetchOptions, FetchOneFallbackPolicy, fetchMany, renderTemplate, fetchManyFromAllDatasets } from './middleware.builders.js'
 import * as v from 'valibot'
 import { pagination } from '../utils/pagination.js'
 import datasette from '../services/datasette.js'
@@ -199,6 +199,29 @@ export const fetchResources = fetchMany({
   result: 'resources'
 })
 
+export const fetchDatasetResources = fetchManyFromAllDatasets({
+  query: ({ req }) => `
+    SELECT * FROM dataset_resource WHERE end_date = ''
+  `,
+  dataset: FetchOptions.performanceDb,
+  result: 'datasetResources'
+})
+
+export const addLineCountsToResources = (req, res, next) => {
+  const { resources, datasetResources } = req
+
+  req.resources = resources.map(resource => {
+    const thisDatasetResource = datasetResources[resource.dataset]
+    const datasetResource = thisDatasetResource.find(
+      _datasetResource =>
+        _datasetResource.resource === resource.resource
+    )
+    return { ...resource, entry_count: datasetResource.entry_count }
+  })
+
+  next()
+}
+
 // Specification
 
 export const fetchSpecification = fetchOne({

src/middleware/datasetTaskList.middleware.js

@@ -5,7 +5,10 @@ import {
   processEntitiesMiddlewares,
   fetchOrgInfo,
   fetchEntityIssueCounts,
-  fetchEntryIssueCounts
+  fetchEntryIssueCounts,
+  logPageError,
+  fetchDatasetResources,
+  addLineCountsToResources
 } from './common.middleware.js'
 import { fetchOne, renderTemplate } from './middleware.builders.js'
 import performanceDbApi from '../services/performanceDbApi.js'
@@ -163,10 +166,13 @@ export default [
   fetchOrgInfo,
   fetchDatasetInfo,
   fetchResources,
+  fetchDatasetResources,
+  addLineCountsToResources,
   ...processEntitiesMiddlewares,
   fetchEntityIssueCounts,
   fetchEntryIssueCounts,
   prepareTasks,
   prepareDatasetTaskListTemplateParams,
-  getDatasetTaskList
+  getDatasetTaskList,
+  logPageError
 ]

src/middleware/middleware.builders.js

@@ -13,6 +13,7 @@ import { templateSchema } from '../routes/schemas.js'
 import { render } from '../utils/custom-renderer.js'
 import datasette from '../services/datasette.js'
 import * as v from 'valibot'
+import { availableDatasets } from '../utils/utils.js'
 
 export const FetchOptions = {
   /**
@@ -114,6 +115,40 @@ export async function fetchManyFn (req, res, next) {
   }
 }
 
+export async function fetchOneFromAllDatasetsFn (req, res, next) {
+  try {
+    const query = this.query({ req, params: req.params })
+    const promises = availableDatasets.map((dataset) => {
+      return datasette.runQuery(query, dataset)
+    })
+    const result = await Promise.all(promises)
+    req[this.result] = Object.fromEntries(result.map(({ formattedData }, i) => [availableDatasets[i], formattedData[0]]))
+    logger.debug({ type: types.DataFetch, message: 'fetchOneFromAllDatasets', resultKey: this.result })
+    next()
+  } catch (error) {
+    logger.debug('fetchMany: failed', { type: types.DataFetch, errorMessage: error.message, endpoint: req.originalUrl, resultKey: this.result })
+    req.handlerName = `fetching '${this.result}'`
+    next(error)
+  }
+}
+
+export async function fetchManyFromAllDatasetsFn (req, res, next) {
+  try {
+    const query = this.query({ req, params: req.params })
+    const promises = availableDatasets.map((dataset) => {
+      return datasette.runQuery(query, dataset)
+    })
+    const result = await Promise.all(promises)
+    req[this.result] = Object.fromEntries(result.map(({ formattedData }, i) => [availableDatasets[i], formattedData]))
+    logger.debug({ type: types.DataFetch, message: 'fetchManyFromAllDatasets', resultKey: this.result })
+    next()
+  } catch (error) {
+    logger.debug('fetchMany: failed', { type: types.DataFetch, errorMessage: error.message, endpoint: req.originalUrl, resultKey: this.result })
+    req.handlerName = `fetching '${this.result}'`
+    next(error)
+  }
+}
+
 /**
    * Middleware. Does a conditional fetch. Optionally invokes `else` if condition is false.
    *
@@ -178,6 +213,24 @@ export function fetchMany (context) {
   return fetchManyFn.bind(context)
 }
 
+/**
+   * Fetches a single record from each dataset databases and stores them in `req` under key specified by `result` entry.
+   *
+   * @param {{query: ({req, params}) => object, result: string, dataset?: FetchParams | (req) => string}} context
+   */
+export function fetchOneFromAllDatasets (context) {
+  return fetchOneFromAllDatasetsFn.bind(context)
+}
+
+/**
+   * Fetches a collection of records from all dataset databases and stores them in `req` under key specified by `result` entry.
+   *
+   * @param {{query: ({req, params}) => object, result: string, dataset?: FetchParams | (req) => string}} context
+   */
+export function fetchManyFromAllDatasets (context) {
+  return fetchManyFromAllDatasetsFn.bind(context)
+}
+
 /**
  * Looks up schema for name in @{link templateSchema} (defaults to any()), validates and renders the template.
  *