perf(api): optimise count retrieval

.changeset/nine-trees-eat.md

@@ -0,0 +1,5 @@
+---
+"@blobscan/api": patch
+---
+
+Optimized element count performance by retrieving pre-calculated values from the stats table when filtering by rollup, date, or with no filters applied

packages/api/src/middlewares/withFilters.ts

@@ -1,5 +1,5 @@
 import type { Prisma } from "@blobscan/db";
-import type { Rollup } from "@blobscan/db/prisma/enums";
+import type { Category, Rollup } from "@blobscan/db/prisma/enums";
 import { z } from "@blobscan/zod";
 
 import { t } from "../trpc-client";
@@ -26,8 +26,8 @@ export type Filters = Partial<{
   blockSlot: NumberRange;
   blockType: Prisma.TransactionForkListRelationFilter;
   transactionAddresses: Prisma.TransactionWhereInput["OR"];
-  transactionCategory: Prisma.TransactionWhereInput["category"];
-  transactionRollup: Prisma.TransactionWhereInput["rollup"];
+  transactionCategory: Category;
+  transactionRollup: Rollup | null;
 
   sort: Prisma.SortOrder;
 }>;

packages/api/src/routers/blob/getCount.ts

@@ -3,68 +3,87 @@ import { z } from "@blobscan/zod";
 
 import type { Filters } from "../../middlewares/withFilters";
 import {
-  hasCustomFilters,
   withAllFiltersSchema,
   withFilters,
 } from "../../middlewares/withFilters";
 import { publicProcedure } from "../../procedures";
+import { buildStatsWhereClause, requiresDirectCount } from "../../utils/count";
 
 const inputSchema = withAllFiltersSchema;
 
 const outputSchema = z.object({
   totalBlobs: z.number(),
 });
 
+/**
+ * Counts blobs based on the provided filters.
+ *
+ * This function decides between counting blobs directly from the blob table
+ * or using pre-calculated aggregated data from daily or overall blob stats
+ * to improve performance.
+ *
+ * The choice depends on the specificity of the filters provided.
+ */
 export async function countBlobs(
   prisma: BlobscanPrismaClient,
   filters: Filters
 ) {
-  if (!hasCustomFilters(filters)) {
-    const overallStats = await prisma.blobOverallStats.findFirst({
-      select: {
-        totalBlobs: true,
-      },
-      where: {
-        AND: [{ category: null }, { rollup: null }],
-      },
-    });
-
-    return overallStats?.totalBlobs ?? 0;
-  }
-
   const {
+    blockNumber,
+    blockTimestamp,
     transactionAddresses,
     transactionCategory,
     transactionRollup,
     blockSlot,
     blockType,
   } = filters;
 
-  const txFiltersExists =
-    transactionRollup !== undefined ||
-    transactionAddresses ||
-    transactionCategory;
-  const blockFiltersExists = blockSlot || blockType;
+  if (requiresDirectCount(filters)) {
+    const transactionFiltersEnabled =
+      transactionCategory || transactionRollup || transactionAddresses;
 
-  return prisma.blobsOnTransactions.count({
-    where: {
-      blockNumber: filters.blockNumber,
-      blockTimestamp: filters.blockTimestamp,
-      block: blockFiltersExists
-        ? {
-            slot: filters.blockSlot,
-            transactionForks: filters.blockType,
-          }
-        : undefined,
-      transaction: txFiltersExists
-        ? {
-            category: filters.transactionCategory,
-            rollup: filters.transactionRollup,
-            OR: filters.transactionAddresses,
-          }
-        : undefined,
+    return prisma.blobsOnTransactions.count({
+      where: {
+        blockNumber,
+        blockTimestamp,
+        block: {
+          slot: blockSlot,
+          transactionForks: blockType,
+        },
+        transaction: transactionFiltersEnabled
+          ? {
+              category: transactionCategory,
+              rollup: transactionRollup,
+              OR: transactionAddresses,
+            }
+          : undefined,
+      },
+    });
+  }
+
+  const where = buildStatsWhereClause(filters);
+
+  // Get count by summing daily total transaction stats data if a date range is provided in filters
+  if (filters.blockTimestamp) {
+    const dailyStats = await prisma.blobDailyStats.findMany({
+      select: {
+        day: true,
+        totalBlobs: true,
+      },
+      where,
+    });
+
+    return dailyStats.reduce((acc, { totalBlobs }) => acc + totalBlobs, 0);
+  }
+
+  const overallStats = await prisma.blobOverallStats.findFirst({
+    select: {
+      totalBlobs: true,
     },
+    where,
   });
+
+  return overallStats?.totalBlobs ?? 0;
 }
 
 export const getCount = publicProcedure

packages/api/src/routers/tx/getCount.ts

@@ -3,32 +3,29 @@ import { z } from "@blobscan/zod";
 
 import type { Filters } from "../../middlewares/withFilters";
 import {
-  hasCustomFilters,
   withAllFiltersSchema,
   withFilters,
 } from "../../middlewares/withFilters";
 import { publicProcedure } from "../../procedures";
+import { buildStatsWhereClause, requiresDirectCount } from "../../utils/count";
 
 const inputSchema = withAllFiltersSchema;
 
 const outputSchema = z.object({
   totalTransactions: z.number(),
 });
 
+/**
+ * Counts transactions based on the provided filters.
+ *
+ * This function decides between counting transactions directly from the transaction table
+ * or using pre-calculated aggregated data from daily or overall transaction stats to
+ * improve performance.
+ *
+ * The choice depends on the specificity of the filters provided.
+ *
+ */
 export async function countTxs(prisma: BlobscanPrismaClient, filters: Filters) {
-  if (!hasCustomFilters(filters)) {
-    const overallStats = await prisma.transactionOverallStats.findFirst({
-      select: {
-        totalTransactions: true,
-      },
-      where: {
-        AND: [{ category: null }, { rollup: null }],
-      },
-    });
-
-    return overallStats?.totalTransactions ?? 0;
-  }
-
   const {
     blockNumber,
     blockTimestamp,
@@ -39,23 +36,47 @@ export async function countTxs(prisma: BlobscanPrismaClient, filters: Filters) {
     blockType,
   } = filters;
 
-  const blockFiltersExists = blockSlot || blockType;
+  if (requiresDirectCount(filters)) {
+    return prisma.transaction.count({
+      where: {
+        blockNumber: blockNumber,
+        blockTimestamp: blockTimestamp,
+        category: transactionCategory,
+        rollup: transactionRollup,
+        OR: transactionAddresses,
+        block: {
+          slot: blockSlot,
+          transactionForks: blockType,
+        },
+      },
+    });
+  }
+
+  const where = buildStatsWhereClause(filters);
 
-  return prisma.transaction.count({
-    where: {
-      blockNumber: blockNumber,
-      blockTimestamp: blockTimestamp,
-      category: transactionCategory,
-      rollup: transactionRollup,
-      OR: transactionAddresses,
-      block: blockFiltersExists
-        ? {
-            slot: blockSlot,
-            transactionForks: blockType,
-          }
-        : undefined,
+  // Get count by summing daily total transaction stats data if a date range is provided in filters
+  if (filters.blockTimestamp) {
+    const dailyStats = await prisma.transactionDailyStats.findMany({
+      select: {
+        totalTransactions: true,
+      },
+      where,
+    });
+
+    return dailyStats.reduce(
+      (acc, { totalTransactions }) => acc + totalTransactions,
+      0
+    );
+  }
+
+  const overallStats = await prisma.transactionOverallStats.findFirst({
+    select: {
+      totalTransactions: true,
     },
+    where,
   });
+
+  return overallStats?.totalTransactions ?? 0;
 }
 
 export const getCount = publicProcedure

packages/api/src/utils/count.ts

@@ -0,0 +1,89 @@
+import { toDailyDatePeriod } from "@blobscan/dayjs";
+import { env } from "@blobscan/env";
+import { getRollupByAddress } from "@blobscan/rollups";
+
+import type { Filters } from "../middlewares/withFilters";
+
+function getRollupFromAddressFilter(
+  addressesFilter: Filters["transactionAddresses"]
+) {
+  if (!addressesFilter) {
+    return;
+  }
+
+  const fromAddress = addressesFilter.find(({ fromId }) => !!fromId)?.fromId;
+
+  if (!fromAddress || typeof fromAddress !== "string") return;
+
+  return getRollupByAddress(fromAddress, env.CHAIN_ID);
+}
+
+/**
+ * Determines if a direct count operation must be performed or the value can be obtained by using
+ * pre-calculated values from aggregated stats tables, based on the provided filters.
+ *
+ * Aggregated stats are not available for certain filters, including:
+ * - Reorged blocks (`blockType` filter)
+ * - Specific `blockNumber` or `blockSlot` ranges
+ * - Filters involving multiple addresses or non-rollup addresses
+ *
+ * This function checks for those cases and returns `true` if a direct count is needed.
+ *
+ * @returns A boolean indicating if a direct count is required.
+ */
+export function requiresDirectCount({
+  blockNumber,
+  blockSlot,
+  transactionAddresses,
+  blockType,
+}: Filters) {
+  const blockNumberRangeFilterEnabled = !!blockNumber;
+  const reorgedFilterEnabled = !!blockType?.some;
+  const slotRangeFilterEnabled = !!blockSlot;
+  const addressFiltersCount = transactionAddresses?.length ?? 0;
+  const severalAddressesFilterEnabled = addressFiltersCount > 1;
+  const nonRollupAddressFilterEnabled =
+    addressFiltersCount === 1 &&
+    !getRollupFromAddressFilter(transactionAddresses);
+
+  return (
+    blockNumberRangeFilterEnabled ||
+    slotRangeFilterEnabled ||
+    reorgedFilterEnabled ||
+    severalAddressesFilterEnabled ||
+    nonRollupAddressFilterEnabled
+  );
+}
+
+export function buildStatsWhereClause({
+  blockTimestamp,
+  transactionCategory,
+  transactionRollup,
+  transactionAddresses,
+}: Filters) {
+  const clauses = [];
+  // We set 'category' or 'rollup' to null when there are no corresponding filters
+  // because the db stores total statistics for each grouping in rows where
+  // 'category' or 'rollup' is null.
+  const rollup =
+    transactionRollup ??
+    getRollupFromAddressFilter(transactionAddresses) ??
+    null;
+
+  const category = (rollup ? null : transactionCategory) ?? null;
+
+  clauses.push({ category }, { rollup });
+
+  const { from, to } = toDailyDatePeriod({
+    from: blockTimestamp?.gte,
+    to: blockTimestamp?.lt,
+  });
+
+  if (from || to) {
+    clauses.push({ day: { gte: from, lt: to } });
+  }
+
+  return {
+    AND: clauses,
+  };
+}