Merge pull request #477 from desci-labs/m0ar/debug-by-publish-date

Debug route only investigates published nodes

Dockerfile

@@ -2,7 +2,7 @@ FROM node:20.8.1-bullseye-slim
 
 VOLUME /root/.yarn
 
-RUN apt-get -qy update && apt-get -qy install openssl
+RUN apt-get -qy update && apt-get -qy install openssl curl
 
 RUN npm install -g npm@9.8.1
 

desci-server/scripts/import-ipfs-content.sh

@@ -1,16 +1,7 @@
 # import required images from ipfs to local
 
-check_and_install() {
-  echo "check and install: $1"
-  # local shell_cmd = $1
-  if ! command -v $1 &> /dev/null;
-  then
-    apt-get install -y $1
-  fi
-}
+set -eux
 
-check_and_install "wget"
-check_and_install "curl"
-
-wget https://ipfs.desci.com/ipfs/bafkreih6yx7ywj7trvpp45vergrnytad7ezsku75tefyro4qrrcfrrmrt4 -O /tmp/cover.png
-curl -F file=@/tmp/cover.png "http://host.docker.internal:5001/api/v0/add?cid-version=1"
+# curl and wget are installed in the docker image
+curl https://ipfs.desci.com/ipfs/bafkreih6yx7ywj7trvpp45vergrnytad7ezsku75tefyro4qrrcfrrmrt4 -o /tmp/cover.png
+curl -F file=@/tmp/cover.png "http://host.docker.internal:5001/api/v0/add?cid-version=1"

desci-server/src/controllers/admin/debug.ts

@@ -1,7 +1,7 @@
 import { Request, Response } from 'express';
 import { logger as parentLogger } from '../../logger.js';
 import { prisma } from '../../client.js';
-import { DataType, Node, NodeVersion, Prisma, PublicDataReference } from '@prisma/client';
+import { DataType, Node, NodeVersion, PublicDataReference } from '@prisma/client';
 import { ensureUuidEndsWithDot } from '../../utils.js';
 import { directStreamLookup } from '../../services/ceramic.js';
 import { getAliasRegistry, getHotWallet } from '../../services/chain.js';
@@ -21,9 +21,9 @@ export const debugNodeHandler = async (
 };
 
 type DebugAllNodesQueryParams = {
+  event?: "first_publish" | "last_publish",
   fromDate?: string,
   toDate?: string,
-  timeColumn?: "createdAt" | "updatedAt",
 };
 
 /** Be gentle with the search scope, as it'll put a fair amount of load on
@@ -33,25 +33,28 @@ export const debugAllNodesHandler = async (
   req: Request<never, never, never, DebugAllNodesQueryParams>,
   res: Response,
 ) => {
-  const { fromDate, toDate, timeColumn } = req.query;
+  const { event, fromDate, toDate } = req.query;
+
+  const timeClause = makeTimeClause(event, fromDate, toDate);
+  const nodes = await prisma.$queryRawUnsafe<{uuid: string}[]>(`
+    select uuid from
+    (
+        select
+          n.uuid,
+          min(nv."createdAt") as first_publish,
+          max(nv."createdAt") as last_publish
+        from "Node" n
+        left join "NodeVersion" nv on n.id = nv."nodeId"
+        where
+            (nv."transactionId" is not null or nv."commitId" is not null)
+        group by n.uuid
+    ) as all_versions
+    ${timeClause};
+  `);
 
-  const nodes = await prisma.node.findMany({
-    select: {
-      uuid: true,
-    },
-    where: {
-      owner: {
-        email: {
-          // Cuts out about 90% :p
-          not: "noreply+test@desci.com"
-        }
-      },
-      ...makeTimeFilter(timeColumn ?? "createdAt", { fromDate, toDate }),
-    }
-  });
   logger.info(
-    { ...req.query, uuids: nodes.map(n => n.uuid) },
-    "handling debugAll query",
+    { event, fromDate, toDate, uuids: nodes.map(n => n.uuid) },
+    "found nodes matching debug range",
   );
 
   const results = await Promise.all(
@@ -60,8 +63,30 @@ export const debugAllNodesHandler = async (
   res.send(results);
 };
 
+const makeTimeClause = (
+  event?: "first_publish" | "last_publish",
+  fromDate?: string,
+  toDate?: string
+) => {
+  if (!fromDate && !toDate) {
+    return "";
+  }
+  let clause = `where all_versions.${event ?? "last_publish"}`;
+
+  if (fromDate) {
+    clause += ` > '${fromDate}'`;
+  };
+
+  if (toDate && fromDate) {
+    clause += `and all_versions.${event ?? "last_publish"} < '${toDate}'`;
+  } else if (toDate) {
+    clause += `< '${toDate}'`;
+  };
+  return clause;
+};
+
 const debugNode = async (uuid: string) => {
-  const node = await prisma.node.findFirst({
+  const node: NodeDbClosure = await prisma.node.findFirst({
     where: {
       uuid: ensureUuidEndsWithDot(uuid),
     },
@@ -98,32 +123,13 @@ const debugNode = async (uuid: string) => {
     createdAt: node.createdAt,
     hasError,
     nVersionsAgree,
-    stream: stream,
-    dpid: dpid,
-    db: database,
-    indexer: indexer,
+    stream,
+    dpid,
+    database,
+    indexer,
   };
 };
 
-const makeTimeFilter = (
-  timeColumn: "createdAt" | "updatedAt",
-  bounds: { fromDate?: string, toDate?: string }
-): Prisma.NodeWhereInput => {
-  const { fromDate, toDate } = bounds;
-
-  let filter: Prisma.DateTimeFilter = {};
-
-  if (fromDate) {
-    filter.gte = new Date(fromDate).toISOString();
-  };
-
-  if (toDate){
-    filter.lte = new Date(toDate).toString();
-  };
-
-  return { [timeColumn]: filter };
-};
-
 type NodeDbClosure = Node & {
   versions: NodeVersion[],
   PublicDataReference: PublicDataReference[],
@@ -171,10 +177,17 @@ const debugStream = async (
 const debugDpid = async (dpid?: number) => {
   if (!dpid) return { present: false, error: false };
 
-  const wallet = await getHotWallet();
-  const registry = getAliasRegistry(wallet);
+  let mappedStream: string;
+  try {
+    const wallet = await getHotWallet();
+    const registry = getAliasRegistry(wallet);
+
+    mappedStream = await registry.resolve(dpid);
+  } catch (e) {
+    const err = e as Error;
+    return { present: true, error: true, name: err.name, msg: err.message, stack: err.stack };
+  };
 
-  const mappedStream = await registry.resolve(dpid);
   if (!mappedStream) {
     return { present: true, error: true, mappedStream: null };
   };

desci-server/src/logger.ts

@@ -133,6 +133,13 @@ function omitBuffer(array) {
   });
 }
 
+// These should probably exit the process as it is not safe to continue
+// execution after a generic exception has occurred:
+// https://nodejs.org/api/process.html#warning-using-uncaughtexception-correctly
 process.on('uncaughtException', (err) => {
   logger.fatal(err, 'uncaught exception');
 });
+
+process.on('unhandledRejection', (reason, promise) => {
+  logger.fatal({ reason, promise }, 'unhandled rejection');
+});

dockerDev.sh

@@ -42,6 +42,7 @@ assert_command_available "docker"
 assert_command_available "docker-compose"
 assert_command_available "lsof"
 assert_command_available "make"
+assert_command_available "curl"
 
 init_node
 npm i -g hardhat