feat: cron and pinpin rewire (#531)

.github/workflows/cron.yml

@@ -0,0 +1,23 @@
+name: cron
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - 'packages/cron/**'
+      - '.github/workflows/cron.yml'
+  pull_request:
+    paths:
+      - 'packages/cron/**'
+      - '.github/workflows/cron.yml'
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    name: Test
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v2
+        with:
+          node-version: 16
+      - uses: bahmutov/npm-install@v1
+      - run: npm test --workspace packages/cron

packages/cron/README.md

@@ -15,8 +15,22 @@ To run this locally you will need the following in your `packages/cron/.env` fil
 
 ```ini
 ENV=dev
+DATABASE=postgres
+
+# PostgREST API URL
+DEV_PG_REST_URL=http://localhost:3000
+# PostgREST API token, for role "postgres", using secret value PGRST_JWT_SECRET from './postgres/docker/docker-compose.yml'
+# https://postgrest.org/en/v8.0/tutorials/tut1.html#step-3-sign-a-token
+DEV_PG_REST_JWT=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJyb2xlIjoicG9zdGdyZXMifQ.oM0SXF31Vs1nfwCaDxjlczE237KcNKhTpKEYxMX-jEU
+
+# Connection string for locally running postgres used in tests
+DEV_PG_CONNECTION=postgres://postgres:postgres@127.0.0.1:5432/postgres
+
+# Cluster
 CLUSTER_API_URL=http://127.0.0.1:9094/
 CLUSTER_IPFS_PROXY_API_URL=http://127.0.0.1:9095/api/v0/
+
+# Fauna
 DEV_FAUNA_KEY="<your key here>"
 ```
 
@@ -31,25 +45,3 @@ Run the job:
 ```sh
 npm run start:pins
 ```
-
-### pinata
-
-Fetch the oldest 600 PinRequests from the DB and pin them on Piñata
-
-To run this locally you will need the following in your `packages/cron/.env` file:
-
-```ini
-ENV=dev
-DEV_FAUNA_KEY="<your key here>"
-PINATA_JWT="<your jwt here>"
-```
-
-You also need to have:
-
-- a dev account and db set up on FaunaDB with the latest schema imported as per [../db/README.md](../db/README.md)
-
-Run the job:
-
-```sh
-npm run start:pinata
-```

packages/cron/package.json

@@ -9,7 +9,11 @@
     "start": "run-s start:*",
     "start:metrics": "node src/bin/metrics.js",
     "start:pins": "node src/bin/pins.js",
-    "start:pinata": "node src/bin/pinata.js"
+    "start:dagcargo:views": "NODE_TLS_REJECT_UNAUTHORIZED=0 node src/bin/dagcargo-views.js",
+    "test": "npm-run-all -p -r mock:cluster mock:pgrest test:e2e",
+    "test:e2e": "mocha test/*.spec.js --exit",
+    "mock:cluster": "smoke -p 9094 test/mocks/cluster",
+    "mock:pgrest": "smoke -p 9087 test/mocks/pgrest"
   },
   "author": "Alan Shaw",
   "license": "(Apache-2.0 AND MIT)",
@@ -20,12 +24,16 @@
     "debug": "^4.3.1",
     "dotenv": "^9.0.2",
     "limiter": "2.0.1",
+    "pg": "^8.7.1",
     "node-fetch": "^2.6.1",
     "p-retry": "^4.6.1",
     "piggybacker": "^2.0.0"
   },
   "devDependencies": {
     "@types/node": "^16.3.1",
-    "npm-run-all": "^4.1.5"
+    "execa": "^5.1.1",
+    "mocha": "^8.3.2",
+    "npm-run-all": "^4.1.5",
+    "smoke": "^3.1.1"
   }
 }

packages/cron/src/bin/dagcargo-views.js

@@ -0,0 +1,19 @@
+#!/usr/bin/env node
+
+import dotenv from 'dotenv'
+import { refreshMaterializedViews } from '../jobs/dagcargo.js'
+import { getPg } from '../lib/utils.js'
+
+async function main () {
+  const pg = getPg(process.env)
+  await pg.connect()
+
+  try {
+    await refreshMaterializedViews({ pg })
+  } finally {
+    await pg.end()
+  }
+}
+
+dotenv.config()
+main()

packages/cron/src/jobs/dagcargo.js

@@ -0,0 +1,26 @@
+import debug from 'debug'
+
+/**
+ * Refreshes the materialized views.
+ *
+ * @param {{ pg: import('pg').Client }} config
+ */
+export async function refreshMaterializedViews ({ pg }) {
+  const log = debug('dagcargo:refreshMaterializedViews')
+  if (!log.enabled) {
+    console.log(
+      'ℹ️ Enable logging by setting DEBUG=dagcargo:refreshMaterializedViews'
+    )
+  }
+
+  log('🔁 REFRESH MATERIALIZED VIEW CONCURRENTLY public.deal;')
+  await pg.query('REFRESH MATERIALIZED VIEW CONCURRENTLY public.deal;')
+
+  log('🔁 REFRESH MATERIALIZED VIEW CONCURRENTLY public.aggregate;')
+  await pg.query('REFRESH MATERIALIZED VIEW CONCURRENTLY public.aggregate;')
+
+  log('🔁 REFRESH MATERIALIZED VIEW CONCURRENTLY public.aggregate_entry;')
+  await pg.query('REFRESH MATERIALIZED VIEW CONCURRENTLY public.aggregate_entry;')
+
+  log('✅ Done')
+}