fix: move to cluster3 (#962)

- [x] set cf secret `CLUSTER_SERVICE` to `IpfsCluster3` - [x] needs `alter type service_type add value 'IpfsCluster3';` in production and staging - [x] cron jobs needs to support nft3 cluster but we need to figure out if we are moving everything. If yes we can just have one single cluster.
nftstorage · Jan 17, 2022 · b6fc2c3 · b6fc2c3
1 parent 24df1f6
commit b6fc2c3
Show file tree

Hide file tree

Showing 12 changed files with 73 additions and 24 deletions.
diff --git a/packages/api/README.md b/packages/api/README.md
@@ -121,7 +121,7 @@ wrangler secret put PINATA_JWT --env production # Get from Pinata
 wrangler secret put SENTRY_DSN --env USER # Get from Sentry
 wrangler secret put DATABASE_TOKEN --env production # Get from database account
 wrangler secret put CLUSTER_BASIC_AUTH_TOKEN --env production # Get from nft.storage vault in 1password
-wrangler secret put CLUSTER_SERVICE --env production # Which cluster should be used. Options 'IpfsCluster' or 'IpfsCluster2'
+wrangler secret put CLUSTER_SERVICE --env production # Which cluster should be used. Options 'IpfsCluster' / 'IpfsCluster2' / 'IpfsCluster3'
 wrangler secret put MAILCHIMP_API_KEY --env production # Get from mailchimp
 wrangler secret put LOGTAIL_TOKEN --env production # Get from Logtail
 wrangler secret put PSA_ALLOW --env production # CSV user ID list, get from 1password vault

diff --git a/packages/api/db/tables.sql b/packages/api/db/tables.sql
@@ -18,7 +18,9 @@ CREATE TYPE service_type AS ENUM (
     -- The original NFT.Storage cluster.
     'IpfsCluster',
     -- The current cluster, originally commissioned for niftysave.
-    'IpfsCluster2'
+    'IpfsCluster2',
+    -- New cluster with flatfs and better DHT
+    'IpfsCluster3'
     );
 
 -- Upload type is the type of received upload data.

diff --git a/packages/api/src/bindings.d.ts b/packages/api/src/bindings.d.ts
@@ -8,7 +8,7 @@ import { Logging } from './utils/logs.js'
 declare global {
   const SALT: string
   const DEBUG: string
-  const CLUSTER_SERVICE: 'IpfsCluster' | 'IpfsCluster2'
+  const CLUSTER_SERVICE: 'IpfsCluster' | 'IpfsCluster2' | 'IpfsCluster3'
   const CLUSTER_API_URL: string
   const CLUSTER_BASIC_AUTH_TOKEN: string
   const MAGIC_SECRET_KEY: string

diff --git a/packages/api/src/constants.js b/packages/api/src/constants.js
@@ -12,6 +12,7 @@ export const secrets = {
 
 const CLUSTER1 = 'https://nft.storage.ipfscluster.io/api/'
 const CLUSTER2 = 'https://nft2.storage.ipfscluster.io/api/'
+const CLUSTER3 = 'https://nft3.storage.ipfscluster.io/api/'
 let clusterUrl
 
 switch (CLUSTER_SERVICE) {
@@ -21,6 +22,9 @@ switch (CLUSTER_SERVICE) {
   case 'IpfsCluster2':
     clusterUrl = CLUSTER2
     break
+  case 'IpfsCluster3':
+    clusterUrl = CLUSTER3
+    break
   default:
     clusterUrl = CLUSTER_API_URL
     break

diff --git a/packages/api/src/utils/db-client.js b/packages/api/src/utils/db-client.js
@@ -7,7 +7,12 @@ import { PostgrestClient, PostgrestQueryBuilder } from '@supabase/postgrest-js'
 /** @type {Array<definitions['upload']['type']>} */
 export const UPLOAD_TYPES = ['Car', 'Blob', 'Multipart', 'Remote', 'Nft']
 /** @type {Array<definitions['pin']['service']>} */
-export const PIN_SERVICES = ['IpfsCluster2', 'IpfsCluster', 'Pinata']
+export const PIN_SERVICES = [
+  'IpfsCluster3',
+  'IpfsCluster2',
+  'IpfsCluster',
+  'Pinata',
+]
 /** @type {Array<definitions['pin']['status']>} */
 export const PIN_STATUSES = ['PinQueued', 'Pinning', 'Pinned', 'PinError']
 
@@ -74,7 +79,7 @@ export class DBClient {
     const defaultPins = [
       {
         status: 'PinQueued',
-        service: 'IpfsCluster2',
+        service: 'IpfsCluster3',
       },
       {
         status: 'PinQueued',
@@ -128,8 +133,12 @@ export class DBClient {
       .eq('source_cid', cid)
       .eq('user_id', userId)
       .is('deleted_at', null)
-      // @ts-ignore
-      .filter('content.pin.service', 'in', '(IpfsCluster,IpfsCluster2)')
+      .filter(
+        // @ts-ignore
+        'content.pin.service',
+        'in',
+        '(IpfsCluster,IpfsCluster2,IpfsCluster3)'
+      )
       .single()
 
     if (status === 406 || !upload) {
@@ -156,8 +165,12 @@ export class DBClient {
       .select(this.uploadQuery)
       .eq('user_id', userId)
       .is('deleted_at', null)
-      // @ts-ignore
-      .filter('content.pin.service', 'in', '(IpfsCluster,IpfsCluster2)')
+      .filter(
+        // @ts-ignore
+        'content.pin.service',
+        'in',
+        '(IpfsCluster,IpfsCluster2,IpfsCluster3)'
+      )
       .limit(opts.limit || 10)
       .order('inserted_at', { ascending: false })
 
@@ -263,7 +276,7 @@ export class DBClient {
         pins:pin(status, service, inserted_at)`
       )
       // @ts-ignore
-      .filter('pins.service', 'in', '(IpfsCluster,IpfsCluster2)')
+      .filter('pins.service', 'in', '(IpfsCluster,IpfsCluster2,IpfsCluster3)')
       .eq('cid', cid)
       .single()
 

diff --git a/packages/api/src/utils/db-types.d.ts b/packages/api/src/utils/db-types.d.ts
@@ -758,7 +758,7 @@ export interface definitions {
      * This is a Foreign Key to `content.cid`.<fk table='content' column='cid'/>
      */
     content_cid: string
-    service: 'Pinata' | 'IpfsCluster' | 'IpfsCluster2'
+    service: 'Pinata' | 'IpfsCluster' | 'IpfsCluster2' | 'IpfsCluster3'
     inserted_at: string
     updated_at: string
   }

diff --git a/packages/api/test/nfts-store.spec.js b/packages/api/test/nfts-store.spec.js
@@ -41,7 +41,8 @@ describe('NFT store', () => {
       result,
       {
         ipnft: 'bafyreicnwbboevx6g6fykitf4nebz2kqgkqz35qvlnlcgfulhrris66m6i',
-        url: 'ipfs://bafyreicnwbboevx6g6fykitf4nebz2kqgkqz35qvlnlcgfulhrris66m6i/metadata.json',
+        url:
+          'ipfs://bafyreicnwbboevx6g6fykitf4nebz2kqgkqz35qvlnlcgfulhrris66m6i/metadata.json',
         data: {
           name: 'name',
           description: 'stuff',
@@ -78,7 +79,7 @@ describe('NFT store', () => {
         content_cid:
           'bafyreicnwbboevx6g6fykitf4nebz2kqgkqz35qvlnlcgfulhrris66m6i',
         status: 'PinQueued',
-        service: 'IpfsCluster2',
+        service: 'IpfsCluster3',
       },
       {
         content_cid:
@@ -115,7 +116,8 @@ describe('NFT store', () => {
       result,
       {
         ipnft: 'bafyreihihnpztkkjaegm2cldghihv4otaa23v2lf6uhmbm2avoolgkfynm',
-        url: 'ipfs://bafyreihihnpztkkjaegm2cldghihv4otaa23v2lf6uhmbm2avoolgkfynm/metadata.json',
+        url:
+          'ipfs://bafyreihihnpztkkjaegm2cldghihv4otaa23v2lf6uhmbm2avoolgkfynm/metadata.json',
         data: {
           name: 'name',
           description: 'stuff',
@@ -143,7 +145,7 @@ describe('NFT store', () => {
         content_cid:
           'bafyreihihnpztkkjaegm2cldghihv4otaa23v2lf6uhmbm2avoolgkfynm',
         status: 'PinQueued',
-        service: 'IpfsCluster2',
+        service: 'IpfsCluster3',
       },
       {
         content_cid:

diff --git a/packages/api/test/nfts-upload.spec.js b/packages/api/test/nfts-upload.spec.js
@@ -252,9 +252,13 @@ describe('NFT Upload ', () => {
       .from('upload')
       .select('*,content(dag_size, pin(status, service, inserted_at))')
       .match({ source_cid: cid, user_id: client.userId })
-      .filter('content.pin.service', 'in', '(IpfsCluster,IpfsCluster2)')
+      .filter(
+        'content.pin.service',
+        'in',
+        '(IpfsCluster,IpfsCluster2,IpfsCluster3)'
+      )
       .single()
 
-    assert.equal(data.content.pin[0].service, 'IpfsCluster2')
+    assert.equal(data.content.pin[0].service, 'IpfsCluster3')
   })
 })
diff --git a/packages/cron/src/bin/pins.js b/packages/cron/src/bin/pins.js
@@ -5,7 +5,12 @@ import { fileURLToPath } from 'url'
 import dotenv from 'dotenv'
 import fetch from 'node-fetch'
 import { updatePendingPinStatuses } from '../jobs/pins.js'
-import { getDBClient, getCluster1, getCluster2 } from '../lib/utils.js'
+import {
+  getDBClient,
+  getCluster1,
+  getCluster2,
+  getCluster3,
+} from '../lib/utils.js'
 
 const __dirname = path.dirname(fileURLToPath(import.meta.url))
 /** @ts-ignore */
@@ -15,8 +20,9 @@ async function main() {
   const db = getDBClient(process.env)
   const cluster1 = getCluster1(process.env)
   const cluster2 = getCluster2(process.env)
+  const cluster3 = getCluster3(process.env)
 
-  await updatePendingPinStatuses({ db, cluster1, cluster2 })
+  await updatePendingPinStatuses({ db, cluster1, cluster2, cluster3 })
 }
 
 dotenv.config({ path: path.join(__dirname, '../../../../.env') })

diff --git a/packages/cron/src/jobs/pinata.js b/packages/cron/src/jobs/pinata.js
@@ -59,7 +59,7 @@ export async function pinToPinata({ db, pinata }) {
       break
     }
 
-    /** @type {PinUpdate[]} */
+    /** @type {Pin[]} */
     const updatedPins = []
     for (const pin of pins) {
       try {

diff --git a/packages/cron/src/jobs/pins.js b/packages/cron/src/jobs/pins.js
@@ -8,6 +8,7 @@ const log = debug('pins:updatePinStatuses')
  *   db: import('../../../api/src/utils/db-client').DBClient
  *   cluster1: import('@nftstorage/ipfs-cluster').Cluster
  *   cluster2: import('@nftstorage/ipfs-cluster').Cluster
+ *   cluster3: import('@nftstorage/ipfs-cluster').Cluster
  * }} Config
  * @typedef {import('../../../api/src/utils/db-types').definitions} definitions
  * @typedef {Pick<definitions['pin'], 'id'|'status'|'content_cid'|'service'|'inserted_at'|'updated_at'>} Pin
@@ -24,7 +25,7 @@ export async function updatePendingPinStatuses(conf) {
     const { count, error: countError } = await conf.db.client
       .from('pin')
       .select('*', { count: 'exact', head: true })
-      .in('service', ['IpfsCluster', 'IpfsCluster2'])
+      .in('service', ['IpfsCluster', 'IpfsCluster2', 'IpfsCluster3'])
       .neq('status', 'Pinned')
       .neq('status', 'PinError')
       .range(0, 1)
@@ -75,7 +76,7 @@ export async function checkFailedPinStatuses(config) {
     const { count, error: countError } = await db.client
       .from('pin')
       .select('*', { count: 'exact', head: true })
-      .in('service', ['IpfsCluster', 'IpfsCluster2'])
+      .in('service', ['IpfsCluster', 'IpfsCluster2', 'IpfsCluster3'])
       .eq('status', 'PinError')
       .gt('inserted_at', after.toISOString())
       .range(0, 1)
@@ -97,7 +98,7 @@ export async function checkFailedPinStatuses(config) {
     const query = db.client.from('pin')
     const { data: pins, error } = await query
       .select('id,status,content_cid,service')
-      .in('service', ['IpfsCluster', 'IpfsCluster2'])
+      .in('service', ['IpfsCluster', 'IpfsCluster2', 'IpfsCluster3'])
       .eq('status', 'PinError')
       .gt('inserted_at', after.toISOString())
       .range(offset, offset + limit - 1)
@@ -123,7 +124,7 @@ export async function checkFailedPinStatuses(config) {
  * }} config
  */
 async function updatePinStatuses(config) {
-  const { countPins, fetchPins, db, cluster1, cluster2 } = config
+  const { countPins, fetchPins, db, cluster1, cluster2, cluster3 } = config
   if (!log.enabled) {
     console.log('ℹ️ Enable logging by setting DEBUG=pins:updatePinStatuses')
   }
@@ -152,6 +153,9 @@ async function updatePinStatuses(config) {
         case 'IpfsCluster2':
           statusRes = await cluster2.status(pin.content_cid)
           break
+        case 'IpfsCluster3':
+          statusRes = await cluster3.status(pin.content_cid)
+          break
         default:
           throw new Error(`Service ${pin.service} not supported.`)
       }

diff --git a/packages/cron/src/lib/utils.js b/packages/cron/src/lib/utils.js
@@ -31,6 +31,20 @@ export function getCluster2(env) {
   })
 }
 
+/**
+ * Create a new IPFS Cluster instance from the passed environment variables.
+ * @param {Record<string, string|undefined>} env
+ */
+export function getCluster3(env) {
+  const clusterApiUrl = env.CLUSTER3_API_URL
+  if (!clusterApiUrl) throw new Error('missing IPFS Cluster API URL')
+  const basicAuthToken = env.CLUSTER3_BASIC_AUTH_TOKEN
+  if (!basicAuthToken) throw new Error('missing IPFS Cluster credentials')
+  return new Cluster(clusterApiUrl, {
+    headers: { Authorization: `Basic ${basicAuthToken}` },
+  })
+}
+
 /**
  * Create a new IPFS client instance from the passed environment variables.
  * @param {Record<string, string|undefined>} env