-
Notifications
You must be signed in to change notification settings - Fork 167
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Alan Shaw
committed
May 21, 2021
1 parent
604e1b0
commit 2e8efa2
Showing
5 changed files
with
182 additions
and
59 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,16 @@ | ||
| #!/usr/bin/env node | ||
|
|
||
| import dotenv from 'dotenv' | ||
| import { syncPinata } from '../jobs/pinata.js' | ||
| import { getCloudflare, getPinata } from '../lib/utils.js' | ||
|
|
||
| async function main() { | ||
| const env = process.env.ENV || 'dev' | ||
| const cf = getCloudflare(process.env) | ||
| const pinata = getPinata(process.env) | ||
| const hostNodes = (process.env.CLUSTER_ADDRS || '').split(',').filter(Boolean) | ||
| await syncPinata({ cf, pinata, env, hostNodes }) | ||
| } | ||
|
|
||
| dotenv.config() | ||
| main() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,56 @@ | ||
| import debug from 'debug' | ||
| import { findNs } from '../lib/utils.js' | ||
|
|
||
| const log = debug('pinata:syncPinata') | ||
|
|
||
| /** | ||
| * Syncs our pinset with Pinata. | ||
| * | ||
| * @param {import('../types').Config & { | ||
| * pinata: import('../lib/pinata').Pinata | ||
| * hostNodes: string[] | ||
| * }} config | ||
| */ | ||
| export async function syncPinata({ cf, env, pinata, hostNodes }) { | ||
| const namespaces = await cf.fetchKVNamespaces() | ||
| const pinsNs = findNs(namespaces, env, 'PINS') | ||
| log(`🎯 Syncing ${pinsNs.title} to Pinata`) | ||
|
|
||
| let i = 0 | ||
| for await (const keys of cf.fetchKVKeys(pinsNs.id)) { | ||
| log(`📥 Processing ${i} -> ${i + keys.length}`) | ||
|
|
||
| /** @type {import('../lib/cloudflare.js').BulkWritePair[]} */ | ||
| const bulkWrites = [] | ||
|
|
||
| await Promise.all( | ||
| keys.map(async (k) => { | ||
| const { name: cid, metadata: pin } = k | ||
|
|
||
| // if not pinned by us or already pinned on Pinata | ||
| if (pin.status !== 'pinned' || pin.pinataStatus === 'pinned') { | ||
| return | ||
| } | ||
|
|
||
| const pinned = await pinata.isPinned(cid) | ||
| // if pinata has finally pinned it then update status in our KV | ||
| if (pinned) { | ||
| log(`📌 ${cid} became pinned on Pinata!`) | ||
| const metadata = { ...pin, pinataStatus: 'pinned' } | ||
| return bulkWrites.push({ key: cid, value: '', metadata }) | ||
| } | ||
|
|
||
| // submit to Pinata | ||
| log(`🙏 asking Pinata to pin ${cid}`) | ||
| return pinata.pinByHash(cid, { pinataOptions: { hostNodes } }) | ||
| }) | ||
| ) | ||
|
|
||
| if (bulkWrites.length) { | ||
| log(`🗂 updating pinata status for ${bulkWrites.length} pins`) | ||
| await cf.writeKVMulti(pinsNs.id, bulkWrites) | ||
| } | ||
|
|
||
| i += keys.length | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,55 @@ | ||
| import fetch from 'node-fetch' | ||
| import retry from 'p-retry' | ||
| import debug from 'debug' | ||
|
|
||
| const log = debug('fetchJSON') | ||
|
|
||
| const REQUEST_TIMEOUT = 60000 | ||
| const RETRY_INTERVAL = 60000 | ||
| const RETRY_ATTEMPTS = 5 | ||
|
|
||
| /** | ||
| * @param {import('limiter').RateLimiter} limiter | ||
| * @param {string} url | ||
| * @param {import('node-fetch').RequestInit} [init] | ||
| * @returns {Promise<any>} | ||
| */ | ||
| export async function fetchJSON(limiter, url, init) { | ||
| await limiter.removeTokens(1) | ||
| const res = await retry( | ||
| async () => { | ||
| const controller = new AbortController() | ||
| const abortID = setTimeout(() => controller.abort(), REQUEST_TIMEOUT) | ||
| init = init || {} | ||
| // @ts-ignore | ||
| init.signal = controller.signal | ||
| try { | ||
| const res = await fetch(url, init) | ||
| const text = await res.text() | ||
| if (!res.ok) { | ||
| throw Object.assign( | ||
| new Error(`${res.status} ${res.statusText}: ${text}`), | ||
| { response: res } | ||
| ) | ||
| } | ||
| return text === '' ? null : JSON.parse(text) | ||
| } finally { | ||
| clearTimeout(abortID) | ||
| } | ||
| }, | ||
| { | ||
| onFailedAttempt: async (err) => { | ||
| // @ts-ignore | ||
| if (err.response && err.response.status === 429) { | ||
| log(`🚦 rate limited ${url}`) | ||
| } else { | ||
| log(`💥 fetch ${url}`, err) | ||
| } | ||
| await limiter.removeTokens(1) | ||
| }, | ||
| retries: RETRY_ATTEMPTS, | ||
| minTimeout: RETRY_INTERVAL, | ||
| } | ||
| ) | ||
| return res | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters