Skip to content

Commit

Permalink
fix: pins cron (#1811)
Browse files Browse the repository at this point in the history
`node-fetch` does not support `getReader` (or at least the version we were using does not):

```
 TypeError: stream.getReader is not a function
    at ndjsonParse (file:///home/runner/work/nft.storage/nft.storage/node_modules/@nftstorage/ipfs-cluster/src/index.js:627:25)
    at ndjsonParse.next (<anonymous>)
    at streamRequest (file:///home/runner/work/nft.storage/nft.storage/node_modules/@nftstorage/ipfs-cluster/src/index.js:332:10)
    at runMicrotasks (<anonymous>)
    at processTicksAndRejections (node:internal/process/task_queues:96:5)
    at async statusAll (file:///home/runner/work/nft.storage/nft.storage/node_modules/@nftstorage/ipfs-cluster/src/index.js:202:20)
    at async file:///home/runner/work/nft.storage/nft.storage/packages/cron/src/jobs/pins.js:185:24
    at async mapAndQueue (file:///home/runner/work/nft.storage/nft.storage/node_modules/streaming-iterables/dist/index.mjs:836:27)
```
  • Loading branch information
Alan Shaw authored Apr 12, 2022
1 parent 16e5333 commit f112573
Show file tree
Hide file tree
Showing 6 changed files with 17 additions and 15 deletions.
3 changes: 1 addition & 2 deletions packages/cron/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -22,19 +22,18 @@
"license": "(Apache-2.0 OR MIT)",
"dependencies": {
"@nftstorage/ipfs-cluster": "^5.0.1",
"@web-std/fetch": "^4.0.0",
"debug": "^4.3.2",
"dotenv": "^10.0.0",
"form-data": "^4.0.0",
"limiter": "2.0.1",
"nft.storage": "^6.0.0",
"node-fetch": "^2.6.1",
"p-retry": "^4.6.1",
"p-settle": "^5.0.0",
"pg": "^8.7.1",
"streaming-iterables": "^6.0.0"
},
"devDependencies": {
"@types/node-fetch": "^2.5.10",
"npm-run-all": "^4.1.5"
}
}
2 changes: 1 addition & 1 deletion packages/cron/src/bin/pins-failed.js
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
import path from 'path'
import { fileURLToPath } from 'url'
import dotenv from 'dotenv'
import fetch from 'node-fetch'
import fetch from '@web-std/fetch'
import { checkFailedPinStatuses } from '../jobs/pins.js'
import { getPg, getCluster1, getCluster2, getCluster3 } from '../lib/utils.js'

Expand Down
2 changes: 1 addition & 1 deletion packages/cron/src/bin/pins.js
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
import path from 'path'
import { fileURLToPath } from 'url'
import dotenv from 'dotenv'
import fetch from 'node-fetch'
import fetch from '@web-std/fetch'
import { updatePendingPinStatuses } from '../jobs/pins.js'
import { getPg, getCluster1, getCluster2, getCluster3 } from '../lib/utils.js'

Expand Down
4 changes: 2 additions & 2 deletions packages/cron/src/lib/fetch.js
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import fetch from 'node-fetch'
import fetch from '@web-std/fetch'
import retry from 'p-retry'
import debug from 'debug'

Expand All @@ -11,7 +11,7 @@ const RETRY_ATTEMPTS = 5
/**
* @param {import('limiter').RateLimiter} limiter
* @param {string} url
* @param {import('node-fetch').RequestInit} [init]
* @param {RequestInit} [init]
* @returns {Promise<any>}
*/
export async function fetchJSON(limiter, url, init) {
Expand Down
2 changes: 1 addition & 1 deletion packages/cron/src/lib/ipfs.js
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { URL } from 'url'
import fetch from 'node-fetch'
import fetch from '@web-std/fetch'

export class IPFS {
/**
Expand Down
19 changes: 11 additions & 8 deletions yarn.lock
Original file line number Diff line number Diff line change
Expand Up @@ -2930,14 +2930,6 @@
resolved "https://registry.yarnpkg.com/@types/ms/-/ms-0.7.31.tgz#31b7ca6407128a3d2bbc27fe2d21b345397f6197"
integrity sha512-iiUgKzV9AuaEkZqkOLDIvlQiL6ltuZd9tGcW3gwpnX8JbuiuhFlEGmmFXEXkN50Cvq7Os88IY2v0dkDqXYWVgA==

"@types/node-fetch@^2.5.10":
version "2.6.1"
resolved "https://registry.yarnpkg.com/@types/node-fetch/-/node-fetch-2.6.1.tgz#8f127c50481db65886800ef496f20bbf15518975"
integrity sha512-oMqjURCaxoSIsHSr1E47QHzbmzNR5rK8McHuNb11BOM9cHcIK3Avy0s/b2JlXHoQGTYS3NsvWzV1M0iK7l0wbA==
dependencies:
"@types/node" "*"
form-data "^3.0.0"

"@types/node@*", "@types/node@>=13.7.0", "@types/node@^17.0.21":
version "17.0.23"
resolved "https://registry.yarnpkg.com/@types/node/-/node-17.0.23.tgz#3b41a6e643589ac6442bdbd7a4a3ded62f33f7da"
Expand Down Expand Up @@ -3191,6 +3183,17 @@
"@web3-storage/multipart-parser" "^1.0.0"
data-uri-to-buffer "^3.0.1"

"@web-std/fetch@^4.0.0":
version "4.0.0"
resolved "https://registry.yarnpkg.com/@web-std/fetch/-/fetch-4.0.0.tgz#32cb02e38c25518599843a51c4518b7ccb108d2c"
integrity sha512-RZUY1m7WoSsNGLfBeef3oAsmskU/IrlDSCMEakQZjD1csC91bdq3MJG7GiKijKJNg/PKRC45YxOo5yeSrAz5mA==
dependencies:
"@web-std/blob" "^3.0.3"
"@web-std/form-data" "^3.0.2"
"@web3-storage/multipart-parser" "^1.0.0"
data-uri-to-buffer "^3.0.1"
mrmime "^1.0.0"

"@web-std/file@^3.0.0":
version "3.0.2"
resolved "https://registry.yarnpkg.com/@web-std/file/-/file-3.0.2.tgz#b84cc9ed754608b18dcf78ac62c40dbcc6a94692"
Expand Down

0 comments on commit f112573

Please sign in to comment.