feat: add nft.storage source

classic-nft.storage.js

@@ -0,0 +1,116 @@
+export const API = 'https://api.nft.storage'
+
+export function getClient ({
+  api = API,
+  token
+}) {
+  if (!token) {
+    console.log('! run `nft token` to set an API token to use')
+    process.exit(-1)
+  }
+  const endpoint = new URL(api)
+  if (api !== API) {
+    // note if we're using something other than prod.
+    console.log(`using ${endpoint.hostname}`)
+  }
+  return new NftStorage({ token, endpoint })
+}
+
+/**
+ * @typedef {object} Service
+ * @property {URL} [endpoint]
+ * @property {string} token
+ */
+
+class NftStorage {
+  constructor ({ token, endpoint }) {
+    this.token = token
+    this.endpoint = endpoint
+  }
+
+  /**
+   * @hidden
+   * @param {string} token
+   * @returns {Record<string, string>}
+   */
+  static headers (token) {
+    if (!token) throw new Error('missing token')
+    return {
+      Authorization: `Bearer ${token}`,
+      'Content-Type': 'application/json',
+    }
+  }
+
+  /**
+   * @param {{before?: string, size?: number}} opts
+   */
+  async* list (opts = {}) {
+    const service = {
+      token: this.token,
+      endpoint: this.endpoint
+    }
+    /**
+     * @param {Service} service
+     * @param {{before: string, size: number, signal: any}} opts
+     * @returns {Promise<Response>}
+     */
+    async function listPage ({ endpoint, token }, { before, size }) {
+      const params = new URLSearchParams()
+      // Only add params if defined
+      if (before) {
+        params.append('before', before)
+      }
+      if (size) {
+        params.append('limit', String(size))
+      }
+      const url = new URL(`?${params}`, endpoint)
+      return fetch(url.toString(), {
+        method: 'GET',
+        headers: {
+          ...NftStorage.headers(token)
+        },
+      })
+    }
+
+    for await (const res of paginator(listPage, service, opts)) {
+      for (const upload of res.value) {
+        yield upload
+      }
+    }
+  }
+}
+
+/**
+ * Follow before with last item, to fetch all the things.
+ *
+ * @param {(service: Service, opts: any) => Promise<Response>} fn
+ * @param {Service} service
+ * @param {{}} opts
+ */
+async function * paginator (fn, service, opts) {
+  let res = await fn(service, opts)
+  if (!res.ok) {
+    if (res.status === 429) {
+      throw new Error('rate limited')
+    }
+
+    const errorMessage = await res.json()
+    throw new Error(`${res.status} ${res.statusText} ${errorMessage ? '- ' + errorMessage.message : ''}`)
+  }
+  let body = await res.json()
+  yield body
+
+  // Iterate through next pages
+  while (body && body.value.length) {
+    // Get before timestamp with less 1ms
+    const before = (new Date((new Date(body.value[body.value.length-1].created)).getTime() - 1)).toISOString()
+    res = await fn(service, {
+      ...opts,
+      before
+    })
+
+    body = await res.json()
+
+    yield body
+  }
+}

migrate-to-w3up.js

@@ -23,6 +23,7 @@ import { parseW3Proof } from "./w3-env.js";
 import inquirer from 'inquirer';
 import readNDJSONStream from 'ndjson-readablestream'
 import { stringToCarCid } from "./utils.js";
+import { getClient as getNftStorageClassicClient } from './classic-nft.storage.js'
 
 // if this file is being executed directly, run main() function
 const isMain = (url, argv = process.argv) => fileURLToPath(url) === fs.realpathSync(argv[1])
@@ -338,10 +339,28 @@ async function promptForSpace(w3upUrl) {
  * @returns {Promise<AsyncIterable<W32023Upload> & { length: Promise<number> }>} uploads
  */
 async function getUploadsFromPrompts() {
-  const confirmation = await confirm({
+  const oldW3sConfirmation = await confirm({
     message: 'no uploads were piped in. Do you want to migrate uploads from old.web3.storage?',
   })
-  if (!confirmation) throw new Error('unable to find a source of uploads to migrate')
+  if (!oldW3sConfirmation) {
+    const classicNftConfirmation = await confirm({
+      message: 'no uploads were piped in. Do you want to migrate uploads from classic-app.nft.storage?',
+    })
+    if (!classicNftConfirmation) {
+      throw new Error('unable to find a source of uploads to migrate')
+    }
+    return getUploadsFromClassicNftStorage()
+  }
+  return getUploadsFromOldWeb3Storage()
+}
+
+/**
+ * get a stream of w32023 uploads via
+ * interactive prompts using inquirer
+ * + old web3.storage client library
+ * @returns {Promise<AsyncIterable<W32023Upload> & { length: Promise<number> }>} uploads
+ */
+async function getUploadsFromOldWeb3Storage() {
   const envToken = process.env.WEB3_TOKEN
   let token;
   if (envToken && await confirm({ message: 'found WEB3_TOKEN in env. Use that?' })) {
@@ -367,6 +386,37 @@ async function getUploadsFromPrompts() {
   return Object.assign(uploads, { length: count })
 }
 
+/**
+ * get a stream of nft.storage uploads via
+ * interactive prompts using inquirer
+ *
+ * @returns {Promise<AsyncIterable<W32023Upload> & { length: Promise<number> }>} uploads
+ */
+async function getUploadsFromClassicNftStorage() {
+  const envToken = process.env.NFT_STORAGE_TOKEN
+  let token;
+  if (envToken && await confirm({ message: 'found NFT_STORAGE_TOKEN in env. Use that?' })) {
+    token = envToken
+  } else {
+    token = await promptForPassword({
+      message: 'enter API token for classic-app.nft.storage',
+    })
+  }
+
+  const classicNftStorage = getNftStorageClassicClient({ token })
+
+  const uploads = (async function* () {
+    for await (const u of classicNftStorage.list()) {
+      if (u) {
+        yield new W32023Upload(u)
+      }
+    }
+  }())
+
+  // @ts-expect-error no length available
+  return uploads
+}
+
 /**
  * cli for `migrate-to-w3up log ` ...
  * `migrate-to-w3up log uploads-from-failures` should extract uploads from UploadMigrationFailure events in the log