refactor(migrate-to-w3up): refactor and cleanup to be used as library

migrate-to-w3up.js → bin/migrate-to-w3up.js

@@ -1,6 +1,6 @@
 #!/usr/bin/env node
 /* eslint-disable @typescript-eslint/no-unused-vars */
-import { W32023Upload, W32023UploadSummary, W32023UploadsFromNdjson } from "./w32023.js";
+import { W32023Upload, W32023UploadSummary, W32023UploadsFromNdjson } from "../src/w32023.js";
 import { fileURLToPath } from 'node:url'
 import fs, { createReadStream, createWriteStream } from 'node:fs'
 import { Readable } from 'node:stream'
@@ -10,20 +10,20 @@ import { DID } from "@ucanto/validator"
 import { StoreConf } from '@web3-storage/access/stores/store-conf'
 import { select } from '@inquirer/prompts';
 import confirm from '@inquirer/confirm';
-import { Web3Storage } from 'web3.storage'
 import promptForPassword from '@inquirer/password';
-import { carPartToStoreAddNb, migrate } from "./w32023-to-w3up.js";
-import { UploadMigrationFailure, UploadMigrationSuccess, UploadPartMigrationFailure, receiptToJson } from "./w3up-migration.js";
+import { carPartToStoreAddNb, migrate } from "../src/w32023-to-w3up.js";
+import { UploadMigrationFailure, UploadMigrationSuccess, UploadPartMigrationFailure, receiptToJson } from "../src/w3up-migration.js";
 import { Store } from "@web3-storage/capabilities";
 import { connect } from '@ucanto/client'
 import { CAR, HTTP } from '@ucanto/transport'
 import { StoreMemory } from "@web3-storage/access/stores/store-memory";
 import * as ed25519Principal from '@ucanto/principal/ed25519'
-import { parseW3Proof } from "./w3-env.js";
+import { parseW3Proof } from "../src/w3-env.js";
 import inquirer from 'inquirer';
 import readNDJSONStream from 'ndjson-readablestream'
-import { stringToCarCid } from "./utils.js";
-import { getClient as getNftStorageClassicClient } from './classic-nft.storage.js'
+import { stringToCarCid } from "../src/utils.js";
+import { getUploads as getNftStorageClassicUploads } from '../src/classic-nft.storage.js'
+import { getUploads as getOldWeb3StorageUploads } from '../src/old-web3.storage.js'
 
 // if this file is being executed directly, run main() function
 const isMain = (url, argv = process.argv) => fileURLToPath(url) === fs.realpathSync(argv[1])
@@ -342,79 +342,59 @@ async function getUploadsFromPrompts() {
   const oldW3sConfirmation = await confirm({
     message: 'no uploads were piped in. Do you want to migrate uploads from old.web3.storage?',
   })
+  let getTokenFn = getOldWeb3StorageToken
+  let getUploadsFn = getOldWeb3StorageUploads
   if (!oldW3sConfirmation) {
     const classicNftConfirmation = await confirm({
       message: 'no uploads were piped in. Do you want to migrate uploads from classic-app.nft.storage?',
     })
     if (!classicNftConfirmation) {
       throw new Error('unable to find a source of uploads to migrate')
     }
-    return getUploadsFromClassicNftStorage()
+    getTokenFn = getClassicNftStorageToken
+    getUploadsFn = getNftStorageClassicUploads
+  }
+  const token = await getTokenFn()
+  try {
+    return getUploadsFn({token})
+  } catch (e) {
+    console.log(e)
+    process.exit(-1)
   }
-  return getUploadsFromOldWeb3Storage()
 }
 
 /**
  * get a stream of w32023 uploads via
  * interactive prompts using inquirer
  * + old web3.storage client library
- * @returns {Promise<AsyncIterable<W32023Upload> & { length: Promise<number> }>} uploads
+ * @returns {Promise<string>} uploads
  */
-async function getUploadsFromOldWeb3Storage() {
+async function getOldWeb3StorageToken() {
   const envToken = process.env.WEB3_TOKEN
-  let token;
   if (envToken && await confirm({ message: 'found WEB3_TOKEN in env. Use that?' })) {
-    token = envToken
+    return envToken
   } else {
-    token = await promptForPassword({
+    return await promptForPassword({
       message: 'enter API token for old.web3.storage',
     })
   }
-  const oldW3 = new Web3Storage({ token })
-  const uploads = (async function* () {
-    for await (const u of oldW3.list()) {
-      if (u) {
-        yield new W32023Upload(u)
-      }
-    }
-  }())
-  // get count
-  const userUploadsResponse = fetch(`https://api.web3.storage/user/uploads`, {
-    headers: { authorization: `Bearer ${token}` },
-  })
-  const count = userUploadsResponse.then(r => parseInt(r.headers.get('count'), 10)).then(c => isNaN(c) ? undefined : c)
-  return Object.assign(uploads, { length: count })
 }
 
 /**
  * get a stream of nft.storage uploads via
  * interactive prompts using inquirer
  *
- * @returns {Promise<AsyncIterable<W32023Upload> & { length: Promise<number> }>} uploads
+ * @returns {Promise<string>} uploads
  */
-async function getUploadsFromClassicNftStorage() {
+async function getClassicNftStorageToken() {
   const envToken = process.env.NFT_STORAGE_TOKEN
-  let token;
   if (envToken && await confirm({ message: 'found NFT_STORAGE_TOKEN in env. Use that?' })) {
-    token = envToken
+    return envToken
   } else {
-    token = await promptForPassword({
+    return await promptForPassword({
       message: 'enter API token for classic-app.nft.storage',
     })
   }
-
-  const classicNftStorage = getNftStorageClassicClient({ token })
-
-  const uploads = (async function* () {
-    for await (const u of classicNftStorage.list()) {
-      if (u) {
-        yield new W32023Upload(u)
-      }
-    }
-  }())
-
-  // @ts-expect-error no length available
-  return uploads
 }
 
 /**

package.json

@@ -2,10 +2,10 @@
   "name": "migrate-to-w3up",
   "version": "1.0.3",
   "description": "Migrate data from old web3.storage to w3up",
-  "main": "index.js",
+  "main": "src/index.js",
   "exports": {
-    ".": "./index.js",
-    "./from-w32023": "./from-w32023.js"
+    ".": "./src/index.js",
+    "./from-w32023": "./src/from-w32023.js"
   },
   "type": "module",
   "scripts": {
@@ -83,7 +83,7 @@
     "@ucanto/transport": "^9.0.2"
   },
   "bin": {
-    "migrate-to-w3up": "./migrate-to-w3up.js"
+    "migrate-to-w3up": "./bin/migrate-to-w3up.js"
   },
   "devDependencies": {
     "@typescript-eslint/eslint-plugin": "^6.19.1",

classic-nft.storage.js → src/classic-nft.storage.js

@@ -1,19 +1,37 @@
+import { W32023Upload } from "../src/w32023.js";
+
 export const API = 'https://api.nft.storage'
 
-export function getClient ({
+/**
+ * get a stream of w32023 uploads from nft.storage
+ * @param {object} options - options
+ * @param {string} [options.api] - optional API endpoint override
+ * @param {string} options.token
+ * @returns {AsyncIterable<W32023Upload> & { length: Promise<number> }} uploads
+ */
+export function getUploads ({
   api = API,
   token
 }) {
   if (!token) {
-    console.log('! run `nft token` to set an API token to use')
-    process.exit(-1)
+    throw new Error('! run `nft token` to set an API token to use')
   }
   const endpoint = new URL(api)
   if (api !== API) {
     // note if we're using something other than prod.
-    console.log(`using ${endpoint.hostname}`)
+    console.info(`using ${endpoint.hostname}`)
   }
-  return new NftStorage({ token, endpoint })
+  const classicNftStorage = new NFTStorage({ token, endpoint })
+
+  const uploads = (async function* () {
+    for await (const u of classicNftStorage.list()) {
+      if (u) {
+        yield new W32023Upload(u)
+      }
+    }
+  }())
+
+  return Object.assign(uploads, { length: classicNftStorage.count() })
 }
 
 /**
@@ -22,7 +40,7 @@ export function getClient ({
  * @property {string} token
  */
 
-class NftStorage {
+class NFTStorage {
   constructor ({ token, endpoint }) {
     this.token = token
     this.endpoint = endpoint
@@ -41,6 +59,16 @@ class NftStorage {
     }
   }
 
+  count() {
+    const res = fetch(this.endpoint.toString(), {
+      method: 'GET',
+      headers: {
+        ...NFTStorage.headers(this.token)
+      },
+    })
+    return res.then(r => parseInt(r.headers.get('count'), 10)).then(c => isNaN(c) ? undefined : c)
+  }
+
   /**
    * @param {{before?: string, size?: number}} opts
    */
@@ -67,7 +95,7 @@ class NftStorage {
       return fetch(url.toString(), {
         method: 'GET',
         headers: {
-          ...NftStorage.headers(token)
+          ...NFTStorage.headers(token)
         },
       })
     }

src/index.js

@@ -0,0 +1,3 @@
+export * as w32023ToW3up from "./w32023-to-w3up.js"
+export * as NFTStorage from './classic-nft.storage.js'
+export * as OldWeb3Storage from './old-web3.storage.js'

src/old-web3.storage.js

@@ -0,0 +1,41 @@
+import { Web3Storage } from "web3.storage"
+import { W32023Upload } from "../src/w32023.js";
+
+const API = 'https://api.web3.storage'
+
+/**
+ * get a stream of w32023 uploads via
+ * interactive prompts using inquirer
+ * + old web3.storage client library 
+ * @param {object} options - options
+ * @param {string} [options.api] - optional API endpoint override
+ * @param {string} options.token
+ * @returns {AsyncIterable<W32023Upload> & { length: Promise<number> }} uploads
+ */
+export function getUploads({
+  api = API,
+  token
+}) {
+  if (!token) {
+    throw new Error('! run `nft token` to set an API token to use')
+  }
+  const endpoint = new URL(api)
+  if (api !== API) {
+    // note if we're using something other than prod.
+    console.info(`using ${endpoint.hostname}`)
+  }
+  const oldW3 = new Web3Storage({ token, endpoint })
+  const uploads = (async function* () {
+    for await (const u of oldW3.list()) {
+      if (u) {
+        yield new W32023Upload(u)
+      }
+    }
+  }())
+  // get count
+  const userUploadsResponse = fetch(`${api}/user/uploads`, {
+    headers: { authorization: `Bearer ${token}` },
+  })
+  const count = userUploadsResponse.then(r => parseInt(r.headers.get('count'), 10)).then(c => isNaN(c) ? undefined : c)
+  return Object.assign(uploads, { length: count })
+}

test/migrate-to-w3up-logging.test.js

@@ -6,10 +6,10 @@ import {
   locate,
   createUploadsStream,
   setupSpaceMigrationScenario,
-} from "../test-utils.js"
+} from "./test-utils.js"
 import { createServer } from 'node:http'
 import * as ed25519 from '@ucanto/principal/ed25519'
-import { encodeDelegationAsCid } from '../w3-env.js'
+import { encodeDelegationAsCid } from '../src/w3-env.js'
 import { pipeline } from 'node:stream/promises'
 import { join } from 'node:path'
 import { text } from 'node:stream/consumers'
@@ -19,7 +19,7 @@ import * as fs from "fs/promises"
 import { tmpdir } from 'node:os'
 import readNDJSONStream from 'ndjson-readablestream'
 import { Readable } from 'node:stream'
-import { readUploadsFromUploadMigrationFailuresNdjson } from '../w3up-migration.js'
+import { readUploadsFromUploadMigrationFailuresNdjson } from '../src/w3up-migration.js'
 
 /** make a temporary file path that can be used for test migration logfiles  */
 async function getTmpLogFilePath() {

migrate-to-w3up.test.js → test/migrate-to-w3up.test.js

@@ -1,13 +1,13 @@
 import { test } from 'node:test'
 import assert from 'node:assert'
-import { exampleUpload1 } from './w32023.js'
+import { exampleUpload1 } from '../src/w32023.js'
 import * as ed25519 from '@ucanto/principal/ed25519'
 import { ReadableStream } from 'node:stream/web'
 import { pipeline } from 'node:stream/promises'
 import { createCarFinder, locate } from './test-utils.js'
 import { createServer } from 'node:http'
 import { delegate } from '@ucanto/core'
-import { encodeDelegationAsCid } from './w3-env.js'
+import { encodeDelegationAsCid } from '../src/w3-env.js'
 import { createMockW3up, spawnMigration } from "./test-utils.js"
 import { text } from 'node:stream/consumers'
 

test-utils.js → test/test-utils.js

@@ -6,7 +6,7 @@ import { Store, Upload } from '@web3-storage/capabilities'
 import * as consumers from 'stream/consumers'
 import * as CAR from '@ucanto/transport/car'
 import * as ed25519 from '@ucanto/principal/ed25519'
-import { exampleUpload1 } from './w32023.js'
+import { exampleUpload1 } from '../src/w32023.js'
 import { ReadableStream } from 'node:stream/web'
 import { delegate } from '@ucanto/core'
 
@@ -100,7 +100,7 @@ export function locate(server) {
 }
 
 
-export const migrateToW3upPath = fileURLToPath(new URL('./migrate-to-w3up.js', import.meta.url))
+export const migrateToW3upPath = fileURLToPath(new URL('../bin/migrate-to-w3up.js', import.meta.url))
 
 /**
  * create a RequestListener that can be a mock up.web3.storage

w32023-to-w3up.test.js → test/w32023-to-w3up.test.js

@@ -1,15 +1,15 @@
 import { test } from 'node:test'
-import { W32023Upload, W32023UploadsFromNdjson } from './w32023.js'
+import { W32023Upload, W32023UploadsFromNdjson } from '../src/w32023.js'
 import assert from 'assert'
 import * as CAR from "@ucanto/transport/car"
 import * as Client from '@ucanto/client'
 import * as ed25519 from '@ucanto/principal/ed25519'
 import * as Server from "@ucanto/server"
-import { migrate } from './w32023-to-w3up.js'
+import { migrate } from '../src/w32023-to-w3up.js'
 import { createServer } from 'http'
-import { MapCidToPromiseResolvers } from './utils.js'
+import { MapCidToPromiseResolvers } from '../src/utils.js'
 import { ReadableStream, TransformStream } from 'stream/web'
-import { UploadMigrationFailure, UploadPartMigrationFailure } from './w3up-migration.js'
+import { UploadMigrationFailure, UploadPartMigrationFailure } from '../src/w3up-migration.js'
 import { createCarFinder, locate } from './test-utils.js'
 
 /** example uploads from `w3 list --json` */

tsconfig.json

@@ -10,7 +10,7 @@
     "noEmit": true,
     "lib": ["DOM"]
   },
-  "include": ["./*.js", "./src", "./test"],
+  "include": ["./*.js", "./src", "./test", "bin/migrate-to-w3up.js"],
   "references": [
   ]
 }