chore: update ipfs (now with typing) dependencies (#1337)

Co-authored-by: Rod Vagg <rod@vagg.org>
Co-authored-by: Spencer T Brody <spencer@3box.io>

packages/cli/package.json

@@ -43,19 +43,19 @@
     "@ceramicnetwork/streamid": "^1.0.1",
     "@stablelib/random": "^1.0.0",
     "aws-sdk": "^2.902.0",
+    "blockcodec-to-ipld-format": "^1.0.0",
     "commander": "^7.0.0",
     "cors": "^2.8.5",
     "dag-jose": "^0.3.0",
     "did-resolver": "^3.1.0",
     "dids": "^2.1.0",
     "express": "^4.17.1",
     "flat": "^5.0.2",
-    "ipfs-http-client": "~49.0.3",
+    "ipfs-http-client": "^50.1.0",
     "key-did-provider-ed25519": "^1.1.0",
     "key-did-resolver": "^1.2.1",
     "levelup": "^4.4.0",
     "morgan": "^1.10.0",
-    "multiformats": "~4.6.1",
     "s3leveldown": "^2.2.1",
     "stream-to-array": "^2.3.0",
     "uint8arrays": "^2.0.5"
@@ -68,7 +68,7 @@
     "@types/node": "^13.13.15",
     "@types/stream-to-array": "^2.3.0",
     "get-port": "^5.1.1",
-    "ipfs-core": "~0.5.2",
+    "ipfs-core": "^0.7.0",
     "rxjs": "^7.0.0",
     "tmp-promise": "^2.0.2"
   },

packages/cli/src/__tests__/create-ipfs.ts

@@ -1,7 +1,6 @@
 import { IpfsApi } from '@ceramicnetwork/common';
 import getPort from 'get-port';
-import { sha256 } from 'multiformats/hashes/sha2';
-import legacy from 'multiformats/legacy';
+import { convert } from 'blockcodec-to-ipld-format'
 import dagJose from 'dag-jose';
 import IPFS from 'ipfs-core';
 
@@ -10,9 +9,7 @@ import IPFS from 'ipfs-core';
  */
 export async function createIPFS(path: string): Promise<IpfsApi> {
   const port = await getPort();
-  const hasher = {};
-  hasher[sha256.code] = sha256;
-  const format = legacy(dagJose, { hashes: hasher });
+  const format = convert(dagJose);
 
   const config = {
     ipld: { formats: [format] },
@@ -24,6 +21,5 @@ export async function createIPFS(path: string): Promise<IpfsApi> {
     },
   };
 
-  // @ts-ignore
   return IPFS.create(config);
 }

packages/cli/src/build-ipfs-connection.util.ts

@@ -1,23 +1,20 @@
 import {IpfsDaemon} from "@ceramicnetwork/ipfs-daemon";
 
 import dagJose from 'dag-jose'
-import legacy from 'multiformats/legacy'
+import { convert } from 'blockcodec-to-ipld-format'
 import ipfsClient from "ipfs-http-client"
 import { IpfsApi } from "@ceramicnetwork/common"
 import { DiagnosticsLogger } from "@ceramicnetwork/common";
-import { sha256 } from 'multiformats/hashes/sha2'
 
-const hasher = {}
-hasher[sha256.code] = sha256
-const dagJoseFormat = legacy(dagJose, {hashes: hasher})
+const dagJoseFormat = convert(dagJose)
 
 
 const IPFS_DHT_SERVER_MODE = process.env.IPFS_DHT_SERVER_MODE === 'true'
 const IPFS_GET_TIMEOUT = 60000 // 1 minute
 
 export async function buildIpfsConnection(network: string, logger: DiagnosticsLogger, ipfsEndpoint?: string): Promise<IpfsApi>{
     if (ipfsEndpoint) {
-        return ipfsClient({ url: ipfsEndpoint, ipld: { formats: [dagJoseFormat] }, timeout: IPFS_GET_TIMEOUT })
+        return ipfsClient.create({ url: ipfsEndpoint, ipld: { formats: [dagJoseFormat] }, timeout: IPFS_GET_TIMEOUT })
     } else {
         const ipfsDaemon = await IpfsDaemon.create({
             ipfsDhtServerMode: IPFS_DHT_SERVER_MODE,

packages/common/package.json

@@ -46,7 +46,8 @@
     "@types/logfmt": "^1.2.1",
     "@types/node": "^13.13.15",
     "dids": "^2.1.0",
-    "ipfs-core": "~0.5.2",
+    "ipfs-core": "^0.7.0",
+    "ipfs-core-types": "^0.5.0",
     "json-schema-to-typescript": "^9.1.1",
     "typescript-json-schema": "^0.42.0"
   },

packages/common/src/index.ts

@@ -16,7 +16,5 @@ export * from './running-state-like'
 export * from './stream-state-subject'
 export * from './subscription-set'
 
-// eslint-disable-next-line @typescript-eslint/ban-ts-comment
-// @ts-ignore
-import type { IPFSAPI as IpfsApi } from 'ipfs-core/dist/src/components'
-export type IpfsApi = typeof IpfsApi
+import type { IPFS } from 'ipfs-core-types'
+export type IpfsApi = IPFS

packages/common/src/utils/stream-utils.ts

@@ -155,10 +155,9 @@ export class StreamUtils {
     static async convertCommitToSignedCommitContainer(commit: CeramicCommit, ipfs: IpfsApi): Promise<CeramicCommit> {
         if (StreamUtils.isSignedCommit(commit)) {
             const block = await ipfs.block.get((commit as DagJWS).link)
-            const linkedBlock = block.data instanceof Uint8Array ? block.data : new Uint8Array(block.data.buffer)
             return {
                 jws: commit as DagJWS,
-                linkedBlock,
+                linkedBlock: block.data,
             }
         }
         return commit

packages/core/README.md

@@ -14,13 +14,11 @@ $ npm install @ceramicnetwork/core
 import Ceramic from '@ceramicnetwork/core'
 import TileDocument from '@ceramicnetwork/stream-tile'
 
-import IPFS from 'ipfs'
+import IPFS from 'ipfs-core'
 import dagJose from 'dag-jose'
-import basicsImport from 'multiformats/cjs/src/basics-import.js'
-import legacy from 'multiformats/cjs/src/legacy.js'
+import { convert } from 'blockcodec-to-ipld-format'
 
-basicsImport.multicodec.add(dagJose)
-const format = legacy(basicsImport, dagJose.name)
+const format = convert(dagJose)
 
 const ipfs = Ipfs.create({
     ipld: { formats: [format] },

packages/core/package.json

@@ -44,12 +44,13 @@
     "ajv": "^8.1.0",
     "ajv-formats": "^2.0.2",
     "await-semaphore": "^0.1.3",
+    "blockcodec-to-ipld-format": "^1.0.0",
     "cids": "~1.1.6",
     "dids": "^2.1.0",
-    "ipld-dag-cbor": "^0.17.0",
+    "ipld-dag-cbor": "^1.0.0",
     "level-ts": "^2.0.5",
     "lodash.clonedeep": "^4.5.0",
-    "multihashes": "^3.1.2",
+    "multihashes": "^4.0.2",
     "p-queue": "^6.6.1",
     "rxjs": "^7.0.0",
     "uint8arrays": "^2.0.5"
@@ -61,11 +62,10 @@
     "dag-jose": "^0.3.0",
     "did-resolver": "^3.1.0",
     "get-port": "^5.1.1",
-    "ipfs": "~0.54.2",
+    "ipfs-core": "^0.7.0",
     "key-did-provider-ed25519": "^1.1.0",
     "key-did-resolver": "^1.2.1",
     "mockdate": "^3.0.5",
-    "multiformats": "~4.6.1",
     "tmp-promise": "^2.0.2"
   },
   "jest": {

packages/core/src/__tests__/dispatcher.test.ts

@@ -139,7 +139,7 @@ describe('Dispatcher', () => {
     // Store the query ID sent when the stream is registered so we can use it as the response ID later
     const publishArgs = ipfs.pubsub.publish.mock.calls[0];
     expect(publishArgs[0]).toEqual(TOPIC);
-    const queryMessageSent = JSON.parse(publishArgs[1]);
+    const queryMessageSent = JSON.parse(new TextDecoder().decode(publishArgs[1]));
     const queryID = queryMessageSent.id;
 
     // Handle UPDATE message

packages/core/src/__tests__/ipfs-util.ts

@@ -1,7 +1,6 @@
 import dagJose from 'dag-jose';
-import { sha256 } from 'multiformats/hashes/sha2'
-import legacy from 'multiformats/legacy'
-import IPFS from 'ipfs';
+import { convert } from 'blockcodec-to-ipld-format'
+import IPFS from 'ipfs-core';
 import { IpfsApi } from '@ceramicnetwork/common';
 import tmp from 'tmp-promise';
 import getPort from 'get-port';
@@ -11,9 +10,7 @@ import getPort from 'get-port';
  * @param overrideConfig - IFPS config for override
  */
 export async function createIPFS(overrideConfig: Record<string, unknown> = {}): Promise<IpfsApi> {
-  const hasher = {}
-  hasher[sha256.code] = sha256
-  const format = legacy(dagJose, {hashes: hasher})
+  const format = convert(dagJose)
   const tmpFolder = await tmp.dir({ unsafeCleanup: true });
 
   const port = await getPort();
@@ -27,8 +24,6 @@ export async function createIPFS(overrideConfig: Record<string, unknown> = {}):
   };
 
   const config = { ...defaultConfig, ...overrideConfig };
-  // eslint-disable-next-line @typescript-eslint/ban-ts-comment
-  // @ts-ignore
   const instance = await IPFS.create(config);
 
   // IPFS does not notify you when it stops.
@@ -51,6 +46,6 @@ export async function createIPFS(overrideConfig: Record<string, unknown> = {}):
  * @param b - Receives connection
  */
 export async function swarmConnect(a: IpfsApi, b: IpfsApi) {
-  const addressB = (await b.id()).addresses[0].toString();
+  const addressB = (await b.id()).addresses[0];
   await a.swarm.connect(addressB);
 }

packages/core/src/dispatcher.ts

@@ -75,7 +75,8 @@ export class Dispatcher {
    */
   async retrieveCommit (cid: CID | string): Promise<any> {
     try {
-      const record = await this._ipfs.dag.get(cid, {timeout: IPFS_GET_TIMEOUT})
+      const asCid = typeof cid === 'string' ? new CID(cid) : cid
+      const record = await this._ipfs.dag.get(asCid, {timeout: IPFS_GET_TIMEOUT})
       await this._restrictRecordSize(cid)
       return cloneDeep(record.value)
     } catch (e) {
@@ -91,7 +92,8 @@ export class Dispatcher {
    */
   async retrieveFromIPFS (cid: CID | string, path?: string): Promise<any> {
     try {
-      const record = await this._ipfs.dag.get(cid, {timeout: IPFS_GET_TIMEOUT, path})
+      const asCid = typeof cid === 'string' ? new CID(cid) : cid
+      const record = await this._ipfs.dag.get(asCid, {timeout: IPFS_GET_TIMEOUT, path})
       return cloneDeep(record.value)
     } catch (e) {
       this._logger.err(`Error while loading CID ${cid.toString()} from IPFS: ${e}`)

packages/core/src/pubsub/__tests__/as-ipfs-message.ts

@@ -10,10 +10,9 @@ import * as random from '@stablelib/random';
  * @param from - Peer ID that ostensibly sent the message.
  */
 export function asIpfsMessage(data: PubsubMessage, from?: string): IPFSPubsubMessage {
-  const asBytes = uint8arrays.fromString(serialize(data));
   return {
     from: from || 'outer-space',
-    data: asBytes,
+    data: serialize(data),
     topicIDs: ['topic'],
     seqno: random.randomBytes(10),
     signature: random.randomBytes(10),

packages/core/src/pubsub/pubsub-message.ts

@@ -4,7 +4,7 @@ import { UnreachableCaseError } from '@ceramicnetwork/common';
 import dagCBOR from 'ipld-dag-cbor';
 import * as multihashes from 'multihashes';
 import * as sha256 from '@stablelib/sha256';
-import { TextDecoder } from 'util';
+import { TextDecoder, TextEncoder } from 'util';
 import * as uint8arrays from 'uint8arrays';
 
 /**
@@ -36,6 +36,9 @@ export type ResponseMessage = {
 
 export type PubsubMessage = UpdateMessage | QueryMessage | ResponseMessage;
 
+const textEncoder = new TextEncoder()
+const textDecoder = new TextDecoder('utf-8')
+
 function messageHash(message: any): string {
   // DAG-CBOR encoding
   const encoded = dagCBOR.util.serialize(message);
@@ -59,14 +62,14 @@ export function buildQueryMessage(streamId: StreamID): QueryMessage {
   };
 }
 
-export function serialize(message: PubsubMessage): string {
+export function serialize(message: PubsubMessage): Uint8Array {
   switch (message.typ) {
     case MsgType.QUERY: {
-      return JSON.stringify({
+      return textEncoder.encode(JSON.stringify({
         ...message,
         doc: message.stream.toString(), // todo remove once we no longer support interop with nodes older than v1.0.0
         stream: message.stream.toString(),
-      });
+      }));
     }
     case MsgType.RESPONSE: {
       const tips = {};
@@ -75,20 +78,20 @@ export function serialize(message: PubsubMessage): string {
         ...message,
         tips: tips,
       };
-      return JSON.stringify(payload);
+      return textEncoder.encode(JSON.stringify(payload));
     }
     case MsgType.UPDATE: {
       // todo remove 'doc' once we no longer support interop with nodes older than v1.0.0
       const payload = { typ: MsgType.UPDATE, doc: message.stream.toString(), stream: message.stream.toString(), tip: message.tip.toString() };
-      return JSON.stringify(payload);
+      return textEncoder.encode(JSON.stringify(payload));
     }
     default:
       throw new UnreachableCaseError(message, 'Unknown message type');
   }
 }
 
 export function deserialize(message: any): PubsubMessage {
-  const asString = new TextDecoder('utf-8').decode(message.data);
+  const asString = textDecoder.decode(message.data);
   const parsed = JSON.parse(asString);
 
   const typ = parsed.typ as MsgType;

packages/core/src/pubsub/pubsub.ts

@@ -4,6 +4,9 @@ import { IpfsApi } from '@ceramicnetwork/common';
 import { map, catchError, mergeMap, withLatestFrom } from 'rxjs/operators';
 import { IncomingChannel, filterExternal, IPFSPubsubMessage } from './incoming-channel';
 import { DiagnosticsLogger, ServiceLogger } from '@ceramicnetwork/common';
+import { TextDecoder } from 'util';
+
+const textDecoder = new TextDecoder('utf-8')
 
 /**
  * Deserialize incoming message in an internal observable that does not emit if error happens.
@@ -25,7 +28,7 @@ function ipfsToPubsub(
         map((incoming) => {
           const message = deserialize(incoming);
           const serializedMessage = serialize(message);
-          const logMessage = { ...incoming, ...JSON.parse(serializedMessage) };
+          const logMessage = { ...incoming, ...JSON.parse(textDecoder.decode(serializedMessage)) };
           delete logMessage.key;
           delete logMessage.signature;
           pubsubLogger.log({ peer: peerId, event: 'received', topic: topic, message: logMessage });
@@ -79,7 +82,7 @@ export class Pubsub extends Observable<PubsubMessage> {
       )
       .subscribe({
         next: ({ peerId, serializedMessage }) => {
-          const logMessage = { ...message, ...JSON.parse(serializedMessage) };
+          const logMessage = { ...message, ...JSON.parse(textDecoder.decode(serializedMessage)) };
           this.pubsubLogger.log({ peer: peerId, event: 'published', topic: this.topic, message: logMessage });
         },
         error: (error) => {

packages/ipfs-daemon/package.json

@@ -33,18 +33,19 @@
     "@ceramicnetwork/common": "^1.0.4",
     "@ceramicnetwork/ipfs-topology": "^1.1.3",
     "aws-sdk": "^2.902.0",
+    "blockcodec-to-ipld-format": "^1.0.0",
     "dag-jose": "^0.3.0",
     "datastore-fs": "^4.0.0",
     "datastore-level": "^5.0.1",
     "datastore-s3": "^5.0.0",
     "express": "^4.17.1",
-    "ipfs": "~0.54.2",
-    "ipfs-http-gateway": "~0.3.2",
-    "ipfs-http-server": "~0.3.3",
-    "multiformats": "~4.6.1"
+    "ipfs-core-types": "^0.5.0",
+    "ipfs-http-gateway": "^0.4.1",
+    "ipfs-http-server": "^0.5.0"
   },
   "devDependencies": {
     "@types/express": "^4.17.8",
+    "@types/jest": "^26.0.23",
     "@types/node": "^13.13.15"
   }
 }

packages/ipfs-daemon/src/create-repo.ts

@@ -10,13 +10,14 @@ const notALock = {
   getLockfilePath: () => {
     // Do Nothing
   },
-  lock: () => notALock.getCloser(),
+  lock: () => Promise.resolve(notALock.getCloser()),
   getCloser: () => ({
     close: () => {
       // Do Nothing
+      return Promise.resolve()
     },
   }),
-  locked: () => false,
+  locked: () => Promise.resolve(false),
 };
 
 export enum StorageBackend {