diff --git a/packages/ipfs-unixfs-exporter/src/index.ts b/packages/ipfs-unixfs-exporter/src/index.ts index fcbbc2cf..37b38c88 100644 --- a/packages/ipfs-unixfs-exporter/src/index.ts +++ b/packages/ipfs-unixfs-exporter/src/index.ts @@ -4,7 +4,7 @@ import resolve from './resolvers/index.js' import last from 'it-last' import type { UnixFS } from 'ipfs-unixfs' import type { PBNode } from '@ipld/dag-pb' -import type { Blockstore as InterfaceBlockstore } from 'interface-blockstore' +import type { Blockstore } from 'interface-blockstore' import type { Bucket } from 'hamt-sharding' import type { ProgressOptions } from 'progress-events' @@ -65,13 +65,13 @@ export interface ResolveResult { next?: NextResult } -export interface Resolve { (cid: CID, name: string, path: string, toResolve: string[], depth: number, blockstore: Blockstore, options: ExporterOptions): Promise } -export interface Resolver { (cid: CID, name: string, path: string, toResolve: string[], resolve: Resolve, depth: number, blockstore: Blockstore, options: ExporterOptions): Promise } +export interface Resolve { (cid: CID, name: string, path: string, toResolve: string[], depth: number, blockstore: ReadableStorage, options: ExporterOptions): Promise } +export interface Resolver { (cid: CID, name: string, path: string, toResolve: string[], resolve: Resolve, depth: number, blockstore: ReadableStorage, options: ExporterOptions): Promise } export type UnixfsV1FileContent = AsyncIterable | Iterable export type UnixfsV1DirectoryContent = AsyncIterable | Iterable export type UnixfsV1Content = UnixfsV1FileContent | UnixfsV1DirectoryContent -export interface UnixfsV1Resolver { (cid: CID, node: PBNode, unixfs: UnixFS, path: string, resolve: Resolve, depth: number, blockstore: Blockstore): (options: ExporterOptions) => UnixfsV1Content } +export interface UnixfsV1Resolver { (cid: CID, node: PBNode, unixfs: UnixFS, path: string, resolve: Resolve, depth: number, blockstore: ReadableStorage): (options: ExporterOptions) => UnixfsV1Content } export interface ShardTraversalContext { hamtDepth: number @@ -79,7 +79,7 @@ export interface ShardTraversalContext { lastBucket: Bucket } -export type Blockstore = Pick +export type ReadableStorage = Pick const toPathComponents = (path: string = ''): string[] => { // split on / unless escaped with \ @@ -121,7 +121,7 @@ const cidAndRest = (path: string | Uint8Array | CID): { cid: CID, toResolve: str throw errCode(new Error(`Unknown path type ${path}`), 'ERR_BAD_PATH') } -export async function * walkPath (path: string | CID, blockstore: Blockstore, options: ExporterOptions = {}): AsyncGenerator { +export async function * walkPath (path: string | CID, blockstore: ReadableStorage, options: ExporterOptions = {}): AsyncGenerator { let { cid, toResolve @@ -153,7 +153,7 @@ export async function * walkPath (path: string | CID, blockstore: Blockstore, op } } -export async function exporter (path: string | CID, blockstore: Blockstore, options: ExporterOptions = {}): Promise { +export async function exporter (path: string | CID, blockstore: ReadableStorage, options: ExporterOptions = {}): Promise { const result = await last(walkPath(path, blockstore, options)) if (result == null) { @@ -163,7 +163,7 @@ export async function exporter (path: string | CID, blockstore: Blockstore, opti return result } -export async function * recursive (path: string | CID, blockstore: Blockstore, options: ExporterOptions = {}): AsyncGenerator { +export async function * recursive (path: string | CID, blockstore: ReadableStorage, options: ExporterOptions = {}): AsyncGenerator { const node = await exporter(path, blockstore, options) if (node == null) { diff --git a/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/file.ts b/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/file.ts index c5f80b51..51f4bd4e 100644 --- a/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/file.ts +++ b/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/file.ts @@ -9,9 +9,9 @@ import parallel from 'it-parallel' import { pipe } from 'it-pipe' import map from 'it-map' import PQueue from 'p-queue' -import type { ExporterOptions, UnixfsV1FileContent, UnixfsV1Resolver, Blockstore } from '../../../index.js' +import type { ExporterOptions, UnixfsV1FileContent, UnixfsV1Resolver, ReadableStorage } from '../../../index.js' -async function walkDAG (blockstore: Blockstore, node: dagPb.PBNode | Uint8Array, queue: Pushable, streamPosition: bigint, start: bigint, end: bigint, options: ExporterOptions): Promise { +async function walkDAG (blockstore: ReadableStorage, node: dagPb.PBNode | Uint8Array, queue: Pushable, streamPosition: bigint, start: bigint, end: bigint, options: ExporterOptions): Promise { // a `raw` node if (node instanceof Uint8Array) { queue.push(extractDataFromBlock(node, streamPosition, start, end)) diff --git a/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/hamt-sharded-directory.ts b/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/hamt-sharded-directory.ts index bc6fbcc0..3dbb94d4 100644 --- a/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/hamt-sharded-directory.ts +++ b/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/hamt-sharded-directory.ts @@ -2,7 +2,7 @@ import parallel from 'it-parallel' import { pipe } from 'it-pipe' import map from 'it-map' import { decode, PBNode } from '@ipld/dag-pb' -import type { ExporterOptions, Resolve, UnixfsV1DirectoryContent, UnixfsV1Resolver, Blockstore } from '../../../index.js' +import type { ExporterOptions, Resolve, UnixfsV1DirectoryContent, UnixfsV1Resolver, ReadableStorage } from '../../../index.js' const hamtShardedDirectoryContent: UnixfsV1Resolver = (cid, node, unixfs, path, resolve, depth, blockstore) => { function yieldHamtDirectoryContent (options: ExporterOptions = {}): UnixfsV1DirectoryContent { @@ -12,7 +12,7 @@ const hamtShardedDirectoryContent: UnixfsV1Resolver = (cid, node, unixfs, path, return yieldHamtDirectoryContent } -async function * listDirectory (node: PBNode, path: string, resolve: Resolve, depth: number, blockstore: Blockstore, options: ExporterOptions): UnixfsV1DirectoryContent { +async function * listDirectory (node: PBNode, path: string, resolve: Resolve, depth: number, blockstore: ReadableStorage, options: ExporterOptions): UnixfsV1DirectoryContent { const links = node.Links const results = pipe( diff --git a/packages/ipfs-unixfs-exporter/src/utils/find-cid-in-shard.ts b/packages/ipfs-unixfs-exporter/src/utils/find-cid-in-shard.ts index 44ecfe22..b029ad2a 100644 --- a/packages/ipfs-unixfs-exporter/src/utils/find-cid-in-shard.ts +++ b/packages/ipfs-unixfs-exporter/src/utils/find-cid-in-shard.ts @@ -2,7 +2,7 @@ import { Bucket, BucketPosition, createHAMT } from 'hamt-sharding' import { decode, PBLink, PBNode } from '@ipld/dag-pb' import { murmur3128 } from '@multiformats/murmur3' -import type { ExporterOptions, ShardTraversalContext, Blockstore } from '../index.js' +import type { ExporterOptions, ShardTraversalContext, ReadableStorage } from '../index.js' import type { CID } from 'multiformats/cid' // FIXME: this is copy/pasted from ipfs-unixfs-importer/src/options.js @@ -61,7 +61,7 @@ const toBucketPath = (position: BucketPosition): Array> return path.reverse() } -const findShardCid = async (node: PBNode, name: string, blockstore: Blockstore, context?: ShardTraversalContext, options?: ExporterOptions): Promise => { +const findShardCid = async (node: PBNode, name: string, blockstore: ReadableStorage, context?: ShardTraversalContext, options?: ExporterOptions): Promise => { if (context == null) { const rootBucket = createHAMT({ hashFn diff --git a/packages/ipfs-unixfs-importer/src/dag-builder/dir.ts b/packages/ipfs-unixfs-importer/src/dag-builder/dir.ts index e789fb68..43a7c437 100644 --- a/packages/ipfs-unixfs-importer/src/dag-builder/dir.ts +++ b/packages/ipfs-unixfs-importer/src/dag-builder/dir.ts @@ -1,7 +1,7 @@ import { UnixFS } from 'ipfs-unixfs' import { persist } from '../utils/persist.js' import { encode, prepare } from '@ipld/dag-pb' -import type { Directory, InProgressImportResult, Blockstore } from '../index.js' +import type { Directory, InProgressImportResult, WritableStorage } from '../index.js' import type { Version } from 'multiformats/cid' import type { ProgressOptions } from 'progress-events' @@ -10,7 +10,7 @@ export interface DirBuilderOptions extends ProgressOptions { signal?: AbortSignal } -export const dirBuilder = async (dir: Directory, blockstore: Blockstore, options: DirBuilderOptions): Promise => { +export const dirBuilder = async (dir: Directory, blockstore: WritableStorage, options: DirBuilderOptions): Promise => { const unixfs = new UnixFS({ type: 'directory', mtime: dir.mtime, diff --git a/packages/ipfs-unixfs-importer/src/dag-builder/file.ts b/packages/ipfs-unixfs-importer/src/dag-builder/file.ts index 6ded9e18..a74a22ef 100644 --- a/packages/ipfs-unixfs-importer/src/dag-builder/file.ts +++ b/packages/ipfs-unixfs-importer/src/dag-builder/file.ts @@ -3,7 +3,7 @@ import { persist } from '../utils/persist.js' import { encode, PBLink, prepare } from '@ipld/dag-pb' import parallelBatch from 'it-parallel-batch' import * as rawCodec from 'multiformats/codecs/raw' -import type { BufferImporter, File, InProgressImportResult, Blockstore, SingleBlockImportResult } from '../index.js' +import type { BufferImporter, File, InProgressImportResult, WritableStorage, SingleBlockImportResult } from '../index.js' import type { FileLayout, Reducer } from '../layout/index.js' import type { Version } from 'multiformats/cid' import type { ProgressOptions } from 'progress-events' @@ -13,7 +13,7 @@ interface BuildFileBatchOptions { blockWriteConcurrency: number } -async function * buildFileBatch (file: File, blockstore: Blockstore, options: BuildFileBatchOptions): AsyncGenerator { +async function * buildFileBatch (file: File, blockstore: WritableStorage, options: BuildFileBatchOptions): AsyncGenerator { let count = -1 let previous: SingleBlockImportResult | undefined @@ -60,7 +60,7 @@ function isSingleBlockImport (result: any): result is SingleBlockImportResult { return result.single === true } -const reduce = (file: File, blockstore: Blockstore, options: ReduceOptions): Reducer => { +const reduce = (file: File, blockstore: WritableStorage, options: ReduceOptions): Reducer => { const reducer: Reducer = async function (leaves) { if (leaves.length === 1 && isSingleBlockImport(leaves[0]) && options.reduceSingleLeafToSelf) { const leaf = leaves[0] @@ -163,6 +163,6 @@ export interface FileBuilderOptions extends BuildFileBatchOptions, ReduceOptions layout: FileLayout } -export const fileBuilder = async (file: File, block: Blockstore, options: FileBuilderOptions): Promise => { +export const fileBuilder = async (file: File, block: WritableStorage, options: FileBuilderOptions): Promise => { return await options.layout(buildFileBatch(file, block, options), reduce(file, block, options)) } diff --git a/packages/ipfs-unixfs-importer/src/dag-builder/index.ts b/packages/ipfs-unixfs-importer/src/dag-builder/index.ts index 3779cc7b..70e88b56 100644 --- a/packages/ipfs-unixfs-importer/src/dag-builder/index.ts +++ b/packages/ipfs-unixfs-importer/src/dag-builder/index.ts @@ -1,7 +1,7 @@ import { dirBuilder, DirBuilderOptions } from './dir.js' import { fileBuilder, FileBuilderOptions } from './file.js' import errCode from 'err-code' -import type { Directory, File, FileCandidate, ImportCandidate, InProgressImportResult, Blockstore } from '../index.js' +import type { Directory, File, FileCandidate, ImportCandidate, InProgressImportResult, WritableStorage } from '../index.js' import type { ChunkValidator } from './validate-chunks.js' import type { Chunker } from '../chunker/index.js' @@ -42,7 +42,7 @@ export interface DagBuilderOptions extends FileBuilderOptions, DirBuilderOptions export type ImporterSourceStream = AsyncIterable | Iterable export interface DAGBuilder { - (source: ImporterSourceStream, blockstore: Blockstore): AsyncIterable<() => Promise> + (source: ImporterSourceStream, blockstore: WritableStorage): AsyncIterable<() => Promise> } export function defaultDagBuilder (options: DagBuilderOptions): DAGBuilder { diff --git a/packages/ipfs-unixfs-importer/src/dir.ts b/packages/ipfs-unixfs-importer/src/dir.ts index 8bf65583..952541c8 100644 --- a/packages/ipfs-unixfs-importer/src/dir.ts +++ b/packages/ipfs-unixfs-importer/src/dir.ts @@ -1,4 +1,4 @@ -import type { Blockstore, ImportResult, InProgressImportResult } from './index.js' +import type { WritableStorage, ImportResult, InProgressImportResult } from './index.js' import type { Mtime, UnixFS } from 'ipfs-unixfs' import { CID } from 'multiformats/cid' import type { PersistOptions } from './utils/persist.js' @@ -50,7 +50,7 @@ export abstract class Dir { abstract put (name: string, value: InProgressImportResult | Dir): Promise abstract get (name: string): Promise abstract eachChildSeries (): AsyncIterable<{ key: string, child: InProgressImportResult | Dir }> - abstract flush (blockstore: Blockstore): AsyncGenerator + abstract flush (blockstore: WritableStorage): AsyncGenerator abstract estimateNodeSize (): number abstract childCount (): number } diff --git a/packages/ipfs-unixfs-importer/src/index.ts b/packages/ipfs-unixfs-importer/src/index.ts index 82d69c29..3c4283cd 100644 --- a/packages/ipfs-unixfs-importer/src/index.ts +++ b/packages/ipfs-unixfs-importer/src/index.ts @@ -3,7 +3,7 @@ import { DAGBuilder, defaultDagBuilder } from './dag-builder/index.js' import { defaultTreeBuilder } from './tree-builder.js' import type { UnixFS, Mtime } from 'ipfs-unixfs' import type { CID, Version as CIDVersion } from 'multiformats/cid' -import type { Blockstore as InterfaceBlockstore } from 'interface-blockstore' +import type { Blockstore } from 'interface-blockstore' import { ChunkValidator, defaultChunkValidator } from './dag-builder/validate-chunks.js' import { fixedSize } from './chunker/fixed-size.js' import type { Chunker } from './chunker/index.js' @@ -17,7 +17,7 @@ import type { ProgressOptions } from 'progress-events' export type ByteStream = AwaitIterable export type ImportContent = ByteStream | Uint8Array -export type Blockstore = Pick +export type WritableStorage = Pick export interface FileCandidate { path?: string @@ -73,8 +73,8 @@ export interface BufferImporterResult extends ImportResult { } export interface HamtHashFn { (value: Uint8Array): Promise } -export interface TreeBuilder { (source: AsyncIterable, blockstore: Blockstore): AsyncIterable } -export interface BufferImporter { (file: File, blockstore: Blockstore): AsyncIterable<() => Promise> } +export interface TreeBuilder { (source: AsyncIterable, blockstore: WritableStorage): AsyncIterable } +export interface BufferImporter { (file: File, blockstore: WritableStorage): AsyncIterable<() => Promise> } export type ImportProgressEvents = BufferImportProgressEvents @@ -227,7 +227,7 @@ export type ImportCandidateStream = AsyncIterable { +export async function * importer (source: ImportCandidateStream, blockstore: WritableStorage, options: ImporterOptions = {}): AsyncGenerator { let candidates: AsyncIterable | Iterable if (Symbol.asyncIterator in source || Symbol.iterator in source) { @@ -302,7 +302,7 @@ export async function * importer (source: ImportCandidateStream, blockstore: Blo * const entry = await importFile(input, blockstore) * ``` */ -export async function importFile (content: FileCandidate, blockstore: Blockstore, options: ImporterOptions = {}): Promise { +export async function importFile (content: FileCandidate, blockstore: WritableStorage, options: ImporterOptions = {}): Promise { const result = await first(importer([content], blockstore, options)) if (result == null) { @@ -333,7 +333,7 @@ export async function importFile (content: FileCandidate, blockstore: Blockstore * const entry = await importDirectory(input, blockstore) * ``` */ -export async function importDirectory (content: DirectoryCandidate, blockstore: Blockstore, options: ImporterOptions = {}): Promise { +export async function importDirectory (content: DirectoryCandidate, blockstore: WritableStorage, options: ImporterOptions = {}): Promise { const result = await first(importer([content], blockstore, options)) if (result == null) { @@ -361,7 +361,7 @@ export async function importDirectory (content: DirectoryCandidate, blockstore: * const entry = await importBytes(input, blockstore) * ``` */ -export async function importBytes (buf: ImportContent, blockstore: Blockstore, options: ImporterOptions = {}): Promise { +export async function importBytes (buf: ImportContent, blockstore: WritableStorage, options: ImporterOptions = {}): Promise { return await importFile({ content: buf }, blockstore, options) @@ -388,7 +388,7 @@ export async function importBytes (buf: ImportContent, blockstore: Blockstore, o * const entry = await importByteStream(input, blockstore) * ``` */ -export async function importByteStream (bufs: ByteStream, blockstore: Blockstore, options: ImporterOptions = {}): Promise { +export async function importByteStream (bufs: ByteStream, blockstore: WritableStorage, options: ImporterOptions = {}): Promise { return await importFile({ content: bufs }, blockstore, options) diff --git a/packages/ipfs-unixfs-importer/src/tree-builder.ts b/packages/ipfs-unixfs-importer/src/tree-builder.ts index 2b5946a5..05ee3906 100644 --- a/packages/ipfs-unixfs-importer/src/tree-builder.ts +++ b/packages/ipfs-unixfs-importer/src/tree-builder.ts @@ -2,7 +2,7 @@ import { DirFlat } from './dir-flat.js' import { flatToShard } from './flat-to-shard.js' import { Dir } from './dir.js' import { toPathComponents } from './utils/to-path-components.js' -import type { ImportResult, InProgressImportResult, TreeBuilder, Blockstore } from './index.js' +import type { ImportResult, InProgressImportResult, TreeBuilder, WritableStorage } from './index.js' import type { PersistOptions } from './utils/persist.js' export interface AddToTreeOptions extends PersistOptions { @@ -54,7 +54,7 @@ async function addToTree (elem: InProgressImportResult, tree: Dir, options: AddT return tree } -async function * flushAndYield (tree: Dir | InProgressImportResult, blockstore: Blockstore): AsyncGenerator { +async function * flushAndYield (tree: Dir | InProgressImportResult, blockstore: WritableStorage): AsyncGenerator { if (!(tree instanceof Dir)) { if (tree.unixfs?.isDirectory() === true) { yield tree diff --git a/packages/ipfs-unixfs-importer/src/utils/persist.ts b/packages/ipfs-unixfs-importer/src/utils/persist.ts index 7a86fa03..2bbe36e1 100644 --- a/packages/ipfs-unixfs-importer/src/utils/persist.ts +++ b/packages/ipfs-unixfs-importer/src/utils/persist.ts @@ -1,7 +1,7 @@ import { CID } from 'multiformats/cid' import * as dagPb from '@ipld/dag-pb' import { sha256 } from 'multiformats/hashes/sha2' -import type { Blockstore } from '../index.js' +import type { WritableStorage } from '../index.js' import type { BlockCodec } from 'multiformats/codecs/interface' import type { Version as CIDVersion } from 'multiformats/cid' import type { ProgressOptions } from 'progress-events' @@ -12,7 +12,7 @@ export interface PersistOptions extends ProgressOptions { signal?: AbortSignal } -export const persist = async (buffer: Uint8Array, blockstore: Blockstore, options: PersistOptions): Promise => { +export const persist = async (buffer: Uint8Array, blockstore: WritableStorage, options: PersistOptions): Promise => { if (options.codec == null) { options.codec = dagPb }