Skip to content

Commit

Permalink
fix: align blockstore pick interface name (#301)
Browse files Browse the repository at this point in the history
  • Loading branch information
achingbrain authored Mar 16, 2023
1 parent 18e5b94 commit ca10d79
Show file tree
Hide file tree
Showing 11 changed files with 37 additions and 37 deletions.
16 changes: 8 additions & 8 deletions packages/ipfs-unixfs-exporter/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import resolve from './resolvers/index.js'
import last from 'it-last'
import type { UnixFS } from 'ipfs-unixfs'
import type { PBNode } from '@ipld/dag-pb'
import type { Blockstore as InterfaceBlockstore } from 'interface-blockstore'
import type { Blockstore } from 'interface-blockstore'
import type { Bucket } from 'hamt-sharding'
import type { ProgressOptions } from 'progress-events'

Expand Down Expand Up @@ -65,21 +65,21 @@ export interface ResolveResult {
next?: NextResult
}

export interface Resolve { (cid: CID, name: string, path: string, toResolve: string[], depth: number, blockstore: Blockstore, options: ExporterOptions): Promise<ResolveResult> }
export interface Resolver { (cid: CID, name: string, path: string, toResolve: string[], resolve: Resolve, depth: number, blockstore: Blockstore, options: ExporterOptions): Promise<ResolveResult> }
export interface Resolve { (cid: CID, name: string, path: string, toResolve: string[], depth: number, blockstore: ReadableStorage, options: ExporterOptions): Promise<ResolveResult> }
export interface Resolver { (cid: CID, name: string, path: string, toResolve: string[], resolve: Resolve, depth: number, blockstore: ReadableStorage, options: ExporterOptions): Promise<ResolveResult> }

export type UnixfsV1FileContent = AsyncIterable<Uint8Array> | Iterable<Uint8Array>
export type UnixfsV1DirectoryContent = AsyncIterable<UnixFSEntry> | Iterable<UnixFSEntry>
export type UnixfsV1Content = UnixfsV1FileContent | UnixfsV1DirectoryContent
export interface UnixfsV1Resolver { (cid: CID, node: PBNode, unixfs: UnixFS, path: string, resolve: Resolve, depth: number, blockstore: Blockstore): (options: ExporterOptions) => UnixfsV1Content }
export interface UnixfsV1Resolver { (cid: CID, node: PBNode, unixfs: UnixFS, path: string, resolve: Resolve, depth: number, blockstore: ReadableStorage): (options: ExporterOptions) => UnixfsV1Content }

export interface ShardTraversalContext {
hamtDepth: number
rootBucket: Bucket<boolean>
lastBucket: Bucket<boolean>
}

export type Blockstore = Pick<InterfaceBlockstore, 'get'>
export type ReadableStorage = Pick<Blockstore, 'get'>

const toPathComponents = (path: string = ''): string[] => {
// split on / unless escaped with \
Expand Down Expand Up @@ -121,7 +121,7 @@ const cidAndRest = (path: string | Uint8Array | CID): { cid: CID, toResolve: str
throw errCode(new Error(`Unknown path type ${path}`), 'ERR_BAD_PATH')
}

export async function * walkPath (path: string | CID, blockstore: Blockstore, options: ExporterOptions = {}): AsyncGenerator<UnixFSEntry, void, any> {
export async function * walkPath (path: string | CID, blockstore: ReadableStorage, options: ExporterOptions = {}): AsyncGenerator<UnixFSEntry, void, any> {
let {
cid,
toResolve
Expand Down Expand Up @@ -153,7 +153,7 @@ export async function * walkPath (path: string | CID, blockstore: Blockstore, op
}
}

export async function exporter (path: string | CID, blockstore: Blockstore, options: ExporterOptions = {}): Promise<UnixFSEntry> {
export async function exporter (path: string | CID, blockstore: ReadableStorage, options: ExporterOptions = {}): Promise<UnixFSEntry> {
const result = await last(walkPath(path, blockstore, options))

if (result == null) {
Expand All @@ -163,7 +163,7 @@ export async function exporter (path: string | CID, blockstore: Blockstore, opti
return result
}

export async function * recursive (path: string | CID, blockstore: Blockstore, options: ExporterOptions = {}): AsyncGenerator<UnixFSEntry, void, any> {
export async function * recursive (path: string | CID, blockstore: ReadableStorage, options: ExporterOptions = {}): AsyncGenerator<UnixFSEntry, void, any> {
const node = await exporter(path, blockstore, options)

if (node == null) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,9 @@ import parallel from 'it-parallel'
import { pipe } from 'it-pipe'
import map from 'it-map'
import PQueue from 'p-queue'
import type { ExporterOptions, UnixfsV1FileContent, UnixfsV1Resolver, Blockstore } from '../../../index.js'
import type { ExporterOptions, UnixfsV1FileContent, UnixfsV1Resolver, ReadableStorage } from '../../../index.js'

async function walkDAG (blockstore: Blockstore, node: dagPb.PBNode | Uint8Array, queue: Pushable<Uint8Array>, streamPosition: bigint, start: bigint, end: bigint, options: ExporterOptions): Promise<void> {
async function walkDAG (blockstore: ReadableStorage, node: dagPb.PBNode | Uint8Array, queue: Pushable<Uint8Array>, streamPosition: bigint, start: bigint, end: bigint, options: ExporterOptions): Promise<void> {
// a `raw` node
if (node instanceof Uint8Array) {
queue.push(extractDataFromBlock(node, streamPosition, start, end))
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import parallel from 'it-parallel'
import { pipe } from 'it-pipe'
import map from 'it-map'
import { decode, PBNode } from '@ipld/dag-pb'
import type { ExporterOptions, Resolve, UnixfsV1DirectoryContent, UnixfsV1Resolver, Blockstore } from '../../../index.js'
import type { ExporterOptions, Resolve, UnixfsV1DirectoryContent, UnixfsV1Resolver, ReadableStorage } from '../../../index.js'

const hamtShardedDirectoryContent: UnixfsV1Resolver = (cid, node, unixfs, path, resolve, depth, blockstore) => {
function yieldHamtDirectoryContent (options: ExporterOptions = {}): UnixfsV1DirectoryContent {
Expand All @@ -12,7 +12,7 @@ const hamtShardedDirectoryContent: UnixfsV1Resolver = (cid, node, unixfs, path,
return yieldHamtDirectoryContent
}

async function * listDirectory (node: PBNode, path: string, resolve: Resolve, depth: number, blockstore: Blockstore, options: ExporterOptions): UnixfsV1DirectoryContent {
async function * listDirectory (node: PBNode, path: string, resolve: Resolve, depth: number, blockstore: ReadableStorage, options: ExporterOptions): UnixfsV1DirectoryContent {
const links = node.Links

const results = pipe(
Expand Down
4 changes: 2 additions & 2 deletions packages/ipfs-unixfs-exporter/src/utils/find-cid-in-shard.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
import { Bucket, BucketPosition, createHAMT } from 'hamt-sharding'
import { decode, PBLink, PBNode } from '@ipld/dag-pb'
import { murmur3128 } from '@multiformats/murmur3'
import type { ExporterOptions, ShardTraversalContext, Blockstore } from '../index.js'
import type { ExporterOptions, ShardTraversalContext, ReadableStorage } from '../index.js'
import type { CID } from 'multiformats/cid'

// FIXME: this is copy/pasted from ipfs-unixfs-importer/src/options.js
Expand Down Expand Up @@ -61,7 +61,7 @@ const toBucketPath = (position: BucketPosition<boolean>): Array<Bucket<boolean>>
return path.reverse()
}

const findShardCid = async (node: PBNode, name: string, blockstore: Blockstore, context?: ShardTraversalContext, options?: ExporterOptions): Promise<CID | undefined> => {
const findShardCid = async (node: PBNode, name: string, blockstore: ReadableStorage, context?: ShardTraversalContext, options?: ExporterOptions): Promise<CID | undefined> => {
if (context == null) {
const rootBucket = createHAMT<boolean>({
hashFn
Expand Down
4 changes: 2 additions & 2 deletions packages/ipfs-unixfs-importer/src/dag-builder/dir.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import { UnixFS } from 'ipfs-unixfs'
import { persist } from '../utils/persist.js'
import { encode, prepare } from '@ipld/dag-pb'
import type { Directory, InProgressImportResult, Blockstore } from '../index.js'
import type { Directory, InProgressImportResult, WritableStorage } from '../index.js'
import type { Version } from 'multiformats/cid'
import type { ProgressOptions } from 'progress-events'

Expand All @@ -10,7 +10,7 @@ export interface DirBuilderOptions extends ProgressOptions {
signal?: AbortSignal
}

export const dirBuilder = async (dir: Directory, blockstore: Blockstore, options: DirBuilderOptions): Promise<InProgressImportResult> => {
export const dirBuilder = async (dir: Directory, blockstore: WritableStorage, options: DirBuilderOptions): Promise<InProgressImportResult> => {
const unixfs = new UnixFS({
type: 'directory',
mtime: dir.mtime,
Expand Down
8 changes: 4 additions & 4 deletions packages/ipfs-unixfs-importer/src/dag-builder/file.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import { persist } from '../utils/persist.js'
import { encode, PBLink, prepare } from '@ipld/dag-pb'
import parallelBatch from 'it-parallel-batch'
import * as rawCodec from 'multiformats/codecs/raw'
import type { BufferImporter, File, InProgressImportResult, Blockstore, SingleBlockImportResult } from '../index.js'
import type { BufferImporter, File, InProgressImportResult, WritableStorage, SingleBlockImportResult } from '../index.js'
import type { FileLayout, Reducer } from '../layout/index.js'
import type { Version } from 'multiformats/cid'
import type { ProgressOptions } from 'progress-events'
Expand All @@ -13,7 +13,7 @@ interface BuildFileBatchOptions {
blockWriteConcurrency: number
}

async function * buildFileBatch (file: File, blockstore: Blockstore, options: BuildFileBatchOptions): AsyncGenerator<InProgressImportResult> {
async function * buildFileBatch (file: File, blockstore: WritableStorage, options: BuildFileBatchOptions): AsyncGenerator<InProgressImportResult> {
let count = -1
let previous: SingleBlockImportResult | undefined

Expand Down Expand Up @@ -60,7 +60,7 @@ function isSingleBlockImport (result: any): result is SingleBlockImportResult {
return result.single === true
}

const reduce = (file: File, blockstore: Blockstore, options: ReduceOptions): Reducer => {
const reduce = (file: File, blockstore: WritableStorage, options: ReduceOptions): Reducer => {
const reducer: Reducer = async function (leaves) {
if (leaves.length === 1 && isSingleBlockImport(leaves[0]) && options.reduceSingleLeafToSelf) {
const leaf = leaves[0]
Expand Down Expand Up @@ -163,6 +163,6 @@ export interface FileBuilderOptions extends BuildFileBatchOptions, ReduceOptions
layout: FileLayout
}

export const fileBuilder = async (file: File, block: Blockstore, options: FileBuilderOptions): Promise<InProgressImportResult> => {
export const fileBuilder = async (file: File, block: WritableStorage, options: FileBuilderOptions): Promise<InProgressImportResult> => {
return await options.layout(buildFileBatch(file, block, options), reduce(file, block, options))
}
4 changes: 2 additions & 2 deletions packages/ipfs-unixfs-importer/src/dag-builder/index.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import { dirBuilder, DirBuilderOptions } from './dir.js'
import { fileBuilder, FileBuilderOptions } from './file.js'
import errCode from 'err-code'
import type { Directory, File, FileCandidate, ImportCandidate, InProgressImportResult, Blockstore } from '../index.js'
import type { Directory, File, FileCandidate, ImportCandidate, InProgressImportResult, WritableStorage } from '../index.js'
import type { ChunkValidator } from './validate-chunks.js'
import type { Chunker } from '../chunker/index.js'

Expand Down Expand Up @@ -42,7 +42,7 @@ export interface DagBuilderOptions extends FileBuilderOptions, DirBuilderOptions
export type ImporterSourceStream = AsyncIterable<ImportCandidate> | Iterable<ImportCandidate>

export interface DAGBuilder {
(source: ImporterSourceStream, blockstore: Blockstore): AsyncIterable<() => Promise<InProgressImportResult>>
(source: ImporterSourceStream, blockstore: WritableStorage): AsyncIterable<() => Promise<InProgressImportResult>>
}

export function defaultDagBuilder (options: DagBuilderOptions): DAGBuilder {
Expand Down
4 changes: 2 additions & 2 deletions packages/ipfs-unixfs-importer/src/dir.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import type { Blockstore, ImportResult, InProgressImportResult } from './index.js'
import type { WritableStorage, ImportResult, InProgressImportResult } from './index.js'
import type { Mtime, UnixFS } from 'ipfs-unixfs'
import { CID } from 'multiformats/cid'
import type { PersistOptions } from './utils/persist.js'
Expand Down Expand Up @@ -50,7 +50,7 @@ export abstract class Dir {
abstract put (name: string, value: InProgressImportResult | Dir): Promise<void>
abstract get (name: string): Promise<InProgressImportResult | Dir | undefined>
abstract eachChildSeries (): AsyncIterable<{ key: string, child: InProgressImportResult | Dir }>
abstract flush (blockstore: Blockstore): AsyncGenerator<ImportResult>
abstract flush (blockstore: WritableStorage): AsyncGenerator<ImportResult>
abstract estimateNodeSize (): number
abstract childCount (): number
}
Expand Down
18 changes: 9 additions & 9 deletions packages/ipfs-unixfs-importer/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import { DAGBuilder, defaultDagBuilder } from './dag-builder/index.js'
import { defaultTreeBuilder } from './tree-builder.js'
import type { UnixFS, Mtime } from 'ipfs-unixfs'
import type { CID, Version as CIDVersion } from 'multiformats/cid'
import type { Blockstore as InterfaceBlockstore } from 'interface-blockstore'
import type { Blockstore } from 'interface-blockstore'
import { ChunkValidator, defaultChunkValidator } from './dag-builder/validate-chunks.js'
import { fixedSize } from './chunker/fixed-size.js'
import type { Chunker } from './chunker/index.js'
Expand All @@ -17,7 +17,7 @@ import type { ProgressOptions } from 'progress-events'
export type ByteStream = AwaitIterable<Uint8Array>
export type ImportContent = ByteStream | Uint8Array

export type Blockstore = Pick<InterfaceBlockstore, 'put'>
export type WritableStorage = Pick<Blockstore, 'put'>

export interface FileCandidate {
path?: string
Expand Down Expand Up @@ -73,8 +73,8 @@ export interface BufferImporterResult extends ImportResult {
}

export interface HamtHashFn { (value: Uint8Array): Promise<Uint8Array> }
export interface TreeBuilder { (source: AsyncIterable<InProgressImportResult>, blockstore: Blockstore): AsyncIterable<ImportResult> }
export interface BufferImporter { (file: File, blockstore: Blockstore): AsyncIterable<() => Promise<BufferImporterResult>> }
export interface TreeBuilder { (source: AsyncIterable<InProgressImportResult>, blockstore: WritableStorage): AsyncIterable<ImportResult> }
export interface BufferImporter { (file: File, blockstore: WritableStorage): AsyncIterable<() => Promise<BufferImporterResult>> }

export type ImportProgressEvents =
BufferImportProgressEvents
Expand Down Expand Up @@ -227,7 +227,7 @@ export type ImportCandidateStream = AsyncIterable<FileCandidate | DirectoryCandi
* }
* ```
*/
export async function * importer (source: ImportCandidateStream, blockstore: Blockstore, options: ImporterOptions = {}): AsyncGenerator<ImportResult, void, unknown> {
export async function * importer (source: ImportCandidateStream, blockstore: WritableStorage, options: ImporterOptions = {}): AsyncGenerator<ImportResult, void, unknown> {
let candidates: AsyncIterable<FileCandidate | DirectoryCandidate> | Iterable<FileCandidate | DirectoryCandidate>

if (Symbol.asyncIterator in source || Symbol.iterator in source) {
Expand Down Expand Up @@ -302,7 +302,7 @@ export async function * importer (source: ImportCandidateStream, blockstore: Blo
* const entry = await importFile(input, blockstore)
* ```
*/
export async function importFile (content: FileCandidate, blockstore: Blockstore, options: ImporterOptions = {}): Promise<ImportResult> {
export async function importFile (content: FileCandidate, blockstore: WritableStorage, options: ImporterOptions = {}): Promise<ImportResult> {
const result = await first(importer([content], blockstore, options))

if (result == null) {
Expand Down Expand Up @@ -333,7 +333,7 @@ export async function importFile (content: FileCandidate, blockstore: Blockstore
* const entry = await importDirectory(input, blockstore)
* ```
*/
export async function importDirectory (content: DirectoryCandidate, blockstore: Blockstore, options: ImporterOptions = {}): Promise<ImportResult> {
export async function importDirectory (content: DirectoryCandidate, blockstore: WritableStorage, options: ImporterOptions = {}): Promise<ImportResult> {
const result = await first(importer([content], blockstore, options))

if (result == null) {
Expand Down Expand Up @@ -361,7 +361,7 @@ export async function importDirectory (content: DirectoryCandidate, blockstore:
* const entry = await importBytes(input, blockstore)
* ```
*/
export async function importBytes (buf: ImportContent, blockstore: Blockstore, options: ImporterOptions = {}): Promise<ImportResult> {
export async function importBytes (buf: ImportContent, blockstore: WritableStorage, options: ImporterOptions = {}): Promise<ImportResult> {
return await importFile({
content: buf
}, blockstore, options)
Expand All @@ -388,7 +388,7 @@ export async function importBytes (buf: ImportContent, blockstore: Blockstore, o
* const entry = await importByteStream(input, blockstore)
* ```
*/
export async function importByteStream (bufs: ByteStream, blockstore: Blockstore, options: ImporterOptions = {}): Promise<ImportResult> {
export async function importByteStream (bufs: ByteStream, blockstore: WritableStorage, options: ImporterOptions = {}): Promise<ImportResult> {
return await importFile({
content: bufs
}, blockstore, options)
Expand Down
4 changes: 2 additions & 2 deletions packages/ipfs-unixfs-importer/src/tree-builder.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import { DirFlat } from './dir-flat.js'
import { flatToShard } from './flat-to-shard.js'
import { Dir } from './dir.js'
import { toPathComponents } from './utils/to-path-components.js'
import type { ImportResult, InProgressImportResult, TreeBuilder, Blockstore } from './index.js'
import type { ImportResult, InProgressImportResult, TreeBuilder, WritableStorage } from './index.js'
import type { PersistOptions } from './utils/persist.js'

export interface AddToTreeOptions extends PersistOptions {
Expand Down Expand Up @@ -54,7 +54,7 @@ async function addToTree (elem: InProgressImportResult, tree: Dir, options: AddT
return tree
}

async function * flushAndYield (tree: Dir | InProgressImportResult, blockstore: Blockstore): AsyncGenerator<ImportResult> {
async function * flushAndYield (tree: Dir | InProgressImportResult, blockstore: WritableStorage): AsyncGenerator<ImportResult> {
if (!(tree instanceof Dir)) {
if (tree.unixfs?.isDirectory() === true) {
yield tree
Expand Down
4 changes: 2 additions & 2 deletions packages/ipfs-unixfs-importer/src/utils/persist.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import { CID } from 'multiformats/cid'
import * as dagPb from '@ipld/dag-pb'
import { sha256 } from 'multiformats/hashes/sha2'
import type { Blockstore } from '../index.js'
import type { WritableStorage } from '../index.js'
import type { BlockCodec } from 'multiformats/codecs/interface'
import type { Version as CIDVersion } from 'multiformats/cid'
import type { ProgressOptions } from 'progress-events'
Expand All @@ -12,7 +12,7 @@ export interface PersistOptions extends ProgressOptions {
signal?: AbortSignal
}

export const persist = async (buffer: Uint8Array, blockstore: Blockstore, options: PersistOptions): Promise<CID> => {
export const persist = async (buffer: Uint8Array, blockstore: WritableStorage, options: PersistOptions): Promise<CID> => {
if (options.codec == null) {
options.codec = dagPb
}
Expand Down

0 comments on commit ca10d79

Please sign in to comment.