Skip to content

Commit

Permalink
feat! embrace writable streams (#29)
Browse files Browse the repository at this point in the history
  • Loading branch information
Gozala authored Jul 29, 2022
1 parent af49439 commit d8fe3e1
Show file tree
Hide file tree
Showing 22 changed files with 954 additions and 867 deletions.
92 changes: 44 additions & 48 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -31,41 +31,31 @@ This library provides functionality similar to [ipfs-unixfs-importer][], but it
You can encode a file as follows

```js
import { TransformStream } from "@web-std/stream"
import * as UnixFS from "@ipld/unixfs"

// Create a web `TransformStream` with additional filesystem specific interface
// that allows encoding files and directories into `writable` end and reading
// IPLD blocks from `readable` end.
const archive = UnixFS.create()
// Create a redable & writable streams with internal queue that can
// hold around 32 blocks
const { readable, writable } = new TransformStream(
{},
UnixFS.withCapacity(1048576 * 32)
)
// Next we create a writer with filesystem like API for encoding files and
// directories into IPLD blocks that will come out on `readable` end.
const writer = UnixFS.createWriter({ writable })

// Create file writer that can be used to encode UnixFS file.
const file = UnixFS.createFileWriter(fs)
const file = UnixFS.createFileWriter(writer)
// write some content
file.write(new TextEncoder().encode("hello world"))
// Finalize file by closing it.
const { cid } = await file.close()

// close the archive to close underlying block stream.
archive.close()
// close the writer to close underlying block stream.
writer.close()

// We could encode all this as car file
encodeCAR({ roots: [cid], blocks: archive.blocks })
```

If your runtime provides [`TransformStream`][] or a [`WritableStream`][] APIs you can create filesystem writer from the `WritableStream<Block>` directly (this will allow you to control exactly how `readable`/`writable` streams deal with back pressure)

```ts
import * as UnixFS from "@ipld/unixfs"

const writeFile = async (blob: Blob, writable: WritableStream<Block>) => {
const fs = UnixFS.createWriter({ writable })
const file = UnixFS.createFileWriter(fs)
file.write(new TextEncoder().encode("hello world"))
const { cid } = await file.close()
fs.close()

return cid
}
encodeCAR({ roots: [cid], blocks: readable })
```

You can encode (non sharded) directories with provided API as well
Expand All @@ -74,23 +64,24 @@ You can encode (non sharded) directories with provided API as well
import * as UnixFS from "@ipld/unixfs"

export const demo = async () => {
const fs = UnixFS.create()
const { readable, writable } = new TransformStream()
const writer = UnixFS.createWriter({ writable })

// write a file
const file = UnixFS.createFileWriter(file)
const file = UnixFS.createFileWriter(writer)
file.write(new TextEncoder().encode("hello world"))
const fileLink = await file.close()

// create directory and add a file we encoded above
const dir = UnixFS.createDirectoryWriter(fs)
dir.write("intro.md", fileLink)
const dir = UnixFS.createDirectoryWriter(writer)
dir.set("intro.md", fileLink)
const dirLink = await dir.close()

// now wrap above directory with another and also add the same file
// there
const root = UnixFS.createDirectoryWriter(fs)
root.write("user", dirLink)
root.write("hello.md", fileLink)
root.set("user", dirLink)
root.set("hello.md", fileLink)

// Creates following UnixFS structure where intro.md and hello.md link to same
// IPFS file.
Expand All @@ -99,37 +90,42 @@ export const demo = async () => {
// ./hello.md
const rootLink = await root.close()
// ...
fs.close()
writer.close()
}
```

### Configure importer
### Configuration

You can configure importer by passing chunker, layout and encored implementations
You can configure DAG layout, chunking and bunch of other things by providing API compatible components. Library provides bunch of them but you can also bring your own.

```js
import * as UnixFS from "@ipld/unixfs"
import * as Rabin from "@ipld/unixfs/src/file/chunker/rabin.js"
import * as Trickle from "@ipld/unixfs/src/file/layout/trickle.js"
import * as Rabin from "@ipld/unixfs/file/chunker/rabin"
import * as Trickle from "@ipld/unixfs/file/layout/trickle"
import * as RawLeaf from "multiformats/codecs/raw"
import { sha256 } from "multiformats/hashes/sha2"

const demo = async blob => {
const { writer, blocks } = UnixFS.create({
fileChunker: await Rabin.create({
avg: 60000,
min: 100,
max: 662144,
}),
fileLayout: Trickle.configure({ maxDirectLeaves: 100 }),
// Encode leaf nodes as raw blocks
fileChunkEncoder: RawLeaf,
smallFileEncoder: RawLeaf,
fileEncoder: UnixFS,
hasher: sha256,
const { readable, writable } = new TransformStream()
const writer = UnixFS.createWriter({
writable,
// you can pass only things you want to override
settings: {
fileChunker: await Rabin.create({
avg: 60000,
min: 100,
max: 662144,
}),
fileLayout: Trickle.configure({ maxDirectLeaves: 100 }),
// Encode leaf nodes as raw blocks
fileChunkEncoder: RawLeaf,
smallFileEncoder: RawLeaf,
fileEncoder: UnixFS,
hasher: sha256,
},
})

const file = UnixFS.createFileWriter(fs, { mode: 0644 })
const file = UnixFS.createFileWriter(writer)
// ...
}
```
Expand Down
14 changes: 7 additions & 7 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
"gen:types": "tsc --build",
"prepare": "tsc --build",
"test:web": "playwright-test test/**/*.spec.js --cov && nyc report",
"test:node": "c8 --check-coverage --branches 93 --functions 80 --lines 93 mocha test/**/*.spec.js",
"test:node": "c8 --check-coverage --branches 95 --functions 83 --lines 94 mocha test/**/*.spec.js",
"test": "npm run test:node",
"coverage": "c8 --reporter=html mocha test/test-*.js && npm_config_yes=true npx st -d coverage -p 8080",
"typecheck": "tsc --build",
Expand Down Expand Up @@ -62,27 +62,27 @@
"types": "./dist/src/directory.d.ts",
"import": "./src/directory.js"
},
"./src/file/layout/trickle.js": {
"./file/layout/trickle": {
"types": "./dist/src/file/layout/trickle.d.ts",
"import": "./src/file/layout/trickle.js"
},
"./src/file/layout/balanced.js": {
"./file/layout/balanced": {
"types": "./dist/src/file/layout/balanced.d.ts",
"import": "./src/file/layout/balanced.js"
},
"./src/file/layout/queue.js": {
"./src/file/layout/queue": {
"types": "./dist/src/file/layout/queue.d.ts",
"import": "./src/file/layout/queue.js"
},
"./src/file/chunker/fixed.js": {
"./file/chunker/fixed": {
"types": "./dist/src/file/chunker/fixed.d.ts",
"import": "./src/file/chunker/fixed.js"
},
"./src/file/chunker/rabin.js": {
"./file/chunker/rabin": {
"types": "./dist/src/file/chunker/rabin.d.ts",
"import": "./src/file/chunker/rabin.js"
},
"./src/file/chunker/buffer.js": {
"./file/chunker/buffer": {
"types": "./dist/src/file/chunker/buffer.d.ts",
"import": "./src/file/chunker/buffer.js"
}
Expand Down
115 changes: 73 additions & 42 deletions src/api.ts
Original file line number Diff line number Diff line change
@@ -1,72 +1,103 @@
import * as UnixFS from "./unixfs.js"
import type {
WriterConfig,
EncoderConfig,
FileWriter,
WriterOptions,
EncoderSettings,
WritableBlockStream,
BlockWriter,
View as FileWriterView,
Writer as FileWriter,
Options as FileWriterOptions,
State as FileWriterSate,
CloseOptions,
Chunker,
LayoutEngine,
Block,
MultihashHasher,
MultihashDigest,
EncodedFile,
} from "./file.js"
import type { DirectoryWriter } from "./directory.js"

export type { Channel, Writer } from "./writer/channel.js"
import type {
DirectoryEntry,
Writer as DirectoryWriter,
View as DirectoryWriterView,
Options as DirectoryWriterOptions,
State as DirectoryWriterState,
} from "./directory.js"
import { Metadata } from "./unixfs.js"

export type {
WriterConfig,
EncoderConfig,
WriterOptions,
CloseOptions,
EncoderSettings,
FileWriterOptions,
FileWriterView,
FileWriter,
FileWriterSate,
EncodedFile,
BlockWriter,
FileWriterConfig,
WritableBlockStream,
} from "./file/api.js"
export {
DirectoryWriterView,
DirectoryWriter,
DirectoryWriterOptions,
DirectoryWriterState,
DirectoryEntry,
DirectoryConfig,
} from "./directory/api.js"
Chunker,
LayoutEngine,
Block,
MultihashHasher,
MultihashDigest,
Metadata,
}

/**
*
*/
export interface Writer {
/**
* Closes this writer and corresponding
*/
close(options?: CloseOptions): Promise<this>
}

/**
* Represents an IPLD [block][] channel with `reader` and `writer` halves. The
* `writer` half provides filesystem like interface for encoding files &
* directories into a [UnixFS][] DAG. The `reader` half emits all the IPLD
* [blocks][] that are created in the process.
* Represents [UnixFS][] DAG writer with a filesystem like API for
* encoding files & directories into a [UnixFS][] DAG.
*
* [block]:https://ipld.io/docs/intro/primer/#blocks-vs-nodes
* [UnixFS]:https://github.com/ipfs/specs/blob/main/UNIXFS.md
*/
export interface FileSystem<Layout extends unknown = unknown>
extends FileSystemWriter<Layout> {
readonly readable: ReadableStream<UnixFS.Block>

readonly writable: WritableBlockStream
export interface View<L extends unknown = unknown> extends Writer {
/**
* Underlaying stream where [UnixFS][] blocks will be written into.
*/
readonly writer: BlockWriter
/**
* Encoder configuration of this writer.
*/

blocks: AsyncIterableIterator<UnixFS.Block>
}
readonly settings: EncoderSettings<L>

export interface FileSystemWriter<L extends unknown = unknown>
extends FileSystemConfig<L> {
/**
* Creates new file writer that will write blocks into the same `BlockQueue`
* as this `DirectoryWriter`.
*
* ⚠️ Please note that file represented by the returned writer is not added to
* to this directory, you need to do that explicitly via `write` call.
* Creates new file writer that will write blocks into the same underlying
* stream. It is mostly convinience function for passing same stream and
* encoder configuration.
*/
createFileWriter<Layout>(
options?: WriterConfig<Layout>
): FileWriter<L | Layout>
settings?: WriterOptions<Layout>
): FileWriterView<L | Layout>

/**
* Creates new directory writer that will write blocks into the same
* `BlockQueue` as this `DirectoryWriter`.
*
* * ⚠️ Please note that directory represented by returned writer is not
* added to this directory, you need to do that explicitly via `write` call.
* underlying stream as this writer. It is mostly convinienc function for
* passing same stream and encoder configuration.
*
*/
createDirectoryWriter(options?: WriterConfig): DirectoryWriter

close(): Promise<void>
createDirectoryWriter<Layout>(
settings?: WriterOptions<Layout>
): DirectoryWriterView<L | Layout>
}

export interface FileSystemConfig<Layout extends unknown = unknown> {
export interface Options<Layout extends unknown = unknown> {
writable: WritableBlockStream
config?: EncoderConfig<Layout>
settings?: EncoderSettings<Layout>
}
Loading

0 comments on commit d8fe3e1

Please sign in to comment.