From cc04ff553b0bc8d63ea0e9bf63fa7a17e467540f Mon Sep 17 00:00:00 2001 From: Irakli Gozalishvili Date: Tue, 16 Apr 2024 17:09:44 -0700 Subject: [PATCH 1/5] feat: w3 index spec --- w3-index.md | 110 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 110 insertions(+) create mode 100644 w3-index.md diff --git a/w3-index.md b/w3-index.md new file mode 100644 index 0000000..f2b232e --- /dev/null +++ b/w3-index.md @@ -0,0 +1,110 @@ +# Index + +# Introduction + +## Abstract + +W3 Indexing protocol allows authorized agents to submit verifiable claims about content-addressable data to be publish on [InterPlanetary Network Indexer (IPNI)][IPNI], making it publicly queryable. + +## Concepts + +### Space + +A namespace, often referred as a "space", is an owned resource that can be shared. It corresponds to a unique asymmetric cryptographic keypair and is identified by a [`did:key`] URI. + +# Capabilities + +## Index Add + +Authorized agent MAY invoke `/space/index/add` capability on the [space] subject to submit [content index] for publishing on [InterPlanetary Network Indexer (IPNI)][IPNI]. + +### Index Add Invocation Example + +Invocation example illustrates Alice requesting to add index under "bafy..blob" to be published on the network. + +```js +{ + "cmd": "/space/index/add", + "sub": "did:key:zAlice", + "iss": "did:key:zAlice", + "aud": "did:web:web3.storage", + "args": { + "index": { "/": "bafy...blob" } + } +} +``` + +### Index Add Capability + +#### Index Add Capability Schema + +```ts +type AddIndex = { + cmd: "/space/index/add" + sub: SpaceDID + args: { + // Link is the Content Archive (CAR) containing + // the `Index`. + index: Link> + } +} + +// Type describes a CAR format +type ContentArchive = ByteView<{ + roots: [Block] + blocks: Block[] +}> +``` + +### Index + +#### Index Schema + +Index schema is variant type keyed by the format descriptor label designed to allow format evolution through versioning and additional schema variants. + +```ts +type Index = Variant<{ + "index/sharded/dag@0.1": ShardedDAGIndex +}> +``` + +#### Sharded DAG Index + +Sharded DAG index MUST describe complete set of blocks that make up the `content` DAG in terms of `BlobSlice`s. + +##### Sharded DAG Index Schema + +```ts + +type ShardedDAGIndex = { + // content root CID + content: Link + // links to blob indexes that contain blocks of the content DAG + shards: Link[] +} + +type BlobIndex = [ + // hash digest of the blob + digest: Multihash + // Index of blob slices + slices: BlobSlice +] + +type BlobSlice = [ + // hash digest of the slice + digest: Multihash + // Slice offset + offset: Int + // Slice size in bytes + length: Int +] + +type Multihash = bytes +``` + +ℹ️ Please note that `shards` is a list of `BlobIndex` links. This provide a flexibility of bundling blob indexes or externalizing them by linking to them. + +It is RECOMMENDED to bundle all the `BlobIndex`s inside the Content Archive of the `Index`. + +[IPNI]:https://github.com/ipni/specs/blob/main/IPNI.md +[`did:key`]:https://w3c-ccg.github.io/did-method-key/ From 9c27f49656f7253be787b6dc091a76d05ab4eb4e Mon Sep 17 00:00:00 2001 From: Irakli Gozalishvili Date: Tue, 16 Apr 2024 17:14:50 -0700 Subject: [PATCH 2/5] chore: ignore some words --- .github/workflows/words-to-ignore.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/words-to-ignore.txt b/.github/workflows/words-to-ignore.txt index f4eb67b..971dc48 100644 --- a/.github/workflows/words-to-ignore.txt +++ b/.github/workflows/words-to-ignore.txt @@ -171,3 +171,6 @@ unlinked 2MiB ok cryptographically +InterPlanetary +queryable +bafy From 42652569a5c27f91b47f4113ba438533364c30bc Mon Sep 17 00:00:00 2001 From: Irakli Gozalishvili Date: Tue, 16 Apr 2024 17:27:29 -0700 Subject: [PATCH 3/5] chore: add example --- w3-index.md | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/w3-index.md b/w3-index.md index f2b232e..09d76d1 100644 --- a/w3-index.md +++ b/w3-index.md @@ -106,5 +106,41 @@ type Multihash = bytes It is RECOMMENDED to bundle all the `BlobIndex`s inside the Content Archive of the `Index`. +##### Sharded DAG Index Example + +> For the reader convenience we use `link` function to denote external blocks that should be linked + +```js +{ + "index/sharded/dag@0.1": { + "content": { "/": "bafy..dag" }, + "shards": [ + link([ + // blob multihash + { "/": { "bytes": "blb...left" } }, + // sliced within the blob + [ + [{ "/": { "bytes": "block..1"} }, 0, 128], + [{ "/": { "bytes": "block..2"} }, 129, 256], + [{ "/": { "bytes": "block..3"} }, 257, 384], + [{ "/": { "bytes": "block..4"} }, 385, 512] + ] + ]), + link([ + // blob multihash + { "/": { "bytes": "blb...right" } }, + // sliced within the blob + [ + [{ "/": { "bytes": "block..5"} }, 0, 128], + [{ "/": { "bytes": "block..6"} }, 129, 256], + [{ "/": { "bytes": "block..7"} }, 257, 384], + [{ "/": { "bytes": "block..8"} }, 385, 512] + ] + ]) + ] + } +} +``` + [IPNI]:https://github.com/ipni/specs/blob/main/IPNI.md [`did:key`]:https://w3c-ccg.github.io/did-method-key/ From 5b7da3ee6ec5645223c3bbcf8459045851de87c9 Mon Sep 17 00:00:00 2001 From: Irakli Gozalishvili Date: Mon, 22 Apr 2024 09:13:40 -0700 Subject: [PATCH 4/5] Update w3-index.md --- w3-index.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/w3-index.md b/w3-index.md index 09d76d1..91e7152 100644 --- a/w3-index.md +++ b/w3-index.md @@ -1,5 +1,6 @@ -# Index +# W3 Index +![wip](https://img.shields.io/badge/status-wip-orange.svg?style=flat-square) # Introduction ## Abstract From 5d7f13538198aa1bec8c2697d70f18cbf2334e7c Mon Sep 17 00:00:00 2001 From: Irakli Gozalishvili Date: Mon, 22 Apr 2024 09:13:58 -0700 Subject: [PATCH 5/5] Update w3-index.md --- w3-index.md | 1 + 1 file changed, 1 insertion(+) diff --git a/w3-index.md b/w3-index.md index 91e7152..6112647 100644 --- a/w3-index.md +++ b/w3-index.md @@ -1,6 +1,7 @@ # W3 Index ![wip](https://img.shields.io/badge/status-wip-orange.svg?style=flat-square) + # Introduction ## Abstract