Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: disallow CAR of single block with links #344

Merged
merged 5 commits into from
Aug 12, 2021
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
72 changes: 53 additions & 19 deletions packages/api/src/car.js
Original file line number Diff line number Diff line change
Expand Up @@ -139,10 +139,8 @@ export async function carPost (request, env, ctx) {
const bytes = new Uint8Array(await blob.arrayBuffer())
const reader = await CarReader.fromBytes(bytes)

const chunkSize = await getBlocksSize(reader)
if (chunkSize === 0) {
throw new Error('empty CAR')
}
const stat = await carStat(reader)
await validateCar(reader, stat)

// Ensure car blob.type is set; it is used by the cluster client to set the foramt=car flag on the /add call.
const content = blob.slice(0, blob.size, 'application/car')
Expand Down Expand Up @@ -188,7 +186,7 @@ export async function carPost (request, env, ctx) {
try {
await env.db.query(INCREMENT_USER_USED_STORAGE, {
user: user._id,
amount: chunkSize
amount: stat.size
})
} catch (err) {
console.error(`failed to update user used storage: ${err.stack}`)
Expand Down Expand Up @@ -263,41 +261,77 @@ export async function sizeOf (response) {
}

/**
* Returns the sum of all block sizes in the received CAR. Throws if any block
* @param {CarReader} reader
* @param {CarStat} stat
*/
async function validateCar (reader, stat) {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What is the case that leads to the creation of such CAR files? I am not sure if this is the best place for such validation?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

go-ipfs can create them where it's using a url-store and the url can no longer be reached. it's an edge case that we hit pretty hard, but is otherwise pretty uncommon.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ah I see, so it is a valid Car file by the end of the day. Probably better to have a better name for this like isCompatibleCar

if (stat.blocks === 0) {
throw new Error('empty CAR')
}

const roots = await reader.getRoots()
if (roots.length === 0) {
throw new Error('missing roots')
}
if (roots.length > 1) {
throw new Error('too many roots')
}

const rootBlock = await getBlock(reader, roots[0])
const numLinks = Array.from(rootBlock.links()).length

// If the root block has links, then we should have at least 2 blocks in the CAR
if (numLinks > 0 && stat.blocks < 2) {
throw new Error('CAR must contain at least one non-root block')
}
}

/**
* Returns the sum of all block sizes and total blocks. Throws if any block
* is bigger than MAX_BLOCK_SIZE (1MiB).
*
* @typedef {{ size: number, blocks: number }} CarStat
* @param {CarReader} reader
* @returns {Promise<CarStat>}
*/
async function getBlocksSize (reader) {
async function carStat (reader) {
let blocks = 0
let size = 0
for await (const block of reader.blocks()) {
const blockSize = block.bytes.byteLength
if (blockSize > MAX_BLOCK_SIZE) {
throw new Error(`block too big: ${blockSize} > ${MAX_BLOCK_SIZE}`)
}
size += blockSize
blocks++
}
return size
return { size, blocks }
}

const decoders = [pb, raw, cbor]

/**
* @param {CarReader} reader
* @param {import('multiformats').CID} cid
*/
async function getBlock (reader, cid) {
const rawBlock = await reader.get(cid)
if (!rawBlock) throw new Error(`missing block for ${cid}`)
const { bytes } = rawBlock
const decoder = decoders.find(d => d.code === cid.code)
if (!decoder) throw new Error(`missing decoder for ${cid.code}`)
return new Block({ cid, bytes, value: decoder.decode(bytes) })
}

/**
* Returns the DAG size of the CAR but only if the graph is complete.
* @param {CarReader} reader
*/
async function getDagSize (reader) {
const decoders = [pb, raw, cbor]
const [rootCid] = await reader.getRoots()

const getBlock = async cid => {
const rawBlock = await reader.get(cid)
if (!rawBlock) throw new Error(`missing block for ${cid}`)
const { bytes } = rawBlock
const decoder = decoders.find(d => d.code === cid.code)
if (!decoder) throw new Error(`missing decoder for ${cid.code}`)
return new Block({ cid, bytes, value: decoder.decode(bytes) })
}

const getSize = async cid => {
const block = await getBlock(cid)
const block = await getBlock(reader, cid)
let size = block.bytes.byteLength
for (const [, cid] of block.links()) {
size += await getSize(cid)
Expand Down
98 changes: 97 additions & 1 deletion packages/api/test/car.spec.js
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,102 @@ describe('POST /car', () => {

assert.strictEqual(res.ok, false)
const { message } = await res.json()
assert.ok(message.includes('empty CAR'))
assert.strictEqual(message, 'empty CAR')
})

it('should throw for CAR with no roots', async () => {
const token = await getTestJWT()

const bytes = pb.encode({ Data: new Uint8Array(), Links: [] })
const hash = await sha256.digest(bytes)
const cid = CID.create(1, pb.code, hash)

const { writer, out } = CarWriter.create([])
writer.put({ cid, bytes })
writer.close()

const carBytes = []
for await (const chunk of out) {
carBytes.push(chunk)
}

const res = await fetch(new URL('car', endpoint), {
method: 'POST',
headers: {
Authorization: `Bearer ${token}`,
'Content-Type': 'application/car'
},
body: new Blob(carBytes)
})

assert.strictEqual(res.ok, false)
const { message } = await res.json()
assert.strictEqual(message, 'missing roots')
})

it('should throw for CAR with multiple roots', async () => {
const token = await getTestJWT()

const bytes = pb.encode({ Data: new Uint8Array(), Links: [] })
const hash = await sha256.digest(bytes)
const cid = CID.create(1, pb.code, hash)

const { writer, out } = CarWriter.create([
cid,
CID.parse('bafybeibqmrg5e5bwhx2ny4kfcjx2mm3ohh2cd4i54wlygquwx7zbgwqs4e')
])
writer.put({ cid, bytes })
writer.close()

const carBytes = []
for await (const chunk of out) {
carBytes.push(chunk)
}

const res = await fetch(new URL('car', endpoint), {
method: 'POST',
headers: {
Authorization: `Bearer ${token}`,
'Content-Type': 'application/car'
},
body: new Blob(carBytes)
})

assert.strictEqual(res.ok, false)
const { message } = await res.json()
assert.strictEqual(message, 'too many roots')
})

it('should throw for CAR with one root block that has links', async () => {
const token = await getTestJWT()

const bytes = pb.encode({
Data: new Uint8Array(),
Links: [{ Hash: CID.parse('bafybeibqmrg5e5bwhx2ny4kfcjx2mm3ohh2cd4i54wlygquwx7zbgwqs4e') }]
})
const hash = await sha256.digest(bytes)
const cid = CID.create(1, pb.code, hash)

const { writer, out } = CarWriter.create(cid)
writer.put({ cid, bytes })
writer.close()

const carBytes = []
for await (const chunk of out) {
carBytes.push(chunk)
}

const res = await fetch(new URL('car', endpoint), {
method: 'POST',
headers: {
Authorization: `Bearer ${token}`,
'Content-Type': 'application/car'
},
body: new Blob(carBytes)
})

assert.strictEqual(res.ok, false)
const { message } = await res.json()
assert.strictEqual(message, 'CAR must contain at least one non-root block')
})
})