fix: check bytes match cid (#1069)

Do the bytes match the CID?

For blocks whose multihash was calculated with sha2-256 (the default hashing function) we rehash each block's bytes to check it matches blocks proposed CID multihash.

For other hashing functions, we allow the blocks... The cluster ipfs nodes will drop blocks that are invalid, so such DAGs wont ever be marked as pinned. It's not a good UX but at least we wont say we are storing a full DAG that was sent to us malformed.

License: (Apache-2.0 AND MIT)
Signed-off-by: Oli Evans <oli@tableflip.io>

packages/api/src/car.js

@@ -1,7 +1,7 @@
 /* eslint-env serviceworker */
 import { PutObjectCommand } from '@aws-sdk/client-s3/dist-es/commands/PutObjectCommand.js'
 import { CarBlockIterator } from '@ipld/car'
-import { toString } from 'uint8arrays'
+import { toString, equals } from 'uint8arrays'
 import { Block } from 'multiformats/block'
 import { sha256 } from 'multiformats/hashes/sha2'
 import * as raw from 'multiformats/codecs/raw'
@@ -293,6 +293,12 @@ async function carStat (carBlob) {
     if (blockSize > MAX_BLOCK_SIZE) {
       throw new InvalidCarError(`block too big: ${blockSize} > ${MAX_BLOCK_SIZE}`)
     }
+    if (block.cid.multihash.code === sha256.code) {
+      const ourHash = await sha256.digest(block.bytes)
+      if (!equals(ourHash.digest, block.cid.multihash.digest)) {
+        throw new InvalidCarError(`block data does not match CID for ${block.cid.toString()}`)
+      }
+    }
     if (!rawRootBlock && block.cid.equals(rootCid)) {
       rawRootBlock = block
     }

packages/api/test/car.spec.js

@@ -1,7 +1,7 @@
 /* eslint-env mocha */
 import assert from 'assert'
 import { CID } from 'multiformats/cid'
-import { sha256 } from 'multiformats/hashes/sha2'
+import { sha256, sha512 } from 'multiformats/hashes/sha2'
 import * as pb from '@ipld/dag-pb'
 import { CarWriter } from '@ipld/car'
 import fetch, { Blob } from '@web-std/fetch'
@@ -209,4 +209,68 @@ describe('POST /car', () => {
     const { message } = await res.json()
     assert.strictEqual(message, 'Invalid CAR file received: CAR must contain at least one non-root block')
   })
+
+  it('should allow a CAR with unsupported hash function', async () => {
+    const token = await getTestJWT('test-upload', 'test-upload')
+
+    const bytes = pb.encode({ Data: new Uint8Array(), Links: [] })
+    // we dont support sha512 yet!
+    const hash = await sha512.digest(bytes)
+    const cid = CID.create(1, pb.code, hash)
+
+    const { writer, out } = CarWriter.create([cid])
+    writer.put({ cid, bytes })
+    writer.close()
+
+    const carBytes = []
+    for await (const chunk of out) {
+      carBytes.push(chunk)
+    }
+
+    const res = await fetch(new URL('car', endpoint), {
+      method: 'POST',
+      headers: {
+        Authorization: `Bearer ${token}`,
+        'Content-Type': 'application/car'
+      },
+      body: new Blob(carBytes)
+    })
+
+    assert(res, 'Server responded')
+    assert(res.ok, 'Server response ok')
+    const resBody = await res.json()
+    assert(resBody.cid, 'Server response payload has `cid` property')
+    assert.strictEqual(resBody.cid, cid.toString(), 'Server responded with expected CID')
+  })
+
+  it('should throw for CAR with a block where the bytes do match the CID', async () => {
+    const token = await getTestJWT('test-upload', 'test-upload')
+
+    const bytes = pb.encode({ Data: new Uint8Array(), Links: [] })
+    const hash = await sha256.digest(bytes)
+    const cid = CID.create(1, pb.code, hash)
+
+    const { writer, out } = CarWriter.create([cid])
+    bytes[bytes.length - 1] = bytes[bytes.length - 1] + 1 // mangle a byte
+    writer.put({ cid, bytes })
+    writer.close()
+
+    const carBytes = []
+    for await (const chunk of out) {
+      carBytes.push(chunk)
+    }
+
+    const res = await fetch(new URL('car', endpoint), {
+      method: 'POST',
+      headers: {
+        Authorization: `Bearer ${token}`,
+        'Content-Type': 'application/car'
+      },
+      body: new Blob(carBytes)
+    })
+
+    assert.strictEqual(res.ok, false)
+    const { message } = await res.json()
+    assert.strictEqual(message, `Invalid CAR file received: block data does not match CID for ${cid.toString()}`)
+  })
 })

packages/api/test/scripts/worker-globals.js

@@ -1,4 +1,4 @@
-export const ENV = 'dev'
+export const ENV = 'test'
 export const MAINTENANCE_MODE = 'rw'
 export const SALT = 'test-salt'
 export const MAGIC_SECRET_KEY = 'test-magic-secret-key'