fix: add backup url parts to list #2675
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,4 +1,6 @@ | ||
| import * as Link from 'multiformats/link' | ||
| import * as Digest from 'multiformats/hashes/digest' | ||
| import { fromString } from 'uint8arrays' | ||
|
|
||
| /** | ||
| * We mixed upload type and content type. This was split into `type` and | ||
|
|
@@ -20,6 +22,13 @@ const typeMap = { | |
| * @param {string} [sourceCid] - User input CID so we can return the same cid version back | ||
| */ | ||
| export function toNFTResponse(upload, sourceCid) { | ||
| // get hash links to CARs that contain parts of this upload | ||
| /** @type {string[]} */ | ||
| const parts = [ | ||
| // from upload table 'backup_urls' column | ||
| ...carCidV1Base32sFromBackupUrls(upload.backup_urls ?? []), | ||
| ] | ||
|
|
||
| /** @type {import('../bindings').NFTResponse} */ | ||
|
Comment on lines
+25
to
+31
There was a problem hiding this comment. adapted from https://github.com/web3-storage/web3.storage/blob/main/packages/db/utils.js#L19 in nft.storage we ran the migration to get backUrls from Migration: https://gist.github.com/yocontra/4ae719b42a54c4582ca28fcac74199d0 I double checked a few rows of |
||
| const nft = { | ||
| cid: sourceCid || upload.source_cid, | ||
|
|
@@ -29,6 +38,7 @@ export function toNFTResponse(upload, sourceCid) { | |
| files: upload.files, | ||
| size: upload.content.dag_size || 0, | ||
| name: upload.name, | ||
| parts, | ||
| pin: { | ||
| cid: sourceCid || upload.source_cid, | ||
| created: upload.content.pin[0].inserted_at, | ||
|
|
@@ -101,3 +111,53 @@ function transformPinStatus(status) { | |
| return 'failed' | ||
| } | ||
| } | ||
|
|
||
| /** | ||
| * given array of backup_urls from uploads table, return a corresponding set of CAR CIDv1 using base32 multihash | ||
| * for any CAR files in the backup_urls. | ||
| * | ||
| * @param {unknown[]} backupUrls | ||
| * @returns {Iterable<string>} | ||
| */ | ||
| function carCidV1Base32sFromBackupUrls(backupUrls) { | ||
| const carCidStrings = new Set() | ||
|
There was a problem hiding this comment. |
||
| for (const backupUrl of backupUrls) { | ||
| let carCid | ||
| try { | ||
| // @ts-expect-error database exported types assumes unknown | ||
| carCid = bucketKeyToPartCID(backupUrl) | ||
| } catch (error) { | ||
| console.warn('error extracting car CID from bucket URL', error) | ||
| } | ||
| if (!carCid) continue | ||
| carCidStrings.add(carCid.toString()) | ||
| } | ||
| return carCidStrings | ||
| } | ||
|
|
||
| const CAR_CODE = 0x0202 | ||
|
|
||
| /** | ||
| * Attempts to extract a CAR CID from a bucket key. | ||
| * | ||
| * @param {string} key | ||
| */ | ||
| const bucketKeyToPartCID = (key) => { | ||
| const filename = String(key.split('/').at(-1)) | ||
|
There was a problem hiding this comment. There was a problem hiding this comment. I think should be fine looking at the code but worth a double check - it needs to now also account for backup URL in this format: There was a problem hiding this comment. I am going to add a test |
||
| const [hash] = filename.split('.') | ||
| try { | ||
| // recent buckets encode CAR CID in filename | ||
| const cid = Link.parse(hash).toV1() | ||
| if (cid.code === CAR_CODE) return cid | ||
| throw new Error('not a CAR CID') | ||
| } catch (err) { | ||
| // older buckets base32 encode a CAR multihash <base32(car-multihash)>.car | ||
| try { | ||
| const digestBytes = fromString(hash, 'base32') | ||
| const digest = Digest.decode(digestBytes) | ||
| return Link.create(CAR_CODE, digest) | ||
| } catch (error) { | ||
| // console.warn('error trying to create CID from s3 key', error) | ||
| } | ||
| } | ||
| } | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
needed to upgrade to use Link, which made needed to upgrade IPLD deps to decrease TS issues