From fae8de1befec616869f24aade5cc97954c509e74 Mon Sep 17 00:00:00 2001 From: Johnathon Roach Date: Mon, 25 Apr 2022 10:29:18 -0500 Subject: [PATCH 01/12] feat: nft up docs (#1837) * add react-device-detect * create arch switch content display for md files * create nftup docs page * update quickstart with nftup info * refine arch-switch component --- .../website/components/mdx/arch-switch.js | 24 ++++++++++ packages/website/package.json | 1 + packages/website/pages/docs/how-to/meta.json | 5 +- packages/website/pages/docs/how-to/nftup.md | 48 +++++++++++++++++++ packages/website/pages/docs/quickstart.md | 15 ++++++ 5 files changed, 91 insertions(+), 2 deletions(-) create mode 100644 packages/website/components/mdx/arch-switch.js create mode 100644 packages/website/pages/docs/how-to/nftup.md diff --git a/packages/website/components/mdx/arch-switch.js b/packages/website/components/mdx/arch-switch.js new file mode 100644 index 0000000000..cb1fdfeb0d --- /dev/null +++ b/packages/website/components/mdx/arch-switch.js @@ -0,0 +1,24 @@ +import React from 'react' +import { osName } from 'react-device-detect' + +/** + * @param {any} props + */ +export function ArchSwitch({ children }) { + return ( +
+ {children.length && + children.map((/** @type {{ props: { case: string; }; }} */ element) => { + if ( + !element.props || + !element.props.case || + element.props.case === osName + ) { + return element + } + + return null + })} +
+ ) +} diff --git a/packages/website/package.json b/packages/website/package.json index 379bcbd52b..f4edea6c37 100644 --- a/packages/website/package.json +++ b/packages/website/package.json @@ -29,6 +29,7 @@ "nft.storage": "^6.0.0", "rc-tooltip": "^5.1.1", "react": "17.0.2", + "react-device-detect": "^2.2.2", "react-dom": "17.0.2", "react-icons": "^4.3.1", "react-if": "4.0.1", diff --git a/packages/website/pages/docs/how-to/meta.json b/packages/website/pages/docs/how-to/meta.json index f72e2bde59..8ece24609d 100644 --- a/packages/website/pages/docs/how-to/meta.json +++ b/packages/website/pages/docs/how-to/meta.json @@ -6,5 +6,6 @@ "store-directory": "Store a directory of files", "get-status": "Get upload status", "ucan": "Use UCAN tokens for delegated authorization", - "pinning-service": "Pinning Services API" -} \ No newline at end of file + "pinning-service": "Pinning Services API", + "nftup": "Upload with NFTUp" +} diff --git a/packages/website/pages/docs/how-to/nftup.md b/packages/website/pages/docs/how-to/nftup.md new file mode 100644 index 0000000000..acb65cf9b3 --- /dev/null +++ b/packages/website/pages/docs/how-to/nftup.md @@ -0,0 +1,48 @@ +--- +title: Upload files and directories with NFTUp +--- + +import { ArchSwitch } from 'components/mdx/arch-switch'; +import Callout from 'nextra-theme-docs/callout'; + +# Upload files and directories with NFTUp + +NFTUp is the easiest way for content creators to upload their metadata and assets, ready to be minted into NFTs by smart contracts and then traded on marketplaces, and browsed in galleries. + + + + **Download for Mac OS** + - [Mac Universal](https://github.com/nftstorage/nftup/releases/download/v1.0.0-beta.0/NFT-UP-1.0.0-beta.0-universal.dmg) + - [Mac M1](https://github.com/nftstorage/nftup/releases/download/v1.0.0-beta.0/NFT-UP-1.0.0-beta.0-arm64.dmg) + - [Mac x86](https://github.com/nftstorage/nftup/releases/download/v1.0.0-beta.0/NFT-UP-1.0.0-beta.0.dmg) + + + **Download for Windows** + - [Windows](https://github.com/nftstorage/nftup/releases/download/v1.0.0-beta.0/NFT-UP-Setup-1.0.0-beta.0.exe) + + [All Releases](https://github.com/nftstorage/nftup/releases) + + +While your download is happening, here is a quick preview of what you can expect. First you'll download the disk image and drag and drop NFTUp to your Applications folder. + +Install Application + +Then double click to launch the application and you'll see the main screen. + +Screenshot 2022-03-31 at 15 09 35 + +Now you can drag and drop some files or folders on it. You might be prompted for your NFT.Storage API key. You can [get your key on your account page](https://nft.storage/manage/). + +Screen Shot 2022-04-04 at 11 54 35 AM + +You can upload any file structure you want. If you are doing something like a HashLips build, you can upload the assets and metadata at the same time by dragging them both to the upload box. + +Screenshot 2022-03-31 at 15 14 17 + +Once the upload is complete, you'll get a link to your content. Click it to fetch your data from the IPFS network. + +Screenshot 2022-03-31 at 15 19 49 + +Now you are ready to set your [smart contract base URL](https://nft.storage/blog/post/2022-02-15-base-url-apis/), or otherwise share your content with NFT minting services. + +Enjoy NFTUp, and if you want to contribute, [the code is available on GitHub.](https://github.com/nftstorage/nftup) diff --git a/packages/website/pages/docs/quickstart.md b/packages/website/pages/docs/quickstart.md index f7dbef0710..a49d1866ef 100644 --- a/packages/website/pages/docs/quickstart.md +++ b/packages/website/pages/docs/quickstart.md @@ -16,6 +16,7 @@ NFT.Storage is especially useful for individual creators who are minting NFTs, o 1. [Creating an NFT.Storage account](#create-an-account) 1. [Uploading a file via the website](#uploading-a-file-using-the-website) +1. [Uploading files & directories via the NFTUp application](#uploading-files--directories-via-the-nftup-application) 1. [Getting a free API token](#get-an-api-token) 1. [Using the JavaScript API](#using-the-javascript-api) @@ -63,6 +64,20 @@ Once you're logged in, you can get started with NFT.Storage right away by upload Once the upload is complete, you'll be able to view your file in the [file listing page](https://nft.storage/files/). +### Uploading files & directories via the NFTUp application + +NFTUp is the easiest way for content creators to upload their metadata and assets, ready to be minted into NFTs by smart contracts and then traded on marketplaces, and browsed in galleries. + +Screenshot 2022-03-31 at 15 09 35 + +1. [Download and install](./how-to/nftup/) NFTUp for your OS +1. Launch the application. +1. Drag and drop some files or folders on it. You might be prompted for your NFT.Storage API key. You can [get your key on your account page](https://nft.storage/manage/). +1. Upload any file structure you want. If you are doing something like a HashLips build, you can upload the assets and metadata at the same time by dragging them both to the upload box. +1. Once the upload is complete, you'll get a link to your content. Click it to fetch your data from the IPFS network. +1. Set your [smart contract base URL](https://nft.storage/blog/post/2022-02-15-base-url-apis/), or otherwise share your content with NFT minting services. +1. Contribute on [GitHub.](https://github.com/nftstorage/nftup) + ### Get an API Token It only takes a few moments to get a free API token from NFT.Storage. This token enables you to interact with the NFT.Storage service without using the main website, enabling you to incorporate files stored using NFT.Storage directly into your applications and services. From 1388217ab0c3a7cd7c99045c7d6b86a2e941d163 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 25 Apr 2022 08:38:40 -0700 Subject: [PATCH 02/12] chore(main): release website 1.46.0 (#1831) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- packages/website/CHANGELOG.md | 13 +++++++++++++ packages/website/package.json | 2 +- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/packages/website/CHANGELOG.md b/packages/website/CHANGELOG.md index 002157f3a7..6696f7d422 100644 --- a/packages/website/CHANGELOG.md +++ b/packages/website/CHANGELOG.md @@ -1,5 +1,18 @@ # Changelog +## [1.46.0](https://github.com/nftstorage/nft.storage/compare/website-v1.45.0...website-v1.46.0) (2022-04-25) + + +### Features + +* add permissions endpoint ([#1753](https://github.com/nftstorage/nft.storage/issues/1753)) ([2f5b6fb](https://github.com/nftstorage/nft.storage/commit/2f5b6fb2660231e4591f37e29389c41ac7045605)) +* nft up docs ([#1837](https://github.com/nftstorage/nft.storage/issues/1837)) ([fae8de1](https://github.com/nftstorage/nft.storage/commit/fae8de1befec616869f24aade5cc97954c509e74)) + + +### Bug Fixes + +* remove deps in favor of web apis ([#1701](https://github.com/nftstorage/nft.storage/issues/1701)) ([7f4ae7a](https://github.com/nftstorage/nft.storage/commit/7f4ae7a77da1e5901da10ecda10dfe52e00cb691)) + ## [1.45.0](https://github.com/nftstorage/nft.storage/compare/website-v1.44.0...website-v1.45.0) (2022-04-14) diff --git a/packages/website/package.json b/packages/website/package.json index f4edea6c37..f02891e259 100644 --- a/packages/website/package.json +++ b/packages/website/package.json @@ -1,6 +1,6 @@ { "name": "website", - "version": "1.45.0", + "version": "1.46.0", "description": "nft.storage website", "private": true, "license": "(Apache-2.0 OR MIT)", From baca52a7d992fa5157595010c98f7d313d6a94f2 Mon Sep 17 00:00:00 2001 From: Daniel Ashcraft Date: Mon, 25 Apr 2022 11:00:52 -0500 Subject: [PATCH 03/12] fix: updated nftup route (#1848) --- packages/website/next.config.js | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/website/next.config.js b/packages/website/next.config.js index f64b77729f..113ba23e78 100644 --- a/packages/website/next.config.js +++ b/packages/website/next.config.js @@ -53,6 +53,7 @@ const nextConfig = withBundleAnalyzer({ '/docs/how-to/store-directory': { page: '/docs/how-to/store-directory' }, '/docs/how-to/get-status': { page: '/docs/how-to/get-status' }, '/docs/how-to/ucan': { page: '/docs/how-to/ucan' }, + '/docs/how-to/nftup': { page: '/docs/how-to/nftup' }, '/docs/client': { page: '/docs/client/js' }, '/docs/client/https': { page: '/docs/client/http' }, '/docs/client/generated': { page: '/docs/client/generated' }, From 94c98b08291ca697b29d3257d60eb2d0c11838cc Mon Sep 17 00:00:00 2001 From: Chiu-Hsiang Hsu <2716047+wdv4758h@users.noreply.github.com> Date: Mon, 25 Apr 2022 16:07:43 +0000 Subject: [PATCH 04/12] fix: OpenAPI generated Rust client URL (#1847) --- packages/website/pages/docs/client/generated.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/website/pages/docs/client/generated.md b/packages/website/pages/docs/client/generated.md index 55d37ff956..41d8f8f552 100644 --- a/packages/website/pages/docs/client/generated.md +++ b/packages/website/pages/docs/client/generated.md @@ -13,7 +13,7 @@ Generated clients are available for the following languages: - [PHP](https://github.com/nftstorage/php-client) - [Python](https://github.com/nftstorage/python-client) - [Ruby](https://github.com/nftstorage/ruby-client) -- [Rust](https://github.com/nftstorage/python-client) +- [Rust](https://github.com/nftstorage/rust-client) Please see the repository for each client library for usage details. From 325d45e93557bd4320919002388591602a31c7e7 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 25 Apr 2022 11:19:39 -0500 Subject: [PATCH 05/12] chore(main): release website 1.46.1 (#1849) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- packages/website/CHANGELOG.md | 8 ++++++++ packages/website/package.json | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/packages/website/CHANGELOG.md b/packages/website/CHANGELOG.md index 6696f7d422..c13fc314c5 100644 --- a/packages/website/CHANGELOG.md +++ b/packages/website/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +### [1.46.1](https://github.com/nftstorage/nft.storage/compare/website-v1.46.0...website-v1.46.1) (2022-04-25) + + +### Bug Fixes + +* OpenAPI generated Rust client URL ([#1847](https://github.com/nftstorage/nft.storage/issues/1847)) ([94c98b0](https://github.com/nftstorage/nft.storage/commit/94c98b08291ca697b29d3257d60eb2d0c11838cc)) +* updated nftup route ([#1848](https://github.com/nftstorage/nft.storage/issues/1848)) ([baca52a](https://github.com/nftstorage/nft.storage/commit/baca52a7d992fa5157595010c98f7d313d6a94f2)) + ## [1.46.0](https://github.com/nftstorage/nft.storage/compare/website-v1.45.0...website-v1.46.0) (2022-04-25) diff --git a/packages/website/package.json b/packages/website/package.json index f02891e259..308a18e154 100644 --- a/packages/website/package.json +++ b/packages/website/package.json @@ -1,6 +1,6 @@ { "name": "website", - "version": "1.46.0", + "version": "1.46.1", "description": "nft.storage website", "private": true, "license": "(Apache-2.0 OR MIT)", From 57948d36ae766336847efd4fe2736c3d7eeb1d40 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 25 Apr 2022 10:12:38 -0700 Subject: [PATCH 06/12] chore(main): release api 2.21.0 (#1832) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- packages/api/CHANGELOG.md | 14 ++++++++++++++ packages/api/package.json | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/packages/api/CHANGELOG.md b/packages/api/CHANGELOG.md index 405d47dc37..dfef3e64ae 100644 --- a/packages/api/CHANGELOG.md +++ b/packages/api/CHANGELOG.md @@ -1,5 +1,19 @@ # Changelog +## [2.21.0](https://github.com/nftstorage/nft.storage/compare/api-v2.20.0...api-v2.21.0) (2022-04-21) + + +### Features + +* add HasSuperHotAccess user tag ([#1838](https://github.com/nftstorage/nft.storage/issues/1838)) ([019a505](https://github.com/nftstorage/nft.storage/commit/019a505e8f4bb93a24b8c480646779f5e4b66326)) +* add permissions endpoint ([#1753](https://github.com/nftstorage/nft.storage/issues/1753)) ([2f5b6fb](https://github.com/nftstorage/nft.storage/commit/2f5b6fb2660231e4591f37e29389c41ac7045605)) + + +### Bug Fixes + +* pin magic admin ([#1841](https://github.com/nftstorage/nft.storage/issues/1841)) ([1bec0b0](https://github.com/nftstorage/nft.storage/commit/1bec0b03dcf13f700ceeaf2a0e3cce8f291f0072)) +* remove cluster API URL from /version endpoint ([#1843](https://github.com/nftstorage/nft.storage/issues/1843)) ([cece7ce](https://github.com/nftstorage/nft.storage/commit/cece7ce7773b9c37a18d8c0f33b0ec307474e294)) + ## [2.20.0](https://github.com/nftstorage/nft.storage/compare/api-v2.19.0...api-v2.20.0) (2022-04-12) diff --git a/packages/api/package.json b/packages/api/package.json index 3a4c4fe983..a8163bc8c6 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -1,6 +1,6 @@ { "name": "api", - "version": "2.20.0", + "version": "2.21.0", "description": "NFT Storage API", "private": true, "type": "module", From d7c38cedb97f7c2331d3775db32c0fa4aa890519 Mon Sep 17 00:00:00 2001 From: Johnathon Roach Date: Mon, 25 Apr 2022 16:38:56 -0500 Subject: [PATCH 07/12] feat: update logo text and link to github (#1833) * feat: update logo text and link to github * Add rel and title to link * Refine logo request copy and ux * Improve contrast ratio of link --- packages/website/components/trustedByLogos.js | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/packages/website/components/trustedByLogos.js b/packages/website/components/trustedByLogos.js index d27a3b94a0..1bcc1f0262 100644 --- a/packages/website/components/trustedByLogos.js +++ b/packages/website/components/trustedByLogos.js @@ -34,7 +34,21 @@ export const TrustedBy = ({ logos }) => { /> ))} -

and 20,000+ other users!

+

+ and 30,000+ other users! +
+ Request to add your logo{' '} + + here + + . +

) } From 4fd9089b383576b469f4649f80876531f9a62d49 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 25 Apr 2022 17:57:08 -0500 Subject: [PATCH 08/12] chore(main): release website 1.47.0 (#1850) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- packages/website/CHANGELOG.md | 7 +++++++ packages/website/package.json | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/website/CHANGELOG.md b/packages/website/CHANGELOG.md index c13fc314c5..7fcf8ccafb 100644 --- a/packages/website/CHANGELOG.md +++ b/packages/website/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## [1.47.0](https://github.com/nftstorage/nft.storage/compare/website-v1.46.1...website-v1.47.0) (2022-04-25) + + +### Features + +* update logo text and link to github ([#1833](https://github.com/nftstorage/nft.storage/issues/1833)) ([d7c38ce](https://github.com/nftstorage/nft.storage/commit/d7c38cedb97f7c2331d3775db32c0fa4aa890519)) + ### [1.46.1](https://github.com/nftstorage/nft.storage/compare/website-v1.46.0...website-v1.46.1) (2022-04-25) diff --git a/packages/website/package.json b/packages/website/package.json index 308a18e154..272d03f6bb 100644 --- a/packages/website/package.json +++ b/packages/website/package.json @@ -1,6 +1,6 @@ { "name": "website", - "version": "1.46.1", + "version": "1.47.0", "description": "nft.storage website", "private": true, "license": "(Apache-2.0 OR MIT)", From a3b28e2df4d09fd914a58e14fe362f4ec991279a Mon Sep 17 00:00:00 2001 From: Daniel Ashcraft Date: Mon, 25 Apr 2022 19:44:33 -0500 Subject: [PATCH 09/12] fix: update db-types and add documentation (#1619) * feat: recreate db-types * chore: update readme about db-types * Update packages/api/README.md Co-authored-by: Alan Shaw * regenerated types Co-authored-by: Alan Shaw --- packages/api/README.md | 9 +++++ packages/api/src/utils/db-types.d.ts | 56 ++++++++++++++-------------- 2 files changed, 37 insertions(+), 28 deletions(-) diff --git a/packages/api/README.md b/packages/api/README.md index 2c42c39161..01b21403f2 100644 --- a/packages/api/README.md +++ b/packages/api/README.md @@ -97,6 +97,15 @@ When prompted for a value enter one of the following permission combinations: - `r-` = read only mode - `rw` = read and write (normal operation) +## DB Types + +The postgres rest api can generate automatic type defs based on the table and column +names. To use this, make sure you've spun down your nft dev environment and run +`yarn db-types` from packages/api root directory. This will auto generate the +`packages/api/utils/db-types.d.ts` file. + +Common errors would be "cannot read version of schema", this typically indicates that another service running on localhost:3000 which is the default port and url for the postgres rest api. + ## S3 Setup We use [S3](https://aws.amazon.com/s3/) for backup and disaster recovery. For production deployment an account on AWS is required. diff --git a/packages/api/src/utils/db-types.d.ts b/packages/api/src/utils/db-types.d.ts index 7cdede28c8..4cad90fadc 100644 --- a/packages/api/src/utils/db-types.d.ts +++ b/packages/api/src/utils/db-types.d.ts @@ -568,6 +568,7 @@ export interface paths { files?: parameters['rowFilter.upload.files'] origins?: parameters['rowFilter.upload.origins'] meta?: parameters['rowFilter.upload.meta'] + backup_urls?: parameters['rowFilter.upload.backup_urls'] inserted_at?: parameters['rowFilter.upload.inserted_at'] updated_at?: parameters['rowFilter.upload.updated_at'] deleted_at?: parameters['rowFilter.upload.deleted_at'] @@ -632,6 +633,7 @@ export interface paths { files?: parameters['rowFilter.upload.files'] origins?: parameters['rowFilter.upload.origins'] meta?: parameters['rowFilter.upload.meta'] + backup_urls?: parameters['rowFilter.upload.backup_urls'] inserted_at?: parameters['rowFilter.upload.inserted_at'] updated_at?: parameters['rowFilter.upload.updated_at'] deleted_at?: parameters['rowFilter.upload.deleted_at'] @@ -660,6 +662,7 @@ export interface paths { files?: parameters['rowFilter.upload.files'] origins?: parameters['rowFilter.upload.origins'] meta?: parameters['rowFilter.upload.meta'] + backup_urls?: parameters['rowFilter.upload.backup_urls'] inserted_at?: parameters['rowFilter.upload.inserted_at'] updated_at?: parameters['rowFilter.upload.updated_at'] deleted_at?: parameters['rowFilter.upload.deleted_at'] @@ -901,14 +904,11 @@ export interface paths { } } } - '/rpc/json_arr_to_text_arr': { + '/rpc/pgrst_watch': { post: { parameters: { body: { - args: { - /** Format: json */ - _json: string - } + args: { [key: string]: unknown } } header: { /** Preference */ @@ -921,13 +921,13 @@ export interface paths { } } } - '/rpc/create_upload': { + '/rpc/find_deals_by_content_cids': { post: { parameters: { body: { args: { - /** Format: json */ - data: string + /** Format: text[] */ + cids: string } } header: { @@ -941,13 +941,13 @@ export interface paths { } } } - '/rpc/find_deals_by_content_cids': { + '/rpc/json_arr_to_text_arr': { post: { parameters: { body: { args: { - /** Format: text[] */ - cids: string + /** Format: json */ + _json: string } } header: { @@ -961,11 +961,14 @@ export interface paths { } } } - '/rpc/pgrst_watch': { + '/rpc/create_upload': { post: { parameters: { body: { - args: { [key: string]: unknown } + args: { + /** Format: json */ + data: string + } } header: { /** Preference */ @@ -988,12 +991,8 @@ export interface definitions { email?: string /** Format: text */ token?: string - /** - * Format: bigint - * @description Note: - * This is a Primary Key. - */ - token_id?: number + /** Format: text */ + token_id?: string /** Format: timestamp with time zone */ deleted_at?: string /** Format: timestamp with time zone */ @@ -1165,11 +1164,8 @@ export interface definitions { origins?: string /** Format: jsonb */ meta?: string - /** - * Format: text[] - * @description Note: - */ - backup_urls: string[] + /** Format: ARRAY */ + backup_urls?: unknown[] /** * Format: timestamp with time zone * @default timezone('utc'::text, now()) @@ -1231,12 +1227,14 @@ export interface definitions { */ user_id: number /** Format: public.user_tag_type */ - tag: 'HasAccountRestriction' | 'HasPsaAccess' | 'StorageLimitBytes' + tag: + | 'HasAccountRestriction' + | 'HasPsaAccess' + | 'HasSuperHotAccess' + | 'StorageLimitBytes' /** Format: text */ value: string /** Format: text */ - user_tag_value_type: string - /** Format: text */ reason: string /** * Format: timestamp with time zone @@ -1280,7 +1278,7 @@ export interface parameters { 'rowFilter.admin_search.email': string /** Format: text */ 'rowFilter.admin_search.token': string - /** Format: bigint */ + /** Format: text */ 'rowFilter.admin_search.token_id': string /** Format: timestamp with time zone */ 'rowFilter.admin_search.deleted_at': string @@ -1378,6 +1376,8 @@ export interface parameters { 'rowFilter.upload.origins': string /** Format: jsonb */ 'rowFilter.upload.meta': string + /** Format: ARRAY */ + 'rowFilter.upload.backup_urls': string /** Format: timestamp with time zone */ 'rowFilter.upload.inserted_at': string /** Format: timestamp with time zone */ From 97b680de080a4b34df55177f2997784100af864d Mon Sep 17 00:00:00 2001 From: Chris Anderson Date: Tue, 26 Apr 2022 10:45:03 -0700 Subject: [PATCH 10/12] fix:blog post type thanks @olizilla for spotting this --- packages/website/posts/2022-02-15-base-url-apis.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/website/posts/2022-02-15-base-url-apis.mdx b/packages/website/posts/2022-02-15-base-url-apis.mdx index 949808b5a4..8f9035f031 100644 --- a/packages/website/posts/2022-02-15-base-url-apis.mdx +++ b/packages/website/posts/2022-02-15-base-url-apis.mdx @@ -15,7 +15,7 @@ Sharing a link to a directory of data is a basic pattern so we shouldn’t be su It's little affordances like this which can make a big difference in the industry. If the easiest way to accept data from users is via a base URL, that is a thumb on the scale for user's data ownership, interoperability and reuse. Sites that accept a URL of uploads benefit from accepting similar data formats as competitive sites. By reusing the upload format, developers not only save time, but can utilize open source data handling libraries, etc. All this means developers can spend more time working on features that distinguish their app. -All this has been possible for years. What’s new today is that with content addressable systems like IPFS you don’t have trust a third party, or control a data repository in the cloud, to upload verifiable data to the network and share the link with service providers. IPFS URLs make up a significant fraction of the assets and metadata in today’s NFTs, as the market sees the value in the verifiable storage that CIDs provide. Content-based identifiers are locked to their content, so any on-chain references can be cryptographically trusted. This makes a shared upload URL as good as sending the files directly. +All this has been possible for years. What’s new today is that with content addressable systems like IPFS you don’t have to trust a third party, or control a data repository in the cloud, to upload verifiable data to the network and share the link with service providers. IPFS URLs make up a significant fraction of the assets and metadata in today’s NFTs, as the market sees the value in the verifiable storage that CIDs provide. Content-based identifiers are locked to their content, so any on-chain references can be cryptographically trusted. This makes a shared upload URL as good as sending the files directly. Even without cryptographic CIDs, a URL to a directory full of content is a valuable baseline abstraction. Users can easily arrange their files in the format expected of the service, whether that is for a static HTML app deploy, or a 10k drop. This separates the value from the toolchain, so even beginners can produce the output format, and begin contributing even while they are learning. This shares some of the accesiblity benefits of “view source,” lowering the barrier to entry for new developers and users to understand the system. From fe2ce9c3b084687104ce51d1cd972b0b957c9cea Mon Sep 17 00:00:00 2001 From: Daniel Ashcraft Date: Wed, 27 Apr 2022 11:00:22 -0500 Subject: [PATCH 11/12] feat: cargo schema migration (#1856) * feat: recreate db-types * Add import sql statement for migration * cleaning up files * Updated migration to only import tables that are new --- .../api/db/migrations/002-import-metrics-cargo-schema.sql | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 packages/api/db/migrations/002-import-metrics-cargo-schema.sql diff --git a/packages/api/db/migrations/002-import-metrics-cargo-schema.sql b/packages/api/db/migrations/002-import-metrics-cargo-schema.sql new file mode 100644 index 0000000000..c3565cb9f0 --- /dev/null +++ b/packages/api/db/migrations/002-import-metrics-cargo-schema.sql @@ -0,0 +1,5 @@ +-- Import dag cargo schema +IMPORT FOREIGN SCHEMA cargo + LIMIT TO (metrics, metrics_log) + FROM SERVER dag_cargo_server + INTO cargo; \ No newline at end of file From 72480d6a9719308bf4a8a81c34764a51f5f58cfa Mon Sep 17 00:00:00 2001 From: Daniel Ashcraft Date: Wed, 27 Apr 2022 11:31:29 -0500 Subject: [PATCH 12/12] =?UTF-8?q?feat:=20add=20cargo=20metric=20statements?= =?UTF-8?q?=20and=20dummy=20data=20inserts=20for=20testing=20=E2=80=A6=20(?= =?UTF-8?q?#1835)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: add cargo metric statements and dummy data inserts for testing purposes * chore: update stats to use new dag cargo metrics vs hard coded * feat: move multiple asyncs to promise.all * Updated to add a simple test * Add more demo data and improve test with hard coded demo data. * Updated website to reflect growth rate not total deals * chore: remove unnecessary yarn lock changes * Updated per Alan's feedback and comments * fix error issue * Updated types and db-client to respect types * typed the getStats output as well * Update packages/api/db/cargo.testing.sql Co-authored-by: Alan Shaw * Update packages/api/src/utils/db-client.js Co-authored-by: Alan Shaw * Update the variable names to match camelcase request Co-authored-by: Alan Shaw --- packages/api/db/cargo.sql | 2 +- packages/api/db/cargo.testing.sql | 58 ++++++++++++++++- packages/api/docker/docker-compose.yml | 2 +- packages/api/src/routes/metrics.js | 3 +- packages/api/src/utils/db-client-types.ts | 12 ++++ packages/api/src/utils/db-client.js | 78 +++++++++++++++++++---- packages/api/test/stats-get.spec.js | 35 ++++++++++ packages/cron/src/jobs/metrics.js | 32 ---------- packages/website/lib/statsUtils.js | 7 +- packages/website/pages/stats.js | 10 +-- yarn.lock | 7 +- 11 files changed, 189 insertions(+), 57 deletions(-) create mode 100644 packages/api/test/stats-get.spec.js diff --git a/packages/api/db/cargo.sql b/packages/api/db/cargo.sql index a424b522b8..e8b8a42072 100644 --- a/packages/api/db/cargo.sql +++ b/packages/api/db/cargo.sql @@ -2,6 +2,6 @@ CREATE SCHEMA IF NOT EXISTS cargo; -- Import dag cargo schema IMPORT FOREIGN SCHEMA cargo - LIMIT TO (aggregate_entries, aggregates, deals, dags) + LIMIT TO (aggregate_entries, aggregates, deals, dags, metrics, metrics_log) FROM SERVER dag_cargo_server INTO cargo; diff --git a/packages/api/db/cargo.testing.sql b/packages/api/db/cargo.testing.sql index 112f018506..4ae6f1a29b 100644 --- a/packages/api/db/cargo.testing.sql +++ b/packages/api/db/cargo.testing.sql @@ -35,7 +35,29 @@ CREATE TABLE IF NOT EXISTS cargo.deals ( entry_last_updated TIMESTAMP WITH TIME ZONE NOT NULL ); --- Test data +CREATE TABLE IF NOT EXISTS cargo.metrics ( + name TEXT NOT NULL, + dimensions TEXT[], + description TEXT NOT NULL, + value BIGINT, + collected_at TIMESTAMP WITH TIME ZONE, + collection_took_seconds NUMERIC NOT NULL +); + +CREATE TABLE IF NOT EXISTS cargo.metrics_log ( + name TEXT NOT NULL, + dimensions TEXT[], + value BIGINT, + collected_at TIMESTAMP WITH TIME ZONE +); + +-- Test data for cargo tables + +INSERT INTO cargo.metrics_log (name, dimensions, value, collected_at) VALUES + ('dagcargo_project_bytes_in_active_deals', '{{project,staging.nft.storage}}', 167859554927623, '2022-04-01 13:41:08.479404+00'); + +INSERT INTO cargo.metrics_log (name, dimensions, value, collected_at) VALUES + ('dagcargo_project_bytes_in_active_deals', '{{project,nft.storage}}', 169334115720738, '2022-03-01 16:33:28.505513+00'); INSERT INTO cargo.aggregate_entries ("aggregate_cid", "cid_v1", "datamodel_selector") VALUES ('bafybeiek5gau46j4dxoyty27qtirb3iuoq7aax4l3xt25mfk2igyt35bme', 'bafybeiaj5yqocsg5cxsuhtvclnh4ulmrgsmnfbhbrfxrc3u2kkh35mts4e', 'Links/19/Hash/Links/46/Hash/Links/0/Hash'); @@ -46,6 +68,38 @@ INSERT INTO cargo.aggregates ("aggregate_cid", "piece_cid", "sha256hex", "export INSERT INTO cargo.deals ("deal_id", "aggregate_cid", "client", "provider", "status", "start_epoch", "end_epoch", "entry_created", "entry_last_updated", "status_meta", "start_time", "sector_start_epoch", "sector_start_time", "end_time") VALUES (2424132, 'bafybeiek5gau46j4dxoyty27qtirb3iuoq7aax4l3xt25mfk2igyt35bme', 'f144zep4gitj73rrujd3jw6iprljicx6vl4wbeavi', 'f0678914', 'active', 1102102, 2570902, '2021-09-09 16:30:52.252233+00', '2021-09-10 00:45:50.408956+00', 'containing sector active as of 2021-09-10 00:36:30 at epoch 1097593', '2021-09-11 14:11:00+00', 1097593, '2021-09-10 00:36:30+00', '2023-02-03 14:11:00+00'); +INSERT INTO cargo.metrics (name, dimensions, description, value, collected_at, collection_took_seconds) VALUES +('dagcargo_project_items_in_active_deals', '{{project,staging.web3.storage}}', 'Count of aggregated items with at least one active deal per project', 1438, '2022-04-14 23:56:46.803497+00', 405.292); + +INSERT INTO cargo.metrics (name, dimensions, description, value, collected_at, collection_took_seconds) VALUES +('dagcargo_project_items_in_active_deals', '{{project,nft.storage}}', 'Count of aggregated items with at least one active deal per project', 56426047, '2022-04-14 23:56:46.806892+00', 405.292); + +INSERT INTO cargo.metrics (name, dimensions, description, value, collected_at, collection_took_seconds) VALUES +('dagcargo_project_bytes_in_active_deals', '{{project,nft.storage}}', 'Amount of per-DAG-deduplicated bytes with at least one active deal per project', 169389985753391, '2022-04-14 23:51:45.76915+00', 104.256); + +INSERT INTO cargo.metrics (name, dimensions, description, value, collected_at, collection_took_seconds) VALUES +('dagcargo_project_bytes_in_active_deals', '{{project,staging.web3.storage}}', 'Amount of per-DAG-deduplicated bytes with at least one active deal per project', 133753809372, '2022-04-14 23:51:45.76712+00', 104.256); + +INSERT INTO cargo.metrics (name, dimensions, description, value, collected_at, collection_took_seconds) VALUES +('dagcargo_project_bytes_in_active_deals', '{{project,web3.storage}}', 'Amount of per-DAG-deduplicated bytes with at least one active deal per project', 181663391277785, '2022-04-14 23:51:45.768323+00', 104.256); + +INSERT INTO public.metric (name, value, updated_at) + VALUES ('uploads_past_7_total', 2011366, TIMEZONE('utc', NOW())); + +INSERT INTO public.metric (name, value, updated_at) + VALUES ('uploads_nft_total', 685866, TIMEZONE('utc', NOW())); + +INSERT INTO public.metric (name, value, updated_at) + VALUES ('uploads_remote_total', 11077834, TIMEZONE('utc', NOW())); + +INSERT INTO public.metric (name, value, updated_at) + VALUES ('uploads_car_total', 17711308, TIMEZONE('utc', NOW())); + +INSERT INTO public.metric (name, value, updated_at) + VALUES ('uploads_multipart_total', 1456388, TIMEZONE('utc', NOW())); + +INSERT INTO public.metric (name, value, updated_at) + VALUES ('uploads_blob_total', 12420729, TIMEZONE('utc', NOW())); INSERT INTO public."user" (magic_link_id, github_id, name, email, public_address) VALUES ('did:ethr:0x65007A739ab7AC5c537161249b81250E49e2853C', 'github|000000', 'mock user', 'test@gmail.com', '0x65007A739ab7AC5c537161249b81250E49e2853C'); -INSERT INTO public.auth_key (name, secret, user_id) VALUES ('main', 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJkaWQ6ZXRocjoweDY1MDA3QTczOWFiN0FDNWM1MzcxNjEyNDliODEyNTBFNDllMjg1M0MiLCJpc3MiOiJuZnQtc3RvcmFnZSIsImlhdCI6MTYzOTc1NDczNjYzOCwibmFtZSI6Im1haW4ifQ.wKwJIRXXHsgwVp8mOQp6r3_F4Lz5lnoAkgVP8wqwA_Y', 1); \ No newline at end of file +INSERT INTO public.auth_key (name, secret, user_id) VALUES ('main', 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJkaWQ6ZXRocjoweDY1MDA3QTczOWFiN0FDNWM1MzcxNjEyNDliODEyNTBFNDllMjg1M0MiLCJpc3MiOiJuZnQtc3RvcmFnZSIsImlhdCI6MTYzOTc1NDczNjYzOCwibmFtZSI6Im1haW4ifQ.wKwJIRXXHsgwVp8mOQp6r3_F4Lz5lnoAkgVP8wqwA_Y', 1); diff --git a/packages/api/docker/docker-compose.yml b/packages/api/docker/docker-compose.yml index e03d3b177e..d534557238 100644 --- a/packages/api/docker/docker-compose.yml +++ b/packages/api/docker/docker-compose.yml @@ -9,7 +9,7 @@ services: - 3000:3000/tcp environment: PGRST_DB_URI: postgres://postgres:postgres@db:5432/postgres - PGRST_DB_SCHEMA: public + PGRST_DB_SCHEMAS: public,cargo PGRST_DB_ANON_ROLE: postgres PGRST_JWT_SECRET: super-secret-jwt-token-with-at-least-32-characters-long db: diff --git a/packages/api/src/routes/metrics.js b/packages/api/src/routes/metrics.js index 65a88bccc2..66518f99ad 100644 --- a/packages/api/src/routes/metrics.js +++ b/packages/api/src/routes/metrics.js @@ -11,8 +11,7 @@ export async function metrics(_, { db }) { /** @type {import('../bindings').Handler} */ export async function getStats(_, { db }) { - let stats = await db.getStats() - + const stats = await db.getStats() return new JSONResponse( { ok: true, diff --git a/packages/api/src/utils/db-client-types.ts b/packages/api/src/utils/db-client-types.ts index daf0ae4e63..e8543cee4b 100644 --- a/packages/api/src/utils/db-client-types.ts +++ b/packages/api/src/utils/db-client-types.ts @@ -89,3 +89,15 @@ export interface ListUploadsOptions { limit?: number meta?: unknown } + +export type StatsPayload = { + [key: string]: number + deals_size_total: number + deals_size_total_prev: number + uploads_blob_total: number + uploads_car_total: number + uploads_multipart_total: number + uploads_nft_total: number + uploads_past_7_total: number + uploads_remote_total: number +} diff --git a/packages/api/src/utils/db-client.js b/packages/api/src/utils/db-client.js index 9df8e72f78..d24f512c87 100644 --- a/packages/api/src/utils/db-client.js +++ b/packages/api/src/utils/db-client.js @@ -30,6 +30,14 @@ export class DBClient { apikey: `${token}`, }, }) + + this.cargoClient = new PostgrestClient(url, { + headers: { + Authorization: `Bearer ${token}`, + apikey: `${token}`, + }, + schema: 'cargo', + }) } /** @@ -538,14 +546,20 @@ export class DBClient { return data[0].value } + /** + * Get stats for uploads and cargo deals + * + * @returns {Promise} + */ async getStats() { /** @type {PostgrestQueryBuilder} */ - const query = this.client.from('metric') - const { data, error } = await query + const nonCargoMetricQuery = this.client.from('metric') + const metricsQuery = this.cargoClient.from('metrics') + const metricsLogQuery = this.cargoClient.from('metrics_log') + + const primaryMetricQuery = nonCargoMetricQuery .select('name, value') .in('name', [ - 'deals_total', - 'deals_size_total', 'uploads_past_7_total', 'uploads_blob_total', 'uploads_car_total', @@ -554,19 +568,57 @@ export class DBClient { 'uploads_multipart_total', ]) - if (error) { - throw new DBError(error) + const dagByteSizeQuery = metricsQuery + .select('name, dimensions, value') + .match({ + name: 'dagcargo_project_bytes_in_active_deals', + dimensions: '{{project,nft.storage}}', + }) + .single() + + const weekAgo = new Date() + weekAgo.setDate(weekAgo.getDate() - 7) + + const dagByteSizeHistory = metricsLogQuery + .select('name, dimensions, value') + .match({ + name: 'dagcargo_project_bytes_in_active_deals', + dimensions: '{{project,nft.storage}}', + }) + .lte('collected_at', weekAgo.toISOString()) + .order('collected_at', { ascending: false }) + .range(0, 1) + .single() + + const [primaryRes, dagSizeRes, dagSizeHistRes] = await Promise.all([ + primaryMetricQuery, + dagByteSizeQuery, + dagByteSizeHistory, + ]) + + if (primaryRes.error || dagSizeHistRes.error || dagSizeRes.error) { + // this allows us to avoid changing the construtor of db error to allow null + const err = Object.assign( + {}, + primaryRes.error || dagSizeHistRes.error || dagSizeRes.error + ) + throw new DBError(err) } - if (!data || !data.length) { - return undefined + /** @type {import('./db-client-types').StatsPayload} */ + const stats = {} + + // Simple splatting of the metrics from first query + if (primaryRes.data && primaryRes.data.length) { + for (const metric of primaryRes.data) { + stats[metric.name] = metric.value + } } - return data.reduce((obj, curr) => { - // @ts-ignore - obj[curr.name] = curr.value - return obj - }, {}) + stats.deals_size_total = dagSizeRes.data.value + stats.deals_size_total_prev = dagSizeHistRes.data.value + + return stats } } diff --git a/packages/api/test/stats-get.spec.js b/packages/api/test/stats-get.spec.js new file mode 100644 index 0000000000..12ff961ed8 --- /dev/null +++ b/packages/api/test/stats-get.spec.js @@ -0,0 +1,35 @@ +import assert from 'assert' +import { createClientWithUser, DBTestClient } from './scripts/helpers.js' +import { fixtures } from './scripts/fixtures.js' + +describe('Get Stats', () => { + /** @type{DBTestClient} */ + let client + + before(async () => { + client = await createClientWithUser() + }) + + it('Should return proper response for /stats route, based on seeded demoData', async () => { + const demoData = { + deals_size_total: 169389985753391, + deals_size_total_prev: 169334115720738, + uploads_blob_total: 12420729, + uploads_car_total: 17711308, + uploads_multipart_total: 1456388, + uploads_nft_total: 685866, + uploads_past_7_total: 2011366, + uploads_remote_total: 11077834, + } + const res = await fetch('/stats', { + headers: { Authorization: `Bearer ${client.token}` }, + }) + + const { ok, data } = await res.json() + + assert.equal(ok, true) + + // this is brittle, but it's simple + assert.deepStrictEqual(data, demoData) + }) +}) diff --git a/packages/cron/src/jobs/metrics.js b/packages/cron/src/jobs/metrics.js index 8acb4ac831..edefa5e30e 100644 --- a/packages/cron/src/jobs/metrics.js +++ b/packages/cron/src/jobs/metrics.js @@ -21,11 +21,6 @@ const COUNT_UPLOADS = 'SELECT COUNT(*) AS total FROM upload WHERE type = $1' const UPLOADS_PAST_7_TOTAL = 'SELECT COUNT(*) FROM upload WHERE inserted_at > CURRENT_DATE - 7' -const DEALS_TOTAL = 'SELECT COUNT(*) from cargo.deals' - -const DEALS_SIZE_TOTAL = - 'SELECT SUM(export_size) as deals_size_total from cargo.aggregates' - const COUNT_PINS = 'SELECT COUNT(*) AS total FROM pin WHERE service = $1 AND status = $2' @@ -58,10 +53,6 @@ export async function updateMetrics({ roPg, rwPg }) { withTimeLog('updateTotalUploadPast7', () => updateTotalUploadPast7(roPg, rwPg) ), - withTimeLog('updateTotalDeals', () => updateTotalDeals(roPg, rwPg)), - withTimeLog('updateTotalDealsSize', () => - updateTotalDealsSize(roPg, rwPg) - ), ...PIN_SERVICES.map((svc) => PIN_STATUSES.map((s) => withTimeLog(`updatePinsCount[${svc}][${s}]`, () => @@ -114,29 +105,6 @@ async function updateTotalUploadPast7(roPg, rwPg) { await rwPg.query(UPDATE_METRIC, ['uploads_past_7_total', rows[0].count]) } -/** - * @param {Client} roPg - * @param {Client} rwPg - */ -async function updateTotalDeals(roPg, rwPg) { - const { rows } = await roPg.query(DEALS_TOTAL) - if (!rows.length) throw new Error(`no rows returned counting total deals`) - await rwPg.query(UPDATE_METRIC, [`deals_total`, rows[0].count]) -} - -/** - * @param {Client} roPg - * @param {Client} rwPg - */ -async function updateTotalDealsSize(roPg, rwPg) { - const { rows } = await roPg.query(DEALS_SIZE_TOTAL) - if (!rows.length) throw new Error(`no rows returned counting total deal size`) - await rwPg.query(UPDATE_METRIC, [ - `deals_size_total`, - rows[0].deals_size_total, - ]) -} - /** * @param {Client} roPg * @param {Client} rwPg diff --git a/packages/website/lib/statsUtils.js b/packages/website/lib/statsUtils.js index 5030110df1..dbda58e83f 100644 --- a/packages/website/lib/statsUtils.js +++ b/packages/website/lib/statsUtils.js @@ -56,5 +56,10 @@ export function calculateStats(stats) { growthRate = calcuateGrowthRate(totalUploads, previousTotal) } - return { ...stats, totalUploads, growthRate } + const dealsSizeGrowthRate = calcuateGrowthRate( + stats.deals_size_total, + stats.deals_size_total_prev + ) + + return { ...stats, totalUploads, growthRate, dealsSizeGrowthRate } } diff --git a/packages/website/pages/stats.js b/packages/website/pages/stats.js index aef9f4251a..6c9165b569 100644 --- a/packages/website/pages/stats.js +++ b/packages/website/pages/stats.js @@ -67,10 +67,10 @@ export default function Stats({ logos }) { ok: true, data: { deals_size_total: 249523372029443, + deals_size_total_prev: 249523372020000, uploads_past_7_total: 2011366, uploads_nft_total: 685866, uploads_remote_total: 11077834, - deals_total: 34959, uploads_car_total: 17711308, uploads_multipart_total: 1456388, uploads_blob_total: 12420729, @@ -164,12 +164,14 @@ export default function Stats({ logos }) {

0 ? 'stat-green' : 'stat-red' + stats.dealsSizeGrowthRate > 0 + ? 'stat-green stat-green-plus' + : 'stat-red' }`} > - {stats.deals_total?.toLocaleString() || 0} + {stats.dealsSizeGrowthRate || 0}%

-

[Total deals]

+

[Week over week change]

diff --git a/yarn.lock b/yarn.lock index 6344487aaf..4da5f3f526 100644 --- a/yarn.lock +++ b/yarn.lock @@ -13665,7 +13665,12 @@ typedoc@^0.22.14: minimatch "^5.0.1" shiki "^0.10.1" -typescript@4.4.4, typescript@4.5.3: +typescript@4.4.4: + version "4.4.4" + resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.4.4.tgz#2cd01a1a1f160704d3101fd5a58ff0f9fcb8030c" + integrity sha512-DqGhF5IKoBl8WNf8C1gu8q0xZSInh9j1kJJMqT3a94w1JzVaBU4EXOSMrz9yDqMT0xt3selp83fuFMQ0uzv6qA== + +typescript@4.5.3: version "4.5.3" resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.5.3.tgz#afaa858e68c7103317d89eb90c5d8906268d353c" integrity sha512-eVYaEHALSt+s9LbvgEv4Ef+Tdq7hBiIZgii12xXJnukryt3pMgJf6aKhoCZ3FWQsu6sydEnkg11fYXLzhLBjeQ==