From bacaeaea4d4610672b48c5d422100fccf78918ca Mon Sep 17 00:00:00 2001 From: Joe Spencer Date: Mon, 12 Sep 2022 03:25:49 -0700 Subject: [PATCH] feat: adding google malware detection (#36) Co-authored-by: Vasco Santos --- .github/workflows/cid-verifier.yml | 107 ++++++++++++ package.json | 6 +- packages/cid-verifier/ava.config.js | 6 + packages/cid-verifier/package.json | 47 ++++++ packages/cid-verifier/scripts/build.js | 60 +++++++ packages/cid-verifier/scripts/cli.js | 16 ++ packages/cid-verifier/src/bindings.d.ts | 45 +++++ packages/cid-verifier/src/cors.js | 34 ++++ packages/cid-verifier/src/env.js | 68 ++++++++ packages/cid-verifier/src/error-handler.js | 22 +++ packages/cid-verifier/src/errors.js | 14 ++ packages/cid-verifier/src/index.js | 52 ++++++ packages/cid-verifier/src/utils/denylist.js | 34 ++++ packages/cid-verifier/src/verification.js | 157 ++++++++++++++++++ packages/cid-verifier/src/version.js | 15 ++ .../v1eap1@evaluateUri/post_.js | 67 ++++++++ packages/cid-verifier/test/mocks/package.json | 5 + packages/cid-verifier/test/utils/miniflare.js | 25 +++ packages/cid-verifier/test/utils/setup.js | 13 ++ .../cid-verifier/test/verification.spec.js | 141 ++++++++++++++++ packages/cid-verifier/test/version.spec.js | 29 ++++ packages/cid-verifier/tsconfig.json | 11 ++ packages/cid-verifier/wrangler.toml | 61 +++++++ packages/edge-gateway/src/bindings.d.ts | 2 + packages/edge-gateway/src/errors.js | 5 +- packages/edge-gateway/src/gateway.js | 90 +++++----- packages/edge-gateway/src/utils/deny-list.js | 13 -- .../edge-gateway/test/cid-verifier.spec.js | 91 ++++++++++ packages/edge-gateway/test/denylist.spec.js | 84 ---------- .../test/fixtures/website/malware.txt | 1 + .../test/fixtures/website/sample.html | 2 + .../mocks/cf-ipfs.com/get_ipfs#@cid#path.js | 13 +- .../test/mocks/cf-ipfs.com/get_ipfs#@cid.js | 14 ++ .../test/mocks/ipfs.io/get_ipfs#@cid#path.js | 13 +- .../test/mocks/ipfs.io/get_ipfs#@cid.js | 14 ++ packages/edge-gateway/test/utils/fixtures.js | 21 ++- packages/edge-gateway/test/utils/miniflare.js | 8 +- .../test/utils/scripts/cid-verifier.js | 41 +++++ packages/edge-gateway/wrangler.toml | 20 ++- pnpm-lock.yaml | 86 +++++++--- 40 files changed, 1371 insertions(+), 182 deletions(-) create mode 100644 .github/workflows/cid-verifier.yml create mode 100644 packages/cid-verifier/ava.config.js create mode 100644 packages/cid-verifier/package.json create mode 100644 packages/cid-verifier/scripts/build.js create mode 100644 packages/cid-verifier/scripts/cli.js create mode 100644 packages/cid-verifier/src/bindings.d.ts create mode 100644 packages/cid-verifier/src/cors.js create mode 100644 packages/cid-verifier/src/env.js create mode 100644 packages/cid-verifier/src/error-handler.js create mode 100644 packages/cid-verifier/src/errors.js create mode 100644 packages/cid-verifier/src/index.js create mode 100644 packages/cid-verifier/src/utils/denylist.js create mode 100644 packages/cid-verifier/src/verification.js create mode 100644 packages/cid-verifier/src/version.js create mode 100644 packages/cid-verifier/test/mocks/google-cloud.io/v1eap1@evaluateUri/post_.js create mode 100644 packages/cid-verifier/test/mocks/package.json create mode 100644 packages/cid-verifier/test/utils/miniflare.js create mode 100644 packages/cid-verifier/test/utils/setup.js create mode 100644 packages/cid-verifier/test/verification.spec.js create mode 100644 packages/cid-verifier/test/version.spec.js create mode 100644 packages/cid-verifier/tsconfig.json create mode 100644 packages/cid-verifier/wrangler.toml delete mode 100644 packages/edge-gateway/src/utils/deny-list.js create mode 100644 packages/edge-gateway/test/cid-verifier.spec.js delete mode 100644 packages/edge-gateway/test/denylist.spec.js create mode 100644 packages/edge-gateway/test/fixtures/website/malware.txt create mode 100644 packages/edge-gateway/test/fixtures/website/sample.html create mode 100644 packages/edge-gateway/test/utils/scripts/cid-verifier.js diff --git a/.github/workflows/cid-verifier.yml b/.github/workflows/cid-verifier.yml new file mode 100644 index 0000000..5211ed3 --- /dev/null +++ b/.github/workflows/cid-verifier.yml @@ -0,0 +1,107 @@ +name: cid-verifier +on: + push: + branches: + - main + paths: + - 'packages/cid-verifier/**' + - '.github/workflows/cid-verifier.yml' + - 'pnpm-lock.yaml' + pull_request: + paths: + - 'packages/cid-verifier/**' + - '.github/workflows/cid-verifier.yml' + - 'pnpm-lock.yaml' +jobs: + check: + runs-on: ubuntu-latest + name: Test + steps: + - uses: actions/checkout@v2 + - uses: pnpm/action-setup@v2.0.1 + with: + version: 6.32.x + - uses: actions/setup-node@v2 + - run: pnpm install + - run: pnpm build:cid-verifier + - name: Lint + run: pnpm lint + test: + runs-on: ubuntu-latest + name: Test + steps: + - uses: actions/checkout@v2 + - uses: pnpm/action-setup@v2.0.1 + with: + version: 6.32.x + - uses: actions/setup-node@v2 + - run: pnpm install + - run: pnpm test:cid-verifier + deploy-staging: + name: Deploy Staging + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + runs-on: ubuntu-latest + needs: test + steps: + - uses: actions/checkout@v2 + - uses: pnpm/action-setup@v2.0.1 + with: + version: 6.32.x + - uses: actions/setup-node@v2 + with: + cache: 'pnpm' + - run: pnpm install + - name: Publish app + uses: cloudflare/wrangler-action@2.0.0 + env: + ENV: 'staging' # inform the build process what the env is + SENTRY_TOKEN: ${{secrets.SENTRY_TOKEN}} + SENTRY_UPLOAD: ${{ secrets.SENTRY_UPLOAD }} + with: + apiToken: ${{secrets.CF_GATEWAY_TOKEN }} + workingDirectory: 'packages/cid-verifier' + environment: 'staging' + changelog: + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + name: Release + runs-on: ubuntu-latest + outputs: + releases_created: ${{ steps.tag-release.outputs.releases_created }} + steps: + - uses: GoogleCloudPlatform/release-please-action@v3 + id: tag-release + with: + path: packages/cid-verifier + token: ${{ secrets.GITHUB_TOKEN }} + release-type: node + monorepo-tags: true + package-name: cid-verifier + release: + if: github.event_name == 'push' && github.ref == 'refs/heads/main' && needs.changelog.outputs.releases_created + name: Release + runs-on: ubuntu-latest + needs: + - check + - test + - changelog + steps: + - uses: actions/checkout@v2 + - uses: pnpm/action-setup@v2.0.1 + with: + version: 6.32.x + - uses: actions/setup-node@v2 + with: + cache: 'pnpm' + node-version: 16 + registry-url: 'https://registry.npmjs.org' + - run: pnpm install + - name: Deploy + uses: cloudflare/wrangler-action@2.0.0 + env: + ENV: 'production' # inform the build process what the env is + SENTRY_TOKEN: ${{ secrets.SENTRY_TOKEN }} + SENTRY_UPLOAD: ${{ secrets.SENTRY_UPLOAD }} + with: + apiToken: ${{ secrets.CF_GATEWAY_TOKEN }} + workingDirectory: 'packages/cid-verifier' + environment: 'production' diff --git a/package.json b/package.json index 4c856e0..26619ee 100644 --- a/package.json +++ b/package.json @@ -8,17 +8,21 @@ "license": "Apache-2.0 OR MIT", "scripts": { "lint": "run-s lint:no-fix-*", + "lint:no-fix-cid-verifier": "pnpm --filter cid-verifier lint", "lint:no-fix-edge-gateway": "pnpm --filter edge-gateway lint", "lint:no-fix-ipfs-gateway-race": "pnpm --filter ipfs-gateway-race lint", "lint:fix": "run-s lint:fix-*", + "lint:fix-cid-verifier": "pnpm --filter cid-verifier lint --fix", "lint:fix-edge-gateway": "pnpm --filter edge-gateway lint --fix", "lint:fix-ipfs-gateway-race": "pnpm --filter ipfs-gateway-race lint --fix", "build": "run-s build:*", + "build:cid-verifier": "pnpm --filter cid-verifier build", "build:edge-gateway": "pnpm --filter edge-gateway build", "build:ipfs-gateway-race": "pnpm --filter ipfs-gateway-race build", "test": "run-s test:*", - "test:ipfs-gateway-race": "pnpm --filter ipfs-gateway-race test", + "test:cid-verifier": "pnpm --filter cid-verifier test", "test:edge-gateway": "pnpm --filter edge-gateway test", + "test:ipfs-gateway-race": "pnpm --filter ipfs-gateway-race test", "clean": "rm -rf node_modules pnpm-lock.yml packages/*/{pnpm-lock.yml,.next,out,coverage,.nyc_output,worker,dist,node_modules}" }, "simple-git-hooks": { diff --git a/packages/cid-verifier/ava.config.js b/packages/cid-verifier/ava.config.js new file mode 100644 index 0000000..f620f52 --- /dev/null +++ b/packages/cid-verifier/ava.config.js @@ -0,0 +1,6 @@ +export default { + files: ['test/*.spec.js'], + timeout: '5m', + concurrency: 1, + nodeArguments: ['--experimental-vm-modules'] +} diff --git a/packages/cid-verifier/package.json b/packages/cid-verifier/package.json new file mode 100644 index 0000000..eaa1ff3 --- /dev/null +++ b/packages/cid-verifier/package.json @@ -0,0 +1,47 @@ +{ + "name": "cid-verifier", + "version": "1.0.0", + "description": "Verify that a CID's content is safe for the web", + "private": true, + "type": "module", + "main": "./dist/worker.js", + "scripts": { + "lint": "standard", + "build": "tsc && node scripts/cli.js build", + "dev": "miniflare dist/worker.js --watch --debug -m", + "test": "npm run test:setup && npm-run-all -p -r mock:google-cloud.io test:worker", + "test:worker": "ava --verbose test/*.spec.js", + "test:setup": "npm run build", + "mock:google-cloud.io": "smoke -p 9111 test/mocks/google-cloud.io" + }, + "dependencies": { + "@web3-storage/worker-utils": "^0.3.0-dev", + "itty-router": "^2.4.5", + "multiformats": "^9.6.4", + "p-retry": "^5.0.0", + "toucan-js": "^2.5.0", + "uint8arrays": "^3.0.0" + }, + "devDependencies": { + "@cloudflare/workers-types": "^3.7.1", + "@sentry/cli": "^1.71.0", + "@types/git-rev-sync": "^2.0.0", + "@web-std/fetch": "^4.0.0", + "ava": "^3.15.0", + "esbuild": "^0.14.2", + "git-rev-sync": "^3.0.1", + "miniflare": "^2.5.0", + "npm-run-all": "^4.1.5", + "sade": "^1.7.4", + "smoke": "^3.1.1", + "standard": "^17.0.0", + "typescript": "4.7.3" + }, + "standard": { + "ignore": [ + "dist" + ] + }, + "author": "jsdevel ", + "license": "Apache-2.0 OR MIT" +} diff --git a/packages/cid-verifier/scripts/build.js b/packages/cid-verifier/scripts/build.js new file mode 100644 index 0000000..8e97dbe --- /dev/null +++ b/packages/cid-verifier/scripts/build.js @@ -0,0 +1,60 @@ +import fs from 'fs' +import path from 'path' +import { fileURLToPath } from 'url' +import { build } from 'esbuild' +import git from 'git-rev-sync' +import Sentry from '@sentry/cli' + +const __dirname = path.dirname(fileURLToPath(import.meta.url)) +const pkg = JSON.parse( + fs.readFileSync(path.join(__dirname, '..', 'package.json'), 'utf8') +) + +export async function buildCmd (opts) { + const sentryRelease = `cid-verifier@${pkg.version}-${opts.env}+${git.short( + __dirname + )}` + console.log(`Building ${sentryRelease}`) + + await build({ + entryPoints: [path.join(__dirname, '..', 'src', 'index.js')], + bundle: true, + format: 'esm', + outfile: path.join(__dirname, '..', 'dist', 'worker.js'), + legalComments: 'external', + define: { + SENTRY_RELEASE: JSON.stringify(sentryRelease), + VERSION: JSON.stringify(pkg.version), + COMMITHASH: JSON.stringify(git.long(__dirname)), + BRANCH: JSON.stringify(git.branch(__dirname)), + global: 'globalThis' + }, + minify: opts.env !== 'dev', + sourcemap: 'external' + }) + + // Sentry release and sourcemap upload + if (process.env.SENTRY_UPLOAD === 'true') { + const cli = new Sentry(undefined, { + authToken: process.env.SENTRY_TOKEN, + org: 'protocol-labs-it', + project: 'cid-verifier', + dist: git.short(__dirname) + }) + + await cli.releases.new(sentryRelease) + await cli.releases.setCommits(sentryRelease, { + auto: true, + ignoreEmpty: true, + ignoreMissing: true + }) + await cli.releases.uploadSourceMaps(sentryRelease, { + include: [path.join(__dirname, '..', 'dist')], + ext: ['map', 'js'] + }) + await cli.releases.finalize(sentryRelease) + await cli.releases.newDeploy(sentryRelease, { + env: opts.env + }) + } +} diff --git a/packages/cid-verifier/scripts/cli.js b/packages/cid-verifier/scripts/cli.js new file mode 100644 index 0000000..c2e2fd9 --- /dev/null +++ b/packages/cid-verifier/scripts/cli.js @@ -0,0 +1,16 @@ +#!/usr/bin/env node + +import sade from 'sade' + +import { buildCmd } from './build.js' + +const env = process.env.ENV || 'dev' +const prog = sade('cid-verifier') + +prog + .command('build') + .describe('Build the worker.') + .option('--env', 'Environment', env) + .action(buildCmd) + +prog.parse(process.argv) diff --git a/packages/cid-verifier/src/bindings.d.ts b/packages/cid-verifier/src/bindings.d.ts new file mode 100644 index 0000000..fbc92c5 --- /dev/null +++ b/packages/cid-verifier/src/bindings.d.ts @@ -0,0 +1,45 @@ +import Toucan from "toucan-js"; +import { Logging } from "@web3-storage/worker-utils/loki"; + +export {}; + +export interface EnvInput { + ENV: string; + DEBUG: string; + GOOGLE_EVALUATE_SAFE_CONFIDENCE_LEVELS: Array; + GOOGLE_CLOUD_API_URL: string; + GOOGLE_CLOUD_API_KEY: string; + SENTRY_DSN?: string; + LOKI_URL?: string; + LOKI_TOKEN?: string; + DENYLIST: KVNamespace; + CID_VERIFIER_RESULTS: KVNamespace; +} + +export interface EnvTransformed { + VERSION: string; + BRANCH: string; + COMMITHASH: string; + SENTRY_RELEASE: string; + sentry?: Toucan; + log: Logging; +} + +export type Env = EnvInput & EnvTransformed; + +export interface GoogleEvaluateAPIResultScore { + threatType: 'THREAT_TYPE_UNSPECIFIED' | 'SOCIAL_ENGINEERING' | 'UNWANTED_SOFTWARE' | 'MALWARE'; + confidenceLevel: 'CONFIDENCE_LEVEL_UNSPECIFIED' | 'SAFE' | 'LOW' | 'MEDIUM' | 'HIGH' | 'VERY_HIGH' | 'EXTREMELY_HIGH'; +} + +export interface GoogleEvaluateAPIResult { + scores: Array; +} + +declare global { + const BRANCH: string; + const VERSION: string; + const COMMITHASH: string; + const SENTRY_RELEASE: string; + const ENV: string; +} diff --git a/packages/cid-verifier/src/cors.js b/packages/cid-verifier/src/cors.js new file mode 100644 index 0000000..cc670b4 --- /dev/null +++ b/packages/cid-verifier/src/cors.js @@ -0,0 +1,34 @@ +/* eslint-env serviceworker */ + +/** + * @param {import('itty-router').RouteHandler} handler + */ +export function withCorsHeaders (handler) { + /** + * @param {Request} request + * @returns {Promise} + */ + return async (request, /** @type {any} */ ...rest) => { + const response = await handler(request, ...rest) + return addCorsHeaders(request, response) + } +} + +/** + * @param {Request} request + * @param {Response} response + * @returns {Response} + */ +export function addCorsHeaders (request, response) { + // Clone the response so that it's no longer immutable (like if it comes from cache or fetch) + response = new Response(response.body, response) + const origin = request.headers.get('origin') + if (origin) { + response.headers.set('Access-Control-Allow-Origin', origin) + response.headers.set('Vary', 'Origin') + } else { + response.headers.set('Access-Control-Allow-Origin', '*') + } + response.headers.set('Access-Control-Expose-Headers', 'Link') + return response +} diff --git a/packages/cid-verifier/src/env.js b/packages/cid-verifier/src/env.js new file mode 100644 index 0000000..3c34af4 --- /dev/null +++ b/packages/cid-verifier/src/env.js @@ -0,0 +1,68 @@ +/* global BRANCH, VERSION, COMMITHASH, SENTRY_RELEASE */ +import Toucan from 'toucan-js' + +import { Logging } from '@web3-storage/worker-utils/loki' + +import pkg from '../package.json' + +/** + * @typedef {import('./bindings').Env} Env + * @typedef {import('.').Ctx} Ctx + */ + +/** + * @param {Request} request + * @param {Env} env + * @param {Ctx} ctx + */ +export function envAll (request, env, ctx) { + // These values are replaced at build time by esbuild `define` + env.BRANCH = BRANCH + env.VERSION = VERSION + env.COMMITHASH = COMMITHASH + env.SENTRY_RELEASE = SENTRY_RELEASE + env.sentry = getSentry(request, env, ctx) + + env.log = new Logging(request, ctx, { + // @ts-ignore TODO: url should be optional together with token + url: env.LOKI_URL, + token: env.LOKI_TOKEN, + debug: Boolean(env.DEBUG), + version: env.VERSION, + commit: env.COMMITHASH, + branch: env.BRANCH, + worker: 'cid-verifier', + env: env.ENV, + sentry: env.sentry + }) + env.log.time('request') +} + +/** + * Get sentry instance if configured + * + * @param {Request} request + * @param {Env} env + * @param {Ctx} ctx + */ +function getSentry (request, env, ctx) { + if (!env.SENTRY_DSN) { + return + } + + return new Toucan({ + request, + dsn: env.SENTRY_DSN, + context: ctx, + allowedHeaders: ['user-agent'], + allowedSearchParams: /(.*)/, + debug: false, + environment: env.ENV || 'dev', + rewriteFrames: { + // sourcemaps only work if stack filepath are absolute like `/worker.js` + root: '/' + }, + release: env.SENTRY_RELEASE, + pkg + }) +} diff --git a/packages/cid-verifier/src/error-handler.js b/packages/cid-verifier/src/error-handler.js new file mode 100644 index 0000000..10da5ec --- /dev/null +++ b/packages/cid-verifier/src/error-handler.js @@ -0,0 +1,22 @@ +/* eslint-env serviceworker, browser */ + +/** + * @param {Error & {status?: number;code?: string; contentType?: string;}} err + * @param {import('./env').Env} env + */ +export function errorHandler (err, env) { + console.error(err.stack) + + const status = err.status || 500 + + if (env.sentry && status >= 500) { + env.log.error(err) + } + + return new Response(err.message || 'Server Error', { + status, + headers: { + 'content-type': err.contentType || 'text/plain' + } + }) +} diff --git a/packages/cid-verifier/src/errors.js b/packages/cid-verifier/src/errors.js new file mode 100644 index 0000000..06d0bcf --- /dev/null +++ b/packages/cid-verifier/src/errors.js @@ -0,0 +1,14 @@ +export class ServiceUnavailableError extends Error { + /** + * @param {string} message + */ + constructor (message = 'Service Unavailable') { + const status = 503 + super(message) + this.name = 'ServiceUnavailableError' + this.status = status + this.code = ServiceUnavailableError.CODE + this.contentType = 'text/html' + } +} +ServiceUnavailableError.CODE = 'ERROR_SERVICE_UNAVAILABLE' diff --git a/packages/cid-verifier/src/index.js b/packages/cid-verifier/src/index.js new file mode 100644 index 0000000..85cfd9c --- /dev/null +++ b/packages/cid-verifier/src/index.js @@ -0,0 +1,52 @@ +/* eslint-env serviceworker */ + +import { Router } from 'itty-router' + +import { verificationGet, verificationPost } from './verification.js' +import { versionGet } from './version.js' + +import { addCorsHeaders, withCorsHeaders } from './cors.js' +import { errorHandler } from './error-handler.js' +import { envAll } from './env.js' + +const router = Router() + +// https://developer.mozilla.org/en-US/docs/Web/API/FetchEvent +/** @typedef {ExecutionContext} Ctx */ + +router + .all('*', envAll) + .get('/version', withCorsHeaders(versionGet)) + .get('/denylist', withCorsHeaders(verificationGet)) + .post('/', withCorsHeaders(verificationPost)) + +/** + * @param {Error} error + * @param {Request} request + * @param {import('./env').Env} env + */ +function serverError (error, request, env) { + return addCorsHeaders(request, errorHandler(error, env)) +} + +export default { + /** + * + * @param {Request} request + * @param {import("./bindings").Env} env + * @param {Ctx} ctx + */ + async fetch (request, env, ctx) { + try { + const res = await router.handle(request, env, ctx) + env.log.timeEnd('request') + return env.log.end(res) + } catch (/** @type {any} */ error) { + if (env.log) { + env.log.timeEnd('request') + return env.log.end(serverError(error, request, env)) + } + return serverError(error, request, env) + } + } +} diff --git a/packages/cid-verifier/src/utils/denylist.js b/packages/cid-verifier/src/utils/denylist.js new file mode 100644 index 0000000..186b8b8 --- /dev/null +++ b/packages/cid-verifier/src/utils/denylist.js @@ -0,0 +1,34 @@ +import pRetry from 'p-retry' +import * as uint8arrays from 'uint8arrays' +import { sha256 } from 'multiformats/hashes/sha2' + +/** + * Get denylist anchor with badbits format. + * + * @param {string} cid + */ +export async function toDenyListAnchor (cid) { + const multihash = await sha256.digest(uint8arrays.fromString(`${cid}/`)) + const digest = multihash.bytes.subarray(2) + return uint8arrays.toString(digest, 'hex') +} + +/** + * Get a given entry from the deny list if CID exists. + * + * @param {string} cid + * @param {import('../env').Env} env + */ +export async function getFromDenyList (cid, env) { + const datastore = env.DENYLIST + if (!datastore) { + throw new Error('db not ready') + } + + const anchor = await toDenyListAnchor(cid) + // TODO: Remove once https://github.com/nftstorage/nftstorage.link/issues/51 is fixed + return await pRetry( + () => datastore.get(anchor), + { retries: 5 } + ) +} diff --git a/packages/cid-verifier/src/verification.js b/packages/cid-verifier/src/verification.js new file mode 100644 index 0000000..36ad5c9 --- /dev/null +++ b/packages/cid-verifier/src/verification.js @@ -0,0 +1,157 @@ +/* eslint-env serviceworker, browser */ +/* global Response */ +import pRetry from 'p-retry' + +import { getFromDenyList } from './utils/denylist' +import { ServiceUnavailableError } from './errors' + +const GOOGLE_EVALUATE = 'google-evaluate' + +/** + * Get verification results from 3rd parties stored in KV. + * + * @param {string} cid + * @param {import('./env').Env} env + */ +async function getResults (cid, env) { + const datastore = env.CID_VERIFIER_RESULTS + if (!datastore) { + throw new Error('CID_VERIFIER_RESULTS db not ready') + } + + return (await datastore.list({ prefix: cid }))?.keys?.reduce((acc, key) => { + // @ts-ignore + acc[key?.name] = key?.metadata?.value + return acc + }, {}) +} + +/** + * @param {Array} params + * @param {(params: Array, request: Request, env: import('./env').Env) => Promise} fn + * @returns {import('itty-router').RouteHandler} + */ +function withRequiredQueryParams (params, fn) { + /** + * @param {Request} request + * @param {import('./env').Env} env + */ + return async function (request, env) { + const searchParams = (new URL(request.url)).searchParams + + for (const param of params) { + if (!searchParams.get(param)) { + return new Response(`${param} is a required query param`, { status: 400 }) + } + } + + return await fn(params.map(param => String(searchParams.get(param))), request, env) + } +} + +export const verificationGet = withRequiredQueryParams(['cid'], + /** + * Returns google malware result. + */ + async function (params, request, env) { + const [cid] = params + + const denyListResource = await getFromDenyList(cid, env) + if (denyListResource) { + const { status } = JSON.parse(denyListResource) + if (status === 451) { + return new Response('BLOCKED FOR LEGAL REASONS', { status: 451 }) + } else { + return new Response('MALWARE DETECTED', { status: 403 }) + } + } + return new Response('', { status: 204 }) + } +) + +/** + * Process CID with malware verification parties. + */ +export const verificationPost = withRequiredQueryParams(['cid', 'url'], + async function verificationPost (params, request, env) { + const [cid, url] = params + const resultKey = `${cid}/${GOOGLE_EVALUATE}` + const lockKey = `${cid}/${GOOGLE_EVALUATE}.lock` + const cidVerifyResults = await getResults(cid, env) + // @ts-ignore + const googleEvaluateResult = cidVerifyResults[resultKey] + // @ts-ignore + const googleEvaluateLock = cidVerifyResults[lockKey] + + if (!googleEvaluateResult && !googleEvaluateLock) { + await fetchGoogleMalwareResults(cid, url, env) + return new Response('cid malware detection processed', { status: 201 }) + } + + return new Response('cid malware detection processed', { status: 202 }) + + /** + * Fetch malware results for the url from Google's Evaluate API. + * + * @param {string} cid + * @param {string} url + * @param {import('./env').Env} env + */ + async function fetchGoogleMalwareResults (cid, url, env) { + try { + await env.CID_VERIFIER_RESULTS.put(lockKey, 'true', { metadata: { value: 'true' } }) + const googleCloudResponse = await fetch( + `${ + env.GOOGLE_CLOUD_API_URL + }/v1eap1:evaluateUri?key=${env.GOOGLE_CLOUD_API_KEY}`, + { + body: JSON.stringify({ + uri: url, + threatTypes: ['SOCIAL_ENGINEERING', 'MALWARE', 'UNWANTED_SOFTWARE'], + allowScan: 'true' + }), + headers: { + 'content-type': 'application/json; charset=utf-8' + }, + method: 'POST' + } + ) + + if (googleCloudResponse.status !== 200) { + throw new ServiceUnavailableError(`GOOGLE CLOUD UNABLE TO VERIFY URL "${url}" status code "${googleCloudResponse?.status}"`) + } + + const evaluateJson = await googleCloudResponse.json() + const stringifiedJSON = JSON.stringify(evaluateJson) + + await pRetry( + () => env.CID_VERIFIER_RESULTS.put(resultKey, stringifiedJSON, { metadata: { value: stringifiedJSON } }), + { retries: 5 } + ) + // @ts-ignore + // if any score isn't what we consider to be safe we add it to the DENYLIST + const threats = evaluateJson?.scores?.filter(score => !env.GOOGLE_EVALUATE_SAFE_CONFIDENCE_LEVELS.includes(score.confidenceLevel)).map(score => score.threatType) + if (threats.length) { + await pRetry( + () => env.DENYLIST.put(cid, JSON.stringify({ + status: 403, + reason: threats.join(', ') + })), + { retries: 5 } + ) + // @ts-ignore + env.log.log(`MALWARE DETECTED for cid "${cid}" ${threats.join(', ')}`, 'info') + } + } catch (e) { + // @ts-ignore + env.log.log(e, 'error') + throw e + } finally { + await pRetry( + () => env.CID_VERIFIER_RESULTS.delete(lockKey), + { retries: 5 } + ) + } + } + } +) diff --git a/packages/cid-verifier/src/version.js b/packages/cid-verifier/src/version.js new file mode 100644 index 0000000..deb9544 --- /dev/null +++ b/packages/cid-verifier/src/version.js @@ -0,0 +1,15 @@ +import { JSONResponse } from '@web3-storage/worker-utils/response' + +/** + * Get edge gateway API version information. + * + * @param {Request} request + * @param {import('./env').Env} env + */ +export async function versionGet (request, env) { + return new JSONResponse({ + version: env.VERSION, + commit: env.COMMITHASH, + branch: env.BRANCH + }) +} diff --git a/packages/cid-verifier/test/mocks/google-cloud.io/v1eap1@evaluateUri/post_.js b/packages/cid-verifier/test/mocks/google-cloud.io/v1eap1@evaluateUri/post_.js new file mode 100644 index 0000000..57a8a8a --- /dev/null +++ b/packages/cid-verifier/test/mocks/google-cloud.io/v1eap1@evaluateUri/post_.js @@ -0,0 +1,67 @@ +/** + * https://github.com/sinedied/smoke#javascript-mocks + */ +module.exports = ({ body: { uri } }) => { + if (uri === 'http://malicious/url') { + return { + statusCode: 200, + body: { + scores: [ + { + threatType: 'SOCIAL_ENGINEERING', + confidenceLevel: 'LOW' + }, + { + threatType: 'UNWANTED_SOFTWARE', + confidenceLevel: 'HIGH' + }, + { + threatType: 'MALWARE', + confidenceLevel: 'SAFE' + } + ] + } + } + } else if (uri === 'http://safe/url') { + return { + statusCode: 200, + body: { + scores: [ + { + threatType: 'SOCIAL_ENGINEERING', + confidenceLevel: 'SAFE' + }, + { + threatType: 'UNWANTED_SOFTWARE', + confidenceLevel: 'SAFE' + }, + { + threatType: 'MALWARE', + confidenceLevel: 'SAFE' + } + ] + } + } + } + + return { + statusCode: 400, + body: { + error: { + code: 400, + message: 'API key not valid. Please pass a valid API key.', + status: 'INVALID_ARGUMENT', + details: [ + { + '@type': 'type.googleapis.com/google.rpc.ErrorInfo', + reason: 'API_KEY_INVALID', + domain: 'googleapis.com', + metadata: { + service: 'webrisk.googleapis.com' + } + } + ] + } + } + } +} diff --git a/packages/cid-verifier/test/mocks/package.json b/packages/cid-verifier/test/mocks/package.json new file mode 100644 index 0000000..024ddd1 --- /dev/null +++ b/packages/cid-verifier/test/mocks/package.json @@ -0,0 +1,5 @@ +{ + "name": "mocks", + "version": "1.0.0", + "description": "just here to fix cjs loading" +} diff --git a/packages/cid-verifier/test/utils/miniflare.js b/packages/cid-verifier/test/utils/miniflare.js new file mode 100644 index 0000000..15ff5fb --- /dev/null +++ b/packages/cid-verifier/test/utils/miniflare.js @@ -0,0 +1,25 @@ +import fs from 'fs' +import path from 'path' +import { Miniflare } from 'miniflare' + +export function getMiniflare () { + let envPath = path.join(process.cwd(), '../../.env') + if (!fs.statSync(envPath, { throwIfNoEntry: false })) { + // @ts-ignore + envPath = true + } + + return new Miniflare({ + envPath, + scriptPath: 'dist/worker.js', + port: 8788, + packagePath: true, + wranglerConfigPath: true, + // We don't want to rebuild our worker for each test, we're already doing + // it once before we run all tests in package.json, so disable it here. + // This will override the option in wrangler.toml. + buildCommand: undefined, + wranglerConfigEnv: 'test', + modules: true + }) +} diff --git a/packages/cid-verifier/test/utils/setup.js b/packages/cid-verifier/test/utils/setup.js new file mode 100644 index 0000000..ef33e9f --- /dev/null +++ b/packages/cid-verifier/test/utils/setup.js @@ -0,0 +1,13 @@ +import anyTest from 'ava' +export * from './miniflare.js' + +/** + * @typedef {import('miniflare').Miniflare} Miniflare + * + * @typedef {Object} Context + * @property {Miniflare} mf + * + * @typedef {import('ava').TestInterface} TestFn + */ + +export const test = /** @type {TestFn} */ (anyTest) diff --git a/packages/cid-verifier/test/verification.spec.js b/packages/cid-verifier/test/verification.spec.js new file mode 100644 index 0000000..b5624d7 --- /dev/null +++ b/packages/cid-verifier/test/verification.spec.js @@ -0,0 +1,141 @@ +import { test, getMiniflare } from './utils/setup.js' +import { toDenyListAnchor } from '../src/utils/denylist.js' + +// TODO: use valid cids and test 400 scenarios +const cidInDenyList = 'asdfasdf' +const cidInDenyListBlockedForLeganReasons = 'blocked for legal reasons' +const pendingCid = 'pending' +const emptyCid = 'empty' +const notMalwareCid = 'notMalware' +const malwareCid = 'malware' +const newCid = 'newCid' +const maliciousUrl = encodeURIComponent('http://malicious/url') +const safeUrl = encodeURIComponent('http://safe/url') +const errorUrl = encodeURIComponent('http://error/url') + +// Create a new Miniflare environment for each test +test.before(async (t) => { + const mf = getMiniflare() + t.context = { + mf + } + const googleMalwareResultsKv = await mf.getKVNamespace('CID_VERIFIER_RESULTS') + await googleMalwareResultsKv.put(`${pendingCid}/google-evaluate.lock`, 'true', { metadata: { value: 'true' } }) + await googleMalwareResultsKv.put(`${notMalwareCid}/google-evaluate`, '{}', { metadata: { value: '{}' } }) + await googleMalwareResultsKv.put(`${malwareCid}/google-evaluate`, JSON.stringify({ + threat: { + threatTypes: ['MALWARE'], + expireTime: '2022-08-28T07:54:04.936398042Z' + } + }), { + metadata: { + value: JSON.stringify({ + threat: { + threatTypes: ['MALWARE'], + expireTime: '2022-08-28T07:54:04.936398042Z' + } + }) + } + }) + const denylistKv = await mf.getKVNamespace('DENYLIST') + await denylistKv.put(await toDenyListAnchor(cidInDenyList), JSON.stringify({ status: 410, reason: 'bad' })) + await denylistKv.put(await toDenyListAnchor(cidInDenyListBlockedForLeganReasons), JSON.stringify({ status: 451, reason: 'blocked for legal reasons' })) +}) + +test('GET /denylist handles cids in DENYLIST', async (t) => { + const { mf } = t.context + const response = await mf.dispatchFetch(`http://localhost:8787/denylist?cid=${cidInDenyList}`) + t.is(await response.text(), 'MALWARE DETECTED') + t.is(response.status, 403) +}) + +test('GET /denylist handles cids in DENYLIST blocked for legal reasons', async (t) => { + const { mf } = t.context + const response = await mf.dispatchFetch(`http://localhost:8787/denylist?cid=${cidInDenyListBlockedForLeganReasons}`) + t.is(await response.text(), 'BLOCKED FOR LEGAL REASONS') + t.is(response.status, 451) +}) + +test('GET /denylist handles no cid', async (t) => { + const { mf } = t.context + const response = await mf.dispatchFetch('http://localhost:8787/denylist?') + t.is(await response.text(), 'cid is a required query param') + t.is(response.status, 400) +}) + +test('GET /denylist handles no results', async (t) => { + const { mf } = t.context + const response = await mf.dispatchFetch(`http://localhost:8787/denylist?cid=${emptyCid}`) + t.is(await response.text(), '') + t.is(response.status, 204) +}) + +test('GET /denylist handles pending results', async (t) => { + const { mf } = t.context + const response = await mf.dispatchFetch(`http://localhost:8787/denylist?cid=${pendingCid}`) + t.is(await response.text(), '') + t.is(response.status, 204) +}) + +test('GET /denylist handles successful results', async (t) => { + const { mf } = t.context + const response = await mf.dispatchFetch(`http://localhost:8787/denylist?cid=${notMalwareCid}`) + t.is(await response.text(), '') + t.is(response.status, 204) +}) + +test('POST / handles no cid', async (t) => { + const { mf } = t.context + const response = await mf.dispatchFetch('http://localhost:8787/?', { method: 'POST' }) + t.is(await response.text(), 'cid is a required query param') + t.is(response.status, 400) +}) + +test('POST / handles no url', async (t) => { + const { mf } = t.context + const response = await mf.dispatchFetch('http://localhost:8787/?cid=asdf', { method: 'POST' }) + t.is(await response.text(), 'url is a required query param') + t.is(response.status, 400) +}) + +test('POST / handles malicious urls', async (t) => { + const { mf } = t.context + const response = await mf.dispatchFetch(`http://localhost:8787/?cid=${newCid}&url=${maliciousUrl}`, { method: 'POST' }) + t.is(await response.text(), 'cid malware detection processed') + t.is(response.status, 201) +}) + +test('POST / handles safe urls', async (t) => { + const { mf } = t.context + const response = await mf.dispatchFetch(`http://localhost:8787/?cid=${Math.random()}&url=${safeUrl}`, { method: 'POST' }) + t.is(await response.text(), 'cid malware detection processed') + t.is(response.status, 201) +}) + +test('POST / handles invalid or error urls', async (t) => { + const { mf } = t.context + const response = await mf.dispatchFetch(`http://localhost:8787/?cid=${Math.random()}&url=${errorUrl}`, { method: 'POST' }) + t.is(await response.text(), 'GOOGLE CLOUD UNABLE TO VERIFY URL "http://error/url" status code "400"') + t.is(response.status, 503) +}) + +test('POST / handles no results', async (t) => { + const { mf } = t.context + const response = await mf.dispatchFetch(`http://localhost:8787/?cid=${newCid}&url=${maliciousUrl}`, { method: 'POST' }) + t.is(await response.text(), 'cid malware detection processed') + t.is(response.status, 201) +}) + +test('POST / handles pending results', async (t) => { + const { mf } = t.context + const response = await mf.dispatchFetch(`http://localhost:8787/?cid=${pendingCid}&url=${maliciousUrl}`, { method: 'POST' }) + t.is(await response.text(), 'cid malware detection processed') + t.is(response.status, 202) +}) + +test('POST / handles overriding existing malware cid', async (t) => { + const { mf } = t.context + const response = await mf.dispatchFetch(`http://localhost:8787/?cid=${malwareCid}&url=${maliciousUrl}`, { method: 'POST' }) + t.is(await response.text(), 'cid malware detection processed') + t.is(response.status, 202) +}) diff --git a/packages/cid-verifier/test/version.spec.js b/packages/cid-verifier/test/version.spec.js new file mode 100644 index 0000000..9766243 --- /dev/null +++ b/packages/cid-verifier/test/version.spec.js @@ -0,0 +1,29 @@ +import fs from 'fs' +import path from 'path' +import { fileURLToPath } from 'url' +import git from 'git-rev-sync' + +import { test, getMiniflare } from './utils/setup.js' + +const __dirname = path.dirname(fileURLToPath(import.meta.url)) +const pkg = JSON.parse( + fs.readFileSync(path.join(__dirname, '..', 'package.json'), 'utf8') +) + +// Create a new Miniflare environment for each test +test.before((t) => { + t.context = { + mf: getMiniflare() + } +}) + +test('Gets Version', async (t) => { + const { mf } = t.context + + const response = await mf.dispatchFetch('http://localhost:8787/version') + const { version, commit, branch } = await response.json() + + t.is(version, pkg.version) + t.is(commit, git.long(__dirname)) + t.is(branch, git.branch(__dirname)) +}) diff --git a/packages/cid-verifier/tsconfig.json b/packages/cid-verifier/tsconfig.json new file mode 100644 index 0000000..23575d6 --- /dev/null +++ b/packages/cid-verifier/tsconfig.json @@ -0,0 +1,11 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "outDir": "dist", + "noEmit": true, + "lib": ["ESNext"], + "types": ["@cloudflare/workers-types"] + }, + "include": ["src", "test", "package.json"], + "exclude": ["**/node_modules/**", "test/mocks/**"] +} diff --git a/packages/cid-verifier/wrangler.toml b/packages/cid-verifier/wrangler.toml new file mode 100644 index 0000000..7f7b7fb --- /dev/null +++ b/packages/cid-verifier/wrangler.toml @@ -0,0 +1,61 @@ +# dotstorage cid verifier wrangler config. +name = "dotstorage-cid-verifier" +main = "./dist/worker.js" +compatibility_date = "2022-07-01" +compatibility_flags = [ "url_standard" ] +no_bundle = true + +[build] +command = "npm run build" + +[vars] +DEBUG = "true" +ENV = "dev" +GOOGLE_EVALUATE_SAFE_CONFIDENCE_LEVELS = [ "CONFIDENCE_LEVEL_UNSPECIFIED", "SAFE" ] + +# PROD! +[env.production] +# name = "dotstorage-cid-verifier-production" +account_id = "fffa4b4363a7e5250af8357087263b3a" # Protocol Labs CF account +routes = [ + "cid-verifier.dag.haus/*" +] +kv_namespaces = [ + { binding = "CID_VERIFIER_RESULTS", id = "9be91fbb7fe34afea6629732981fc8e9" }, + { binding = "DENYLIST", id = "785cf627e913468ca5319523ae929def" } +] + +[env.production.vars] +DEBUG = "false" +ENV = "production" + +# Staging! +[env.staging] +# name = "dotstorage-cid-verifier-staging" +account_id = "fffa4b4363a7e5250af8357087263b3a" # Protocol Labs CF account +routes = [ + "cid-verifier-staging.dag.haus/*" +] +kv_namespaces = [ + { binding = "CID_VERIFIER_RESULTS", id = "c742e87b92da4a77bdce7de8105778ff" }, + { binding = "DENYLIST", id = "f4eb0eca32e14e28b643604a82e00cb3" } +] + +[env.staging.vars] +DEBUG = "true" +ENV = "staging" + +# Test! +[env.test] +workers_dev = true +kv_namespaces = [ + { binding = "CID_VERIFIER_RESULTS" }, + { binding = "DENYLIST" } +] + +[env.test.vars] +GOOGLE_CLOUD_API_URL = 'http://127.0.0.1:9111' +GOOGLE_CLOUD_API_KEY = 'TEST' +DEBUG = "true" +ENV = "test" +GOOGLE_EVALUATE_SAFE_CONFIDENCE_LEVELS = [ "CONFIDENCE_LEVEL_UNSPECIFIED", "SAFE" ] diff --git a/packages/edge-gateway/src/bindings.d.ts b/packages/edge-gateway/src/bindings.d.ts index a79ccc1..37372c0 100644 --- a/packages/edge-gateway/src/bindings.d.ts +++ b/packages/edge-gateway/src/bindings.d.ts @@ -17,6 +17,8 @@ export interface AnalyticsEngineEvent { export interface EnvInput { ENV: string DEBUG: string + CID_VERIFIER_URL: string + CID_VERIFIER: Fetcher IPFS_GATEWAYS: string GATEWAY_HOSTNAME: string EDGE_GATEWAY_API_URL: string diff --git a/packages/edge-gateway/src/errors.js b/packages/edge-gateway/src/errors.js index a3c7d21..269d6b1 100644 --- a/packages/edge-gateway/src/errors.js +++ b/packages/edge-gateway/src/errors.js @@ -28,7 +28,10 @@ export class TimeoutError extends Error { } TimeoutError.CODE = 'ERROR_TIMEOUT' -export const createErrorHtmlContent = (/** @type {number} */ status, /** @type {string} */ message) => ` +export const createErrorHtmlContent = ( + /** @type {number} */ status, + /** @type {string} */ message +) => ` ${status} ${message}
diff --git a/packages/edge-gateway/src/gateway.js b/packages/edge-gateway/src/gateway.js index 1bc1f23..99ec2ac 100644 --- a/packages/edge-gateway/src/gateway.js +++ b/packages/edge-gateway/src/gateway.js @@ -3,13 +3,11 @@ import pAny, { AggregateError } from 'p-any' import pDefer from 'p-defer' -import pRetry from 'p-retry' import pSettle from 'p-settle' import { FilterError } from 'p-some' import { getCidFromSubdomainUrl } from './utils/cid.js' import { getHeaders } from './utils/headers.js' -import { toDenyListAnchor } from './utils/deny-list.js' import { TimeoutError } from './errors.js' import { CF_CACHE_MAX_OBJECT_SIZE, @@ -51,11 +49,9 @@ export async function gatewayGet (request, env, ctx) { const cid = await getCidFromSubdomainUrl(reqUrl) const pathname = reqUrl.pathname - // Validation layer - root CID validation - const denyListRootCidEntry = await getFromDenyList(env.DENYLIST, cid) - if (denyListRootCidEntry) { - const { status, reason } = JSON.parse(denyListRootCidEntry) - return new Response(reason || '', { status: status || 410 }) + const cidVerificationResponse = await env.CID_VERIFIER.fetch(`${env.CID_VERIFIER_URL}/verification?cid=${cid}`) + if (cidVerificationResponse.status !== 204) { + return cidVerificationResponse } // 1st layer resolution - CDN @@ -101,17 +97,21 @@ export async function gatewayGet (request, env, ctx) { const winnerUrl = await winnerUrlPromise.promise // Validation layer - resource CID - if (winnerGwResponse && pathname !== '/') { - const resourceCid = decodeURIComponent( - winnerGwResponse.headers.get('etag') || '' - ) - const denyListResource = await getFromDenyList(env.DENYLIST, resourceCid) - if (denyListResource) { - const { status, reason } = JSON.parse(denyListResource) - return new Response(reason || '', { status: status || 410 }) + const resourceCid = decodeURIComponent(winnerGwResponse.headers.get('etag') || '') + if (winnerGwResponse && pathname !== '/' && resourceCid) { + const cidResourceVerificationResponse = await env.CID_VERIFIER.fetch(`${env.CID_VERIFIER_URL}/denylist?cid=${resourceCid}`) + if (cidResourceVerificationResponse.status !== 204) { + return cidResourceVerificationResponse } } + // Ask CID verifier to validate HTML content + if (winnerGwResponse.headers.get('content-type')?.includes('text/html')) { + const verifyCid = pathname !== '/' ? resourceCid : cid + // fire and forget. Let cid-verifier process this cid and url if it needs to + env.CID_VERIFIER.fetch(`${env.CID_VERIFIER_URL}/?cid=${verifyCid}&url=${encodeURIComponent(request.url)}`, { method: 'POST' }) + } + // Cache response if (winnerGwResponse && winnerGwResponse.ok) { ctx.waitUntil(putToCache(request, winnerGwResponse, cache)) @@ -174,9 +174,10 @@ async function getFromDotstorage (request, cid, { pathname = '' } = {}) { // @ts-ignore custom entry in cf object if (request.cf?.onlyIfCachedGateways) { /** @type {URL[]} */ - // @ts-ignore custom entry in cf object - const onlyIfCachedGateways = (JSON.parse(request.cf?.onlyIfCachedGateways)) - .map((/** @type {string} */ gw) => new URL(gw)) + const onlyIfCachedGateways = JSON.parse( + // @ts-ignore custom entry in cf object + request.cf?.onlyIfCachedGateways + ).map((/** @type {string} */ gw) => new URL(gw)) onlyIfCachedGateways.forEach((gw) => hosts.push(gw.host)) } @@ -185,21 +186,22 @@ async function getFromDotstorage (request, cid, { pathname = '' } = {}) { const headers = getHeaders(request) headers.set('Cache-Control', 'only-if-cached') - const proxiedResponse = await pAny(hosts.map(async (host) => { - const response = await fetch( - `https://${cid}.ipfs.${host}${pathname}`, - { headers } - ) + const proxiedResponse = await pAny( + hosts.map(async (host) => { + const response = await fetch(`https://${cid}.ipfs.${host}${pathname}`, { + headers + }) - if (!response.ok) { - throw new Error() - } + if (!response.ok) { + throw new Error() + } - return { - response, - resolutionIdentifier: host - } - })) + return { + response, + resolutionIdentifier: host + } + }) + ) return proxiedResponse } catch (_) {} @@ -253,27 +255,6 @@ async function getFromPermaCache (request, env) { } } -/** - * Get a given entry from the deny list if CID exists. - * - * @param {KVNamespace} datastore - * @param {string} cid - */ -async function getFromDenyList (datastore, cid) { - if (!datastore) { - return undefined - } - - const anchor = await toDenyListAnchor(cid) - // TODO: Remove once https://github.com/nftstorage/nftstorage.link/issues/51 is fixed - return await pRetry( - // TODO: in theory we should check each subcomponent of the pathname also. - // https://github.com/nftstorage/nft.storage/issues/1737 - () => datastore.get(anchor), - { retries: 5 } - ) -} - /** * Put receives response to cache. * @@ -304,7 +285,10 @@ function getResponseWithCustomHeaders ( const clonedResponse = new Response(response.body, response) clonedResponse.headers.set('x-dotstorage-resolution-layer', resolutionLayer) - clonedResponse.headers.set('x-dotstorage-resolution-id', resolutionIdentifier) + clonedResponse.headers.set( + 'x-dotstorage-resolution-id', + resolutionIdentifier + ) return clonedResponse } diff --git a/packages/edge-gateway/src/utils/deny-list.js b/packages/edge-gateway/src/utils/deny-list.js deleted file mode 100644 index abee945..0000000 --- a/packages/edge-gateway/src/utils/deny-list.js +++ /dev/null @@ -1,13 +0,0 @@ -import * as uint8arrays from 'uint8arrays' -import { sha256 } from 'multiformats/hashes/sha2' - -/** - * Get denylist anchor with badbits format. - * - * @param {string} cid - */ -export async function toDenyListAnchor (cid) { - const multihash = await sha256.digest(uint8arrays.fromString(`${cid}/`)) - const digest = multihash.bytes.subarray(2) - return uint8arrays.toString(digest, 'hex') -} diff --git a/packages/edge-gateway/test/cid-verifier.spec.js b/packages/edge-gateway/test/cid-verifier.spec.js new file mode 100644 index 0000000..e1a398d --- /dev/null +++ b/packages/edge-gateway/test/cid-verifier.spec.js @@ -0,0 +1,91 @@ +import pWaitFor from 'p-wait-for' +import { GenericContainer, Wait } from 'testcontainers' + +import { test, getMiniflare } from './utils/setup.js' +import { addFixtures } from './utils/fixtures.js' + +test.before(async (t) => { + const container = await new GenericContainer('ipfs/go-ipfs:v0.13.0') + .withExposedPorts( + { + container: 8080, + host: 9081 + }, + 5001 + ) + .withWaitStrategy(Wait.forLogMessage('Daemon is ready')) + .start() + + // Add fixtures + await addFixtures(container.getMappedPort(5001)) + + t.context = { + container, + mf: getMiniflare() + } +}) + +test.beforeEach(async (t) => { + t.context = { + ...t.context, + mf: getMiniflare() + } +}) + +test.after(async (t) => { + await t.context.container?.stop() +}) + +test('Returns 403 when requested CID is recorded as malicious', async (t) => { + const { mf } = t.context + const cid = 'bafkreibx45dh23bkcli5qxevg2zq5pa7dbzpdd45h4uugk6qgjlyifulj4' + + const res = await mf.dispatchFetch(`https://${cid}.ipfs.localhost:8787`) + t.is(res.status, 403) + t.is(await res.text(), 'malicious') +}) + +test('Returns 403 when requested resource CID is recorded as malicious', async (t) => { + const { mf } = t.context + const cid = 'bafybeiaekuoonpqpmems3uapy27zsas5p6ylku53lzkaufnvt4s5n6a7au' + + const res = await mf.dispatchFetch(`https://${cid}.ipfs.localhost:8787/malware.txt`) + t.is(res.status, 403) + t.is(await res.text(), 'malicious') +}) + +test('Request cid-verifier to validate response when HTML file is requested with root cid+path', async (t) => { + const { mf } = t.context + const htmlDirectoryRootCid = 'bafybeiaekuoonpqpmems3uapy27zsas5p6ylku53lzkaufnvt4s5n6a7au' + + const res = await mf.dispatchFetch(`https://${htmlDirectoryRootCid}.ipfs.localhost:8787/sample.html`) + t.is(res.status, 200) + t.is(res.headers.get('Content-Type'), 'text/html') + + // Validate call in progress to verify CID + const TEST_NAMESPACE = await mf.getKVNamespace('TEST_NAMESPACE') + + const kvKey = `${htmlDirectoryRootCid}/test.lock` + await pWaitFor(async () => { + const value = await TEST_NAMESPACE.get(kvKey) + return value === 'LOCK' + }, { timeout: 1000 }) +}) + +test('Request cid-verifier to validate response when HTML file is requested with resource cid', async (t) => { + const { mf } = t.context + const htmlResourceCid = 'bafkreib6uzgr2noyzup3uuqcp6gafddnx6n3iinkyflbrkhdhfpcoggc5u' + + const res = await mf.dispatchFetch(`https://${htmlResourceCid}.ipfs.localhost:8787`) + t.is(res.status, 200) + t.true(res.headers.get('Content-Type')?.includes('text/html')) + + // Validate call in progress to verify CID + const TEST_NAMESPACE = await mf.getKVNamespace('TEST_NAMESPACE') + + const kvKey = `${htmlResourceCid}/test.lock` + await pWaitFor(async () => { + const value = await TEST_NAMESPACE.get(kvKey) + return value === 'LOCK' + }, { timeout: 1000 }) +}) diff --git a/packages/edge-gateway/test/denylist.spec.js b/packages/edge-gateway/test/denylist.spec.js deleted file mode 100644 index 5c2d3cb..0000000 --- a/packages/edge-gateway/test/denylist.spec.js +++ /dev/null @@ -1,84 +0,0 @@ -import { GenericContainer, Wait } from 'testcontainers' -import { test, getMiniflare } from './utils/setup.js' -import { addFixtures } from './utils/fixtures.js' -import { toDenyListAnchor } from '../src/utils/deny-list.js' - -test.before(async (t) => { - const container = await new GenericContainer('ipfs/go-ipfs:v0.13.0') - .withExposedPorts({ - container: 8080, - host: 9081 - }, - 5001 - ) - .withWaitStrategy(Wait.forLogMessage('Daemon is ready')) - .start() - - // Add fixtures - await addFixtures(container.getMappedPort(5001)) - - t.context = { - container, - mf: getMiniflare() - } -}) - -test.beforeEach(async (t) => { - t.context = { - ...t.context, - mf: getMiniflare() - } -}) - -test.after(async (t) => { - await t.context.container?.stop() -}) - -test('Blocks access to a CID on the deny list', async (t) => { - const { mf } = t.context - const cid = 'bafkreidyeivj7adnnac6ljvzj2e3rd5xdw3revw4da7mx2ckrstapoupoq' - - // add the CID to the deny list - const denyListKv = await mf.getKVNamespace('DENYLIST') - const anchor = await toDenyListAnchor(cid) - - await denyListKv.put(anchor, '{}') - - const res = await mf.dispatchFetch(`https://${cid}.ipfs.localhost:8787`) - t.is(res.status, 410) - t.is(await res.text(), '') -}) - -test('Blocks access to a CID resource on the deny list', async (t) => { - const { mf } = t.context - const resourceCid = - 'bafkreia4d2wzubczuknsuwcrta2psy7rjkso4xxryjep44yvddtp6pe5vu' - - // add the resourceCid to the deny list - const denyListKv = await mf.getKVNamespace('DENYLIST') - const anchor = await toDenyListAnchor(resourceCid) - - await denyListKv.put(anchor, '{}') - - const res = await mf.dispatchFetch( - 'https://bafybeih74zqc6kamjpruyra4e4pblnwdpickrvk4hvturisbtveghflovq.ipfs.localhost:8787/path' - ) - t.is(res.status, 410) - t.is(await res.text(), '') -}) - -test('Blocks access to a CID on the deny list with custom status and reason', async (t) => { - const { mf } = t.context - const cid = 'bafkreidyeivj7adnnac6ljvzj2e3rd5xdw3revw4da7mx2ckrstapoupoq' - - // add the CID to the deny list - const denyListKv = await mf.getKVNamespace('DENYLIST') - const anchor = await toDenyListAnchor(cid) - - // 451: Unavailable For Legal Reasons - await denyListKv.put(anchor, JSON.stringify({ status: 451, reason: 'bad' })) - - const res = await mf.dispatchFetch(`https://${cid}.ipfs.localhost:8787`) - t.is(res.status, 451) - t.is(await res.text(), 'bad') -}) diff --git a/packages/edge-gateway/test/fixtures/website/malware.txt b/packages/edge-gateway/test/fixtures/website/malware.txt new file mode 100644 index 0000000..84fd04c --- /dev/null +++ b/packages/edge-gateway/test/fixtures/website/malware.txt @@ -0,0 +1 @@ +This is fixture malware \ No newline at end of file diff --git a/packages/edge-gateway/test/fixtures/website/sample.html b/packages/edge-gateway/test/fixtures/website/sample.html new file mode 100644 index 0000000..bf1d7a4 --- /dev/null +++ b/packages/edge-gateway/test/fixtures/website/sample.html @@ -0,0 +1,2 @@ + + diff --git a/packages/edge-gateway/test/mocks/cf-ipfs.com/get_ipfs#@cid#path.js b/packages/edge-gateway/test/mocks/cf-ipfs.com/get_ipfs#@cid#path.js index df27b79..ff005b8 100644 --- a/packages/edge-gateway/test/mocks/cf-ipfs.com/get_ipfs#@cid#path.js +++ b/packages/edge-gateway/test/mocks/cf-ipfs.com/get_ipfs#@cid#path.js @@ -1,7 +1,18 @@ /** * https://github.com/sinedied/smoke#javascript-mocks */ -module.exports = () => { +module.exports = (request) => { + if (request.url.includes('bafybeiaekuoonpqpmems3uapy27zsas5p6ylku53lzkaufnvt4s5n6a7au')) { + return { + statusCode: 200, + headers: { + 'Content-Type': 'text/plain', + Etag: 'bafkreibv3ecfm3wpoawshuqhir3cn2w4dewlr6jit3hfx6cjqgmzbsq22y' + }, + body: 'This is fixture malware' + } + } + return { statusCode: 200, headers: { diff --git a/packages/edge-gateway/test/mocks/cf-ipfs.com/get_ipfs#@cid.js b/packages/edge-gateway/test/mocks/cf-ipfs.com/get_ipfs#@cid.js index a396e5f..0b4a290 100644 --- a/packages/edge-gateway/test/mocks/cf-ipfs.com/get_ipfs#@cid.js +++ b/packages/edge-gateway/test/mocks/cf-ipfs.com/get_ipfs#@cid.js @@ -25,6 +25,20 @@ module.exports = async ({ params, headers }) => { headers: responseHeaders, body: 'Hello dot.storage! 😎👻' } + } else if ( + cid === 'bafkreib6uzgr2noyzup3uuqcp6gafddnx6n3iinkyflbrkhdhfpcoggc5u' + ) { + return { + statusCode: 200, + headers: { + ...headers, + 'Content-Type': 'text/html; charset=utf-8' + }, + body: ` + + + ` + } } return { diff --git a/packages/edge-gateway/test/mocks/ipfs.io/get_ipfs#@cid#path.js b/packages/edge-gateway/test/mocks/ipfs.io/get_ipfs#@cid#path.js index df27b79..ff005b8 100644 --- a/packages/edge-gateway/test/mocks/ipfs.io/get_ipfs#@cid#path.js +++ b/packages/edge-gateway/test/mocks/ipfs.io/get_ipfs#@cid#path.js @@ -1,7 +1,18 @@ /** * https://github.com/sinedied/smoke#javascript-mocks */ -module.exports = () => { +module.exports = (request) => { + if (request.url.includes('bafybeiaekuoonpqpmems3uapy27zsas5p6ylku53lzkaufnvt4s5n6a7au')) { + return { + statusCode: 200, + headers: { + 'Content-Type': 'text/plain', + Etag: 'bafkreibv3ecfm3wpoawshuqhir3cn2w4dewlr6jit3hfx6cjqgmzbsq22y' + }, + body: 'This is fixture malware' + } + } + return { statusCode: 200, headers: { diff --git a/packages/edge-gateway/test/mocks/ipfs.io/get_ipfs#@cid.js b/packages/edge-gateway/test/mocks/ipfs.io/get_ipfs#@cid.js index 9447294..bca5536 100644 --- a/packages/edge-gateway/test/mocks/ipfs.io/get_ipfs#@cid.js +++ b/packages/edge-gateway/test/mocks/ipfs.io/get_ipfs#@cid.js @@ -23,6 +23,20 @@ module.exports = async ({ params, headers }) => { headers: responseHeaders, body: 'Hello dot.storage! 😎👻' } + } else if ( + cid === 'bafkreib6uzgr2noyzup3uuqcp6gafddnx6n3iinkyflbrkhdhfpcoggc5u' + ) { + return { + statusCode: 200, + headers: { + ...headers, + 'Content-Type': 'text/html; charset=utf-8' + }, + body: ` + + + ` + } } return { diff --git a/packages/edge-gateway/test/utils/fixtures.js b/packages/edge-gateway/test/utils/fixtures.js index 956b7e1..b68b0b9 100644 --- a/packages/edge-gateway/test/utils/fixtures.js +++ b/packages/edge-gateway/test/utils/fixtures.js @@ -24,6 +24,10 @@ export async function addFixtures (port) { // bafkreifbh4or5yoti7bahifd3gwx5m2qiwmrvpxsx3nsquf7r4wwkiruve client.add('Hello dot.storage! 😎😎😎', { rawLeaves: true + }), + // bafkreibx45dh23bkcli5qxevg2zq5pa7dbzpdd45h4uugk6qgjlyifulj4 + client.add('MALICIOUS', { + rawLeaves: true }) ]) @@ -37,6 +41,19 @@ export async function addFixtures (port) { cidVersion: 1 } // eslint-disable-next-line no-empty - )) { - } + )) {} + + // bafybeiaekuoonpqpmems3uapy27zsas5p6ylku53lzkaufnvt4s5n6a7au - / + // bafkreib6uzgr2noyzup3uuqcp6gafddnx6n3iinkyflbrkhdhfpcoggc5u - /sample.html + // bafkreibv3ecfm3wpoawshuqhir3cn2w4dewlr6jit3hfx6cjqgmzbsq22y - /malware.txt + // eslint-disable-next-line no-unused-vars + for await (const _ of client.addAll( + globSource(path.join(__dirname, '../fixtures/website'), '**/*'), + { + rawLeaves: true, + wrapWithDirectory: true, + cidVersion: 1 + } + // eslint-disable-next-line no-empty + )) {} } diff --git a/packages/edge-gateway/test/utils/miniflare.js b/packages/edge-gateway/test/utils/miniflare.js index e37afa4..58dc05a 100644 --- a/packages/edge-gateway/test/utils/miniflare.js +++ b/packages/edge-gateway/test/utils/miniflare.js @@ -25,10 +25,16 @@ export function getMiniflare () { api: { scriptPath: './test/utils/scripts/api.js', modules: true + }, + cid_verifier: { + scriptPath: './test/utils/scripts/cid-verifier.js', + modules: true, + kvNamespaces: ['TEST_NAMESPACE'] } }, serviceBindings: { - API: 'api' + API: 'api', + CID_VERIFIER: 'cid_verifier' }, bindings: { PUBLIC_RACE_WINNER: createAnalyticsEngine(), diff --git a/packages/edge-gateway/test/utils/scripts/cid-verifier.js b/packages/edge-gateway/test/utils/scripts/cid-verifier.js new file mode 100644 index 0000000..84d83c0 --- /dev/null +++ b/packages/edge-gateway/test/utils/scripts/cid-verifier.js @@ -0,0 +1,41 @@ +/* eslint-env serviceworker, browser */ + +const maliciousRootCid = 'bafkreibx45dh23bkcli5qxevg2zq5pa7dbzpdd45h4uugk6qgjlyifulj4' +const maliciousResourceCid = 'bafkreibv3ecfm3wpoawshuqhir3cn2w4dewlr6jit3hfx6cjqgmzbsq22y' +const htmlDirectoryRootCid = 'bafybeiaekuoonpqpmems3uapy27zsas5p6ylku53lzkaufnvt4s5n6a7au' +const htmlResourceCid = 'bafkreib6uzgr2noyzup3uuqcp6gafddnx6n3iinkyflbrkhdhfpcoggc5u' + +export default { + /** + * @param {Request} request + * @param {any} env + */ + async fetch (request, env) { + // GET for denylist validation + if ( + request.method === 'GET' && + (request.url.includes(maliciousRootCid) || request.url.includes(maliciousResourceCid)) + ) { + return new Response('malicious', { + status: 403 + }) + } + + // POST for cid validation + if ( + request.method === 'POST' && + request.url.includes(htmlDirectoryRootCid) + ) { + await env.TEST_NAMESPACE.put(`${htmlDirectoryRootCid}/test.lock`, 'LOCK') + } else if ( + request.method === 'POST' && + request.url.includes(htmlResourceCid) + ) { + await env.TEST_NAMESPACE.put(`${htmlResourceCid}/test.lock`, 'LOCK') + } + + return new Response('', { + status: 204 + }) + } +} diff --git a/packages/edge-gateway/wrangler.toml b/packages/edge-gateway/wrangler.toml index 8336bb9..166ddd3 100644 --- a/packages/edge-gateway/wrangler.toml +++ b/packages/edge-gateway/wrangler.toml @@ -29,11 +29,11 @@ routes = [ { pattern = "*.ipns.dag.haus/*", zone_id = "f2f8a5b1c557202c6e3d0ce0e98e4c8e" }, { pattern = "*.ipns.dag.haus", zone_id = "f2f8a5b1c557202c6e3d0ce0e98e4c8e" } ] -kv_namespaces = [{ binding = "DENYLIST", id = "785cf627e913468ca5319523ae929def" }] [env.production.vars] IPFS_GATEWAYS = "[\"https://ipfs.io\", \"https://cf.dag.haus\"]" GATEWAY_HOSTNAME = 'ipfs.dag.haus' +CID_VERIFIER_URL = 'https://cid-verifier.dag.haus' EDGE_GATEWAY_API_URL = 'https://api.nftstorage.link' DEBUG = "false" ENV = "production" @@ -45,6 +45,12 @@ type = "service" service = "nftstorage-link-api-production" environment = "production" +[[env.production.services]] +binding = "CID_VERIFIER" +type = "service" +service = "dotstorage-cid-verifier-production" +environment = "production" + [[env.production.unsafe.bindings]] type = "analytics_engine" dataset = "PUBLIC_RACE_WINNER_PRODUCTION" @@ -62,7 +68,7 @@ name = "PUBLIC_RACE_STATUS_CODE" # Staging! [env.staging] -# name = "gateway-nft-storage-staging" +# name = "gateway-edge-gateway-staging" account_id = "fffa4b4363a7e5250af8357087263b3a" # Protocol Labs CF account routes = [ { pattern = "*.ipfs-staging.dag.haus/*", zone_id = "f2f8a5b1c557202c6e3d0ce0e98e4c8e" }, @@ -71,11 +77,11 @@ routes = [ { pattern = "*.ipns-staging.dag.haus/*", zone_id = "f2f8a5b1c557202c6e3d0ce0e98e4c8e" }, { pattern = "*.ipns-staging.dag.haus", zone_id = "f2f8a5b1c557202c6e3d0ce0e98e4c8e" } ] -kv_namespaces = [{ binding = "DENYLIST", id = "f4eb0eca32e14e28b643604a82e00cb3" }] [env.staging.vars] IPFS_GATEWAYS = "[\"https://ipfs.io\", \"https://cf.dag.haus\", \"https://pinata.dag.haus\"]" GATEWAY_HOSTNAME = 'ipfs-staging.dag.haus' +CID_VERIFIER_URL = 'https://cid-verifier-staging.dag.haus' EDGE_GATEWAY_API_URL = 'https://api.nftstorage.link' DEBUG = "true" ENV = "staging" @@ -87,6 +93,12 @@ type = "service" service = "nftstorage-link-api-staging" environment = "production" +[[env.staging.services]] +binding = "CID_VERIFIER" +type = "service" +service = "dotstorage-cid-verifier-staging" +environment = "production" + [[env.staging.unsafe.bindings]] type = "analytics_engine" dataset = "PUBLIC_RACE_WINNER_STAGING" @@ -105,11 +117,11 @@ name = "PUBLIC_RACE_STATUS_CODE" # Test! [env.test] workers_dev = true -kv_namespaces = [{ binding = "DENYLIST" }] [env.test.vars] IPFS_GATEWAYS = "[\"http://127.0.0.1:9081\", \"http://localhost:9082\", \"http://localhost:9083\"]" GATEWAY_HOSTNAME = 'ipfs.localhost:8787' +CID_VERIFIER_URL = 'http://cid-verifier.localhost:8787' EDGE_GATEWAY_API_URL = 'http://localhost:8787' DEBUG = "true" ENV = "test" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 9089042..a55e907 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -16,6 +16,49 @@ importers: simple-git-hooks: 2.8.0 wrangler: 2.0.23 + packages/cid-verifier: + specifiers: + '@cloudflare/workers-types': ^3.7.1 + '@sentry/cli': ^1.71.0 + '@types/git-rev-sync': ^2.0.0 + '@web-std/fetch': ^4.0.0 + '@web3-storage/worker-utils': ^0.3.0-dev + ava: ^3.15.0 + esbuild: ^0.14.2 + git-rev-sync: ^3.0.1 + itty-router: ^2.4.5 + miniflare: ^2.5.0 + multiformats: ^9.6.4 + npm-run-all: ^4.1.5 + p-retry: ^5.0.0 + sade: ^1.7.4 + smoke: ^3.1.1 + standard: ^17.0.0 + toucan-js: ^2.5.0 + typescript: 4.7.3 + uint8arrays: ^3.0.0 + dependencies: + '@web3-storage/worker-utils': 0.3.0-dev + itty-router: 2.6.1 + multiformats: 9.7.0 + p-retry: 5.1.1 + toucan-js: 2.6.1 + uint8arrays: 3.0.0 + devDependencies: + '@cloudflare/workers-types': 3.14.0 + '@sentry/cli': 1.74.4 + '@types/git-rev-sync': 2.0.0 + '@web-std/fetch': 4.1.0 + ava: 3.15.0 + esbuild: 0.14.48 + git-rev-sync: 3.0.2 + miniflare: 2.6.0 + npm-run-all: 4.1.5 + sade: 1.8.1 + smoke: 3.1.1 + standard: 17.0.0 + typescript: 4.7.3 + packages/edge-gateway: specifiers: '@cloudflare/workers-types': ^3.7.1 @@ -54,7 +97,7 @@ importers: uint8arrays: ^3.0.0 dependencies: '@web3-storage/worker-utils': 0.3.0-dev - ipfs-core-utils: 0.15.1 + ipfs-core-utils: 0.15.1_undici@5.5.1 ipfs-gateway-race: link:../ipfs-gateway-race itty-router: 2.6.1 multiformats: 9.7.0 @@ -259,11 +302,11 @@ packages: dependencies: multiformats: 9.7.0 - /@libp2p/interfaces/2.0.4: + /@libp2p/interfaces/2.0.4_undici@5.5.1: resolution: {integrity: sha512-MfwkTFyHJtvwNxkjOjzkXyIVvKFtEW2Q3IGRJPyPQMrtB6ll0rGMTlyJ3BQS1bcD0YkNhggFm+8XiU2/0LCBhQ==} engines: {node: '>=16.0.0', npm: '>=7.0.0'} dependencies: - '@multiformats/multiaddr': 10.3.3 + '@multiformats/multiaddr': 10.3.3_undici@5.5.1 err-code: 3.0.1 interface-datastore: 6.1.1 it-pushable: 2.0.2 @@ -274,11 +317,11 @@ packages: - undici dev: false - /@libp2p/logger/1.1.6: + /@libp2p/logger/1.1.6_undici@5.5.1: resolution: {integrity: sha512-ZKoRUt7cyHlbxHYDZ1Fn3A+ByqGABdmd4z07+1TfVvpEQSpn2IVcV0mt6ff5kUUtGuVeSrqK1/ZDzWqhgg56vg==} engines: {node: '>=16.0.0', npm: '>=7.0.0'} dependencies: - '@libp2p/interfaces': 2.0.4 + '@libp2p/interfaces': 2.0.4_undici@5.5.1 debug: 4.3.4 interface-datastore: 6.1.1 multiformats: 9.7.0 @@ -585,20 +628,20 @@ packages: - utf-8-validate dev: true - /@multiformats/multiaddr-to-uri/9.0.1: + /@multiformats/multiaddr-to-uri/9.0.1_undici@5.5.1: resolution: {integrity: sha512-kSyHZ2lKjoEzHu/TM4ZVwFj4AWV1B9qFBFJjYb/fK1NqrnrNb/M3uhoyckJvP7WZvpDsnEc7fUCpmPipDY6LMw==} dependencies: - '@multiformats/multiaddr': 10.3.3 + '@multiformats/multiaddr': 10.3.3_undici@5.5.1 transitivePeerDependencies: - supports-color - undici dev: false - /@multiformats/multiaddr/10.3.3: + /@multiformats/multiaddr/10.3.3_undici@5.5.1: resolution: {integrity: sha512-+LX9RovG7DJsANb+U6VchV/tApcdJzeafbi5+MPUam90oL91BbEh6ozNZOz4Qf5ZEeilexc12oomatmODJh1/w==} engines: {node: '>=16.0.0', npm: '>=7.0.0'} dependencies: - dns-over-http-resolver: 2.1.0 + dns-over-http-resolver: 2.1.0_undici@5.5.1 err-code: 3.0.1 is-ip: 4.0.0 multiformats: 9.7.0 @@ -2007,12 +2050,12 @@ packages: - supports-color dev: true - /dns-over-http-resolver/2.1.0: + /dns-over-http-resolver/2.1.0_undici@5.5.1: resolution: {integrity: sha512-eb8RGy6k54JdD7Rjw8g65y1MyA4z3m3IIYh7uazkgZuKIdFn8gYt8dydMm3op+2UshDdk9EexrXcDluKNY/CDg==} engines: {node: '>=16.0.0', npm: '>=7.0.0'} dependencies: debug: 4.3.4 - native-fetch: 4.0.2 + native-fetch: 4.0.2_undici@5.5.1 receptacle: 1.3.2 transitivePeerDependencies: - supports-color @@ -3631,11 +3674,11 @@ packages: engines: {node: '>= 0.10'} dev: true - /ipfs-core-types/0.11.1: + /ipfs-core-types/0.11.1_undici@5.5.1: resolution: {integrity: sha512-K7ZSx9EPEvT4At2kExKyMCqY4Jo3Nb/VnC/iSWqFKRaqb0MTB4IJgvWrwwDuN541tn+dvUnTfOp2wWQSov1UAw==} dependencies: '@ipld/dag-pb': 2.1.17 - '@multiformats/multiaddr': 10.3.3 + '@multiformats/multiaddr': 10.3.3_undici@5.5.1 interface-datastore: 6.1.1 ipfs-unixfs: 6.0.9 multiformats: 9.7.0 @@ -3683,17 +3726,17 @@ packages: - supports-color dev: true - /ipfs-core-utils/0.15.1: + /ipfs-core-utils/0.15.1_undici@5.5.1: resolution: {integrity: sha512-nZmUiLctJWMFrEciVkKdDUO2xLpXWy7Ilt0VMJ35Y5+OJznCXxMHUQo1WUALATlo9ziHgDdHFrAUuyW0yB2rww==} dependencies: - '@libp2p/logger': 1.1.6 - '@multiformats/multiaddr': 10.3.3 - '@multiformats/multiaddr-to-uri': 9.0.1 + '@libp2p/logger': 1.1.6_undici@5.5.1 + '@multiformats/multiaddr': 10.3.3_undici@5.5.1 + '@multiformats/multiaddr-to-uri': 9.0.1_undici@5.5.1 any-signal: 3.0.1 blob-to-it: 1.0.4 browser-readablestream-to-it: 1.0.3 err-code: 3.0.1 - ipfs-core-types: 0.11.1 + ipfs-core-types: 0.11.1_undici@5.5.1 ipfs-unixfs: 6.0.9 ipfs-utils: 9.0.7 it-all: 1.0.6 @@ -4779,10 +4822,12 @@ packages: node-fetch: 3.2.6 dev: true - /native-fetch/4.0.2: + /native-fetch/4.0.2_undici@5.5.1: resolution: {integrity: sha512-4QcVlKFtv2EYVS5MBgsGX5+NWKtbDbIECdUXDBGDMAZXq3Jkv9zf+y8iS7Ub8fEdga3GpYeazp9gauNqXHJOCg==} peerDependencies: undici: '*' + dependencies: + undici: 5.5.1 dev: false /natural-compare/1.4.0: @@ -6590,7 +6635,6 @@ packages: /undici/5.5.1: resolution: {integrity: sha512-MEvryPLf18HvlCbLSzCW0U00IMftKGI5udnjrQbC5D4P0Hodwffhv+iGfWuJwg16Y/TK11ZFK8i+BPVW2z/eAw==} engines: {node: '>=12.18'} - dev: true /unique-string/2.0.0: resolution: {integrity: sha512-uNaeirEPvpZWSgzwsPGtU2zVSTrn/8L5q/IexZmH0eH6SA73CmAA5U4GwORTxQAZs95TAXLNqeLoPPNO5gZfWg==} @@ -6805,7 +6849,7 @@ packages: /wide-align/1.1.5: resolution: {integrity: sha512-eDMORYaPNZ4sQIuuYPDHdQvf4gyCF9rEEV/yPxGfwPkRodwEgiMUUXTx/dex+Me0wxx53S+NgUHaP7y3MGlDmg==} dependencies: - string-width: 1.0.2 + string-width: 4.2.3 dev: true /widest-line/3.1.0: