feat: adding google malware detection (#36)

Co-authored-by: Vasco Santos <santos.vasco10@gmail.com>

.github/workflows/cid-verifier.yml

@@ -0,0 +1,107 @@
+name: cid-verifier
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - 'packages/cid-verifier/**'
+      - '.github/workflows/cid-verifier.yml'
+      - 'pnpm-lock.yaml'
+  pull_request:
+    paths:
+      - 'packages/cid-verifier/**'
+      - '.github/workflows/cid-verifier.yml'
+      - 'pnpm-lock.yaml'
+jobs:
+  check:
+    runs-on: ubuntu-latest
+    name: Test
+    steps:
+      - uses: actions/checkout@v2
+      - uses: pnpm/action-setup@v2.0.1
+        with:
+          version: 6.32.x
+      - uses: actions/setup-node@v2
+      - run: pnpm install
+      - run: pnpm build:cid-verifier
+      - name: Lint
+        run: pnpm lint
+  test:
+    runs-on: ubuntu-latest
+    name: Test
+    steps:
+      - uses: actions/checkout@v2
+      - uses: pnpm/action-setup@v2.0.1
+        with:
+          version: 6.32.x
+      - uses: actions/setup-node@v2
+      - run: pnpm install
+      - run: pnpm test:cid-verifier
+  deploy-staging:
+    name: Deploy Staging
+    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+    runs-on: ubuntu-latest
+    needs: test
+    steps:
+      - uses: actions/checkout@v2
+      - uses: pnpm/action-setup@v2.0.1
+        with:
+          version: 6.32.x
+      - uses: actions/setup-node@v2
+        with:
+          cache: 'pnpm'
+      - run: pnpm install
+      - name: Publish app
+        uses: cloudflare/wrangler-action@2.0.0
+        env:
+          ENV: 'staging' # inform the build process what the env is
+          SENTRY_TOKEN: ${{secrets.SENTRY_TOKEN}}
+          SENTRY_UPLOAD: ${{ secrets.SENTRY_UPLOAD }}
+        with:
+          apiToken: ${{secrets.CF_GATEWAY_TOKEN }}
+          workingDirectory: 'packages/cid-verifier'
+          environment: 'staging'
+  changelog:
+    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+    name: Release
+    runs-on: ubuntu-latest
+    outputs:
+      releases_created: ${{ steps.tag-release.outputs.releases_created }}
+    steps:
+      - uses: GoogleCloudPlatform/release-please-action@v3
+        id: tag-release
+        with:
+          path: packages/cid-verifier
+          token: ${{ secrets.GITHUB_TOKEN }}
+          release-type: node
+          monorepo-tags: true
+          package-name: cid-verifier
+  release:
+    if: github.event_name == 'push' && github.ref == 'refs/heads/main' && needs.changelog.outputs.releases_created
+    name: Release
+    runs-on: ubuntu-latest
+    needs:
+      - check
+      - test
+      - changelog
+    steps:
+      - uses: actions/checkout@v2
+      - uses: pnpm/action-setup@v2.0.1
+        with:
+          version: 6.32.x
+      - uses: actions/setup-node@v2
+        with:
+          cache: 'pnpm'
+          node-version: 16
+          registry-url: 'https://registry.npmjs.org'
+      - run: pnpm install
+      - name: Deploy
+        uses: cloudflare/wrangler-action@2.0.0
+        env:
+          ENV: 'production' # inform the build process what the env is
+          SENTRY_TOKEN: ${{ secrets.SENTRY_TOKEN }}
+          SENTRY_UPLOAD: ${{ secrets.SENTRY_UPLOAD }}
+        with:
+          apiToken: ${{ secrets.CF_GATEWAY_TOKEN }}
+          workingDirectory: 'packages/cid-verifier'
+          environment: 'production'

package.json

@@ -8,17 +8,21 @@
   "license": "Apache-2.0 OR MIT",
   "scripts": {
     "lint": "run-s lint:no-fix-*",
+    "lint:no-fix-cid-verifier": "pnpm --filter cid-verifier lint",
     "lint:no-fix-edge-gateway": "pnpm --filter edge-gateway lint",
     "lint:no-fix-ipfs-gateway-race": "pnpm --filter ipfs-gateway-race lint",
     "lint:fix": "run-s lint:fix-*",
+    "lint:fix-cid-verifier": "pnpm --filter cid-verifier lint --fix",
     "lint:fix-edge-gateway": "pnpm --filter edge-gateway lint --fix",
     "lint:fix-ipfs-gateway-race": "pnpm --filter ipfs-gateway-race lint --fix",
     "build": "run-s build:*",
+    "build:cid-verifier": "pnpm --filter cid-verifier build",
     "build:edge-gateway": "pnpm --filter edge-gateway build",
     "build:ipfs-gateway-race": "pnpm --filter ipfs-gateway-race build",
     "test": "run-s test:*",
-    "test:ipfs-gateway-race": "pnpm --filter ipfs-gateway-race test",
+    "test:cid-verifier": "pnpm --filter cid-verifier test",
     "test:edge-gateway": "pnpm --filter edge-gateway test",
+    "test:ipfs-gateway-race": "pnpm --filter ipfs-gateway-race test",
     "clean": "rm -rf node_modules pnpm-lock.yml packages/*/{pnpm-lock.yml,.next,out,coverage,.nyc_output,worker,dist,node_modules}"
   },
   "simple-git-hooks": {

packages/cid-verifier/ava.config.js

@@ -0,0 +1,6 @@
+export default {
+  files: ['test/*.spec.js'],
+  timeout: '5m',
+  concurrency: 1,
+  nodeArguments: ['--experimental-vm-modules']
+}

packages/cid-verifier/package.json

@@ -0,0 +1,47 @@
+{
+  "name": "cid-verifier",
+  "version": "1.0.0",
+  "description": "Verify that a CID's content is safe for the web",
+  "private": true,
+  "type": "module",
+  "main": "./dist/worker.js",
+  "scripts": {
+    "lint": "standard",
+    "build": "tsc && node scripts/cli.js build",
+    "dev": "miniflare dist/worker.js --watch --debug -m",
+    "test": "npm run test:setup && npm-run-all -p -r mock:google-cloud.io test:worker",
+    "test:worker": "ava --verbose test/*.spec.js",
+    "test:setup": "npm run build",
+    "mock:google-cloud.io": "smoke -p 9111 test/mocks/google-cloud.io"
+  },
+  "dependencies": {
+    "@web3-storage/worker-utils": "^0.3.0-dev",
+    "itty-router": "^2.4.5",
+    "multiformats": "^9.6.4",
+    "p-retry": "^5.0.0",
+    "toucan-js": "^2.5.0",
+    "uint8arrays": "^3.0.0"
+  },
+  "devDependencies": {
+    "@cloudflare/workers-types": "^3.7.1",
+    "@sentry/cli": "^1.71.0",
+    "@types/git-rev-sync": "^2.0.0",
+    "@web-std/fetch": "^4.0.0",
+    "ava": "^3.15.0",
+    "esbuild": "^0.14.2",
+    "git-rev-sync": "^3.0.1",
+    "miniflare": "^2.5.0",
+    "npm-run-all": "^4.1.5",
+    "sade": "^1.7.4",
+    "smoke": "^3.1.1",
+    "standard": "^17.0.0",
+    "typescript": "4.7.3"
+  },
+  "standard": {
+    "ignore": [
+      "dist"
+    ]
+  },
+  "author": "jsdevel <jsdevel@kogo.io>",
+  "license": "Apache-2.0 OR MIT"
+}

packages/cid-verifier/scripts/build.js

@@ -0,0 +1,60 @@
+import fs from 'fs'
+import path from 'path'
+import { fileURLToPath } from 'url'
+import { build } from 'esbuild'
+import git from 'git-rev-sync'
+import Sentry from '@sentry/cli'
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url))
+const pkg = JSON.parse(
+  fs.readFileSync(path.join(__dirname, '..', 'package.json'), 'utf8')
+)
+
+export async function buildCmd (opts) {
+  const sentryRelease = `cid-verifier@${pkg.version}-${opts.env}+${git.short(
+    __dirname
+  )}`
+  console.log(`Building ${sentryRelease}`)
+
+  await build({
+    entryPoints: [path.join(__dirname, '..', 'src', 'index.js')],
+    bundle: true,
+    format: 'esm',
+    outfile: path.join(__dirname, '..', 'dist', 'worker.js'),
+    legalComments: 'external',
+    define: {
+      SENTRY_RELEASE: JSON.stringify(sentryRelease),
+      VERSION: JSON.stringify(pkg.version),
+      COMMITHASH: JSON.stringify(git.long(__dirname)),
+      BRANCH: JSON.stringify(git.branch(__dirname)),
+      global: 'globalThis'
+    },
+    minify: opts.env !== 'dev',
+    sourcemap: 'external'
+  })
+
+  // Sentry release and sourcemap upload
+  if (process.env.SENTRY_UPLOAD === 'true') {
+    const cli = new Sentry(undefined, {
+      authToken: process.env.SENTRY_TOKEN,
+      org: 'protocol-labs-it',
+      project: 'cid-verifier',
+      dist: git.short(__dirname)
+    })
+
+    await cli.releases.new(sentryRelease)
+    await cli.releases.setCommits(sentryRelease, {
+      auto: true,
+      ignoreEmpty: true,
+      ignoreMissing: true
+    })
+    await cli.releases.uploadSourceMaps(sentryRelease, {
+      include: [path.join(__dirname, '..', 'dist')],
+      ext: ['map', 'js']
+    })
+    await cli.releases.finalize(sentryRelease)
+    await cli.releases.newDeploy(sentryRelease, {
+      env: opts.env
+    })
+  }
+}

packages/cid-verifier/scripts/cli.js

@@ -0,0 +1,16 @@
+#!/usr/bin/env node
+
+import sade from 'sade'
+
+import { buildCmd } from './build.js'
+
+const env = process.env.ENV || 'dev'
+const prog = sade('cid-verifier')
+
+prog
+  .command('build')
+  .describe('Build the worker.')
+  .option('--env', 'Environment', env)
+  .action(buildCmd)
+
+prog.parse(process.argv)

packages/cid-verifier/src/bindings.d.ts

@@ -0,0 +1,45 @@
+import Toucan from "toucan-js";
+import { Logging } from "@web3-storage/worker-utils/loki";
+
+export {};
+
+export interface EnvInput {
+  ENV: string;
+  DEBUG: string;
+  GOOGLE_EVALUATE_SAFE_CONFIDENCE_LEVELS: Array<string>;
+  GOOGLE_CLOUD_API_URL: string;
+  GOOGLE_CLOUD_API_KEY: string;
+  SENTRY_DSN?: string;
+  LOKI_URL?: string;
+  LOKI_TOKEN?: string;
+  DENYLIST: KVNamespace;
+  CID_VERIFIER_RESULTS: KVNamespace;
+}
+
+export interface EnvTransformed {
+  VERSION: string;
+  BRANCH: string;
+  COMMITHASH: string;
+  SENTRY_RELEASE: string;
+  sentry?: Toucan;
+  log: Logging;
+}
+
+export type Env = EnvInput & EnvTransformed;
+
+export interface GoogleEvaluateAPIResultScore {
+  threatType: 'THREAT_TYPE_UNSPECIFIED' | 'SOCIAL_ENGINEERING' | 'UNWANTED_SOFTWARE' | 'MALWARE';
+  confidenceLevel: 'CONFIDENCE_LEVEL_UNSPECIFIED' | 'SAFE' | 'LOW' | 'MEDIUM' | 'HIGH' | 'VERY_HIGH' | 'EXTREMELY_HIGH';
+}
+
+export interface GoogleEvaluateAPIResult {
+  scores: Array<GoogleEvaluateAPIResultScore>;
+}
+
+declare global {
+  const BRANCH: string;
+  const VERSION: string;
+  const COMMITHASH: string;
+  const SENTRY_RELEASE: string;
+  const ENV: string;
+}

packages/cid-verifier/src/cors.js

@@ -0,0 +1,34 @@
+/* eslint-env serviceworker */
+
+/**
+ * @param {import('itty-router').RouteHandler<Request>} handler
+ */
+export function withCorsHeaders (handler) {
+  /**
+   * @param {Request} request
+   * @returns {Promise<Response>}
+   */
+  return async (request, /** @type {any} */ ...rest) => {
+    const response = await handler(request, ...rest)
+    return addCorsHeaders(request, response)
+  }
+}
+
+/**
+ * @param {Request} request
+ * @param {Response} response
+ * @returns {Response}
+ */
+export function addCorsHeaders (request, response) {
+  // Clone the response so that it's no longer immutable (like if it comes from cache or fetch)
+  response = new Response(response.body, response)
+  const origin = request.headers.get('origin')
+  if (origin) {
+    response.headers.set('Access-Control-Allow-Origin', origin)
+    response.headers.set('Vary', 'Origin')
+  } else {
+    response.headers.set('Access-Control-Allow-Origin', '*')
+  }
+  response.headers.set('Access-Control-Expose-Headers', 'Link')
+  return response
+}