Adding google malware detection.

* For now adding this to edge-gateway.  Updates our existing env.DENYLIST store.
* TODO: Add a smoke server for local testing and add tests for pass/fail scenarios.

packages/edge-gateway/src/bindings.d.ts

@@ -19,6 +19,8 @@ export interface EnvInput {
   DEBUG: string
   IPFS_GATEWAYS: string
   GATEWAY_HOSTNAME: string
+  GOOGLE_CLOUD_API_URL: string
+  GOOGLE_CLOUD_API_KEY: string
   EDGE_GATEWAY_API_URL: string
   REQUEST_TIMEOUT?: number
   SENTRY_DSN?: string

packages/edge-gateway/src/gateway.js

@@ -110,6 +110,13 @@ export async function gatewayGet (request, env, ctx) {
       const { status, reason } = JSON.parse(denyListResource)
       return new Response(reason || '', { status: status || 410 })
     }
+
+    if (winnerGwResponse.headers.get('content-type') === 'text/html') {
+      // we don't wait for google because it adds to the overall latency, but in the
+      // background we'll update KV so future requests coming in will properly be
+      // blocked as CIDs/malware are flagged.
+      fetchGoogleMalwareDetection(winnerUrl, cid, env)
+    }
   }
 
   // Cache response
@@ -274,6 +281,37 @@ async function getFromDenyList (datastore, cid) {
   )
 }
 
+/**
+ * Fetch malware state from google's malware detection API and update our DENYLIST.
+ *
+ * TODO: notify badbits
+ *
+ * @param {string} url
+ * @param {string} originalRequestCid
+ * @param {Env} env
+ */
+async function fetchGoogleMalwareDetection (url, originalRequestCid, env) {
+  const response = await fetch(
+    `${env.GOOGLE_CLOUD_API_URL}/v1/uris:search?threatTypes=MALWARE&threatTypes=SOCIAL_ENGINEERING&threatTypes=UNWANTED_SOFTWARE&uri=${encodeURIComponent(url)}&key=${env.GOOGLE_CLOUD_API_KEY}`
+  )
+
+  if (response.status === 200) {
+    const jsonBody = await response.json()
+    /* Example body
+    {
+      threat: {
+        threatTypes: [ 'MALWARE' ],
+        expireTime: '2022-08-28T07:54:04.936398042Z'
+      }
+    }
+    */
+    if (jsonBody.threat) {
+      const anchor = await toDenyListAnchor(originalRequestCid)
+      env.DENYLIST.put(anchor, { status: 410, reason: 'This was flagged by Google\'s malware API' })
+    }
+  }
+}
+
 /**
  * Put receives response to cache.
  *

packages/edge-gateway/wrangler.toml

@@ -108,6 +108,9 @@ workers_dev = true
 kv_namespaces = [{ binding = "DENYLIST" }]
 
 [env.test.vars]
+# TODO: setup smoke server for tests
+GOOGLE_CLOUD_API_URL = 'https://webrisk.googleapis.com'
+GOOGLE_CLOUD_API_KEY = 'TODO'
 IPFS_GATEWAYS = "[\"http://127.0.0.1:9081\", \"http://localhost:9082\", \"http://localhost:9083\"]"
 GATEWAY_HOSTNAME = 'ipfs.localhost:8787'
 EDGE_GATEWAY_API_URL = 'http://localhost:8787'