feat: denylist update remote command and workflow

.github/workflows/denylist-update.yml

@@ -0,0 +1,39 @@
+name: Update remote denylist
+
+on:
+  workflow_dispatch:
+    inputs:
+      url:
+        required: true
+        description: URL of the file to use to update denylist
+        type: string
+      operation:
+        required: true
+        description: Operation to be performed in the remote denylist. Default is add
+        options: 
+          - add
+          - delete
+        default: add
+
+jobs:
+  update:
+    name: Update remote denylist
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        env: ['staging', 'production']
+    timeout-minutes: 20
+    steps:
+      - uses: actions/checkout@v2
+      - uses: pnpm/action-setup@v2.0.1
+        with:
+          version: 6.32.x
+      - uses: actions/setup-node@v2
+        with:
+          node-version: 16
+          cache: 'pnpm'
+      - run: pnpm install
+      - name: Run job
+        env:
+          CF_API_TOKEN: ${{ secrets.CF_GATEWAY_TOKEN }}
+        run: node packages/denylist/scripts/cli.js denylist update-remote ${{ github.event.inputs.url }} --env ${{ matrix.env }} --operation ${{ github.event.inputs.operation }}

packages/denylist/scripts/cli.js

@@ -3,7 +3,7 @@
 import sade from 'sade'
 
 import { buildCmd } from './build.js'
-import { denylistSyncCmd, denylistAddCmd } from './denylist.js'
+import { denylistSyncCmd, denylistAddCmd, denylistUpdateRemoteCmd } from './denylist.js'
 
 const env = process.env.ENV || 'dev'
 const prog = sade('denylist')
@@ -24,5 +24,12 @@ prog
   .option('--status', 'HTTP status to send in response.')
   .option('--reason', 'Reason for deny. Note: may be communicated in response')
   .action(denylistAddCmd)
+  .command('denylist update-remote <url>')
+  .option('--env', 'Wrangler environment to use.', env)
+  .option('--operation', 'Operation to be performed in the remote denylist for items in list ("add" or "delete")', 'add')
+  .describe(
+    'Add the content of a given file to the remote deny list.'
+  )
+  .action(denylistUpdateRemoteCmd)
 
 prog.parse(process.argv)

packages/denylist/scripts/denylist.js

@@ -22,28 +22,27 @@ const DENY_LIST_SOURCES = [
   denyListPath
 ]
 
-export async function denylistSyncCmd ({ env }) {
-  const cfApiToken = mustGetEnv('CF_API_TOKEN')
-
-  const wranglerConfig = await getWranglerToml(wranglerConfigPath)
-  const wranglerEnvConfig = wranglerConfig.env[env]
-  if (!wranglerEnvConfig) {
-    throw new Error(`missing wrangler configuration for env: ${env}`)
-  }
-  console.log(`🧩 using wrangler config: ${wranglerConfigPath}`)
+export async function denylistAddCmd (cid, options) {
+  console.log(`🦴 fetching ${denyListPath}`)
+  const denyList = await getDenyList(denyListPath)
+  const entry = await cidToAnchor(cid, options)
 
-  const cfAccountId = wranglerEnvConfig.account_id
-  if (!cfAccountId) {
-    throw new Error(`missing Cloudflare account_id in env: ${env}`)
+  if (denyList.some((e) => e.anchor === entry.anchor)) {
+    throw new Error('already exists')
   }
-  console.log(`🏕 using env: ${env} (${cfAccountId})`)
+  denyList.push(entry)
+  console.log('📝 writing update')
+  fs.writeFileSync(
+    denyListPath.replace('file://', ''),
+    JSON.stringify(denyList, null, 2)
+  )
+  console.log('✅ Done')
+}
 
-  const kvNamespaces = wranglerEnvConfig.kv_namespaces || []
-  const denyListKv = kvNamespaces.find((kv) => kv.binding === 'DENYLIST')
-  if (!denyListKv) {
-    throw new Error('missing binding in kv_namespaces: DENYLIST')
-  }
-  console.log(`🪢 using KV binding: DENYLIST (${denyListKv.id})`)
+export async function denylistSyncCmd ({ env }) {
+  const {
+    cfApiToken, cfAccountId, denyListKv
+  } = await getDenylistProperties({ env })
 
   for (const url of DENY_LIST_SOURCES) {
     console.log(`🦴 fetching ${url}`)
@@ -58,48 +57,56 @@ export async function denylistSyncCmd ({ env }) {
   console.log('✅ Done')
 }
 
-async function getWranglerToml (url) {
-  const res = await fetch(url)
-  if (!res.ok) {
-    throw new Error(`unexpected status fetching wrangler.toml: ${res.status}`)
+export async function denylistUpdateRemoteCmd (url, { env, reason, operation }) {
+  if (operation !== 'add' && operation !== 'delete') {
+    throw new Error(`operation must be one of {add, delete}. Received: ${operation}`)
   }
-  return toml.parse(await res.text())
-}
-
-async function getDenyList (url) {
-  const headers = new Headers()
-  headers.append('cache-control', 'no-cache')
-  headers.append('pragma', 'no-cache')
-
-  const res = await fetch(url, {
-    headers
+  const {
+    cfApiToken, cfAccountId, denyListKv
+  } = await getDenylistProperties({ env })
+
+  const entries = await getListFromUrl(url)
+  const bulk = await Promise.all(entries.map(async entry => {
+    return {
+      key: (await cidToAnchor(entry)).anchor,
+      value: { reason }
+    }
+  }))
+  console.log(`update ${bulk.length} remote denylist: ${operation}`)
+  await writeKVMulti(cfApiToken, cfAccountId, denyListKv.id, bulk, {
+    delete: operation === 'delete'
   })
-  if (!res.ok) {
-    throw new Error(`unexpected status fetching denylist.json: ${res.status}`)
-  }
-  const list = await res.json()
-  return list
+  console.log('✅ Done')
 }
 
 /**
  * @param {string} apiToken Cloudflare API token
  * @param {string} accountId Cloudflare account ID
  * @param {string} nsId KV namespace ID
  * @param {Array<BulkWritePair>} kvs
- * @returns {Promise<{ success: boolean, errors: Array<{ code: number, message: string }>, messages: any[] }>}
+ * @param {object} [options]
+ * @param {boolean} [options.delete]
  */
-async function writeKVMulti (apiToken, accountId, nsId, kvs) {
+async function writeKVMulti (apiToken, accountId, nsId, kvs, options = {}) {
+  const isDelete = options.delete
   const url = `https://api.cloudflare.com/client/v4/accounts/${accountId}/storage/kv/namespaces/${nsId}/bulk`
-  kvs = kvs.map((kv) => ({
-    ...kv,
-    value: JSON.stringify(kv.value)
-  }))
+
+  // Delete
+  if (isDelete) {
+    kvs = kvs.map((kv) => kv.key)
+  // Add
+  } else {
+    kvs = kvs.map((kv) => ({
+      ...kv,
+      value: JSON.stringify(kv.value)
+    }))
+  }
 
   const chunkSize = 10000
   for (let i = 0; i < kvs.length; i += chunkSize) {
     const kvsChunk = kvs.slice(i, i + chunkSize)
     const res = await fetch(url, {
-      method: 'PUT',
+      method: isDelete ? 'DELETE' : 'PUT',
       headers: {
         Authorization: `Bearer ${apiToken}`,
         'Content-Type': 'application/json'
@@ -113,21 +120,36 @@ async function writeKVMulti (apiToken, accountId, nsId, kvs) {
         error ? `${error.code}: ${error.message}` : 'failed to write to KV'
       )
     }
+
+    // Delay to avoid going into
+    await new Promise(resolve => setTimeout(() => resolve(true), 10000))
   }
 }
 
-/**
- * @param {string} key
- * @returns {string}
- */
-function mustGetEnv (key) {
-  if (!process.env[key]) {
-    throw new Error(`missing environment variable: ${key}`)
+async function getListFromUrl (url) {
+  console.log(`🦴 fetching ${url}`)
+  const response = await fetch(url)
+  if (!response.ok) {
+    throw new Error(`unexpected status fetching given url: ${response.status}`)
+  }
+
+  let entries
+  const contentType = response.headers.get('content-type')
+  if (contentType === 'application/json') {
+    entries = await response.json()
+  } else if (contentType === 'text/plain') {
+    const textBody = await response.text()
+    // Separate entries by new line and trim them
+    entries = textBody.split(/\r?\n/)
+  } else {
+    console.log(`content type must be "application/json" or "text/plain". Received ${contentType}`)
+    process.exit(1)
   }
-  return process.env[key]
+
+  return entries
 }
 
-export async function denylistAddCmd (cid, options) {
+async function cidToAnchor (cid, options = {}) {
   const parts = cid.split('/')
   const cidv1 = CID.parse(parts[0]).toV1().toString(base32.encoder)
   const cidv1Path = `${cidv1}/${parts.slice(1).join('/')}`
@@ -139,17 +161,70 @@ export async function denylistAddCmd (cid, options) {
   const entry = { anchor, status: options.status, reason: options.reason }
   console.log(`🎫 entry: ${JSON.stringify(entry)}`)
 
-  console.log(`🦴 fetching ${denyListPath}`)
-  const denyList = await getDenyList(denyListPath)
-  if (denyList.some((e) => e.anchor === anchor)) {
-    throw new Error('already exists')
+  return entry
+}
+
+async function getDenylistProperties ({ env }) {
+  const cfApiToken = mustGetEnv('CF_API_TOKEN')
+
+  const wranglerConfig = await getWranglerToml(wranglerConfigPath)
+  const wranglerEnvConfig = wranglerConfig.env[env]
+  if (!wranglerEnvConfig) {
+    throw new Error(`missing wrangler configuration for env: ${env}`)
   }
+  console.log(`🧩 using wrangler config: ${wranglerConfigPath}`)
 
-  denyList.push(entry)
-  console.log('📝 writing update')
-  fs.writeFileSync(
-    denyListPath.replace('file://', ''),
-    JSON.stringify(denyList, null, 2)
-  )
-  console.log('✅ Done')
+  const cfAccountId = wranglerEnvConfig.account_id
+  if (!cfAccountId) {
+    throw new Error(`missing Cloudflare account_id in env: ${env}`)
+  }
+  console.log(`🏕 using env: ${env} (${cfAccountId})`)
+
+  const kvNamespaces = wranglerEnvConfig.kv_namespaces || []
+  const denyListKv = kvNamespaces.find((kv) => kv.binding === 'DENYLIST')
+  if (!denyListKv) {
+    throw new Error('missing binding in kv_namespaces: DENYLIST')
+  }
+  console.log(`🪢 using KV binding: DENYLIST (${denyListKv.id})`)
+
+  return {
+    cfAccountId,
+    cfApiToken,
+    denyListKv
+  }
+}
+
+async function getDenyList (url) {
+  const headers = new Headers()
+  headers.append('cache-control', 'no-cache')
+  headers.append('pragma', 'no-cache')
+
+  const res = await fetch(url, {
+    headers
+  })
+  if (!res.ok) {
+    throw new Error(`unexpected status fetching denylist.json: ${res.status}`)
+  }
+  const list = await res.json()
+  return list
+}
+
+async function getWranglerToml (url) {
+  const res = await fetch(url)
+  if (!res.ok) {
+    throw new Error(`unexpected status fetching wrangler.toml: ${res.status}`)
+  }
+  return toml.parse(await res.text())
+}
+
+/**
+ * @param {string} key
+ * @returns {string}
+ */
+function mustGetEnv (key) {
+  const value = process.env[key]
+  if (value) {
+    return value
+  }
+  throw new Error(`missing environment variable: ${key}`)
 }