From 4caf6b107c6bf01e3a4edc20bdd5bbebd22de132 Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Tue, 11 Jul 2023 17:31:44 +0200 Subject: [PATCH] feat: denylist update remote command and workflow --- .github/workflows/denylist-update.yml | 39 +++++ packages/denylist/scripts/cli.js | 9 +- packages/denylist/scripts/denylist.js | 205 ++++++++++++++++++-------- 3 files changed, 187 insertions(+), 66 deletions(-) create mode 100644 .github/workflows/denylist-update.yml diff --git a/.github/workflows/denylist-update.yml b/.github/workflows/denylist-update.yml new file mode 100644 index 0000000..4d61a55 --- /dev/null +++ b/.github/workflows/denylist-update.yml @@ -0,0 +1,39 @@ +name: Update remote denylist + +on: + workflow_dispatch: + inputs: + url: + required: true + description: URL of the file to use to update denylist + type: string + operation: + required: true + description: Operation to be performed in the remote denylist. Default is add + options: + - add + - delete + default: add + +jobs: + update: + name: Update remote denylist + runs-on: ubuntu-latest + strategy: + matrix: + env: ['staging', 'production'] + timeout-minutes: 20 + steps: + - uses: actions/checkout@v2 + - uses: pnpm/action-setup@v2.0.1 + with: + version: 6.32.x + - uses: actions/setup-node@v2 + with: + node-version: 16 + cache: 'pnpm' + - run: pnpm install + - name: Run job + env: + CF_API_TOKEN: ${{ secrets.CF_GATEWAY_TOKEN }} + run: node packages/denylist/scripts/cli.js denylist update-remote ${{ github.event.inputs.url }} --env ${{ matrix.env }} --operation ${{ github.event.inputs.operation }} \ No newline at end of file diff --git a/packages/denylist/scripts/cli.js b/packages/denylist/scripts/cli.js index 15a4f70..6dff765 100644 --- a/packages/denylist/scripts/cli.js +++ b/packages/denylist/scripts/cli.js @@ -3,7 +3,7 @@ import sade from 'sade' import { buildCmd } from './build.js' -import { denylistSyncCmd, denylistAddCmd } from './denylist.js' +import { denylistSyncCmd, denylistAddCmd, denylistUpdateRemoteCmd } from './denylist.js' const env = process.env.ENV || 'dev' const prog = sade('denylist') @@ -24,5 +24,12 @@ prog .option('--status', 'HTTP status to send in response.') .option('--reason', 'Reason for deny. Note: may be communicated in response') .action(denylistAddCmd) + .command('denylist update-remote ') + .option('--env', 'Wrangler environment to use.', env) + .option('--operation', 'Operation to be performed in the remote denylist for items in list ("add" or "delete")', 'add') + .describe( + 'Add the content of a given file to the remote deny list.' + ) + .action(denylistUpdateRemoteCmd) prog.parse(process.argv) diff --git a/packages/denylist/scripts/denylist.js b/packages/denylist/scripts/denylist.js index b563786..ff40d43 100644 --- a/packages/denylist/scripts/denylist.js +++ b/packages/denylist/scripts/denylist.js @@ -22,28 +22,27 @@ const DENY_LIST_SOURCES = [ denyListPath ] -export async function denylistSyncCmd ({ env }) { - const cfApiToken = mustGetEnv('CF_API_TOKEN') - - const wranglerConfig = await getWranglerToml(wranglerConfigPath) - const wranglerEnvConfig = wranglerConfig.env[env] - if (!wranglerEnvConfig) { - throw new Error(`missing wrangler configuration for env: ${env}`) - } - console.log(`🧩 using wrangler config: ${wranglerConfigPath}`) +export async function denylistAddCmd (cid, options) { + console.log(`🦴 fetching ${denyListPath}`) + const denyList = await getDenyList(denyListPath) + const entry = await cidToAnchor(cid, options) - const cfAccountId = wranglerEnvConfig.account_id - if (!cfAccountId) { - throw new Error(`missing Cloudflare account_id in env: ${env}`) + if (denyList.some((e) => e.anchor === entry.anchor)) { + throw new Error('already exists') } - console.log(`🏕 using env: ${env} (${cfAccountId})`) + denyList.push(entry) + console.log('📝 writing update') + fs.writeFileSync( + denyListPath.replace('file://', ''), + JSON.stringify(denyList, null, 2) + ) + console.log('✅ Done') +} - const kvNamespaces = wranglerEnvConfig.kv_namespaces || [] - const denyListKv = kvNamespaces.find((kv) => kv.binding === 'DENYLIST') - if (!denyListKv) { - throw new Error('missing binding in kv_namespaces: DENYLIST') - } - console.log(`🪢 using KV binding: DENYLIST (${denyListKv.id})`) +export async function denylistSyncCmd ({ env }) { + const { + cfApiToken, cfAccountId, denyListKv + } = await getDenylistProperties({ env }) for (const url of DENY_LIST_SOURCES) { console.log(`🦴 fetching ${url}`) @@ -58,27 +57,26 @@ export async function denylistSyncCmd ({ env }) { console.log('✅ Done') } -async function getWranglerToml (url) { - const res = await fetch(url) - if (!res.ok) { - throw new Error(`unexpected status fetching wrangler.toml: ${res.status}`) +export async function denylistUpdateRemoteCmd (url, { env, reason, operation }) { + if (operation !== 'add' && operation !== 'delete') { + throw new Error(`operation must be one of {add, delete}. Received: ${operation}`) } - return toml.parse(await res.text()) -} - -async function getDenyList (url) { - const headers = new Headers() - headers.append('cache-control', 'no-cache') - headers.append('pragma', 'no-cache') - - const res = await fetch(url, { - headers + const { + cfApiToken, cfAccountId, denyListKv + } = await getDenylistProperties({ env }) + + const entries = await getListFromUrl(url) + const bulk = await Promise.all(entries.map(async entry => { + return { + key: (await cidToAnchor(entry)).anchor, + value: { reason } + } + })) + console.log(`update ${bulk.length} remote denylist: ${operation}`) + await writeKVMulti(cfApiToken, cfAccountId, denyListKv.id, bulk, { + delete: operation === 'delete' }) - if (!res.ok) { - throw new Error(`unexpected status fetching denylist.json: ${res.status}`) - } - const list = await res.json() - return list + console.log('✅ Done') } /** @@ -86,20 +84,29 @@ async function getDenyList (url) { * @param {string} accountId Cloudflare account ID * @param {string} nsId KV namespace ID * @param {Array} kvs - * @returns {Promise<{ success: boolean, errors: Array<{ code: number, message: string }>, messages: any[] }>} + * @param {object} [options] + * @param {boolean} [options.delete] */ -async function writeKVMulti (apiToken, accountId, nsId, kvs) { +async function writeKVMulti (apiToken, accountId, nsId, kvs, options = {}) { + const isDelete = options.delete const url = `https://api.cloudflare.com/client/v4/accounts/${accountId}/storage/kv/namespaces/${nsId}/bulk` - kvs = kvs.map((kv) => ({ - ...kv, - value: JSON.stringify(kv.value) - })) + + // Delete + if (isDelete) { + kvs = kvs.map((kv) => kv.key) + // Add + } else { + kvs = kvs.map((kv) => ({ + ...kv, + value: JSON.stringify(kv.value) + })) + } const chunkSize = 10000 for (let i = 0; i < kvs.length; i += chunkSize) { const kvsChunk = kvs.slice(i, i + chunkSize) const res = await fetch(url, { - method: 'PUT', + method: isDelete ? 'DELETE' : 'PUT', headers: { Authorization: `Bearer ${apiToken}`, 'Content-Type': 'application/json' @@ -113,21 +120,36 @@ async function writeKVMulti (apiToken, accountId, nsId, kvs) { error ? `${error.code}: ${error.message}` : 'failed to write to KV' ) } + + // Delay to avoid going into + await new Promise(resolve => setTimeout(() => resolve(true), 10000)) } } -/** - * @param {string} key - * @returns {string} - */ -function mustGetEnv (key) { - if (!process.env[key]) { - throw new Error(`missing environment variable: ${key}`) +async function getListFromUrl (url) { + console.log(`🦴 fetching ${url}`) + const response = await fetch(url) + if (!response.ok) { + throw new Error(`unexpected status fetching given url: ${response.status}`) + } + + let entries + const contentType = response.headers.get('content-type') + if (contentType === 'application/json') { + entries = await response.json() + } else if (contentType === 'text/plain') { + const textBody = await response.text() + // Separate entries by new line and trim them + entries = textBody.split(/\r?\n/) + } else { + console.log(`content type must be "application/json" or "text/plain". Received ${contentType}`) + process.exit(1) } - return process.env[key] + + return entries } -export async function denylistAddCmd (cid, options) { +async function cidToAnchor (cid, options = {}) { const parts = cid.split('/') const cidv1 = CID.parse(parts[0]).toV1().toString(base32.encoder) const cidv1Path = `${cidv1}/${parts.slice(1).join('/')}` @@ -139,17 +161,70 @@ export async function denylistAddCmd (cid, options) { const entry = { anchor, status: options.status, reason: options.reason } console.log(`🎫 entry: ${JSON.stringify(entry)}`) - console.log(`🦴 fetching ${denyListPath}`) - const denyList = await getDenyList(denyListPath) - if (denyList.some((e) => e.anchor === anchor)) { - throw new Error('already exists') + return entry +} + +async function getDenylistProperties ({ env }) { + const cfApiToken = mustGetEnv('CF_API_TOKEN') + + const wranglerConfig = await getWranglerToml(wranglerConfigPath) + const wranglerEnvConfig = wranglerConfig.env[env] + if (!wranglerEnvConfig) { + throw new Error(`missing wrangler configuration for env: ${env}`) } + console.log(`🧩 using wrangler config: ${wranglerConfigPath}`) - denyList.push(entry) - console.log('📝 writing update') - fs.writeFileSync( - denyListPath.replace('file://', ''), - JSON.stringify(denyList, null, 2) - ) - console.log('✅ Done') + const cfAccountId = wranglerEnvConfig.account_id + if (!cfAccountId) { + throw new Error(`missing Cloudflare account_id in env: ${env}`) + } + console.log(`🏕 using env: ${env} (${cfAccountId})`) + + const kvNamespaces = wranglerEnvConfig.kv_namespaces || [] + const denyListKv = kvNamespaces.find((kv) => kv.binding === 'DENYLIST') + if (!denyListKv) { + throw new Error('missing binding in kv_namespaces: DENYLIST') + } + console.log(`🪢 using KV binding: DENYLIST (${denyListKv.id})`) + + return { + cfAccountId, + cfApiToken, + denyListKv + } +} + +async function getDenyList (url) { + const headers = new Headers() + headers.append('cache-control', 'no-cache') + headers.append('pragma', 'no-cache') + + const res = await fetch(url, { + headers + }) + if (!res.ok) { + throw new Error(`unexpected status fetching denylist.json: ${res.status}`) + } + const list = await res.json() + return list +} + +async function getWranglerToml (url) { + const res = await fetch(url) + if (!res.ok) { + throw new Error(`unexpected status fetching wrangler.toml: ${res.status}`) + } + return toml.parse(await res.text()) +} + +/** + * @param {string} key + * @returns {string} + */ +function mustGetEnv (key) { + const value = process.env[key] + if (value) { + return value + } + throw new Error(`missing environment variable: ${key}`) }