Streamline packaging

lib/adBlockRustUtils.js

@@ -1,7 +1,8 @@
-import { Engine, FilterSet, uBlockResources } from 'adblock-rs'
+import { Engine, FilterSet, RuleTypes, uBlockResources } from 'adblock-rs'
 
+import fs from 'fs-extra'
 import path from 'path'
-import { promises as fs } from 'fs'
+import util from '../lib/util.js'
 
 const uBlockLocalRoot = 'submodules/uBlock'
 const uBlockWebAccessibleResources = path.join(uBlockLocalRoot, 'src/web_accessible_resources')
@@ -50,9 +51,6 @@ const getListCatalog = lazyInit(async () => {
 // Legacy logic requires a distinction between default and regional lists.
 // This can be removed once DAT support is no longer needed by iOS.
 const isDefaultList = entry => entry.default_enabled && entry.hidden
-const getDefaultLists = () => getListCatalog().then(catalog => {
-  return catalog.filter(isDefaultList)
-})
 const getRegionalLists = () => getListCatalog().then(catalog => {
   return catalog.filter(entry => !isDefaultList(entry))
 })
@@ -126,6 +124,72 @@ const generateResourcesFile = async (outLocation) => {
   return fs.writeFile(outLocation, await generateResources(), 'utf8')
 }
 
+// Removes Brave-specific scriptlet injections from non-Brave lists
+const enforceBraveDirectives = (title, data) => {
+  if (!title || !title.startsWith('Brave ')) {
+    return data.split('\n').filter(line => {
+      const hasBraveScriptlet = line.indexOf('+js(brave-') >= 0
+      if (hasBraveScriptlet) {
+        console.log('List ' + title + ' attempted to include brave-specific directive: ' + line)
+      }
+      return !hasBraveScriptlet
+    }).join('\n')
+  } else {
+    return data
+  }
+}
+
+/**
+ * Parses the passed in filter rule data and serializes a data file to disk.
+ *
+ * @param filterRuleData An array of { format, data, includeRedirectUrls, ruleTypes } where format is one of `adblock-rust`'s supported filter parsing formats and data is a newline-separated list of such filters.
+ * includeRedirectUrls is a boolean: https://github.com/brave/adblock-rust/pull/184. We only support redirect URLs on filter lists we maintain and trust.
+ * ruleTypes was added with https://github.com/brave/brave-core-crx-packager/pull/298 and allows for { RuleTypes.ALL, RuleTypes.NETWORK_ONLY, RuleTypes.COSMETIC_ONLY }
+ * @param outputDATFilename The filename of the DAT file to create.
+ */
+const generateDataFileFromLists = (filterRuleData, outPath, defaultRuleType = RuleTypes.ALL) => {
+  const filterSet = new FilterSet(false)
+  for (let { title, format, data, includeRedirectUrls, ruleTypes } of filterRuleData) {
+    includeRedirectUrls = Boolean(includeRedirectUrls)
+    ruleTypes = ruleTypes || defaultRuleType
+    const parseOpts = { format, includeRedirectUrls, ruleTypes }
+    filterSet.addFilters(enforceBraveDirectives(title, data).split('\n'), parseOpts)
+  }
+  const client = new Engine(filterSet, true)
+  const arrayBuffer = client.serializeRaw()
+  fs.writeFileSync(outPath, Buffer.from(arrayBuffer))
+}
+
+/**
+ * Serializes the provided lists to disk in one file as `list.txt` under the given component subdirectory.
+ */
+const generatePlaintextListFromLists = (listBuffers, outPath) => {
+  const fullList = listBuffers.map(({ data, title }) => enforceBraveDirectives(title, data)).join('\n')
+  fs.writeFileSync(outPath, fullList)
+}
+
+/**
+ * Returns a promise that resolves to the contents of each list from the entry's sources.
+ * Throws if any of the lists fail the sanity check.
+ */
+const downloadListsForEntry = (entry) => {
+  const lists = entry.sources
+
+  const promises = []
+  lists.forEach((l) => {
+    console.log(`${entry.langs} ${l.url}...`)
+    promises.push(util.fetchTextFromURL(l.url)
+      .then(data => ({ title: l.title || entry.title, format: l.format, data }))
+      .then(listBuffer => {
+        sanityCheckList(listBuffer)
+        return listBuffer
+      })
+    )
+  })
+
+  return Promise.all(promises)
+}
+
 /**
  * A list of requests that should not be blocked unless the list has some serious issue.
  *
@@ -178,13 +242,15 @@ const sanityCheckList = ({ title, format, data }) => {
 }
 
 export {
+  downloadListsForEntry,
   regionalCatalogComponentId,
   regionalCatalogPubkey,
   resourcesComponentId,
   resourcesPubkey,
   sanityCheckList,
+  generateDataFileFromLists,
+  generatePlaintextListFromLists,
   generateResourcesFile,
   getListCatalog,
-  getDefaultLists,
   getRegionalLists
 }

lib/util.js

@@ -64,6 +64,49 @@ const fetchTextFromURL = (listURL) => {
   return p
 }
 
+const prepareNextVersionCRX = async (binary, publisherProofKey, endpoint, region, componentDescriptor, privateKeyFile, localRun) => {
+  mkdirp.sync(componentDescriptor.stagingDir)
+
+  let contentHash
+  if (componentDescriptor.contentHash !== undefined) {
+    try {
+      contentHash = componentDescriptor.contentHash()
+    } catch (_) {
+      // ignore failure if e.g. previous version does not exist yet
+    }
+  }
+
+  let version
+  if (!localRun) {
+    version = '1.0.0'
+  } else {
+    version = await getNextVersion(endpoint, region, componentDescriptor.componentId, contentHash)
+    if (version === undefined) {
+      console.log(`content for ${componentDescriptor.componentId} was not updated, skipping!`)
+      return
+    }
+  }
+
+  const contentHashFile = componentDescriptor.contentHashFile
+
+  // Remove any existing `.contentHash` file for determinism
+  if (contentHashFile !== undefined && fs.existsSync(contentHashFile)) {
+    fs.unlinkSync(contentHashFile)
+  }
+
+  await componentDescriptor.stageFiles(version, componentDescriptor.stagingDir)
+
+  if (!localRun) {
+    generateCRXFile(binary, componentDescriptor.crxFile, privateKeyFile, publisherProofKey, componentDescriptor.stagingDir)
+  }
+
+  if (contentHash !== undefined && contentHashFile !== undefined) {
+    fs.writeFileSync(contentHashFile, contentHash)
+  }
+
+  console.log(`Generated ${componentDescriptor.crxFile} with version number ${version}`)
+}
+
 const generateCRXFile = (binary, crxFile, privateKeyFile, publisherProofKey,
   inputDir) => {
   if (!binary) {
@@ -76,8 +119,12 @@ const generateCRXFile = (binary, crxFile, privateKeyFile, publisherProofKey,
     throw new Error(`Private key file '${privateKeyFile}' is missing, was it uploaded?`)
   }
 
+  const crxOutputDir = path.dirname(crxFile)
+  mkdirp.sync(crxOutputDir)
+
   const tmp = tmpdir()
   const tempUserDataDir = fs.mkdtempSync(path.join(tmp, 'crx-package-job-'))
+
   const args = [
     `--pack-extension=${path.resolve(inputDir)}`,
     `--pack-extension-key=${path.resolve(privateKeyFile)}`,
@@ -551,18 +598,6 @@ const updateDynamoDB = (endpoint, region, id, version, hash, name, disabled, con
   return dynamodb.send(new PutItemCommand(params))
 }
 
-const addCommonScriptOptions = (command) => {
-  return command
-    .option('-b, --binary <binary>',
-      'Path to the Chromium based executable to use to generate the CRX file')
-    .option('-p, --publisher-proof-key <file>',
-      'File containing private key for generating publisher proof')
-
-    // If setup locally, use  --endpoint http://localhost:8000
-    .option('-e, --endpoint <endpoint>', 'DynamoDB endpoint to connect to', '')
-    .option('-r, --region <region>', 'The AWS region to use', 'us-west-2')
-}
-
 const escapeStringForJSON = str => {
   if (typeof str !== 'string') {
     throw new Error('Not a string: ' + JSON.stringify(str))
@@ -613,18 +648,16 @@ export default {
   fetchTextFromURL,
   createTableIfNotExists,
   fetchPreviousVersions,
-  generateCRXFile,
   generatePuffPatches,
   generateSHA256HashOfFile,
-  getNextVersion,
   getIDFromBase64PublicKey,
   installErrorHandlers,
   parseManifest,
   uploadCRXFile,
   updateDBForCRXFile,
-  addCommonScriptOptions,
   escapeStringForJSON,
   copyManifestWithVersion,
   stageDir,
-  stageFiles
+  stageFiles,
+  prepareNextVersionCRX
 }

package.json

@@ -39,12 +39,12 @@
     "generate-user-model-installer-updates": "node scripts/generateBraveAdsResourcesComponentInputFiles.js",
     "lint": "standard lib scripts test",
     "package-brave-player": "node ./scripts/packageBravePlayer",
-    "package-ethereum-remote-client": "node ./scripts/packageComponent --type ethereum-remote-client",
+    "package-ethereum-remote-client": "node ./scripts/packageEthereumRemoteClient",
     "package-ad-block": "node ./scripts/packageAdBlock",
     "package-tor-client": "node ./scripts/packageTorClient",
     "package-tor-pluggable-transports": "node ./scripts/packageTorPluggableTransports",
     "package-ipfs-daemon": "node ./scripts/packageIpfsDaemon",
-    "package-wallet-data-files": "node ./scripts/packageComponent --type wallet-data-files-updater",
+    "package-wallet-data-files": "node ./scripts/packageWalletDataFiles",
     "package-local-data-files": "node ./scripts/packageLocalDataFiles",
     "package-ntp-background-images": "node ./scripts/packageNTPBackgroundImagesComponent",
     "package-ntp-sponsored-images": "node ./scripts/ntp-sponsored-images/package",

scripts/generateAdBlockRustDataFiles.js

@@ -2,143 +2,5 @@
  * License, v. 2.0. If a copy of the MPL was not distributed with this file,
  * You can obtain one at http://mozilla.org/MPL/2.0/. */
 
-import {
-  generateResourcesFile,
-  getListCatalog,
-  getDefaultLists,
-  getRegionalLists,
-  resourcesComponentId,
-  regionalCatalogComponentId,
-  sanityCheckList
-} from '../lib/adBlockRustUtils.js'
-import Sentry from '../lib/sentry.js'
-import util from '../lib/util.js'
-import path from 'path'
-import fs from 'fs'
-
-/**
- * Obtains the output path to store a file given the specied name and subdir
- */
-const getOutPath = (outputFilename, outSubdir) => {
-  let outPath = path.join('build')
-  if (!fs.existsSync(outPath)) {
-    fs.mkdirSync(outPath)
-  }
-  outPath = path.join(outPath, 'ad-block-updater')
-  if (!fs.existsSync(outPath)) {
-    fs.mkdirSync(outPath)
-  }
-  outPath = path.join(outPath, outSubdir)
-  if (!fs.existsSync(outPath)) {
-    fs.mkdirSync(outPath)
-  }
-  return path.join(outPath, outputFilename)
-}
-
-// Removes Brave-specific scriptlet injections from non-Brave lists
-const enforceBraveDirectives = (title, data) => {
-  if (!title || !title.startsWith('Brave ')) {
-    return data.split('\n').filter(line => {
-      const hasBraveScriptlet = line.indexOf('+js(brave-') >= 0
-      if (hasBraveScriptlet) {
-        console.log('List ' + title + ' attempted to include brave-specific directive: ' + line)
-      }
-      return !hasBraveScriptlet
-    }).join('\n')
-  } else {
-    return data
-  }
-}
-
-/**
- * Serializes the provided lists to disk in one file as `list.txt` under the given component subdirectory.
- */
-const generatePlaintextListFromLists = (listBuffers, outSubdir) => {
-  const fullList = listBuffers.map(({ data, title }) => enforceBraveDirectives(title, data)).join('\n')
-  fs.writeFileSync(getOutPath('list.txt', outSubdir), fullList)
-}
-
-/**
- * Convenience function that generates component files for a given catalog entry.
- *
- * If any list source cannot be downloaded, the promise will resolve but the new files will _not_ be generated.
- *
- * @param entry the corresponding entry directly from one of Brave's list catalogs
- * @param doIos boolean, whether or not filters for iOS should be created (currently only used by default list)
- * @return a Promise which resolves upon completion
- */
-const generateDataFilesForCatalogEntry = (entry) => {
-  const lists = entry.sources
-
-  const promises = []
-  lists.forEach((l) => {
-    console.log(`${entry.langs} ${l.url}...`)
-    promises.push(util.fetchTextFromURL(l.url)
-      .then(data => ({ title: l.title || entry.title, format: l.format, data }))
-      .then(listBuffer => {
-        sanityCheckList(listBuffer)
-        return listBuffer
-      })
-    )
-  })
-  return Promise.all(promises)
-    .then(
-      listBuffers => generatePlaintextListFromLists(listBuffers, entry.list_text_component.component_id),
-      e => {
-        console.error(`Not publishing a new version of ${entry.title} due to failure downloading a source: ${e.message}`)
-        if (Sentry) {
-          Sentry.captureException(e, { level: 'warning' })
-        }
-      }
-    )
-}
-
-/**
- * Convenience function that generates a DAT file for each region, and writes
- * the catalog of available regional lists to the default list directory and
- * regional catalog component directory.
- */
-const generateDataFilesForAllRegions = () => {
-  console.log('Processing per region list updates...')
-  return getRegionalLists().then(regions => {
-    return new Promise((resolve, reject) => {
-      const catalogString = JSON.stringify(regions)
-      fs.writeFileSync(getOutPath('regional_catalog.json', regionalCatalogComponentId), catalogString)
-      getListCatalog().then(listCatalog => {
-        const catalogString = JSON.stringify(listCatalog)
-        fs.writeFileSync(getOutPath('list_catalog.json', regionalCatalogComponentId), catalogString)
-        resolve()
-      })
-    }).then(() => Promise.all(regions.map(region =>
-      generateDataFilesForCatalogEntry(region)
-    )))
-  })
-}
-
-const generateDataFilesForResourcesComponent = () => {
-  return generateResourcesFile(getOutPath('resources.json', resourcesComponentId))
-}
-
-const generateDataFilesForDefaultAdblock = () => getDefaultLists()
-  .then(defaultLists => Promise.all(defaultLists.map(list => generateDataFilesForCatalogEntry(list))))
-
-generateDataFilesForDefaultAdblock()
-  .then(generateDataFilesForResourcesComponent)
-  .then(generateDataFilesForAllRegions)
-  .then(() => {
-    console.log('Thank you for updating the data files, don\'t forget to upload them too!')
-  })
-  .catch((e) => {
-    console.error(`Something went wrong, aborting: ${e} ${e.stack} ${e.message}`)
-    process.exit(1)
-  })
-
-process.on('uncaughtException', (err) => {
-  console.error('Caught exception:', err)
-  process.exit(1)
-})
-
-process.on('unhandledRejection', (err) => {
-  console.error('Unhandled rejection:', err)
-  process.exit(1)
-})
+// TODO remove this after the Jenkins job is updated
+console.log('Data files for adblock-rust are now generated as part of the package-ad-block job.')