chore(benchmarks): Add ability to fix broken images for contentful sites

benchmarks/source-contentful/README.md

@@ -7,16 +7,28 @@ Those individual article pages and the homepage share a common "Layout" componen
 
 ## Setup Contentful benchmark site
 
-1. TODO <Setup data source>
+1. Setup will-it-build data source
 2. Copy `.env.example` to `.env.development` and make sure all variables are set
 3. Run `yarn setup`
 
 Note that the script is idempotent, so you can re-run it on failures.
 
 Also use `yarn setup --skip [N:number]` to skip first `N` articles
-(for example articles created during a previous run)
+(for example articles created during a previous run which failed)
+
+### Fixing broken images
+
+Sometimes Contentful silently fails to process images which causes builds to fail.
+Use following approach to fix those:
+
+1. Run `yarn site find-broken-images`
+2. Change image URLs in will-it-build dataset for this site to some other images
+   (or just use one of the larger sites and set `BENCHMARK_SITE_ID` appropriately)
+3. Run `yarn site fix-broken-images imageid1 imageid2 imageid3`
+   This command updates broken images with images from the `BENCHMARK_SITE_ID` dataset
 
 ## Build a site
 
-1. Copy `.env.example` to `.env.production` and make sure all variables are set
-2. Run `yarn build`
+1. Copy `.env.example` to `.env.production`
+2. Set `BENCHMARK_CONTENTFUL_SPACE_ID` and `BENCHMARK_CONTENTFUL_ACCESS_TOKEN` variables
+3. Run `yarn build`

benchmarks/source-contentful/bin/setup.js → benchmarks/source-contentful/bin/site.js

@@ -2,6 +2,7 @@ const fs = require(`fs`)
 const path = require(`path`)
 const contentful = require(`contentful-management`)
 const chalk = require("chalk")
+const yargs = require("yargs")
 
 require("dotenv").config({
   path: `.env.${process.env.NODE_ENV}`,
@@ -29,12 +30,43 @@ const contentfulConfig = {
 const { spaceId, managementToken } = contentfulConfig
 
 if (!spaceId || !managementToken) {
-  console.error(
-    `Contentful space id and management API token are required.`
-  )
+  console.error(`Contentful space id and management API token are required.`)
   process.exit(1)
 }
 
+yargs
+  .scriptName("site")
+  .usage("$0 <command> [arguments]")
+  .command({
+    command: `setup [--skip=number]`,
+    desc: `Setup new Contentful Benchmark Site from the dataset`,
+    builder: yargs =>
+      yargs.option(`skip`, {
+        type: `number`,
+        default: 0,
+        description: `Skip this number of entries`,
+      }),
+    handler: ({ skip = 0 }) => {
+      runSetup({ skip }).catch(console.error)
+    },
+  })
+  .command({
+    command: "find-broken-images",
+    desc: `Find broken images in current contentful site`,
+    handler: () => {
+      runFindBrokenImages().catch(console.error)
+    },
+  })
+  .command({
+    command: "fix-broken-images <ids...>",
+    desc: `Fix images found by find-broken-images`,
+    handler: ({ ids }) => {
+      runFixBrokenImages(ids).catch(console.error)
+    },
+  })
+  .demandCommand(1)
+  .help().argv
+
 /**
  * Transforms an article from source dataset to contentful data model
  */
@@ -201,16 +233,7 @@ async function createArticle(env, articleData) {
   }
 }
 
-const resolveSkip = () => {
-  const index = process.argv.findIndex(param => param === `--skip`)
-  if (index >= 0) {
-    const skipValue = process.argv[index + 1]
-    return Number(skipValue) || 0
-  }
-  return 0
-}
-
-async function createEntries(env) {
+async function createEntries({ env, skip = 0 }) {
   const processBatch = sourceArticles =>
     Promise.all(
       sourceArticles.map(async sourceArticle => {
@@ -219,14 +242,12 @@ async function createEntries(env) {
         const articleCreated = await createArticle(env, article)
         console.log(
           `Processed ${chalk.green(article.sys.id)} (` +
-          `asset ${assetCreated ? `created` : chalk.yellow(`exists`)}, ` +
-          `article ${articleCreated ? `created` : chalk.yellow(`exists`)})`
+            `asset ${assetCreated ? `created` : chalk.yellow(`exists`)}, ` +
+            `article ${articleCreated ? `created` : chalk.yellow(`exists`)})`
         )
       })
     )
 
-  const skip = resolveSkip()
-
   if (skip) {
     console.log(`Skipping first ${chalk.yellow(skip)} articles`)
   }
@@ -248,28 +269,87 @@ async function createEntries(env) {
   }
 }
 
-async function run() {
+async function updateAssets({ env, assetIds }) {
+  for await (const sourceArticle of readSourceArticles(inputDir)) {
+    const { asset: assetData } = extractEntities(sourceArticle)
+    if (assetIds.has(assetData.sys.id)) {
+      try {
+        let asset = await env.getAsset(assetData.sys.id)
+        try {
+          asset = await asset.unpublish()
+        } catch (e) {}
+        asset = await asset.delete()
+        asset = await createAsset(env, assetData)
+        console.log(`Updated asset ${chalk.yellow(assetData.sys.id)}`)
+      } catch (e) {
+        console.warn(`Could not update asset ${chalk.yellow(assetData.sys.id)}`)
+        console.log(e)
+      }
+    }
+  }
+}
+
+async function findBrokenImages(env) {
+  let assets
+  let skip = 0
+  let ids = []
+
+  do {
+    assets = await env.getAssets({ skip })
+    for (let asset of assets.items) {
+      const details = asset.fields.file[`en-US`].details
+      if (!details || !details.image || !details.image.width) {
+        ids.push(asset.sys.id)
+      }
+    }
+    skip = assets.skip + assets.limit
+  } while (assets && assets.items.length > 0)
+  return ids
+}
+
+async function createClient() {
   const client = contentful.createClient({
     accessToken: contentfulConfig.managementToken,
   })
 
   const space = await client.getSpace(contentfulConfig.spaceId)
   const env = await space.getEnvironment(`master`)
 
-  // Create content model only:
-  createContentModel(env)
-    .then(() => {
-      console.log(`Content model ${chalk.green(`created`)}`)
-    })
-    .then(() => createEntries(env))
-    .then(() => {
-      console.log(
-        `All set! You can now run ${chalk.yellow(
-          "gatsby develop"
-        )} to see the site in action.`
-      )
-    })
-    .catch(error => console.error(error))
+  return { client, space, env }
+}
+
+async function runSetup({ skip }) {
+  const { env } = await createClient()
+
+  await createContentModel(env)
+  console.log(`Content model ${chalk.green(`created`)}`)
+  await createEntries({ env, skip })
+
+  console.log(
+    `All set! You can now run ${chalk.yellow(
+      "gatsby develop"
+    )} to see the site in action.`
+  )
+}
+
+async function runFindBrokenImages() {
+  const { env } = await createClient()
+  const ids = await findBrokenImages(env)
+  if (ids.length) {
+    console.log(chalk.yellow(`Broken images:`))
+    console.log(ids.join(` `))
+    console.log(``)
+  } else {
+    console.log(chalk.green(`No broken images!`))
+  }
 }
 
-run().catch(console.error)
+async function runFixBrokenImages(ids) {
+  if (!ids.length) {
+    console.log(`Nothing to do: no broken images!`)
+    return
+  }
+  const { env } = await createClient()
+  console.log(`Fixing ${chalk.yellow(ids.length)} broken images`)
+  await updateAssets({ env, assetIds: new Set(ids) })
+}

benchmarks/source-contentful/package.json

@@ -10,7 +10,8 @@
     "develop": "gatsby develop",
     "format": "prettier --write \"**/*.{js,jsx,json,md}\"",
     "serve": "gatsby serve",
-    "setup": "cross-env NODE_ENV=development node bin/setup.js",
+    "site": "cross-env NODE_ENV=development node bin/site.js",
+    "setup": "cross-env NODE_ENV=development node bin/site.js setup",
     "start": "npm run develop",
     "test": "echo \"Write tests! -> https://gatsby.dev/unit-testing\" && exit 1"
   },
@@ -31,7 +32,8 @@
     "chalk": "^2.4.2",
     "cross-env": "^7.0.0",
     "gatsby-plugin-benchmark-reporting": "*",
-    "prettier": "^1.19.1"
+    "prettier": "^1.19.1",
+    "yargs": "^15.3.1"
   },
   "repository": {
     "type": "git",