diff --git a/benchmarks/source-contentful/README.md b/benchmarks/source-contentful/README.md index c1b09f04c6655..25e876bbf3e91 100644 --- a/benchmarks/source-contentful/README.md +++ b/benchmarks/source-contentful/README.md @@ -7,16 +7,28 @@ Those individual article pages and the homepage share a common "Layout" componen ## Setup Contentful benchmark site -1. TODO +1. Setup will-it-build data source 2. Copy `.env.example` to `.env.development` and make sure all variables are set 3. Run `yarn setup` Note that the script is idempotent, so you can re-run it on failures. Also use `yarn setup --skip [N:number]` to skip first `N` articles -(for example articles created during a previous run) +(for example articles created during a previous run which failed) + +### Fixing broken images + +Sometimes Contentful silently fails to process images which causes builds to fail. +Use following approach to fix those: + +1. Run `yarn site find-broken-images` +2. Change image URLs in will-it-build dataset for this site to some other images + (or just use one of the larger sites and set `BENCHMARK_SITE_ID` appropriately) +3. Run `yarn site fix-broken-images imageid1 imageid2 imageid3` + This command updates broken images with images from the `BENCHMARK_SITE_ID` dataset ## Build a site -1. Copy `.env.example` to `.env.production` and make sure all variables are set -2. Run `yarn build` +1. Copy `.env.example` to `.env.production` +2. Set `BENCHMARK_CONTENTFUL_SPACE_ID` and `BENCHMARK_CONTENTFUL_ACCESS_TOKEN` variables +3. Run `yarn build` diff --git a/benchmarks/source-contentful/bin/setup.js b/benchmarks/source-contentful/bin/site.js similarity index 63% rename from benchmarks/source-contentful/bin/setup.js rename to benchmarks/source-contentful/bin/site.js index e53e762865da5..633179c15a97b 100644 --- a/benchmarks/source-contentful/bin/setup.js +++ b/benchmarks/source-contentful/bin/site.js @@ -2,6 +2,7 @@ const fs = require(`fs`) const path = require(`path`) const contentful = require(`contentful-management`) const chalk = require("chalk") +const yargs = require("yargs") require("dotenv").config({ path: `.env.${process.env.NODE_ENV}`, @@ -29,12 +30,43 @@ const contentfulConfig = { const { spaceId, managementToken } = contentfulConfig if (!spaceId || !managementToken) { - console.error( - `Contentful space id and management API token are required.` - ) + console.error(`Contentful space id and management API token are required.`) process.exit(1) } +yargs + .scriptName("site") + .usage("$0 [arguments]") + .command({ + command: `setup [--skip=number]`, + desc: `Setup new Contentful Benchmark Site from the dataset`, + builder: yargs => + yargs.option(`skip`, { + type: `number`, + default: 0, + description: `Skip this number of entries`, + }), + handler: ({ skip = 0 }) => { + runSetup({ skip }).catch(console.error) + }, + }) + .command({ + command: "find-broken-images", + desc: `Find broken images in current contentful site`, + handler: () => { + runFindBrokenImages().catch(console.error) + }, + }) + .command({ + command: "fix-broken-images ", + desc: `Fix images found by find-broken-images`, + handler: ({ ids }) => { + runFixBrokenImages(ids).catch(console.error) + }, + }) + .demandCommand(1) + .help().argv + /** * Transforms an article from source dataset to contentful data model */ @@ -201,16 +233,7 @@ async function createArticle(env, articleData) { } } -const resolveSkip = () => { - const index = process.argv.findIndex(param => param === `--skip`) - if (index >= 0) { - const skipValue = process.argv[index + 1] - return Number(skipValue) || 0 - } - return 0 -} - -async function createEntries(env) { +async function createEntries({ env, skip = 0 }) { const processBatch = sourceArticles => Promise.all( sourceArticles.map(async sourceArticle => { @@ -219,14 +242,12 @@ async function createEntries(env) { const articleCreated = await createArticle(env, article) console.log( `Processed ${chalk.green(article.sys.id)} (` + - `asset ${assetCreated ? `created` : chalk.yellow(`exists`)}, ` + - `article ${articleCreated ? `created` : chalk.yellow(`exists`)})` + `asset ${assetCreated ? `created` : chalk.yellow(`exists`)}, ` + + `article ${articleCreated ? `created` : chalk.yellow(`exists`)})` ) }) ) - const skip = resolveSkip() - if (skip) { console.log(`Skipping first ${chalk.yellow(skip)} articles`) } @@ -248,7 +269,45 @@ async function createEntries(env) { } } -async function run() { +async function updateAssets({ env, assetIds }) { + for await (const sourceArticle of readSourceArticles(inputDir)) { + const { asset: assetData } = extractEntities(sourceArticle) + if (assetIds.has(assetData.sys.id)) { + try { + let asset = await env.getAsset(assetData.sys.id) + try { + asset = await asset.unpublish() + } catch (e) {} + asset = await asset.delete() + asset = await createAsset(env, assetData) + console.log(`Updated asset ${chalk.yellow(assetData.sys.id)}`) + } catch (e) { + console.warn(`Could not update asset ${chalk.yellow(assetData.sys.id)}`) + console.log(e) + } + } + } +} + +async function findBrokenImages(env) { + let assets + let skip = 0 + let ids = [] + + do { + assets = await env.getAssets({ skip }) + for (let asset of assets.items) { + const details = asset.fields.file[`en-US`].details + if (!details || !details.image || !details.image.width) { + ids.push(asset.sys.id) + } + } + skip = assets.skip + assets.limit + } while (assets && assets.items.length > 0) + return ids +} + +async function createClient() { const client = contentful.createClient({ accessToken: contentfulConfig.managementToken, }) @@ -256,20 +315,41 @@ async function run() { const space = await client.getSpace(contentfulConfig.spaceId) const env = await space.getEnvironment(`master`) - // Create content model only: - createContentModel(env) - .then(() => { - console.log(`Content model ${chalk.green(`created`)}`) - }) - .then(() => createEntries(env)) - .then(() => { - console.log( - `All set! You can now run ${chalk.yellow( - "gatsby develop" - )} to see the site in action.` - ) - }) - .catch(error => console.error(error)) + return { client, space, env } +} + +async function runSetup({ skip }) { + const { env } = await createClient() + + await createContentModel(env) + console.log(`Content model ${chalk.green(`created`)}`) + await createEntries({ env, skip }) + + console.log( + `All set! You can now run ${chalk.yellow( + "gatsby develop" + )} to see the site in action.` + ) +} + +async function runFindBrokenImages() { + const { env } = await createClient() + const ids = await findBrokenImages(env) + if (ids.length) { + console.log(chalk.yellow(`Broken images:`)) + console.log(ids.join(` `)) + console.log(``) + } else { + console.log(chalk.green(`No broken images!`)) + } } -run().catch(console.error) +async function runFixBrokenImages(ids) { + if (!ids.length) { + console.log(`Nothing to do: no broken images!`) + return + } + const { env } = await createClient() + console.log(`Fixing ${chalk.yellow(ids.length)} broken images`) + await updateAssets({ env, assetIds: new Set(ids) }) +} diff --git a/benchmarks/source-contentful/package.json b/benchmarks/source-contentful/package.json index b52d69b4da110..fe26a7a92cb4e 100644 --- a/benchmarks/source-contentful/package.json +++ b/benchmarks/source-contentful/package.json @@ -10,7 +10,8 @@ "develop": "gatsby develop", "format": "prettier --write \"**/*.{js,jsx,json,md}\"", "serve": "gatsby serve", - "setup": "cross-env NODE_ENV=development node bin/setup.js", + "site": "cross-env NODE_ENV=development node bin/site.js", + "setup": "cross-env NODE_ENV=development node bin/site.js setup", "start": "npm run develop", "test": "echo \"Write tests! -> https://gatsby.dev/unit-testing\" && exit 1" }, @@ -31,7 +32,8 @@ "chalk": "^2.4.2", "cross-env": "^7.0.0", "gatsby-plugin-benchmark-reporting": "*", - "prettier": "^1.19.1" + "prettier": "^1.19.1", + "yargs": "^15.3.1" }, "repository": { "type": "git",