feat(gatsby-recipes): fix MDX rendering for exports (#25133)

* feat(gatsby-recipes): Handle exports, render MDX directly

* Continue working towards proper exports

* Continue implementing MDX renderer v2

* More MDX rendering hacking

* Finish basic export handling

* Small fixes

Co-authored-by: John Otander <johnotander@gmail.com>

packages/gatsby-recipes/package.json

@@ -11,7 +11,7 @@
     "@babel/generator": "^7.9.6",
     "@babel/helper-plugin-utils": "^7.8.3",
     "@babel/plugin-transform-react-jsx": "^7.9.4",
-    "@babel/standalone": "^7.9.6",
+    "@babel/standalone": "^7.10.2",
     "@babel/template": "^7.8.6",
     "@babel/types": "^7.9.6",
     "@emotion/core": "^10.0.14",
@@ -24,13 +24,15 @@
     "acorn": "^7.2.0",
     "acorn-jsx": "^5.2.0",
     "ansi-html": "^0.0.7",
+    "babel-plugin-remove-export-keywords": "^1.6.5",
     "contentful-management": "^5.26.3",
     "cors": "^2.8.5",
     "debug": "^4.1.1",
     "detect-port": "^1.3.0",
     "execa": "^4.0.1",
     "express": "^4.17.1",
     "express-graphql": "^0.9.0",
+    "flatted": "^3.0.0",
     "formik": "^2.0.8",
     "fs-extra": "^8.1.0",
     "gatsby-core-utils": "^1.3.3",
@@ -75,6 +77,7 @@
     "subscriptions-transport-ws": "^0.9.16",
     "svg-tag-names": "^2.0.1",
     "unified": "^8.4.2",
+    "unist-util-remove": "^2.0.0",
     "unist-util-visit": "^2.0.2",
     "urql": "^1.9.7",
     "ws": "^7.3.0",
@@ -90,10 +93,17 @@
   },
   "homepage": "https://github.com/gatsbyjs/gatsby/tree/master/packages/gatsby-recipes#readme",
   "jest": {
-    "testPathIgnorePatterns": ["/.cache/", "dist"],
+    "testPathIgnorePatterns": [
+      "/.cache/",
+      "dist"
+    ],
     "testEnvironment": "node"
   },
-  "keywords": ["gatsby", "gatsby-recipes", "mdx"],
+  "keywords": [
+    "gatsby",
+    "gatsby-recipes",
+    "mdx"
+  ],
   "license": "MIT",
   "main": "dist/index.js",
   "repository": {

packages/gatsby-recipes/src/components/mdx.js

@@ -0,0 +1,45 @@
+import React from 'react'
+import { transform } from '@babel/standalone'
+import mdx from '@mdx-js/mdx'
+import { mdx as createElement, MDXProvider } from '@mdx-js/react'
+import babelPluginTransformReactJsx from '@babel/plugin-transform-react-jsx'
+import babelPluginRemoveExportKeywords from 'babel-plugin-remove-export-keywords'
+
+const transformJsx = jsx => {
+  const { code } = transform(jsx, {
+    plugins: [
+      babelPluginRemoveExportKeywords,
+      [babelPluginTransformReactJsx, { useBuiltIns: true }],
+    ],
+  })
+
+  return code
+}
+
+const transformCodeForEval = jsx => {
+  return `${jsx}
+
+  return React.createElement(MDXProvider, { components },
+    React.createElement(MDXContent, props)
+  );`
+}
+
+export default ({ children: mdxSrc, scope, components, ...props }) => {
+  const fullScope = {
+    mdx: createElement,
+    MDXProvider,
+    React,
+    components,
+    props,
+    ...scope
+  }
+  const scopeKeys = Object.keys(fullScope)
+  const scopeValues = Object.values(fullScope)
+
+  const jsxFromMdx = mdx.sync(mdxSrc, { skipExport: true })
+  const srcCode = transformJsx(jsxFromMdx)
+
+  const fn = new Function(...scopeKeys, transformCodeForEval(srcCode))
+
+  return fn(...scopeValues)
+}

packages/gatsby-recipes/src/create-plan.js

@@ -1,7 +1,11 @@
 const render = require(`./renderer`)
 
 module.exports = async (context, cb) => {
-  const stepAsMdx = context.steps.join(`\n`)
+  console.log(context)
+  const stepAsMdx = [
+    ...context.steps,
+    ...context.exports
+  ].join(`\n`)
 
   try {
     const result = await render(stepAsMdx, cb, context.inputs)

packages/gatsby-recipes/src/graphql-server/server.js

@@ -54,7 +54,7 @@ const applyPlan = ({ recipePath, projectRoot }) => {
         currentStep: state.context.currentStep,
       })
       // Wait until plans are created before updating the UI
-      if (state.value !== `creatingPlan`) {
+      if (state.value !== `creatingPlan` || state.value !== 'validateSteps') {
         emitUpdate({
           context: state.context,
           lastEvent: state.event,

packages/gatsby-recipes/src/gui.js

@@ -1,13 +1,14 @@
 /** @jsx jsx */
-import * as GatsbyComponents from "gatsby-interface"
 import { jsx, ThemeProvider as ThemeUIProvider, Styled } from "theme-ui"
 const lodash = require(`lodash`)
 const React = require(`react`)
 const { useState } = require(`react`)
-const MDX = require(`@mdx-js/runtime`).default
 const ansi2HTML = require(`ansi-html`)
+const remove = require("unist-util-remove")
 import { MdRefresh, MdBrightness1 } from "react-icons/md"
 import { keyframes } from "@emotion/core"
+import MDX from "./components/mdx"
+
 const {
   Button,
   ThemeProvider,
@@ -46,9 +47,8 @@ const makeResourceId = res => {
 
 let sendEvent
 
-const PROJECT_ROOT =
-  `/Users/kylemathews/programs/recipes-test` ||
-  `/Users/johno-mini/c/gatsby/starters/blog`
+const PROJECT_ROOT = `/Users/kylemathews/programs/recipes-test`
+// `/Users/johno-mini/c/gatsby/starters/blog`
 
 const Color = `span`
 const Spinner = () => <span>Loading...</span>
@@ -279,6 +279,19 @@ log(
   `======================================= ${new Date().toJSON()}`
 )
 
+const removeJsx = () => tree => {
+  remove(tree, "mdxBlockElement", node => {
+    return node.name === "File"
+  })
+
+  remove(tree, "export", node => {
+    console.log({ node })
+    return true
+  })
+  console.log({ tree })
+  return tree
+}
+
 const RecipeGui = ({
   recipe = `./test.mdx`,
   // recipe = `jest.mdx`,
@@ -496,8 +509,13 @@ const RecipeGui = ({
                   },
                 }}
               >
-                <MDX key="DOC" components={components} scope={{ sendEvent }}>
-                  {step}
+                <MDX
+                  key="DOC"
+                  components={components}
+                  scope={{ sendEvent }}
+                  remarkPlugins={[removeJsx]}
+                >
+                  {state.context.exports.join("\n") + "\n\n" + step}
                 </MDX>
               </div>
             </div>
@@ -569,95 +587,6 @@ const RecipeGui = ({
         })
       }
 
-      const PresentStep = ({ step }) => {
-        // const isPlan = state.context.plan && state.context.plan.length > 0
-        // const isPresetPlanState = state.value === `presentPlan`
-        // const isRunningStep = state.value === `applyingPlan`
-        // console.log(`PresentStep`, { isRunningStep, isPlan, isPresetPlanState })
-        // if (isRunningStep) {
-        // console.log("running step")
-        // return null
-        // }
-        // if (!isPlan || !isPresetPlanState) {
-        // return (
-        // <div margin-top={1}>
-        // <button onClick={() => sendEvent({ event: `CONTINUE` })}>
-        // Go!
-        // </button>
-        // </div>
-        // )
-        // }
-        //
-        // {plan.map((p, i) => (
-        // <div margin-top={1} key={`${p.resourceName} plan ${i}`}>
-        // <Styled.p>{p.resourceName}:</Styled.p>
-        // <Styled.p> * {p.describe}</Styled.p>
-        // {p.diff && p.diff !== `` && (
-        // <>
-        // <Styled.p>---</Styled.p>
-        // <pre
-        // sx={{
-        // lineHeight: 0.7,
-        // background: `#f5f3f2`,
-        // padding: [3],
-        // "& > span": {
-        // display: `block`,
-        // },
-        // }}
-        // dangerouslySetInnerHTML={{ __html: ansi2HTML(p.diff) }}
-        // />
-        // <Styled.p>---</Styled.p>
-        // </>
-        // )}
-        // </div>
-        // ))}
-        // <div margin-top={1}>
-        // <button onClick={() => sendEvent({ event: "CONTINUE" })}>
-        // Go!
-        // </button>
-        // </div>
-      }
-
-      const RunningStep = ({ state }) => {
-        const isPlan = state.context.plan && state.context.plan.length > 0
-        const isRunningStep = state.value === `applyingPlan`
-
-        if (!isPlan || !isRunningStep) {
-          return null
-        }
-
-        return (
-          <div>
-            {state.context.plan.map((p, i) => (
-              <div key={`${p.resourceName}-${i}`}>
-                <Styled.p>{p.resourceName}:</Styled.p>
-                <Styled.p>
-                  {` `}
-                  <Spinner /> {p.describe}
-                  {` `}
-                  {state.context.elapsed > 0 && (
-                    <Styled.p>
-                      ({state.context.elapsed / 1000}s elapsed)
-                    </Styled.p>
-                  )}
-                </Styled.p>
-              </div>
-            ))}
-          </div>
-        )
-      }
-
-      const Error = ({ state }) => {
-        log(`errors`, state)
-        if (state && state.context && state.context.error) {
-          return (
-            <Color red>{JSON.stringify(state.context.error, null, 2)}</Color>
-          )
-        }
-
-        return null
-      }
-
       const staticMessages = {}
       for (let step = 0; step < state.context.currentStep; step++) {
         staticMessages[step] = [
@@ -798,8 +727,14 @@ const RecipeGui = ({
               }}
             >
               <div sx={{ "*:last-child": { mb: 0 } }}>
-                <MDX components={components} scope={{ sendEvent }}>
-                  {state.context.steps[0]}
+                <MDX
+                  components={components}
+                  scope={{ sendEvent }}
+                  remarkPlugins={[removeJsx]}
+                >
+                  {state.context.exports.join("\n") +
+                    "\n\n" +
+                    state.context.steps[0]}
                 </MDX>
               </div>
               <Button

packages/gatsby-recipes/src/parser/index.js

@@ -8,6 +8,7 @@ const fs = require(`fs-extra`)
 const isUrl = require(`is-url`)
 const path = require(`path`)
 const visit = require(`unist-util-visit`)
+const remove = require(`unist-util-remove`)
 
 const { uuid } = require(`./util`)
 
@@ -20,6 +21,17 @@ const asRoot = node => {
   }
 }
 
+const pluckExports = tree => {
+  let exports = []
+  visit(tree, `export`, node => {
+    exports.push(node)
+  })
+
+  remove(tree, `export`)
+
+  return exports
+}
+
 const applyUuid = tree => {
   visit(tree, `mdxBlockElement`, node => {
     if (!IGNORED_COMPONENTS.includes(node.name)) {
@@ -28,6 +40,11 @@ const applyUuid = tree => {
         name: `_uuid`,
         value: uuid(),
       })
+      node.attributes.push({
+        type: `mdxAttribute`,
+        name: `_type`,
+        value: node.name,
+      })
     }
   })
 
@@ -60,15 +77,14 @@ const partitionSteps = ast => {
 const toMdx = nodes => {
   const stepAst = applyUuid(asRoot(nodes))
   const mdxSrc = u.stringify(stepAst)
-  // const regex = new RegExp(`^[ \\t]`, "gm")
-  const regex = /^(\s*)export/gm
 
-  return mdxSrc.replace(regex, `\nexport`)
+  return mdxSrc
 }
 
 const parse = async src => {
   try {
     const ast = u.parse(src)
+    const exportNodes = pluckExports(ast)
     const [intro, ...resourceSteps] = partitionSteps(ast)
 
     const wrappedIntroStep = {
@@ -79,7 +95,6 @@ const parse = async src => {
     }
 
     const wrappedResourceSteps = resourceSteps.map((step, i) => {
-      console.log(step)
       return {
         type: `mdxBlockElement`,
         name: `RecipeStep`,
@@ -100,10 +115,13 @@ const parse = async src => {
     })
 
     const steps = [wrappedIntroStep, ...wrappedResourceSteps]
+    ast.children = [...exportNodes, ...ast.children]
 
     return {
       ast,
       steps,
+      exports: exportNodes,
+      exportsAsMdx: exportNodes.map(toMdx),
       stepsAsMdx: steps.map(toMdx),
     }
   } catch (e) {

packages/gatsby-recipes/src/recipe-machine/index.js

@@ -18,10 +18,12 @@ const recipeMachine = Machine(
       projectRoot: null,
       currentStep: 0,
       steps: [],
+      exports: [],
       plan: [],
       commands: [],
       stepResources: [],
       stepsAsMdx: [],
+      exportsAsMdx: [],
       inputs: {},
     },
     states: {
@@ -72,6 +74,7 @@ const recipeMachine = Machine(
             target: `validateSteps`,
             actions: assign({
               steps: (context, event) => event.data.stepsAsMdx,
+              exports: (context, event) => event.data.exportsAsMdx
             }),
           },
         },