fix(v2): Fix MDX docs being considered as partials when siteDir match the _ prefix convention

.github/workflows/v2-build-size-report.yml

@@ -26,7 +26,7 @@ jobs:
         with:
           repo-token: '${{ secrets.GITHUB_TOKEN }}'
           build-script: 'build:v2:en'
-          pattern: '{website/build/assets/js/main*js,website/build/assets/css/styles*css,website/.docusaurus/globalData.json,website/build/index.html,website/build/blog/index.html,website/build/blog/**/introducing-docusaurus/*,website/build/docs/index.html,website/build/docs/installation/index.html,website/build/docs-tests/index.html,website/build/docs-tests/standalone/index.html}'
+          pattern: '{website/build/assets/js/main*js,website/build/assets/css/styles*css,website/.docusaurus/globalData.json,website/build/index.html,website/build/blog/index.html,website/build/blog/**/introducing-docusaurus/*,website/build/docs/index.html,website/build/docs/installation/index.html,website/build/tests/docs/index.html,website/build/tests/docs/standalone/index.html}'
           strip-hash: '\.([^;]\w{7})\.'
           minimum-change-threshold: 30
           compression: 'none'

packages/docusaurus-mdx-loader/package.json

@@ -24,6 +24,7 @@
     "@docusaurus/utils": "2.0.0-beta.3",
     "@mdx-js/mdx": "^1.6.21",
     "@mdx-js/react": "^1.6.21",
+    "chalk": "^4.1.1",
     "escape-html": "^1.0.3",
     "file-loader": "^6.2.0",
     "fs-extra": "^10.0.0",

packages/docusaurus-mdx-loader/src/index.js

@@ -7,6 +7,7 @@
 
 const {readFile} = require('fs-extra');
 const mdx = require('@mdx-js/mdx');
+const chalk = require('chalk');
 const emoji = require('remark-emoji');
 const {
   parseFrontMatter,
@@ -69,8 +70,8 @@ module.exports = async function docusaurusMdxLoader(fileString) {
     ],
     filepath: filePath,
   };
-  let result;
 
+  let result;
   try {
     result = await mdx(content, options);
   } catch (err) {
@@ -90,9 +91,19 @@ module.exports = async function docusaurusMdxLoader(fileString) {
     : false;
 
   if (isMDXPartial && hasFrontMatter) {
-    return callback(
-      new Error(`MDX partial should not contain FrontMatter: ${filePath}`),
-    );
+    const errorMessage = `Docusaurus MDX partial files should not contain FrontMatter.
+Those partial files use the _ prefix as a convention by default, but this is configurable.
+File at ${filePath} contains FrontMatter that will be ignored: \n${JSON.stringify(
+      frontMatter,
+      null,
+      2,
+    )}`;
+    const shouldError = process.env.NODE_ENV === 'test' || process.env.CI;
+    if (shouldError) {
+      return callback(new Error(errorMessage));
+    } else {
+      console.warn(chalk.yellow(errorMessage));
+    }
   }
 
   if (!isMDXPartial) {

packages/docusaurus-plugin-content-blog/src/index.ts

@@ -16,7 +16,7 @@ import {
   reportMessage,
   posixPath,
   addTrailingPathSeparator,
-  createMatcher,
+  createAbsoluteFilePathMatcher,
 } from '@docusaurus/utils';
 import {
   STATIC_DIR_NAME,
@@ -437,6 +437,7 @@ export default function pluginContentBlog(
         },
       };
 
+      const contentDirs = getContentPathList(contentPaths);
       return {
         resolve: {
           alias: {
@@ -447,7 +448,7 @@ export default function pluginContentBlog(
           rules: [
             {
               test: /(\.mdx?)$/,
-              include: getContentPathList(contentPaths)
+              include: contentDirs
                 // Trailing slash is important, see https://github.com/facebook/docusaurus/pull/3970
                 .map(addTrailingPathSeparator),
               use: [
@@ -460,7 +461,10 @@ export default function pluginContentBlog(
                     beforeDefaultRemarkPlugins,
                     beforeDefaultRehypePlugins,
                     staticDir: path.join(siteDir, STATIC_DIR_NAME),
-                    isMDXPartial: createMatcher(options.exclude),
+                    isMDXPartial: createAbsoluteFilePathMatcher(
+                      options.exclude,
+                      contentDirs,
+                    ),
                     metadataPath: (mdxPath: string) => {
                       // Note that metadataPath must be the same/in-sync as
                       // the path from createData for each MDX.

packages/docusaurus-plugin-content-docs/src/index.ts

@@ -18,7 +18,7 @@ import {
   reportMessage,
   posixPath,
   addTrailingPathSeparator,
-  createMatcher,
+  createAbsoluteFilePathMatcher,
 } from '@docusaurus/utils';
 import {LoadContext, Plugin, RouteConfig} from '@docusaurus/types';
 import {loadSidebars, createSidebarsUtils, processSidebars} from './sidebars';
@@ -394,9 +394,10 @@ export default function pluginContentDocs(
       };
 
       function createMDXLoaderRule(): RuleSetRule {
+        const contentDirs = flatten(versionsMetadata.map(getDocsDirPaths));
         return {
           test: /(\.mdx?)$/,
-          include: flatten(versionsMetadata.map(getDocsDirPaths))
+          include: contentDirs
             // Trailing slash is important, see https://github.com/facebook/docusaurus/pull/3970
             .map(addTrailingPathSeparator),
           use: compact([
@@ -409,7 +410,10 @@ export default function pluginContentDocs(
                 beforeDefaultRehypePlugins,
                 beforeDefaultRemarkPlugins,
                 staticDir: path.join(siteDir, STATIC_DIR_NAME),
-                isMDXPartial: createMatcher(options.exclude),
+                isMDXPartial: createAbsoluteFilePathMatcher(
+                  options.exclude,
+                  contentDirs,
+                ),
                 metadataPath: (mdxPath: string) => {
                   // Note that metadataPath must be the same/in-sync as
                   // the path from createData for each MDX.

packages/docusaurus-plugin-content-pages/src/index.ts

@@ -16,7 +16,8 @@ import {
   getFolderContainingFile,
   addTrailingPathSeparator,
   Globby,
-  createMatcher,
+  createAbsoluteFilePathMatcher,
+  normalizeUrl,
 } from '@docusaurus/utils';
 import {
   LoadContext,
@@ -124,8 +125,11 @@ export default function pluginContentPages(
 
         const source = path.join(contentPath, relativeSource);
         const aliasedSourcePath = aliasedSitePath(source, siteDir);
-        const pathName = encodePath(fileToPath(relativeSource));
-        const permalink = pathName.replace(/^\//, baseUrl || '');
+        const permalink = normalizeUrl([
+          baseUrl,
+          options.routeBasePath,
+          encodePath(fileToPath(relativeSource)),
+        ]);
         if (isMarkdownSource(relativeSource)) {
           return {
             type: 'mdx',
@@ -194,6 +198,7 @@ export default function pluginContentPages(
         beforeDefaultRehypePlugins,
         beforeDefaultRemarkPlugins,
       } = options;
+      const contentDirs = getContentPathList(contentPaths);
       return {
         resolve: {
           alias: {
@@ -204,7 +209,7 @@ export default function pluginContentPages(
           rules: [
             {
               test: /(\.mdx?)$/,
-              include: getContentPathList(contentPaths)
+              include: contentDirs
                 // Trailing slash is important, see https://github.com/facebook/docusaurus/pull/3970
                 .map(addTrailingPathSeparator),
               use: [
@@ -217,7 +222,10 @@ export default function pluginContentPages(
                     beforeDefaultRehypePlugins,
                     beforeDefaultRemarkPlugins,
                     staticDir: path.join(siteDir, STATIC_DIR_NAME),
-                    isMDXPartial: createMatcher(options.exclude),
+                    isMDXPartial: createAbsoluteFilePathMatcher(
+                      options.exclude,
+                      contentDirs,
+                    ),
                     metadataPath: (mdxPath: string) => {
                       // Note that metadataPath must be the same/in-sync as
                       // the path from createData for each MDX.

packages/docusaurus-utils/src/__tests__/globUtils.test.ts

@@ -0,0 +1,109 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+import {
+  GlobExcludeDefault,
+  createMatcher,
+  createAbsoluteFilePathMatcher,
+} from '../globUtils';
+
+describe('createMatcher', () => {
+  const matcher = createMatcher(GlobExcludeDefault);
+
+  test('match default exclude MD/MDX partials correctly', () => {
+    expect(matcher('doc.md')).toEqual(false);
+    expect(matcher('category/doc.md')).toEqual(false);
+    expect(matcher('category/subcategory/doc.md')).toEqual(false);
+    //
+    expect(matcher('doc.mdx')).toEqual(false);
+    expect(matcher('category/doc.mdx')).toEqual(false);
+    expect(matcher('category/subcategory/doc.mdx')).toEqual(false);
+    //
+    expect(matcher('_doc.md')).toEqual(true);
+    expect(matcher('category/_doc.md')).toEqual(true);
+    expect(matcher('category/subcategory/_doc.md')).toEqual(true);
+    expect(matcher('_category/doc.md')).toEqual(true);
+    expect(matcher('_category/subcategory/doc.md')).toEqual(true);
+    expect(matcher('category/_subcategory/doc.md')).toEqual(true);
+  });
+
+  test('match default exclude tests correctly', () => {
+    expect(matcher('xyz.js')).toEqual(false);
+    expect(matcher('xyz.ts')).toEqual(false);
+    expect(matcher('xyz.jsx')).toEqual(false);
+    expect(matcher('xyz.tsx')).toEqual(false);
+    expect(matcher('folder/xyz.js')).toEqual(false);
+    expect(matcher('folder/xyz.ts')).toEqual(false);
+    expect(matcher('folder/xyz.jsx')).toEqual(false);
+    expect(matcher('folder/xyz.tsx')).toEqual(false);
+    //
+    expect(matcher('xyz.test.js')).toEqual(true);
+    expect(matcher('xyz.test.ts')).toEqual(true);
+    expect(matcher('xyz.test.jsx')).toEqual(true);
+    expect(matcher('xyz.test.tsx')).toEqual(true);
+    expect(matcher('folder/xyz.test.js')).toEqual(true);
+    expect(matcher('folder/xyz.test.ts')).toEqual(true);
+    expect(matcher('folder/xyz.test.jsx')).toEqual(true);
+    expect(matcher('folder/xyz.test.tsx')).toEqual(true);
+    expect(matcher('folder/subfolder/xyz.test.js')).toEqual(true);
+    expect(matcher('folder/subfolder/xyz.test.ts')).toEqual(true);
+    expect(matcher('folder/subfolder/xyz.test.jsx')).toEqual(true);
+    expect(matcher('folder/subfolder/xyz.test.tsx')).toEqual(true);
+    //
+    expect(matcher('__tests__/subfolder/xyz.js')).toEqual(true);
+    expect(matcher('__tests__/subfolder/xyz.ts')).toEqual(true);
+    expect(matcher('__tests__/subfolder/xyz.jsx')).toEqual(true);
+    expect(matcher('__tests__/subfolder/xyz.tsx')).toEqual(true);
+    expect(matcher('folder/__tests__/xyz.js')).toEqual(true);
+    expect(matcher('folder/__tests__/xyz.ts')).toEqual(true);
+    expect(matcher('folder/__tests__/xyz.jsx')).toEqual(true);
+    expect(matcher('folder/__tests__/xyz.tsx')).toEqual(true);
+  });
+});
+
+describe('createAbsoluteFilePathMatcher', () => {
+  const rootFolders = ['/_root/docs', '/root/_docs/', '/__test__/website/src'];
+
+  const matcher = createAbsoluteFilePathMatcher(
+    GlobExcludeDefault,
+    rootFolders,
+  );
+
+  test('match default exclude MD/MDX partials correctly', () => {
+    expect(matcher('/_root/docs/myDoc.md')).toEqual(false);
+    expect(matcher('/_root/docs/myDoc.mdx')).toEqual(false);
+    expect(matcher('/root/_docs/myDoc.md')).toEqual(false);
+    expect(matcher('/root/_docs/myDoc.mdx')).toEqual(false);
+    expect(matcher('/_root/docs/category/myDoc.md')).toEqual(false);
+    expect(matcher('/_root/docs/category/myDoc.mdx')).toEqual(false);
+    expect(matcher('/root/_docs/category/myDoc.md')).toEqual(false);
+    expect(matcher('/root/_docs/category/myDoc.mdx')).toEqual(false);
+    //
+    expect(matcher('/_root/docs/_myDoc.md')).toEqual(true);
+    expect(matcher('/_root/docs/_myDoc.mdx')).toEqual(true);
+    expect(matcher('/root/_docs/_myDoc.md')).toEqual(true);
+    expect(matcher('/root/_docs/_myDoc.mdx')).toEqual(true);
+    expect(matcher('/_root/docs/_category/myDoc.md')).toEqual(true);
+    expect(matcher('/_root/docs/_category/myDoc.mdx')).toEqual(true);
+    expect(matcher('/root/_docs/_category/myDoc.md')).toEqual(true);
+    expect(matcher('/root/_docs/_category/myDoc.mdx')).toEqual(true);
+  });
+
+  test('match default exclude tests correctly', () => {
+    expect(matcher('/__test__/website/src/xyz.js')).toEqual(false);
+    expect(matcher('/__test__/website/src/__test__/xyz.js')).toEqual(true);
+    expect(matcher('/__test__/website/src/xyz.test.js')).toEqual(true);
+  });
+
+  test('throw if file is not contained in any root doc', () => {
+    expect(() =>
+      matcher('/bad/path/myDoc.md'),
+    ).toThrowErrorMatchingInlineSnapshot(
+      `"createAbsoluteFilePathMatcher unexpected error, absoluteFilePath=/bad/path/myDoc.md was not contained in any of the root folders [\\"/_root/docs\\",\\"/root/_docs/\\",\\"/__test__/website/src\\"]"`,
+    );
+  });
+});

packages/docusaurus-utils/src/globUtils.ts

@@ -8,8 +8,8 @@
 // Globby/Micromatch are the 2 libs we use in Docusaurus consistently
 
 export {default as Globby} from 'globby';
-
 import Micromatch from 'micromatch'; // Note: Micromatch is used by Globby
+import path from 'path';
 
 // The default patterns we ignore when globbing
 // using _ prefix for exclusion by convention
@@ -33,3 +33,31 @@ export function createMatcher(patterns: string[]): Matcher {
   );
   return (str) => regexp.test(str);
 }
+
+// We use match patterns like '**/_*/**',
+// This function permits to help to:
+// Match /user/sebastien/website/docs/_partials/xyz.md
+// Ignore /user/_sebastien/website/docs/partials/xyz.md
+export function createAbsoluteFilePathMatcher(
+  patterns: string[],
+  rootFolders: string[],
+): Matcher {
+  const matcher = createMatcher(patterns);
+
+  function getRelativeFilePath(absoluteFilePath: string) {
+    const rootFolder = rootFolders.find((folderPath) =>
+      absoluteFilePath.startsWith(folderPath),
+    );
+    if (!rootFolder) {
+      throw new Error(
+        `createAbsoluteFilePathMatcher unexpected error, absoluteFilePath=${absoluteFilePath} was not contained in any of the root folders ${JSON.stringify(
+          rootFolders,
+        )}`,
+      );
+    }
+    return path.relative(rootFolder, absoluteFilePath);
+  }
+
+  return (absoluteFilePath: string) =>
+    matcher(getRelativeFilePath(absoluteFilePath));
+}

packages/docusaurus-utils/src/index.ts

@@ -31,7 +31,12 @@ export * from './markdownParser';
 export * from './markdownLinks';
 export * from './escapePath';
 export {md5Hash, simpleHash, docuHash} from './hashUtils';
-export {Globby, GlobExcludeDefault, createMatcher} from './globUtils';
+export {
+  Globby,
+  GlobExcludeDefault,
+  createMatcher,
+  createAbsoluteFilePathMatcher,
+} from './globUtils';
 
 const fileHash = new Map();
 export async function generate(

website/_dogfooding/README.md

@@ -0,0 +1,13 @@
+# Docusaurus website dogfooding
+
+This is where we test edge cases of Docusaurus by using fancy configs, ensuring they all don't fail during a real site build.
+
+Eventually, we could move these tests later so another test site? Note there is value to keep seeing the live result of those tests.
+
+Fancy things we can test for here:
+
+- Plugin Multi-instance
+- Symlinks support
+- Webpack configs
+- \_ prefix convention
+- Huge sidebars impact

website/_dogfooding/_pages-tests/_pagePartial.md

@@ -0,0 +1,3 @@
+### Page partial content
+
+This is text coming from a page partial

website/_dogfooding/_pages-tests/index.md

@@ -0,0 +1,9 @@
+## Page
+
+Let's import some MDX partial:
+
+```mdx-code-block
+import PagePartial from "./_pagePartial.md"
+
+<PagePartial />
+```

website/_dogfooding/docs-tests-symlink

@@ -0,0 +1 @@
+_docs-tests