Skip to content

Commit

Permalink
Merge pull request #374 from easyops-cn/steve/unlisted-content
Browse files Browse the repository at this point in the history
feat: do not index unlisted content
  • Loading branch information
weareoutman authored Nov 24, 2023
2 parents 683687d + b0c425d commit d45732f
Show file tree
Hide file tree
Showing 6 changed files with 53 additions and 9 deletions.
1 change: 1 addition & 0 deletions .eslintrc
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
}
],
"rules": {
"@typescript-eslint/no-explicit-any": "warn",
"react-hooks/rules-of-hooks": "error",
"react-hooks/exhaustive-deps": "warn"
}
Expand Down
20 changes: 19 additions & 1 deletion docusaurus-search-local/src/server/utils/parse.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import {
import { parse } from "./parse";

describe("parse", () => {
test.each<[string, "docs" | "blog" | "page", ParsedDocument]>([
test.each<[string, "docs" | "blog" | "page", ParsedDocument | null]>([
[
`<body>
<article>
Expand Down Expand Up @@ -84,6 +84,24 @@ describe("parse", () => {
breadcrumb: [],
},
],
[
`<head>
<meta name="robots" content="noindex, nofollow" />
</head>
<body>
<article>
<header>
<h1>Hello World</h1>
</header>
<main>
<span class="ignore">Test</span>
Peace.
</main>
</article>
</body>`,
"docs",
null,
],
])("parse(...) should work", (html, type, doc) => {
expect(
parse(html, type, "", {
Expand Down
9 changes: 8 additions & 1 deletion docusaurus-search-local/src/server/utils/parse.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,15 @@ export function parse(
type: "docs" | "blog" | "page",
url: string,
{ ignoreCssSelectors }: ProcessedPluginOptions
): ParsedDocument {
): ParsedDocument | null {
const $ = cheerio.load(html);

const robotsMeta = $('meta[name="robots"]');
if (robotsMeta.attr("content")?.includes("noindex")) {
// Unlisted content
return null;
}

// Remove copy buttons from code boxes
$('div[class^="mdxCodeBlock_"] button').remove();

Expand Down
10 changes: 9 additions & 1 deletion docusaurus-search-local/src/server/utils/scanDocuments.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,12 @@ describe("scanDocuments", () => {
url: "/2",
type: "page",
},
{
// Unlisted
filePath: "/tmp/3",
url: "/3",
type: "docs",
},
];
mockParse.mockImplementation((html) => {
if (html.includes("1")) {
Expand All @@ -52,7 +58,7 @@ describe("scanDocuments", () => {
],
breadcrumb: ["Docs"],
};
} else {
} else if (html.includes("2")) {
return {
pageTitle: "Hello First Page",
sections: [
Expand All @@ -64,6 +70,8 @@ describe("scanDocuments", () => {
],
breadcrumb: [],
};
} else {
return null;
}
});
const allDocuments = await scanDocuments(
Expand Down
13 changes: 7 additions & 6 deletions docusaurus-search-local/src/server/utils/scanDocuments.ts
Original file line number Diff line number Diff line change
Expand Up @@ -35,12 +35,13 @@ export async function scanDocuments(
);

const html = await readFileAsync(filePath, { encoding: "utf8" });
const { pageTitle, sections, breadcrumb } = parse(
html,
type,
url,
config
);

const parsed = parse(html, type, url, config);
if (!parsed) {
// Unlisted content
return;
}
const { pageTitle, sections, breadcrumb } = parsed;

const titleId = getNextDocId();

Expand Down
9 changes: 9 additions & 0 deletions website/docs/unlisted-post.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
---
unlisted: true
---

# Unlisted Post

This unlisted blog post should be "hidden" in production, but remain accessible.

It is filtered from the sidebar, sitemap, SEO indexation...

0 comments on commit d45732f

Please sign in to comment.