Skip to content

Commit

Permalink
feat: use effect
Browse files Browse the repository at this point in the history 
MDX Compilation is now functional!
(as in, FP, not as in, working; that's been the case for a while)
  • Loading branch information
@lishaduck
lishaduck committed Jun 12, 2024
1 parent 8899262 commit 2889a25
Show file tree
Hide file tree
Showing 3 changed files with 141 additions and 114 deletions.
5 changes: 3 additions & 2 deletions deno.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,11 +32,13 @@
"@preact/signals": "https://esm.sh/v135/*@preact/signals@1.2.3",
"@preact/signals-core": "https://esm.sh/v135/*@preact/signals-core@1.6.0",
"@std/assert": "jsr:@std/assert@^0.224.0",
"@std/fs": "jsr:@std/fs@^0.224.0",
"@std/path": "jsr:@std/path@^0.224.0",
"@tailwindcss/forms": "npm:@tailwindcss/forms@0.5.7",
"@tailwindcss/typography": "npm:@tailwindcss/typography@0.5.13",
"@vendor/": "./vendor/",
"clsx": "npm:clsx@2.1.1",
"effect": "npm:effect@3.1.5",
"idb-keyval": "npm:idb-keyval@6.2.1",
"openai": "https://deno.land/x/openai@v4.47.1/mod.ts",
"openai/": "https://deno.land/x/openai@v4.47.1/",
Expand Down Expand Up @@ -117,7 +119,6 @@
],
"include": [
"ban-untagged-todo",
"explicit-function-return-type",
"explicit-module-boundary-types",
"guard-for-in",
"no-await-in-loop",
Expand All @@ -142,4 +143,4 @@
"include": [],
"entrypoint": "src/main.ts"
}
}
}
228 changes: 116 additions & 112 deletions tool/compile-mdx.ts
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import { type CompileOptions, compile } from "@mdx-js/mdx";
import { dirname, fromFileUrl, join, resolve } from "@std/path";
import { dirname, fromFileUrl, join, relative, resolve } from "@std/path";
import { Cause, Chunk, Console, Effect, Order, Stream } from "effect";
import rehypeMathjax from "rehype-mathjax";
import remarkFrontmatter from "remark-frontmatter";
import remarkLintCheckboxContentIndent from "remark-lint-checkbox-content-indent";
Expand All @@ -23,6 +24,7 @@ import {
solutionPagesSchema,
titleList,
} from "../src/utils/solutions.ts";
import { readTextFile, walkDir, writeTextFile } from "./effect-deno.ts";

declare module "vfile" {
export interface DataMap {
Expand All @@ -44,26 +46,22 @@ const utilsDir = join(srcDir, "utils");
/**
* Compile the MDX files into JS.
*/
async function run(): Promise<void> {
const program = Effect.gen(function* () {
const initialFiles = getSolutions(contentDir);
const compiledFiles = compileSolutions(initialFiles);

const files: VFile[] = [];
for await (const file of compiledFiles) {
files.push(file);
}
files.sort(sortFiles);

lint(files);
const chunk = yield* compiledFiles.pipe(Stream.runCollect);
const files = chunk.pipe(Chunk.sort(sortFiles)).pipe(Chunk.toArray);

await Promise.all([
yield* Effect.all([
lint(files),
writeSolutions(files),
staticImports(files),
categories(files),
]);

console.info(`Compiled ${files.length} MDX files into JS.`);
}
yield* Console.info(`Compiled ${files.length} MDX files into JS.`);
});

/**
* Get all of the MDX files in a directory.
Expand All @@ -74,19 +72,23 @@ async function run(): Promise<void> {
* @remarks
* This is an async generator because it's recursive.
*/
async function* getSolutions(
const getSolutions = (
basePath: string,
currentPath = "",
): AsyncGenerator<VFile, void, unknown> {
for await (const entry of Deno.readDir(resolve(basePath, currentPath))) {
const fullPath = resolve(basePath, currentPath, entry.name);
if (entry.isFile && entry.name.match(/\.mdx?$/) !== null) {
yield getSolution(fullPath, currentPath, entry.name);
} else if (entry.isDirectory) {
yield* getSolutions(basePath, join(currentPath, entry.name));
}
}
}
): Stream.Stream<VFile, Cause.UnknownException> =>
walkDir(basePath, (e) => new Cause.UnknownException(e), {
exts: ["mdx"],
})
.pipe(Stream.filter((entry) => entry.isFile))
.pipe(
Stream.mapEffect((entry) =>
getSolution(
entry.path,
dirname(relative(basePath, entry.path)),
basePath,
entry.name,
),
),
);

/**
* Get the contents of a file.
Expand All @@ -96,20 +98,22 @@ async function* getSolutions(
* @param fileName - The name of the file.
* @returns The file's contents.
*/
async function getSolution(
const getSolution = (
fullPath: string,
relPath: string,
basePath: string,
fileName: string,
): Promise<VFile> {
const fileContent = await Deno.readTextFile(fullPath);

return new VFile({
value: fileContent,
dirname: relPath,
basename: fileName,
cwd: fullPath,
): Effect.Effect<VFile, Cause.UnknownException> =>
Effect.gen(function* () {
const fileContent = yield* readTextFile(fullPath);

return new VFile({
value: fileContent,
dirname: relPath,
basename: fileName,
cwd: basePath,
});
});
}

/** Options for the lint reporter. */
const lintReportOptions = {
Expand All @@ -121,27 +125,24 @@ const lintReportOptions = {
*
* @param files Markdown files.
*/
function lint(files: VFile[]): void {
const lints = reporter(files, lintReportOptions);
if (lints !== "") {
console.error(lints);
Deno.exit(1);
}
}
const lint = (files: VFile[]): Effect.Effect<void, string> =>
Effect.gen(function* () {
const lints = yield* Effect.sync(() => reporter(files, lintReportOptions));
if (lints !== "") {
yield* Effect.fail(lints);
}
});

/**
* Compile MDX files into Preact JSX files.
*
* @param initialFiles A list of virtual MDX files for compilation.
* @returns A list of virtual JS files.
*/
async function* compileSolutions(
initialFiles: AsyncIterable<VFile>,
): AsyncGenerator<VFile, void, unknown> {
for await (const entry of initialFiles) {
yield compileSolution(entry);
}
}
const compileSolutions = (
initialFiles: Stream.Stream<VFile, Cause.UnknownException>,
): Stream.Stream<VFile, Cause.UnknownException> =>
initialFiles.pipe(Stream.mapEffect(compileSolution));

/**
* Plugins for the MDX compilation.
Expand Down Expand Up @@ -177,78 +178,81 @@ const compileOptions = {
* @param file A virtual MDX file.
* @returns A virtual JS file.
*/
async function compileSolution(file: VFile): Promise<VFile> {
matter(file); // Extract the frontmatter into `data.matter`.
const compileSolution = (file: VFile) =>
Effect.gen(function* () {
yield* Effect.sync(() => matter(file)); // Extract the frontmatter into `data.matter`.

const compiled = await compile(file, compileOptions);
compiled.extname = ".jsx";
const compiled = yield* Effect.promise(() => compile(file, compileOptions));
compiled.extname = ".jsx";

// @ts-expect-error: The types are a bit off, but I'm feeling lazy.
compiled.data.matter.category =
compiled.dirname !== "." ? compiled.dirname : compiled.stem;
// @ts-expect-error: The types are a bit off, but I'm feeling lazy.
compiled.data.matter.category =
compiled.dirname !== "." ? compiled.dirname : compiled.stem;

return compiled;
}

async function writeSolutions(solutions: VFile[]): Promise<void> {
const promises: Promise<void>[] = [];
for (const solution of solutions) {
promises.push(writeSolution(solution));
}
return compiled;
});

await Promise.all(promises);
}
const writeSolutions = (
solutions: VFile[],
): Effect.Effect<void, Cause.UnknownException> =>
Effect.all(
solutions.map((solution) => writeSolution(solution)),
{ concurrency: "unbounded" },
);

/**
* Write the file to the disk.
*
* @param solution A file to write.
* @returns A promise resolving when the file's written.
*/
async function writeSolution(solution: VFile): Promise<void> {
return await Deno.writeTextFile(
join(contentDir, solution.path),
solution.toString(),
);
}
const writeSolution = (
solution: VFile,
): Effect.Effect<void, Cause.UnknownException, never> =>
writeTextFile(join(contentDir, solution.path), solution.toString());

function sortFiles(a: VFile, b: VFile): number {
// Adjusting compareByCategory to use Order.Order<VFile>
const compareByCategory: Order.Order<VFile> = (a, b) => {
const aCategory = a.data.matter?.category ?? "";
const bCategory = b.data.matter?.category ?? "";
const aSlug = a.stem ?? "";
const bSlug = b.stem ?? "";

const aCategoryIndex = categoryList.indexOf(aCategory);
const bCategoryIndex = categoryList.indexOf(bCategory);
const aSlugIndex = titleList.indexOf(aSlug);
const bSlugIndex = titleList.indexOf(bSlug);

// If the category is not found in the list, set the index to Infinity.
const categoryComparison =
(aCategoryIndex === -1 ? Number.POSITIVE_INFINITY : aCategoryIndex) -
(bCategoryIndex === -1 ? Number.POSITIVE_INFINITY : bCategoryIndex);

// If the slug is not found in the list, sort alphabetically.
const slugComparison =
aSlugIndex === -1 || bSlugIndex === -1
? aSlug.localeCompare(bSlug)
: aSlugIndex - bSlugIndex;

// If the categories are the same, sort by title, otherwise, sort by category.
return categoryComparison === 0 ? slugComparison : categoryComparison;
}
return Order.number(
aCategoryIndex === -1 ? Number.POSITIVE_INFINITY : aCategoryIndex,
bCategoryIndex === -1 ? Number.POSITIVE_INFINITY : bCategoryIndex,
);
};

const compareBySlug = Order.combine<VFile>(
(a, b) => {
const aSlugIndex = titleList.indexOf(a.stem ?? "");
const bSlugIndex = titleList.indexOf(b.stem ?? "");

// Compare indices if both slugs are found, otherwise return 0 to defer to the next orderer
return aSlugIndex !== -1 && bSlugIndex !== -1
? Order.number(aSlugIndex, bSlugIndex)
: 0;
},
Order.mapInput(Order.string, (file) => file.stem ?? ""),
);

// Main sorting function that uses category and slug comparisons.
const sortFiles = Order.combine(compareByCategory, compareBySlug);

/**
* Write a file containing static imports for all the files.
*/
async function staticImports(files: VFile[]): Promise<void> {
const fileNames = files.map((file): string => file.path);
const icons = files.flatMap((file) =>
file.data.matter?.icon ? [file.data.matter.icon] : [],
);
const fileContent = staticImportsFile(fileNames, icons);
await writeGenFile(fileContent, "imports");
}
const staticImports = (
files: VFile[],
): Effect.Effect<void, Cause.UnknownException> =>
Effect.gen(function* () {
const fileNames = files.map((file): string => file.path);
const icons = files.flatMap((file) =>
file.data.matter?.icon ? [file.data.matter.icon] : [],
);
const fileContent = staticImportsFile(fileNames, icons);
yield* writeGenFile(fileContent, "imports");
});

function createImport(url: string): string {
return `(async () => await import("${url}"));`;
Expand Down Expand Up @@ -282,10 +286,14 @@ function staticImportsFile(files: string[], icons: string[]): string {
/**
* Write a file containing the categories of all the files.
*/
async function categories(files: VFile[]): Promise<void> {
const fileContent = categoriesFile(files);
await writeGenFile(fileContent, "categories");
}
const categories = (
files: VFile[],
): Effect.Effect<void, Cause.UnknownException, never> =>
Effect.gen(function* () {
const fileContent = categoriesFile(files);

yield* writeGenFile(fileContent, "categories");
});

/**
* Create a file containing the categories of all the files.
Expand Down Expand Up @@ -315,14 +323,10 @@ export const solutions = ${json} as const satisfies SolutionPages;
/**
* Write a file to the `utils` directory.
*/
async function writeGenFile(
const writeGenFile = (
fileContent: string,
fileName: string,
): Promise<void> {
await Deno.writeTextFile(
resolve(utilsDir, `${fileName}.gen.ts`),
fileContent,
);
}
): Effect.Effect<void, Cause.UnknownException, never> =>
writeTextFile(resolve(utilsDir, `${fileName}.gen.ts`), fileContent);

await run();
await Effect.runPromise(program);
22 changes: 22 additions & 0 deletions tool/effect-deno.ts
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
import { type WalkEntry, type WalkOptions, walk } from "@std/fs";
import { type Cause, Effect, Stream } from "effect";

export const writeTextFile = (
path: string | URL,
data: string | ReadableStream<string>,
options?: Deno.WriteFileOptions,
): Effect.Effect<void, Cause.UnknownException, never> =>
Effect.tryPromise(() => Deno.writeTextFile(path, data, options));

export const readTextFile = (
path: string | URL,
options?: Deno.ReadFileOptions,
): Effect.Effect<string, Cause.UnknownException, never> =>
Effect.tryPromise(() => Deno.readTextFile(path, options));

export const walkDir = <E>(
root: string | URL,
onError: (e: unknown) => E,
walkOptions?: WalkOptions,
): Stream.Stream<WalkEntry, E, never> =>
Stream.fromAsyncIterable(walk(root, walkOptions), onError);

0 comments on commit 2889a25

Please sign in to comment.