dc.require functionality for importing other scripts (#49)

* make `DatacoreJSRenderer.convert` static since it doesn't use instance data

* implement ability to import/require external scripts

* replace `cachedRead` with regular read

* extract codeblock language union to its own exported type
to avoid repitition

* add script cache

* - set `DatacoreScript.code` to null upon succesful script loading
  (this is to free up memory)
- switch to array for loaded scripts tracking
- add `source` property to `DatacoreScript`

* incorporate `ScriptCache` into local api

* fix order

* nits

(cherry picked from commit 2bdfa8e)

* change `ScriptCache.load`
- this method now takes a parent `DatacoreLocalApi` instance as a parameter
- create a new instance of `DatacoreLocalApi` for each recursively loaded script
- throw on failure instead of returning null (so that it displays properly in a codeblock)

* change `dc` to `parentContext` in failure case

* formatting nits

* corrections to script codeblock detection

* implement suggested changes
- move loaded script cache into ScriptCache
- use `cachedRead`
- use a map instead of an array
- extract `convert` from class into separate utility function
- lift `code` property of `DatacoreScript` up into the return value of `resolveSource`

* more changes
- add promise property to `DatacoreScript` interface
- return a previous promise containing the evaluation result if it exists

* add back anti-cycle guard, make scriptCache class field not readonly

src/api/local-api.tsx

@@ -4,7 +4,7 @@ import { Datacore } from "index/datacore";
 import { SearchResult } from "index/datastore";
 import { IndexQuery } from "index/types/index-query";
 import { Indexable } from "index/types/indexable";
-import { MarkdownPage } from "index/types/markdown";
+import { MarkdownCodeblock, MarkdownPage } from "index/types/markdown";
 import { App } from "obsidian";
 import { useFileMetadata, useFullQuery, useIndexUpdates, useInterning, useQuery } from "ui/hooks";
 import * as luxon from "luxon";
@@ -21,6 +21,8 @@ import { VanillaTable } from "./ui/views/vanilla-table";
 import { Callout } from "./ui/views/callout";
 import { DataArray } from "./data-array";
 import { Coerce } from "./coerce";
+import { DatacoreJSRenderer, asyncEvalInContext } from "ui/javascript";
+import { DatacoreScript, ScriptCache } from "./script-cache";
 
 /** Local API provided to specific codeblocks when they are executing. */
 export class DatacoreLocalApi {
@@ -56,6 +58,28 @@ export class DatacoreLocalApi {
         return this.api.core;
     }
 
+    //////////////////////////////
+    // Script loading utilities //
+    //////////////////////////////
+
+    // Note: Script loading is a bit jank, since it has to be asynchronous due to IO (unless of course we wanted to cache
+    // every single script in the vault in memory, which seems terrible for performance). It functions by essentially
+    // returning a lazy proxy.
+
+    /**
+     * Asynchronously load a javascript block from the given path or link; this method supports loading code blocks
+     * from markdown files via the link option
+     *
+     */
+
+    public scriptCache: ScriptCache = new ScriptCache(this.core.datastore);
+
+    public async require(path: string | Link): Promise<any> {
+        let result = await this.scriptCache.load(path, this);
+        if (!result.successful) throw new Error(result.error);
+        return result.value;
+    }
+
     ///////////////////////
     // General utilities //
     ///////////////////////

src/api/script-cache.ts

@@ -0,0 +1,125 @@
+import { Link } from "expression/link";
+import { Datastore } from "index/datastore";
+import { Failure, Result } from "./result";
+import { MarkdownCodeblock, MarkdownSection } from "index/types/markdown";
+import { DatacoreJSRenderer, ScriptLanguage, asyncEvalInContext, convert } from "ui/javascript";
+import { DatacoreLocalApi } from "./local-api";
+import { Fragment, h } from "preact";
+import { Deferred, deferred } from "utils/deferred";
+export interface DatacoreScript {
+    language: ScriptLanguage;
+    id: string;
+    state: LoadingState;
+    source: string;
+    promise: Deferred<Result<any, string>>;
+}
+export const enum LoadingState {
+    UNDEFINED = -1,
+    LOADING,
+    LOADED,
+}
+/** A simple caching script loader that can load any DAG of script dependencies. */
+export class ScriptCache {
+    /** A static placeholder placed into the `scripts` section to catch recursive loading loops. */
+    private static readonly LOADING_SENTINEL: unique symbol = Symbol("CURRENTLY_LOADING");
+    private static SCRIPT_TAGS: string[] = ["javascript", "typescript", "ts", "js", "tsx", "jsx"];
+
+    public constructor(private store: Datastore) {}
+    /**
+     * Load the given script at the given path, recursively loading any subscripts by correctly injecting `import` into the child contexts.
+     */
+    public loadedScripts: Map<string, DatacoreScript> = new Map<string, DatacoreScript>();
+    public async load(
+        path: string | Link,
+        parentContext: DatacoreLocalApi,
+        sourcePath?: string
+    ): Promise<Result<any, string>> {
+        const source = await this.resolveSource(path, sourcePath);
+        if (!source.successful) return source;
+
+        const {
+            value: { scriptInfo, code },
+        } = source;
+        scriptInfo.state = LoadingState.LOADING;
+        if (!this.loadedScripts.has(scriptInfo.source)) {
+            this.loadedScripts.set(scriptInfo.source, scriptInfo);
+            let dc = new DatacoreLocalApi(parentContext.api, scriptInfo.source);
+            dc.scriptCache = this;
+            const res = await asyncEvalInContext(code, {
+                h,
+                Fragment,
+                dc,
+            });
+            scriptInfo.state = LoadingState.LOADED;
+            scriptInfo.promise.resolve(Result.success(res));
+            this.loadedScripts.set(scriptInfo.source, scriptInfo);
+        }
+        let element = this.loadedScripts.get(scriptInfo.source);
+        if (element?.state === LoadingState.LOADING) {
+            let res: Failure<any, string> = Result.failure(
+                `Script import cycle detected; currently loaded scripts are:\n${[...this.loadedScripts.values()]
+                    .map((x) => `		- ${x.source}`)
+                    .join("\n")}`
+            ) as Failure<any, string>;
+            scriptInfo.promise.reject(res.error);
+            return res;
+        }
+        return element!.promise;
+    }
+
+    /** Attempts to resolve the source to load given a path or link to a markdown section. */
+    private async resolveSource(
+        path: string | Link,
+        sourcePath?: string
+    ): Promise<Result<{ code: string; scriptInfo: DatacoreScript }, string>> {
+        const object = this.store.resolveLink(path);
+        const prefixRegex = /datacore\s?/i;
+        if (!object) return Result.failure("Could not find a script at the given path: " + path.toString());
+
+        const tfile = this.store.vault.getFileByPath(object.$file!);
+        if (!tfile) return Result.failure(`File "${object.$file}" not found`);
+
+        let codeBlock: MarkdownCodeblock | null | undefined;
+        // If the object is a markdown section, search for any javascript codeblocks; otherwise, check if it is a full script file.
+        if (object instanceof MarkdownSection) {
+            codeBlock = object.$blocks
+                .filter((b): b is MarkdownCodeblock => b.$type === "codeblock")
+                .find((cb) =>
+                    cb.$languages.some((language) =>
+                        ScriptCache.SCRIPT_TAGS.includes(language.replace(prefixRegex, ""))
+                    )
+                );
+
+            // Load the script.
+        } else if (object instanceof MarkdownCodeblock) {
+            if (object.$languages.some((x) => ScriptCache.SCRIPT_TAGS.includes(x.replace(prefixRegex, ""))))
+                codeBlock = object;
+        }
+
+        if (!codeBlock)
+            return Result.failure("Could not find a script in the given markdown section: " + path.toString());
+
+        let lang = codeBlock.$languages
+            .find((a) => ScriptCache.SCRIPT_TAGS.includes(a.replace(prefixRegex, "")))!
+            .replace(prefixRegex, "");
+        if (lang?.toLocaleLowerCase() === "typescript") lang = "ts";
+        else if (lang?.toLocaleLowerCase() === "javascript") lang = "js";
+
+        const rawCode = (await this.store.vault.read(tfile))
+            .split(/\r?\n|\r/)
+            .slice(codeBlock.$contentPosition.start, codeBlock.$contentPosition.end + 1)
+            .join("\n");
+        let code = convert(rawCode, lang as ScriptLanguage);
+
+        return Result.success({
+            scriptInfo: {
+                id: codeBlock.$id,
+                language: lang as ScriptLanguage,
+                state: LoadingState.UNDEFINED,
+                source: codeBlock.$file,
+                promise: deferred<Result<string, any>>(),
+            },
+            code,
+        });
+    }
+}

src/ui/javascript.tsx

@@ -4,7 +4,7 @@ import { DatacoreLocalApi } from "api/local-api";
 import { JSX, createElement, h, isValidElement, render, Fragment } from "preact";
 import { unmountComponentAtNode } from "preact/compat";
 import { transform } from "sucrase";
-
+export type ScriptLanguage = "js" | "ts" | "jsx" | "tsx";
 /**
  * Renders a script by executing it and handing it the appropriate React context to execute
  * automatically.
@@ -17,7 +17,7 @@ export class DatacoreJSRenderer extends MarkdownRenderChild {
         public container: HTMLElement,
         public path: string,
         public script: string,
-        public language: "js" | "ts" | "jsx" | "tsx"
+        public language: ScriptLanguage
     ) {
         super(container);
     }
@@ -27,7 +27,7 @@ export class DatacoreJSRenderer extends MarkdownRenderChild {
 
         // Attempt to parse and evaluate the script to produce either a renderable JSX object or a function.
         try {
-            const primitiveScript = this.convert(this.script, this.language);
+            const primitiveScript = convert(this.script, this.language);
 
             const renderable = await asyncEvalInContext(primitiveScript, {
                 dc: this.api,
@@ -69,25 +69,23 @@ export class DatacoreJSRenderer extends MarkdownRenderChild {
     }
 
     /** Attempts to convert the script in the given language to plain javascript; will throw an Error on failure. */
-    private convert(script: string, language: "js" | "ts" | "jsx" | "tsx"): string {
-        switch (language) {
-            case "js":
-                return script;
-            case "jsx":
-                return transform(this.script, { transforms: ["jsx"], jsxPragma: "h", jsxFragmentPragma: "Fragment" })
-                    .code;
-            case "ts":
-                return transform(this.script, { transforms: ["typescript"] }).code;
-            case "tsx":
-                return transform(this.script, {
-                    transforms: ["typescript", "jsx"],
-                    jsxPragma: "h",
-                    jsxFragmentPragma: "Fragment",
-                }).code;
-        }
+}
+export function convert(script: string, language: ScriptLanguage): string {
+    switch (language) {
+        case "js":
+            return script;
+        case "jsx":
+            return transform(script, { transforms: ["jsx"], jsxPragma: "h", jsxFragmentPragma: "Fragment" }).code;
+        case "ts":
+            return transform(script, { transforms: ["typescript"] }).code;
+        case "tsx":
+            return transform(script, {
+                transforms: ["typescript", "jsx"],
+                jsxPragma: "h",
+                jsxFragmentPragma: "Fragment",
+            }).code;
     }
 }
-
 /** Make a renderable element from the returned object; if this transformation is not possible, throw an exception. */
 export function makeRenderableElement(object: any, sourcePath: string): JSX.Element {
     if (typeof object === "function") {