🤩 Inline expressions

README.md

@@ -24,6 +24,41 @@ To install the extension, execute:
 pip install jupyterlab_myst
 ```
 
+## Features
+
+JupyterLab MyST is a fully featured markdown renderer for technical documents, [get started with MyST Markdown](https://myst-tools.org/docs/mystjs/quickstart-myst-markdown). It supports the MyST `{eval}` inline role, which facilitates the interweaving of code outputs and prose. For example, we can use inline expressions to explore the properties of a NumPy array.
+
+In the code cell:
+
+```python
+import numpy as np
+array = np.arange(4)
+```
+
+In the markdown cell:
+
+```markdown
+Let's consider the following array: {eval}`array`.
+
+We can compute the total: {eval}`array.sum()` and the maximum value is {eval}`array.max()`.
+```
+
+This will evaluate inline, and show:
+
+```text
+Let's consider the following array: array([0, 1, 2, 3]).
+
+We can compute the total: 6 and the maximum value is 3.
+```
+
+You can also use this with `ipywidgets`, and have inline interactive text:
+
+![](./images/cookies.gif)
+
+Or with `matplotlib` to show inline spark-lines:
+
+![](./images/stock-price.gif)
+
 ## Usage
 
 [MyST][myst-quickstart] is a flavour of Markdown, which combines the fluid experience of writing Markdown with the programmable extensibility of reStructuredText. This extension for JupyterLab makes it easier to develop rich, computational narratives, technical documentation, and open scientific communication.

binder/environment.yml

@@ -18,4 +18,6 @@ dependencies:
   - pip
   - wheel
   # additional packages for demos
-  # - ipywidgets
+  - numpy
+  - matplotlib
+  - ipywidgets

src/CellMetadataProvider.tsx

@@ -0,0 +1,35 @@
+import { IRenderMimeRegistry } from '@jupyterlab/rendermime';
+import React, { createContext, useContext } from 'react';
+import { IUserExpressionMetadata } from './metadata';
+
+type MetadataState = {
+  metadata?: IUserExpressionMetadata[];
+  trusted?: boolean;
+  rendermime?: IRenderMimeRegistry;
+};
+
+const MetadataContext = createContext<MetadataState | undefined>(undefined);
+
+// Create a provider for components to consume and subscribe to changes
+export function CellMetadataProvider({
+  metadata,
+  trusted,
+  rendermime,
+  children
+}: {
+  metadata: IUserExpressionMetadata[] | undefined;
+  trusted: boolean;
+  rendermime: IRenderMimeRegistry;
+  children: React.ReactNode;
+}): JSX.Element {
+  return (
+    <MetadataContext.Provider value={{ metadata, trusted, rendermime }}>
+      {children}
+    </MetadataContext.Provider>
+  );
+}
+
+export function useCellMetadata(): MetadataState {
+  const state = useContext(MetadataContext) ?? {};
+  return state;
+}

src/MySTContentFactory.ts

@@ -10,6 +10,6 @@ export class MySTContentFactory extends NotebookPanel.ContentFactory {
     if (!options.contentFactory) {
       options.contentFactory = this;
     }
-    return new MySTMarkdownCell(options).initializeState();
+    return new MySTMarkdownCell(options, parent).initializeState();
   }
 }

src/MySTMarkdownCell.tsx

@@ -17,11 +17,28 @@ import { useParse } from 'myst-to-react';
 import { parseContent } from './myst';
 import { IMySTMarkdownCell } from './types';
 import { linkFactory } from './links';
+import { selectAll } from 'unist-util-select';
+
+import { PromiseDelegate } from '@lumino/coreutils';
+import { metadataSection, IUserExpressionMetadata } from './metadata';
+import { CellMetadataProvider } from './CellMetadataProvider';
+import { IRenderMimeRegistry } from '@jupyterlab/rendermime';
 
 export class MySTMarkdownCell
   extends MarkdownCell
   implements IMySTMarkdownCell
 {
+  private _doneRendering = new PromiseDelegate<void>();
+
+  private __rendermime: IRenderMimeRegistry;
+
+  constructor(options: MarkdownCell.IOptions, parent: StaticNotebook) {
+    super(options);
+    // Note we cannot clone this, and it must be the parents (the notebooks)
+    this.__rendermime = parent.rendermime;
+    // this.__rendermime.addFactory(textRendererFactory);
+  }
+
   myst: {
     pre?: GenericParent;
     post?: GenericParent;
@@ -34,13 +51,34 @@ export class MySTMarkdownCell
       const node = document.createElement('div');
       this.myst = { node };
     }
+
+    this._doneRendering = new PromiseDelegate<void>();
     const notebook = this.parent as StaticNotebook;
     this.myst.pre = undefined;
-    parseContent(notebook);
+    const parseComplete = parseContent(notebook);
     const widget = new Widget({ node: this.myst.node });
     widget.addClass('myst');
     this.addClass('jp-MySTMarkdownCell');
     this.inputArea.renderInput(widget);
+    if (parseComplete) {
+      parseComplete.then(() => this._doneRendering.resolve());
+    } else {
+      // Something went wrong, reject the rendering promise
+      this._doneRendering.reject('Unknown error with parsing MyST Markdown.');
+    }
+  }
+
+  /**
+   * Whether the Markdown renderer has finished rendering.
+   */
+  get doneRendering(): Promise<void> {
+    return this._doneRendering.promise;
+  }
+
+  get expressions(): string[] {
+    const { post: mdast } = this.myst ?? {};
+    const expressions = selectAll('inlineExpression', mdast);
+    return expressions.map(node => (node as any).value);
   }
 
   mystRender(): void {
@@ -56,19 +94,30 @@ export class MySTMarkdownCell
     const { references, frontmatter } = notebook.myst;
 
     const children = useParse(mdast as any, renderers);
-
+    const metadata = this.model.metadata.get(
+      metadataSection
+    ) as IUserExpressionMetadata[];
     render(
       <ThemeProvider
         theme={Theme.light}
         Link={linkFactory(notebook)}
         renderers={renderers}
       >
-        <TabStateProvider>
-          <ReferencesProvider references={references} frontmatter={frontmatter}>
-            {isFirstCell && <FrontmatterBlock frontmatter={frontmatter} />}
-            {children}
-          </ReferencesProvider>
-        </TabStateProvider>
+        <CellMetadataProvider
+          metadata={metadata}
+          trusted={this.model.trusted}
+          rendermime={this.__rendermime}
+        >
+          <TabStateProvider>
+            <ReferencesProvider
+              references={references}
+              frontmatter={frontmatter}
+            >
+              {isFirstCell && <FrontmatterBlock frontmatter={frontmatter} />}
+              {children}
+            </ReferencesProvider>
+          </TabStateProvider>
+        </CellMetadataProvider>
       </ThemeProvider>,
       this.myst.node
     );

src/actions.ts

@@ -0,0 +1,131 @@
+import { ISessionContext } from '@jupyterlab/apputils';
+import { Cell, IMarkdownCellModel } from '@jupyterlab/cells';
+import { KernelMessage } from '@jupyterlab/services';
+import { JSONObject } from '@lumino/coreutils';
+import { IExpressionResult } from './userExpressions';
+import { IUserExpressionMetadata, metadataSection } from './metadata';
+import {
+  Notebook,
+  NotebookPanel,
+  INotebookTracker
+} from '@jupyterlab/notebook';
+import { IMySTMarkdownCell } from './types';
+
+function isMarkdownCell(cell: Cell): cell is IMySTMarkdownCell {
+  return cell.model.type === 'markdown';
+}
+
+/**
+ * Load user expressions for given XMarkdown cell from kernel.
+ * Store results in cell attachments.
+ */
+export async function executeUserExpressions(
+  cell: IMySTMarkdownCell,
+  sessionContext: ISessionContext
+): Promise<void> {
+  // Check we have a kernel
+  const kernel = sessionContext.session?.kernel;
+  if (!kernel) {
+    throw new Error('Session has no kernel.');
+  }
+
+  const model = cell.model as IMarkdownCellModel;
+  const cellId = { cellId: model.id };
+
+  // Can simplify with `Object.fromEntries` here
+  // requires ts compiler upgrade!
+
+  // Build ordered map from string index to node
+  const namedExpressions = new Map(
+    cell.expressions.map((expr, index) => [`${index}`, expr])
+  );
+
+  // Extract expression values
+  const userExpressions: JSONObject = {};
+  namedExpressions.forEach((expr, key) => {
+    userExpressions[key] = expr;
+  });
+
+  // Populate request data
+  const content: KernelMessage.IExecuteRequestMsg['content'] = {
+    code: '',
+    user_expressions: userExpressions
+  };
+
+  // Perform request
+  console.debug('Performing kernel request', content);
+  const future = kernel.requestExecute(content, false, {
+    ...model.metadata.toJSON(),
+    ...cellId
+  });
+
+  // Set response handler
+  future.onReply = (msg: KernelMessage.IExecuteReplyMsg) => {
+    console.debug('Handling kernel response', msg);
+    // Only work with `ok` results
+    const content = msg.content;
+    if (content.status !== 'ok') {
+      console.error('Kernel response was not OK', msg);
+      return;
+    }
+
+    console.debug('Clear existing metadata');
+    // Clear metadata if present
+    cell.model.metadata.delete(metadataSection);
+
+    // Store results as metadata
+    const expressions: IUserExpressionMetadata[] = [];
+    for (const key in content.user_expressions) {
+      const expr = namedExpressions.get(key);
+
+      if (expr === undefined) {
+        console.error(
+          "namedExpressions doesn't have key. This should never happen"
+        );
+        continue;
+      }
+      const result = content.user_expressions[key] as IExpressionResult;
+
+      const expressionMetadata: IUserExpressionMetadata = {
+        expression: expr,
+        result: result
+      };
+      expressions.push(expressionMetadata);
+
+      console.debug(`Saving ${expr} to cell attachments`, expressionMetadata);
+    }
+
+    // Update cell metadata
+    cell.model.metadata.set(metadataSection, expressions);
+    // Rerender the cell with React
+    console.debug('Render cell after the metadata is added');
+    cell.mystRender();
+  };
+
+  await future.done;
+}
+
+export function notebookExecuted(
+  notebook: Notebook,
+  cell: Cell,
+  tracker: INotebookTracker
+): void {
+  // Find the Notebook panel
+  const panel = tracker.find((w: NotebookPanel) => {
+    return w.content === notebook;
+  });
+  // Retrieve the kernel context
+  const ctx = panel?.sessionContext;
+  if (ctx === undefined) {
+    return;
+  }
+  // Load the user expressions for the given cell.
+  if (!isMarkdownCell(cell)) {
+    return;
+  }
+  console.debug(
+    `Markdown cell ${cell.model.id} was executed, waiting for render to complete ...`
+  );
+
+  cell.doneRendering.then(() => executeUserExpressions(cell, ctx));
+}

src/index.ts

@@ -9,8 +9,11 @@ import {
   INotebookTracker,
   INotebookWidgetFactory,
   NotebookPanel,
-  NotebookWidgetFactory
+  NotebookWidgetFactory,
+  NotebookActions,
+  Notebook
 } from '@jupyterlab/notebook';
+import { Cell } from '@jupyterlab/cells';
 import { MySTContentFactory } from './MySTContentFactory';
 
 import { ISessionContextDialogs } from '@jupyterlab/apputils';
@@ -20,6 +23,7 @@ import { ITranslator } from '@jupyterlab/translation';
 import { LabIcon } from '@jupyterlab/ui-components';
 
 import mystIconSvg from '../style/mystlogo.svg';
+import { notebookExecuted } from './actions';
 
 const mystIcon = new LabIcon({
   name: 'myst-notebook-extension:mystIcon',
@@ -117,4 +121,25 @@ const legacyPlugin: JupyterFrontEndPlugin<void> = {
   }
 };
 
-export default [plugin, legacyPlugin];
+/**
+ * The notebook cell executor.
+ */
+const executor: JupyterFrontEndPlugin<void> = {
+  id: 'jupyterlab-myst:executor',
+  requires: [INotebookTracker],
+  autoStart: true,
+  activate: (app: JupyterFrontEnd, tracker: INotebookTracker) => {
+    console.log('Using jupyterlab-myst:executor');
+
+    NotebookActions.executed.connect(
+      (sender: any, value: { notebook: Notebook; cell: Cell }) => {
+        const { notebook, cell } = value;
+        notebookExecuted(notebook, cell, tracker);
+      }
+    );
+
+    return;
+  }
+};
+
+export default [plugin, legacyPlugin, executor];