Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

🔧 MAINTAIN: Split sphinx extension into separate module #409

Merged
merged 1 commit into from
Apr 28, 2022
Merged

🔧 MAINTAIN: Split sphinx extension into separate module #409

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion myst_nb/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
def setup(app):
"""Sphinx extension setup."""
# we import this locally, so sphinx is not automatically imported
from .sphinx_ import sphinx_setup
from .sphinx_ext import sphinx_setup

return sphinx_setup(app)

Expand Down
2 changes: 1 addition & 1 deletion myst_nb/docutils_.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
"""A parser for docutils."""
"""The docutils parser implementation for myst-nb."""
from __future__ import annotations

from functools import partial
Expand Down
196 changes: 2 additions & 194 deletions myst_nb/sphinx_.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,17 +1,14 @@
"""An extension for sphinx"""
"""The sphinx parser implementation for myst-nb."""
from __future__ import annotations

from collections import defaultdict
from importlib import resources as import_resources
import json
import os
from pathlib import Path
from typing import Any, DefaultDict, cast

from docutils import nodes
from markdown_it.token import Token
from markdown_it.tree import SyntaxTreeNode
from myst_parser import setup_sphinx as setup_myst_parser
from myst_parser.docutils_renderer import token_line
from myst_parser.main import MdParserConfig, create_md_parser
from myst_parser.sphinx_parser import MystParser
Expand All @@ -22,33 +19,23 @@
from sphinx.environment.collectors import EnvironmentCollector
from sphinx.transforms.post_transforms import SphinxPostTransform
from sphinx.util import logging as sphinx_logging
from sphinx.util.fileutil import copy_asset_file

from myst_nb import __version__, static
from myst_nb._compat import findall
from myst_nb.core.config import NbParserConfig
from myst_nb.core.execute import ExecutionResult, execute_notebook
from myst_nb.core.loggers import DEFAULT_LOG_TYPE, SphinxDocLogger
from myst_nb.core.parse import nb_node_to_dict, notebook_to_tokens
from myst_nb.core.preprocess import preprocess_notebook
from myst_nb.core.read import UnexpectedCellDirective, create_nb_reader
from myst_nb.core.read import create_nb_reader
from myst_nb.core.render import (
MimeData,
NbElementRenderer,
create_figure_context,
get_mime_priority,
load_renderer,
)
from myst_nb.ext.download import NbDownloadRole
from myst_nb.glue.crossref import ReplacePendingGlueReferences
from myst_nb.glue.domain import NbGlueDomain

SPHINX_LOGGER = sphinx_logging.getLogger(__name__)
OUTPUT_FOLDER = "jupyter_execute"

# used for deprecated config values,
# so we can tell if they have been set by a user, and warn them
UNSET = "--unset--"


class SphinxEnvType(BuildEnvironment):
Expand All @@ -60,185 +47,6 @@ class SphinxEnvType(BuildEnvironment):
nb_new_exec_data: bool


def sphinx_setup(app: Sphinx):
"""Initialize Sphinx extension."""
# note, for core events overview, see:
# https://www.sphinx-doc.org/en/master/extdev/appapi.html#sphinx-core-events

# Add myst-parser configuration and transforms (but does not add the parser)
setup_myst_parser(app)

# add myst-nb configuration variables
for name, default, field in NbParserConfig().as_triple():
if not field.metadata.get("sphinx_exclude"):
# TODO add types?
app.add_config_value(f"nb_{name}", default, "env", Any)
if "legacy_name" in field.metadata:
app.add_config_value(
f"{field.metadata['legacy_name']}", UNSET, "env", Any
)
# Handle non-standard deprecation
app.add_config_value("nb_render_priority", UNSET, "env", Any)

# generate notebook configuration from Sphinx configuration
# this also validates the configuration values
app.connect("builder-inited", create_mystnb_config)

# add parser and default associated file suffixes
app.add_source_parser(Parser)
app.add_source_suffix(".md", "myst-nb", override=True)
app.add_source_suffix(".ipynb", "myst-nb")
# add additional file suffixes for parsing
app.connect("config-inited", add_nb_custom_formats)
# ensure notebook checkpoints are excluded from parsing
app.connect("config-inited", add_exclude_patterns)
# add collector for myst nb specific data
app.add_env_collector(NbMetadataCollector)

# TODO add an event which, if any files have been removed,
# all jupyter-cache stage records with a non-existent path are removed
# (just to keep it "tidy", but won't affect run)

# add directive to ensure all notebook cells are converted
app.add_directive("code-cell", UnexpectedCellDirective, override=True)
app.add_directive("raw-cell", UnexpectedCellDirective, override=True)

# add directive for downloading an executed notebook
app.add_role("nb-download", NbDownloadRole())

# add post-transform for selecting mime type from a bundle
app.add_post_transform(SelectMimeType)
app.add_post_transform(ReplacePendingGlueReferences)

# add HTML resources
app.add_css_file("mystnb.css")
app.connect("build-finished", add_global_html_resources)
# note, this event is only available in Sphinx >= 3.5
app.connect("html-page-context", add_per_page_html_resources)

# add configuration for hiding cell input/output
# TODO replace this, or make it optional
app.setup_extension("sphinx_togglebutton")
app.connect("config-inited", update_togglebutton_classes)

# Note lexers are registered as `pygments.lexers` entry-points
# and so do not need to be added here.

# setup extension for execution statistics tables
# import here, to avoid circular import
from myst_nb.ext.execution_tables import setup_exec_table_extension

setup_exec_table_extension(app)

# add glue roles and directives
# note, we have to add this as a domain, to allow for ':' in the names,
# without a sphinx warning
app.add_domain(NbGlueDomain)

return {
"version": __version__,
"parallel_read_safe": True,
"parallel_write_safe": True,
}


def add_nb_custom_formats(app: Sphinx, config):
"""Add custom conversion formats."""
for suffix in config.nb_custom_formats:
app.add_source_suffix(suffix, "myst-nb", override=True)


def create_mystnb_config(app):
"""Generate notebook configuration from Sphinx configuration"""

# Ignore type checkers because the attribute is dynamically assigned
from sphinx.util.console import bold # type: ignore[attr-defined]

values = {}
for name, _, field in NbParserConfig().as_triple():
if not field.metadata.get("sphinx_exclude"):
values[name] = app.config[f"nb_{name}"]
if "legacy_name" in field.metadata:
legacy_value = app.config[field.metadata["legacy_name"]]
if legacy_value != UNSET:
legacy_name = field.metadata["legacy_name"]
SPHINX_LOGGER.warning(
f"{legacy_name!r} is deprecated for 'nb_{name}' "
f"[{DEFAULT_LOG_TYPE}.config]",
type=DEFAULT_LOG_TYPE,
subtype="config",
)
values[name] = legacy_value
if app.config["nb_render_priority"] != UNSET:
SPHINX_LOGGER.warning(
"'nb_render_priority' is deprecated for 'nb_mime_priority_overrides'"
f"{DEFAULT_LOG_TYPE}.config",
type=DEFAULT_LOG_TYPE,
subtype="config",
)

try:
app.env.mystnb_config = NbParserConfig(**values)
SPHINX_LOGGER.info(
bold("myst-nb v%s:") + " %s", __version__, app.env.mystnb_config
)
except (TypeError, ValueError) as error:
SPHINX_LOGGER.critical("myst-nb configuration invalid: %s", error.args[0])
raise

# update the output_folder (for writing external files like images),
# and the execution_cache_path (for caching notebook outputs)
# to a set path within the sphinx build folder
output_folder = Path(app.outdir).parent.joinpath(OUTPUT_FOLDER).resolve()
exec_cache_path: None | str | Path = app.env.mystnb_config.execution_cache_path
if not exec_cache_path:
exec_cache_path = Path(app.outdir).parent.joinpath(".jupyter_cache").resolve()
app.env.mystnb_config = app.env.mystnb_config.copy(
output_folder=str(output_folder), execution_cache_path=str(exec_cache_path)
)
SPHINX_LOGGER.info(f"Using jupyter-cache at: {exec_cache_path}")


def add_exclude_patterns(app: Sphinx, config):
"""Add default exclude patterns (if not already present)."""
if "**.ipynb_checkpoints" not in config.exclude_patterns:
config.exclude_patterns.append("**.ipynb_checkpoints")


def add_global_html_resources(app: Sphinx, exception):
"""Add HTML resources that apply to all pages."""
# see https://github.com/sphinx-doc/sphinx/issues/1379
if app.builder is not None and app.builder.format == "html" and not exception:
with import_resources.path(static, "mystnb.css") as source_path:
destination = os.path.join(app.builder.outdir, "_static", "mystnb.css")
copy_asset_file(str(source_path), destination)


def add_per_page_html_resources(
app: Sphinx, pagename: str, *args: Any, **kwargs: Any
) -> None:
"""Add JS files for this page, identified from the parsing of the notebook."""
if app.env is None or app.builder is None or app.builder.format != "html":
return
js_files = NbMetadataCollector.get_js_files(app.env, pagename) # type: ignore
for path, kwargs in js_files.values():
app.add_js_file(path, **kwargs) # type: ignore


def update_togglebutton_classes(app: Sphinx, config):
"""Update togglebutton classes to recognise hidden cell inputs/outputs."""
to_add = [
".tag_hide_input div.cell_input",
".tag_hide-input div.cell_input",
".tag_hide_output div.cell_output",
".tag_hide-output div.cell_output",
".tag_hide_cell.cell",
".tag_hide-cell.cell",
]
for selector in to_add:
config.togglebutton_selector += f", {selector}"


class Parser(MystParser):
"""Sphinx parser for Jupyter Notebook formats, containing MyST Markdown."""

Expand Down
Loading