Add support for pluto (#121)

DemoCard Generation from Pluto Notebooks, no tests and no explicit docs
Co-authored-by: Carlo Lucibello <carlo.lucibello@gmail.com>

.github/workflows/UnitTest.yml

@@ -16,7 +16,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        julia-version: ['1.0', '1', 'nightly']
+        julia-version: ['1.6', '1', 'nightly']
         os: [ubuntu-latest, windows-latest, macOS-latest]
 
     steps:

Project.toml

@@ -12,11 +12,15 @@ ImageCore = "a09fc81d-aa75-5fe9-8630-4744c3626534"
 JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
 Literate = "98b081ad-f1c9-55d3-8b20-4c87d4299306"
 Mustache = "ffc61752-8dc7-55ee-8c37-f3e9cdd09e70"
+Pluto = "c3e4b0f8-55cb-11ea-2926-15256bba5781"
+PlutoStaticHTML = "359b1769-a58e-495b-9770-312e911026ad"
 Suppressor = "fd094767-a336-5f1f-9728-57cf17d0bbfb"
 YAML = "ddb6d928-2868-570f-bddf-ab3f9cf99eb6"
 
 [compat]
 Documenter = "0.22, 0.23, 0.24, 0.25, 0.26, 0.27"
+Pluto = "^0.19"
+PlutoStaticHTML = "^6"
 FileIO = "1"
 HTTP = "0.6, 0.7, 0.8, 0.9, 1"
 ImageCore = "0.7, 0.8, 0.9"
@@ -25,7 +29,7 @@ Literate = "2"
 Mustache = "0.5, 1.0"
 Suppressor = "0.2"
 YAML = "0.3, 0.4"
-julia = "1"
+julia = "1.6"
 
 [extras]
 ImageMagick = "6218d12a-5da1-5696-b52f-db25d2ecc6d1"

src/DemoCards.jl

@@ -6,6 +6,8 @@ using Mustache
 using Literate
 using ImageCore
 using FileIO, JSON, YAML
+using Pluto
+using PlutoStaticHTML
 using Suppressor # suppress log generated by 3rd party tools, e.g., Literate
 import HTTP
 using Documenter

src/types/card.jl

@@ -23,7 +23,11 @@ function democard(path::String)::AbstractDemoCard
     if ext in markdown_exts
         return MarkdownDemoCard(path)
     elseif ext in julia_exts
-        return JuliaDemoCard(path)
+        if is_pluto_notebook(path)
+          return PlutoDemoCard(path)
+        else
+          return JuliaDemoCard(path)
+        end
     else
         return UnmatchedCard(path)
     end
@@ -122,3 +126,4 @@ end
 
 include("markdown.jl")
 include("julia.jl")
+include("pluto.jl")

src/types/pluto.jl

@@ -0,0 +1,178 @@
+const pluto_footer = raw"""
+
+---
+
+*This page was generated using [DemoCards.jl](https://github.com/johnnychen94/DemoCards.jl). and [PlutoStaticHTML.jl](https://github.com/rikhuijzer/PlutoStaticHTML.jl)*
+
+
+"""
+
+"""
+    struct PlutoDemoCard <: AbstractDemoCard
+    PlutoDemoCard(path::AbstractString)
+
+Constructs a markdown-format demo card from a pluto notebook.
+
+
+# Fields
+
+Besides `path`, this struct has some other fields:
+
+* `path`: path to the source markdown file
+* `cover`: path to the cover image
+* `id`: cross-reference id
+* `title`: one-line description of the demo card
+* `author`: author(s) of this demo.
+* `date`: the update date of this demo.
+* `julia`: Julia version compatibility
+* `description`: multi-line description of the demo card
+* `hidden`: whether this card is shown in the generated index page
+
+# Configuration
+
+You can pass additional information by adding a YAML front matter to the markdown file.
+Supported items are:
+
+* `cover`: an URL or a relative path to the cover image. If not specified, it will use the first available image link, or all-white image if there's no image links.
+* `description`: a multi-line description to this file, will be displayed when the demo card is hovered. By default it uses `title`.
+* `id`: specify the `id` tag for cross-references. By default it's infered from the filename, e.g., `simple_demo` from `simple demo.md`.
+* `title`: one-line description to this file, will be displayed under the cover image. By default, it's the name of the file (without extension).
+* `author`: author name. If there are multiple authors, split them with semicolon `;`.
+* `julia`: Julia version compatibility. Any string that can be converted to `VersionNumber`
+* `date`: any string contents that can be passed to `Dates.DateTime`. For example, `2020-09-13`.
+* `hidden`: whether this card is shown in the layout of index page. The default value is `false`.
+
+An example of the front matter:
+
+```text
+---
+title: passing extra information
+cover: cover.png
+id: non_ambiguious_id
+author: Jane Doe; John Roe
+date: 2020-01-31
+description: this demo shows how you can pass extra demo information to DemoCards package. All these are optional.
+hidden: false
+---
+```
+
+See also: [`PlutoDemoCard`](@ref DemoCards.PlutoDemoCard), [`DemoSection`](@ref DemoCards.DemoSection), [`DemoPage`](@ref DemoCards.DemoPage)
+"""
+mutable struct PlutoDemoCard <: AbstractDemoCard
+    path::String
+    cover::Union{String, Nothing}
+    id::String
+    title::String
+    description::String
+    author::String
+    date::DateTime
+    julia::Union{Nothing, VersionNumber}
+    hidden::Bool
+end
+
+function PlutoDemoCard(path::AbstractString)::PlutoDemoCard
+    # first consturct an incomplete democard, and then load the config
+    card = PlutoDemoCard(path, "", "", "", "", "", DateTime(0), JULIA_COMPAT, false)
+
+    config = parse(card)
+    card.cover = load_config(card, "cover"; config=config)
+    card.title =  load_config(card, "title"; config=config)
+    card.date = load_config(card, "date"; config=config)
+    card.author = load_config(card, "author"; config=config)
+    card.julia = load_config(card, "julia"; config=config)
+    # default id requires a title
+    card.id = load_config(card, "id"; config=config)
+    # default description requires a title
+    card.description = load_config(card, "description"; config=config)
+    card.hidden = load_config(card, "hidden"; config=config)
+
+    return card
+end
+
+
+"""
+    save_democards(card_dir::AbstractString, card::PlutoDemoCard;
+                   project_dir,
+                   src,
+                   credit,
+                   nbviewer_root_url)
+
+process the original julia file and save it.
+
+The processing pipeline is:
+
+1. preprocess and copy source file
+3. generate markdown file
+4. insert header and footer to generated markdown file
+"""
+function save_democards(card_dir::AbstractString,
+                        card::PlutoDemoCard;
+                        credit,
+                        nbviewer_root_url,
+                        project_dir=Base.source_dir(),
+                        src="src",
+                        throw_error = false,
+                        properties = Dict{String, Any}(),
+                        kwargs...)
+    if !isabspath(card_dir)
+      card_dir = abspath(card_dir)
+    end
+    isdir(card_dir) || mkpath(card_dir)
+    @debug card.path
+
+    # copy to card dir and do things
+    cardname = splitext(basename(card.path))[1]
+    curr_dir = pwd()
+    src_dir = dirname(card.path)
+    # pluto outputs are expensive, we save the output to a cache dir
+    # these cache dir contains the render files from previous runs,
+    # saves time, while rendering
+    render_dir = joinpath(src_dir, "..", "..", "pluto_output") |> abspath
+    isdir(render_dir) || mkpath(render_dir)
+
+    nb_path = joinpath(card_dir, "$(cardname).jl")
+    card_path = joinpath(card_dir, "$(cardname).md")
+
+    _, _, body = split_pluto_frontmatter(readlines(card.path))
+    write(nb_path, join(body, "\n"))
+
+    if VERSION < card.julia
+        # It may work, it may not work; I hope it would work.
+        @warn "The running Julia version `$(VERSION)` is older than the declared compatible version `$(card.julia)`. You might need to upgrade your Julia."
+    end
+
+    oopts = OutputOptions(; append_build_context=false)
+    output_format = documenter_output
+    bopts = BuildOptions(card_dir;previous_dir=render_dir,
+                         output_format=output_format)
+    # don't run notebooks in parallel
+    build_notebooks(bopts, ["$(cardname).jl"], oopts)
+
+    cache_path = joinpath(render_dir, basename(card_path))
+    cp(card_path, cache_path; force=true)
+
+    badges = make_badges(card;
+                         src=src,
+                         card_dir=card_dir,
+                         nbviewer_root_url=nbviewer_root_url,
+                         project_dir=project_dir,
+                         build_notebook=false) 
+
+    header = "# [$(card.title)](@id $(card.id))\n"
+    footer = pluto_footer
+
+    body = join(readlines(card_path), "\n")
+    write(card_path, header, badges * "\n\n", body, footer)
+
+    return nothing
+end
+
+function make_badges(card::PlutoDemoCard; src, card_dir, nbviewer_root_url, project_dir, build_notebook)
+    cardname = splitext(basename(card.path))[1]
+    badges = []
+    push!(badges, "[![Source code]($download_badge)]($(cardname).jl)")
+
+    push!(badges, invoke(make_badges, Tuple{AbstractDemoCard}, card))
+
+    join(badges, " ")
+end

src/utils.jl

@@ -143,6 +143,28 @@ function get_regex(::Val{:Markdown}, regex_type)
     end
 end
 
+
+function get_regex(::Val{:Pluto}, regex_type)
+    if regex_type == :image
+        # Example: ![title](path)
+        return r"^\s*!\[(?<title>[^\]]*)\]\((?<path>[^\s]*)\)"
+    elseif regex_type == :title
+        # Example: # [title](@id id)
+        regex_title = r"md[\"]{1,3}.*\n#\s(\S.*)\n"
+        # Example: # title
+        regex_simple_title = r"md[\"]{1,3}.*\n#\s(\S.*)\n"
+
+        # Note: return the complete one first
+        return (regex_title, regex_simple_title)
+    elseif regex_type == :content
+        # lines that are not title, image, link, list
+        # FIXME: list is also captured by this regex
+        return r"^\s*(?<content>[^#\-*!].*)"
+    else
+        error("Unrecognized regex type: $(regex_type)")
+    end
+end
+
 function get_regex(::Val{:Julia}, regex_type)
     if regex_type == :image
         # Example: #md ![title](path)
@@ -189,8 +211,14 @@ Currently supported items are: `title`, `id`, `cover`, `description`.
     They also need to validate the values.
 """
 function parse(T::Val, card::AbstractDemoCard)
-    header, frontmatter, body = split_frontmatter(readlines(card.path))
-    config = parse(T, body)
+    # TODO: generalize
+    if T === Val(:Pluto)
+      header, frontmatter, body = split_pluto_frontmatter(readlines(card.path))
+      config = parse(T, body) # should ideally not pickup
+    else
+      header, frontmatter, body = split_frontmatter(readlines(card.path))
+      config = parse(T, body)
+    end
     # frontmatter has higher priority
     if !isempty(frontmatter)
         yaml_config = try
@@ -207,6 +235,7 @@ function parse(T::Val, card::AbstractDemoCard)
 end
 parse(card::JuliaDemoCard) = parse(Val(:Julia), card)
 parse(card::MarkdownDemoCard) = parse(Val(:Markdown), card)
+parse(card::PlutoDemoCard) = parse(Val(:Pluto), card)
 
 
 function parse(T::Val, contents::String)
@@ -313,7 +342,7 @@ function split_frontmatter(contents::AbstractArray{<:AbstractString})
         m isa RegexMatch
     end
     offsets = findall(offsets)
-    
+
     length(offsets) < 2 && return "", "", contents
     if offsets[1] != 1 && !all(x->isempty(strip(x)) || startswith(strip(x), "#"), contents[1:offsets[1]-1])
         # For julia demo, comments and empty lines are allowed before frontmatter
@@ -341,6 +370,28 @@ function split_frontmatter(contents::AbstractArray{<:AbstractString})
 end
 
 
+# special case, generalize it with rest of functions later
+function split_pluto_frontmatter(contents)
+    first_cell_regex = r"#\s╔═╡\sCell\sorder:\n#\s?[╠╟][─═]\s?([0-9a-f\-]{36})"
+    content = join(contents, "\n")
+    first_cell_id = match(first_cell_regex, content)[1]
+    m1 = r"#\s╔═╡\s"
+    m2 = Regex("$(first_cell_id)\n")
+    m3 = r"""md\"\"\"\n([\s\S]*?)\"\"\""""
+    first_cell_regex = m1 * m2 * m3
+    frontmatter = match(first_cell_regex, content)[1]
+    frontmatter = split(frontmatter, "\n") |> Vector{String}
+    content_id_repr = r"#\s?[╠╟][─═]\s?" * Regex("$(first_cell_id)")
+
+    offset = map(contents) do line
+        m = match(content_id_repr, line)
+        m isa RegexMatch
+    end |> findlast
+
+    return String[], frontmatter, vcat(contents[1:offset-1], contents[offset+1:end])
+end
+
+
 function get_default_title(x::Union{AbstractDemoCard, DemoSection, DemoPage})
     name_without_ext = splitext(basename(x))[1]
     strip(replace(uppercasefirst(name_without_ext), r"[_-]" => " "))
@@ -369,3 +420,5 @@ function input_bool(prompt)
         # Otherwise loop and repeat the prompt
     end
 end
+
+is_pluto_notebook(path::String) = any(occursin.(r"#\s?╔═╡\s?[0-9a-f\-]{36}", readlines(path)))