diff --git a/.github/workflows/UnitTest.yml b/.github/workflows/UnitTest.yml index 5f8c7d42..97e672ff 100644 --- a/.github/workflows/UnitTest.yml +++ b/.github/workflows/UnitTest.yml @@ -16,7 +16,7 @@ jobs: strategy: fail-fast: false matrix: - julia-version: ['1.0', '1', 'nightly'] + julia-version: ['1.6', '1', 'nightly'] os: [ubuntu-latest, windows-latest, macOS-latest] steps: diff --git a/Project.toml b/Project.toml index 0c2cdd63..2ea91a07 100644 --- a/Project.toml +++ b/Project.toml @@ -12,11 +12,15 @@ ImageCore = "a09fc81d-aa75-5fe9-8630-4744c3626534" JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" Literate = "98b081ad-f1c9-55d3-8b20-4c87d4299306" Mustache = "ffc61752-8dc7-55ee-8c37-f3e9cdd09e70" +Pluto = "c3e4b0f8-55cb-11ea-2926-15256bba5781" +PlutoStaticHTML = "359b1769-a58e-495b-9770-312e911026ad" Suppressor = "fd094767-a336-5f1f-9728-57cf17d0bbfb" YAML = "ddb6d928-2868-570f-bddf-ab3f9cf99eb6" [compat] Documenter = "0.22, 0.23, 0.24, 0.25, 0.26, 0.27" +Pluto = "^0.19" +PlutoStaticHTML = "^6" FileIO = "1" HTTP = "0.6, 0.7, 0.8, 0.9, 1" ImageCore = "0.7, 0.8, 0.9" @@ -25,7 +29,7 @@ Literate = "2" Mustache = "0.5, 1.0" Suppressor = "0.2" YAML = "0.3, 0.4" -julia = "1" +julia = "1.6" [extras] ImageMagick = "6218d12a-5da1-5696-b52f-db25d2ecc6d1" diff --git a/src/DemoCards.jl b/src/DemoCards.jl index fe5586d5..ba38ccdf 100644 --- a/src/DemoCards.jl +++ b/src/DemoCards.jl @@ -6,6 +6,8 @@ using Mustache using Literate using ImageCore using FileIO, JSON, YAML +using Pluto +using PlutoStaticHTML using Suppressor # suppress log generated by 3rd party tools, e.g., Literate import HTTP using Documenter diff --git a/src/types/card.jl b/src/types/card.jl index dc47f11d..ab9d1b35 100644 --- a/src/types/card.jl +++ b/src/types/card.jl @@ -23,7 +23,11 @@ function democard(path::String)::AbstractDemoCard if ext in markdown_exts return MarkdownDemoCard(path) elseif ext in julia_exts - return JuliaDemoCard(path) + if is_pluto_notebook(path) + return PlutoDemoCard(path) + else + return JuliaDemoCard(path) + end else return UnmatchedCard(path) end @@ -122,3 +126,4 @@ end include("markdown.jl") include("julia.jl") +include("pluto.jl") diff --git a/src/types/pluto.jl b/src/types/pluto.jl new file mode 100644 index 00000000..15762927 --- /dev/null +++ b/src/types/pluto.jl @@ -0,0 +1,178 @@ +const pluto_footer = raw""" + +--- + +*This page was generated using [DemoCards.jl](https://github.com/johnnychen94/DemoCards.jl). and [PlutoStaticHTML.jl](https://github.com/rikhuijzer/PlutoStaticHTML.jl)* + + +""" + +""" + struct PlutoDemoCard <: AbstractDemoCard + PlutoDemoCard(path::AbstractString) + +Constructs a markdown-format demo card from a pluto notebook. + + +# Fields + +Besides `path`, this struct has some other fields: + +* `path`: path to the source markdown file +* `cover`: path to the cover image +* `id`: cross-reference id +* `title`: one-line description of the demo card +* `author`: author(s) of this demo. +* `date`: the update date of this demo. +* `julia`: Julia version compatibility +* `description`: multi-line description of the demo card +* `hidden`: whether this card is shown in the generated index page + +# Configuration + +You can pass additional information by adding a YAML front matter to the markdown file. +Supported items are: + +* `cover`: an URL or a relative path to the cover image. If not specified, it will use the first available image link, or all-white image if there's no image links. +* `description`: a multi-line description to this file, will be displayed when the demo card is hovered. By default it uses `title`. +* `id`: specify the `id` tag for cross-references. By default it's infered from the filename, e.g., `simple_demo` from `simple demo.md`. +* `title`: one-line description to this file, will be displayed under the cover image. By default, it's the name of the file (without extension). +* `author`: author name. If there are multiple authors, split them with semicolon `;`. +* `julia`: Julia version compatibility. Any string that can be converted to `VersionNumber` +* `date`: any string contents that can be passed to `Dates.DateTime`. For example, `2020-09-13`. +* `hidden`: whether this card is shown in the layout of index page. The default value is `false`. + +An example of the front matter: + +```text +--- +title: passing extra information +cover: cover.png +id: non_ambiguious_id +author: Jane Doe; John Roe +date: 2020-01-31 +description: this demo shows how you can pass extra demo information to DemoCards package. All these are optional. +hidden: false +--- +``` + +See also: [`PlutoDemoCard`](@ref DemoCards.PlutoDemoCard), [`DemoSection`](@ref DemoCards.DemoSection), [`DemoPage`](@ref DemoCards.DemoPage) +""" +mutable struct PlutoDemoCard <: AbstractDemoCard + path::String + cover::Union{String, Nothing} + id::String + title::String + description::String + author::String + date::DateTime + julia::Union{Nothing, VersionNumber} + hidden::Bool +end + +function PlutoDemoCard(path::AbstractString)::PlutoDemoCard + # first consturct an incomplete democard, and then load the config + card = PlutoDemoCard(path, "", "", "", "", "", DateTime(0), JULIA_COMPAT, false) + + config = parse(card) + card.cover = load_config(card, "cover"; config=config) + card.title = load_config(card, "title"; config=config) + card.date = load_config(card, "date"; config=config) + card.author = load_config(card, "author"; config=config) + card.julia = load_config(card, "julia"; config=config) + # default id requires a title + card.id = load_config(card, "id"; config=config) + # default description requires a title + card.description = load_config(card, "description"; config=config) + card.hidden = load_config(card, "hidden"; config=config) + + return card +end + + +""" + save_democards(card_dir::AbstractString, card::PlutoDemoCard; + project_dir, + src, + credit, + nbviewer_root_url) + +process the original julia file and save it. + +The processing pipeline is: + +1. preprocess and copy source file +3. generate markdown file +4. insert header and footer to generated markdown file +""" +function save_democards(card_dir::AbstractString, + card::PlutoDemoCard; + credit, + nbviewer_root_url, + project_dir=Base.source_dir(), + src="src", + throw_error = false, + properties = Dict{String, Any}(), + kwargs...) + if !isabspath(card_dir) + card_dir = abspath(card_dir) + end + isdir(card_dir) || mkpath(card_dir) + @debug card.path + + # copy to card dir and do things + cardname = splitext(basename(card.path))[1] + curr_dir = pwd() + src_dir = dirname(card.path) + # pluto outputs are expensive, we save the output to a cache dir + # these cache dir contains the render files from previous runs, + # saves time, while rendering + render_dir = joinpath(src_dir, "..", "..", "pluto_output") |> abspath + isdir(render_dir) || mkpath(render_dir) + + nb_path = joinpath(card_dir, "$(cardname).jl") + card_path = joinpath(card_dir, "$(cardname).md") + + _, _, body = split_pluto_frontmatter(readlines(card.path)) + write(nb_path, join(body, "\n")) + + if VERSION < card.julia + # It may work, it may not work; I hope it would work. + @warn "The running Julia version `$(VERSION)` is older than the declared compatible version `$(card.julia)`. You might need to upgrade your Julia." + end + + oopts = OutputOptions(; append_build_context=false) + output_format = documenter_output + bopts = BuildOptions(card_dir;previous_dir=render_dir, + output_format=output_format) + # don't run notebooks in parallel + build_notebooks(bopts, ["$(cardname).jl"], oopts) + + cache_path = joinpath(render_dir, basename(card_path)) + cp(card_path, cache_path; force=true) + + badges = make_badges(card; + src=src, + card_dir=card_dir, + nbviewer_root_url=nbviewer_root_url, + project_dir=project_dir, + build_notebook=false) + + header = "# [$(card.title)](@id $(card.id))\n" + footer = pluto_footer + + body = join(readlines(card_path), "\n") + write(card_path, header, badges * "\n\n", body, footer) + + return nothing +end + +function make_badges(card::PlutoDemoCard; src, card_dir, nbviewer_root_url, project_dir, build_notebook) + cardname = splitext(basename(card.path))[1] + badges = [] + push!(badges, "[![Source code]($download_badge)]($(cardname).jl)") + + push!(badges, invoke(make_badges, Tuple{AbstractDemoCard}, card)) + + join(badges, " ") +end diff --git a/src/utils.jl b/src/utils.jl index 9431de40..fb132771 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -143,6 +143,28 @@ function get_regex(::Val{:Markdown}, regex_type) end end + +function get_regex(::Val{:Pluto}, regex_type) + if regex_type == :image + # Example: ![title](path) + return r"^\s*!\[(?[^\]]*)\]\((?<path>[^\s]*)\)" + elseif regex_type == :title + # Example: # [title](@id id) + regex_title = r"md[\"]{1,3}.*\n#\s(\S.*)\n" + # Example: # title + regex_simple_title = r"md[\"]{1,3}.*\n#\s(\S.*)\n" + + # Note: return the complete one first + return (regex_title, regex_simple_title) + elseif regex_type == :content + # lines that are not title, image, link, list + # FIXME: list is also captured by this regex + return r"^\s*(?<content>[^#\-*!].*)" + else + error("Unrecognized regex type: $(regex_type)") + end +end + function get_regex(::Val{:Julia}, regex_type) if regex_type == :image # Example: #md ![title](path) @@ -189,8 +211,14 @@ Currently supported items are: `title`, `id`, `cover`, `description`. They also need to validate the values. """ function parse(T::Val, card::AbstractDemoCard) - header, frontmatter, body = split_frontmatter(readlines(card.path)) - config = parse(T, body) + # TODO: generalize + if T === Val(:Pluto) + header, frontmatter, body = split_pluto_frontmatter(readlines(card.path)) + config = parse(T, body) # should ideally not pickup + else + header, frontmatter, body = split_frontmatter(readlines(card.path)) + config = parse(T, body) + end # frontmatter has higher priority if !isempty(frontmatter) yaml_config = try @@ -207,6 +235,7 @@ function parse(T::Val, card::AbstractDemoCard) end parse(card::JuliaDemoCard) = parse(Val(:Julia), card) parse(card::MarkdownDemoCard) = parse(Val(:Markdown), card) +parse(card::PlutoDemoCard) = parse(Val(:Pluto), card) function parse(T::Val, contents::String) @@ -313,7 +342,7 @@ function split_frontmatter(contents::AbstractArray{<:AbstractString}) m isa RegexMatch end offsets = findall(offsets) - + length(offsets) < 2 && return "", "", contents if offsets[1] != 1 && !all(x->isempty(strip(x)) || startswith(strip(x), "#"), contents[1:offsets[1]-1]) # For julia demo, comments and empty lines are allowed before frontmatter @@ -341,6 +370,28 @@ function split_frontmatter(contents::AbstractArray{<:AbstractString}) end +# special case, generalize it with rest of functions later +function split_pluto_frontmatter(contents) + first_cell_regex = r"#\s╔═╡\sCell\sorder:\n#\s?[╠╟][─═]\s?([0-9a-f\-]{36})" + content = join(contents, "\n") + first_cell_id = match(first_cell_regex, content)[1] + m1 = r"#\s╔═╡\s" + m2 = Regex("$(first_cell_id)\n") + m3 = r"""md\"\"\"\n([\s\S]*?)\"\"\"""" + first_cell_regex = m1 * m2 * m3 + frontmatter = match(first_cell_regex, content)[1] + frontmatter = split(frontmatter, "\n") |> Vector{String} + content_id_repr = r"#\s?[╠╟][─═]\s?" * Regex("$(first_cell_id)") + + offset = map(contents) do line + m = match(content_id_repr, line) + m isa RegexMatch + end |> findlast + + return String[], frontmatter, vcat(contents[1:offset-1], contents[offset+1:end]) +end + + function get_default_title(x::Union{AbstractDemoCard, DemoSection, DemoPage}) name_without_ext = splitext(basename(x))[1] strip(replace(uppercasefirst(name_without_ext), r"[_-]" => " ")) @@ -369,3 +420,5 @@ function input_bool(prompt) # Otherwise loop and repeat the prompt end end + +is_pluto_notebook(path::String) = any(occursin.(r"#\s?╔═╡\s?[0-9a-f\-]{36}", readlines(path)))