From 80a2a94d5a646859239cb8acab3ecc3c508e741e Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Fri, 21 Apr 2017 00:32:23 +0200 Subject: [PATCH 1/5] Re-enable hoedown by default --- src/librustdoc/html/markdown.rs | 335 +++++++++++++++--- src/librustdoc/html/render.rs | 44 ++- src/librustdoc/lib.rs | 20 +- src/librustdoc/markdown.rs | 14 +- src/librustdoc/test.rs | 55 +-- src/test/rustdoc/check-hard-break.rs | 20 -- src/test/rustdoc/check-rule-image-footnote.rs | 44 --- 7 files changed, 382 insertions(+), 150 deletions(-) delete mode 100644 src/test/rustdoc/check-hard-break.rs delete mode 100644 src/test/rustdoc/check-rule-image-footnote.rs diff --git a/src/librustdoc/html/markdown.rs b/src/librustdoc/html/markdown.rs index 4bf856240f66a..614d59c512da2 100644 --- a/src/librustdoc/html/markdown.rs +++ b/src/librustdoc/html/markdown.rs @@ -32,6 +32,7 @@ use std::ascii::AsciiExt; use std::cell::RefCell; use std::collections::{HashMap, VecDeque}; use std::default::Default; +use std::ffi::CString; use std::fmt::{self, Write}; use std::str; use syntax::feature_gate::UnstableFeatures; @@ -40,21 +41,28 @@ use syntax::codemap::Span; use html::render::derive_id; use html::toc::TocBuilder; use html::highlight; +use html::escape::Escape; use test; use pulldown_cmark::{html, Event, Tag, Parser}; use pulldown_cmark::{Options, OPTION_ENABLE_FOOTNOTES, OPTION_ENABLE_TABLES}; +#[derive(PartialEq, Debug, Clone, Copy)] +pub enum RenderType { + Hoedown, + Pulldown, +} + /// A unit struct which has the `fmt::Display` trait implemented. When /// formatted, this struct will emit the HTML corresponding to the rendered /// version of the contained markdown string. // The second parameter is whether we need a shorter version or not. -pub struct Markdown<'a>(pub &'a str); +pub struct Markdown<'a>(pub &'a str, pub RenderType); /// A unit struct like `Markdown`, that renders the markdown with a /// table of contents. -pub struct MarkdownWithToc<'a>(pub &'a str); +pub struct MarkdownWithToc<'a>(pub &'a str, pub RenderType); /// A unit struct like `Markdown`, that renders the markdown escaping HTML tags. -pub struct MarkdownHtml<'a>(pub &'a str); +pub struct MarkdownHtml<'a>(pub &'a str, pub RenderType); /// A unit struct like `Markdown`, that renders only the first paragraph. pub struct MarkdownSummaryLine<'a>(pub &'a str); @@ -73,6 +81,14 @@ fn stripped_filtered_line<'a>(s: &'a str) -> Option<&'a str> { } } +/// Returns a new string with all consecutive whitespace collapsed into +/// single spaces. +/// +/// Any leading or trailing whitespace will be trimmed. +fn collapse_whitespace(s: &str) -> String { + s.split_whitespace().collect::>().join(" ") +} + /// Convert chars from a title for an id. /// /// "Hello, world!" -> "hello-world" @@ -368,6 +384,7 @@ const HOEDOWN_EXT_AUTOLINK: libc::c_uint = 1 << 3; const HOEDOWN_EXT_STRIKETHROUGH: libc::c_uint = 1 << 4; const HOEDOWN_EXT_SUPERSCRIPT: libc::c_uint = 1 << 8; const HOEDOWN_EXT_FOOTNOTES: libc::c_uint = 1 << 2; +const HOEDOWN_HTML_ESCAPE: libc::c_uint = 1 << 1; const HOEDOWN_EXTENSIONS: libc::c_uint = HOEDOWN_EXT_NO_INTRA_EMPHASIS | HOEDOWN_EXT_TABLES | @@ -462,6 +479,13 @@ struct hoedown_buffer { unit: libc::size_t, } +struct MyOpaque { + dfltblk: extern "C" fn(*mut hoedown_buffer, *const hoedown_buffer, + *const hoedown_buffer, *const hoedown_renderer_data, + libc::size_t), + toc_builder: Option, +} + extern { fn hoedown_html_renderer_new(render_flags: libc::c_uint, nesting_level: libc::c_int) @@ -478,6 +502,7 @@ extern { fn hoedown_document_free(md: *mut hoedown_document); fn hoedown_buffer_new(unit: libc::size_t) -> *mut hoedown_buffer; + fn hoedown_buffer_puts(b: *mut hoedown_buffer, c: *const libc::c_char); fn hoedown_buffer_free(b: *mut hoedown_buffer); } @@ -487,6 +512,208 @@ impl hoedown_buffer { } } +pub fn render(w: &mut fmt::Formatter, + s: &str, + print_toc: bool, + html_flags: libc::c_uint) -> fmt::Result { + extern fn block(ob: *mut hoedown_buffer, orig_text: *const hoedown_buffer, + lang: *const hoedown_buffer, data: *const hoedown_renderer_data, + line: libc::size_t) { + unsafe { + if orig_text.is_null() { return } + + let opaque = (*data).opaque as *mut hoedown_html_renderer_state; + let my_opaque: &MyOpaque = &*((*opaque).opaque as *const MyOpaque); + let text = (*orig_text).as_bytes(); + let origtext = str::from_utf8(text).unwrap(); + let origtext = origtext.trim_left(); + debug!("docblock: ==============\n{:?}\n=======", text); + let rendered = if lang.is_null() || origtext.is_empty() { + false + } else { + let rlang = (*lang).as_bytes(); + let rlang = str::from_utf8(rlang).unwrap(); + if !LangString::parse(rlang).rust { + (my_opaque.dfltblk)(ob, orig_text, lang, + opaque as *const hoedown_renderer_data, + line); + true + } else { + false + } + }; + + let lines = origtext.lines().filter(|l| { + stripped_filtered_line(*l).is_none() + }); + let text = lines.collect::>().join("\n"); + if rendered { return } + PLAYGROUND.with(|play| { + // insert newline to clearly separate it from the + // previous block so we can shorten the html output + let mut s = String::from("\n"); + let playground_button = play.borrow().as_ref().and_then(|&(ref krate, ref url)| { + if url.is_empty() { + return None; + } + let test = origtext.lines().map(|l| { + stripped_filtered_line(l).unwrap_or(l) + }).collect::>().join("\n"); + let krate = krate.as_ref().map(|s| &**s); + let test = test::maketest(&test, krate, false, + &Default::default()); + let channel = if test.contains("#![feature(") { + "&version=nightly" + } else { + "" + }; + // These characters don't need to be escaped in a URI. + // FIXME: use a library function for percent encoding. + fn dont_escape(c: u8) -> bool { + (b'a' <= c && c <= b'z') || + (b'A' <= c && c <= b'Z') || + (b'0' <= c && c <= b'9') || + c == b'-' || c == b'_' || c == b'.' || + c == b'~' || c == b'!' || c == b'\'' || + c == b'(' || c == b')' || c == b'*' + } + let mut test_escaped = String::new(); + for b in test.bytes() { + if dont_escape(b) { + test_escaped.push(char::from(b)); + } else { + write!(test_escaped, "%{:02X}", b).unwrap(); + } + } + Some(format!( + r#"Run"#, + url, test_escaped, channel + )) + }); + s.push_str(&highlight::render_with_highlighting( + &text, + Some("rust-example-rendered"), + None, + playground_button.as_ref().map(String::as_str))); + let output = CString::new(s).unwrap(); + hoedown_buffer_puts(ob, output.as_ptr()); + }) + } + } + + extern fn header(ob: *mut hoedown_buffer, text: *const hoedown_buffer, + level: libc::c_int, data: *const hoedown_renderer_data, + _: libc::size_t) { + // hoedown does this, we may as well too + unsafe { hoedown_buffer_puts(ob, "\n\0".as_ptr() as *const _); } + + // Extract the text provided + let s = if text.is_null() { + "".to_owned() + } else { + let s = unsafe { (*text).as_bytes() }; + str::from_utf8(&s).unwrap().to_owned() + }; + + // Discard '', '' tags and some escaped characters, + // transform the contents of the header into a hyphenated string + // without non-alphanumeric characters other than '-' and '_'. + // + // This is a terrible hack working around how hoedown gives us rendered + // html for text rather than the raw text. + let mut id = s.clone(); + let repl_sub = vec!["", "", "", "", + "", "", + "<", ">", "&", "'", """]; + for sub in repl_sub { + id = id.replace(sub, ""); + } + let id = id.chars().filter_map(|c| { + if c.is_alphanumeric() || c == '-' || c == '_' { + if c.is_ascii() { + Some(c.to_ascii_lowercase()) + } else { + Some(c) + } + } else if c.is_whitespace() && c.is_ascii() { + Some('-') + } else { + None + } + }).collect::(); + + let opaque = unsafe { (*data).opaque as *mut hoedown_html_renderer_state }; + let opaque = unsafe { &mut *((*opaque).opaque as *mut MyOpaque) }; + + let id = derive_id(id); + + let sec = opaque.toc_builder.as_mut().map_or("".to_owned(), |builder| { + format!("{} ", builder.push(level as u32, s.clone(), id.clone())) + }); + + // Render the HTML + let text = format!("\ + {sec}{}", + s, lvl = level, id = id, sec = sec); + + let text = CString::new(text).unwrap(); + unsafe { hoedown_buffer_puts(ob, text.as_ptr()) } + } + + extern fn codespan( + ob: *mut hoedown_buffer, + text: *const hoedown_buffer, + _: *const hoedown_renderer_data, + _: libc::size_t + ) -> libc::c_int { + let content = if text.is_null() { + "".to_owned() + } else { + let bytes = unsafe { (*text).as_bytes() }; + let s = str::from_utf8(bytes).unwrap(); + collapse_whitespace(s) + }; + + let content = format!("{}", Escape(&content)); + let element = CString::new(content).unwrap(); + unsafe { hoedown_buffer_puts(ob, element.as_ptr()); } + // Return anything except 0, which would mean "also print the code span verbatim". + 1 + } + + unsafe { + let ob = hoedown_buffer_new(DEF_OUNIT); + let renderer = hoedown_html_renderer_new(html_flags, 0); + let mut opaque = MyOpaque { + dfltblk: (*renderer).blockcode.unwrap(), + toc_builder: if print_toc {Some(TocBuilder::new())} else {None} + }; + (*((*renderer).opaque as *mut hoedown_html_renderer_state)).opaque + = &mut opaque as *mut _ as *mut libc::c_void; + (*renderer).blockcode = Some(block); + (*renderer).header = Some(header); + (*renderer).codespan = Some(codespan); + + let document = hoedown_document_new(renderer, HOEDOWN_EXTENSIONS, 16); + hoedown_document_render(document, ob, s.as_ptr(), + s.len() as libc::size_t); + hoedown_document_free(document); + + hoedown_html_renderer_free(renderer); + + let mut ret = opaque.toc_builder.map_or(Ok(()), |builder| { + write!(w, "", builder.into_toc()) + }); + + if ret.is_ok() { + let buf = (*ob).as_bytes(); + ret = w.write_str(str::from_utf8(buf).unwrap()); + } + hoedown_buffer_free(ob); + ret + } +} + pub fn old_find_testable_code(doc: &str, tests: &mut ::test::Collector, position: Span) { extern fn block(_ob: *mut hoedown_buffer, text: *const hoedown_buffer, @@ -511,7 +738,22 @@ pub fn old_find_testable_code(doc: &str, tests: &mut ::test::Collector, position stripped_filtered_line(l).unwrap_or(l) }); let filename = tests.get_filename(); - tests.add_old_test(lines.collect::>().join("\n"), filename); + + if tests.render_type == RenderType::Hoedown { + let text = (*text).as_bytes(); + let text = str::from_utf8(text).unwrap(); + let lines = text.lines().map(|l| { + stripped_filtered_line(l).unwrap_or(l) + }); + let text = lines.collect::>().join("\n"); + tests.add_test(text.to_owned(), + block_info.should_panic, block_info.no_run, + block_info.ignore, block_info.test_harness, + block_info.compile_fail, block_info.error_codes, + line, filename); + } else { + tests.add_old_test(lines.collect::>().join("\n"), filename); + } } } @@ -533,7 +775,6 @@ pub fn old_find_testable_code(doc: &str, tests: &mut ::test::Collector, position } tests.set_position(position); - unsafe { let ob = hoedown_buffer_new(DEF_OUNIT); let renderer = hoedown_html_renderer_new(0, 0); @@ -702,72 +943,84 @@ impl LangString { impl<'a> fmt::Display for Markdown<'a> { fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { - let Markdown(md) = *self; + let Markdown(md, render_type) = *self; + // This is actually common enough to special-case if md.is_empty() { return Ok(()) } + if render_type == RenderType::Hoedown { + render(fmt, md, false, 0) + } else { + let mut opts = Options::empty(); + opts.insert(OPTION_ENABLE_TABLES); + opts.insert(OPTION_ENABLE_FOOTNOTES); - let mut opts = Options::empty(); - opts.insert(OPTION_ENABLE_TABLES); - opts.insert(OPTION_ENABLE_FOOTNOTES); - - let p = Parser::new_ext(md, opts); + let p = Parser::new_ext(md, opts); - let mut s = String::with_capacity(md.len() * 3 / 2); + let mut s = String::with_capacity(md.len() * 3 / 2); - html::push_html(&mut s, - Footnotes::new(CodeBlocks::new(HeadingLinks::new(p, None)))); + html::push_html(&mut s, + Footnotes::new(CodeBlocks::new(HeadingLinks::new(p, None)))); - fmt.write_str(&s) + fmt.write_str(&s) + } } } impl<'a> fmt::Display for MarkdownWithToc<'a> { fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { - let MarkdownWithToc(md) = *self; + let MarkdownWithToc(md, render_type) = *self; - let mut opts = Options::empty(); - opts.insert(OPTION_ENABLE_TABLES); - opts.insert(OPTION_ENABLE_FOOTNOTES); + if render_type == RenderType::Hoedown { + render(fmt, md, true, 0) + } else { + let mut opts = Options::empty(); + opts.insert(OPTION_ENABLE_TABLES); + opts.insert(OPTION_ENABLE_FOOTNOTES); - let p = Parser::new_ext(md, opts); + let p = Parser::new_ext(md, opts); - let mut s = String::with_capacity(md.len() * 3 / 2); + let mut s = String::with_capacity(md.len() * 3 / 2); - let mut toc = TocBuilder::new(); + let mut toc = TocBuilder::new(); - html::push_html(&mut s, - Footnotes::new(CodeBlocks::new(HeadingLinks::new(p, Some(&mut toc))))); + html::push_html(&mut s, + Footnotes::new(CodeBlocks::new(HeadingLinks::new(p, Some(&mut toc))))); - write!(fmt, "", toc.into_toc())?; + write!(fmt, "", toc.into_toc())?; - fmt.write_str(&s) + fmt.write_str(&s) + } } } impl<'a> fmt::Display for MarkdownHtml<'a> { fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { - let MarkdownHtml(md) = *self; + let MarkdownHtml(md, render_type) = *self; + // This is actually common enough to special-case if md.is_empty() { return Ok(()) } + if render_type == RenderType::Hoedown { + render(fmt, md, false, HOEDOWN_HTML_ESCAPE) + } else { + let mut opts = Options::empty(); + opts.insert(OPTION_ENABLE_TABLES); + opts.insert(OPTION_ENABLE_FOOTNOTES); - let mut opts = Options::empty(); - opts.insert(OPTION_ENABLE_TABLES); - opts.insert(OPTION_ENABLE_FOOTNOTES); - - let p = Parser::new_ext(md, opts); + let p = Parser::new_ext(md, opts); - // Treat inline HTML as plain text. - let p = p.map(|event| match event { - Event::Html(text) | Event::InlineHtml(text) => Event::Text(text), - _ => event - }); + // Treat inline HTML as plain text. + let p = p.map(|event| match event { + Event::Html(text) | Event::InlineHtml(text) => Event::Text(text), + _ => event + }); - let mut s = String::with_capacity(md.len() * 3 / 2); + let mut s = String::with_capacity(md.len() * 3 / 2); - html::push_html(&mut s, - Footnotes::new(CodeBlocks::new(HeadingLinks::new(p, None)))); + html::push_html(&mut s, + Footnotes::new(CodeBlocks::new(HeadingLinks::new(p, None)))); - fmt.write_str(&s) + fmt.write_str(&s) + } } } diff --git a/src/librustdoc/html/render.rs b/src/librustdoc/html/render.rs index d55a0640562ae..57d71e6c4e004 100644 --- a/src/librustdoc/html/render.rs +++ b/src/librustdoc/html/render.rs @@ -72,7 +72,7 @@ use html::format::{TyParamBounds, WhereClause, href, AbiSpace}; use html::format::{VisSpace, Method, UnsafetySpace, MutableSpace}; use html::format::fmt_impl_for_trait_page; use html::item_type::ItemType; -use html::markdown::{self, Markdown, MarkdownHtml, MarkdownSummaryLine}; +use html::markdown::{self, Markdown, MarkdownHtml, MarkdownSummaryLine, RenderType}; use html::{highlight, layout}; /// A pair of name and its optional document. @@ -98,6 +98,7 @@ pub struct Context { /// publicly reused items to redirect to the right location. pub render_redirect_pages: bool, pub shared: Arc, + pub render_type: RenderType, } pub struct SharedContext { @@ -433,7 +434,8 @@ pub fn run(mut krate: clean::Crate, dst: PathBuf, passes: FxHashSet, css_file_extension: Option, - renderinfo: RenderInfo) -> Result<(), Error> { + renderinfo: RenderInfo, + render_type: RenderType) -> Result<(), Error> { let src_root = match krate.src.parent() { Some(p) => p.to_path_buf(), None => PathBuf::new(), @@ -495,6 +497,7 @@ pub fn run(mut krate: clean::Crate, dst: dst, render_redirect_pages: false, shared: Arc::new(scx), + render_type: render_type, }; // Crawl the crate to build various caches used for the output @@ -1638,11 +1641,12 @@ fn plain_summary_line(s: Option<&str>) -> String { fn document(w: &mut fmt::Formatter, cx: &Context, item: &clean::Item) -> fmt::Result { document_stability(w, cx, item)?; - document_full(w, item)?; + document_full(w, item, cx.render_type)?; Ok(()) } -fn document_short(w: &mut fmt::Formatter, item: &clean::Item, link: AssocItemLink) -> fmt::Result { +fn document_short(w: &mut fmt::Formatter, item: &clean::Item, link: AssocItemLink, + render_type: RenderType) -> fmt::Result { if let Some(s) = item.doc_value() { let markdown = if s.contains('\n') { format!("{} [Read more]({})", @@ -1651,7 +1655,7 @@ fn document_short(w: &mut fmt::Formatter, item: &clean::Item, link: AssocItemLin format!("{}", &plain_summary_line(Some(s))) }; write!(w, "
{}
", - Markdown(&markdown))?; + Markdown(&markdown, render_type))?; } Ok(()) } @@ -1681,10 +1685,11 @@ fn get_doc_value(item: &clean::Item) -> Option<&str> { } } -fn document_full(w: &mut fmt::Formatter, item: &clean::Item) -> fmt::Result { +fn document_full(w: &mut fmt::Formatter, item: &clean::Item, + render_type: RenderType) -> fmt::Result { if let Some(s) = get_doc_value(item) { write!(w, "
{}
", - Markdown(&format!("{}{}", md_render_assoc_item(item), s)))?; + Markdown(&format!("{}{}", md_render_assoc_item(item), s), render_type))?; } Ok(()) } @@ -1872,7 +1877,13 @@ fn item_module(w: &mut fmt::Formatter, cx: &Context, ", name = *myitem.name.as_ref().unwrap(), stab_docs = stab_docs, - docs = MarkdownSummaryLine(doc_value), + docs = if cx.render_type == RenderType::Hoedown { + format!("{}", + shorter(Some(&Markdown(doc_value, + RenderType::Hoedown).to_string()))) + } else { + format!("{}", MarkdownSummaryLine(doc_value)) + }, class = myitem.type_(), stab = myitem.stability_class().unwrap_or("".to_string()), unsafety_flag = unsafety_flag, @@ -1915,7 +1926,9 @@ fn short_stability(item: &clean::Item, cx: &Context, show_reason: bool) -> Vec{}", text)) }; @@ -1944,7 +1957,8 @@ fn short_stability(item: &clean::Item, cx: &Context, show_reason: bool) -> Vec🔬 \ This is a nightly-only experimental API. {}\ {}", - unstable_extra, MarkdownHtml(&stab.unstable_reason)); + unstable_extra, + MarkdownHtml(&stab.unstable_reason, cx.render_type)); stability.push(format!("
{}
", text)); } @@ -1964,7 +1978,7 @@ fn short_stability(item: &clean::Item, cx: &Context, show_reason: bool) -> Vec{}", text)) } @@ -2900,7 +2914,7 @@ fn render_impl(w: &mut fmt::Formatter, cx: &Context, i: &Impl, link: AssocItemLi write!(w, "")?; write!(w, "\n")?; if let Some(ref dox) = i.impl_item.doc_value() { - write!(w, "
{}
", Markdown(dox))?; + write!(w, "
{}
", Markdown(dox, cx.render_type))?; } } @@ -2999,11 +3013,11 @@ fn render_impl(w: &mut fmt::Formatter, cx: &Context, i: &Impl, link: AssocItemLi // because impls can't have a stability. document_stability(w, cx, it)?; if get_doc_value(item).is_some() { - document_full(w, item)?; + document_full(w, item, cx.render_type)?; } else { // In case the item isn't documented, // provide short documentation from the trait. - document_short(w, it, link)?; + document_short(w, it, link, cx.render_type)?; } } } else { @@ -3011,7 +3025,7 @@ fn render_impl(w: &mut fmt::Formatter, cx: &Context, i: &Impl, link: AssocItemLi } } else { document_stability(w, cx, item)?; - document_short(w, item, link)?; + document_short(w, item, link, cx.render_type)?; } } Ok(()) diff --git a/src/librustdoc/lib.rs b/src/librustdoc/lib.rs index 0ca267bb82d2e..2a6134fde5c3d 100644 --- a/src/librustdoc/lib.rs +++ b/src/librustdoc/lib.rs @@ -93,6 +93,8 @@ pub mod test; use clean::AttributesExt; +use html::markdown::RenderType; + struct Output { krate: clean::Crate, renderinfo: html::render::RenderInfo, @@ -169,6 +171,7 @@ pub fn opts() -> Vec { "URL to send code snippets to, may be reset by --markdown-playground-url \ or `#![doc(html_playground_url=...)]`", "URL")), + unstable(optflag("", "enable-commonmark", "to enable commonmark doc rendering/testing")), ] } @@ -250,6 +253,12 @@ pub fn main_args(args: &[String]) -> isize { let css_file_extension = matches.opt_str("e").map(|s| PathBuf::from(&s)); let cfgs = matches.opt_strs("cfg"); + let render_type = if matches.opt_present("enable-commonmark") { + RenderType::Pulldown + } else { + RenderType::Hoedown + }; + if let Some(ref p) = css_file_extension { if !p.is_file() { writeln!( @@ -273,15 +282,17 @@ pub fn main_args(args: &[String]) -> isize { match (should_test, markdown_input) { (true, true) => { - return markdown::test(input, cfgs, libs, externs, test_args, maybe_sysroot) + return markdown::test(input, cfgs, libs, externs, test_args, maybe_sysroot, render_type) } (true, false) => { - return test::run(input, cfgs, libs, externs, test_args, crate_name, maybe_sysroot) + return test::run(input, cfgs, libs, externs, test_args, crate_name, maybe_sysroot, + render_type) } (false, true) => return markdown::render(input, output.unwrap_or(PathBuf::from("doc")), &matches, &external_html, - !matches.opt_present("markdown-no-toc")), + !matches.opt_present("markdown-no-toc"), + render_type), (false, false) => {} } @@ -295,7 +306,8 @@ pub fn main_args(args: &[String]) -> isize { output.unwrap_or(PathBuf::from("doc")), passes.into_iter().collect(), css_file_extension, - renderinfo) + renderinfo, + render_type) .expect("failed to generate documentation"); 0 } diff --git a/src/librustdoc/markdown.rs b/src/librustdoc/markdown.rs index f75144c23aca9..b9ed0eeaef736 100644 --- a/src/librustdoc/markdown.rs +++ b/src/librustdoc/markdown.rs @@ -26,6 +26,7 @@ use html::render::reset_ids; use html::escape::Escape; use html::markdown; use html::markdown::{Markdown, MarkdownWithToc, find_testable_code, old_find_testable_code}; +use html::markdown::RenderType; use test::{TestOptions, Collector}; /// Separate any lines at the start of the file that begin with `# ` or `%`. @@ -50,7 +51,8 @@ fn extract_leading_metadata<'a>(s: &'a str) -> (Vec<&'a str>, &'a str) { /// Render `input` (e.g. "foo.md") into an HTML file in `output` /// (e.g. output = "bar" => "bar/foo.html"). pub fn render(input: &str, mut output: PathBuf, matches: &getopts::Matches, - external_html: &ExternalHtml, include_toc: bool) -> isize { + external_html: &ExternalHtml, include_toc: bool, + render_type: RenderType) -> isize { let input_p = Path::new(input); output.push(input_p.file_stem().unwrap()); output.set_extension("html"); @@ -94,9 +96,9 @@ pub fn render(input: &str, mut output: PathBuf, matches: &getopts::Matches, reset_ids(false); let rendered = if include_toc { - format!("{}", MarkdownWithToc(text)) + format!("{}", MarkdownWithToc(text, render_type)) } else { - format!("{}", Markdown(text)) + format!("{}", Markdown(text, render_type)) }; let err = write!( @@ -147,7 +149,8 @@ pub fn render(input: &str, mut output: PathBuf, matches: &getopts::Matches, /// Run any tests/code examples in the markdown file `input`. pub fn test(input: &str, cfgs: Vec, libs: SearchPaths, externs: Externs, - mut test_args: Vec, maybe_sysroot: Option) -> isize { + mut test_args: Vec, maybe_sysroot: Option, + render_type: RenderType) -> isize { let input_str = match load_string(input) { Ok(s) => s, Err(LoadStringError::ReadFail) => return 1, @@ -158,7 +161,8 @@ pub fn test(input: &str, cfgs: Vec, libs: SearchPaths, externs: Externs, opts.no_crate_inject = true; let mut collector = Collector::new(input.to_string(), cfgs, libs, externs, true, opts, maybe_sysroot, None, - Some(input.to_owned())); + Some(input.to_owned()), + render_type); old_find_testable_code(&input_str, &mut collector, DUMMY_SP); find_testable_code(&input_str, &mut collector, DUMMY_SP); test_args.insert(0, "rustdoctest".to_string()); diff --git a/src/librustdoc/test.rs b/src/librustdoc/test.rs index 3206b5021075d..a30ec25de60d7 100644 --- a/src/librustdoc/test.rs +++ b/src/librustdoc/test.rs @@ -43,7 +43,7 @@ use errors; use errors::emitter::ColorConfig; use clean::Attributes; -use html::markdown; +use html::markdown::{self, RenderType}; #[derive(Clone, Default)] pub struct TestOptions { @@ -57,7 +57,8 @@ pub fn run(input: &str, externs: Externs, mut test_args: Vec, crate_name: Option, - maybe_sysroot: Option) + maybe_sysroot: Option, + render_type: RenderType) -> isize { let input_path = PathBuf::from(input); let input = config::Input::File(input_path.clone()); @@ -106,7 +107,8 @@ pub fn run(input: &str, opts, maybe_sysroot, Some(codemap), - None); + None, + render_type); { let dep_graph = DepGraph::new(false); @@ -396,12 +398,15 @@ pub struct Collector { position: Span, codemap: Option>, filename: Option, + // to be removed when hoedown will be removed as well + pub render_type: RenderType, } impl Collector { pub fn new(cratename: String, cfgs: Vec, libs: SearchPaths, externs: Externs, use_headers: bool, opts: TestOptions, maybe_sysroot: Option, - codemap: Option>, filename: Option) -> Collector { + codemap: Option>, filename: Option, + render_type: RenderType) -> Collector { Collector { tests: Vec::new(), old_tests: HashMap::new(), @@ -418,6 +423,7 @@ impl Collector { position: DUMMY_SP, codemap: codemap, filename: filename, + render_type: render_type, } } @@ -458,20 +464,22 @@ impl Collector { as_test_harness: bool, compile_fail: bool, error_codes: Vec, line: usize, filename: String) { let name = self.generate_name(line, &filename); - let name_beg = self.generate_name_beginning(&filename); - let mut found = false; - // to be removed when hoedown is removed - let test = test.trim().to_owned(); - if let Some(entry) = self.old_tests.get_mut(&name_beg) { - found = entry.remove_item(&test).is_some(); - } - if !found { - let _ = writeln!(&mut io::stderr(), - "WARNING: {} Code block is not currently run as a test, but will in \ - future versions of rustdoc. Please ensure this code block is a \ - runnable test, or use the `ignore` directive.", - name); - return + if self.render_type == RenderType::Pulldown { + let name_beg = self.generate_name_beginning(&filename); + let mut found = false; + // to be removed when hoedown is removed + let test = test.trim().to_owned(); + if let Some(entry) = self.old_tests.get_mut(&name_beg) { + found = entry.remove_item(&test).is_some(); + } + if !found { + let _ = writeln!(&mut io::stderr(), + "WARNING: {} Code block is not currently run as a test, but will in \ + future versions of rustdoc. Please ensure this code block is a \ + runnable test, or use the `ignore` directive.", + name); + return + } } let cfgs = self.cfgs.clone(); let libs = self.libs.clone(); @@ -587,10 +595,15 @@ impl<'a, 'hir> HirCollector<'a, 'hir> { attrs.unindent_doc_comments(); if let Some(doc) = attrs.doc_value() { self.collector.cnt = 0; - markdown::old_find_testable_code(doc, self.collector, + if self.collector.render_type == RenderType::Pulldown { + markdown::old_find_testable_code(doc, self.collector, + attrs.span.unwrap_or(DUMMY_SP)); + markdown::find_testable_code(doc, self.collector, attrs.span.unwrap_or(DUMMY_SP)); - markdown::find_testable_code(doc, self.collector, - attrs.span.unwrap_or(DUMMY_SP)); + } else { + markdown::old_find_testable_code(doc, self.collector, + attrs.span.unwrap_or(DUMMY_SP)); + } } nested(self); diff --git a/src/test/rustdoc/check-hard-break.rs b/src/test/rustdoc/check-hard-break.rs deleted file mode 100644 index f048b64d104ab..0000000000000 --- a/src/test/rustdoc/check-hard-break.rs +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright 2017 The Rust Project Developers. See the COPYRIGHT -// file at the top-level directory of this distribution and at -// http://rust-lang.org/COPYRIGHT. -// -// Licensed under the Apache License, Version 2.0 or the MIT license -// , at your -// option. This file may not be copied, modified, or distributed -// except according to those terms. - -#![crate_name = "foo"] - -// ignore-tidy-end-whitespace - -// @has foo/fn.f.html -// @has - '

hard break:
' -// @has - 'after hard break

' -/// hard break: -/// after hard break -pub fn f() {} diff --git a/src/test/rustdoc/check-rule-image-footnote.rs b/src/test/rustdoc/check-rule-image-footnote.rs deleted file mode 100644 index 46542677857fc..0000000000000 --- a/src/test/rustdoc/check-rule-image-footnote.rs +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright 2017 The Rust Project Developers. See the COPYRIGHT -// file at the top-level directory of this distribution and at -// http://rust-lang.org/COPYRIGHT. -// -// Licensed under the Apache License, Version 2.0 or the MIT license -// , at your -// option. This file may not be copied, modified, or distributed -// except according to those terms. - -#![crate_name = "foo"] - -// ignore-tidy-linelength - -// @has foo/fn.f.html -// @has - '

markdown test

' -// @has - '

this is a link.

' -// @has - '
' -// @has - '

a footnote1.

' -// @has - '

another footnote2.

' -// @has - '

Rust

' -// @has - '
  1. ' -// @has - '

    Thing â†©

  2. ' -// @has - '

    Another Thing â†©

' -/// markdown test -/// -/// this is a [link]. -/// -/// [link]: https://example.com "this is a title" -/// -/// ----------- -/// -/// a footnote[^footnote]. -/// -/// another footnote[^footnotebis]. -/// -/// [^footnote]: Thing -/// -/// -/// [^footnotebis]: Another Thing -/// -/// -/// ![Rust](https://www.rust-lang.org/logos/rust-logo-128x128-blk-v2.png) -pub fn f() {} From 295f25b19862eed40ed12e13db1afae1b4f41d35 Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Fri, 21 Apr 2017 12:27:48 +0200 Subject: [PATCH 2/5] Set hoedown to generate error index --- src/tools/error_index_generator/main.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tools/error_index_generator/main.rs b/src/tools/error_index_generator/main.rs index efadde992277f..ca383b5add011 100644 --- a/src/tools/error_index_generator/main.rs +++ b/src/tools/error_index_generator/main.rs @@ -24,7 +24,7 @@ use std::path::PathBuf; use syntax::diagnostics::metadata::{get_metadata_dir, ErrorMetadataMap, ErrorMetadata}; -use rustdoc::html::markdown::{Markdown, PLAYGROUND}; +use rustdoc::html::markdown::{Markdown, PLAYGROUND, RenderType}; use rustc_serialize::json; enum OutputFormat { @@ -100,7 +100,7 @@ impl Formatter for HTMLFormatter { // Description rendered as markdown. match info.description { - Some(ref desc) => write!(output, "{}", Markdown(desc))?, + Some(ref desc) => write!(output, "{}", Markdown(desc, RenderType::Hoedown))?, None => write!(output, "

No description.

\n")?, } From d79b511f5c0eb6092c1cc4c3fd41a673cf5e1d56 Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Fri, 21 Apr 2017 16:09:28 +0200 Subject: [PATCH 3/5] Fix invalid linkage --- src/libcollections/binary_heap.rs | 2 +- src/libcollections/btree/map.rs | 2 +- src/libcollections/btree/set.rs | 2 +- src/libcollections/linked_list.rs | 2 +- src/libcollections/vec_deque.rs | 2 +- src/libstd/collections/hash/map.rs | 2 +- src/libstd/collections/hash/set.rs | 2 +- src/libstd/path.rs | 2 +- src/libstd/sync/mpsc/mod.rs | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/libcollections/binary_heap.rs b/src/libcollections/binary_heap.rs index 149c285a72a98..b10a655c205b7 100644 --- a/src/libcollections/binary_heap.rs +++ b/src/libcollections/binary_heap.rs @@ -1042,7 +1042,7 @@ impl<'a, T> FusedIterator for Iter<'a, T> {} /// An owning iterator over the elements of a `BinaryHeap`. /// -/// This `struct` is created by the [`into_iter`] method on [`BinaryHeap`] +/// This `struct` is created by the [`into_iter`] method on [`BinaryHeap`][`BinaryHeap`] /// (provided by the `IntoIterator` trait). See its documentation for more. /// /// [`into_iter`]: struct.BinaryHeap.html#method.into_iter diff --git a/src/libcollections/btree/map.rs b/src/libcollections/btree/map.rs index fb0b852d10271..c5e4c76525243 100644 --- a/src/libcollections/btree/map.rs +++ b/src/libcollections/btree/map.rs @@ -298,7 +298,7 @@ pub struct IterMut<'a, K: 'a, V: 'a> { /// An owning iterator over the entries of a `BTreeMap`. /// -/// This `struct` is created by the [`into_iter`] method on [`BTreeMap`] +/// This `struct` is created by the [`into_iter`] method on [`BTreeMap`][`BTreeMap`] /// (provided by the `IntoIterator` trait). See its documentation for more. /// /// [`into_iter`]: struct.BTreeMap.html#method.into_iter diff --git a/src/libcollections/btree/set.rs b/src/libcollections/btree/set.rs index e05533aa50e3c..d32460da93923 100644 --- a/src/libcollections/btree/set.rs +++ b/src/libcollections/btree/set.rs @@ -97,7 +97,7 @@ impl<'a, T: 'a + fmt::Debug> fmt::Debug for Iter<'a, T> { /// An owning iterator over the items of a `BTreeSet`. /// -/// This `struct` is created by the [`into_iter`] method on [`BTreeSet`] +/// This `struct` is created by the [`into_iter`] method on [`BTreeSet`][`BTreeSet`] /// (provided by the `IntoIterator` trait). See its documentation for more. /// /// [`BTreeSet`]: struct.BTreeSet.html diff --git a/src/libcollections/linked_list.rs b/src/libcollections/linked_list.rs index 1cc5e10418f25..adfd91bec4893 100644 --- a/src/libcollections/linked_list.rs +++ b/src/libcollections/linked_list.rs @@ -115,7 +115,7 @@ impl<'a, T: 'a + fmt::Debug> fmt::Debug for IterMut<'a, T> { /// An owning iterator over the elements of a `LinkedList`. /// -/// This `struct` is created by the [`into_iter`] method on [`LinkedList`] +/// This `struct` is created by the [`into_iter`] method on [`LinkedList`][`LinkedList`] /// (provided by the `IntoIterator` trait). See its documentation for more. /// /// [`into_iter`]: struct.LinkedList.html#method.into_iter diff --git a/src/libcollections/vec_deque.rs b/src/libcollections/vec_deque.rs index a9e795f9378a6..079d3acf3764a 100644 --- a/src/libcollections/vec_deque.rs +++ b/src/libcollections/vec_deque.rs @@ -2070,7 +2070,7 @@ impl<'a, T> FusedIterator for IterMut<'a, T> {} /// An owning iterator over the elements of a `VecDeque`. /// -/// This `struct` is created by the [`into_iter`] method on [`VecDeque`] +/// This `struct` is created by the [`into_iter`] method on [`VecDeque`][`VecDeque`] /// (provided by the `IntoIterator` trait). See its documentation for more. /// /// [`into_iter`]: struct.VecDeque.html#method.into_iter diff --git a/src/libstd/collections/hash/map.rs b/src/libstd/collections/hash/map.rs index eacb59d375a50..e7102115caf5c 100644 --- a/src/libstd/collections/hash/map.rs +++ b/src/libstd/collections/hash/map.rs @@ -1377,7 +1377,7 @@ pub struct IterMut<'a, K: 'a, V: 'a> { /// An owning iterator over the entries of a `HashMap`. /// -/// This `struct` is created by the [`into_iter`] method on [`HashMap`] +/// This `struct` is created by the [`into_iter`] method on [`HashMap`][`HashMap`] /// (provided by the `IntoIterator` trait). See its documentation for more. /// /// [`into_iter`]: struct.HashMap.html#method.into_iter diff --git a/src/libstd/collections/hash/set.rs b/src/libstd/collections/hash/set.rs index e3fad28502573..7215e1bde8503 100644 --- a/src/libstd/collections/hash/set.rs +++ b/src/libstd/collections/hash/set.rs @@ -890,7 +890,7 @@ pub struct Iter<'a, K: 'a> { /// An owning iterator over the items of a `HashSet`. /// -/// This `struct` is created by the [`into_iter`] method on [`HashSet`] +/// This `struct` is created by the [`into_iter`] method on [`HashSet`][`HashSet`] /// (provided by the `IntoIterator` trait). See its documentation for more. /// /// [`HashSet`]: struct.HashSet.html diff --git a/src/libstd/path.rs b/src/libstd/path.rs index 15bc74a834010..9d66430bc9303 100644 --- a/src/libstd/path.rs +++ b/src/libstd/path.rs @@ -10,7 +10,7 @@ //! Cross-platform path manipulation. //! -//! This module provides two types, [`PathBuf`] and [`Path`] (akin to [`String`] +//! This module provides two types, [`PathBuf`] and [`Path`][`Path`] (akin to [`String`] //! and [`str`]), for working with paths abstractly. These types are thin wrappers //! around [`OsString`] and [`OsStr`] respectively, meaning that they work directly //! on strings according to the local platform's path syntax. diff --git a/src/libstd/sync/mpsc/mod.rs b/src/libstd/sync/mpsc/mod.rs index 852675edc0238..6c8839224f77d 100644 --- a/src/libstd/sync/mpsc/mod.rs +++ b/src/libstd/sync/mpsc/mod.rs @@ -452,7 +452,7 @@ pub struct SendError(#[stable(feature = "rust1", since = "1.0.0")] pub T); /// An error returned from the [`recv`] function on a [`Receiver`]. /// /// The [`recv`] operation can only fail if the sending half of a -/// [`channel`] (or [`sync_channel`]) is disconnected, implying that no further +/// [`channel`][`channel`] (or [`sync_channel`]) is disconnected, implying that no further /// messages will ever be received. /// /// [`recv`]: struct.Receiver.html#method.recv From 91fb6bc1eb9874096e30cdac5152cdbd9d25dbf4 Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Fri, 21 Apr 2017 18:41:47 +0200 Subject: [PATCH 4/5] Fix tests --- src/librustdoc/html/markdown.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/librustdoc/html/markdown.rs b/src/librustdoc/html/markdown.rs index 614d59c512da2..0dc7ae2873b71 100644 --- a/src/librustdoc/html/markdown.rs +++ b/src/librustdoc/html/markdown.rs @@ -1094,6 +1094,7 @@ pub fn plain_summary_line(md: &str) -> String { mod tests { use super::{LangString, Markdown, MarkdownHtml}; use super::plain_summary_line; + use super::RenderType; use html::render::reset_ids; #[test] @@ -1134,14 +1135,14 @@ mod tests { #[test] fn issue_17736() { let markdown = "# title"; - format!("{}", Markdown(markdown)); + format!("{}", Markdown(markdown, RenderType::Pulldown)); reset_ids(true); } #[test] fn test_header() { fn t(input: &str, expect: &str) { - let output = format!("{}", Markdown(input)); + let output = format!("{}", Markdown(input, RenderType::Pulldown)); assert_eq!(output, expect, "original: {}", input); reset_ids(true); } @@ -1163,7 +1164,7 @@ mod tests { #[test] fn test_header_ids_multiple_blocks() { fn t(input: &str, expect: &str) { - let output = format!("{}", Markdown(input)); + let output = format!("{}", Markdown(input, RenderType::Pulldown)); assert_eq!(output, expect, "original: {}", input); } @@ -1204,7 +1205,7 @@ mod tests { #[test] fn test_markdown_html_escape() { fn t(input: &str, expect: &str) { - let output = format!("{}", MarkdownHtml(input)); + let output = format!("{}", MarkdownHtml(input, RenderType::Pulldown)); assert_eq!(output, expect, "original: {}", input); } From 9c978820550994809940fa91436b6fbab9e6db49 Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Sat, 22 Apr 2017 14:56:36 +0200 Subject: [PATCH 5/5] Fix line display --- src/librustdoc/html/markdown.rs | 12 ++++-------- src/librustdoc/test.rs | 8 ++++---- 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/src/librustdoc/html/markdown.rs b/src/librustdoc/html/markdown.rs index 0dc7ae2873b71..85a28bbfbc6d0 100644 --- a/src/librustdoc/html/markdown.rs +++ b/src/librustdoc/html/markdown.rs @@ -719,7 +719,7 @@ pub fn old_find_testable_code(doc: &str, tests: &mut ::test::Collector, position text: *const hoedown_buffer, lang: *const hoedown_buffer, data: *const hoedown_renderer_data, - _line: libc::size_t) { + line: libc::size_t) { unsafe { if text.is_null() { return } let block_info = if lang.is_null() { @@ -737,22 +737,18 @@ pub fn old_find_testable_code(doc: &str, tests: &mut ::test::Collector, position let lines = text.lines().map(|l| { stripped_filtered_line(l).unwrap_or(l) }); + let text = lines.collect::>().join("\n"); let filename = tests.get_filename(); if tests.render_type == RenderType::Hoedown { - let text = (*text).as_bytes(); - let text = str::from_utf8(text).unwrap(); - let lines = text.lines().map(|l| { - stripped_filtered_line(l).unwrap_or(l) - }); - let text = lines.collect::>().join("\n"); + let line = tests.get_line() + line; tests.add_test(text.to_owned(), block_info.should_panic, block_info.no_run, block_info.ignore, block_info.test_harness, block_info.compile_fail, block_info.error_codes, line, filename); } else { - tests.add_old_test(lines.collect::>().join("\n"), filename); + tests.add_old_test(text, filename); } } } diff --git a/src/librustdoc/test.rs b/src/librustdoc/test.rs index a30ec25de60d7..5b9ab304db0a9 100644 --- a/src/librustdoc/test.rs +++ b/src/librustdoc/test.rs @@ -464,19 +464,19 @@ impl Collector { as_test_harness: bool, compile_fail: bool, error_codes: Vec, line: usize, filename: String) { let name = self.generate_name(line, &filename); + // to be removed when hoedown is removed if self.render_type == RenderType::Pulldown { let name_beg = self.generate_name_beginning(&filename); let mut found = false; - // to be removed when hoedown is removed let test = test.trim().to_owned(); if let Some(entry) = self.old_tests.get_mut(&name_beg) { found = entry.remove_item(&test).is_some(); } if !found { let _ = writeln!(&mut io::stderr(), - "WARNING: {} Code block is not currently run as a test, but will in \ - future versions of rustdoc. Please ensure this code block is a \ - runnable test, or use the `ignore` directive.", + "WARNING: {} Code block is not currently run as a test, but will \ + in future versions of rustdoc. Please ensure this code block is \ + a runnable test, or use the `ignore` directive.", name); return }