Skip to content

Commit

Permalink
Rollup merge of rust-lang#69164 - GuillaumeGomez:update-pulldown-cmar…
Browse files Browse the repository at this point in the history 
…k, r=Dylan-DPC

Update pulldown-cmark dependency

r? @kinnison

cc @ollie27

Reopening of  rust-lang#65894.
  • Loading branch information
@JohnTitor
JohnTitor authored Feb 14, 2020
2 parents 72def9a + d8589de commit a6ff1db
Show file tree
Hide file tree
Showing 3 changed files with 108 additions and 94 deletions.
30 changes: 12 additions & 18 deletions Cargo.lock
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1564,7 +1564,7 @@ dependencies = [
"rand_xoshiro",
"sized-chunks",
"typenum",
"version_check 0.9.1",
"version_check",
]

[[package]]
Expand Down Expand Up @@ -2014,9 +2014,9 @@ dependencies = [

[[package]]
name = "memchr"
version = "2.2.0"
version = "2.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2efc7bc57c883d4a4d6e3246905283d8dae951bb3bd32f49d6ef297f546e1c39"
checksum = "53445de381a1f436797497c61d851644d0e8e88e6140f22872ad33a704933978"

[[package]]
name = "memmap"
Expand Down Expand Up @@ -2604,23 +2604,23 @@ dependencies = [

[[package]]
name = "pulldown-cmark"
version = "0.5.3"
version = "0.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "77043da1282374688ee212dc44b3f37ff929431de9c9adc3053bd3cee5630357"
checksum = "1c205cc82214f3594e2d50686730314f817c67ffa80fe800cf0db78c3c2b9d9e"
dependencies = [
"bitflags",
"getopts",
"memchr",
"unicase",
]

[[package]]
name = "pulldown-cmark"
version = "0.6.1"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1c205cc82214f3594e2d50686730314f817c67ffa80fe800cf0db78c3c2b9d9e"
checksum = "2c2d7fd131800e0d63df52aff46201acaab70b431a4a1ec6f0343fe8e64f35a4"
dependencies = [
"bitflags",
"getopts",
"memchr",
"unicase",
]
Expand Down Expand Up @@ -4160,7 +4160,7 @@ version = "0.0.0"
dependencies = [
"itertools 0.8.0",
"minifier",
"pulldown-cmark 0.5.3",
"pulldown-cmark 0.7.0",
"rustc-rayon",
"serde",
"serde_json",
Expand Down Expand Up @@ -5160,11 +5160,11 @@ checksum = "535c204ee4d8434478593480b8f86ab45ec9aae0e83c568ca81abf0fd0e88f86"

[[package]]
name = "unicase"
version = "2.5.1"
version = "2.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2e2e6bd1e59e56598518beb94fd6db628ded570326f0a98c679a304bd9f00150"
checksum = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6"
dependencies = [
"version_check 0.1.5",
"version_check",
]

[[package]]
Expand Down Expand Up @@ -5334,12 +5334,6 @@ dependencies = [
"failure",
]

[[package]]
name = "version_check"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "914b1a6776c4c929a602fafd8bc742e06365d4bcbe48c30f9cca5824f70dc9dd"

[[package]]
name = "version_check"
version = "0.9.1"
Expand Down
2 changes: 1 addition & 1 deletion src/librustdoc/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ name = "rustdoc"
path = "lib.rs"

[dependencies]
pulldown-cmark = { version = "0.5.3", default-features = false }
pulldown-cmark = { version = "0.7", default-features = false }
minifier = "0.0.33"
rayon = { version = "0.3.0", package = "rustc-rayon" }
serde = { version = "1.0", features = ["derive"] }
Expand Down
170 changes: 95 additions & 75 deletions src/librustdoc/html/markdown.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ use crate::html::highlight;
use crate::html::toc::TocBuilder;
use crate::test;

use pulldown_cmark::{html, CowStr, Event, Options, Parser, Tag};
use pulldown_cmark::{html, CodeBlockKind, CowStr, Event, Options, Parser, Tag};

#[cfg(test)]
mod tests;
Expand Down Expand Up @@ -189,10 +189,15 @@ impl<'a, I: Iterator<Item = Event<'a>>> Iterator for CodeBlocks<'_, 'a, I> {
let compile_fail;
let ignore;
let edition;
if let Some(Event::Start(Tag::CodeBlock(lang))) = event {
let parse_result = LangString::parse(&lang, self.check_error_codes, false);
if let Some(Event::Start(Tag::CodeBlock(kind))) = event {
let parse_result = match kind {
CodeBlockKind::Fenced(ref lang) => {
LangString::parse(&lang, self.check_error_codes, false)
}
CodeBlockKind::Indented => LangString::all_false(),
};
if !parse_result.rust {
return Some(Event::Start(Tag::CodeBlock(lang)));
return Some(Event::Start(Tag::CodeBlock(kind)));
}
compile_fail = parse_result.compile_fail;
ignore = parse_result.ignore;
Expand Down Expand Up @@ -370,11 +375,11 @@ impl<'a, 'b, 'ids, I: Iterator<Item = Event<'a>>> Iterator for HeadingLinks<'a,
}

let event = self.inner.next();
if let Some(Event::Start(Tag::Header(level))) = event {
if let Some(Event::Start(Tag::Heading(level))) = event {
let mut id = String::new();
for event in &mut self.inner {
match &event {
Event::End(Tag::Header(..)) => break,
Event::End(Tag::Heading(..)) => break,
Event::Text(text) | Event::Code(text) => {
id.extend(text.chars().filter_map(slugify));
}
Expand All @@ -391,18 +396,18 @@ impl<'a, 'b, 'ids, I: Iterator<Item = Event<'a>>> Iterator for HeadingLinks<'a,
let mut html_header = String::new();
html::push_html(&mut html_header, self.buf.iter().cloned());
let sec = builder.push(level as u32, html_header, id.clone());
self.buf.push_front(Event::InlineHtml(format!("{} ", sec).into()));
self.buf.push_front(Event::Html(format!("{} ", sec).into()));
}

self.buf.push_back(Event::InlineHtml(format!("</a></h{}>", level).into()));
self.buf.push_back(Event::Html(format!("</a></h{}>", level).into()));

let start_tags = format!(
"<h{level} id=\"{id}\" class=\"section-header\">\
<a href=\"#{id}\">",
id = id,
level = level
);
return Some(Event::InlineHtml(start_tags.into()));
return Some(Event::Html(start_tags.into()));
}
event
}
Expand Down Expand Up @@ -556,40 +561,44 @@ pub fn find_testable_code<T: test::Tester>(
error_codes: ErrorCodes,
enable_per_target_ignores: bool,
) {
let mut parser = Parser::new(doc);
let mut parser = Parser::new(doc).into_offset_iter();
let mut prev_offset = 0;
let mut nb_lines = 0;
let mut register_header = None;
while let Some(event) = parser.next() {
while let Some((event, offset)) = parser.next() {
match event {
Event::Start(Tag::CodeBlock(s)) => {
let offset = parser.get_offset();

let block_info = if s.is_empty() {
LangString::all_false()
} else {
LangString::parse(&*s, error_codes, enable_per_target_ignores)
Event::Start(Tag::CodeBlock(kind)) => {
let block_info = match kind {
CodeBlockKind::Fenced(ref lang) => {
if lang.is_empty() {
LangString::all_false()
} else {
LangString::parse(lang, error_codes, enable_per_target_ignores)
}
}
CodeBlockKind::Indented => LangString::all_false(),
};
if !block_info.rust {
continue;
}

let mut test_s = String::new();

while let Some(Event::Text(s)) = parser.next() {
while let Some((Event::Text(s), _)) = parser.next() {
test_s.push_str(&s);
}

let text = test_s
.lines()
.map(|l| map_line(l).for_code())
.collect::<Vec<Cow<'_, str>>>()
.join("\n");
nb_lines += doc[prev_offset..offset].lines().count();
let line = tests.get_line() + nb_lines;

nb_lines += doc[prev_offset..offset.start].lines().count();
let line = tests.get_line() + nb_lines + 1;
tests.add_test(text, block_info, line);
prev_offset = offset;
prev_offset = offset.start;
}
Event::Start(Tag::Header(level)) => {
Event::Start(Tag::Heading(level)) => {
register_header = Some(level as u32);
}
Event::Text(ref s) if register_header.is_some() => {
Expand Down Expand Up @@ -783,7 +792,7 @@ impl MarkdownHtml<'_> {

// Treat inline HTML as plain text.
let p = p.map(|event| match event {
Event::Html(text) | Event::InlineHtml(text) => Event::Text(text),
Event::Html(text) => Event::Text(text),
_ => event,
});

Expand Down Expand Up @@ -842,10 +851,10 @@ pub fn plain_summary_line(md: &str) -> String {
let next_event = next_event.unwrap();
let (ret, is_in) = match next_event {
Event::Start(Tag::Paragraph) => (None, 1),
Event::Start(Tag::Header(_)) => (None, 1),
Event::Start(Tag::Heading(_)) => (None, 1),
Event::Code(code) => (Some(format!("`{}`", code)), 0),
Event::Text(ref s) if self.is_in > 0 => (Some(s.as_ref().to_owned()), 0),
Event::End(Tag::Paragraph) | Event::End(Tag::Header(_)) => (None, -1),
Event::End(Tag::Paragraph) | Event::End(Tag::Heading(_)) => (None, -1),
_ => (None, 0),
};
if is_in > 0 || (is_in < 0 && self.is_in > 0) {
Expand Down Expand Up @@ -940,68 +949,79 @@ crate fn rust_code_blocks(md: &str) -> Vec<RustCodeBlock> {
return code_blocks;
}

let mut p = Parser::new_ext(md, opts());

let mut code_block_start = 0;
let mut code_start = 0;
let mut is_fenced = false;
let mut previous_offset = 0;
let mut in_rust_code_block = false;
while let Some(event) = p.next() {
let offset = p.get_offset();
let mut p = Parser::new_ext(md, opts()).into_offset_iter();

while let Some((event, offset)) = p.next() {
match event {
Event::Start(Tag::CodeBlock(syntax)) => {
let lang_string = if syntax.is_empty() {
LangString::all_false()
} else {
LangString::parse(&*syntax, ErrorCodes::Yes, false)
};

if lang_string.rust {
in_rust_code_block = true;

code_start = offset;
code_block_start = match md[previous_offset..offset].find("```") {
Some(fence_idx) => {
is_fenced = true;
previous_offset + fence_idx
let (syntax, code_start, code_end, range, is_fenced) = match syntax {
CodeBlockKind::Fenced(syntax) => {
let syntax = syntax.as_ref();
let lang_string = if syntax.is_empty() {
LangString::all_false()
} else {
LangString::parse(&*syntax, ErrorCodes::Yes, false)
};
if !lang_string.rust {
continue;
}
None => {
is_fenced = false;
offset
let syntax = if syntax.is_empty() { None } else { Some(syntax.to_owned()) };
let (code_start, mut code_end) = match p.next() {
Some((Event::Text(_), offset)) => (offset.start, offset.end),
Some((_, sub_offset)) => {
let code = Range { start: sub_offset.start, end: sub_offset.start };
code_blocks.push(RustCodeBlock {
is_fenced: true,
range: offset,
code,
syntax,
});
continue;
}
None => {
let code = Range { start: offset.end, end: offset.end };
code_blocks.push(RustCodeBlock {
is_fenced: true,
range: offset,
code,
syntax,
});
continue;
}
};
while let Some((Event::Text(_), offset)) = p.next() {
code_end = offset.end;
}
};
}
}
Event::End(Tag::CodeBlock(syntax)) if in_rust_code_block => {
in_rust_code_block = false;

let code_block_end = if is_fenced {
let fence_str = &md[previous_offset..offset].chars().rev().collect::<String>();
fence_str
.find("```")
.map(|fence_idx| offset - fence_idx)
.unwrap_or_else(|| offset)
} else if md.as_bytes().get(offset).map(|b| *b == b'\n').unwrap_or_default() {
offset - 1
} else {
offset
(syntax, code_start, code_end, offset, true)
}
CodeBlockKind::Indented => {
// The ending of the offset goes too far sometime so we reduce it by one in
// these cases.
if offset.end > offset.start
&& md.get(offset.end..=offset.end) == Some(&"\n")
{
(
None,
offset.start,
offset.end,
Range { start: offset.start, end: offset.end - 1 },
false,
)
} else {
(None, offset.start, offset.end, offset, false)
}
}
};

let code_end = if is_fenced { previous_offset } else { code_block_end };

code_blocks.push(RustCodeBlock {
is_fenced,
range: Range { start: code_block_start, end: code_block_end },
range,
code: Range { start: code_start, end: code_end },
syntax: if !syntax.is_empty() { Some(syntax.into_string()) } else { None },
syntax,
});
}
_ => (),
}

previous_offset = offset;
}

code_blocks
Expand Down

0 comments on commit a6ff1db

Please sign in to comment.