Auto merge of #90 - servo:url-functions, r=mbrubeck

Treat url(<string>) as a normal functions, per spec change.

Only unquoted URLs are special tokens now. Use `Parser::expect_url`.

This is a [breaking-change]. The version number was incremented accordingly.

This change will help with servo/servo#7767

This triggers rust-lang/rust#28934 and fails to build in the current Rust nightly, but works fine in the Rust version that Servo currently use. Hopefully that rustc bug will be fixed before we need to upgrade Rust in Servo.

r? @mbrubeck

<!-- Reviewable:start -->
[<img src="https://reviewable.io/review_button.png" height=40 alt="Review on Reviewable"/>](https://reviewable.io/reviews/servo/rust-cssparser/90)
<!-- Reviewable:end -->

Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 
 name = "cssparser"
-version = "0.3.9"
+version = "0.4.0"
 authors = [ "Simon Sapin <simon.sapin@exyr.org>" ]
 
 description = "Rust implementation of CSS Syntax Level 3"

src/parser.rs

@@ -331,7 +331,7 @@ impl<'i, 't> Parser<'i, 't> {
     /// This can help tell e.g. `color: green;` from `color: green 4px;`
     #[inline]
     pub fn parse_entirely<F, T>(&mut self, parse: F) -> Result<T, ()>
-    where F: FnOnce(&mut Parser) -> Result<T, ()> {
+    where F: FnOnce(&mut Parser<'i, 't>) -> Result<T, ()> {
         let result = parse(self);
         try!(self.expect_exhausted());
         result
@@ -374,7 +374,7 @@ impl<'i, 't> Parser<'i, 't> {
     /// The result is overridden to `Err(())` if the closure leaves some input before that point.
     #[inline]
     pub fn parse_nested_block<F, T>(&mut self, parse: F) -> Result <T, ()>
-    where F: FnOnce(&mut Parser) -> Result<T, ()> {
+    where F: for<'tt> FnOnce(&mut Parser<'i, 'tt>) -> Result<T, ()> {
         let block_type = self.at_start_of.take().expect("\
             A nested parser can only be created when a Function, \
             ParenthesisBlock, SquareBracketBlock, or CurlyBracketBlock \
@@ -412,7 +412,7 @@ impl<'i, 't> Parser<'i, 't> {
     #[inline]
     pub fn parse_until_before<F, T>(&mut self, delimiters: Delimiters, parse: F)
                                     -> Result <T, ()>
-    where F: FnOnce(&mut Parser) -> Result<T, ()> {
+    where F: for<'tt> FnOnce(&mut Parser<'i, 'tt>) -> Result<T, ()> {
         let delimiters = self.stop_before | delimiters;
         let result;
         // Introduce a new scope to limit duration of nested_parser’s borrow
@@ -451,7 +451,7 @@ impl<'i, 't> Parser<'i, 't> {
     #[inline]
     pub fn parse_until_after<F, T>(&mut self, delimiters: Delimiters, parse: F)
                                    -> Result <T, ()>
-    where F: FnOnce(&mut Parser) -> Result<T, ()> {
+    where F: for<'tt> FnOnce(&mut Parser<'i, 'tt>) -> Result<T, ()> {
         let result = self.parse_until_before(delimiters, parse);
         let next_byte = self.tokenizer.next_byte();
         if next_byte.is_some() && !self.stop_before.contains(Delimiters::from_byte(next_byte)) {
@@ -481,7 +481,7 @@ impl<'i, 't> Parser<'i, 't> {
 
     /// Parse a <ident-token> whose unescaped value is an ASCII-insensitive match for the given value.
     #[inline]
-    pub fn expect_ident_matching<'a>(&mut self, expected_value: &str) -> Result<(), ()> {
+    pub fn expect_ident_matching(&mut self, expected_value: &str) -> Result<(), ()> {
         match try!(self.next()) {
             Token::Ident(ref value) if value.eq_ignore_ascii_case(expected_value) => Ok(()),
             _ => Err(())
@@ -511,7 +511,10 @@ impl<'i, 't> Parser<'i, 't> {
     #[inline]
     pub fn expect_url(&mut self) -> Result<Cow<'i, str>, ()> {
         match try!(self.next()) {
-            Token::Url(value) => Ok(value),
+            Token::UnquotedUrl(value) => Ok(value),
+            Token::Function(ref name) if name.eq_ignore_ascii_case("url") => {
+                self.parse_nested_block(|input| input.expect_string())
+            },
             _ => Err(())
         }
     }
@@ -520,8 +523,11 @@ impl<'i, 't> Parser<'i, 't> {
     #[inline]
     pub fn expect_url_or_string(&mut self) -> Result<Cow<'i, str>, ()> {
         match try!(self.next()) {
-            Token::Url(value) => Ok(value),
+            Token::UnquotedUrl(value) => Ok(value),
             Token::QuotedString(value) => Ok(value),
+            Token::Function(ref name) if name.eq_ignore_ascii_case("url") => {
+                self.parse_nested_block(|input| input.expect_string())
+            },
             _ => Err(())
         }
     }

src/serializer.rs

@@ -81,9 +81,9 @@ impl<'a> ToCss for Token<'a> {
                 try!(serialize_identifier(&**value, dest));
             }
             Token::QuotedString(ref value) => try!(serialize_string(&**value, dest)),
-            Token::Url(ref value) => {
+            Token::UnquotedUrl(ref value) => {
                 try!(dest.write_str("url("));
-                try!(serialize_string(&**value, dest));
+                try!(serialize_unquoted_url(&**value, dest));
                 try!(dest.write_str(")"));
             },
             Token::Delim(value) => try!(write!(dest, "{}", value)),
@@ -121,7 +121,7 @@ impl<'a> ToCss for Token<'a> {
                     if truncated_start != 0 {
                         try!(write!(dest, "{:X}", truncated_start));
                     }
-                    for _ in (0..question_marks) {
+                    for _ in 0..question_marks {
                         try!(dest.write_str("?"));
                     }
                 } else {
@@ -213,6 +213,26 @@ fn serialize_name<W>(value: &str, dest: &mut W) -> fmt::Result where W:fmt::Writ
 }
 
 
+fn serialize_unquoted_url<W>(value: &str, dest: &mut W) -> fmt::Result where W:fmt::Write {
+    let mut chunk_start = 0;
+    for (i, b) in value.bytes().enumerate() {
+        let hex = match b {
+            b'\0' ... b' ' | b'\x7F' => true,
+            b'(' | b')' | b'"' | b'\'' | b'\\' => false,
+            _ => continue
+        };
+        try!(dest.write_str(&value[chunk_start..i]));
+        if hex {
+            try!(write!(dest, "\\{:X} ", b));
+        } else {
+            try!(write!(dest, "\\{}", b as char));
+        }
+        chunk_start = i + 1;
+    }
+    dest.write_str(&value[chunk_start..])
+}
+
+
 /// Write a double-quoted CSS string token, escaping content as necessary.
 pub fn serialize_string<W>(value: &str, dest: &mut W) -> fmt::Result where W: fmt::Write {
     try!(dest.write_str("\""));
@@ -382,7 +402,7 @@ impl<'a> Token<'a> {
         TokenSerializationType(match *self {
             Token::Ident(_) => Ident,
             Token::AtKeyword(_) | Token::Hash(_) | Token::IDHash(_) => AtKeywordOrHash,
-            Token::Url(_) | Token::BadUrl => UrlOrBadUrl,
+            Token::UnquotedUrl(_) | Token::BadUrl => UrlOrBadUrl,
             Token::Delim('#') => DelimHash,
             Token::Delim('@') => DelimAt,
             Token::Delim('.') | Token::Delim('+') => DelimDotOrPlus,

src/tests.rs

@@ -2,12 +2,11 @@
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
-use std::borrow::Cow::Borrowed;
+use std::borrow::Cow::{self, Borrowed};
 use std::fs::File;
 use std::io::{self, Write};
 use std::path::Path;
 use std::process::Command;
-use std::mem;
 use rustc_serialize::json::{self, Json, ToJson};
 use tempdir::TempDir;
 
@@ -74,14 +73,8 @@ fn almost_equals(a: &Json, b: &Json) -> bool {
 fn normalize(json: &mut Json) {
     match *json {
         Json::Array(ref mut list) => {
-            match find_url(list) {
-                Some(Ok(url)) => *list = vec!["url".to_json(), Json::String(url)],
-                Some(Err(())) => *list = vec!["error".to_json(), "bad-url".to_json()],
-                None => {
-                    for item in list.iter_mut() {
-                        normalize(item)
-                    }
-                }
+            for item in list.iter_mut() {
+                normalize(item)
             }
         }
         Json::String(ref mut s) => {
@@ -93,26 +86,6 @@ fn normalize(json: &mut Json) {
     }
 }
 
-fn find_url(list: &mut [Json]) -> Option<Result<String, ()>> {
-    if list.len() < 2 ||
-        list[0].as_string() != Some("function") ||
-        list[1].as_string() != Some("url") {
-        return None
-    }
-
-    let mut args = list[2..].iter_mut().filter(|a| a.as_string() != Some(" "));
-    if let (Some(&mut Json::Array(ref mut arg)), None) = (args.next(), args.next()) {
-        if arg.len() == 2 && arg[0].as_string() == Some("string") {
-            if let &mut Json::String(ref mut value) = &mut arg[1] {
-                return Some(Ok(mem::replace(value, String::new())))
-            }
-        }
-    }
-
-    Some(Err(()))
-}
-
-
 fn assert_json_eq(results: json::Json, mut expected: json::Json, message: String) {
     normalize(&mut expected);
     if !almost_equals(&results, &expected) {
@@ -281,6 +254,42 @@ fn outer_block_end_consumed() {
     assert_eq!(input.next(), Err(()));
 }
 
+#[test]
+fn unquoted_url_escaping() {
+    let token = Token::UnquotedUrl("\
+        \x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\
+        \x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f \
+        !\"#$%&\'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]\
+        ^_`abcdefghijklmnopqrstuvwxyz{|}~\x7fé\
+    ".into());
+    let serialized = token.to_css_string();
+    assert_eq!(serialized, "\
+        url(\
+            \\1 \\2 \\3 \\4 \\5 \\6 \\7 \\8 \\9 \\A \\B \\C \\D \\E \\F \\10 \
+            \\11 \\12 \\13 \\14 \\15 \\16 \\17 \\18 \\19 \\1A \\1B \\1C \\1D \\1E \\1F \\20 \
+            !\\\"#$%&\\'\\(\\)*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\\\]\
+            ^_`abcdefghijklmnopqrstuvwxyz{|}~\\7F é\
+        )\
+    ");
+    assert_eq!(Parser::new(&serialized).next(), Ok(token))
+}
+
+#[test]
+fn test_expect_url() {
+    fn parse(s: &str) -> Result<Cow<str>, ()> {
+        Parser::new(s).expect_url()
+    }
+    assert_eq!(parse("url()").unwrap(), "");
+    assert_eq!(parse("url( ").unwrap(), "");
+    assert_eq!(parse("url( abc").unwrap(), "abc");
+    assert_eq!(parse("url( abc \t)").unwrap(), "abc");
+    assert_eq!(parse("url( 'abc' \t)").unwrap(), "abc");
+    assert_eq!(parse("url(abc more stuff)"), Err(()));
+    // The grammar at https://drafts.csswg.org/css-values/#urls plans for `<url-modifier>*`
+    // at the position of "more stuff", but no such modifier is defined yet.
+    assert_eq!(parse("url('abc' more stuff)"), Err(()));
+}
+
 
 fn run_color_tests<F: Fn(Result<Color, ()>) -> Json>(json_data: &str, to_json: F) {
     run_json_tests(json_data, |input| {
@@ -606,7 +615,7 @@ fn one_component_value_to_json(token: Token, input: &mut Parser) -> Json {
         Token::Hash(value) => JArray!["hash", value, "unrestricted"],
         Token::IDHash(value) => JArray!["hash", value, "id"],
         Token::QuotedString(value) => JArray!["string", value],
-        Token::Url(value) => JArray!["url", value],
+        Token::UnquotedUrl(value) => JArray!["url", value],
         Token::Delim('\\') => "\\".to_json(),
         Token::Delim(value) => value.to_string().to_json(),
 

src/tokenizer.rs

@@ -48,7 +48,7 @@ pub enum Token<'a> {
     /// A [`<url-token>`](https://drafts.csswg.org/css-syntax/#url-token-diagram) or `url( <string-token> )` function
     ///
     /// The value does not include the `url(` `)` markers or the quotes.
-    Url(Cow<'a, str>),
+    UnquotedUrl(Cow<'a, str>),
 
     /// A `<delim-token>`
     Delim(char),
@@ -628,7 +628,7 @@ fn consume_ident_like<'a>(tokenizer: &mut Tokenizer<'a>) -> Token<'a> {
     if !tokenizer.is_eof() && tokenizer.next_char() == '(' {
         tokenizer.advance(1);
         if value.eq_ignore_ascii_case("url") {
-            consume_url(tokenizer)
+            consume_unquoted_url(tokenizer).unwrap_or(Function(value))
         } else {
             if tokenizer.var_functions == VarFunctions::LookingForThem &&
                 value.eq_ignore_ascii_case("var") {
@@ -791,31 +791,30 @@ fn consume_numeric<'a>(tokenizer: &mut Tokenizer<'a>) -> Token<'a> {
 }
 
 
-fn consume_url<'a>(tokenizer: &mut Tokenizer<'a>) -> Token<'a> {
-    while !tokenizer.is_eof() {
-        match tokenizer.next_char() {
-            ' ' | '\t' | '\n' | '\r' | '\x0C' => tokenizer.advance(1),
-            '"' => return consume_quoted_url(tokenizer, false),
-            '\'' => return consume_quoted_url(tokenizer, true),
-            ')' => { tokenizer.advance(1); break },
-            _ => return consume_unquoted_url(tokenizer),
-        }
-    }
-    return Url(Borrowed(""));
-
-    fn consume_quoted_url<'a>(tokenizer: &mut Tokenizer<'a>, single_quote: bool) -> Token<'a> {
-        match consume_quoted_string(tokenizer, single_quote) {
-            Ok(value) => consume_url_end(tokenizer, value),
-            Err(()) => consume_bad_url(tokenizer),
+fn consume_unquoted_url<'a>(tokenizer: &mut Tokenizer<'a>) -> Result<Token<'a>, ()> {
+    for (offset, c) in tokenizer.input[tokenizer.position..].char_indices() {
+        match c {
+            ' ' | '\t' | '\n' | '\r' | '\x0C' => {},
+            '"' | '\'' => return Err(()),  // Do not advance
+            ')' => {
+                tokenizer.advance(offset + 1);
+                return Ok(UnquotedUrl(Borrowed("")));
+            }
+            _ => {
+                tokenizer.advance(offset);
+                return Ok(consume_unquoted_url(tokenizer))
+            }
         }
     }
+    tokenizer.position = tokenizer.input.len();
+    return Ok(UnquotedUrl(Borrowed("")));
 
     fn consume_unquoted_url<'a>(tokenizer: &mut Tokenizer<'a>) -> Token<'a> {
         let start_pos = tokenizer.position();
         let mut string;
         loop {
             if tokenizer.is_eof() {
-                return Url(Borrowed(tokenizer.slice_from(start_pos)))
+                return UnquotedUrl(Borrowed(tokenizer.slice_from(start_pos)))
             }
             match tokenizer.next_char() {
                 ' ' | '\t' | '\n' | '\r' | '\x0C' => {
@@ -826,7 +825,7 @@ fn consume_url<'a>(tokenizer: &mut Tokenizer<'a>) -> Token<'a> {
                 ')' => {
                     let value = tokenizer.slice_from(start_pos);
                     tokenizer.advance(1);
-                    return Url(Borrowed(value))
+                    return UnquotedUrl(Borrowed(value))
                 }
                 '\x01'...'\x08' | '\x0B' | '\x0E'...'\x1F' | '\x7F'  // non-printable
                     | '"' | '\'' | '(' => {
@@ -861,7 +860,7 @@ fn consume_url<'a>(tokenizer: &mut Tokenizer<'a>) -> Token<'a> {
             };
             string.push(next_char)
         }
-        Url(Owned(string))
+        UnquotedUrl(Owned(string))
     }
 
     fn consume_url_end<'a>(tokenizer: &mut Tokenizer<'a>, string: Cow<'a, str>) -> Token<'a> {
@@ -872,7 +871,7 @@ fn consume_url<'a>(tokenizer: &mut Tokenizer<'a>) -> Token<'a> {
                 _ => return consume_bad_url(tokenizer)
             }
         }
-        Url(string)
+        UnquotedUrl(string)
     }
 
     fn consume_bad_url<'a>(tokenizer: &mut Tokenizer<'a>) -> Token<'a> {