test.rs

// Copyright 2012-2015 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

use super::UnicodeSegmentation;

use std::prelude::v1::*;

#[test]
fn test_graphemes() {
    use crate::testdata::{TEST_DIFF, TEST_SAME};

    pub const EXTRA_DIFF: &'static [(
        &'static str,
        &'static [&'static str],
        &'static [&'static str],
    )] = &[
        // Official test suite doesn't include two Prepend chars between two other chars.
        (
            "\u{20}\u{600}\u{600}\u{20}",
            &["\u{20}", "\u{600}\u{600}\u{20}"],
            &["\u{20}", "\u{600}", "\u{600}", "\u{20}"],
        ),
        // Test for Prepend followed by two Any chars
        (
            "\u{600}\u{20}\u{20}",
            &["\u{600}\u{20}", "\u{20}"],
            &["\u{600}", "\u{20}", "\u{20}"],
        ),
    ];

    pub const EXTRA_SAME: &'static [(&'static str, &'static [&'static str])] = &[
        // family emoji (more than two emoji joined by ZWJ)
        (
            "\u{1f468}\u{200d}\u{1f467}\u{200d}\u{1f466}",
            &["\u{1f468}\u{200d}\u{1f467}\u{200d}\u{1f466}"],
        ),
        // cartwheel emoji followed by two fitzpatrick skin tone modifiers
        // (test case from issue #19)
        (
            "\u{1F938}\u{1F3FE}\u{1F3FE}",
            &["\u{1F938}\u{1F3FE}\u{1F3FE}"],
        ),
    ];

    for &(s, g) in TEST_SAME.iter().chain(EXTRA_SAME) {
        if s.starts_with("क\u{94d}") || s.starts_with("क\u{93c}") {
            continue; // TODO: fix these
        }
        // test forward iterator
        assert!(UnicodeSegmentation::graphemes(s, true).eq(g.iter().cloned()));
        assert!(UnicodeSegmentation::graphemes(s, false).eq(g.iter().cloned()));

        // test reverse iterator
        assert!(UnicodeSegmentation::graphemes(s, true)
            .rev()
            .eq(g.iter().rev().cloned()));
        assert!(UnicodeSegmentation::graphemes(s, false)
            .rev()
            .eq(g.iter().rev().cloned()));
    }

    for &(s, gt, gf) in TEST_DIFF.iter().chain(EXTRA_DIFF) {
        // test forward iterator
        assert!(UnicodeSegmentation::graphemes(s, true).eq(gt.iter().cloned()));
        assert!(UnicodeSegmentation::graphemes(s, false).eq(gf.iter().cloned()));

        // test reverse iterator
        assert!(UnicodeSegmentation::graphemes(s, true)
            .rev()
            .eq(gt.iter().rev().cloned()));
        assert!(UnicodeSegmentation::graphemes(s, false)
            .rev()
            .eq(gf.iter().rev().cloned()));
    }

    // test the indices iterators
    let s = "a̐éö̲\r\n";
    let gr_inds = UnicodeSegmentation::grapheme_indices(s, true).collect::<Vec<(usize, &str)>>();
    let b: &[_] = &[(0, "a̐"), (3, "é"), (6, "ö̲"), (11, "\r\n")];
    assert_eq!(gr_inds, b);
    let gr_inds = UnicodeSegmentation::grapheme_indices(s, true)
        .rev()
        .collect::<Vec<(usize, &str)>>();
    let b: &[_] = &[(11, "\r\n"), (6, "ö̲"), (3, "é"), (0, "a̐")];
    assert_eq!(gr_inds, b);
    let mut gr_inds_iter = UnicodeSegmentation::grapheme_indices(s, true);
    {
        let gr_inds = gr_inds_iter.by_ref();
        let e1 = gr_inds.size_hint();
        assert_eq!(e1, (1, Some(13)));
        let c = gr_inds.count();
        assert_eq!(c, 4);
    }
    let e2 = gr_inds_iter.size_hint();
    assert_eq!(e2, (0, Some(0)));

    // make sure the reverse iterator does the right thing with "\n" at beginning of string
    let s = "\n\r\n\r";
    let gr = UnicodeSegmentation::graphemes(s, true)
        .rev()
        .collect::<Vec<&str>>();
    let b: &[_] = &["\r", "\r\n", "\n"];
    assert_eq!(gr, b);
}

#[test]
fn test_words() {
    use crate::testdata::TEST_WORD;

    // Unicode's official tests don't really test longer chains of flag emoji
    // TODO This could be improved with more tests like flag emoji with interspersed Extend chars and ZWJ
    const EXTRA_TESTS: &'static [(&'static str, &'static [&'static str])] = &[
        (
            "🇦🇫🇦🇽🇦🇱🇩🇿🇦🇸🇦🇩🇦🇴",
            &["🇦🇫", "🇦🇽", "🇦🇱", "🇩🇿", "🇦🇸", "🇦🇩", "🇦🇴"],
        ),
        ("🇦🇫🇦🇽🇦🇱🇩🇿🇦🇸🇦🇩🇦", &["🇦🇫", "🇦🇽", "🇦🇱", "🇩🇿", "🇦🇸", "🇦🇩", "🇦"]),
        (
            "🇦a🇫🇦🇽a🇦🇱🇩🇿🇦🇸🇦🇩🇦",
            &["🇦", "a", "🇫🇦", "🇽", "a", "🇦🇱", "🇩🇿", "🇦🇸", "🇦🇩", "🇦"],
        ),
        (
            "\u{1f468}\u{200d}\u{1f468}\u{200d}\u{1f466}",
            &["\u{1f468}\u{200d}\u{1f468}\u{200d}\u{1f466}"],
        ),
        ("😌👎🏼", &["😌", "👎🏼"]),
        // perhaps wrong, spaces should not be included?
        ("hello world", &["hello", " ", "world"]),
        ("🇨🇦🇨🇭🇿🇲🇿 hi", &["🇨🇦", "🇨🇭", "🇿🇲", "🇿", " ", "hi"]),
    ];
    for &(s, w) in TEST_WORD.iter().chain(EXTRA_TESTS.iter()) {
        macro_rules! assert_ {
            ($test:expr, $exp:expr, $name:expr) => {
                // collect into vector for better diagnostics in failure case
                let testing = $test.collect::<Vec<_>>();
                let expected = $exp.collect::<Vec<_>>();
                assert_eq!(
                    testing, expected,
                    "{} test for testcase ({:?}, {:?}) failed.",
                    $name, s, w
                )
            };
        }
        // test forward iterator
        assert_!(
            s.split_word_bounds(),
            w.iter().cloned(),
            "Forward word boundaries"
        );

        // test reverse iterator
        assert_!(
            s.split_word_bounds().rev(),
            w.iter().rev().cloned(),
            "Reverse word boundaries"
        );

        // generate offsets from word string lengths
        let mut indices = vec![0];
        for i in w.iter().cloned().map(|s| s.len()).scan(0, |t, n| {
            *t += n;
            Some(*t)
        }) {
            indices.push(i);
        }
        indices.pop();
        let indices = indices;

        // test forward indices iterator
        assert_!(
            s.split_word_bound_indices().map(|(l, _)| l),
            indices.iter().cloned(),
            "Forward word indices"
        );

        // test backward indices iterator
        assert_!(
            s.split_word_bound_indices().rev().map(|(l, _)| l),
            indices.iter().rev().cloned(),
            "Reverse word indices"
        );
    }
}

#[test]
fn test_sentences() {
    use crate::testdata::TEST_SENTENCE;

    for &(s, w) in TEST_SENTENCE.iter() {
        macro_rules! assert_ {
            ($test:expr, $exp:expr, $name:expr) => {
                // collect into vector for better diagnostics in failure case
                let testing = $test.collect::<Vec<_>>();
                let expected = $exp.collect::<Vec<_>>();
                assert_eq!(
                    testing, expected,
                    "{} test for testcase ({:?}, {:?}) failed.",
                    $name, s, w
                )
            };
        }

        assert_!(
            s.split_sentence_bounds(),
            w.iter().cloned(),
            "Forward sentence boundaries"
        );
    }
}

#[test]
fn test_syriac_abbr_mark() {
    use crate::tables::word as wd;
    let (_, _, cat) = wd::word_category('\u{70f}');
    assert_eq!(cat, wd::WC_ALetter);
}

#[test]
fn test_end_of_ayah_cat() {
    use crate::tables::word as wd;
    let (_, _, cat) = wd::word_category('\u{6dd}');
    assert_eq!(cat, wd::WC_Numeric);
}

quickcheck! {
    fn quickcheck_forward_reverse_graphemes_extended(s: String) -> bool {
        let a = s.graphemes(true).collect::<Vec<_>>();
        let mut b = s.graphemes(true).rev().collect::<Vec<_>>();
        b.reverse();
        a == b
    }

    fn quickcheck_forward_reverse_graphemes_legacy(s: String) -> bool {
        let a = s.graphemes(false).collect::<Vec<_>>();
        let mut b = s.graphemes(false).rev().collect::<Vec<_>>();
        b.reverse();
        a == b
    }

    fn quickcheck_join_graphemes(s: String) -> bool {
        let a = s.graphemes(true).collect::<String>();
        let b = s.graphemes(false).collect::<String>();
        a == s && b == s
    }

    fn quickcheck_forward_reverse_words(s: String) -> bool {
        let a = s.split_word_bounds().collect::<Vec<_>>();
        let mut b = s.split_word_bounds().rev().collect::<Vec<_>>();
        b.reverse();
        a == b
    }

    fn quickcheck_join_words(s: String) -> bool {
        let a = s.split_word_bounds().collect::<String>();
        a == s
    }
}