rustc lexer: regression tests for embedded Idents.

I chose to make two of them because I wanted something close to an
"end-to-end" test (*), but at the same time I wanted a test that
would run on Windows (**).

(*) The run-make test serves as the end-to-end: It constructs an input
that is trying to subvert the hack and we are going to check that it
fails in the attempt).

(**) The compile-fail-fulldeps test serves as a more narrow test that
will be tested on all platforms.  It also attempts to subvert the
hack, testing that when you use `new_parser_from_tts`, the resulting
parser does not support reading embedded Idents.

src/test/auxiliary/macro_crate_test.rs

@@ -31,6 +31,7 @@ macro_rules! unexported_macro (() => (3i))
 #[plugin_registrar]
 pub fn plugin_registrar(reg: &mut Registry) {
     reg.register_macro("make_a_1", expand_make_a_1);
+    reg.register_macro("forged_ident", expand_forged_ident);
     reg.register_syntax_extension(
         token::intern("into_foo"),
         ItemModifier(expand_into_foo));
@@ -52,4 +53,29 @@ fn expand_into_foo(cx: &mut ExtCtxt, sp: Span, attr: Gc<MetaItem>, it: Gc<Item>)
     }
 }
 
+fn expand_forged_ident(cx: &mut ExtCtxt, sp: Span, tts: &[TokenTree]) -> Box<MacResult> {
+    use syntax::ext::quote::rt::*;
+
+    if !tts.is_empty() {
+        cx.span_fatal(sp, "forged_ident takes no arguments");
+    }
+
+    // Most of this is modelled after the expansion of the `quote_expr!`
+    // macro ...
+    let parse_sess = cx.parse_sess();
+    let cfg = cx.cfg();
+
+    // ... except this is where we inject a forged identifier,
+    // and deliberately do not call `cx.parse_tts_with_hygiene`
+    // (because we are testing that this will be *rejected*
+    //  by the default parser).
+
+    let expr = {
+        let tt = cx.parse_tts("\x00name_2,ctxt_0\x00".to_string());
+        let mut parser = new_parser_from_tts(parse_sess, cfg, tt);
+        parser.parse_expr()
+    };
+    MacExpr::new(expr)
+}
+
 pub fn foo() {}

src/test/compile-fail-fulldeps/macro-crate-cannot-read-embedded-ident.rs

@@ -0,0 +1,31 @@
+// Copyright 2014 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+// aux-build:macro_crate_test.rs
+// ignore-stage1
+// ignore-android
+// error-pattern: unknown start of token: \x00
+
+// Issue #15750 and #15962 : this test is checking that the standard
+// parser rejects embedded idents.  pnkfelix did not want to attempt
+// to make a test file that itself used the embedded ident input form,
+// since he worrid that would be difficult to work with in many text
+// editors, so instead he made a macro that expands into the embedded
+// ident form.
+
+#![feature(phase)]
+
+#[phase(plugin)]
+extern crate macro_crate_test;
+
+fn main() {
+    let x = 0;
+    assert_eq!(3, forged_ident!());
+}

src/test/run-make/cannot-read-embedded-idents/Makefile

@@ -0,0 +1,28 @@
+-include ../tools.mk
+
+# Issue #15750, #15962 : This test ensures that our special embedded
+# ident syntax hack is not treated as legitimate input by the lexer in
+# normal mode.
+#
+# It is modelled after the `unicode-input/` test, since we need to
+# create files with syntax that can trip up normal text editting tools
+# (namely text with embedded nul-bytes).
+
+# This test attempts to run rustc itself from the compiled binary; but
+# that means that you need to set the LD_LIBRARY_PATH for rustc itself
+# while running create_and_compile, and that won't work for stage1.
+
+# FIXME ignore windows
+ifndef IS_WINDOWS
+ifeq ($(RUST_BUILD_STAGE),1)
+DOTEST=
+else
+DOTEST=dotest
+endif
+endif
+
+all: $(DOTEST)
+
+dotest:
+	$(RUSTC) create_and_compile.rs
+	$(call RUN,create_and_compile)  "$(RUSTC)" "$(TMPDIR)"

src/test/run-make/cannot-read-embedded-idents/create_and_compile.rs

@@ -0,0 +1,44 @@
+// Copyright 2014 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+use std::os;
+use std::io::{File, Command};
+
+// creates broken.rs, which has the Ident \x00name_0,ctxt_0\x00
+// embedded within it, and then attempts to compile broken.rs with the
+// provided `rustc`
+
+fn main() {
+    let args = os::args();
+    let rustc = args[1].as_slice();
+    let tmpdir = Path::new(args[2].as_slice());
+
+    let main_file = tmpdir.join("broken.rs");
+    let _ = File::create(&main_file).unwrap()
+        .write_str("pub fn main() {
+                   let \x00name_0,ctxt_0\x00 = 3i;
+                   println!(\"{}\", \x00name_0,ctxt_0\x00);
+        }");
+
+    // rustc is passed to us with --out-dir and -L etc., so we
+    // can't exec it directly
+    let result = Command::new("sh")
+        .arg("-c")
+        .arg(format!("{} {}",
+                     rustc,
+                     main_file.as_str()
+                     .unwrap()).as_slice())
+        .output().unwrap();
+    let err = String::from_utf8_lossy(result.error.as_slice());
+
+    // positive test so that this test will be updated when the
+    // compiler changes.
+    assert!(err.as_slice().contains("unknown start of token"))
+}