auto merge of #6938 : jbclements/rust/hygiene_fns_and_cleanup, r=jbcl…

…ements

I'm close to flipping the switch on hygiene for let-bound identifiers.  This commit adds a bunch of support functions for that change... but also a huge amount of cleanup in syntax.rc. The most interesting of these are

- the use of TLS for the interners everywhere.  We had already breached the "no-global-state" dam by using TLS for encoding, and it saves a lot of code just to use it everywhere.
Also, there were many places where two or more interners were passed in attached to different structures, and the danger of having those diverge seemed greater that the danger of having a single one get big and heavy. If the interner size proves to be a problem, it should be quite simple to add a "parameterize"-like dynamic binding form--because we don't have interesting continuation operations (or tail calling, mostly) this should just be a case of a mutation followed by another later mutation. Again, this is only if the interner gets too big.

- I renamed the "repr" field of the identifier to "name". I can see the case for "repr" when there's only one field in the structure, but that's no longer the case; there's now a name and a context (both are uints).

- the interner now just maps between strings and uints, rather than between idents and uints. The former state made perfect sense when identifiers didn't have syntax contexts, but that's no longer the case.

I've run this patch against a fairly recent incoming, and it appears to pass all tests. Let's see if it can be merged....

src/librustc/back/link.rs

@@ -734,7 +734,7 @@ pub fn mangle_internal_name_by_path(ccx: @CrateContext, path: path) -> ~str {
 }
 
 pub fn mangle_internal_name_by_seq(ccx: @CrateContext, flav: &str) -> ~str {
-    return fmt!("%s_%u", flav, (ccx.names)(flav).repr);
+    return fmt!("%s_%u", flav, (ccx.names)(flav).name);
 }
 
 

src/librustc/driver/driver.rs

@@ -39,6 +39,7 @@ use syntax::attr;
 use syntax::codemap;
 use syntax::diagnostic;
 use syntax::parse;
+use syntax::parse::token;
 use syntax::print::{pp, pprust};
 use syntax;
 
@@ -230,7 +231,7 @@ pub fn compile_rest(sess: Session,
                               sess.filesearch,
                               session::sess_os_to_meta_os(sess.targ_cfg.os),
                               sess.opts.is_static,
-                              sess.parse_sess.interner));
+                              token::get_ident_interner()));
 
     let lang_items = time(time_passes, ~"language item collection", ||
                           middle::lang_items::collect_language_items(crate, sess));
@@ -455,7 +456,7 @@ pub fn pretty_print_input(sess: Session, cfg: ast::crate_cfg, input: &input,
     let is_expanded = upto != cu_parse;
     let src = sess.codemap.get_filemap(source_name(input)).src;
     do io::with_str_reader(*src) |rdr| {
-        pprust::print_crate(sess.codemap, sess.parse_sess.interner,
+        pprust::print_crate(sess.codemap, token::get_ident_interner(),
                             sess.span_diagnostic, crate.unwrap(),
                             source_name(input),
                             rdr, io::stdout(), ann, is_expanded);
@@ -754,7 +755,7 @@ pub fn build_session_(sopts: @session::options,
     let target_cfg = build_target_config(sopts, demitter);
     let p_s = parse::new_parse_sess_special_handler(span_diagnostic_handler,
                                                     cm);
-    let cstore = @mut cstore::mk_cstore(p_s.interner);
+    let cstore = @mut cstore::mk_cstore(token::get_ident_interner());
     let filesearch = filesearch::mk_filesearch(
         &sopts.maybe_sysroot,
         sopts.target_triple,
@@ -963,7 +964,7 @@ pub fn early_error(emitter: diagnostic::Emitter, msg: ~str) -> ! {
 
 pub fn list_metadata(sess: Session, path: &Path, out: @io::Writer) {
     metadata::loader::list_file_metadata(
-        sess.parse_sess.interner,
+        token::get_ident_interner(),
         session::sess_os_to_meta_os(sess.targ_cfg.os), path, out);
 }
 

src/librustc/driver/session.rs

@@ -26,6 +26,7 @@ use syntax::diagnostic;
 use syntax::parse::ParseSess;
 use syntax::{ast, codemap};
 use syntax::abi;
+use syntax::parse::token;
 use syntax;
 
 use core::hashmap::HashMap;
@@ -293,14 +294,19 @@ impl Session_ {
         self.opts.optimize == No && !self.debugging_opt(no_debug_borrows)
     }
 
+    // pointless function, now...
     pub fn str_of(@self, id: ast::ident) -> @~str {
-        self.parse_sess.interner.get(id)
+        token::ident_to_str(&id)
     }
+
+    // pointless function, now...
     pub fn ident_of(@self, st: &str) -> ast::ident {
-        self.parse_sess.interner.intern(st)
+        token::str_to_ident(st)
     }
+
+    // pointless function, now...
     pub fn intr(@self) -> @syntax::parse::token::ident_interner {
-        self.parse_sess.interner
+        token::get_ident_interner()
     }
 }
 

src/librustc/front/test.rs

@@ -22,6 +22,7 @@ use syntax::codemap::{dummy_sp, span, ExpandedFrom, CallInfo, NameAndSpan};
 use syntax::codemap;
 use syntax::ext::base::ExtCtxt;
 use syntax::fold;
+use syntax::parse::token;
 use syntax::print::pprust;
 use syntax::{ast, ast_util};
 
@@ -143,7 +144,7 @@ fn fold_item(cx: @mut TestCtxt, i: @ast::item, fld: @fold::ast_fold)
           -> Option<@ast::item> {
     cx.path.push(i.ident);
     debug!("current path: %s",
-           ast_util::path_name_i(copy cx.path, cx.sess.parse_sess.interner));
+           ast_util::path_name_i(copy cx.path));
 
     if is_test_fn(cx, i) || is_bench_fn(i) {
         match i.node {
@@ -411,13 +412,10 @@ fn mk_test_desc_and_fn_rec(cx: &TestCtxt, test: &Test) -> @ast::expr {
 
     let ext_cx = cx.ext_cx;
 
-    debug!("encoding %s", ast_util::path_name_i(path,
-                                                cx.sess.parse_sess.interner));
+    debug!("encoding %s", ast_util::path_name_i(path));
 
     let name_lit: ast::lit =
-        nospan(ast::lit_str(@ast_util::path_name_i(
-            path,
-            cx.sess.parse_sess.interner)));
+        nospan(ast::lit_str(@ast_util::path_name_i(path)));
 
     let name_expr = @ast::expr {
           id: cx.sess.next_node_id(),

src/librustc/metadata/creader.rs

@@ -22,6 +22,7 @@ use core::vec;
 use syntax::attr;
 use syntax::codemap::{span, dummy_sp};
 use syntax::diagnostic::span_handler;
+use syntax::parse::token;
 use syntax::parse::token::ident_interner;
 use syntax::visit;
 use syntax::ast;
@@ -176,7 +177,7 @@ fn visit_item(e: @mut Env, i: @ast::item) {
                             }
                             nn
                         }
-                        None => e.intr.get(i.ident)
+                        None => token::ident_to_str(&i.ident)
                     };
                 if attr::find_attrs_by_name(i.attrs, "nolink").is_empty() {
                     already_added =
@@ -235,7 +236,7 @@ fn resolve_crate(e: @mut Env,
                  hash: @~str,
                  span: span)
               -> ast::crate_num {
-    let metas = metas_with_ident(@/*bad*/copy *e.intr.get(ident), metas);
+    let metas = metas_with_ident(token::ident_to_str(&ident), metas);
 
     match existing_match(e, metas, hash) {
       None => {
@@ -276,7 +277,7 @@ fn resolve_crate(e: @mut Env,
             match attr::last_meta_item_value_str_by_name(load_ctxt.metas,
                                                          "name") {
                 Some(v) => v,
-                None => e.intr.get(ident),
+                None => token::ident_to_str(&ident),
             };
         let cmeta = @cstore::crate_metadata {
             name: cname,
@@ -305,10 +306,11 @@ fn resolve_crate_deps(e: @mut Env, cdata: @~[u8]) -> cstore::cnum_map {
     for decoder::get_crate_deps(e.intr, cdata).each |dep| {
         let extrn_cnum = dep.cnum;
         let cname = dep.name;
+        let cname_str = token::ident_to_str(&dep.name);
         let cmetas = metas_with(dep.vers, @~"vers", ~[]);
         debug!("resolving dep crate %s ver: %s hash: %s",
-               *e.intr.get(dep.name), *dep.vers, *dep.hash);
-        match existing_match(e, metas_with_ident(e.intr.get(cname),
+               *cname_str, *dep.vers, *dep.hash);
+        match existing_match(e, metas_with_ident(cname_str,
                                                  copy cmetas),
                              dep.hash) {
           Some(local_cnum) => {

src/librustc/metadata/decoder.rs

@@ -40,6 +40,7 @@ use syntax::parse::token::{StringRef, ident_interner, special_idents};
 use syntax::print::pprust;
 use syntax::{ast, ast_util};
 use syntax::codemap;
+use syntax::parse::token;
 
 type cmd = @crate_metadata;
 
@@ -297,10 +298,10 @@ fn item_path(intr: @ident_interner, item_doc: ebml::Doc) -> ast_map::path {
     for reader::docs(path_doc) |tag, elt_doc| {
         if tag == tag_path_elt_mod {
             let str = reader::doc_as_str(elt_doc);
-            result.push(ast_map::path_mod(intr.intern(str)));
+            result.push(ast_map::path_mod(token::str_to_ident(str)));
         } else if tag == tag_path_elt_name {
             let str = reader::doc_as_str(elt_doc);
-            result.push(ast_map::path_name(intr.intern(str)));
+            result.push(ast_map::path_name(token::str_to_ident(str)));
         } else {
             // ignore tag_path_len element
         }
@@ -314,8 +315,8 @@ fn item_name(intr: @ident_interner, item: ebml::Doc) -> ast::ident {
     do reader::with_doc_data(name) |data| {
         let string = str::from_bytes_slice(data);
         match intr.find_equiv(&StringRef(string)) {
-            None => intr.intern(string),
-            Some(val) => val,
+            None => token::str_to_ident(string),
+            Some(val) => ast::new_ident(val),
         }
     }
 }
@@ -843,7 +844,7 @@ pub fn get_type_name_if_impl(intr: @ident_interner,
     }
 
     for reader::tagged_docs(item, tag_item_impl_type_basename) |doc| {
-        return Some(intr.intern(str::from_bytes(reader::doc_data(doc))));
+        return Some(token::str_to_ident(str::from_bytes(reader::doc_data(doc))));
     }
 
     return None;
@@ -1095,7 +1096,7 @@ pub fn get_crate_deps(intr: @ident_interner, data: @~[u8]) -> ~[crate_dep] {
     }
     for reader::tagged_docs(depsdoc, tag_crate_dep) |depdoc| {
         deps.push(crate_dep {cnum: crate_num,
-                  name: intr.intern(docstr(depdoc, tag_crate_dep_name)),
+                  name: token::str_to_ident(docstr(depdoc, tag_crate_dep_name)),
                   vers: @docstr(depdoc, tag_crate_dep_vers),
                   hash: @docstr(depdoc, tag_crate_dep_hash)});
         crate_num += 1;
@@ -1109,7 +1110,7 @@ fn list_crate_deps(intr: @ident_interner, data: @~[u8], out: @io::Writer) {
     for get_crate_deps(intr, data).each |dep| {
         out.write_str(
             fmt!("%d %s-%s-%s\n",
-                 dep.cnum, *intr.get(dep.name), *dep.hash, *dep.vers));
+                 dep.cnum, *token::ident_to_str(&dep.name), *dep.hash, *dep.vers));
     }
 
     out.write_str("\n");

src/librustc/metadata/encoder.rs

@@ -43,6 +43,7 @@ use syntax::opt_vec::OptVec;
 use syntax::opt_vec;
 use syntax::parse::token::special_idents;
 use syntax::{ast_util, visit};
+use syntax::parse::token;
 use syntax;
 use writer = extra::ebml::writer;
 
@@ -141,8 +142,7 @@ fn add_to_index(ecx: @EncodeContext,
     full_path.push(name);
     index.push(
         entry {
-            val: ast_util::path_name_i(full_path,
-                                       ecx.tcx.sess.parse_sess.interner),
+            val: ast_util::path_name_i(full_path),
             pos: ebml_w.writer.tell()
         });
 }
@@ -485,8 +485,7 @@ fn encode_info_for_mod(ecx: @EncodeContext,
                         (%?/%?)",
                         *ecx.tcx.sess.str_of(ident),
                         did,
-                        ast_map::node_id_to_str(ecx.tcx.items, did, ecx.tcx
-                                                .sess.parse_sess.interner));
+                        ast_map::node_id_to_str(ecx.tcx.items, did, token::get_ident_interner()));
 
                 ebml_w.start_tag(tag_mod_impl);
                 ebml_w.wr_str(def_to_str(local_def(did)));
@@ -1055,7 +1054,7 @@ fn encode_info_for_item(ecx: @EncodeContext,
                         tcx.sess.span_unimpl(
                             item.span,
                             fmt!("Method %s is both provided and static",
-                                 *tcx.sess.intr().get(method_ty.ident)));
+                                 *token::ident_to_str(&method_ty.ident)));
                     }
                     encode_type_param_bounds(ebml_w, ecx,
                                              &m.generics.ty_params);

src/librustc/metadata/loader.rs

@@ -19,6 +19,7 @@ use metadata::filesearch::FileSearch;
 use metadata::filesearch;
 use syntax::codemap::span;
 use syntax::diagnostic::span_handler;
+use syntax::parse::token;
 use syntax::parse::token::ident_interner;
 use syntax::print::pprust;
 use syntax::{ast, attr};
@@ -59,7 +60,7 @@ pub fn load_library_crate(cx: &Context) -> (~str, @~[u8]) {
       None => {
         cx.diag.span_fatal(
             cx.span, fmt!("can't find crate for `%s`",
-                          *cx.intr.get(cx.ident)));
+                          *token::ident_to_str(&cx.ident)));
       }
     }
 }

src/librustc/middle/astencode.rs

@@ -41,6 +41,7 @@ use syntax::codemap::span;
 use syntax::codemap;
 use syntax::fold::*;
 use syntax::fold;
+use syntax::parse::token;
 use syntax;
 use writer = extra::ebml::writer;
 
@@ -86,7 +87,7 @@ pub fn encode_inlined_item(ecx: @e::EncodeContext,
                            ii: ast::inlined_item,
                            maps: Maps) {
     debug!("> Encoding inlined item: %s::%s (%u)",
-           ast_map::path_to_str(path, ecx.tcx.sess.parse_sess.interner),
+           ast_map::path_to_str(path, token::get_ident_interner()),
            *ecx.tcx.sess.str_of(ii.ident()),
            ebml_w.writer.tell());
 
@@ -99,7 +100,7 @@ pub fn encode_inlined_item(ecx: @e::EncodeContext,
     ebml_w.end_tag();
 
     debug!("< Encoded inlined fn: %s::%s (%u)",
-           ast_map::path_to_str(path, ecx.tcx.sess.parse_sess.interner),
+           ast_map::path_to_str(path, token::get_ident_interner()),
            *ecx.tcx.sess.str_of(ii.ident()),
            ebml_w.writer.tell());
 }
@@ -119,7 +120,7 @@ pub fn decode_inlined_item(cdata: @cstore::crate_metadata,
       None => None,
       Some(ast_doc) => {
         debug!("> Decoding inlined fn: %s::?",
-               ast_map::path_to_str(path, tcx.sess.parse_sess.interner));
+               ast_map::path_to_str(path, token::get_ident_interner()));
         let mut ast_dsr = reader::Decoder(ast_doc);
         let from_id_range = Decodable::decode(&mut ast_dsr);
         let to_id_range = reserve_id_range(dcx.tcx.sess, from_id_range);
@@ -132,7 +133,7 @@ pub fn decode_inlined_item(cdata: @cstore::crate_metadata,
         let ii = renumber_ast(xcx, raw_ii);
         debug!("Fn named: %s", *tcx.sess.str_of(ii.ident()));
         debug!("< Decoded inlined fn: %s::%s",
-               ast_map::path_to_str(path, tcx.sess.parse_sess.interner),
+               ast_map::path_to_str(path, token::get_ident_interner()),
                *tcx.sess.str_of(ii.ident()));
         ast_map::map_decoded_item(tcx.sess.diagnostic(),
                                   dcx.tcx.items, path, &ii);
@@ -1167,7 +1168,7 @@ impl fake_ext_ctxt for fake_session {
         }
     }
     fn ident_of(&self, st: &str) -> ast::ident {
-        self.interner.intern(st)
+        token::str_to_ident(st)
     }
 }
 
@@ -1236,9 +1237,9 @@ fn test_simplification() {
     match (item_out, item_exp) {
       (ast::ii_item(item_out), ast::ii_item(item_exp)) => {
         assert!(pprust::item_to_str(item_out,
-                                         ext_cx.parse_sess().interner)
+                                    token::get_ident_interner())
                      == pprust::item_to_str(item_exp,
-                                            ext_cx.parse_sess().interner));
+                                            token::get_ident_interner()));
       }
       _ => fail!()
     }

src/librustc/middle/borrowck/mod.rs

@@ -30,6 +30,7 @@ use syntax::ast;
 use syntax::ast_map;
 use syntax::visit;
 use syntax::codemap::span;
+use syntax::parse::token;
 
 macro_rules! if_ok(
     ($inp: expr) => (
@@ -710,8 +711,8 @@ impl BorrowckCtxt {
         match *loan_path {
             LpVar(id) => {
                 match self.tcx.items.find(&id) {
-                    Some(&ast_map::node_local(ident)) => {
-                        str::push_str(out, *self.tcx.sess.intr().get(ident));
+                    Some(&ast_map::node_local(ref ident)) => {
+                        str::push_str(out, *token::ident_to_str(ident));
                     }
                     r => {
                         self.tcx.sess.bug(
@@ -724,9 +725,9 @@ impl BorrowckCtxt {
             LpExtend(lp_base, _, LpInterior(mc::InteriorField(fname))) => {
                 self.append_loan_path_to_str_from_interior(lp_base, out);
                 match fname {
-                    mc::NamedField(fname) => {
+                    mc::NamedField(ref fname) => {
                         str::push_char(out, '.');
-                        str::push_str(out, *self.tcx.sess.intr().get(fname));
+                        str::push_str(out, *token::ident_to_str(fname));
                     }
                     mc::PositionalField(idx) => {
                         str::push_char(out, '#'); // invent a notation here