Auto merge of #56869 - GuillaumeGomez:index-size-reduction, r=QuietMi…

…sdreavus Reduce search-index.js size Coming from: ``` 1735683 Dec 16 01:35 build/x86_64-apple-darwin/doc/search-index.js ``` to: ``` 727755 Dec 16 01:51 build/x86_64-apple-darwin/doc/search-index.js ``` r? @QuietMisdreavus
rust-lang · Jan 17, 2019 · 956dba4 · 956dba4
2 parents c40b977 + d405606
commit 956dba4
Show file tree

Hide file tree

Showing 4 changed files with 101 additions and 28 deletions.
diff --git a/Cargo.lock b/Cargo.lock
@@ -1332,7 +1332,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 
 [[package]]
 name = "minifier"
-version = "0.0.20"
+version = "0.0.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
  "macro-utils 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -2692,7 +2692,7 @@ dependencies = [
 name = "rustdoc"
 version = "0.0.0"
 dependencies = [
- "minifier 0.0.20 (registry+https://github.com/rust-lang/crates.io-index)",
+ "minifier 0.0.26 (registry+https://github.com/rust-lang/crates.io-index)",
  "parking_lot 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)",
  "pulldown-cmark 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "tempfile 3.0.5 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -3510,7 +3510,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 "checksum memchr 2.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0a3eb002f0535929f1199681417029ebea04aadc0c7a4224b46be99c7f5d6a16"
 "checksum memmap 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e2ffa2c986de11a9df78620c01eeaaf27d94d3ff02bf81bfcca953102dd0c6ff"
 "checksum memoffset 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0f9dc261e2b62d7a622bf416ea3c5245cdd5d9a7fcc428c0d06804dfce1775b3"
-"checksum minifier 0.0.20 (registry+https://github.com/rust-lang/crates.io-index)" = "96c269bb45c39b333392b2b18ad71760b34ac65666591386b0e959ed58b3f474"
+"checksum minifier 0.0.26 (registry+https://github.com/rust-lang/crates.io-index)" = "f299df45afd73332044ea9f717c816a84fc90c8b631409abf339ba93642a7985"
 "checksum miniz-sys 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)" = "0300eafb20369952951699b68243ab4334f4b10a88f411c221d444b36c40e649"
 "checksum miniz_oxide 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5ad30a47319c16cde58d0314f5d98202a80c9083b5f61178457403dfb14e509c"
 "checksum miniz_oxide_c_api 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "28edaef377517fd9fe3e085c37d892ce7acd1fbeab9239c5a36eec352d8a8b7e"

diff --git a/src/librustdoc/Cargo.toml b/src/librustdoc/Cargo.toml
@@ -9,6 +9,6 @@ path = "lib.rs"
 
 [dependencies]
 pulldown-cmark = { version = "0.1.2", default-features = false }
-minifier = "0.0.20"
+minifier = "0.0.26"
 tempfile = "3"
 parking_lot = "0.6.4"
diff --git a/src/librustdoc/html/render.rs b/src/librustdoc/html/render.rs
@@ -741,8 +741,8 @@ fn build_index(krate: &clean::Crate, cache: &mut Cache) -> String {
 
     let mut crate_data = BTreeMap::new();
     crate_data.insert("doc".to_owned(), Json::String(crate_doc));
-    crate_data.insert("items".to_owned(), Json::Array(crate_items));
-    crate_data.insert("paths".to_owned(), Json::Array(crate_paths));
+    crate_data.insert("i".to_owned(), Json::Array(crate_items));
+    crate_data.insert("p".to_owned(), Json::Array(crate_paths));
 
     // Collect the index into a string
     format!("searchIndex[{}] = {};",
@@ -914,12 +914,44 @@ themePicker.onblur = handleThemeButtonsBlur;
     write(cx.dst.join("COPYRIGHT.txt"),
           static_files::COPYRIGHT)?;
 
-    fn collect(path: &Path, krate: &str, key: &str) -> io::Result<(Vec<String>, Vec<String>)> {
+    fn collect(
+        path: &Path,
+        krate: &str,
+        key: &str,
+        for_search_index: bool,
+    ) -> io::Result<(Vec<String>, Vec<String>, Vec<String>)> {
+        use minifier::js;
+
         let mut ret = Vec::new();
         let mut krates = Vec::new();
+        let mut variables = Vec::new();
+
+        let mut krate = krate.to_owned();
+
         if path.exists() {
             for line in BufReader::new(File::open(path)?).lines() {
                 let line = line?;
+                if for_search_index && line.starts_with("var r_") {
+                    variables.push(line.clone());
+                    // We need to check if the crate name has been put into a variable as well.
+                    let tokens = js::simple_minify(&line).apply(js::clean_tokens);
+                    let mut pos = 0;
+                    while pos < tokens.len() {
+                        if let Some((var_pos, Some(value_pos))) =
+                                js::get_variable_name_and_value_positions(&tokens, pos) {
+                            if let Some(s) = tokens.0[value_pos].get_string() {
+                                if &s[1..s.len() - 1] == krate {
+                                    if let Some(var) = tokens[var_pos].get_other() {
+                                        krate = var.to_owned();
+                                        break
+                                    }
+                                }
+                            }
+                        }
+                        pos += 1;
+                    }
+                    continue;
+                }
                 if !line.starts_with(key) {
                     continue;
                 }
@@ -933,7 +965,7 @@ themePicker.onblur = handleThemeButtonsBlur;
                                                  .unwrap_or_else(|| String::new()));
             }
         }
-        Ok((ret, krates))
+        Ok((ret, krates, variables))
     }
 
     fn show_item(item: &IndexItem, krate: &str) -> String {
@@ -948,7 +980,7 @@ themePicker.onblur = handleThemeButtonsBlur;
 
     let dst = cx.dst.join("aliases.js");
     {
-        let (mut all_aliases, _) = try_err!(collect(&dst, &krate.name, "ALIASES"), &dst);
+        let (mut all_aliases, _, _) = try_err!(collect(&dst, &krate.name, "ALIASES", false), &dst);
         let mut w = try_err!(File::create(&dst), &dst);
         let mut output = String::with_capacity(100);
         for (alias, items) in &cache.aliases {
@@ -1035,7 +1067,9 @@ themePicker.onblur = handleThemeButtonsBlur;
         }
 
         let dst = cx.dst.join("source-files.js");
-        let (mut all_sources, _krates) = try_err!(collect(&dst, &krate.name, "sourcesIndex"), &dst);
+        let (mut all_sources, _krates, _) = try_err!(collect(&dst, &krate.name, "sourcesIndex",
+                                                             false),
+                                                     &dst);
         all_sources.push(format!("sourcesIndex[\"{}\"] = {};",
                                  &krate.name,
                                  hierarchy.to_json_string()));
@@ -1049,20 +1083,22 @@ themePicker.onblur = handleThemeButtonsBlur;
 
     // Update the search index
     let dst = cx.dst.join("search-index.js");
-    let (mut all_indexes, mut krates) = try_err!(collect(&dst, &krate.name, "searchIndex"), &dst);
+    let (mut all_indexes, mut krates, variables) = try_err!(collect(&dst,
+                                                                    &krate.name,
+                                                                    "searchIndex",
+                                                                    true), &dst);
     all_indexes.push(search_index);
 
     // Sort the indexes by crate so the file will be generated identically even
     // with rustdoc running in parallel.
     all_indexes.sort();
     let mut w = try_err!(File::create(&dst), &dst);
-    try_err!(writeln!(&mut w, "var N = null;var searchIndex = {{}};"), &dst);
-    for index in &all_indexes {
-        try_err!(write_minify_replacer(&mut w, &*index, options.enable_minification,
-                                       &[(minifier::js::Keyword::Null, "N")]),
-                 &dst);
-    }
-    try_err!(writeln!(&mut w, "initSearch(searchIndex);addSearchOptions(searchIndex);"), &dst);
+    try_err!(writeln!(&mut w, "var N=null,E=\"\",T=\"t\",U=\"u\",searchIndex={{}};"), &dst);
+    try_err!(write_minify_replacer(&mut w,
+                                   &format!("{}\n{}", variables.join(""), all_indexes.join("\n")),
+                                   options.enable_minification),
+             &dst);
+    try_err!(write!(&mut w, "initSearch(searchIndex);addSearchOptions(searchIndex);"), &dst);
 
     if options.enable_index_page {
         if let Some(index_page) = options.index_page.clone() {
@@ -1161,8 +1197,9 @@ themePicker.onblur = handleThemeButtonsBlur;
                             remote_item_type.css_class(),
                             remote_path[remote_path.len() - 1]));
 
-        let (mut all_implementors, _) = try_err!(collect(&mydst, &krate.name, "implementors"),
-                                                 &mydst);
+        let (mut all_implementors, _, _) = try_err!(collect(&mydst, &krate.name, "implementors",
+                                                            false),
+                                                    &mydst);
         all_implementors.push(implementors);
         // Sort the implementors by crate so the file will be generated
         // identically even with rustdoc running in parallel.
@@ -1216,14 +1253,50 @@ fn write_minify(dst: PathBuf, contents: &str, enable_minification: bool) -> Resu
     }
 }
 
-fn write_minify_replacer<W: Write>(dst: &mut W,
-                                   contents: &str,
-                                   enable_minification: bool,
-                                   keywords_to_replace: &[(minifier::js::Keyword, &str)])
-                                   -> io::Result<()> {
+fn write_minify_replacer<W: Write>(
+    dst: &mut W,
+    contents: &str,
+    enable_minification: bool,
+) -> io::Result<()> {
+    use minifier::js::{Keyword, ReservedChar, Token};
+
     if enable_minification {
         writeln!(dst, "{}",
-                 minifier::js::minify_and_replace_keywords(contents, keywords_to_replace))
+                 minifier::js::simple_minify(contents)
+                              .apply(|f| {
+                                  // We keep backlines.
+                                  minifier::js::clean_tokens_except(f, |c| {
+                                      c.get_char() != Some(ReservedChar::Backline)
+                                  })
+                              })
+                              .apply(|f| {
+                                  minifier::js::replace_token_with(f, |t| {
+                                      match *t {
+                                          Token::Keyword(Keyword::Null) => Some(Token::Other("N")),
+                                          Token::String(s) => {
+                                              let s = &s[1..s.len() -1]; // The quotes are included
+                                              if s.is_empty() {
+                                                  Some(Token::Other("E"))
+                                              } else if s == "t" {
+                                                  Some(Token::Other("T"))
+                                              } else if s == "u" {
+                                                  Some(Token::Other("U"))
+                                              } else {
+                                                  None
+                                              }
+                                          }
+                                          _ => None,
+                                      }
+                                  })
+                              })
+                              .apply(|f| {
+                                  // We add a backline after the newly created variables.
+                                  minifier::js::aggregate_strings_with_separation(
+                                      f,
+                                      Token::Char(ReservedChar::Backline),
+                                  )
+                              })
+                              .to_string())
     } else {
         writeln!(dst, "{}", contents)
     }

diff --git a/src/librustdoc/html/static/main.js b/src/librustdoc/html/static/main.js
@@ -1555,10 +1555,10 @@ if (!DOMTokenList.prototype.remove) {
                 //              (String) description,
                 //              (Number | null) the parent path index to `paths`]
                 //              (Object | null) the type of the function (if any)
-                var items = rawSearchIndex[crate].items;
+                var items = rawSearchIndex[crate].i;
                 // an array of [(Number) item type,
                 //              (String) name]
-                var paths = rawSearchIndex[crate].paths;
+                var paths = rawSearchIndex[crate].p;
 
                 // convert `paths` into an object form
                 var len = paths.length;