Auto merge of #958 - jelford:cache_downloaded_files, r=brson

Store downloaded files in a persistent directory until installation

This PR should go some way to addressing #889 by caching downloaded
components in a persistent directory until they have been installed.

This way, if the download/install process is interrupted, the file
that made it can be re-used. This should help ease the pain of
intermittent connections.

src/download/src/lib.rs

@@ -47,21 +47,6 @@ pub fn download(url: &Url,
     Err("no working backends".into())
 }
 
-pub fn download_to_path(url: &Url,
-                        path: &Path,
-                        callback: Option<&Fn(Event) -> Result<()>>)
-                        -> Result<()> {
-    for &backend in BACKENDS {
-        match download_to_path_with_backend(backend, url, path, callback) {
-            Err(Error(ErrorKind::BackendUnavailable(_), _)) => (),
-            Err(e) => return Err(e),
-            Ok(()) => return Ok(()),
-        }
-    }
-
-    Err("no working backends".into())
-}
-
 pub fn download_with_backend(backend: Backend,
                              url: &Url,
                              callback: &Fn(Event) -> Result<()>)

src/rustup-dist/src/dist.rs

@@ -8,9 +8,12 @@ use manifest::Component;
 use manifest::Manifest as ManifestV2;
 use manifestation::{Manifestation, UpdateStatus, Changes};
 
-use std::path::Path;
+use std::path::{Path, PathBuf};
+use std::ops;
+use url::Url;
 use std::fmt;
 use std::env;
+use std::fs;
 
 use regex::Regex;
 use sha2::{Sha256, Digest};
@@ -482,9 +485,87 @@ pub fn download_and_check<'a>(url_str: &str,
 pub struct DownloadCfg<'a> {
     pub dist_root: &'a str,
     pub temp_cfg: &'a temp::Cfg,
+    pub download_dir: &'a PathBuf,
     pub notify_handler: &'a Fn(Notification),
 }
 
+
+pub struct File {
+    path: PathBuf,
+}
+
+impl ops::Deref for File {
+    type Target = Path;
+
+    fn deref(&self) -> &Path {
+        ops::Deref::deref(&self.path)
+    }
+}
+
+impl<'a> DownloadCfg<'a> {
+
+    pub fn download(&self, url: &Url, hash: &str, notify_handler: &'a Fn(Notification)) -> Result<File> {
+
+        try!(utils::ensure_dir_exists("Download Directory", &self.download_dir, &|n| notify_handler(n.into())));
+        let target_file = self.download_dir.join(Path::new(hash));
+
+        if target_file.exists() {
+            let mut hasher = Sha256::new();
+            use std::io::Read;
+            let mut downloaded = try!(fs::File::open(&target_file).chain_err(|| "opening already downloaded file"));
+            let mut buf = [0; 1024];
+            loop {
+                if let Ok(n) = downloaded.read(&mut buf) {
+                    if n == 0 { break; }
+                    hasher.input(&buf[..n]);
+                } else {
+                    break;
+                }
+            }
+            let cached_result = hasher.result_str();
+            if hash == cached_result {
+                notify_handler(Notification::FileAlreadyDownloaded);
+                notify_handler(Notification::ChecksumValid(&url.to_string()));
+                return Ok(File { path: target_file, });
+            } else {
+                notify_handler(Notification::CachedFileChecksumFailed);
+                try!(fs::remove_file(&target_file).chain_err(|| "cleaning up previous download"));
+            }
+        }
+
+        let mut hasher = Sha256::new();
+
+        try!(utils::download_file(&url,
+                                  &target_file,
+                                  Some(&mut hasher),
+                                  &|n| notify_handler(n.into())));
+
+        let actual_hash = hasher.result_str();
+
+        if hash != actual_hash {
+            // Incorrect hash
+            return Err(ErrorKind::ChecksumFailed {
+                url: url.to_string(),
+                expected: hash.to_string(),
+                calculated: actual_hash,
+            }.into());
+        } else {
+            notify_handler(Notification::ChecksumValid(&url.to_string()));
+            return Ok(File { path: target_file, })
+        }
+    }
+
+    pub fn clean(&self, hashes: &Vec<String>) -> Result<()> {
+        for hash in hashes.iter() {
+            let used_file = self.download_dir.join(hash);
+            if self.download_dir.join(&used_file).exists() {
+                try!(fs::remove_file(used_file).chain_err(|| "cleaning up cached downloads"));
+            }
+        }
+        Ok(())
+    }
+}
+
 pub fn download_hash(url: &str, cfg: DownloadCfg) -> Result<String> {
     let hash_url = try!(utils::parse_url(&(url.to_owned() + ".sha256")));
     let hash_file = try!(cfg.temp_cfg.new_file());
@@ -551,7 +632,7 @@ pub fn update_from_dist_<'a>(download: DownloadCfg<'a>,
         Ok(Some((m, hash))) => {
             return match try!(manifestation.update(&m,
                                                    changes,
-                                                   &download.temp_cfg,
+                                                   &download,
                                                    download.notify_handler.clone())) {
                 UpdateStatus::Unchanged => Ok(None),
                 UpdateStatus::Changed => Ok(Some(hash)),

src/rustup-dist/src/lib.rs

@@ -7,6 +7,7 @@ extern crate walkdir;
 extern crate toml;
 extern crate flate2;
 extern crate tar;
+extern crate url;
 #[macro_use]
 extern crate rustup_utils;
 #[macro_use]

src/rustup-dist/src/manifestation.rs

@@ -3,14 +3,13 @@
 
 use config::Config;
 use manifest::{Component, Manifest, TargetedPackage};
-use dist::{download_and_check, DownloadCfg, TargetTriple, DEFAULT_DIST_SERVER};
+use dist::{download_and_check, DownloadCfg, TargetTriple, DEFAULT_DIST_SERVER, File};
 use component::{Components, Transaction, TarGzPackage, Package};
 use temp;
 use errors::*;
 use notifications::*;
 use rustup_utils::utils;
 use prefix::InstallPrefix;
-use sha2::{Sha256, Digest};
 use std::path::Path;
 
 pub const DIST_MANIFEST: &'static str = "multirust-channel-manifest.toml";
@@ -73,10 +72,11 @@ impl Manifestation {
     pub fn update(&self,
                   new_manifest: &Manifest,
                   changes: Changes,
-                  temp_cfg: &temp::Cfg,
+                  download_cfg: &DownloadCfg,
                   notify_handler: &Fn(Notification)) -> Result<UpdateStatus> {
 
         // Some vars we're going to need a few times
+        let temp_cfg = download_cfg.temp_cfg;
         let prefix = self.installation.prefix();
         let ref rel_installed_manifest_path = prefix.rel_manifest_file(DIST_MANIFEST);
         let ref installed_manifest_path = prefix.path().join(rel_installed_manifest_path);
@@ -125,7 +125,8 @@ impl Manifestation {
         let altered = temp_cfg.dist_server != DEFAULT_DIST_SERVER;
 
         // Download component packages and validate hashes
-        let mut things_to_install: Vec<(Component, temp::File)> = Vec::new();
+        let mut things_to_install: Vec<(Component, File)> = Vec::new();
+        let mut things_downloaded: Vec<String> = Vec::new();
         for (component, url, hash) in components_urls_and_hashes {
 
             notify_handler(Notification::DownloadingComponent(&component.pkg,
@@ -137,32 +138,14 @@ impl Manifestation {
                 url
             };
 
-            // Download each package to temp file
-            let temp_file = try!(temp_cfg.new_file());
             let url_url = try!(utils::parse_url(&url));
 
-            let mut hasher = Sha256::new();
-            try!(utils::download_file(&url_url,
-                                      &temp_file,
-                                      Some(&mut hasher),
-                                      &|n| notify_handler(n.into())).chain_err(|| {
+            let dowloaded_file = try!(download_cfg.download(&url_url, &hash, &notify_handler).chain_err(|| {
                 ErrorKind::ComponentDownloadFailed(component.clone())
             }));
+            things_downloaded.push(hash);
 
-            let actual_hash = hasher.result_str();
-
-            if hash != actual_hash {
-                // Incorrect hash
-                return Err(ErrorKind::ChecksumFailed {
-                    url: url,
-                    expected: hash,
-                    calculated: actual_hash,
-                }.into());
-            } else {
-                notify_handler(Notification::ChecksumValid(&url));
-            }
-
-            things_to_install.push((component, temp_file));
+            things_to_install.push((component, dowloaded_file));
         }
 
         // Begin transaction
@@ -226,6 +209,8 @@ impl Manifestation {
         // End transaction
         tx.commit();
 
+        try!(download_cfg.clean(&things_downloaded));
+
         Ok(UpdateStatus::Changed)
     }
 
@@ -315,8 +300,11 @@ impl Manifestation {
                                                           &self.target_triple,
                                                           Some(&self.target_triple)));
 
+        use std::path::PathBuf;
+        let dld_dir = PathBuf::from("bogus");
         let dlcfg = DownloadCfg {
             dist_root: "bogus",
+            download_dir: &dld_dir,
             temp_cfg: temp_cfg,
             notify_handler: notify_handler
         };

src/rustup-dist/src/notifications.rs

@@ -18,6 +18,8 @@ pub enum Notification<'a> {
     NoUpdateHash(&'a Path),
     ChecksumValid(&'a str),
     SignatureValid(&'a str),
+    FileAlreadyDownloaded,
+    CachedFileChecksumFailed,
     RollingBack,
     ExtensionNotInstalled(&'a Component),
     NonFatalError(&'a Error),
@@ -48,6 +50,7 @@ impl<'a> Notification<'a> {
             Temp(ref n) => n.level(),
             Utils(ref n) => n.level(),
             ChecksumValid(_) | NoUpdateHash(_) |
+            FileAlreadyDownloaded |
             DownloadingLegacyManifest  => NotificationLevel::Verbose,
             Extracting(_, _) | SignatureValid(_)  |
             DownloadingComponent(_, _, _) |
@@ -56,7 +59,7 @@ impl<'a> Notification<'a> {
             ManifestChecksumFailedHack |
             RollingBack | DownloadingManifest(_) => NotificationLevel::Info,
             CantReadUpdateHash(_) | ExtensionNotInstalled(_) |
-            MissingInstalledComponent(_) => NotificationLevel::Warn,
+            MissingInstalledComponent(_) | CachedFileChecksumFailed => NotificationLevel::Warn,
             NonFatalError(_) => NotificationLevel::Error,
         }
     }
@@ -80,6 +83,8 @@ impl<'a> Display for Notification<'a> {
             NoUpdateHash(path) => write!(f, "no update hash at: '{}'", path.display()),
             ChecksumValid(_) => write!(f, "checksum passed"),
             SignatureValid(_) => write!(f, "signature valid"),
+            FileAlreadyDownloaded => write!(f, "reusing previously downloaded file"),
+            CachedFileChecksumFailed => write!(f, "bad checksum for cached download"),
             RollingBack => write!(f, "rolling back changes"),
             ExtensionNotInstalled(c) => {
                 write!(f, "extension '{}' was not installed", c.name())
@@ -106,4 +111,3 @@ impl<'a> Display for Notification<'a> {
         }
     }
 }
-