diff --git a/components/common/Cargo.lock b/components/common/Cargo.lock index a6de118e6b..73f212715e 100644 --- a/components/common/Cargo.lock +++ b/components/common/Cargo.lock @@ -27,6 +27,11 @@ name = "bitflags" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "broadcast" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "byteorder" version = "0.5.1" @@ -76,10 +81,12 @@ dependencies = [ name = "habitat_depot_client" version = "0.5.0" dependencies = [ + "broadcast 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "habitat_core 0.5.0", "habitat_depot_core 0.5.0", "hyper 0.9.3 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", + "pbr 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-serialize 0.3.19 (registry+https://github.com/rust-lang/crates.io-index)", "url 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -271,6 +278,17 @@ dependencies = [ "openssl-sys 0.7.9 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "pbr" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "kernel32-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)", + "time 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "pkg-config" version = "0.3.8" diff --git a/components/depot-client/Cargo.lock b/components/depot-client/Cargo.lock index ea985f0034..fb32ff84d1 100644 --- a/components/depot-client/Cargo.lock +++ b/components/depot-client/Cargo.lock @@ -2,10 +2,12 @@ name = "habitat_depot_client" version = "0.5.0" dependencies = [ + "broadcast 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "habitat_core 0.5.0", "habitat_depot_core 0.5.0", "hyper 0.9.3 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "pbr 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-serialize 0.3.18 (registry+https://github.com/rust-lang/crates.io-index)", "url 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -23,6 +25,11 @@ name = "bitflags" version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "broadcast" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "byteorder" version = "0.5.1" @@ -147,7 +154,7 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "libarchive3-sys 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -155,13 +162,13 @@ name = "libarchive3-sys" version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", "pkg-config 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "libc" -version = "0.2.8" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -177,7 +184,7 @@ name = "libsodium-sys" version = "0.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", "pkg-config 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -186,7 +193,7 @@ name = "log" version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -199,7 +206,7 @@ name = "memchr" version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -215,7 +222,7 @@ name = "num_cpus" version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -226,7 +233,7 @@ dependencies = [ "bitflags 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", "gcc 0.3.25 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 0.1.15 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", "openssl-sys 0.7.6 (registry+https://github.com/rust-lang/crates.io-index)", "openssl-sys-extras 0.7.6 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -237,7 +244,7 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "gdi32-sys 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", "libressl-pnacl-sys 2.1.6 (registry+https://github.com/rust-lang/crates.io-index)", "pkg-config 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", "user32-sys 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", @@ -249,10 +256,21 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "gcc 0.3.25 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", "openssl-sys 0.7.6 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "pbr" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "kernel32-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", + "time 0.1.34 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "pkg-config" version = "0.3.8" @@ -271,7 +289,7 @@ name = "rand" version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -331,7 +349,7 @@ name = "sodiumoxide" version = "0.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", "libsodium-sys 0.0.10 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-serialize 0.3.18 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -359,7 +377,7 @@ version = "0.1.34" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "kernel32-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)", ] diff --git a/components/depot-client/Cargo.toml b/components/depot-client/Cargo.toml index 02c1c73914..3d0accb154 100644 --- a/components/depot-client/Cargo.toml +++ b/components/depot-client/Cargo.toml @@ -4,8 +4,10 @@ version = "0.5.0" authors = ["Adam Jacob ", "Jamie Winsor ", "Fletcher Nichol ", "Joshua Timberman ", "Dave Parfitt "] [dependencies] +broadcast = "*" hyper = "*" log = "*" +pbr = "*" rustc-serialize = "*" url = "*" diff --git a/components/depot-client/src/lib.rs b/components/depot-client/src/lib.rs index 2bd4273415..7926abf097 100644 --- a/components/depot-client/src/lib.rs +++ b/components/depot-client/src/lib.rs @@ -7,10 +7,12 @@ extern crate habitat_core as hcore; extern crate habitat_depot_core as depot_core; +extern crate broadcast; #[macro_use] extern crate hyper; #[macro_use] extern crate log; +extern crate pbr; extern crate rustc_serialize; extern crate url; @@ -19,15 +21,17 @@ pub mod error; pub use error::{Error, Result}; use std::fs::{self, File}; -use std::io::{Read, Write, BufWriter, Seek, SeekFrom}; +use std::io::{self, Read, Seek, SeekFrom}; use std::path::{Path, PathBuf}; +use broadcast::BroadcastWriter; use hcore::package::{PackageArchive, PackageIdent}; use hcore::env::http_proxy_unless_domain_exempted; use depot_core::{XFileName, data_object}; use hyper::client::{Client, Body}; use hyper::status::StatusCode; use hyper::Url; +use pbr::{ProgressBar, Units}; use rustc_serialize::json; /// Download a public key from a remote Depot to the given filepath. @@ -85,7 +89,7 @@ pub fn get_origin_keys(depot: &str, origin: &str, path: &str) -> Result<()> { /// * File cannot be created and written to pub fn fetch_package(depot: &str, package: &PackageIdent, store: &str) -> Result { let url = try!(Url::parse(&format!("{}/pkgs/{}/download", depot, package))); - match download(&package.name, url, store) { + match download(&package.to_string(), url, store) { Ok(file) => { let path = PathBuf::from(file); Ok(PackageArchive::new(path)) @@ -164,7 +168,6 @@ fn download(status: &str, url: Url, path: &str) -> Result { let client = try!(new_client(&url)); let mut res = try!(client.get(url).send()); debug!("Response: {:?}", res); - if res.status != hyper::status::StatusCode::Ok { return Err(Error::HTTP(res.status)); } @@ -173,50 +176,16 @@ fn download(status: &str, url: Url, path: &str) -> Result { Some(filename) => format!("{}", filename), None => return Err(Error::NoXFilename), }; - let length = res.headers - .get::() - .map_or("Unknown".to_string(), |v| format!("{}", v)); - // Here is a moment where you can really like Rust. We create - // a file, wrap it in a BufWriter - which understands how to - // safely batch writes into large buffer sizes on the heap, - // saving us the tax of frequent system calls. We then do - // what we would do in C - create a buffer of bytes, then - // read into that buffer, and write out to the other side. - // - // Under the hood, Hyper uses the "BufReader" to implement - // reading the request body - so on both ends, we are getting - // free buffering on the heap, using our stack buffer just to - // shuttle back and forth. This is essentially with the "tee" - // function does in the stdlib, but with error handling that - // a generic "tee" can't really provide easily. - // - // What you can't see is this - the compiler helped with - // making sure all the edge cases of the pattern were covered, - // and even though its a trivial case, it was pretty great. let tempfile = format!("{}/{}.tmp", path, file_name); let finalfile = format!("{}/{}", path, file_name); - let f = try!(File::create(&tempfile)); - let mut writer = BufWriter::new(&f); - let mut written: i64 = 0; - let mut buf = [0u8; 100000]; // Our byte buffer - loop { - let len = try!(res.read(&mut buf)); // Raise IO errors - match len { - 0 => { - // 0 == EOF, so stop writing and finish progress - progress(status, written, &length, true); - break; - } - _ => { - // Write the buffer to the BufWriter on the Heap - let bytes_written = try!(writer.write(&buf[0..len])); - if bytes_written == 0 { - return Err(Error::WriteSyncFailed); - } - written = written + (bytes_written as i64); - progress(status, written, &length, false); - } - }; + let size: u64 = res.headers.get::().map_or(0, |v| **v); + { + let mut f = try!(File::create(&tempfile)); + let mut pb = ProgressBar::new(size); + pb.set_units(Units::Bytes); + let mut writer = BroadcastWriter::new(&mut f, &mut pb); + println!("Downloading {}", &status); + try!(io::copy(&mut res, &mut writer)); } try!(fs::rename(&tempfile, &finalfile)); Ok(finalfile) @@ -236,47 +205,12 @@ fn upload(url: Url, file: &mut File) -> Result<()> { } } -fn progress(status: &str, written: i64, length: &str, finished: bool) { - let progress = format!("{} {}/{}", status, written, length); - print!("{}", from_char(progress.len(), '\x08')); - if finished { - println!("{}", progress); - } else { - print!("{}", progress); - } -} - -fn from_char(length: usize, ch: char) -> String { - if length == 0 { - return String::new(); - } - - let mut buf = String::new(); - buf.push(ch); - let size = buf.len() * length; - buf.reserve(size); - for _ in 1..length { - buf.push(ch) - } - buf -} - fn new_client(url: &Url) -> Result { match try!(http_proxy_unless_domain_exempted(url.host_str().unwrap_or(""))) { Some((proxy_host, proxy_port)) => { - debug!("Using proxy {}:{}...", &proxy_host, &proxy_port); + println!("Using proxy {}:{}...", &proxy_host, &proxy_port); Ok(Client::with_http_proxy(proxy_host, proxy_port)) } None => Ok(Client::new()), } } - -#[cfg(test)] -mod tests { - use super::from_char; - - #[test] - fn from_char_returns_the_correct_string() { - assert_eq!("xxxx", from_char(4, 'x')); - } -} diff --git a/components/hab/Cargo.lock b/components/hab/Cargo.lock index b9fd839af2..7afa43b8f4 100644 --- a/components/hab/Cargo.lock +++ b/components/hab/Cargo.lock @@ -35,6 +35,11 @@ name = "bitflags" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "broadcast" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "byteorder" version = "0.5.1" @@ -121,10 +126,12 @@ dependencies = [ name = "habitat_depot_client" version = "0.5.0" dependencies = [ + "broadcast 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "habitat_core 0.5.0", "habitat_depot_core 0.5.0", "hyper 0.9.3 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", + "pbr 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-serialize 0.3.19 (registry+https://github.com/rust-lang/crates.io-index)", "url 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -387,6 +394,17 @@ dependencies = [ "openssl-sys 0.7.9 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "pbr" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "kernel32-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)", + "time 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "pkg-config" version = "0.3.8" diff --git a/components/sup/Cargo.lock b/components/sup/Cargo.lock index abe9ba2b2b..1f2440a743 100644 --- a/components/sup/Cargo.lock +++ b/components/sup/Cargo.lock @@ -61,6 +61,11 @@ dependencies = [ "serde_json 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "broadcast" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "byteorder" version = "0.5.1" @@ -164,10 +169,12 @@ dependencies = [ name = "habitat_depot_client" version = "0.5.0" dependencies = [ + "broadcast 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "habitat_core 0.5.0", "habitat_depot_core 0.5.0", "hyper 0.9.3 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", + "pbr 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-serialize 0.3.19 (registry+https://github.com/rust-lang/crates.io-index)", "url 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -481,6 +488,17 @@ dependencies = [ "openssl-sys 0.7.9 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "pbr" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "kernel32-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)", + "time 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "persistent" version = "0.1.0"