Add a timeout to cache lookups

This should help ensure that we don't wait *too* long for the cache to respond
(for example on an excessively slow network) and time out the server
unnecessarily.

src/compiler/compiler.rs

@@ -53,6 +53,7 @@ use std::time::{
     Instant,
 };
 use tempdir::TempDir;
+use tokio_core::reactor::{Handle, Timeout};
 
 use errors::*;
 
@@ -174,6 +175,8 @@ pub enum MissType {
     Normal,
     /// Cache lookup was overridden, recompilation was forced.
     ForcedRecache,
+    /// Cache took too long to respond.
+    TimedOut,
 }
 
 /// Information about a successful cache write.
@@ -298,7 +301,8 @@ impl Compiler {
                                     parsed_args: &ParsedArguments,
                                     cwd: &str,
                                     cache_control: CacheControl,
-                                    pool: &CpuPool)
+                                    pool: &CpuPool,
+                                    handle: &Handle)
                                     -> SFuture<(CompileResult, process::Output)>
         where T: CommandCreatorSync
     {
@@ -319,6 +323,7 @@ impl Compiler {
         let storage = storage.clone();
         let pool = pool.clone();
         let creator = creator.clone();
+        let handle = handle.clone();
 
         Box::new(result.and_then(move |preprocessor_result| -> SFuture<_> {
             // If the preprocessor failed, just return that result.
@@ -355,6 +360,20 @@ impl Compiler {
                 storage.get(&key)
             };
 
+            // Wait at most a minute for the cache to respond before we forge
+            // ahead ourselves with a compilation.
+            let timeout = Duration::new(60, 0);
+            let timeout = Timeout::new(timeout, &handle).into_future().flatten();
+
+            let cache_status = cache_status.map(Some);
+            let timeout = timeout.map(|_| None).chain_err(|| "timeout error");
+            let cache_status = cache_status.select(timeout).then(|r| {
+                match r {
+                    Ok((a, _other)) => Ok(a),
+                    Err((e, _other)) => Err(e),
+                }
+            });
+
             Box::new(cache_status.and_then(move |result| {
                 let duration = start.elapsed();
                 let pwd = Path::new(&cwd);
@@ -363,7 +382,7 @@ impl Compiler {
                     .collect::<HashMap<_, _>>();
 
                 let miss_type = match result {
-                    Cache::Hit(mut entry) => {
+                    Some(Cache::Hit(mut entry)) => {
                         debug!("[{}]: Cache hit!", parsed_args.output_file());
                         let mut stdout = io::Cursor::new(vec!());
                         let mut stderr = io::Cursor::new(vec!());
@@ -386,14 +405,18 @@ impl Compiler {
                             (result, output)
                         })) as SFuture<_>
                     }
-                    Cache::Miss => {
+                    Some(Cache::Miss) => {
                         debug!("[{}]: Cache miss!", parsed_args.output_file());
                         MissType::Normal
                     }
-                    Cache::Recache => {
+                    Some(Cache::Recache) => {
                         debug!("[{}]: Cache recache!", parsed_args.output_file());
                         MissType::ForcedRecache
                     }
+                    None => {
+                        debug!("[{}]: Cache timed out!", parsed_args.output_file());
+                        MissType::TimedOut
+                    }
                 };
                 me.compile(&creator,
                            preprocessor_result,

src/server.rs

@@ -549,7 +549,8 @@ impl<C> SccacheService<C>
                                                     &parsed_arguments,
                                                     &cwd,
                                                     cache_control,
-                                                    &self.pool);
+                                                    &self.pool,
+                                                    &self.handle);
         let me = self.clone();
         let task = result.then(move |result| {
             let mut res = ServerResponse::new();
@@ -575,6 +576,9 @@ impl<C> SccacheService<C>
                                     stats.cache_misses += 1;
                                     stats.forced_recaches += 1;
                                 }
+                                MissType::TimedOut => {
+                                    stats.cache_misses += 1;
+                                }
                             }
                             stats.cache_read_miss_duration += duration;
                             cache_write = Some(future);