Switch to use AWS signature v4

Cargo.toml

@@ -27,7 +27,6 @@ rustdoc-args = [
 ]
 
 [dependencies]
-cargo-registry-s3 = { path = "src/s3", version = "0.2.0" }
 rand = "0.3"
 git2 = "0.6.4"
 flate2 = "1.0"
@@ -39,6 +38,7 @@ openssl = "0.9.14"
 oauth2 = "0.3"
 log = "0.3"
 env_logger = "0.5"
+futures = "0.1.24"
 hex = "0.3"
 htmlescape = "0.3.1"
 license-exprs = "^1.4"
@@ -47,6 +47,9 @@ toml = "0.4"
 diesel = { version = "1.3.0", features = ["postgres", "serde_json", "chrono", "r2d2"] }
 diesel_full_text_search = "1.0.0"
 diesel_ltree = "0.1.3"
+rusoto_core = "0.34.0"
+rusoto_credential = "0.13.0"
+rusoto_s3 = "0.34.0"
 serde_json = "1.0.0"
 serde_derive = "1.0.0"
 serde = "1.0.0"

docker-compose.yml

@@ -28,7 +28,7 @@ services:
     volumes:
       # Mount the src/ directory so we don't have to rebuild the Docker image
       # when we want to change some code
-      - ./src:/app/src:ro
+      - ./src:/app/src
 
       - index:/app/tmp
       - cargo-cache:/usr/local/cargo/registry
@@ -37,7 +37,7 @@ services:
     build:
       context: .
       dockerfile: frontend.Dockerfile
-    entrypoint: npm run start --proxy http://backend:8888
+    entrypoint: npm run start -- --proxy http://backend:8888
     links:
       - backend
     ports:

frontend.Dockerfile

@@ -5,4 +5,7 @@ COPY package.json /app
 
 COPY . /app
 
+RUN npm install -g ember-cli
+RUN npm install
+
 ENTRYPOINT ["npm", "run", "start:staging"]

src/bin/render-readmes.rs

@@ -146,7 +146,6 @@ fn main() {
                 config
                     .uploader
                     .upload(
-                        &reqwest::Client::new(),
                         &readme_path,
                         readme.into_bytes(),
                         "text/html",

src/config.rs

@@ -1,9 +1,7 @@
-use s3;
-
 use std::env;
 use std::path::PathBuf;
 
-use {env, Env, Replica, Uploader};
+use {env, Env, Replica, S3UploaderBuilder, Uploader};
 
 #[derive(Clone, Debug)]
 pub struct Config {
@@ -38,6 +36,7 @@ impl Default for Config {
     /// - `S3_REGION`: The region in which the bucket was created. Optional if US standard.
     /// - `S3_ACCESS_KEY`: The access key to interact with S3. Optional if running a mirror.
     /// - `S3_SECRET_KEY`: The secret key to interact with S3. Optional if running a mirror.
+    /// - `S3_HOST`: The S3 host to use. Optional.
     /// - `SESSION_KEY`: The key used to sign and encrypt session cookies.
     /// - `GH_CLIENT_ID`: The client ID of the associated GitHub application.
     /// - `GH_CLIENT_SECRET`: The client secret of the associated GitHub application.
@@ -56,22 +55,17 @@ impl Default for Config {
         } else {
             Env::Development
         };
+
         let uploader = match (cargo_env, mirror) {
             (Env::Production, Replica::Primary) => {
                 // `env` panics if these vars are not set, and in production for a primary instance,
                 // that's what we want since we don't want to be able to start the server if the
                 // server doesn't know where to upload crates.
-                Uploader::S3 {
-                    bucket: s3::Bucket::new(
-                        env("S3_BUCKET"),
-                        env::var("S3_REGION").ok(),
-                        env("S3_ACCESS_KEY"),
-                        env("S3_SECRET_KEY"),
-                        &api_protocol,
-                    ),
-                    cdn: env::var("S3_CDN").ok(),
-                    proxy: None,
-                }
+                S3UploaderBuilder::new(env("S3_BUCKET"), env("S3_ACCESS_KEY"), env("S3_SECRET_KEY"))
+                    .region(env::var("S3_REGION").ok())
+                    .host(env::var("S3_HOST").ok())
+                    .cdn(env::var("S3_CDN").ok())
+                    .build()
             }
             (Env::Production, Replica::ReadOnlyMirror) => {
                 // Read-only mirrors don't need access key or secret key since by definition,
@@ -82,17 +76,17 @@ impl Default for Config {
                 //
                 // Read-only mirrors definitely need bucket though, so that they know where
                 // to serve crate files from.
-                Uploader::S3 {
-                    bucket: s3::Bucket::new(
-                        env("S3_BUCKET"),
-                        env::var("S3_REGION").ok(),
-                        env::var("S3_ACCESS_KEY").unwrap_or_default(),
-                        env::var("S3_SECRET_KEY").unwrap_or_default(),
-                        &api_protocol,
-                    ),
-                    cdn: env::var("S3_CDN").ok(),
-                    proxy: None,
-                }
+                let mut s3_builder = S3UploaderBuilder::new(
+                    env("S3_BUCKET"),
+                    env::var("S3_ACCESS_KEY").unwrap_or_default(),
+                    env::var("S3_SECRET_KEY").unwrap_or_default(),
+                );
+
+                s3_builder
+                    .region(env::var("S3_REGION").ok())
+                    .host(env::var("S3_HOST").ok())
+                    .cdn(env::var("S3_CDN").ok())
+                    .build()
             }
             // In Development mode, either running as a primary instance or a read-only mirror
             _ => {
@@ -102,17 +96,18 @@ impl Default for Config {
                     // and read from S3 like production does. All values except for bucket are
                     // optional, like production read-only mirrors.
                     println!("Using S3 uploader");
-                    Uploader::S3 {
-                        bucket: s3::Bucket::new(
-                            env("S3_BUCKET"),
-                            env::var("S3_REGION").ok(),
-                            env::var("S3_ACCESS_KEY").unwrap_or_default(),
-                            env::var("S3_SECRET_KEY").unwrap_or_default(),
-                            &api_protocol,
-                        ),
-                        cdn: env::var("S3_CDN").ok(),
-                        proxy: None,
-                    }
+
+                    let mut s3_builder = S3UploaderBuilder::new(
+                        env("S3_BUCKET"),
+                        env::var("S3_ACCESS_KEY").unwrap_or_default(),
+                        env::var("S3_SECRET_KEY").unwrap_or_default(),
+                    );
+
+                    s3_builder
+                        .region(env::var("S3_REGION").ok())
+                        .host(env::var("S3_HOST").ok())
+                        .cdn(env::var("S3_CDN").ok())
+                        .build()
                 } else {
                     // If we don't set the `S3_BUCKET` variable, we'll use a development-only
                     // uploader that makes it possible to run and publish to a locally-running

src/git.rs

@@ -206,11 +206,19 @@ where
 
 pub fn credentials(
     _user: &str,
-    _user_from_url: Option<&str>,
+    user_from_url: Option<&str>,
     _cred: git2::CredentialType,
 ) -> Result<git2::Cred, git2::Error> {
     match (env::var("GIT_HTTP_USER"), env::var("GIT_HTTP_PWD")) {
         (Ok(u), Ok(p)) => git2::Cred::userpass_plaintext(&u, &p),
-        _ => Err(git2::Error::from_str("no authentication set")),
+        _ => {
+            // If there is no user, or password, try and use SSH keys using
+            // the SSH agent.
+            if let Some(user) = user_from_url {
+                git2::Cred::ssh_key_from_agent(user)
+            } else {
+                Err(git2::Error::from_str("no authentication set"))
+            }
+        }
     }
 }

src/lib.rs

@@ -20,6 +20,7 @@ extern crate diesel_full_text_search;
 extern crate diesel_ltree;
 extern crate dotenv;
 extern crate flate2;
+extern crate futures;
 extern crate git2;
 extern crate hex;
 extern crate htmlescape;
@@ -32,7 +33,9 @@ extern crate oauth2;
 extern crate openssl;
 extern crate rand;
 extern crate reqwest;
-extern crate s3;
+extern crate rusoto_core;
+extern crate rusoto_credential;
+extern crate rusoto_s3;
 extern crate scheduled_thread_pool;
 extern crate semver;
 extern crate serde;
@@ -53,7 +56,7 @@ extern crate conduit_router;
 extern crate conduit_static;
 extern crate cookie;
 
-pub use self::uploaders::{Bomb, Uploader};
+pub use self::uploaders::{Bomb, S3UploaderBuilder, Uploader};
 pub use app::App;
 pub use config::Config;
 

src/middleware/security_headers.rs

@@ -22,12 +22,10 @@ impl SecurityHeaders {
 
         let s3_host = match *uploader {
             Uploader::S3 {
-                ref bucket,
-                ref cdn,
-                ..
+                ref host, ref cdn, ..
             } => match *cdn {
                 Some(ref s) => s.clone(),
-                None => bucket.host(),
+                None => host.clone(),
             },
             _ => unreachable!(
                 "This middleware should only be used in the production environment, \