From c2223541c64326c6762e5e82035a1d49fe2049f9 Mon Sep 17 00:00:00 2001 From: Pieter Date: Fri, 21 Oct 2022 19:26:35 +0200 Subject: [PATCH] bug: package Secrets.toml (#422) * bug: package Secrets.toml * refactor: clippy suggestion --- Cargo.lock | 2 + cargo-shuttle/Cargo.toml | 2 + cargo-shuttle/src/client.rs | 12 +-- cargo-shuttle/src/lib.rs | 106 ++++++++++++++++++- examples/rocket/secrets/.gitignore | 1 + examples/rocket/secrets/README.md | 5 + examples/rocket/secrets/Secrets.toml | 1 - examples/rocket/secrets/Secrets.toml.example | 1 + 8 files changed, 118 insertions(+), 12 deletions(-) create mode 100644 examples/rocket/secrets/.gitignore create mode 100644 examples/rocket/secrets/README.md delete mode 100644 examples/rocket/secrets/Secrets.toml create mode 100644 examples/rocket/secrets/Secrets.toml.example diff --git a/Cargo.lock b/Cargo.lock index 7b9447b54..0afac3b59 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1195,6 +1195,7 @@ dependencies = [ "crossbeam-channel", "crossterm", "dirs", + "flate2", "futures", "headers", "indoc", @@ -1209,6 +1210,7 @@ dependencies = [ "shuttle-secrets", "shuttle-service", "sqlx 0.6.1", + "tar", "tempfile", "test-context", "tokio", diff --git a/cargo-shuttle/Cargo.toml b/cargo-shuttle/Cargo.toml index b4bea91e9..8e45de9c3 100644 --- a/cargo-shuttle/Cargo.toml +++ b/cargo-shuttle/Cargo.toml @@ -19,6 +19,7 @@ clap_complete = "3.2.5" crossbeam-channel = "0.5.6" crossterm = "0.25.0" dirs = "4.0.0" +flate2 = "1.0.24" futures = "0.3.23" headers = "0.3.8" indoc = "1.0.7" @@ -30,6 +31,7 @@ reqwest-retry = "0.1.5" serde = { version = "1.0.143", features = ["derive"] } serde_json = "1.0.83" sqlx = { version = "0.6.1", features = ["runtime-tokio-native-tls", "postgres"] } +tar = "0.4.38" tokio = { version = "1.20.1", features = ["macros"] } tokio-tungstenite = { version = "0.17.2", features = ["native-tls"] } toml = "0.5.9" diff --git a/cargo-shuttle/src/client.rs b/cargo-shuttle/src/client.rs index aea1f2d38..9305e6b99 100644 --- a/cargo-shuttle/src/client.rs +++ b/cargo-shuttle/src/client.rs @@ -1,6 +1,4 @@ use std::fmt::Write; -use std::fs::File; -use std::io::Read; use anyhow::{Context, Result}; use async_trait::async_trait; @@ -78,7 +76,7 @@ impl Client { pub async fn deploy( &self, - package_file: File, + data: Vec, project: &ProjectName, no_test: bool, ) -> Result { @@ -92,13 +90,7 @@ impl Client { let _ = write!(path, "?no-test"); } - let mut package_file = package_file; - let mut package_content = Vec::new(); - package_file - .read_to_end(&mut package_content) - .context("failed to convert package content to buf")?; - - self.post(path, Some(package_content)) + self.post(path, Some(data)) .await .context("failed to send deployment to the Shuttle server")? .to_json() diff --git a/cargo-shuttle/src/lib.rs b/cargo-shuttle/src/lib.rs index 291fbe200..6b7ee7a9e 100644 --- a/cargo-shuttle/src/lib.rs +++ b/cargo-shuttle/src/lib.rs @@ -24,10 +24,14 @@ use clap_complete::{generate, Shell}; use config::RequestContext; use crossterm::style::Stylize; use factory::LocalFactory; +use flate2::read::GzDecoder; +use flate2::write::GzEncoder; +use flate2::Compression; use futures::StreamExt; use shuttle_common::models::secret; use shuttle_service::loader::{build_crate, Loader}; use shuttle_service::Logger; +use tar::{Archive, Builder}; use tokio::sync::mpsc; use tracing::trace; use uuid::Uuid; @@ -341,8 +345,10 @@ impl Shuttle { .run_cargo_package(args.allow_dirty) .context("failed to package cargo project")?; + let data = self.package_secret(package_file)?; + let deployment = client - .deploy(package_file, self.ctx.project_name(), args.no_test) + .deploy(data, self.ctx.project_name(), args.no_test) .await?; let mut stream = client @@ -449,6 +455,39 @@ impl Shuttle { let owned = locks.get(0).unwrap().file().try_clone()?; Ok(owned) } + + fn package_secret(&self, file: File) -> Result> { + let tar_read = GzDecoder::new(file); + let mut archive_read = Archive::new(tar_read); + let tar_write = GzEncoder::new(Vec::new(), Compression::best()); + let mut archive_write = Builder::new(tar_write); + + for entry in archive_read.entries()? { + let entry = entry?; + let path = entry.path()?; + let file_name = path.components().nth(1).unwrap(); + + if file_name.as_os_str() == "Secrets.toml" { + println!( + "{}: you may want to fix this", + "Secrets.toml might be tracked by your version control".yellow() + ); + } + + archive_write.append(&entry.header().clone(), entry)?; + } + + let secrets_path = self.ctx.working_directory().join("Secrets.toml"); + if secrets_path.exists() { + archive_write + .append_path_with_name(secrets_path, Path::new("shuttle").join("Secrets.toml"))?; + } + + let encoder = archive_write.into_inner()?; + let data = encoder.finish()?; + + Ok(data) + } } pub enum CommandOutcome { @@ -458,8 +497,13 @@ pub enum CommandOutcome { #[cfg(test)] mod tests { + use flate2::read::GzDecoder; + use tar::Archive; + use crate::args::ProjectArgs; use crate::Shuttle; + use std::fs::{canonicalize, File}; + use std::io::Write; use std::path::PathBuf; fn path_from_workspace_root(path: &str) -> PathBuf { @@ -495,4 +539,64 @@ mod tests { path_from_workspace_root("examples/axum/hello-world/") ); } + + #[test] + fn secrets_file_is_archived() { + let working_directory = + canonicalize(path_from_workspace_root("examples/rocket/secrets")).unwrap(); + + let mut secrets_file = File::create(working_directory.join("Secrets.toml")).unwrap(); + secrets_file + .write_all(b"MY_API_KEY = 'the contents of my API key'") + .unwrap(); + + let mut project_args = ProjectArgs { + working_directory, + name: None, + }; + + let mut shuttle = Shuttle::new().unwrap(); + shuttle.load_project(&mut project_args).unwrap(); + + let file = shuttle.run_cargo_package(true).unwrap(); + + // Make sure the Secrets.toml file is not initially present + let tar = GzDecoder::new(file); + let mut archive = Archive::new(tar); + + for entry in archive.entries().unwrap() { + let entry = entry.unwrap(); + let path = entry.path().unwrap(); + let name = path.components().nth(1).unwrap().as_os_str(); + + assert!( + name != "Secrets.toml", + "no Secrets.toml file should be in the initial archive: {:?}", + path + ); + } + + let file = shuttle.run_cargo_package(true).unwrap(); + let new_file = shuttle.package_secret(file).unwrap(); + let mut found_secrets_file = false; + + // This time the Secrets.toml file should be present + let tar = flate2::bufread::GzDecoder::new(&new_file[..]); + let mut archive = Archive::new(tar); + + for entry in archive.entries().unwrap() { + let entry = entry.unwrap(); + let path = entry.path().unwrap(); + let name = path.components().nth(1).unwrap().as_os_str(); + + if name == "Secrets.toml" { + found_secrets_file = true; + } + } + + assert!( + found_secrets_file, + "Secrets.toml was not added to the archive" + ); + } } diff --git a/examples/rocket/secrets/.gitignore b/examples/rocket/secrets/.gitignore new file mode 100644 index 000000000..bf07a4e9c --- /dev/null +++ b/examples/rocket/secrets/.gitignore @@ -0,0 +1 @@ +Secrets.toml diff --git a/examples/rocket/secrets/README.md b/examples/rocket/secrets/README.md new file mode 100644 index 000000000..988cbb6c3 --- /dev/null +++ b/examples/rocket/secrets/README.md @@ -0,0 +1,5 @@ +## How to use +The secrets resource requires a `Secrets.toml` file to be present in your crate. Each like in this file +should be a key-value pair that you can access using `SecretStore::get(&self, key)`. + +Rename `Secrets.toml.example` to `Secrets.toml` to use this example. diff --git a/examples/rocket/secrets/Secrets.toml b/examples/rocket/secrets/Secrets.toml deleted file mode 100644 index ceedf199e..000000000 --- a/examples/rocket/secrets/Secrets.toml +++ /dev/null @@ -1 +0,0 @@ -MY_API_KEY = 'the contents of my API key' diff --git a/examples/rocket/secrets/Secrets.toml.example b/examples/rocket/secrets/Secrets.toml.example new file mode 100644 index 000000000..455ab4c81 --- /dev/null +++ b/examples/rocket/secrets/Secrets.toml.example @@ -0,0 +1 @@ +MY_API_KEY = 'the contents of my API key' \ No newline at end of file