diff --git a/Cargo.lock b/Cargo.lock index e78ea89b0..a57c5c9d4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -888,6 +888,7 @@ dependencies = [ "secret", "serde_json", "sev 0.1.0", + "storage", "thiserror", "tokio", "ttrpc", @@ -5463,6 +5464,21 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "storage" +version = "0.1.0" +dependencies = [ + "anyhow", + "base64 0.21.4", + "log", + "rstest", + "secret", + "serde", + "serde_json", + "thiserror", + "tokio", +] + [[package]] name = "string_cache" version = "0.8.7" diff --git a/Cargo.toml b/Cargo.toml index 99d52a06e..7ec7bd99d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,6 +16,7 @@ members = [ "confidential-data-hub/kms", "confidential-data-hub/image", "confidential-data-hub/secret", + "confidential-data-hub/storage", "image-rs", "ocicrypt-rs", ] diff --git a/confidential-data-hub/docs/SECURE_STORAGE.md b/confidential-data-hub/docs/SECURE_STORAGE.md new file mode 100644 index 000000000..d93bec0c4 --- /dev/null +++ b/confidential-data-hub/docs/SECURE_STORAGE.md @@ -0,0 +1,13 @@ +# Secure Storage + +## Purpose +The Purpose of this secure storage feature is: +1. Mounting external storage from guest instead of host which would then share it to guest, this is due to performance consideration. +2. The unencrypted data in storage could only be accessed within TEE, that is why we call it secure storage. + +## Architecture +![architecture](./images/secure_storage.png) + +First of all, the sensitive information of external storage is sealed by the key from KBS/KMS, and store in [sealed secret](https://github.com/confidential-containers/guest-components/blob/main/confidential-data-hub/docs/SEALED_SECRET.md). The sensitive information includes access key id/access key secret to storage, the encryption key of the data(such as AI model) stored in the storage, which also means we supported client encryption. +We reuse [direct block device assigned volume feature](https://github.com/kata-containers/kata-containers/blob/main/docs/design/direct-blk-device-assignment.md) to mount external storage from guest directly. CSI plugin, such as [alibaba cloud OSS CSI plugin](https://github.com/kubernetes-sigs/alibaba-cloud-csi-driver/blob/master/docs/oss.md) reads the sensitve information from sealed secret and pass it to kata agent. When secure mount service in CDH receives secure mount request, it calls sealed secret service to unseal the sensitive information mentioned above, this process could be based on remote attestation. If success, the secure mount service would use the unsealed sensitive information to mount the external storage and decrypt the data in storage. + diff --git a/confidential-data-hub/docs/images/secure_storage.png b/confidential-data-hub/docs/images/secure_storage.png new file mode 100644 index 000000000..225d946e9 Binary files /dev/null and b/confidential-data-hub/docs/images/secure_storage.png differ diff --git a/confidential-data-hub/hub/Cargo.toml b/confidential-data-hub/hub/Cargo.toml index 5eed01edb..4dd0a4c3b 100644 --- a/confidential-data-hub/hub/Cargo.toml +++ b/confidential-data-hub/hub/Cargo.toml @@ -21,6 +21,7 @@ lazy_static.workspace = true log.workspace = true protobuf = { workspace = true, optional = true } secret.path = "../secret" +storage.path = "../storage" serde = { workspace = true, optional = true } serde_json.workspace = true sev = { path = "../../attestation-agent/deps/sev", optional = true } diff --git a/confidential-data-hub/hub/protos/api.proto b/confidential-data-hub/hub/protos/api.proto index 97a3c6266..8a8ddaf26 100644 --- a/confidential-data-hub/hub/protos/api.proto +++ b/confidential-data-hub/hub/protos/api.proto @@ -26,6 +26,19 @@ message KeyProviderKeyWrapProtocolOutput { bytes KeyProviderKeyWrapProtocolOutput = 1; } +message SecureMountRequest { + string driver = 1; + repeated string driver_options = 2; + string source = 3; + string fstype = 4; + repeated string options = 5; + string mount_point = 6; +} + +message SecureMountResponse { + string mount_path = 1; +} + service SealedSecretService { rpc UnsealSecret(UnsealSecretInput) returns (UnsealSecretOutput) {}; } @@ -37,3 +50,7 @@ service GetResourceService { service KeyProviderService { rpc UnWrapKey(KeyProviderKeyWrapProtocolInput) returns (KeyProviderKeyWrapProtocolOutput) {}; } + +service SecureMountService { + rpc SecureMount(SecureMountRequest) returns (SecureMountResponse) {}; +} diff --git a/confidential-data-hub/hub/src/api.rs b/confidential-data-hub/hub/src/api.rs index 3069bff3d..880db037e 100644 --- a/confidential-data-hub/hub/src/api.rs +++ b/confidential-data-hub/hub/src/api.rs @@ -6,6 +6,7 @@ use async_trait::async_trait; use crate::Result; +use storage::volume_type::Storage; /// The APIs of the DataHub. See /// for @@ -26,4 +27,6 @@ pub trait DataHub { /// URI is defined in /// async fn get_resource(&self, uri: String) -> Result>; + + async fn secure_mount(&self, storage: Storage) -> Result; } diff --git a/confidential-data-hub/hub/src/bin/confidential-data-hub/api.rs b/confidential-data-hub/hub/src/bin/confidential-data-hub/api.rs index a58d8c2eb..302ac804e 100644 --- a/confidential-data-hub/hub/src/bin/confidential-data-hub/api.rs +++ b/confidential-data-hub/hub/src/bin/confidential-data-hub/api.rs @@ -757,6 +757,340 @@ impl ::protobuf::reflect::ProtobufValue for KeyProviderKeyWrapProtocolOutput { type RuntimeType = ::protobuf::reflect::rt::RuntimeTypeMessage; } +#[derive(PartialEq,Clone,Default,Debug)] +// @@protoc_insertion_point(message:api.SecureMountRequest) +pub struct SecureMountRequest { + // message fields + // @@protoc_insertion_point(field:api.SecureMountRequest.driver) + pub driver: ::std::string::String, + // @@protoc_insertion_point(field:api.SecureMountRequest.driver_options) + pub driver_options: ::std::vec::Vec<::std::string::String>, + // @@protoc_insertion_point(field:api.SecureMountRequest.source) + pub source: ::std::string::String, + // @@protoc_insertion_point(field:api.SecureMountRequest.fstype) + pub fstype: ::std::string::String, + // @@protoc_insertion_point(field:api.SecureMountRequest.options) + pub options: ::std::vec::Vec<::std::string::String>, + // @@protoc_insertion_point(field:api.SecureMountRequest.mount_point) + pub mount_point: ::std::string::String, + // special fields + // @@protoc_insertion_point(special_field:api.SecureMountRequest.special_fields) + pub special_fields: ::protobuf::SpecialFields, +} + +impl<'a> ::std::default::Default for &'a SecureMountRequest { + fn default() -> &'a SecureMountRequest { + ::default_instance() + } +} + +impl SecureMountRequest { + pub fn new() -> SecureMountRequest { + ::std::default::Default::default() + } + + fn generated_message_descriptor_data() -> ::protobuf::reflect::GeneratedMessageDescriptorData { + let mut fields = ::std::vec::Vec::with_capacity(6); + let mut oneofs = ::std::vec::Vec::with_capacity(0); + fields.push(::protobuf::reflect::rt::v2::make_simpler_field_accessor::<_, _>( + "driver", + |m: &SecureMountRequest| { &m.driver }, + |m: &mut SecureMountRequest| { &mut m.driver }, + )); + fields.push(::protobuf::reflect::rt::v2::make_vec_simpler_accessor::<_, _>( + "driver_options", + |m: &SecureMountRequest| { &m.driver_options }, + |m: &mut SecureMountRequest| { &mut m.driver_options }, + )); + fields.push(::protobuf::reflect::rt::v2::make_simpler_field_accessor::<_, _>( + "source", + |m: &SecureMountRequest| { &m.source }, + |m: &mut SecureMountRequest| { &mut m.source }, + )); + fields.push(::protobuf::reflect::rt::v2::make_simpler_field_accessor::<_, _>( + "fstype", + |m: &SecureMountRequest| { &m.fstype }, + |m: &mut SecureMountRequest| { &mut m.fstype }, + )); + fields.push(::protobuf::reflect::rt::v2::make_vec_simpler_accessor::<_, _>( + "options", + |m: &SecureMountRequest| { &m.options }, + |m: &mut SecureMountRequest| { &mut m.options }, + )); + fields.push(::protobuf::reflect::rt::v2::make_simpler_field_accessor::<_, _>( + "mount_point", + |m: &SecureMountRequest| { &m.mount_point }, + |m: &mut SecureMountRequest| { &mut m.mount_point }, + )); + ::protobuf::reflect::GeneratedMessageDescriptorData::new_2::( + "SecureMountRequest", + fields, + oneofs, + ) + } +} + +impl ::protobuf::Message for SecureMountRequest { + const NAME: &'static str = "SecureMountRequest"; + + fn is_initialized(&self) -> bool { + true + } + + fn merge_from(&mut self, is: &mut ::protobuf::CodedInputStream<'_>) -> ::protobuf::Result<()> { + while let Some(tag) = is.read_raw_tag_or_eof()? { + match tag { + 10 => { + self.driver = is.read_string()?; + }, + 18 => { + self.driver_options.push(is.read_string()?); + }, + 26 => { + self.source = is.read_string()?; + }, + 34 => { + self.fstype = is.read_string()?; + }, + 42 => { + self.options.push(is.read_string()?); + }, + 50 => { + self.mount_point = is.read_string()?; + }, + tag => { + ::protobuf::rt::read_unknown_or_skip_group(tag, is, self.special_fields.mut_unknown_fields())?; + }, + }; + } + ::std::result::Result::Ok(()) + } + + // Compute sizes of nested messages + #[allow(unused_variables)] + fn compute_size(&self) -> u64 { + let mut my_size = 0; + if !self.driver.is_empty() { + my_size += ::protobuf::rt::string_size(1, &self.driver); + } + for value in &self.driver_options { + my_size += ::protobuf::rt::string_size(2, &value); + }; + if !self.source.is_empty() { + my_size += ::protobuf::rt::string_size(3, &self.source); + } + if !self.fstype.is_empty() { + my_size += ::protobuf::rt::string_size(4, &self.fstype); + } + for value in &self.options { + my_size += ::protobuf::rt::string_size(5, &value); + }; + if !self.mount_point.is_empty() { + my_size += ::protobuf::rt::string_size(6, &self.mount_point); + } + my_size += ::protobuf::rt::unknown_fields_size(self.special_fields.unknown_fields()); + self.special_fields.cached_size().set(my_size as u32); + my_size + } + + fn write_to_with_cached_sizes(&self, os: &mut ::protobuf::CodedOutputStream<'_>) -> ::protobuf::Result<()> { + if !self.driver.is_empty() { + os.write_string(1, &self.driver)?; + } + for v in &self.driver_options { + os.write_string(2, &v)?; + }; + if !self.source.is_empty() { + os.write_string(3, &self.source)?; + } + if !self.fstype.is_empty() { + os.write_string(4, &self.fstype)?; + } + for v in &self.options { + os.write_string(5, &v)?; + }; + if !self.mount_point.is_empty() { + os.write_string(6, &self.mount_point)?; + } + os.write_unknown_fields(self.special_fields.unknown_fields())?; + ::std::result::Result::Ok(()) + } + + fn special_fields(&self) -> &::protobuf::SpecialFields { + &self.special_fields + } + + fn mut_special_fields(&mut self) -> &mut ::protobuf::SpecialFields { + &mut self.special_fields + } + + fn new() -> SecureMountRequest { + SecureMountRequest::new() + } + + fn clear(&mut self) { + self.driver.clear(); + self.driver_options.clear(); + self.source.clear(); + self.fstype.clear(); + self.options.clear(); + self.mount_point.clear(); + self.special_fields.clear(); + } + + fn default_instance() -> &'static SecureMountRequest { + static instance: SecureMountRequest = SecureMountRequest { + driver: ::std::string::String::new(), + driver_options: ::std::vec::Vec::new(), + source: ::std::string::String::new(), + fstype: ::std::string::String::new(), + options: ::std::vec::Vec::new(), + mount_point: ::std::string::String::new(), + special_fields: ::protobuf::SpecialFields::new(), + }; + &instance + } +} + +impl ::protobuf::MessageFull for SecureMountRequest { + fn descriptor() -> ::protobuf::reflect::MessageDescriptor { + static descriptor: ::protobuf::rt::Lazy<::protobuf::reflect::MessageDescriptor> = ::protobuf::rt::Lazy::new(); + descriptor.get(|| file_descriptor().message_by_package_relative_name("SecureMountRequest").unwrap()).clone() + } +} + +impl ::std::fmt::Display for SecureMountRequest { + fn fmt(&self, f: &mut ::std::fmt::Formatter<'_>) -> ::std::fmt::Result { + ::protobuf::text_format::fmt(self, f) + } +} + +impl ::protobuf::reflect::ProtobufValue for SecureMountRequest { + type RuntimeType = ::protobuf::reflect::rt::RuntimeTypeMessage; +} + +#[derive(PartialEq,Clone,Default,Debug)] +// @@protoc_insertion_point(message:api.SecureMountResponse) +pub struct SecureMountResponse { + // message fields + // @@protoc_insertion_point(field:api.SecureMountResponse.mount_path) + pub mount_path: ::std::string::String, + // special fields + // @@protoc_insertion_point(special_field:api.SecureMountResponse.special_fields) + pub special_fields: ::protobuf::SpecialFields, +} + +impl<'a> ::std::default::Default for &'a SecureMountResponse { + fn default() -> &'a SecureMountResponse { + ::default_instance() + } +} + +impl SecureMountResponse { + pub fn new() -> SecureMountResponse { + ::std::default::Default::default() + } + + fn generated_message_descriptor_data() -> ::protobuf::reflect::GeneratedMessageDescriptorData { + let mut fields = ::std::vec::Vec::with_capacity(1); + let mut oneofs = ::std::vec::Vec::with_capacity(0); + fields.push(::protobuf::reflect::rt::v2::make_simpler_field_accessor::<_, _>( + "mount_path", + |m: &SecureMountResponse| { &m.mount_path }, + |m: &mut SecureMountResponse| { &mut m.mount_path }, + )); + ::protobuf::reflect::GeneratedMessageDescriptorData::new_2::( + "SecureMountResponse", + fields, + oneofs, + ) + } +} + +impl ::protobuf::Message for SecureMountResponse { + const NAME: &'static str = "SecureMountResponse"; + + fn is_initialized(&self) -> bool { + true + } + + fn merge_from(&mut self, is: &mut ::protobuf::CodedInputStream<'_>) -> ::protobuf::Result<()> { + while let Some(tag) = is.read_raw_tag_or_eof()? { + match tag { + 10 => { + self.mount_path = is.read_string()?; + }, + tag => { + ::protobuf::rt::read_unknown_or_skip_group(tag, is, self.special_fields.mut_unknown_fields())?; + }, + }; + } + ::std::result::Result::Ok(()) + } + + // Compute sizes of nested messages + #[allow(unused_variables)] + fn compute_size(&self) -> u64 { + let mut my_size = 0; + if !self.mount_path.is_empty() { + my_size += ::protobuf::rt::string_size(1, &self.mount_path); + } + my_size += ::protobuf::rt::unknown_fields_size(self.special_fields.unknown_fields()); + self.special_fields.cached_size().set(my_size as u32); + my_size + } + + fn write_to_with_cached_sizes(&self, os: &mut ::protobuf::CodedOutputStream<'_>) -> ::protobuf::Result<()> { + if !self.mount_path.is_empty() { + os.write_string(1, &self.mount_path)?; + } + os.write_unknown_fields(self.special_fields.unknown_fields())?; + ::std::result::Result::Ok(()) + } + + fn special_fields(&self) -> &::protobuf::SpecialFields { + &self.special_fields + } + + fn mut_special_fields(&mut self) -> &mut ::protobuf::SpecialFields { + &mut self.special_fields + } + + fn new() -> SecureMountResponse { + SecureMountResponse::new() + } + + fn clear(&mut self) { + self.mount_path.clear(); + self.special_fields.clear(); + } + + fn default_instance() -> &'static SecureMountResponse { + static instance: SecureMountResponse = SecureMountResponse { + mount_path: ::std::string::String::new(), + special_fields: ::protobuf::SpecialFields::new(), + }; + &instance + } +} + +impl ::protobuf::MessageFull for SecureMountResponse { + fn descriptor() -> ::protobuf::reflect::MessageDescriptor { + static descriptor: ::protobuf::rt::Lazy<::protobuf::reflect::MessageDescriptor> = ::protobuf::rt::Lazy::new(); + descriptor.get(|| file_descriptor().message_by_package_relative_name("SecureMountResponse").unwrap()).clone() + } +} + +impl ::std::fmt::Display for SecureMountResponse { + fn fmt(&self, f: &mut ::std::fmt::Formatter<'_>) -> ::std::fmt::Result { + ::protobuf::text_format::fmt(self, f) + } +} + +impl ::protobuf::reflect::ProtobufValue for SecureMountResponse { + type RuntimeType = ::protobuf::reflect::rt::RuntimeTypeMessage; +} + static file_descriptor_proto_data: &'static [u8] = b"\ \n\tapi.proto\x12\x03api\"+\n\x11UnsealSecretInput\x12\x16\n\x06secret\ \x18\x01\x20\x01(\x0cR\x06secret\"2\n\x12UnsealSecretOutput\x12\x1c\n\tp\ @@ -766,12 +1100,20 @@ static file_descriptor_proto_data: &'static [u8] = b"\ \n\x1fKeyProviderKeyWrapProtocolInput\x12H\n\x1fKeyProviderKeyWrapProtoc\ olInput\x18\x01\x20\x01(\x0cR\x1fKeyProviderKeyWrapProtocolInput\"n\n\ \x20KeyProviderKeyWrapProtocolOutput\x12J\n\x20KeyProviderKeyWrapProtoco\ - lOutput\x18\x01\x20\x01(\x0cR\x20KeyProviderKeyWrapProtocolOutput2V\n\ - \x13SealedSecretService\x12?\n\x0cUnsealSecret\x12\x16.api.UnsealSecretI\ - nput\x1a\x17.api.UnsealSecretOutput2V\n\x12GetResourceService\x12@\n\x0b\ - GetResource\x12\x17.api.GetResourceRequest\x1a\x18.api.GetResourceRespon\ - se2n\n\x12KeyProviderService\x12X\n\tUnWrapKey\x12$.api.KeyProviderKeyWr\ - apProtocolInput\x1a%.api.KeyProviderKeyWrapProtocolOutputb\x06proto3\ + lOutput\x18\x01\x20\x01(\x0cR\x20KeyProviderKeyWrapProtocolOutput\"\xbe\ + \x01\n\x12SecureMountRequest\x12\x16\n\x06driver\x18\x01\x20\x01(\tR\x06\ + driver\x12%\n\x0edriver_options\x18\x02\x20\x03(\tR\rdriverOptions\x12\ + \x16\n\x06source\x18\x03\x20\x01(\tR\x06source\x12\x16\n\x06fstype\x18\ + \x04\x20\x01(\tR\x06fstype\x12\x18\n\x07options\x18\x05\x20\x03(\tR\x07o\ + ptions\x12\x1f\n\x0bmount_point\x18\x06\x20\x01(\tR\nmountPoint\"4\n\x13\ + SecureMountResponse\x12\x1d\n\nmount_path\x18\x01\x20\x01(\tR\tmountPath\ + 2V\n\x13SealedSecretService\x12?\n\x0cUnsealSecret\x12\x16.api.UnsealSec\ + retInput\x1a\x17.api.UnsealSecretOutput2V\n\x12GetResourceService\x12@\n\ + \x0bGetResource\x12\x17.api.GetResourceRequest\x1a\x18.api.GetResourceRe\ + sponse2n\n\x12KeyProviderService\x12X\n\tUnWrapKey\x12$.api.KeyProviderK\ + eyWrapProtocolInput\x1a%.api.KeyProviderKeyWrapProtocolOutput2V\n\x12Sec\ + ureMountService\x12@\n\x0bSecureMount\x12\x17.api.SecureMountRequest\x1a\ + \x18.api.SecureMountResponseb\x06proto3\ "; /// `FileDescriptorProto` object which was a source for this generated file @@ -789,13 +1131,15 @@ pub fn file_descriptor() -> &'static ::protobuf::reflect::FileDescriptor { file_descriptor.get(|| { let generated_file_descriptor = generated_file_descriptor_lazy.get(|| { let mut deps = ::std::vec::Vec::with_capacity(0); - let mut messages = ::std::vec::Vec::with_capacity(6); + let mut messages = ::std::vec::Vec::with_capacity(8); messages.push(UnsealSecretInput::generated_message_descriptor_data()); messages.push(UnsealSecretOutput::generated_message_descriptor_data()); messages.push(GetResourceRequest::generated_message_descriptor_data()); messages.push(GetResourceResponse::generated_message_descriptor_data()); messages.push(KeyProviderKeyWrapProtocolInput::generated_message_descriptor_data()); messages.push(KeyProviderKeyWrapProtocolOutput::generated_message_descriptor_data()); + messages.push(SecureMountRequest::generated_message_descriptor_data()); + messages.push(SecureMountResponse::generated_message_descriptor_data()); let mut enums = ::std::vec::Vec::with_capacity(0); ::protobuf::reflect::GeneratedFileDescriptor::new_generated( file_descriptor_proto(), diff --git a/confidential-data-hub/hub/src/bin/confidential-data-hub/api_ttrpc.rs b/confidential-data-hub/hub/src/bin/confidential-data-hub/api_ttrpc.rs index a86192fa2..47b16b330 100644 --- a/confidential-data-hub/hub/src/bin/confidential-data-hub/api_ttrpc.rs +++ b/confidential-data-hub/hub/src/bin/confidential-data-hub/api_ttrpc.rs @@ -165,3 +165,51 @@ pub fn create_key_provider_service(service: Arc Self { + SecureMountServiceClient { + client, + } + } + + pub async fn secure_mount(&self, ctx: ttrpc::context::Context, req: &super::api::SecureMountRequest) -> ::ttrpc::Result { + let mut cres = super::api::SecureMountResponse::new(); + ::ttrpc::async_client_request!(self, ctx, req, "api.SecureMountService", "SecureMount", cres); + } +} + +struct SecureMountMethod { + service: Arc>, +} + +#[async_trait] +impl ::ttrpc::r#async::MethodHandler for SecureMountMethod { + async fn handler(&self, ctx: ::ttrpc::r#async::TtrpcContext, req: ::ttrpc::Request) -> ::ttrpc::Result<::ttrpc::Response> { + ::ttrpc::async_request_handler!(self, ctx, req, api, SecureMountRequest, secure_mount); + } +} + +#[async_trait] +pub trait SecureMountService: Sync { + async fn secure_mount(&self, _ctx: &::ttrpc::r#async::TtrpcContext, _: super::api::SecureMountRequest) -> ::ttrpc::Result { + Err(::ttrpc::Error::RpcStatus(::ttrpc::get_status(::ttrpc::Code::NOT_FOUND, "/api.SecureMountService/SecureMount is not supported".to_string()))) + } +} + +pub fn create_secure_mount_service(service: Arc>) -> HashMap { + let mut ret = HashMap::new(); + let mut methods = HashMap::new(); + let streams = HashMap::new(); + + methods.insert("SecureMount".to_string(), + Box::new(SecureMountMethod{service: service.clone()}) as Box); + + ret.insert("api.SecureMountService".to_string(), ::ttrpc::r#async::Service{ methods, streams }); + ret +} diff --git a/confidential-data-hub/hub/src/bin/confidential-data-hub/main.rs b/confidential-data-hub/hub/src/bin/confidential-data-hub/main.rs index 2737c9da3..6455b98b1 100644 --- a/confidential-data-hub/hub/src/bin/confidential-data-hub/main.rs +++ b/confidential-data-hub/hub/src/bin/confidential-data-hub/main.rs @@ -8,6 +8,7 @@ use std::{path::Path, sync::Arc}; use anyhow::{Context, Result}; use api_ttrpc::{ create_get_resource_service, create_key_provider_service, create_sealed_secret_service, + create_secure_mount_service, }; use clap::Parser; use log::info; @@ -58,11 +59,13 @@ async fn main() -> Result<()> { let sealed_secret_service = ttrpc_service!(create_sealed_secret_service); let get_resource_service = ttrpc_service!(create_get_resource_service); let key_provider_service = ttrpc_service!(create_key_provider_service); + let secure_mount_service = ttrpc_service!(create_secure_mount_service); let mut server = TtrpcServer::new() .bind(&cli.socket) .context("cannot bind cdh ttrpc service")? .register_service(sealed_secret_service) .register_service(get_resource_service) + .register_service(secure_mount_service) .register_service(key_provider_service); server.start().await?; diff --git a/confidential-data-hub/hub/src/bin/confidential-data-hub/server/mod.rs b/confidential-data-hub/hub/src/bin/confidential-data-hub/server/mod.rs index 4565ca9de..501f0850f 100644 --- a/confidential-data-hub/hub/src/bin/confidential-data-hub/server/mod.rs +++ b/confidential-data-hub/hub/src/bin/confidential-data-hub/server/mod.rs @@ -10,15 +10,17 @@ use async_trait::async_trait; use confidential_data_hub::{hub::Hub, DataHub}; use lazy_static::lazy_static; use log::debug; +use storage::volume_type::Storage; use tokio::sync::RwLock; use ttrpc::{asynchronous::TtrpcContext, Code, Error, Status}; use crate::{ api::{ GetResourceRequest, GetResourceResponse, KeyProviderKeyWrapProtocolInput, - KeyProviderKeyWrapProtocolOutput, UnsealSecretInput, UnsealSecretOutput, + KeyProviderKeyWrapProtocolOutput, SecureMountRequest, SecureMountResponse, + UnsealSecretInput, UnsealSecretOutput, }, - api_ttrpc::{GetResourceService, KeyProviderService, SealedSecretService}, + api_ttrpc::{GetResourceService, KeyProviderService, SealedSecretService, SecureMountService}, server::message::{KeyProviderInput, KeyUnwrapOutput, KeyUnwrapResults}, }; @@ -151,3 +153,35 @@ impl KeyProviderService for Server { Ok(reply) } } + +#[async_trait] +impl SecureMountService for Server { + async fn secure_mount( + &self, + _ctx: &TtrpcContext, + req: SecureMountRequest, + ) -> ::ttrpc::Result { + debug!("get new Secure mount request"); + let reader = HUB.read().await; + let reader = reader.as_ref().expect("must be initialized"); + let storage = Storage { + driver: req.driver, + driver_options: req.driver_options, + source: req.source, + fstype: req.fstype, + options: req.options, + mount_point: req.mount_point, + }; + let resource = reader.secure_mount(storage).await.map_err(|e| { + let mut status = Status::new(); + status.set_code(Code::INTERNAL); + status.set_message(format!("[CDH] [ERROR]: secure mount failed: {e}")); + Error::RpcStatus(status) + })?; + + let mut reply = SecureMountResponse::new(); + reply.mount_path = resource; + debug!("send back the resource"); + Ok(reply) + } +} diff --git a/confidential-data-hub/hub/src/error.rs b/confidential-data-hub/hub/src/error.rs index 15169480d..07abc2bf2 100644 --- a/confidential-data-hub/hub/src/error.rs +++ b/confidential-data-hub/hub/src/error.rs @@ -20,4 +20,7 @@ pub enum Error { #[error("unseal secret failed: {0}")] UnsealSecret(String), + + #[error("secure mount failed: {0}")] + SecureMount(String), } diff --git a/confidential-data-hub/hub/src/hub.rs b/confidential-data-hub/hub/src/hub.rs index 704fc75fb..a6e95f2d1 100644 --- a/confidential-data-hub/hub/src/hub.rs +++ b/confidential-data-hub/hub/src/hub.rs @@ -8,6 +8,7 @@ use base64::{engine::general_purpose::STANDARD, Engine}; use image::AnnotationPacket; use kms::{Annotations, ProviderSettings}; use secret::secret::Secret; +use storage::volume_type::Storage; use crate::{DataHub, Error, Result}; @@ -73,4 +74,12 @@ impl DataHub for Hub { .map_err(|e| Error::GetResource(format!("get rersource failed: {e}")))?; Ok(res) } + + async fn secure_mount(&self, storage: Storage) -> Result { + let res = storage + .mount() + .await + .map_err(|e| Error::SecureMount(e.to_string()))?; + Ok(res) + } } diff --git a/confidential-data-hub/storage/Cargo.toml b/confidential-data-hub/storage/Cargo.toml new file mode 100644 index 000000000..fdcd5b685 --- /dev/null +++ b/confidential-data-hub/storage/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "storage" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +serde = "1" +serde_json = "1" +thiserror.workspace = true +tokio = { workspace = true, features = ["fs"] } +anyhow.workspace = true +secret = { path = "../secret" } +base64.workspace = true +log.workspace = true + +[dev-dependencies] +rstest.workspace = true +tokio = { workspace = true, features = ["rt", "macros" ] } + +[build-dependencies] +anyhow.workspace = true + +[features] +default = ["aliyun"] +aliyun = [] diff --git a/confidential-data-hub/storage/src/error.rs b/confidential-data-hub/storage/src/error.rs new file mode 100644 index 000000000..ed29ab401 --- /dev/null +++ b/confidential-data-hub/storage/src/error.rs @@ -0,0 +1,20 @@ +// Copyright (c) 2023 Intel +// +// SPDX-License-Identifier: Apache-2.0 +// + +use thiserror::Error; + +pub type Result = std::result::Result; + +#[derive(Error, Debug)] +pub enum Error { + #[error("secure mount failed: {0}")] + SecureMountFailed(String), + + #[error("file error: {0}")] + FileError(String), + + #[error("unseal secret failed: {0}")] + UnsealSecretFailed(String), +} diff --git a/confidential-data-hub/storage/src/lib.rs b/confidential-data-hub/storage/src/lib.rs new file mode 100644 index 000000000..391985321 --- /dev/null +++ b/confidential-data-hub/storage/src/lib.rs @@ -0,0 +1,9 @@ +// Copyright (c) 2023 Intel +// +// SPDX-License-Identifier: Apache-2.0 +// + +pub mod error; +pub mod volume_type; + +pub use error::*; diff --git a/confidential-data-hub/storage/src/volume_type/alibaba_cloud_oss/mod.rs b/confidential-data-hub/storage/src/volume_type/alibaba_cloud_oss/mod.rs new file mode 100644 index 000000000..aa50157f8 --- /dev/null +++ b/confidential-data-hub/storage/src/volume_type/alibaba_cloud_oss/mod.rs @@ -0,0 +1,6 @@ +// Copyright (c) 2023 Intel +// +// SPDX-License-Identifier: Apache-2.0 +// + +pub mod oss; diff --git a/confidential-data-hub/storage/src/volume_type/alibaba_cloud_oss/oss.rs b/confidential-data-hub/storage/src/volume_type/alibaba_cloud_oss/oss.rs new file mode 100644 index 000000000..42cf891d9 --- /dev/null +++ b/confidential-data-hub/storage/src/volume_type/alibaba_cloud_oss/oss.rs @@ -0,0 +1,166 @@ +// Copyright (c) 2023 Intel +// +// SPDX-License-Identifier: Apache-2.0 +// + +use base64::{engine::general_purpose::STANDARD, Engine}; +use secret::secret::Secret; +use serde::{Deserialize, Serialize}; +use std::fs; +use std::fs::File; +use std::io::Write; +use std::os::unix::fs::PermissionsExt; +use std::process::Command; + +use crate::{Error, Result}; + +const OSSFS_PASSWD_FILE: &str = "/tmp/ossfs_passwd"; +const GOCRYPTFS_PASSWD_FILE: &str = "/tmp/gocryptfs_passwd"; +const OSSFS_BIN: &str = "/usr/local/bin/ossfs"; +const GOCRYPTFS_BIN: &str = "/usr/local/bin/gocryptfs"; + +#[derive(Serialize, Deserialize, PartialEq, Debug)] +pub struct Oss { + #[serde(rename = "akId")] + pub ak_id: String, + #[serde(rename = "akSecret")] + pub ak_secret: String, + #[serde(default)] + pub annotations: String, + pub bucket: String, + #[serde(default)] + pub encrypted: String, + #[serde(rename = "encPasswd", default)] + pub enc_passwd: String, + #[serde(rename = "kmsKeyId", default)] + pub kms_key_id: String, + #[serde(rename = "otherOpts")] + pub other_opts: String, + pub path: String, + pub readonly: String, + #[serde(rename = "targetPath")] + pub target_path: String, + pub url: String, + #[serde(rename = "volumeId")] + pub volume_id: String, +} + +async fn unseal_secret(secret: Vec) -> Result> { + // TODO: verify the jws signature using the key specified by `kid` + // in header. Here we directly get the JWS payload + let payload = secret.split(|c| *c == b'.').nth(1).ok_or_else(|| { + Error::SecureMountFailed("illegal input sealed secret (not a JWS)".into()) + })?; + + let secret_json = STANDARD.decode(payload).map_err(|e| { + Error::SecureMountFailed(format!( + "illegal input sealed secret (JWS body is not standard base64 encoded): {e}" + )) + })?; + let secret: Secret = serde_json::from_slice(&secret_json).map_err(|e| { + Error::SecureMountFailed(format!( + "illegal input sealed secret format (json deseralization failed): {e}" + )) + })?; + + let res = secret + .unseal() + .await + .map_err(|e| Error::UnsealSecretFailed(format!("unseal failed: {e}")))?; + Ok(res) +} + +async fn get_plaintext_secret(secret: &str) -> Result { + if secret.starts_with("sealed.") { + let tmp = secret + .strip_prefix("sealed.") + .ok_or(Error::SecureMountFailed( + "strip_prefix \"sealed.\" failed".to_string(), + ))?; + let unsealed = unseal_secret(tmp.into()).await?; + + return String::from_utf8(unsealed) + .map_err(|e| Error::SecureMountFailed(format!("convert to String failed: {e}"))); + } + Err(Error::SecureMountFailed( + "sealed secret format error!".to_string(), + )) +} + +impl Oss { + pub(crate) async fn mount(&self, source: String, mount_point: String) -> Result { + // unseal secret + let plain_ak_id = get_plaintext_secret(&self.ak_id).await?; + let plain_ak_secret = get_plaintext_secret(&self.ak_secret).await?; + + // create ossfs passwd file + let mut ossfs_passwd = File::create(OSSFS_PASSWD_FILE) + .map_err(|e| Error::FileError(format!("create file failed: {e}")))?; + let metadata = ossfs_passwd + .metadata() + .map_err(|e| Error::FileError(format!("create metadata failed: {e}")))?; + let mut permissions = metadata.permissions(); + permissions.set_mode(0o600); + ossfs_passwd + .set_permissions(permissions) + .map_err(|e| Error::FileError(format!("set permissions failed: {e}")))?; + ossfs_passwd + .write_all(format!("{}:{}:{}", self.bucket, plain_ak_id, plain_ak_secret).as_bytes()) + .map_err(|e| Error::FileError(format!("write file failed: {e}")))?; + + // generate parameters for ossfs command, and execute + let mut opts = self + .other_opts + .split_whitespace() + .map(str::to_string) + .collect(); + let s = if self.encrypted == "gocryptfs" { + fs::create_dir_all("/tmp/oss") + .map_err(|e| Error::FileError(format!("create dir failed: {e}")))?; + "/tmp/oss/".to_string() + } else { + source.clone() + }; + let mut parameters = vec![ + format!("{}:{}", self.bucket, self.path), + s.clone(), + format!("-ourl={}", self.url), + format!("-opasswd_file={}", OSSFS_PASSWD_FILE), + ]; + parameters.append(&mut opts); + + Command::new(OSSFS_BIN) + .args(parameters) + .spawn() + .expect("failed to mount oss"); + std::thread::sleep(std::time::Duration::from_secs(3)); + + // decrypt with gocryptfs if needed + if self.encrypted == "gocryptfs" { + // unseal secret + let plain_passwd = get_plaintext_secret(&self.enc_passwd).await?; + + // create gocryptfs passwd file + let mut gocryptfs_passwd = File::create(GOCRYPTFS_PASSWD_FILE) + .map_err(|e| Error::FileError(format!("create file failed: {e}")))?; + gocryptfs_passwd + .write_all(plain_passwd.as_bytes()) + .map_err(|e| Error::FileError(format!("write file failed: {e}")))?; + + // generate parameters for gocryptfs, and execute + let parameters = vec![ + s, + source, + "-passfile".to_string(), + GOCRYPTFS_PASSWD_FILE.to_string(), + "-nosyslog".to_string(), + ]; + Command::new(GOCRYPTFS_BIN) + .args(parameters) + .spawn() + .expect("failed to decrypt oss"); + std::thread::sleep(std::time::Duration::from_secs(3)); + } + Ok(mount_point) + } +} diff --git a/confidential-data-hub/storage/src/volume_type/mod.rs b/confidential-data-hub/storage/src/volume_type/mod.rs new file mode 100644 index 000000000..b0daa76cf --- /dev/null +++ b/confidential-data-hub/storage/src/volume_type/mod.rs @@ -0,0 +1,55 @@ +// Copyright (c) 2023 Intel +// +// SPDX-License-Identifier: Apache-2.0 +// + +#[cfg(feature = "aliyun")] +pub mod alibaba_cloud_oss; + +#[cfg(feature = "aliyun")] +use self::alibaba_cloud_oss::oss::Oss; +use crate::{Error, Result}; +use log::warn; + +#[derive(PartialEq, Clone, Debug)] +pub struct Storage { + pub driver: String, + pub driver_options: Vec, + pub source: String, + pub fstype: String, + pub options: Vec, + pub mount_point: String, +} + +impl Storage { + pub async fn mount(&self) -> Result { + for driver_option in &self.driver_options { + let (volume_type, metadata) = + driver_option + .split_once('=') + .ok_or(Error::SecureMountFailed( + "split by \"=\" failed".to_string(), + ))?; + + match volume_type { + #[cfg(feature = "aliyun")] + "alibaba-cloud-oss" => { + let oss: Oss = serde_json::from_str(metadata).map_err(|e| { + Error::SecureMountFailed(format!( + "illegal mount info format (json deseralization failed): {e}" + )) + })?; + return oss + .mount(self.source.clone(), self.mount_point.clone()) + .await; + } + other => { + warn!("skip mount info with unsupported volume_type: {other}"); + } + }; + } + Err(Error::SecureMountFailed( + "illegal mount info as no expected driver_options".to_string(), + )) + } +}