From b6da2b681528ccd82e94f8f0186460e36944f04f Mon Sep 17 00:00:00 2001 From: David Estes Date: Mon, 20 May 2024 17:29:18 -0600 Subject: [PATCH] feat: support configuring persistent storage for containers --- keramik/src/advanced_configuration.md | 39 ++++++++++ operator/src/network/cas.rs | 99 ++++++++++-------------- operator/src/network/ceramic.rs | 61 ++++++--------- operator/src/network/controller.rs | 107 +++++++++++++++++++++++++- operator/src/network/ipfs.rs | 40 ++++++---- operator/src/network/mod.rs | 2 + operator/src/network/spec.rs | 31 ++++++++ operator/src/network/storage.rs | 45 +++++++++++ 8 files changed, 312 insertions(+), 112 deletions(-) create mode 100644 operator/src/network/storage.rs diff --git a/keramik/src/advanced_configuration.md b/keramik/src/advanced_configuration.md index d43810d5..fc9e9c9e 100644 --- a/keramik/src/advanced_configuration.md +++ b/keramik/src/advanced_configuration.md @@ -70,6 +70,45 @@ and adding the following environment variables to the `spec/template/spec/contai # Image Resources +## Storage + +Nearly all containers (monitoring outstanding), allow configuring the peristent storage size and class. The storage class must be created out of band, but can be included. The storage configuration has two keys (`size` and `class`) and can be used like so: + +```yaml +apiVersion: "keramik.3box.io/v1alpha1" +kind: Network +metadata: + name: small +spec: + replicas: 2 + bootstrap: + image: keramik/runner:dev + imagePullPolicy: IfNotPresent + cas: + casStorage: + size: "3Gi" + class: "fastDisk" # typically not set + ipfs: + go: + storage: + size: "1Gi" + ganacheStorage: + size: "1Gi" + postgresStorage: + size: "3Gi" + localstackStorage: + size: "5Gi" + ceramic: + - ipfs: + rust: + storage: + size: "3Gi" + +``` + + +## Requests / Limits + During local benchmarking, you may not have enough resources to run the cluster. A simple "fix" is to use the `devMode` flag on the network and simulation specs. This will override the resource requests and limits values to be none, which means it doesn't need available resources to deploy, and can consume as much as it desires. This would be problematic in production and should only be used for testing purposes. ```yaml diff --git a/operator/src/network/cas.rs b/operator/src/network/cas.rs index 2647f7e8..b84c5cc6 100644 --- a/operator/src/network/cas.rs +++ b/operator/src/network/cas.rs @@ -5,9 +5,8 @@ use k8s_openapi::{ apps::v1::StatefulSetSpec, core::v1::{ Container, ContainerPort, EnvVar, EnvVarSource, PersistentVolumeClaim, - PersistentVolumeClaimSpec, PersistentVolumeClaimVolumeSource, PodSecurityContext, - PodSpec, PodTemplateSpec, ResourceRequirements, SecretKeySelector, ServicePort, - ServiceSpec, Volume, VolumeMount, + PersistentVolumeClaimVolumeSource, PodSecurityContext, PodSpec, PodTemplateSpec, + ResourceRequirements, SecretKeySelector, ServicePort, ServiceSpec, Volume, VolumeMount, }, }, apimachinery::pkg::{ @@ -30,6 +29,7 @@ use crate::{ ipfs::{IpfsConfig, IpfsInfo, IPFS_DATA_PV_CLAIM}, node_affinity::NodeAffinityConfig, resource_limits::ResourceLimitsConfig, + storage::PersistentStorageConfig, CasApiSpec, CasSpec, }, utils::override_and_sort_env_vars, @@ -42,9 +42,13 @@ pub struct CasConfig { pub image_pull_policy: String, pub ipfs: IpfsConfig, pub cas_resource_limits: ResourceLimitsConfig, + pub cas_storage: PersistentStorageConfig, pub ganache_resource_limits: ResourceLimitsConfig, + pub ganache_storage: PersistentStorageConfig, pub postgres_resource_limits: ResourceLimitsConfig, + pub postgres_storage: PersistentStorageConfig, pub localstack_resource_limits: ResourceLimitsConfig, + pub localstack_storage: PersistentStorageConfig, pub api: CasApiConfig, } @@ -86,22 +90,38 @@ impl Default for CasConfig { memory: Some(Quantity("1Gi".to_owned())), storage: Quantity("1Gi".to_owned()), }, + cas_storage: PersistentStorageConfig { + size: Quantity("10Gi".to_owned()), + class: None, + }, ipfs: Default::default(), ganache_resource_limits: ResourceLimitsConfig { cpu: Some(Quantity("250m".to_owned())), memory: Some(Quantity("1Gi".to_owned())), storage: Quantity("1Gi".to_owned()), }, + ganache_storage: PersistentStorageConfig { + size: Quantity("10Gi".to_owned()), + class: None, + }, postgres_resource_limits: ResourceLimitsConfig { cpu: Some(Quantity("250m".to_owned())), memory: Some(Quantity("512Mi".to_owned())), storage: Quantity("1Gi".to_owned()), }, + postgres_storage: PersistentStorageConfig { + size: Quantity("10Gi".to_owned()), + class: None, + }, localstack_resource_limits: ResourceLimitsConfig { cpu: Some(Quantity("250m".to_owned())), memory: Some(Quantity("1Gi".to_owned())), storage: Quantity("1Gi".to_owned()), }, + localstack_storage: PersistentStorageConfig { + size: Quantity("10Gi".to_owned()), + class: None, + }, api: Default::default(), } } @@ -126,19 +146,32 @@ impl From for CasConfig { value.cas_resource_limits, default.cas_resource_limits, ), + cas_storage: PersistentStorageConfig::from_spec(value.cas_storage, default.cas_storage), ipfs: value.ipfs.map(Into::into).unwrap_or(default.ipfs), ganache_resource_limits: ResourceLimitsConfig::from_spec( value.ganache_resource_limits, default.ganache_resource_limits, ), + ganache_storage: PersistentStorageConfig::from_spec( + value.ganache_storage, + default.ganache_storage, + ), postgres_resource_limits: ResourceLimitsConfig::from_spec( value.postgres_resource_limits, default.postgres_resource_limits, ), + postgres_storage: PersistentStorageConfig::from_spec( + value.postgres_storage, + default.postgres_storage, + ), localstack_resource_limits: ResourceLimitsConfig::from_spec( value.localstack_resource_limits, default.localstack_resource_limits, ), + localstack_storage: PersistentStorageConfig::from_spec( + value.localstack_storage, + default.localstack_storage, + ), api: value.api.map(Into::into).unwrap_or(default.api), } } @@ -504,17 +537,7 @@ pub fn cas_stateful_set_spec( name: Some("cas-data".to_owned()), ..Default::default() }, - spec: Some(PersistentVolumeClaimSpec { - access_modes: Some(vec!["ReadWriteOnce".to_owned()]), - resources: Some(ResourceRequirements { - requests: Some(BTreeMap::from_iter(vec![( - "storage".to_owned(), - Quantity("10Gi".to_owned()), - )])), - ..Default::default() - }), - ..Default::default() - }), + spec: Some(config.cas_storage.clone().into()), ..Default::default() }]), ..Default::default() @@ -582,17 +605,7 @@ pub fn cas_ipfs_stateful_set_spec( name: Some(IPFS_DATA_PV_CLAIM.to_owned()), ..Default::default() }, - spec: Some(PersistentVolumeClaimSpec { - access_modes: Some(vec!["ReadWriteOnce".to_owned()]), - resources: Some(ResourceRequirements { - requests: Some(BTreeMap::from_iter(vec![( - "storage".to_owned(), - Quantity("10Gi".to_owned()), - )])), - ..Default::default() - }), - ..Default::default() - }), + spec: Some(config.ipfs.storage_config().clone().into()), ..Default::default() }]), ..Default::default() @@ -676,17 +689,7 @@ pub fn ganache_stateful_set_spec( name: Some("ganache-data".to_owned()), ..Default::default() }, - spec: Some(PersistentVolumeClaimSpec { - access_modes: Some(vec!["ReadWriteOnce".to_owned()]), - resources: Some(ResourceRequirements { - requests: Some(BTreeMap::from_iter(vec![( - "storage".to_owned(), - Quantity("10Gi".to_owned()), - )])), - ..Default::default() - }), - ..Default::default() - }), + spec: Some(config.ganache_storage.clone().into()), ..Default::default() }]), ..Default::default() @@ -799,17 +802,7 @@ pub fn postgres_stateful_set_spec( name: Some("postgres-data".to_owned()), ..Default::default() }, - spec: Some(PersistentVolumeClaimSpec { - access_modes: Some(vec!["ReadWriteOnce".to_owned()]), - resources: Some(ResourceRequirements { - requests: Some(BTreeMap::from_iter(vec![( - "storage".to_owned(), - Quantity("10Gi".to_owned()), - )])), - ..Default::default() - }), - ..Default::default() - }), + spec: Some(config.postgres_storage.clone().into()), ..Default::default() }]), ..Default::default() @@ -883,17 +876,7 @@ pub fn localstack_stateful_set_spec( name: Some("localstack-data".to_owned()), ..Default::default() }, - spec: Some(PersistentVolumeClaimSpec { - access_modes: Some(vec!["ReadWriteOnce".to_owned()]), - resources: Some(ResourceRequirements { - requests: Some(BTreeMap::from_iter(vec![( - "storage".to_owned(), - Quantity("10Gi".to_owned()), - )])), - ..Default::default() - }), - ..Default::default() - }), + spec: Some(config.localstack_storage.clone().into()), ..Default::default() }]), ..Default::default() diff --git a/operator/src/network/ceramic.rs b/operator/src/network/ceramic.rs index 6752deb7..4325f939 100644 --- a/operator/src/network/ceramic.rs +++ b/operator/src/network/ceramic.rs @@ -5,10 +5,9 @@ use k8s_openapi::{ apps::v1::{RollingUpdateStatefulSetStrategy, StatefulSetSpec, StatefulSetUpdateStrategy}, core::v1::{ ConfigMapVolumeSource, Container, ContainerPort, EmptyDirVolumeSource, EnvVar, - EnvVarSource, HTTPGetAction, PersistentVolumeClaim, PersistentVolumeClaimSpec, - PersistentVolumeClaimVolumeSource, PodSecurityContext, PodSpec, PodTemplateSpec, Probe, - ResourceRequirements, SecretKeySelector, SecurityContext, ServicePort, ServiceSpec, - Volume, VolumeMount, + EnvVarSource, HTTPGetAction, PersistentVolumeClaim, PersistentVolumeClaimVolumeSource, + PodSecurityContext, PodSpec, PodTemplateSpec, Probe, ResourceRequirements, + SecretKeySelector, SecurityContext, ServicePort, ServiceSpec, Volume, VolumeMount, }, }, apimachinery::pkg::{ @@ -34,7 +33,7 @@ use crate::{ utils::override_and_sort_env_vars, }; -use super::debug_mode_security_context; +use super::{debug_mode_security_context, storage::PersistentStorageConfig}; pub fn config_maps( info: &CeramicInfo, @@ -143,7 +142,9 @@ pub struct CeramicConfig { pub image_pull_policy: String, pub ipfs: IpfsConfig, pub resource_limits: ResourceLimitsConfig, + pub storage: PersistentStorageConfig, pub postgres_resource_limits: ResourceLimitsConfig, + pub postgres_storage: PersistentStorageConfig, pub env: Option>, } @@ -268,11 +269,19 @@ impl Default for CeramicConfig { memory: Some(Quantity("1Gi".to_owned())), storage: Quantity("1Gi".to_owned()), }, + storage: PersistentStorageConfig { + size: Quantity("10Gi".to_owned()), + class: None, + }, postgres_resource_limits: ResourceLimitsConfig { cpu: Some(Quantity("250m".to_owned())), memory: Some(Quantity("1Gi".to_owned())), storage: Quantity("1Gi".to_owned()), }, + postgres_storage: PersistentStorageConfig { + size: Quantity("10Gi".to_owned()), + class: None, + }, env: None, } } @@ -308,10 +317,15 @@ impl From for CeramicConfig { value.resource_limits, default.resource_limits, ), + storage: PersistentStorageConfig::from_spec(value.storage, default.storage), postgres_resource_limits: ResourceLimitsConfig::from_spec( value.postgres_resource_limits, default.postgres_resource_limits, ), + postgres_storage: PersistentStorageConfig::from_spec( + value.postgres_storage, + default.postgres_storage, + ), env: value.env, } } @@ -700,17 +714,7 @@ pub fn stateful_set_spec(ns: &str, bundle: &CeramicBundle<'_>) -> StatefulSetSpe name: Some("ceramic-data".to_owned()), ..Default::default() }, - spec: Some(PersistentVolumeClaimSpec { - access_modes: Some(vec!["ReadWriteOnce".to_owned()]), - resources: Some(ResourceRequirements { - requests: Some(BTreeMap::from_iter(vec![( - "storage".to_owned(), - Quantity("10Gi".to_owned()), - )])), - ..Default::default() - }), - ..Default::default() - }), + spec: Some(bundle.config.storage.clone().into()), ..Default::default() }, PersistentVolumeClaim { @@ -718,18 +722,7 @@ pub fn stateful_set_spec(ns: &str, bundle: &CeramicBundle<'_>) -> StatefulSetSpe name: Some(IPFS_DATA_PV_CLAIM.to_owned()), ..Default::default() }, - spec: Some(PersistentVolumeClaimSpec { - access_modes: Some(vec!["ReadWriteOnce".to_owned()]), - resources: Some(ResourceRequirements { - requests: Some(BTreeMap::from_iter(vec![( - "storage".to_owned(), - Quantity("10Gi".to_owned()), - )])), - ..Default::default() - }), - storage_class_name: bundle.config.ipfs.storage_class_name(), - ..Default::default() - }), + spec: Some(bundle.config.ipfs.storage_config().clone().into()), ..Default::default() }, PersistentVolumeClaim { @@ -737,17 +730,7 @@ pub fn stateful_set_spec(ns: &str, bundle: &CeramicBundle<'_>) -> StatefulSetSpe name: Some("postgres-data".to_owned()), ..Default::default() }, - spec: Some(PersistentVolumeClaimSpec { - access_modes: Some(vec!["ReadWriteOnce".to_owned()]), - resources: Some(ResourceRequirements { - requests: Some(BTreeMap::from_iter(vec![( - "storage".to_owned(), - Quantity("10Gi".to_owned()), - )])), - ..Default::default() - }), - ..Default::default() - }), + spec: Some(bundle.config.postgres_storage.clone().into()), ..Default::default() }, ]), diff --git a/operator/src/network/controller.rs b/operator/src/network/controller.rs index 22b0960d..84eda162 100644 --- a/operator/src/network/controller.rs +++ b/operator/src/network/controller.rs @@ -1087,8 +1087,8 @@ mod tests { ipfs_rpc::{tests::MockIpfsRpcClientTest, Peer}, stub::{CeramicStub, Stub}, BootstrapSpec, CasSpec, CeramicSpec, DataDogSpec, GoIpfsSpec, IpfsSpec, MonitoringSpec, - NetworkSpec, NetworkStatus, NetworkType, PodMonitorSpec, ResourceLimitsSpec, - RustIpfsSpec, + NetworkSpec, NetworkStatus, NetworkType, PersistentStorageSpec, PodMonitorSpec, + ResourceLimitsSpec, RustIpfsSpec, }, utils::{ test::{timeout_after_1s, ApiServerVerifier, WithStatus}, @@ -2384,6 +2384,10 @@ mod tests { memory: Some(Quantity("4Gi".to_owned())), storage: Some(Quantity("4Gi".to_owned())), }), + storage: Some(PersistentStorageSpec { + size: Some(Quantity("1Gi".to_owned())), + class: None, + }), ..Default::default() })), ..Default::default() @@ -2512,6 +2516,15 @@ mod tests { } ] } + @@ -470,7 +444,7 @@ + ], + "resources": { + "requests": { + - "storage": "10Gi" + + "storage": "1Gi" + } + } + } "#]]); let (testctx, api_handle) = Context::test(mock_rpc_client); let fakeserver = ApiServerVerifier::new(api_handle); @@ -2685,6 +2698,10 @@ mod tests { memory: Some(Quantity("4Gi".to_owned())), storage: Some(Quantity("4Gi".to_owned())), }), + storage: Some(PersistentStorageSpec { + size: Some(Quantity("1Gi".to_owned())), + class: Some("fastDisk".to_owned()), + }), env: Some(BTreeMap::from_iter([ ("ENV_KEY_A".to_string(), "ENV_VALUE_A".to_string()), ("ENV_KEY_B".to_string(), "ENV_VALUE_B".to_string()), @@ -2773,6 +2790,19 @@ mod tests { } }, "volumeMounts": [ + @@ -470,9 +482,10 @@ + ], + "resources": { + "requests": { + - "storage": "10Gi" + + "storage": "1Gi" + } + - } + + }, + + "storageClassName": "fastDisk" + } + }, + { "#]]); let (testctx, api_handle) = Context::test(mock_rpc_client); let fakeserver = ApiServerVerifier::new(api_handle); @@ -2870,12 +2900,20 @@ mod tests { memory: Some(Quantity("1Gi".to_owned())), storage: Some(Quantity("1Gi".to_owned())), }), + cas_storage: Some(PersistentStorageSpec { + size: Some(Quantity("2Gi".to_owned())), + class: None, + }), ipfs: Some(IpfsSpec::Rust(RustIpfsSpec { resource_limits: Some(ResourceLimitsSpec { cpu: Some(Quantity("2".to_owned())), memory: Some(Quantity("2Gi".to_owned())), storage: Some(Quantity("2Gi".to_owned())), }), + storage: Some(PersistentStorageSpec { + size: Some(Quantity("3Gi".to_owned())), + class: Some("fastDisk".to_owned()), + }), ..Default::default() })), ganache_resource_limits: Some(ResourceLimitsSpec { @@ -2883,11 +2921,19 @@ mod tests { memory: Some(Quantity("3Gi".to_owned())), storage: Some(Quantity("3Gi".to_owned())), }), + ganache_storage: Some(PersistentStorageSpec { + size: Some(Quantity("4Gi".to_owned())), + class: Some("fastDisk".to_owned()), + }), postgres_resource_limits: Some(ResourceLimitsSpec { cpu: Some(Quantity("4".to_owned())), memory: Some(Quantity("4Gi".to_owned())), storage: Some(Quantity("4Gi".to_owned())), }), + postgres_storage: Some(PersistentStorageSpec { + size: Some(Quantity("5Gi".to_owned())), + class: None, + }), ..Default::default() }), ..Default::default() @@ -2960,6 +3006,15 @@ mod tests { "ephemeral-storage": "1Gi", "memory": "1Gi" } + @@ -470,7 +470,7 @@ + ], + "resources": { + "requests": { + - "storage": "10Gi" + + "storage": "2Gi" + } + } + } "#]]); stub.cas_ipfs_stateful_set.patch(expect![[r#" --- original @@ -2985,6 +3040,19 @@ mod tests { } }, "volumeMounts": [ + @@ -136,9 +136,10 @@ + ], + "resources": { + "requests": { + - "storage": "10Gi" + + "storage": "3Gi" + } + - } + + }, + + "storageClassName": "fastDisk" + } + } + ] "#]]); stub.ganache_stateful_set.patch(expect![[r#" --- original @@ -3010,6 +3078,19 @@ mod tests { } }, "volumeMounts": [ + @@ -92,9 +92,10 @@ + ], + "resources": { + "requests": { + - "storage": "10Gi" + + "storage": "4Gi" + } + - } + + }, + + "storageClassName": "fastDisk" + } + } + ] "#]]); stub.cas_postgres_stateful_set.patch(expect![[r#" --- original @@ -3035,6 +3116,15 @@ mod tests { } }, "volumeMounts": [ + @@ -114,7 +114,7 @@ + ], + "resources": { + "requests": { + - "storage": "10Gi" + + "storage": "5Gi" + } + } + } "#]]); let (testctx, api_handle) = Context::test(mock_rpc_client); let fakeserver = ApiServerVerifier::new(api_handle); @@ -3055,6 +3145,10 @@ mod tests { memory: Some(Quantity("4Gi".to_owned())), storage: Some(Quantity("4Gi".to_owned())), }), + storage: Some(PersistentStorageSpec { + size: Some(Quantity("100Gi".to_owned())), + class: None, + }), ..Default::default() }]), ..Default::default() @@ -3124,6 +3218,15 @@ mod tests { } }, "volumeMounts": [ + @@ -453,7 +453,7 @@ + ], + "resources": { + "requests": { + - "storage": "10Gi" + + "storage": "100Gi" + } + } + } "#]]); let (testctx, api_handle) = Context::test(mock_rpc_client); let fakeserver = ApiServerVerifier::new(api_handle); diff --git a/operator/src/network/ipfs.rs b/operator/src/network/ipfs.rs index 1759996a..a3562a95 100644 --- a/operator/src/network/ipfs.rs +++ b/operator/src/network/ipfs.rs @@ -15,8 +15,8 @@ const IPFS_SERVICE_PORT: i32 = 5001; use crate::{ network::{ ceramic::NetworkConfig, controller::NETWORK_DEV_MODE_RESOURCES, - resource_limits::ResourceLimitsConfig, GoIpfsSpec, IpfsSpec, RustIpfsSpec, - NETWORK_LOCAL_ID, + resource_limits::ResourceLimitsConfig, storage::PersistentStorageConfig, GoIpfsSpec, + IpfsSpec, RustIpfsSpec, NETWORK_LOCAL_ID, }, utils::override_and_sort_env_vars, }; @@ -83,10 +83,10 @@ impl IpfsConfig { IpfsConfig::Go(config) => config.volumes(&info), } } - pub fn storage_class_name(&self) -> Option { + pub fn storage_config(&self) -> &PersistentStorageConfig { match self { - IpfsConfig::Rust(r) => r.storage_class.clone(), - IpfsConfig::Go(g) => g.storage_class.clone(), + IpfsConfig::Rust(r) => &r.storage, + IpfsConfig::Go(g) => &g.storage, } } } @@ -95,7 +95,7 @@ pub struct RustIpfsConfig { image: String, image_pull_policy: String, resource_limits: ResourceLimitsConfig, - storage_class: Option, + storage: PersistentStorageConfig, rust_log: String, env: Option>, } @@ -123,7 +123,10 @@ impl Default for RustIpfsConfig { memory: Some(Quantity("1Gi".to_owned())), storage: Quantity("1Gi".to_owned()), }, - storage_class: None, + storage: PersistentStorageConfig { + size: Quantity("10Gi".to_owned()), + class: None, + }, rust_log: "info,ceramic_one=debug,multipart=error".to_owned(), env: None, } @@ -131,7 +134,11 @@ impl Default for RustIpfsConfig { } impl From for RustIpfsConfig { fn from(value: RustIpfsSpec) -> Self { - let default = RustIpfsConfig::network_default(); + let mut default = RustIpfsConfig::network_default(); + // we prefer the value from PersistentStorageConfig but will fall back to this if provided + if let Some(class) = value.storage_class { + default.storage.class = Some(class); + } Self { image: value.image.unwrap_or(default.image), image_pull_policy: value.image_pull_policy.unwrap_or(default.image_pull_policy), @@ -139,8 +146,8 @@ impl From for RustIpfsConfig { value.resource_limits, default.resource_limits, ), + storage: PersistentStorageConfig::from_spec(value.storage, default.storage), rust_log: value.rust_log.unwrap_or(default.rust_log), - storage_class: value.storage_class, env: value.env, } } @@ -260,7 +267,7 @@ pub struct GoIpfsConfig { image: String, image_pull_policy: String, resource_limits: ResourceLimitsConfig, - storage_class: Option, + storage: PersistentStorageConfig, commands: Vec, } @@ -287,14 +294,21 @@ impl Default for GoIpfsConfig { memory: Some(Quantity("512Mi".to_owned())), storage: Quantity("1Gi".to_owned()), }, - storage_class: None, + storage: PersistentStorageConfig { + size: Quantity("10Gi".to_owned()), + class: None, + }, commands: vec![], } } } impl From for GoIpfsConfig { fn from(value: GoIpfsSpec) -> Self { - let default = GoIpfsConfig::network_default(); + let mut default = GoIpfsConfig::network_default(); + // we prefer the value from PersistentStorageConfig but will fall back to this if provided + if let Some(class) = value.storage_class { + default.storage.class = Some(class); + } Self { image: value.image.unwrap_or(default.image), image_pull_policy: value.image_pull_policy.unwrap_or(default.image_pull_policy), @@ -302,7 +316,7 @@ impl From for GoIpfsConfig { value.resource_limits, default.resource_limits, ), - storage_class: value.storage_class, + storage: PersistentStorageConfig::from_spec(value.storage, default.storage), commands: value.commands.unwrap_or(default.commands), } } diff --git a/operator/src/network/mod.rs b/operator/src/network/mod.rs index 70db7dc8..bd1ecfd5 100644 --- a/operator/src/network/mod.rs +++ b/operator/src/network/mod.rs @@ -25,6 +25,8 @@ mod node_affinity; pub(crate) mod peers; #[cfg(feature = "controller")] pub(crate) mod resource_limits; +#[cfg(feature = "controller")] +pub(crate) mod storage; #[cfg(test)] #[cfg(feature = "controller")] diff --git a/operator/src/network/spec.rs b/operator/src/network/spec.rs index 3c98be8b..66b68852 100644 --- a/operator/src/network/spec.rs +++ b/operator/src/network/spec.rs @@ -161,8 +161,12 @@ pub struct CeramicSpec { pub ipfs: Option, /// Resource limits for ceramic nodes, applies to both requests and limits. pub resource_limits: Option, + /// Storage configuration for the ceramic container. + pub storage: Option, /// Resource limits for postgres container in ceramic nodes, applies to both requests and limits. pub postgres_resource_limits: Option, + /// Storage configuration for the postgres container. + pub postgres_storage: Option, /// Extra env values to pass to the image. /// CAUTION: Any env vars specified in this set will override any predefined values. pub env: Option>, @@ -171,6 +175,9 @@ pub struct CeramicSpec { /// Describes how the IPFS node for a peer should behave. #[derive(Serialize, Deserialize, Debug, PartialEq, Clone, JsonSchema)] #[serde(rename_all = "camelCase")] +#[allow(clippy::large_enum_variant)] +// Clippy seems be warning for a false positive. +// It's saying 0 and at least 216, when it should be at least 248 for RustIpfsSpec. pub enum IpfsSpec { /// Rust IPFS specification Rust(RustIpfsSpec), @@ -188,6 +195,8 @@ pub struct RustIpfsSpec { pub image_pull_policy: Option, /// Resource limits for ipfs nodes, applies to both requests and limits. pub resource_limits: Option, + /// Persistent storage configuration + pub storage: Option, /// Name of the storage class for the PVC of the IPFS container pub storage_class: Option, /// Value of the RUST_LOG env var. @@ -207,6 +216,8 @@ pub struct GoIpfsSpec { pub image_pull_policy: Option, /// Resource limits for ipfs nodes, applies to both requests and limits. pub resource_limits: Option, + /// Persistent storage configuration + pub storage: Option, /// Name of the storage class for the PVC of the IPFS container pub storage_class: Option, /// List of ipfs commands to run during initialization. @@ -225,14 +236,24 @@ pub struct CasSpec { pub ipfs: Option, /// Resource limits for the CAS pod, applies to both requests and limits. pub cas_resource_limits: Option, + /// CAS storage configuration + pub cas_storage: Option, /// Resource limits for the CAS IPFS pod, applies to both requests and limits. pub ipfs_resource_limits: Option, + /// IPFS container storage configuration + pub ipfs_storage: Option, /// Resource limits for the Ganache pod, applies to both requests and limits. pub ganache_resource_limits: Option, + /// Ganache container storage configuration + pub ganache_storage: Option, /// Resource limits for the CAS Postgres pod, applies to both requests and limits. pub postgres_resource_limits: Option, + /// Postgres container container storage configuration + pub postgres_storage: Option, /// Resource limits for the LocalStack pod, applies to both requests and limits. pub localstack_resource_limits: Option, + /// Localstack container storage configuration + pub localstack_storage: Option, /// Configuration for the CAS API pub api: Option, } @@ -270,6 +291,16 @@ pub struct ResourceLimitsSpec { pub storage: Option, } +/// Describes the resources limits and requests for a pod +#[derive(Serialize, Deserialize, Debug, Default, PartialEq, Clone, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub struct PersistentStorageSpec { + /// Size of the persistent disk to request + pub size: Option, + /// Name of the storage class for the PVC of the container + pub class: Option, +} + /// Describes how monitoring resources are deployed for the network #[derive(Serialize, Deserialize, Debug, Default, PartialEq, Clone, JsonSchema)] #[serde(rename_all = "camelCase")] diff --git a/operator/src/network/storage.rs b/operator/src/network/storage.rs new file mode 100644 index 00000000..4083c6db --- /dev/null +++ b/operator/src/network/storage.rs @@ -0,0 +1,45 @@ +use std::collections::BTreeMap; + +use k8s_openapi::{ + api::core::v1::{PersistentVolumeClaimSpec, ResourceRequirements}, + apimachinery::pkg::api::resource::Quantity, +}; + +use crate::network::PersistentStorageSpec; + +#[derive(Clone)] +pub struct PersistentStorageConfig { + /// Persistent storage resource limit + pub size: Quantity, + pub class: Option, +} + +impl PersistentStorageConfig { + pub fn from_spec(spec: Option, defaults: Self) -> Self { + if let Some(spec) = spec { + Self { + size: spec.size.unwrap_or(defaults.size), + class: spec.class.or(defaults.class), + } + } else { + defaults + } + } +} + +impl From for PersistentVolumeClaimSpec { + fn from(value: PersistentStorageConfig) -> Self { + Self { + access_modes: Some(vec!["ReadWriteOnce".to_owned()]), + resources: Some(ResourceRequirements { + requests: Some(BTreeMap::from_iter(vec![( + "storage".to_owned(), + value.size, + )])), + ..Default::default() + }), + storage_class_name: value.class, + ..Default::default() + } + } +}