From 516dc9c80e7e1510c911481e180143dca6147fe7 Mon Sep 17 00:00:00 2001 From: Marc Schreiber Date: Thu, 11 Jan 2024 10:55:41 +0100 Subject: [PATCH 1/6] Bootstrap Companions in Kubernetes Environments This commit introduces the configuration `bootstrapping.containers` which provides a way to parse the configuration of application wide companions because the current available configuration of companions if quite limiting (Current backends, Docker and Kubernetes, offer way more options than the PREvant configuration object allows). For example, PREvant was limited to self-contained applications where each microservice only relies on interactions via network API calls (REST, database connections, messaging, etc.) With this commit PREvant is now able to deploy application companions that are more powerful than the PREvant configuration in Kubernetes backends. If `bootstrapping.containers` is defined, PREvant will start one or more containers on the infrastructure backend that are expected to generate Kubernetes manifests as output on standard out (stdout) that will be parsed by PREvant and supported are: - roles and role bindings - config maps and secrets - service accounts - persistent volume claims - services - pods, deployments, stateful sets, and jobs Then before deploying these manifests PREvant merges all objects with the objects generated from the HTTP request payload. Thus you can add or overwrite configurations. For example, you can change the image used or an environment variable. If you overwrite any configuration the companion will be turned into an instance (as PREvant did before). Ingresses won't be deployed if the bootstrap container outputs one of these. Instead they will be parsed and if they use the ingress class `nginx` they will be transformed into Traefik ingresses and middlewares so that the microservices will be available via web interface. This approach make #143 obsolete and fixes #123 and contributes to #146. --- api/Cargo.lock | 141 +- api/Cargo.toml | 11 +- api/src/apps/mod.rs | 28 +- api/src/apps/routes.rs | 8 +- api/src/config/app_selector.rs | 5 +- api/src/config/companion.rs | 158 ++- api/src/config/mod.rs | 75 +- api/src/config/secret.rs | 4 +- api/src/deployment/deployment_unit.rs | 44 +- api/src/deployment/hooks.rs | 17 +- .../infrastructure/dummy_infrastructure.rs | 6 +- .../kubernetes/deployment_unit.rs | 1192 +++++++++++++++++ .../kubernetes/infrastructure.rs | 509 +++---- api/src/infrastructure/kubernetes/mod.rs | 1 + api/src/infrastructure/kubernetes/payloads.rs | 318 +++-- api/src/infrastructure/traefik.rs | 157 ++- api/src/models/app_name.rs | 6 + api/src/models/service.rs | 8 - assets/bootstrap-companions.svg | 1146 ++++++++++++++++ docs/companions.md | 107 ++ 20 files changed, 3351 insertions(+), 590 deletions(-) create mode 100644 api/src/infrastructure/kubernetes/deployment_unit.rs create mode 100644 assets/bootstrap-companions.svg create mode 100644 docs/companions.md diff --git a/api/Cargo.lock b/api/Cargo.lock index e41a023d..2f6751c6 100644 --- a/api/Cargo.lock +++ b/api/Cargo.lock @@ -186,12 +186,6 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" -[[package]] -name = "base64" -version = "0.20.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ea22880d78093b0cbe17c89f64a7d457941e65759157ec6cb31a31d652b05e5" - [[package]] name = "base64" version = "0.21.7" @@ -547,9 +541,9 @@ dependencies = [ [[package]] name = "darling" -version = "0.14.4" +version = "0.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850" +checksum = "0209d94da627ab5605dcccf08bb18afa5009cfbef48d8a8b7d7bdbc79be25c5e" dependencies = [ "darling_core", "darling_macro", @@ -557,27 +551,27 @@ dependencies = [ [[package]] name = "darling_core" -version = "0.14.4" +version = "0.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "109c1ca6e6b7f82cc233a97004ea8ed7ca123a9af07a8230878fcfda9b158bf0" +checksum = "177e3443818124b357d8e76f53be906d60937f0d3a90773a664fa63fa253e621" dependencies = [ "fnv", "ident_case", "proc-macro2", "quote", "strsim", - "syn 1.0.109", + "syn 2.0.48", ] [[package]] name = "darling_macro" -version = "0.14.4" +version = "0.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e" +checksum = "836a9bbc7ad63342d6d6e7b815ccab164bc77a2d95d84bc3117a8c0d5c98e2d5" dependencies = [ "darling_core", "quote", - "syn 1.0.109", + "syn 2.0.48", ] [[package]] @@ -646,27 +640,6 @@ dependencies = [ "subtle", ] -[[package]] -name = "dirs-next" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1" -dependencies = [ - "cfg-if", - "dirs-sys-next", -] - -[[package]] -name = "dirs-sys-next" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d" -dependencies = [ - "libc", - "redox_users", - "winapi", -] - [[package]] name = "displaydoc" version = "0.2.4" @@ -790,7 +763,7 @@ dependencies = [ "pear", "serde", "tempfile", - "toml 0.8.8", + "toml", "uncased", "version_check", ] @@ -1082,6 +1055,15 @@ dependencies = [ "digest", ] +[[package]] +name = "home" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5" +dependencies = [ + "windows-sys 0.52.0", +] + [[package]] name = "http" version = "0.2.11" @@ -1454,14 +1436,16 @@ dependencies = [ ] [[package]] -name = "jsonpath_lib" -version = "0.3.0" +name = "jsonpath-rust" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eaa63191d68230cccb81c5aa23abd53ed64d83337cacbb25a7b8c7979523774f" +checksum = "06cc127b7c3d270be504572364f9569761a180b981919dd0d87693a7f5fb7829" dependencies = [ - "log", - "serde", + "pest", + "pest_derive", + "regex", "serde_json", + "thiserror", ] [[package]] @@ -1481,9 +1465,9 @@ dependencies = [ [[package]] name = "k8s-openapi" -version = "0.18.0" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd990069640f9db34b3b0f7a1afc62a05ffaa3be9b66aa3c313f58346df7f788" +checksum = "edc3606fd16aca7989db2f84bb25684d0270c6d6fa1dbcd0025af7b4130523a6" dependencies = [ "base64 0.21.7", "bytes 1.5.0", @@ -1495,9 +1479,9 @@ dependencies = [ [[package]] name = "kube" -version = "0.84.0" +version = "0.87.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14bd236a6f6ddeac3fefa2863eb4e363cb3a2c49d66619e181b5b8f8f0787575" +checksum = "3499c8d60c763246c7a213f51caac1e9033f46026904cb89bc8951ae8601f26e" dependencies = [ "k8s-openapi", "kube-client", @@ -1507,22 +1491,22 @@ dependencies = [ [[package]] name = "kube-client" -version = "0.84.0" +version = "0.87.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04a28620131ca89b2509e52f5e1b71bfa3e61a50321836b2ae373bc18e0309e6" +checksum = "033450dfa0762130565890dadf2f8835faedf749376ca13345bcd8ecd6b5f29f" dependencies = [ - "base64 0.20.0", + "base64 0.21.7", "bytes 1.5.0", "chrono", - "dirs-next", "either", "futures 0.3.30", + "home", "http", "http-body", "hyper", "hyper-rustls", "hyper-timeout", - "jsonpath_lib", + "jsonpath-rust", "k8s-openapi", "kube-core", "pem", @@ -1543,9 +1527,9 @@ dependencies = [ [[package]] name = "kube-core" -version = "0.84.0" +version = "0.87.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8227a989f1eeee3bcbf045165d6aca462af3744ecd4dfdcfba81051fb7de428e" +checksum = "b5bba93d054786eba7994d03ce522f368ef7d48c88a1826faa28478d85fb63ae" dependencies = [ "chrono", "form_urlencoded", @@ -1560,15 +1544,15 @@ dependencies = [ [[package]] name = "kube-derive" -version = "0.84.0" +version = "0.87.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19d227fcf3e12f53ea1a38d4766a8c29f8b27795579e4146464effb88d52dd99" +checksum = "91e98dd5e5767c7b894c1f0e41fd628b145f808e981feb8b08ed66455d47f1a4" dependencies = [ "darling", "proc-macro2", "quote", "serde_json", - "syn 1.0.109", + "syn 2.0.48", ] [[package]] @@ -1583,17 +1567,6 @@ version = "0.2.152" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13e3bf6590cbc649f4d1a3eefc9d5d6eb746f5200ffb04e5e142700b8faa56e7" -[[package]] -name = "libredox" -version = "0.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85c833ca1e66078851dba29046874e38f08b2c883700aa29a03ddd3b23814ee8" -dependencies = [ - "bitflags 2.4.1", - "libc", - "redox_syscall", -] - [[package]] name = "linux-raw-sys" version = "0.4.12" @@ -1965,11 +1938,12 @@ dependencies = [ [[package]] name = "pem" -version = "1.1.1" +version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8835c273a76a90455d7344889b0964598e3316e2a79ede8e36f16bdcf2228b8" +checksum = "1b8fcc794035347fb64beda2d3b462595dd2753e3f268d89c5aae77e8cf2c310" dependencies = [ - "base64 0.13.1", + "base64 0.21.7", + "serde", ] [[package]] @@ -2186,6 +2160,7 @@ dependencies = [ "pest", "pest_derive", "regex", + "regex-syntax 0.8.2", "reqwest", "rocket", "schemars", @@ -2200,7 +2175,7 @@ dependencies = [ "shiplift", "tempfile", "tokio", - "toml 0.7.8", + "toml", "url", "uuid", "yansi 0.5.1", @@ -2286,17 +2261,6 @@ dependencies = [ "bitflags 1.3.2", ] -[[package]] -name = "redox_users" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a18479200779601e498ada4e8c1e1f50e3ee19deb0259c25825a98b5603b2cb4" -dependencies = [ - "getrandom", - "libredox", - "thiserror", -] - [[package]] name = "ref-cast" version = "1.0.22" @@ -2738,7 +2702,6 @@ version = "1.0.111" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "176e46fa42316f18edd598015a5166857fc835ec732f5215eac6b7bdbf0a84f4" dependencies = [ - "indexmap", "itoa", "ryu", "serde", @@ -3208,18 +3171,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "toml" -version = "0.7.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd79e69d3b627db300ff956027cc6c3798cef26d22526befdfcd12feeb6d2257" -dependencies = [ - "serde", - "serde_spanned", - "toml_datetime", - "toml_edit 0.19.15", -] - [[package]] name = "toml" version = "0.8.8" @@ -3248,8 +3199,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" dependencies = [ "indexmap", - "serde", - "serde_spanned", "toml_datetime", "winnow", ] diff --git a/api/Cargo.toml b/api/Cargo.toml index 19a6b67e..a1513a36 100644 --- a/api/Cargo.toml +++ b/api/Cargo.toml @@ -26,15 +26,16 @@ futures = { version = "0.3", features = ["compat"] } handlebars = "4.5" http-api-problem = "0.57" jira_query = "1.3" -k8s-openapi = { version = "0.18", default-features = false, features = ["v1_24"] } -kube = { version = "0.84", default-features = false, features = ["client", "derive", "rustls-tls"] } +k8s-openapi = { version = "0.20", default-features = false, features = ["v1_24"] } +kube = { version = "0.87", default-features = false, features = ["client", "derive", "rustls-tls"] } lazy_static = "1.4" log = "0.4" multimap = "0.9" oci-distribution = "0.10" pest = "2.6" pest_derive = "2.6" -regex = "1.9" +regex = "1.10" +regex-syntax = "0.8" reqwest = { version = "0.11", features = ["json"] } rocket = { version = "0.5", features = ["json"] } schemars = "0.8" @@ -46,9 +47,9 @@ serde_json = "1.0" serde_regex = "1.1" serde_yaml = "0.9" tokio = { version = "1.29", features = ["macros", "rt", "rt-multi-thread", "sync", "time"] } -toml = "0.7" +toml = "0.8" url = { version = "2.4", features = ["serde"] } -uuid = { version = "1.3", features = ["serde", "v4"] } +uuid = { version = "1.5", features = ["serde", "v4"] } yansi = "0.5" diff --git a/api/src/apps/mod.rs b/api/src/apps/mod.rs index 2884ce04..a84a28df 100644 --- a/api/src/apps/mod.rs +++ b/api/src/apps/mod.rs @@ -291,6 +291,12 @@ impl AppsService { let deployment_unit = if let Ok(Some(base_traefik_ingress_route)) = self.infrastructure.base_traefik_ingress_route().await { + trace!( + "The base URL for {app_name} is: {:?}", + base_traefik_ingress_route + .to_url() + .map(|url| url.to_string()) + ); deployment_unit_builder .apply_base_traefik_ingress_route(base_traefik_ingress_route) .build() @@ -517,7 +523,7 @@ mod tests { let apps = AppsService::new(config, infrastructure)?; apps.create_or_update( - &AppName::from_str("master").unwrap(), + &AppName::master(), &AppStatusChangeId::new(), None, &vec![sc!("service-a"), sc!("service-b")], @@ -527,7 +533,7 @@ mod tests { apps.create_or_update( &AppName::from_str("branch").unwrap(), &AppStatusChangeId::new(), - Some(AppName::from_str("master").unwrap()), + Some(AppName::master()), &vec![sc!("service-b")], ) .await?; @@ -569,7 +575,7 @@ mod tests { apps.create_or_update( &AppName::from_str("branch").unwrap(), &AppStatusChangeId::new(), - Some(AppName::from_str("master").unwrap()), + Some(AppName::master()), &vec![sc!("service-a")], ) .await?; @@ -602,7 +608,7 @@ mod tests { let apps = AppsService::new(config, infrastructure)?; apps.create_or_update( - &AppName::from_str("master").unwrap(), + &AppName::master(), &AppStatusChangeId::new(), None, &vec![sc!("mariadb")], @@ -666,7 +672,7 @@ mod tests { let infrastructure = Box::new(Dummy::new()); let apps = AppsService::new(config, infrastructure)?; - let app_name = AppName::from_str("master").unwrap(); + let app_name = AppName::master(); apps.create_or_update( &app_name, @@ -863,7 +869,7 @@ Log msg 3 of service-a of app master let infrastructure = Box::new(Dummy::new()); let apps = AppsService::new(config, infrastructure)?; - let app_name = AppName::from_str("master").unwrap(); + let app_name = AppName::master(); apps.create_or_update( &app_name, @@ -948,7 +954,7 @@ Log msg 3 of service-a of app master let infrastructure = Box::new(Dummy::with_delay(std::time::Duration::from_millis(500))); let apps = Arc::new(AppsService::new(config, infrastructure)?); - let app_name = AppName::from_str("master").unwrap(); + let app_name = AppName::master(); apps.create_or_update( &app_name, &AppStatusChangeId::new(), @@ -965,7 +971,7 @@ Log msg 3 of service-a of app master .unwrap(); rt.block_on(apps_clone.delete_app(&app_name, &AppStatusChangeId::new())) }); - let app_name = AppName::from_str("master").unwrap(); + let app_name = AppName::master(); let handle2 = std::thread::spawn(move || { let rt = runtime::Builder::new_current_thread() .enable_time() @@ -1061,7 +1067,7 @@ Log msg 3 of service-a of app master "#; let (_temp_js_file, config) = config_with_deployment_hook(script); - let app_name = &AppName::from_str("master").unwrap(); + let app_name = &AppName::master(); let infrastructure = Box::new(Dummy::new()); let apps = AppsService::new(config, infrastructure)?; @@ -1093,7 +1099,7 @@ Log msg 3 of service-a of app master ))); let apps = AppsService::new(Config::default(), infrastructure)?; - let app_name = &AppName::from_str("master").unwrap(); + let app_name = &AppName::master(); apps.create_or_update( &app_name, &AppStatusChangeId::new(), @@ -1135,7 +1141,7 @@ Log msg 3 of service-a of app master let infrastructure = Box::new(Dummy::new()); let apps = AppsService::new(Config::default(), infrastructure)?; - let app_name = &AppName::from_str("master").unwrap(); + let app_name = &AppName::master(); apps.create_or_update( &app_name, &AppStatusChangeId::new(), diff --git a/api/src/apps/routes.rs b/api/src/apps/routes.rs index b306fec7..f026cba2 100644 --- a/api/src/apps/routes.rs +++ b/api/src/apps/routes.rs @@ -191,7 +191,7 @@ async fn change_status( )] async fn logs( app_name: Result, - service_name: String, + service_name: &str, since: Option, limit: Option, apps: &State>, @@ -214,13 +214,13 @@ async fn logs( let limit = limit.unwrap_or(20_000); let log_chunk = apps - .get_logs(&app_name, &service_name, &since, limit) + .get_logs(&app_name, &service_name.to_string(), &since, limit) .await?; Ok(LogsResponse { log_chunk, app_name, - service_name, + service_name: service_name.to_string(), limit, }) } @@ -810,7 +810,7 @@ mod tests { "type": "https://httpstatuses.com/400", "status": 400, "title": "Bad Request", - "detail": "Invalid image: private-registry.example.com/_/postgres at line 1 column 70" + "detail": "Invalid image: private-registry.example.com/_/postgres at line 1 column 51" }) ); } diff --git a/api/src/config/app_selector.rs b/api/src/config/app_selector.rs index 1f6ddb8f..6922eaf3 100644 --- a/api/src/config/app_selector.rs +++ b/api/src/config/app_selector.rs @@ -23,16 +23,17 @@ * THE SOFTWARE. * =========================LICENSE_END================================== */ +use crate::models::AppName; use regex::Regex; #[derive(Clone)] pub(super) struct AppSelector(Regex); impl AppSelector { - pub fn matches(&self, app_name: &str) -> bool { + pub fn matches(&self, app_name: &AppName) -> bool { match self.0.captures(app_name) { None => false, - Some(captures) => captures.get(0).map_or("", |m| m.as_str()) == app_name, + Some(captures) => captures.get(0).map_or("", |m| m.as_str()) == app_name.as_str(), } } } diff --git a/api/src/config/companion.rs b/api/src/config/companion.rs index aa150b5f..7753b98d 100644 --- a/api/src/config/companion.rs +++ b/api/src/config/companion.rs @@ -25,11 +25,21 @@ */ use crate::config::AppSelector; use crate::models::service::ContainerType; -use crate::models::{Environment, Image, Router, ServiceConfig}; +use crate::models::{AppName, Environment, Image, Router, ServiceConfig}; +use handlebars::Handlebars; use secstr::SecUtf8; use serde_value::Value; use std::collections::BTreeMap; use std::path::PathBuf; +use url::Url; + +#[derive(Clone, Default, Deserialize)] +pub(super) struct Companions { + #[serde(default)] + bootstrapping: Bootstrapping, + #[serde(flatten)] + companions: BTreeMap, +} #[derive(Clone, Deserialize)] #[serde(rename_all = "camelCase")] @@ -78,12 +88,52 @@ pub enum DeploymentStrategy { RedeployNever, } +#[derive(Clone, Default, Deserialize)] +struct Bootstrapping { + containers: Vec, +} + +#[derive(Clone, Deserialize)] +pub struct BootstrappingContainer { + image: Image, + #[serde(default)] + args: Vec, +} + +impl Companions { + pub(super) fn companion_configs

( + &self, + app_name: &AppName, + predicate: P, + ) -> Vec<(ServiceConfig, DeploymentStrategy, StorageStrategy)> + where + P: Fn(&Companion) -> bool, + { + self.companions + .iter() + .filter(|(_, companion)| companion.matches_app_name(app_name)) + .filter(|(_, companion)| predicate(companion)) + .map(|(_, companion)| { + ( + ServiceConfig::from(companion.clone()), + companion.deployment_strategy().clone(), + companion.storage_strategy().clone(), + ) + }) + .collect() + } + + pub(super) fn companion_bootstrapping_containers(&self) -> &Vec { + &self.bootstrapping.containers + } +} + impl Companion { pub fn companion_type(&self) -> &CompanionType { &self.companion_type } - pub fn matches_app_name(&self, app_name: &str) -> bool { + pub fn matches_app_name(&self, app_name: &AppName) -> bool { self.app_selector.matches(app_name) } @@ -149,6 +199,42 @@ impl Default for StorageStrategy { } } +impl BootstrappingContainer { + pub fn image(&self) -> &Image { + &self.image + } + + pub fn templated_args(&self, app_name: &AppName, base_url: &Option) -> Vec { + let handlebars = Handlebars::new(); + + #[derive(Serialize)] + #[serde(rename_all = "camelCase")] + struct AppData<'a> { + name: &'a str, + #[serde(skip_serializing_if = "Option::is_none")] + base_url: &'a Option, + } + // TODO: apply same pattern as for companions. {{application.name}}, {{service.…}}… + #[derive(Serialize)] + struct Data<'a> { + application: AppData<'a>, + } + + let data = Data { + application: AppData { + name: &app_name, + base_url, + }, + }; + + self.args + .iter() + // TODO: handle result + .map(|arg| handlebars.render_template(&arg, &data).unwrap()) + .collect() + } +} + #[cfg(test)] mod tests { use super::*; @@ -160,6 +246,12 @@ mod tests { }; } + macro_rules! companions_from_str { + ( $config_str:expr ) => { + toml::de::from_str::($config_str).unwrap() + }; + } + #[test] fn should_parse_companion_with_required_fields() { let companion = companion_from_str!( @@ -181,4 +273,66 @@ mod tests { DeploymentStrategy::RedeployAlways ); } + + #[test] + fn should_parse_companion_bootstrap_containers() { + let companions = companions_from_str!( + r#" + [[bootstrapping.containers]] + image = "busybox" + "# + ); + + let container = &companions.bootstrapping.containers[0]; + + assert_eq!(container.image, Image::from_str("busybox").unwrap()); + assert_eq!( + container.templated_args(&AppName::master(), &None), + Vec::::new() + ); + } + + #[test] + fn should_parse_companion_bootstrap_containers_and_template_args() { + let companions = companions_from_str!( + r#" + [[bootstrapping.containers]] + image = "busybox" + args = [ "echo", "Hello {{application.name}}" ] + "# + ); + + let container = &companions.bootstrapping.containers[0]; + + assert_eq!(container.image, Image::from_str("busybox").unwrap()); + assert_eq!( + container.templated_args(&AppName::master(), &None), + vec![String::from("echo"), String::from("Hello master")] + ); + } + + #[test] + fn should_parse_companion_bootstrap_containers_and_template_url_args() { + let companions = companions_from_str!( + r#" + [[bootstrapping.containers]] + image = "busybox" + args = [ "echo", "Hello {{application.baseUrl}}" ] + "# + ); + + let container = &companions.bootstrapping.containers[0]; + + assert_eq!(container.image, Image::from_str("busybox").unwrap()); + assert_eq!( + container.templated_args( + &AppName::master(), + &Some(Url::parse("http://example.com").unwrap()) + ), + vec![ + String::from("echo"), + String::from("Hello http://example.com/") + ] + ); + } } diff --git a/api/src/config/mod.rs b/api/src/config/mod.rs index beb054a5..98cfadfd 100644 --- a/api/src/config/mod.rs +++ b/api/src/config/mod.rs @@ -24,11 +24,13 @@ * =========================LICENSE_END================================== */ +pub use self::companion::BootstrappingContainer; pub use self::companion::DeploymentStrategy; pub use self::companion::StorageStrategy; -use self::companion::{Companion, CompanionType}; +use self::companion::{Companion, CompanionType, Companions}; pub use self::container::ContainerConfig; pub use self::runtime::Runtime; +use crate::models::AppName; use crate::models::ServiceConfig; pub(self) use app_selector::AppSelector; use clap::Parser; @@ -146,7 +148,8 @@ pub struct Config { runtime: Runtime, containers: Option, jira: Option, - companions: Option>, + #[serde(default)] + companions: Companions, services: Option>, hooks: Option>, #[serde(default)] @@ -189,7 +192,7 @@ impl Config { pub fn service_companion_configs( &self, - app_name: &str, + app_name: &AppName, ) -> Vec<(ServiceConfig, DeploymentStrategy, StorageStrategy)> { self.companion_configs(app_name, |companion| { companion.companion_type() == &CompanionType::Service @@ -198,39 +201,29 @@ impl Config { pub fn application_companion_configs( &self, - app_name: &str, + app_name: &AppName, ) -> Vec<(ServiceConfig, DeploymentStrategy, StorageStrategy)> { self.companion_configs(app_name, |companion| { companion.companion_type() == &CompanionType::Application }) } + pub fn companion_bootstrapping_containers(&self) -> &Vec { + self.companions.companion_bootstrapping_containers() + } + fn companion_configs

( &self, - app_name: &str, + app_name: &AppName, predicate: P, ) -> Vec<(ServiceConfig, DeploymentStrategy, StorageStrategy)> where P: Fn(&Companion) -> bool, { - match &self.companions { - None => vec![], - Some(companions_map) => companions_map - .iter() - .filter(|(_, companion)| companion.matches_app_name(app_name)) - .filter(|(_, companion)| predicate(companion)) - .map(|(_, companion)| { - ( - companion.clone().into(), - companion.deployment_strategy().clone(), - companion.storage_strategy().clone(), - ) - }) - .collect(), - } + self.companions.companion_configs(app_name, predicate) } - pub fn add_secrets_to(&self, service_config: &mut ServiceConfig, app_name: &str) { + pub fn add_secrets_to(&self, service_config: &mut ServiceConfig, app_name: &AppName) { if let Some(services) = &self.services { if let Some(service) = services.get(service_config.service_name()) { service.add_secrets_to(service_config, app_name); @@ -262,7 +255,7 @@ impl JiraConfig { } impl Service { - pub fn add_secrets_to(&self, service_config: &mut ServiceConfig, app_name: &str) { + pub fn add_secrets_to(&self, service_config: &mut ServiceConfig, app_name: &AppName) { if let Some(secrets) = &self.secrets { for s in secrets.iter().filter(|s| s.matches_app_name(app_name)) { let (path, sec) = s.clone().into(); @@ -340,7 +333,7 @@ mod tests { "# ); - let companion_configs = config.application_companion_configs("master"); + let companion_configs = config.application_companion_configs(&AppName::master()); assert_eq!(companion_configs.len(), 1); companion_configs.iter().for_each(|(config, _, _)| { @@ -375,7 +368,7 @@ mod tests { "# ); - let companion_configs = config.service_companion_configs("master"); + let companion_configs = config.service_companion_configs(&AppName::master()); assert_eq!(companion_configs.len(), 1); companion_configs.iter().for_each(|(config, _, _)| { @@ -401,7 +394,7 @@ mod tests { "# ); - let companion_configs = config.service_companion_configs("master"); + let companion_configs = config.service_companion_configs(&AppName::master()); assert_eq!(companion_configs.len(), 1); companion_configs.iter().for_each(|(_, strategy, _)| { @@ -424,7 +417,7 @@ mod tests { "# ); - let companion_configs = config.application_companion_configs("master"); + let companion_configs = config.application_companion_configs(&AppName::master()); assert_eq!(companion_configs.len(), 1); companion_configs.iter().for_each(|(config, _, _)| { @@ -446,7 +439,7 @@ mod tests { "# ); - let companion_configs = config.application_companion_configs("master"); + let companion_configs = config.application_companion_configs(&AppName::master()); assert_eq!(companion_configs.len(), 1); companion_configs.iter().for_each(|(config, _, _)| { @@ -470,7 +463,7 @@ mod tests { "# ); - let companion_configs = config.application_companion_configs("master"); + let companion_configs = config.application_companion_configs(&AppName::master()); assert_eq!(companion_configs.len(), 1); companion_configs.iter().for_each(|(config, _, _)| { @@ -500,7 +493,8 @@ mod tests { "# ); - let companion_configs = config.application_companion_configs("random-name"); + let companion_configs = + config.application_companion_configs(&AppName::from_str("random-name").unwrap()); assert_eq!(companion_configs.len(), 0); } @@ -517,7 +511,7 @@ mod tests { ); let mut service_config = service_config!("mariadb"); - config.add_secrets_to(&mut service_config, &String::from("master")); + config.add_secrets_to(&mut service_config, &AppName::master()); let secret_file_content = service_config .files() .expect("File content is missing") @@ -539,7 +533,7 @@ mod tests { ); let mut service_config = service_config!("mariadb"); - config.add_secrets_to(&mut service_config, &String::from("master")); + config.add_secrets_to(&mut service_config, &AppName::master()); let secret_file_content = service_config .files() @@ -562,7 +556,10 @@ mod tests { ); let mut service_config = service_config!("mariadb"); - config.add_secrets_to(&mut service_config, &String::from("master-1.x")); + config.add_secrets_to( + &mut service_config, + &AppName::from_str("master-1.x").unwrap(), + ); let secret_file_content = service_config .files() @@ -585,7 +582,10 @@ mod tests { ); let mut service_config = service_config!("mariadb"); - config.add_secrets_to(&mut service_config, &String::from("random-app-name")); + config.add_secrets_to( + &mut service_config, + &AppName::from_str("random-app-name").unwrap(), + ); assert!(service_config.files().is_none()); } @@ -603,7 +603,10 @@ mod tests { ); let mut service_config = service_config!("mariadb"); - config.add_secrets_to(&mut service_config, &String::from("master-1.x")); + config.add_secrets_to( + &mut service_config, + &AppName::from_str("master-1.x").unwrap(), + ); assert_eq!(service_config.files(), None); } @@ -668,7 +671,7 @@ mod tests { "# ); - let companion_configs = config.application_companion_configs("master"); + let companion_configs = config.application_companion_configs(&AppName::master()); assert_eq!(companion_configs.len(), 1); companion_configs.iter().for_each(|(config, _, _)| { @@ -689,7 +692,7 @@ mod tests { "# ); - let companion_configs = config.application_companion_configs("master"); + let companion_configs = config.application_companion_configs(&AppName::master()); assert_eq!(companion_configs.len(), 1); companion_configs diff --git a/api/src/config/secret.rs b/api/src/config/secret.rs index a573a60d..ac46155d 100644 --- a/api/src/config/secret.rs +++ b/api/src/config/secret.rs @@ -23,7 +23,7 @@ * THE SOFTWARE. * =========================LICENSE_END================================== */ -use crate::config::AppSelector; +use crate::{config::AppSelector, models::AppName}; use base64::{engine::general_purpose, Engine}; use secstr::SecUtf8; use serde::{de, Deserialize, Deserializer}; @@ -53,7 +53,7 @@ impl Secret { Ok(SecUtf8::from(sec_value)) } - pub fn matches_app_name(&self, app_name: &str) -> bool { + pub fn matches_app_name(&self, app_name: &AppName) -> bool { self.app_selector.matches(app_name) } } diff --git a/api/src/deployment/deployment_unit.rs b/api/src/deployment/deployment_unit.rs index 8c21e183..bc4701df 100644 --- a/api/src/deployment/deployment_unit.rs +++ b/api/src/deployment/deployment_unit.rs @@ -97,6 +97,7 @@ pub struct WithAppliedHooks { pub struct WithAppliedIngressRoute { app_name: AppName, services: Vec, + route: TraefikIngressRoute, } pub struct DeploymentUnitBuilder { @@ -106,6 +107,7 @@ pub struct DeploymentUnitBuilder { pub struct DeploymentUnit { app_name: AppName, services: Vec, + route: TraefikIngressRoute, } #[derive(Clone, Debug)] @@ -174,6 +176,10 @@ impl DeploymentUnit { pub fn app_name(&self) -> &AppName { &self.app_name } + + pub fn app_base_route(&self) -> &TraefikIngressRoute { + &self.route + } } impl DeploymentUnitBuilder { @@ -548,18 +554,26 @@ impl DeploymentUnitBuilder { service.ingress_route.merge_with(service_route); } + let mut route = route; + route.merge_with(TraefikIngressRoute::with_app_only_defaults( + &self.stage.app_name, + )); + DeploymentUnitBuilder { stage: WithAppliedIngressRoute { app_name: self.stage.app_name, services: self.stage.services, + route, }, } } pub fn build(self) -> DeploymentUnit { + let route = TraefikIngressRoute::with_app_only_defaults(&self.stage.app_name); DeploymentUnit { app_name: self.stage.app_name, services: self.stage.services, + route, } } } @@ -569,6 +583,7 @@ impl DeploymentUnitBuilder { DeploymentUnit { app_name: self.stage.app_name, services: self.stage.services, + route: self.stage.route, } } } @@ -580,13 +595,12 @@ mod tests { use crate::models::{Environment, EnvironmentVariable}; use crate::{config_from_str, sc}; use secstr::SecUtf8; - use std::str::FromStr; #[tokio::test] async fn should_return_unique_images() -> Result<(), AppsServiceError> { let config = Config::default(); let unit = DeploymentUnitBuilder::init( - AppName::from_str("master").unwrap(), + AppName::master(), vec![ sc!("http1", "nginx:1.13"), sc!("wordpress1", "wordpress:alpine"), @@ -617,7 +631,7 @@ mod tests { "# ); - let app_name = AppName::from_str("master").unwrap(); + let app_name = AppName::master(); let service_configs = vec![sc!("http1", "nginx:1.13")]; let unit = DeploymentUnitBuilder::init(app_name, service_configs) @@ -650,7 +664,7 @@ mod tests { "# ); - let app_name = AppName::from_str("master").unwrap(); + let app_name = AppName::master(); let service_configs = vec![sc!( "openid", labels = (), @@ -705,7 +719,7 @@ mod tests { "# ); - let app_name = AppName::from_str("master").unwrap(); + let app_name = AppName::master(); let service_configs = vec![sc!( "openid", labels = (), @@ -754,7 +768,7 @@ mod tests { image = 'postgres:11' "# ); - let app_name = AppName::from_str("master").unwrap(); + let app_name = AppName::master(); let service_configs = vec![ sc!("wordpress", "wordpress:alpine"), sc!("nextcloud", "nextcloud:alpine"), @@ -802,7 +816,7 @@ mod tests { ]))); let config = Config::default(); - let app_name = AppName::from_str("master").unwrap(); + let app_name = AppName::master(); let unit = DeploymentUnitBuilder::init(app_name, vec![service_config]) .extend_with_config(&config) @@ -836,7 +850,7 @@ mod tests { SecUtf8::from("{{application.name}}"), )]))); - let app_name = AppName::from_str("master").unwrap(); + let app_name = AppName::master(); let config = Config::default(); let unit = DeploymentUnitBuilder::init(app_name, vec![service_configs]) @@ -875,7 +889,7 @@ mod tests { "# ); - let app_name = AppName::from_str("master").unwrap(); + let app_name = AppName::master(); let service_configs = vec![sc!("wordpress", "wordpress:latest")]; let unit = DeploymentUnitBuilder::init(app_name, service_configs) @@ -929,7 +943,7 @@ mod tests { "# ); - let app_name = AppName::from_str("master").unwrap(); + let app_name = AppName::master(); let service_configs = vec![sc!("wordpress", "wordpress:alpine")]; let unit = DeploymentUnitBuilder::init(app_name, service_configs) @@ -983,7 +997,7 @@ mod tests { "# ); - let app_name = AppName::from_str("master").unwrap(); + let app_name = AppName::master(); let service_configs = vec![sc!("openid", "private.example.com/library/openid:backup")]; let unit = DeploymentUnitBuilder::init(app_name, service_configs) @@ -1032,7 +1046,7 @@ mod tests { "# ); - let app_name = AppName::from_str("master").unwrap(); + let app_name = AppName::master(); let service_configs = vec![ sc!("wordpress", "wordpress:alpine"), sc!("wordpress-db", "postgres:11-alpine"), @@ -1081,7 +1095,7 @@ mod tests { #[tokio::test] async fn should_determine_deployment_strategy_for_requested_service( ) -> Result<(), AppsServiceError> { - let app_name = AppName::from_str("master").unwrap(); + let app_name = AppName::master(); let service_configs = vec![ sc!("wordpress", "wordpress:alpine"), sc!("wordpress-db", "postgres:11-alpine"), @@ -1131,7 +1145,7 @@ mod tests { "# ); - let app_name = AppName::from_str("master").unwrap(); + let app_name = AppName::master(); let service_configs = vec![sc!("wordpress", "wordpress:alpine")]; let unit = DeploymentUnitBuilder::init(app_name, service_configs) @@ -1161,7 +1175,7 @@ mod tests { async fn apply_base_traefik_router_rule() -> Result<(), AppsServiceError> { let config = config_from_str!(""); - let app_name = AppName::from_str("master").unwrap(); + let app_name = AppName::master(); let service_configs = vec![sc!("wordpress")]; let unit = DeploymentUnitBuilder::init(app_name, service_configs) diff --git a/api/src/deployment/hooks.rs b/api/src/deployment/hooks.rs index 17e6bb10..8c6af8bc 100644 --- a/api/src/deployment/hooks.rs +++ b/api/src/deployment/hooks.rs @@ -236,7 +236,6 @@ mod tests { use crate::deployment::deployment_unit::DeploymentUnitBuilder; use std::collections::HashMap; use std::io::Write; - use std::str::FromStr; use std::vec; use tempfile::NamedTempFile; @@ -269,7 +268,7 @@ mod tests { let (_temp_js_file, config) = config_with_deployment_hook(script); - let app_name = AppName::from_str("master").unwrap(); + let app_name = AppName::master(); let service_configs = vec![crate::sc!("service-a")]; let unit = DeploymentUnitBuilder::init(app_name, service_configs) @@ -314,7 +313,7 @@ mod tests { let (_temp_js_file, config) = config_with_deployment_hook(script); - let app_name = AppName::from_str("master").unwrap(); + let app_name = AppName::master(); let mut service_config = crate::sc!("service-a"); let mut files = BTreeMap::new(); @@ -359,7 +358,7 @@ mod tests { "#; let (_temp_js_file, config) = config_with_deployment_hook(script); - let app_name = AppName::from_str("master").unwrap(); + let app_name = AppName::master(); let service_config = crate::sc!("service-a"); let unit = DeploymentUnitBuilder::init(app_name, vec![service_config]) @@ -401,7 +400,7 @@ mod tests { "#; let (_temp_js_file, config) = config_with_deployment_hook(script); - let app_name = AppName::from_str("master").unwrap(); + let app_name = AppName::master(); let mut service_config = crate::sc!("service-a"); service_config.set_env(Some(Environment::new(vec![EnvironmentVariable::new( @@ -452,7 +451,7 @@ mod tests { let (_temp_js_file, config) = config_with_deployment_hook(script); - let app_name = AppName::from_str("master").unwrap(); + let app_name = AppName::master(); let mut service_config = crate::sc!("service-a"); service_config.set_env(Some(Environment::new(vec![EnvironmentVariable::new( String::from("VARIABLE_X"), @@ -497,7 +496,7 @@ mod tests { "#; let service_config = crate::sc!("service-a"); let (_temp_js_file, config) = config_with_deployment_hook(script); - let app_name = AppName::from_str("master").unwrap(); + let app_name = AppName::master(); let unit = DeploymentUnitBuilder::init(app_name, vec![service_config]) .extend_with_config(&config) @@ -533,7 +532,7 @@ mod tests { let service_config = crate::sc!("service-a"); let (_temp_js_file, config) = config_with_deployment_hook(script); - let app_name = AppName::from_str("master").unwrap(); + let app_name = AppName::master(); let unit = DeploymentUnitBuilder::init(app_name, vec![service_config]) .extend_with_config(&config) @@ -561,7 +560,7 @@ mod tests { let mut deployed_services_error = String::new(); let service_config = crate::sc!("service-a"); let (_temp_js_file, config) = config_with_deployment_hook(script); - let app_name = AppName::from_str("master").unwrap(); + let app_name = AppName::master(); match DeploymentUnitBuilder::init(app_name, vec![service_config]) .extend_with_config(&config) .extend_with_templating_only_service_configs(Vec::new()) diff --git a/api/src/infrastructure/dummy_infrastructure.rs b/api/src/infrastructure/dummy_infrastructure.rs index 9c089e48..8b6abc55 100644 --- a/api/src/infrastructure/dummy_infrastructure.rs +++ b/api/src/infrastructure/dummy_infrastructure.rs @@ -34,6 +34,7 @@ use async_trait::async_trait; use chrono::{DateTime, FixedOffset, Utc}; use multimap::MultiMap; use std::collections::HashSet; +use std::str::FromStr; use std::sync::Mutex; use std::time::Duration; @@ -113,7 +114,7 @@ impl Infrastructure for DummyInfrastructure { .build() .unwrap(); - s.insert(app.clone(), service); + s.insert(AppName::from_str(app).unwrap(), service); } } @@ -155,7 +156,8 @@ impl Infrastructure for DummyInfrastructure { self.delay_if_configured().await; let mut services = self.services.lock().unwrap(); - match services.remove(app_name) { + + match services.remove(&app_name) { Some(services) => Ok(services .into_iter() .map(|sc| { diff --git a/api/src/infrastructure/kubernetes/deployment_unit.rs b/api/src/infrastructure/kubernetes/deployment_unit.rs new file mode 100644 index 00000000..ac86b512 --- /dev/null +++ b/api/src/infrastructure/kubernetes/deployment_unit.rs @@ -0,0 +1,1192 @@ +use super::{ + infrastructure::KubernetesInfrastructureError, + payloads::{ + convert_k8s_ingress_to_traefik_ingress, IngressRoute as TraefikIngressRoute, + Middleware as TraefikMiddleware, + }, +}; +use crate::{ + config::BootstrappingContainer, + deployment::DeploymentUnit, + infrastructure::{APP_NAME_LABEL, CONTAINER_TYPE_LABEL, SERVICE_NAME_LABEL}, + models::{AppName, ContainerType, Image}, +}; +use failure::Error; +use futures::{AsyncBufReadExt, AsyncReadExt, StreamExt, TryStreamExt}; +use k8s_openapi::{ + api::{ + apps::v1::{Deployment, StatefulSet}, + batch::v1::Job, + core::v1::{ + ConfigMap, Container, LocalObjectReference, PersistentVolumeClaim, Pod, PodSpec, + Secret, Service, ServiceAccount, + }, + networking::v1::Ingress, + rbac::v1::{Role, RoleBinding}, + }, + apimachinery::pkg::apis::meta::v1::LabelSelector, + DeepMerge, Metadata, Resource, +}; +use kube::{ + api::{LogParams, Patch, PatchParams, PostParams, WatchParams}, + core::{DynamicObject, ObjectMeta, WatchEvent}, + Api, Client, ResourceExt, +}; +use serde::Deserialize; +use std::{ + collections::{BTreeMap, HashSet}, + str::FromStr, +}; +use url::Url; + +#[derive(Default)] +pub(super) struct K8sDeploymentUnit { + roles: Vec, + role_bindings: Vec, + stateful_sets: Vec, + config_maps: Vec, + secrets: Vec, + pvcs: Vec, + services: Vec, + pods: Vec, + deployments: Vec, + jobs: Vec, + service_accounts: Vec, + traefik_ingresses: Vec, + traefik_middlewares: Vec, +} + +impl K8sDeploymentUnit { + async fn start_bootstrapping_pods( + app_name: &AppName, + client: Client, + bootstrapping_containers: &[BootstrappingContainer], + image_pull_secret: Option, + base_url: Option, + ) -> Result<(String, Vec), Error> { + let image_pull_secrets = match image_pull_secret { + Some(image_pull_secret) => { + let image_pull_secrets = vec![LocalObjectReference { + name: Some(image_pull_secret.metadata.name.clone().unwrap_or_default()), + }]; + create_or_patch(client.clone(), app_name, image_pull_secret).await?; + Some(image_pull_secrets) + } + None => None, + }; + + let containers = bootstrapping_containers + .iter() + .enumerate() + .map(|(i, bc)| Container { + name: format!("bootstrap-{i}"), + image: Some(bc.image().to_string()), + image_pull_policy: Some(String::from("Always")), + args: Some(bc.templated_args(app_name, &base_url)), + ..Default::default() + }) + .collect::>(); + + let pod_name = format!( + "{}-bootstrap-{}", + app_name.to_rfc1123_namespace_id(), + uuid::Uuid::new_v4() + ); + + let pod = Pod { + metadata: ObjectMeta { + name: Some(pod_name.clone()), + labels: Some(BTreeMap::from([( + APP_NAME_LABEL.to_string(), + app_name.to_string(), + )])), + ..Default::default() + }, + spec: Some(PodSpec { + containers, + image_pull_secrets, + restart_policy: Some(String::from("Never")), + ..Default::default() + }), + ..Default::default() + }; + create_or_patch(client.clone(), app_name, pod).await?; + + let api: Api = Api::namespaced(client, &app_name.to_rfc1123_namespace_id()); + + // Wait for a bookmark event to be sure that the log is ready to be consumed + let wp = WatchParams::default() + .fields(&format!("metadata.name={pod_name}")) + .timeout(10); + let mut stream = api.watch(&wp, "0").await?.boxed(); + while let Some(status) = stream.try_next().await? { + trace!("Saw watch event for bootstrapping pod {pod_name} in {app_name}: {status:?}"); + + if let WatchEvent::Bookmark(_bookmark) = status { + debug!("Boot strapping pod {pod_name} for {app_name} ready."); + break; + } + } + + loop { + let pod = api.get_status(&pod_name).await?; + + if let Some(phase) = pod.status.and_then(|status| status.phase) { + match phase.as_str() { + "Running" | "Succeeded" => { + break; + } + "Failed" | "Unknown" => { + return Err(KubernetesInfrastructureError::UnexpectedError { + internal_message: format!( + "Bootstrap pod {pod_name} for {app_name} failed" + ), + } + .into()); + } + phase => { + trace!("Boot strapping pod {pod_name} for {app_name} still not in running phase. Currently in {phase}."); + } + } + } + + tokio::time::sleep(tokio::time::Duration::from_secs(5)).await; + } + + let mut log_streams = Vec::with_capacity(bootstrapping_containers.len()); + + for i in 0..bootstrapping_containers.len() { + log_streams.push( + api.log_stream( + &pod_name, + &LogParams { + container: Some(format!("bootstrap-{i}")), + follow: true, + ..Default::default() + }, + ) + .await?, + ); + } + + Ok((pod_name, log_streams)) + } + + pub(super) async fn bootstrap( + deployment_unit: &DeploymentUnit, + client: Client, + bootstrapping_container: &[BootstrappingContainer], + image_pull_secret: Option, + ) -> Result { + if bootstrapping_container.is_empty() { + return Ok(Default::default()); + } + + let app_name = deployment_unit.app_name(); + + let (bootstrapping_pod_name, mut log_streams) = Self::start_bootstrapping_pods( + app_name, + client.clone(), + bootstrapping_container, + image_pull_secret, + deployment_unit.app_base_route().to_url(), + ) + .await?; + + let result = Self::parse_from_log_streams(deployment_unit, &mut log_streams).await; + + let pod_api: Api = Api::namespaced(client, &app_name.to_rfc1123_namespace_id()); + pod_api + .delete(&bootstrapping_pod_name, &Default::default()) + .await?; + + result + } + + async fn parse_from_log_streams( + deployment_unit: &DeploymentUnit, + log_streams: L, + ) -> Result + where + L: IntoIterator, + ::Item: AsyncBufReadExt, + ::Item: Unpin, + { + let app_name = deployment_unit.app_name(); + let mut roles = Vec::new(); + let mut role_bindings = Vec::new(); + let mut stateful_sets = Vec::new(); + let mut config_maps = Vec::new(); + let mut secrets = Vec::new(); + let mut pvcs = Vec::new(); + let mut services = Vec::new(); + let mut pods = Vec::new(); + let mut deployments = Vec::new(); + let mut jobs = Vec::new(); + let mut service_accounts = Vec::new(); + let mut ingresses = Vec::new(); + + for mut log_stream in log_streams.into_iter() { + let mut stdout = String::new(); + log_stream.read_to_string(&mut stdout).await?; + + trace!( + "Received YAML from bootstrapping container in {app_name}: {}…", + stdout.lines().next().unwrap_or(&stdout) + ); + + for doc in serde_yaml::Deserializer::from_str(&stdout) { + match DynamicObject::deserialize(doc) { + Ok(mut dy) => { + dy.metadata.namespace = Some(app_name.to_rfc1123_namespace_id()); + dy.labels_mut() + .insert(APP_NAME_LABEL.to_string(), app_name.to_string()); + + let api_version = dy + .types + .as_ref() + .map(|t| t.api_version.as_str()) + .unwrap_or_default(); + let kind = dy + .types + .as_ref() + .map(|t| t.kind.as_str()) + .unwrap_or_default(); + + trace!( + "Parsed {} ({api_version}, {kind}) for {app_name} as a bootstrap application element.", + dy.metadata + .name + .as_deref() + .unwrap_or_default() + ); + + match (api_version, kind) { + (Role::API_VERSION, Role::KIND) => match dy.clone().try_parse::() + { + Ok(role) => { + roles.push(role); + } + Err(e) => { + error!("Cannot parse {:?} as Role: {e}", dy.metadata.name); + } + }, + + (RoleBinding::API_VERSION, RoleBinding::KIND) => { + match dy.clone().try_parse::() { + Ok(role_binding) => { + role_bindings.push(role_binding); + } + Err(e) => { + error!( + "Cannot parse {:?} as RoleBinding: {e}", + dy.metadata.name + ); + } + } + } + (StatefulSet::API_VERSION, StatefulSet::KIND) => { + match dy.clone().try_parse::() { + Ok(stateful_set) => { + stateful_sets.push(stateful_set); + } + Err(e) => { + error!( + "Cannot parse {:?} as StatefulSet: {e}", + dy.metadata.name + ); + } + } + } + (ConfigMap::API_VERSION, ConfigMap::KIND) => { + match dy.clone().try_parse::() { + Ok(config_map) => { + config_maps.push(config_map); + } + Err(e) => { + error!( + "Cannot parse {:?} as ConfigMap: {e}", + dy.metadata.name + ); + } + } + } + (Secret::API_VERSION, Secret::KIND) => { + if let serde_json::Value::Object(obj) = &mut dy.data { + obj.entry("data").and_modify(|obj| { + if let serde_json::Value::Object(obj) = obj { + for (_k, v) in obj.iter_mut() { + if let serde_json::Value::String(str) = v { + // replacing new lines here because it is assumed + // that the data is base64 encoded and thus there + // must be no new lines + *v = str.replace('\n', "").into(); + } + } + } + }); + } + + match dy.clone().try_parse::() { + Ok(secret) => { + secrets.push(secret); + } + Err(e) => { + error!( + "Cannot parse {:?} as Secret: {e}", + dy.metadata.name + ); + } + } + } + (PersistentVolumeClaim::API_VERSION, PersistentVolumeClaim::KIND) => { + match dy.clone().try_parse::() { + Ok(pvc) => { + pvcs.push(pvc); + } + Err(e) => { + error!( + "Cannot parse {:?} as PersistentVolumeClaim: {e}", + dy.metadata.name + ); + } + } + } + (Service::API_VERSION, Service::KIND) => { + match dy.clone().try_parse::() { + Ok(service) => { + services.push(service); + } + Err(e) => { + error!( + "Cannot parse {:?} as Service: {e}", + dy.metadata.name + ); + } + } + } + (Deployment::API_VERSION, Deployment::KIND) => { + match dy.clone().try_parse::() { + Ok(mut deployment) => { + let service_name = deployment + .labels() + .get("app.kubernetes.io/component") + .cloned() + .unwrap_or_else(|| { + deployment.metadata.name.clone().unwrap_or_default() + }); + + deployment + .labels_mut() + .insert(SERVICE_NAME_LABEL.to_string(), service_name); + deployment.labels_mut().insert( + CONTAINER_TYPE_LABEL.to_string(), + ContainerType::ApplicationCompanion.to_string(), + ); + + deployments.push(deployment); + } + Err(e) => { + error!( + "Cannot parse {:?} as Deployment: {e}", + dy.metadata.name + ); + } + } + } + (Pod::API_VERSION, Pod::KIND) => match dy.clone().try_parse::() { + Ok(pod) => { + pods.push(pod); + } + Err(e) => { + error!("Cannot parse {:?} as Pod: {e}", dy.metadata.name); + } + }, + (Job::API_VERSION, Job::KIND) => match dy.clone().try_parse::() { + Ok(job) => { + jobs.push(job); + } + Err(e) => { + error!("Cannot parse {:?} as Job: {e}", dy.metadata.name); + } + }, + (ServiceAccount::API_VERSION, ServiceAccount::KIND) => { + match dy.clone().try_parse::() { + Ok(service_account) => { + service_accounts.push(service_account); + } + Err(e) => { + error!( + "Cannot parse {:?} as ServiceAccount: {e}", + dy.metadata.name + ); + } + } + } + (Ingress::API_VERSION, Ingress::KIND) => { + match dy.clone().try_parse::() { + Ok(ingress) => { + ingresses.push(ingress); + } + Err(e) => { + error!( + "Cannot parse {:?} as Ingress: {e}", + dy.metadata.name + ); + } + } + } + _ => { + warn!( + "Cannot parse {name} ({api_version}, {kind}) for {app_name} because its kind is unknown", + name=dy.metadata.name.unwrap_or_default() + ); + } + } + } + Err(err) => { + warn!("The output of a bootstrap container for {app_name} could not be parsed: {stdout}"); + return Err(err.into()); + } + } + } + } + + let mut traefik_ingresses = Vec::new(); + let mut traefik_middlewares = Vec::new(); + + for ingress in ingresses { + let Ok((route, middlewares)) = convert_k8s_ingress_to_traefik_ingress( + ingress, + deployment_unit.app_base_route().clone(), + ) else { + continue; + }; + + traefik_ingresses.push(route); + traefik_middlewares.extend(middlewares); + } + + Ok(Self { + roles, + role_bindings, + stateful_sets, + config_maps, + secrets, + pvcs, + services, + pods, + deployments, + jobs, + service_accounts, + traefik_ingresses, + traefik_middlewares, + }) + } + + pub(super) fn merge( + &mut self, + secret: Option, + service: Service, + deployment: Deployment, + ingress: TraefikIngressRoute, + middlewares: Vec, + ) { + let mut deployment = deployment; + + let service_name = deployment + .metadata + .labels + .as_ref() + .and_then(|labels| labels.get(SERVICE_NAME_LABEL)) + .expect("There must be label providing the service name"); + + let stateful_sets = self + .stateful_sets + .iter_mut() + .filter(|set| Some(service_name) == set.metadata().name.as_ref()) + .filter_map(|set| { + let spec = set.spec.as_mut()?; + Some(( + &mut set.metadata, + spec.template.metadata.as_mut(), + spec.template.spec.as_mut()?, + )) + }); + let deployments = self + .deployments + .iter_mut() + .filter(|set| Some(service_name) == set.metadata().name.as_ref()) + .filter_map(|deployment| { + let spec = deployment.spec.as_mut()?; + Some(( + &mut deployment.metadata, + spec.template.metadata.as_mut(), + spec.template.spec.as_mut()?, + )) + }); + let pods = self + .pods + .iter_mut() + .filter(|pod| Some(service_name) == pod.metadata().name.as_ref()) + .filter_map(|pod| Some((&mut pod.metadata, None, pod.spec.as_mut()?))); + + match stateful_sets.chain(deployments).chain(pods).next() { + Some((metadata, pod_meta, pod_spec)) => { + // Clean everything that might interfere with the original definitions of + // bootstrapped companion before calling merge_from down below. + deployment.metadata.name = None; + + metadata.merge_from(deployment.metadata); + + let mut deployment_spec = deployment + .spec + .expect("There should be a deployment spec created for the deployable service"); + + deployment_spec.selector = LabelSelector::default(); + + let template_to_be_merged = deployment_spec.template; + + if let Some(pod_meta) = pod_meta { + pod_meta.merge_from( + template_to_be_merged.metadata.expect( + "There should be a pod meta created for the deployable service", + ), + ); + } + + let mut pod_spec_to_be_merged = template_to_be_merged + .spec + .expect("There should be a pod spec created for the deployable service"); + pod_spec_to_be_merged.containers[0].name = pod_spec.containers[0].name.clone(); + pod_spec_to_be_merged.containers[0].ports = None; + + pod_spec.merge_from(pod_spec_to_be_merged); + + if let Some(secret) = secret { + self.secrets.push(secret); + } + // Ingress, Service, and Middlewares will be ignored because at this point it can + // be assumed that these configurations are covered by the Kubernetes objects that + // were used for bootstrapping the application. + } + None => { + self.secrets.extend(secret); + self.services.push(service); + self.deployments.push(deployment); + self.traefik_ingresses.push(ingress); + self.traefik_middlewares.extend(middlewares); + } + } + } + + /// This filters bootstrapped [Deployments](Deployment), [Stateful Sets](StatefulSet), or + /// [Pods](Pod) by the existing [services](Service) in already deployed application to avoid + /// that deployments of instances overwrite each other + pub(super) fn filter_by_instances_and_replicas( + &mut self, + services: &[crate::models::service::Service], + ) { + let service_not_to_be_retained = services + .iter() + .filter(|s| { + s.container_type() == &ContainerType::Instance + || s.container_type() == &ContainerType::Replica + }) + .map(|s| s.service_name()) + .collect::>(); + + self.deployments.retain(|deployment| { + let Some(service_name) = deployment + .metadata + .labels + .as_ref() + .and_then(|labels| labels.get(SERVICE_NAME_LABEL)) + else { + return false; + }; + + !service_not_to_be_retained.contains(service_name) + }); + } + + fn images_of_pod_spec(spec: &PodSpec) -> HashSet { + let mut images = HashSet::new(); + + if let Some(init_containers) = &spec.init_containers { + for init_container in init_containers { + if let Some(image) = init_container + .image + .as_ref() + .and_then(|image| Image::from_str(image).ok()) + { + images.insert(image); + } + } + } + + for container in &spec.containers { + if let Some(image) = container + .image + .as_ref() + .and_then(|image| Image::from_str(image).ok()) + { + images.insert(image); + } + } + + images + } + + pub(super) fn images(&self) -> HashSet { + let mut images = HashSet::new(); + + for deployment in &self.deployments { + let Some(spec) = &deployment.spec else { + continue; + }; + let Some(spec) = &spec.template.spec else { + continue; + }; + + images.extend(Self::images_of_pod_spec(spec)); + } + for job in &self.jobs { + let Some(spec) = &job.spec else { + continue; + }; + let Some(spec) = &spec.template.spec else { + continue; + }; + + images.extend(Self::images_of_pod_spec(spec)); + } + for stateful_set in &self.stateful_sets { + let Some(spec) = &stateful_set.spec else { + continue; + }; + let Some(spec) = &spec.template.spec else { + continue; + }; + + images.extend(Self::images_of_pod_spec(spec)); + } + for pod in &self.pods { + let Some(spec) = &pod.spec else { + continue; + }; + + images.extend(Self::images_of_pod_spec(spec)); + } + + images + } + + pub(super) fn apply_image_pull_secret(&mut self, image_pull_secret: Secret) { + let pull_secret_reference = LocalObjectReference { + name: Some(image_pull_secret.metadata.name.clone().unwrap_or_default()), + }; + self.secrets.push(image_pull_secret); + + for deployment in self.deployments.iter_mut() { + let Some(spec) = &mut deployment.spec else { + continue; + }; + let Some(spec) = &mut spec.template.spec else { + continue; + }; + + spec.image_pull_secrets = Some(vec![pull_secret_reference.clone()]); + } + for job in self.jobs.iter_mut() { + let Some(spec) = &mut job.spec else { + continue; + }; + let Some(spec) = &mut spec.template.spec else { + continue; + }; + + spec.image_pull_secrets = Some(vec![pull_secret_reference.clone()]); + } + for stateful_set in self.stateful_sets.iter_mut() { + let Some(spec) = &mut stateful_set.spec else { + continue; + }; + let Some(spec) = &mut spec.template.spec else { + continue; + }; + + spec.image_pull_secrets = Some(vec![pull_secret_reference.clone()]); + } + for pod in self.pods.iter_mut() { + let Some(spec) = &mut pod.spec else { + continue; + }; + + spec.image_pull_secrets = Some(vec![pull_secret_reference.clone()]); + } + } + + pub(super) async fn deploy( + self, + client: Client, + app_name: &AppName, + ) -> Result, Error> { + let mut deployments = Vec::with_capacity(self.deployments.len()); + + for role in self.roles { + create_or_patch(client.clone(), app_name, role).await?; + } + for role_binding in self.role_bindings { + create_or_patch(client.clone(), app_name, role_binding).await?; + } + for config_map in self.config_maps { + create_or_patch(client.clone(), app_name, config_map).await?; + } + for secret in self.secrets { + create_or_patch(client.clone(), app_name, secret).await?; + } + for pvc in self.pvcs { + create_or_patch(client.clone(), app_name, pvc).await?; + } + for service in self.services { + create_or_patch(client.clone(), app_name, service).await?; + } + for service_account in self.service_accounts { + create_or_patch(client.clone(), app_name, service_account).await?; + } + for deployment in self.deployments { + let deployment = create_or_patch(client.clone(), app_name, deployment).await?; + deployments.push(deployment); + } + for job in self.jobs { + create_or_patch(client.clone(), app_name, job).await?; + } + for stateful_set in self.stateful_sets { + create_or_patch(client.clone(), app_name, stateful_set).await?; + } + for ingress in self.traefik_ingresses { + create_or_patch(client.clone(), app_name, ingress).await?; + } + for middleware in self.traefik_middlewares { + create_or_patch(client.clone(), app_name, middleware).await?; + } + for pod in self.pods { + create_or_patch(client.clone(), app_name, pod).await?; + } + + Ok(deployments) + } +} + +async fn create_or_patch(client: Client, app_name: &AppName, payload: T) -> Result +where + T: serde::Serialize + Clone + std::fmt::Debug + for<'a> serde::Deserialize<'a>, + T: kube::core::Resource, + ::DynamicType: std::default::Default, +{ + let api = Api::namespaced(client.clone(), &app_name.to_rfc1123_namespace_id()); + match api.create(&PostParams::default(), &payload).await { + Ok(result) => Ok(result), + Err(kube::error::Error::Api(kube::error::ErrorResponse { code, .. })) if code == 409 => { + let name = payload.meta().name.clone().unwrap_or_default(); + match api + .patch(&name, &PatchParams::default(), &Patch::Merge(&payload)) + .await + { + Ok(result) => Ok(result), + Err(_e) => { + // TODO: how to handle the case? e.g. patching a job may fails + Ok(payload) + } + } + } + Err(e) => Err(e.into()), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{deployment::deployment_unit::DeploymentUnitBuilder, models::ServiceBuilder}; + use k8s_openapi::api::{ + apps::v1::DeploymentSpec, + core::v1::{ContainerPort, EnvVar, PodTemplateSpec}, + }; + use std::collections::HashMap; + + async fn parse_unit(stdout: &'static str) -> K8sDeploymentUnit { + let log_streams = vec![stdout.as_bytes()]; + + let deployment_unit = DeploymentUnitBuilder::init(AppName::master(), Vec::new()) + .extend_with_config(&Default::default()) + .extend_with_templating_only_service_configs(Vec::new()) + .extend_with_image_infos(HashMap::new()) + .apply_templating() + .unwrap() + .apply_hooks(&Default::default()) + .await + .unwrap() + .apply_base_traefik_ingress_route( + crate::infrastructure::TraefikIngressRoute::with_app_only_defaults( + &AppName::master(), + ), + ) + .build(); + + K8sDeploymentUnit::parse_from_log_streams(&deployment_unit, log_streams) + .await + .unwrap() + } + + #[tokio::test] + async fn parse_unit_from_secret_stdout_where_value_is_base64_encoded() { + let unit = parse_unit( + r#" + apiVersion: v1 + kind: Secret + metadata: + name: secret-tls + type: kubernetes.io/tls + data: + # values are base64 encoded, which obscures them but does NOT provide + # any useful level of confidentiality + tls.crt: | + LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUNVakNDQWJzQ0FnMytNQTBHQ1NxR1NJYjNE + UUVCQlFVQU1JR2JNUXN3Q1FZRFZRUUdFd0pLVURFT01Bd0cKQTFVRUNCTUZWRzlyZVc4eEVEQU9C + Z05WQkFjVEIwTm9kVzh0YTNVeEVUQVBCZ05WQkFvVENFWnlZVzVyTkVSRQpNUmd3RmdZRFZRUUxF + dzlYWldKRFpYSjBJRk4xY0hCdmNuUXhHREFXQmdOVkJBTVREMFp5WVc1ck5FUkVJRmRsCllpQkRR + VEVqTUNFR0NTcUdTSWIzRFFFSkFSWVVjM1Z3Y0c5eWRFQm1jbUZ1YXpSa1pDNWpiMjB3SGhjTk1U + TXcKTVRFeE1EUTFNVE01V2hjTk1UZ3dNVEV3TURRMU1UTTVXakJMTVFzd0NRWURWUVFHREFKS1VE + RVBNQTBHQTFVRQpDQXdHWEZSdmEzbHZNUkV3RHdZRFZRUUtEQWhHY21GdWF6UkVSREVZTUJZR0Ex + VUVBd3dQZDNkM0xtVjRZVzF3CmJHVXVZMjl0TUlHYU1BMEdDU3FHU0liM0RRRUJBUVVBQTRHSUFE + Q0JoQUo5WThFaUhmeHhNL25PbjJTbkkxWHgKRHdPdEJEVDFKRjBReTliMVlKanV2YjdjaTEwZjVN + Vm1UQllqMUZTVWZNOU1vejJDVVFZdW4yRFljV29IcFA4ZQpqSG1BUFVrNVd5cDJRN1ArMjh1bklI + QkphVGZlQ09PekZSUFY2MEdTWWUzNmFScG04L3dVVm16eGFLOGtCOWVaCmhPN3F1TjdtSWQxL2pW + cTNKODhDQXdFQUFUQU5CZ2txaGtpRzl3MEJBUVVGQUFPQmdRQU1meTQzeE15OHh3QTUKVjF2T2NS + OEtyNWNaSXdtbFhCUU8xeFEzazlxSGtyNFlUY1JxTVQ5WjVKTm1rWHYxK2VSaGcwTi9WMW5NUTRZ + RgpnWXcxbnlESnBnOTduZUV4VzQyeXVlMFlHSDYyV1hYUUhyOVNVREgrRlowVnQvRGZsdklVTWRj + UUFEZjM4aU9zCjlQbG1kb3YrcE0vNCs5a1h5aDhSUEkzZXZ6OS9NQT09Ci0tLS0tRU5EIENFUlRJ + RklDQVRFLS0tLS0K + # In this example, the key data is not a real PEM-encoded private key + tls.key: | + RXhhbXBsZSBkYXRhIGZvciB0aGUgVExTIGNydCBmaWVsZA== + "#, + ) + .await; + + assert_json_diff::assert_json_eq!( + unit.secrets, + serde_json::json!([{ + "apiVersion": "v1", + "kind": "Secret", + "metadata": { + "name": "secret-tls", + "namespace": "master", + "labels": { + APP_NAME_LABEL: "master" + } + }, + "type": "kubernetes.io/tls", + "data": { + "tls.crt": "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUNVakNDQWJzQ0FnMytNQTBHQ1NxR1NJYjNEUUVCQlFVQU1JR2JNUXN3Q1FZRFZRUUdFd0pLVURFT01Bd0cKQTFVRUNCTUZWRzlyZVc4eEVEQU9CZ05WQkFjVEIwTm9kVzh0YTNVeEVUQVBCZ05WQkFvVENFWnlZVzVyTkVSRQpNUmd3RmdZRFZRUUxFdzlYWldKRFpYSjBJRk4xY0hCdmNuUXhHREFXQmdOVkJBTVREMFp5WVc1ck5FUkVJRmRsCllpQkRRVEVqTUNFR0NTcUdTSWIzRFFFSkFSWVVjM1Z3Y0c5eWRFQm1jbUZ1YXpSa1pDNWpiMjB3SGhjTk1UTXcKTVRFeE1EUTFNVE01V2hjTk1UZ3dNVEV3TURRMU1UTTVXakJMTVFzd0NRWURWUVFHREFKS1VERVBNQTBHQTFVRQpDQXdHWEZSdmEzbHZNUkV3RHdZRFZRUUtEQWhHY21GdWF6UkVSREVZTUJZR0ExVUVBd3dQZDNkM0xtVjRZVzF3CmJHVXVZMjl0TUlHYU1BMEdDU3FHU0liM0RRRUJBUVVBQTRHSUFEQ0JoQUo5WThFaUhmeHhNL25PbjJTbkkxWHgKRHdPdEJEVDFKRjBReTliMVlKanV2YjdjaTEwZjVNVm1UQllqMUZTVWZNOU1vejJDVVFZdW4yRFljV29IcFA4ZQpqSG1BUFVrNVd5cDJRN1ArMjh1bklIQkphVGZlQ09PekZSUFY2MEdTWWUzNmFScG04L3dVVm16eGFLOGtCOWVaCmhPN3F1TjdtSWQxL2pWcTNKODhDQXdFQUFUQU5CZ2txaGtpRzl3MEJBUVVGQUFPQmdRQU1meTQzeE15OHh3QTUKVjF2T2NSOEtyNWNaSXdtbFhCUU8xeFEzazlxSGtyNFlUY1JxTVQ5WjVKTm1rWHYxK2VSaGcwTi9WMW5NUTRZRgpnWXcxbnlESnBnOTduZUV4VzQyeXVlMFlHSDYyV1hYUUhyOVNVREgrRlowVnQvRGZsdklVTWRjUUFEZjM4aU9zCjlQbG1kb3YrcE0vNCs5a1h5aDhSUEkzZXZ6OS9NQT09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K", + "tls.key": "RXhhbXBsZSBkYXRhIGZvciB0aGUgVExTIGNydCBmaWVsZA==" + } + }]) + ) + } + + #[tokio::test] + async fn parse_unit_from_deploymen_stdout() { + let unit = parse_unit( + r#" + apiVersion: apps/v1 + kind: Deployment + metadata: + name: nginx-deployment + labels: + app: nginx + spec: + selector: + matchLabels: + app: nginx + template: + metadata: + labels: + app: nginx + spec: + containers: + - name: nginx + image: nginx:1.14.2 + ports: + - containerPort: 80 + "#, + ) + .await; + + assert_json_diff::assert_json_eq!( + unit.deployments, + serde_json::json!([{ + "apiVersion": "apps/v1", + "kind": "Deployment", + "metadata": { + "name": "nginx-deployment", + "namespace": "master", + "labels": { + "app": "nginx", + APP_NAME_LABEL: "master", + SERVICE_NAME_LABEL: "nginx-deployment", + CONTAINER_TYPE_LABEL: "app-companion" + } + }, + "spec": { + "selector": { + "matchLabels": { + "app": "nginx" + } + }, + "template": { + "metadata": { + "labels": { + "app": "nginx" + } + }, + "spec": { + "containers": [{ + "name": "nginx", + "image": "nginx:1.14.2", + "ports": [{ + "containerPort": 80 + }] + }] + } + } + } + }]) + ) + } + + #[tokio::test] + async fn merge_deployment_into_bootstrapped_deployment() { + let mut unit = parse_unit( + r#" + apiVersion: apps/v1 + kind: Deployment + metadata: + name: nginx + labels: + app: nginx + spec: + selector: + matchLabels: + app: nginx + template: + metadata: + labels: + app: nginx + spec: + containers: + - name: nginx + image: nginx:1.14.2 + ports: + - containerPort: 80 + "#, + ) + .await; + + unit.merge( + None, + Service { + ..Default::default() + }, + Deployment { + metadata: ObjectMeta { + name: Some(String::from("random-name")), + labels: Some(BTreeMap::from([ + (SERVICE_NAME_LABEL.to_string(), String::from("nginx")), + (CONTAINER_TYPE_LABEL.to_string(), String::from("instance")), + ])), + annotations: Some(BTreeMap::from([( + String::from("my-important-annotation"), + String::from("test data"), + )])), + ..Default::default() + }, + spec: Some(DeploymentSpec { + selector: LabelSelector { + match_labels: Some(BTreeMap::from([( + SERVICE_NAME_LABEL.to_string(), + String::from("random-name"), + )])), + ..Default::default() + }, + template: PodTemplateSpec { + metadata: Some(ObjectMeta { + annotations: Some(BTreeMap::from([( + String::from("date"), + String::from("2024-01-01"), + )])), + ..Default::default() + }), + spec: Some(PodSpec { + containers: vec![Container { + name: String::from("random-name"), + image: Some(String::from("nginx:1.29.0")), + env: Some(vec![EnvVar { + name: String::from("NGINX_HOST"), + value: Some(String::from("example.com")), + ..Default::default() + }]), + ports: Some(vec![ContainerPort { + container_port: 4711, + ..Default::default() + }]), + ..Default::default() + }], + ..Default::default() + }), + ..Default::default() + }, + ..Default::default() + }), + ..Default::default() + }, + TraefikIngressRoute { + metadata: Default::default(), + spec: Default::default(), + }, + Vec::new(), + ); + + assert!(unit.secrets.is_empty()); + assert!(unit.services.is_empty()); + assert!(unit.traefik_ingresses.is_empty()); + assert!(unit.traefik_middlewares.is_empty()); + assert_json_diff::assert_json_eq!( + unit.deployments, + serde_json::json!([{ + "apiVersion": "apps/v1", + "kind": "Deployment", + "metadata": { + "name": "nginx", + "namespace": "master", + "labels": { + "app": "nginx", + APP_NAME_LABEL: "master", + SERVICE_NAME_LABEL: "nginx", + CONTAINER_TYPE_LABEL: "instance" + }, + "annotations": { + "my-important-annotation": "test data" + } + }, + "spec": { + "selector": { + "matchLabels": { + "app": "nginx" + } + }, + "template": { + "metadata": { + "labels": { + "app": "nginx" + }, + "annotations": { + "date": "2024-01-01" + } + }, + "spec": { + "containers": [{ + "name": "nginx", + "image": "nginx:1.29.0", + "env": [{ + "name": "NGINX_HOST", + "value": "example.com" + }], + "ports": [{ + "containerPort": 80 + }] + }] + } + } + } + }]) + ) + } + + #[tokio::test] + async fn filter_by_instances_and_replicas() { + let mut unit = parse_unit( + r#" + apiVersion: apps/v1 + kind: Deployment + metadata: + name: nginx + labels: + app: nginx + spec: + selector: + matchLabels: + app: nginx + template: + metadata: + labels: + app: nginx + spec: + containers: + - name: nginx + image: nginx:1.14.2 + ports: + - containerPort: 80 + "#, + ) + .await; + + unit.filter_by_instances_and_replicas(dbg!(&[ServiceBuilder::new() + .app_name(AppName::master().to_string()) + .id(String::from("test")) + .config(crate::sc!("nginx", "nginx:1.15")) + .build() + .unwrap()])); + + assert!(unit.deployments.is_empty()); + } + + #[tokio::test] + async fn filter_not_by_instances_and_replicas() { + let mut unit = parse_unit( + r#" + apiVersion: apps/v1 + kind: Deployment + metadata: + name: nginx + labels: + app: nginx + spec: + selector: + matchLabels: + app: nginx + template: + metadata: + labels: + app: nginx + spec: + containers: + - name: nginx + image: nginx:1.14.2 + ports: + - containerPort: 80 + "#, + ) + .await; + + unit.filter_by_instances_and_replicas(dbg!(&[ServiceBuilder::new() + .app_name(AppName::master().to_string()) + .id(String::from("test")) + .config(crate::sc!("postgres", "postgres")) + .build() + .unwrap()])); + + assert!(!unit.deployments.is_empty()); + } +} diff --git a/api/src/infrastructure/kubernetes/infrastructure.rs b/api/src/infrastructure/kubernetes/infrastructure.rs index 56bca54c..d86b4f30 100644 --- a/api/src/infrastructure/kubernetes/infrastructure.rs +++ b/api/src/infrastructure/kubernetes/infrastructure.rs @@ -27,10 +27,11 @@ use super::super::{ APP_NAME_LABEL, CONTAINER_TYPE_LABEL, IMAGE_LABEL, REPLICATED_ENV_LABEL, SERVICE_NAME_LABEL, STORAGE_TYPE_LABEL, }; +use super::deployment_unit::K8sDeploymentUnit; use super::payloads::{ - deployment_payload, deployment_replicas_payload, image_pull_secret_payload, - ingress_route_payload, middleware_payload, namespace_payload, persistent_volume_claim_payload, - secrets_payload, service_payload, IngressRoute, + deployment_payload, image_pull_secret_payload, ingress_route_payload, middleware_payload, + namespace_payload, persistent_volume_claim_payload, secrets_payload, service_payload, + IngressRoute, Middleware, }; use crate::config::{Config as PREvantConfig, ContainerConfig, Runtime}; use crate::deployment::deployment_unit::{DeployableService, DeploymentUnit}; @@ -43,12 +44,13 @@ use crate::models::{ use async_trait::async_trait; use chrono::{DateTime, FixedOffset, Utc}; use failure::Error; -use futures::future::join_all; +use futures::stream::{FuturesUnordered, Stream}; +use futures::StreamExt; +use k8s_openapi::api::core::v1::PersistentVolumeClaim; use k8s_openapi::api::storage::v1::StorageClass; use k8s_openapi::api::{ apps::v1::Deployment as V1Deployment, core::v1::Namespace as V1Namespace, - core::v1::PersistentVolumeClaim, core::v1::Pod as V1Pod, core::v1::Secret as V1Secret, - core::v1::Service as V1Service, + core::v1::Pod as V1Pod, core::v1::Secret as V1Secret, core::v1::Service as V1Service, }; use kube::{ api::{Api, DeleteParams, ListParams, LogParams, Patch, PatchParams, PostParams}, @@ -62,7 +64,7 @@ use secstr::SecUtf8; use std::collections::{BTreeMap, HashMap}; use std::convert::{From, TryFrom}; use std::net::IpAddr; -use std::path::PathBuf; +use std::pin::Pin; use std::str::FromStr; pub struct KubernetesInfrastructure { @@ -76,11 +78,6 @@ pub enum KubernetesInfrastructureError { internal_message )] UnexpectedError { internal_message: String }, - #[fail( - display = "The deployment {} does not provide a label for service name.", - deployment_name - )] - MissingServiceNameLabel { deployment_name: String }, #[fail( display = "The deployment {} does not provide a label for app name.", deployment_name @@ -118,34 +115,72 @@ impl KubernetesInfrastructure { }) } - async fn create_service_from( + async fn get_deployment( &self, - deployment: V1Deployment, - ) -> Result { - let namespace = deployment.metadata.namespace.clone().unwrap_or_default(); - let mut builder = ServiceBuilder::try_from(deployment)?; + app_name: &AppName, + service_name: &str, + ) -> Result, KubernetesInfrastructureError> { + let client = self.client().await?; + let namespace = app_name.to_rfc1123_namespace_id(); let p = ListParams { - label_selector: Some(format!( - "{}={},{}={}", - APP_NAME_LABEL, - builder - .current_app_name() - .map_or_else(|| "", |name| name.as_str()), - SERVICE_NAME_LABEL, - builder - .current_config() - .map_or_else(|| "", |config| config.service_name()), - )), + label_selector: Some(format!("{SERVICE_NAME_LABEL}={service_name}",)), ..Default::default() }; - if let Some(pod) = Api::::namespaced(self.client().await?, &namespace) + + match Api::::namespaced(client.clone(), &namespace) .list(&p) .await? - .items .into_iter() .next() { + Some(deployment) => Ok(Some(deployment)), + None => Ok(None), + } + } + + async fn get_pod_of_deployment( + &self, + deployment: &V1Deployment, + ) -> Result, KubernetesInfrastructureError> { + let Some(spec) = &deployment.spec else { + return Ok(None); + }; + + match Api::::namespaced( + self.client().await?, + deployment + .metadata + .namespace + .as_ref() + .expect("A namespace should be present for a deployment"), + ) + .list(&ListParams { + label_selector: spec.selector.match_labels.as_ref().map(|labels| { + labels + .iter() + .map(|(k, v)| format!("{k}={v}")) + .collect::>() + .join(",") + }), + ..Default::default() + }) + .await? + .into_iter() + .next() + { + Some(pod) => Ok(Some(pod)), + None => Ok(None), + } + } + + async fn create_service_from( + &self, + deployment: V1Deployment, + ) -> Result { + let mut builder = ServiceBuilder::try_from(deployment.clone())?; + + if let Some(pod) = self.get_pod_of_deployment(&deployment).await? { if let Some(container) = pod.spec.as_ref().and_then(|spec| spec.containers.first()) { builder = builder.started_at( pod.status @@ -179,8 +214,7 @@ impl KubernetesInfrastructure { &self, app_name: &AppName, ) -> Result, KubernetesInfrastructureError> { - let mut services = Vec::new(); - let futures = Api::::namespaced( + let mut futures = Api::::namespaced( self.client().await?, &app_name.to_rfc1123_namespace_id(), ) @@ -188,11 +222,14 @@ impl KubernetesInfrastructure { .await? .items .into_iter() + // FIXME: this performs many network requests to inspect the IPs ip addresses. .map(|deployment| self.create_service_from(deployment)) - .collect::>(); + .map(Box::pin) + .collect::>>>(); - for create_service_result in join_all(futures).await { - let service = match create_service_result { + let mut services = Vec::with_capacity(futures.size_hint().0); + while let Some(service) = futures.next().await { + let service = match service { Ok(service) => service, Err(e) => { debug!("Deployment does not provide required data: {:?}", e); @@ -211,54 +248,14 @@ impl KubernetesInfrastructure { app_name: &AppName, service_name: &str, ) -> Result, KubernetesInfrastructureError> { - let p = ListParams { - label_selector: Some(format!("{SERVICE_NAME_LABEL}={service_name}")), - ..Default::default() - }; + let deployment = self.get_deployment(app_name, service_name).await?; - match Api::::namespaced( - self.client().await?, - &app_name.to_rfc1123_namespace_id(), - ) - .list(&p) - .await? - .items - .into_iter() - .next() - .map(|deployment| self.create_service_from(deployment)) - { - None => Ok(None), + match deployment.map(|deployment| self.create_service_from(deployment)) { Some(service) => Ok(Some(service.await?)), + None => Ok(None), } } - async fn post_service_and_custom_resource_definitions( - &self, - app_name: &AppName, - service: &DeployableService, - ) -> Result<(), KubernetesInfrastructureError> { - let client = self.client().await?; - - Api::namespaced(client.clone(), &app_name.to_rfc1123_namespace_id()) - .create(&PostParams::default(), &service_payload(app_name, service)) - .await?; - - Api::namespaced(client.clone(), &app_name.to_rfc1123_namespace_id()) - .create( - &PostParams::default(), - &ingress_route_payload(app_name, service), - ) - .await?; - - for middleware in middleware_payload(app_name, service) { - Api::namespaced(client.clone(), &app_name.to_rfc1123_namespace_id()) - .create(&PostParams::default(), &middleware) - .await?; - } - - Ok(()) - } - async fn create_namespace_if_necessary( &self, app_name: &AppName, @@ -291,15 +288,13 @@ impl KubernetesInfrastructure { } } - async fn create_pull_secrets_if_necessary( - &self, - app_name: &AppName, - service: &[DeployableService], - ) -> Result<(), KubernetesInfrastructureError> { - let registries_and_credentials: BTreeMap = service - .iter() - .filter_map(|strategy| { - strategy.image().registry().and_then(|registry| { + fn image_pull_secret<'a, I>(&self, app_name: &AppName, images: I) -> Option + where + I: Iterator, + { + let registries_and_credentials: BTreeMap = images + .filter_map(|image| { + image.registry().and_then(|registry| { self.config .registry_credentials(®istry) .map(|(username, password)| (registry, (username, password))) @@ -308,163 +303,49 @@ impl KubernetesInfrastructure { .collect(); if registries_and_credentials.is_empty() { - return Ok(()); + return None; } - match Api::namespaced(self.client().await?, &app_name.to_rfc1123_namespace_id()) - .create( - &PostParams::default(), - &image_pull_secret_payload(app_name, registries_and_credentials), - ) - .await - { - Ok(result) => { - debug!( - "Successfully created image pull secret {}", - result - .metadata - .name - .unwrap_or_else(|| String::from("")) - ); - Ok(()) - } - Err(KubeError::Api(ErrorResponse { code, .. })) if code == 409 => { - debug!("Secrets already exists for {}", app_name); - Ok(()) - } - Err(e) => { - error!("Cannot deploy namespace: {}", e); - Err(e.into()) - } - } + Some(image_pull_secret_payload( + app_name, + registries_and_credentials, + )) } - async fn deploy_service<'a>( + async fn create_payloads( &self, app_name: &AppName, - service: &'a DeployableService, + deployabel_service: &DeployableService, container_config: &ContainerConfig, - ) -> Result<&'a DeployableService, KubernetesInfrastructureError> { - if let Some(files) = service.files() { - self.deploy_secret(app_name, service, files).await?; - } - - let client = self.client().await?; - - let persistence_volume_map = self - .create_persistent_volume_claim(app_name, service) - .await?; - - match Api::namespaced(client.clone(), &app_name.to_rfc1123_namespace_id()) - .create( - &PostParams::default(), - &deployment_payload( - app_name, - service, - container_config, - self.config - .registry_credentials(&service.image().registry().unwrap_or_default()) - .is_some(), - &persistence_volume_map, - ), - ) - .await - { - Ok(result) => { - debug!( - "Successfully deployed {}", - result - .metadata - .name - .unwrap_or_else(|| String::from("")) - ); - self.post_service_and_custom_resource_definitions(app_name, service) - .await?; - Ok(service) - } - - Err(KubeError::Api(ErrorResponse { code, .. })) if code == 409 => { - Api::::namespaced( - client.clone(), - &app_name.to_rfc1123_namespace_id(), - ) - .patch( - &format!( - "{}-{}-deployment", - app_name.to_rfc1123_namespace_id(), - service.service_name() - ), - &PatchParams::default(), - &Patch::Merge(deployment_payload( - app_name, - service, - container_config, - self.config - .registry_credentials(&service.image().registry().unwrap_or_default()) - .is_some(), - &persistence_volume_map, - )), - ) - .await?; - Ok(service) - } - Err(e) => { - error!("Cannot deploy service: {}", e); - Err(e.into()) - } - } - } - - async fn deploy_secret( - &self, - app_name: &AppName, - service_config: &ServiceConfig, - volumes: &BTreeMap, - ) -> Result<(), KubernetesInfrastructureError> { - debug!( - "Deploying volumes as secrets for {} in app {}", - service_config.service_name(), - app_name + ) -> Result< + ( + Option, + V1Service, + V1Deployment, + IngressRoute, + Vec, + ), + KubernetesInfrastructureError, + > { + let secret = deployabel_service + .files() + .map(|files| secrets_payload(app_name, deployabel_service, files)); + + let service = service_payload(app_name, deployabel_service); + + let deployment = deployment_payload( + app_name, + deployabel_service, + container_config, + &self + .create_persistent_volume_claim(app_name, deployabel_service) + .await?, ); - let client = self.client().await?; + let ingress_route = ingress_route_payload(app_name, deployabel_service); + let middlewares = middleware_payload(app_name, deployabel_service.ingress_route()); - match Api::namespaced(client.clone(), &app_name.to_rfc1123_namespace_id()) - .create( - &PostParams::default(), - &secrets_payload(app_name, service_config, volumes), - ) - .await - { - Ok(result) => { - debug!( - "Successfully deployed {}", - result - .metadata - .name - .unwrap_or_else(|| String::from("")) - ); - Ok(()) - } - Err(KubeError::Api(ErrorResponse { code, .. })) if code == 409 => { - Api::::namespaced(client.clone(), &app_name.to_rfc1123_namespace_id()) - .patch( - &format!( - "{}-{}-secret", - app_name.to_rfc1123_namespace_id(), - service_config.service_name() - ), - &PatchParams::default(), - &Patch::Merge(secrets_payload(app_name, service_config, volumes)), - ) - .await?; - Ok(()) - } - Err(e) => { - error!("Cannot deploy secret: {}", e); - Err(e.into()) - } - } + Ok((secret, service, deployment, ingress_route, middlewares)) } async fn create_persistent_volume_claim<'a>( @@ -600,7 +481,6 @@ impl Infrastructure for KubernetesInfrastructure { .collect::>(); let mut apps = MultiMap::new(); - for app_name in app_names { let services = self.get_services_of_app(&app_name).await?; apps.insert_many(app_name, services); @@ -615,24 +495,58 @@ impl Infrastructure for KubernetesInfrastructure { deployment_unit: &DeploymentUnit, container_config: &ContainerConfig, ) -> Result, Error> { - let services = deployment_unit.services(); let app_name = deployment_unit.app_name(); - self.create_namespace_if_necessary(app_name).await?; - self.create_pull_secrets_if_necessary(app_name, services) - .await?; - let futures = services - .iter() - .map(|service| self.deploy_service(app_name, service, container_config)) - .collect::>(); + let client = self.client().await?; + + let bootstrap_image_pull_secret = self.image_pull_secret( + app_name, + self.config + .companion_bootstrapping_containers() + .iter() + .map(|bc| bc.image()), + ); + let mut k8s_deployment_unit = K8sDeploymentUnit::bootstrap( + deployment_unit, + client.clone(), + self.config.companion_bootstrapping_containers(), + bootstrap_image_pull_secret, + ) + .await?; + + let services = self.get_services_of_app(app_name).await?; + k8s_deployment_unit.filter_by_instances_and_replicas(&services); + + // TODO: things like cloning data from existing deployments have to be considered + for deployable_service in deployment_unit.services() { + let (secret, service, deployment, ingress_route, middlewares) = self + .create_payloads(app_name, deployable_service, container_config) + .await?; + + k8s_deployment_unit.merge(secret, service, deployment, ingress_route, middlewares); + } - for deploy_result in join_all(futures).await { - trace!("deployed {:?}", deploy_result); - deploy_result?; + if let Some(image_pull_secret) = + self.image_pull_secret(app_name, k8s_deployment_unit.images().iter()) + { + k8s_deployment_unit.apply_image_pull_secret(image_pull_secret); } - Ok(self.get_services_of_app(app_name).await?) + let deployments = k8s_deployment_unit.deploy(client, app_name).await?; + let mut services = Vec::with_capacity(deployments.len()); + for deployment in deployments.into_iter() { + let service = match self.create_service_from(deployment).await { + Ok(service) => service, + Err(e) => { + debug!("Deployment does not provide required data: {:?}", e); + continue; + } + }; + services.push(service); + } + + Ok(services) } async fn stop_services( @@ -662,19 +576,17 @@ impl Infrastructure for KubernetesInfrastructure { from: &Option>, limit: usize, ) -> Result, String)>>, Error> { - let p = ListParams { - label_selector: Some(format!("{SERVICE_NAME_LABEL}={service_name}",)), - ..Default::default() + let client = self.client().await?; + let namespace = app_name.to_rfc1123_namespace_id(); + + let deployment = match self.get_deployment(app_name, service_name).await? { + Some(deployment) => deployment, + None => { + return Ok(None); + } }; - let pod = match Api::::namespaced( - self.client().await?, - &app_name.to_rfc1123_namespace_id(), - ) - .list(&p) - .await? - .into_iter() - .next() - { + + let pod = match self.get_pod_of_deployment(&deployment).await? { Some(pod) => pod, None => { return Ok(None); @@ -700,10 +612,9 @@ impl Infrastructure for KubernetesInfrastructure { ..Default::default() }; - let logs = - Api::::namespaced(self.client().await?, &app_name.to_rfc1123_namespace_id()) - .logs(&pod.metadata.name.unwrap(), &p) - .await?; + let logs = Api::::namespaced(client, &namespace) + .logs(&pod.metadata.name.unwrap(), &p) + .await?; let logs = logs .split('\n') @@ -745,15 +656,23 @@ impl Infrastructure for KubernetesInfrastructure { None => return Ok(None), }; + let mut deployment = match self.get_deployment(app_name, service_name).await? { + Some(deployment) => deployment, + None => { + return Ok(None); + } + }; + + let Some(spec) = deployment.spec.as_mut() else { + return Ok(None); + }; + spec.replicas = Some(replicas); + Api::::namespaced(self.client().await?, &app_name.to_rfc1123_namespace_id()) .patch( - &format!( - "{}-{}-deployment", - app_name.to_rfc1123_namespace_id(), - service_name - ), + &deployment.metadata.name.clone().unwrap(), &PatchParams::default(), - &Patch::Merge(deployment_replicas_payload(app_name, &service, replicas)), + &Patch::Merge(deployment), ) .await?; @@ -866,9 +785,9 @@ impl TryFrom for ServiceBuilder { deployment .spec .as_ref() - .map(|spec| match (spec.paused, spec.replicas) { - (Some(true), _) => ServiceStatus::Paused, - (Some(false), Some(replicas)) if replicas <= 0 => ServiceStatus::Paused, + .map(|spec| match spec.replicas { + None => ServiceStatus::Paused, + Some(replicas) if replicas <= 0 => ServiceStatus::Paused, _ => ServiceStatus::Running, }) .unwrap_or(ServiceStatus::Paused), @@ -892,24 +811,24 @@ impl TryFrom<&V1Deployment> for ServiceConfig { &deployment.metadata.labels, &deployment.metadata.annotations, ) { - let service_name = match labels.get(SERVICE_NAME_LABEL) { - Some(service_name) => service_name, - None => { - return Err(KubernetesInfrastructureError::MissingServiceNameLabel { - deployment_name: deployment_name.clone(), - }); - } - }; + let service_name = labels.get(SERVICE_NAME_LABEL).unwrap_or(deployment_name); - let image = annotations + let image = match annotations .get(IMAGE_LABEL) - .map(|image| { - Image::from_str(image) - .expect("Kubernetes API should provide valid image string") - }) - .ok_or_else(|| KubernetesInfrastructureError::MissingImageLabel { - deployment_name: deployment_name.clone(), - })?; + .and_then(|image| Image::from_str(image).ok()) + { + Some(img) => img, + None => deployment + .spec + .as_ref() + .and_then(|spec| spec.template.spec.as_ref()) + .and_then(|pod_spec| pod_spec.containers.first()) + .and_then(|container| container.image.as_ref()) + .and_then(|image| Image::from_str(image).ok()) + .ok_or_else(|| KubernetesInfrastructureError::MissingImageLabel { + deployment_name: deployment_name.clone(), + })?, + }; let mut config = ServiceConfig::new(service_name.clone(), image); @@ -1111,7 +1030,7 @@ mod tests { } #[test] - fn should_not_parse_service_from_deployment_spec_missing_service_name_label() { + fn should_parse_service_from_deployment_spec_with_missing_service_name_label() { let deployment = deployment_object!( "master-nginx", Some(String::from("master")), @@ -1120,13 +1039,11 @@ mod tests { None, ); - let err = ServiceBuilder::try_from(deployment).unwrap_err(); - assert_eq!( - err, - KubernetesInfrastructureError::MissingServiceNameLabel { - deployment_name: "master-nginx".to_string() - } - ); + let service = ServiceBuilder::try_from(deployment) + .unwrap() + .build() + .unwrap(); + assert_eq!(service.service_name(), "master-nginx"); } #[test] diff --git a/api/src/infrastructure/kubernetes/mod.rs b/api/src/infrastructure/kubernetes/mod.rs index 38e22459..2f2d1744 100644 --- a/api/src/infrastructure/kubernetes/mod.rs +++ b/api/src/infrastructure/kubernetes/mod.rs @@ -25,5 +25,6 @@ */ pub use infrastructure::KubernetesInfrastructure; +mod deployment_unit; mod infrastructure; mod payloads; diff --git a/api/src/infrastructure/kubernetes/payloads.rs b/api/src/infrastructure/kubernetes/payloads.rs index aac3830b..ff343405 100644 --- a/api/src/infrastructure/kubernetes/payloads.rs +++ b/api/src/infrastructure/kubernetes/payloads.rs @@ -31,17 +31,17 @@ use crate::config::{Config, ContainerConfig}; use crate::deployment::deployment_unit::{DeployableService, DeploymentStrategy}; use crate::infrastructure::traefik::TraefikMiddleware; use crate::infrastructure::{TraefikIngressRoute, TraefikRouterRule}; -use crate::models::service::Service; use crate::models::{AppName, ServiceConfig}; use base64::{engine::general_purpose, Engine}; use bytesize::ByteSize; use chrono::Utc; use k8s_openapi::api::apps::v1::DeploymentSpec; use k8s_openapi::api::core::v1::{ - Container, ContainerPort, EnvVar, KeyToPath, LocalObjectReference, PersistentVolumeClaim, - PersistentVolumeClaimSpec, PersistentVolumeClaimVolumeSource, PodSpec, PodTemplateSpec, - ResourceRequirements, SecretVolumeSource, Volume, VolumeMount, + Container, ContainerPort, EnvVar, KeyToPath, PersistentVolumeClaim, PersistentVolumeClaimSpec, + PersistentVolumeClaimVolumeSource, PodSpec, PodTemplateSpec, ResourceRequirements, + SecretVolumeSource, Volume, VolumeMount, }; +use k8s_openapi::api::networking::v1::Ingress; use k8s_openapi::api::{ apps::v1::Deployment as V1Deployment, core::v1::Namespace as V1Namespace, core::v1::Secret as V1Secret, core::v1::Service as V1Service, @@ -56,8 +56,10 @@ use schemars::JsonSchema; use secstr::SecUtf8; use serde::{Deserialize, Serialize}; use serde_json::{Map, Value}; +use std::collections::hash_map::DefaultHasher; use std::collections::{BTreeMap, HashMap, HashSet}; use std::convert::TryFrom; +use std::hash::Hasher; use std::iter::FromIterator; use std::path::{Component, PathBuf}; use std::str::FromStr; @@ -72,7 +74,7 @@ use std::string::ToString; )] #[serde(rename_all = "camelCase")] pub struct IngressRouteSpec { - pub entrypoints: Option>, + pub entry_points: Option>, pub routes: Option>, pub tls: Option, } @@ -94,13 +96,13 @@ pub struct TraefikRuleService { #[derive(Clone, Debug, Default, Deserialize, Serialize, JsonSchema)] pub struct TraefikRuleMiddleware { - name: String, + pub name: String, } #[derive(Clone, Debug, Default, Deserialize, Serialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct TraefikTls { - cert_resolver: Option, + pub cert_resolver: Option, } #[derive(CustomResource, Clone, Debug, Deserialize, Serialize, JsonSchema)] @@ -111,7 +113,7 @@ pub struct TraefikTls { namespaced )] #[serde(rename_all = "camelCase")] -pub struct MiddlewareSpec(Value); +pub struct MiddlewareSpec(pub Value); macro_rules! secret_name_from_path { ($path:expr) => {{ @@ -139,14 +141,23 @@ macro_rules! secret_name_from_name { } impl TryFrom for TraefikIngressRoute { - type Error = &'static str; + type Error = String; fn try_from(value: IngressRoute) -> Result { - let k8s_route = value.spec.routes.unwrap().into_iter().next().unwrap(); - let rule = TraefikRouterRule::from_str(&k8s_route.r#match).unwrap(); + let Some(routes) = value.spec.routes else { + return Err(String::from( + "The ingress route does not provide any routes", + )); + }; + let Some(k8s_route) = routes.into_iter().next() else { + return Err(String::from( + "The ingress route does not provide any routes", + )); + }; + let rule = TraefikRouterRule::from_str(&k8s_route.r#match)?; Ok(TraefikIngressRoute::with_existing_routing_rules( - value.spec.entrypoints.unwrap_or_default(), + value.spec.entry_points.unwrap_or_default(), rule, k8s_route .middlewares @@ -159,6 +170,134 @@ impl TryFrom for TraefikIngressRoute { } } +pub fn convert_k8s_ingress_to_traefik_ingress( + ingress: Ingress, + base_route: TraefikIngressRoute, +) -> Result<(IngressRoute, Option), &'static str> { + let Some(spec) = ingress.spec else { + return Err("Ingress does not provide spec"); + }; + let Some(rules) = spec.rules else { + return Err("Ingress' spec does not provide rules"); + }; + + let Some(path) = rules + .into_iter() + .filter_map(|rule| rule.http) + .find_map(|http| http.paths.into_iter().next()) + else { + return Err("Ingress' rule does not a provide http paths object"); + }; + + let Some(path_value) = path.path else { + return Err("Ingress' path does not provide a HTTP path value"); + }; + + let (rule, middleware) = match &spec.ingress_class_name { + Some(ingress_class_name) if ingress_class_name == "nginx" => { + let middleware = ingress + .metadata + .annotations + .as_ref() + .filter(|annotations| { + annotations.get("nginx.ingress.kubernetes.io/use-regex") + == Some(&String::from("true")) + }) + .and_then(|annotations| { + annotations + .get("nginx.ingress.kubernetes.io/rewrite-target") + .cloned() + }) + .and_then(|_rewrite_target| { + let hir = regex_syntax::parse(&path_value).ok()?; + let got = regex_syntax::hir::literal::Extractor::new().extract(&hir); + let prefixes = got + .literals()? + .iter() + .map(|l| String::from_utf8_lossy(l.as_bytes()).to_string()) + .map(serde_json::Value::from) + .collect::>(); + + Some(Middleware { + metadata: kube::core::ObjectMeta { + name: Some(uuid::Uuid::new_v4().to_string()), + ..Default::default() + }, + spec: MiddlewareSpec(serde_json::json!({ + "stripPrefix": { + "prefixes": serde_json::Value::from(prefixes) + } + })), + }) + }); + + (None, middleware) + } + _ => { + // TODO warn that ingress class is unknown + ( + Some(TraefikIngressRoute::with_rule( + TraefikRouterRule::path_prefix_rule([path_value.clone()]), + )), + None, + ) + } + }; + + let mut route = base_route; + if let Some(rule) = rule { + route.merge_with(rule); + } + + let mut middlewares = route + .routes() + .iter() + .flat_map(|route| route.middlewares().iter()) + .filter_map(|middleware| match middleware { + crate::infrastructure::traefik::TraefikMiddleware::Ref(name) => { + Some(TraefikRuleMiddleware { name: name.clone() }) + } + crate::infrastructure::traefik::TraefikMiddleware::Spec { .. } => None, + }) + .collect::>(); + middlewares.extend(middleware.as_ref().map(|m| TraefikRuleMiddleware { + name: m.metadata.name.clone().unwrap_or_default(), + })); + + let routes = vec![TraefikRuleSpec { + kind: String::from("Rule"), + r#match: route.routes()[0].rule().to_string(), + middlewares: Some(middlewares), + services: vec![TraefikRuleService { + kind: Some(String::from("Service")), + name: path.backend.service.clone().unwrap().name, + port: Some( + path.backend + .service + .as_ref() + .and_then(|service| service.port.as_ref()) + .and_then(|port| port.number) + .map(|p| p as u16) + // TODO: how to get the if missing + .unwrap_or(80), + ), + }], + }]; + + let route = IngressRoute { + metadata: ingress.metadata, + spec: IngressRouteSpec { + routes: Some(routes), + entry_points: Some(route.entry_points().clone()), + tls: route.tls().as_ref().map(|tls| TraefikTls { + cert_resolver: Some(tls.cert_resolver.clone()), + }), + }, + }; + + Ok((route, middleware)) +} + /// Creates a JSON payload suitable for [Kubernetes' /// Namespaces](https://kubernetes.io/docs/tasks/administer-cluster/namespaces/) pub fn namespace_payload(app_name: &AppName, config: &Config) -> V1Namespace { @@ -202,7 +341,6 @@ pub fn deployment_payload( app_name: &AppName, service: &DeployableService, container_config: &ContainerConfig, - use_image_pull_secret: bool, persistent_volume_map: &Option>, ) -> V1Deployment { let env = service.env().map(|env| { @@ -346,7 +484,7 @@ pub fn deployment_payload( template: PodTemplateSpec { metadata: Some(ObjectMeta { labels: Some(labels), - annotations: Some(deployment_annotations(service)), + annotations: Some(deployment_annotations(service.strategy())), ..Default::default() }), spec: Some(PodSpec { @@ -364,16 +502,6 @@ pub fn deployment_payload( resources, ..Default::default() }], - image_pull_secrets: if use_image_pull_secret { - Some(vec![LocalObjectReference { - name: Some(format!( - "{}-image-pull-secret", - app_name.to_rfc1123_namespace_id() - )), - }]) - } else { - None - }, ..Default::default() }), }, @@ -389,8 +517,8 @@ pub fn deployment_payload( /// For example, this [popular workaround](https://stackoverflow.com/a/55221174/5088458) will be /// applied to ensure that a pod will be recreated everytime a deployment with /// [`DeploymentStrategy::RedeployAlways`] has been initiated. -fn deployment_annotations(service: &DeployableService) -> BTreeMap { - match service.strategy() { +fn deployment_annotations(strategy: &DeploymentStrategy) -> BTreeMap { + match strategy { DeploymentStrategy::RedeployOnImageUpdate(image_id) => { BTreeMap::from([(String::from("imageHash"), image_id.clone())]) } @@ -401,37 +529,6 @@ fn deployment_annotations(service: &DeployableService) -> BTreeMap V1Deployment { - serde_json::from_value(serde_json::json!({ - "apiVersion": "apps/v1", - "kind": "Deployment", - "metadata": { - "name": format!("{}-{}-deployment", app_name.to_rfc1123_namespace_id(), service.service_name()), - "namespace": app_name.to_rfc1123_namespace_id(), - "labels": { - APP_NAME_LABEL: app_name, - SERVICE_NAME_LABEL: service.service_name(), - CONTAINER_TYPE_LABEL: service.container_type().to_string() - } - }, - "spec": { - "replicas": replicas, - "selector": { - "matchLabels": { - APP_NAME_LABEL: app_name, - SERVICE_NAME_LABEL: service.service_name(), - CONTAINER_TYPE_LABEL: service.container_type().to_string() - } - } - } - })) - .expect("Cannot convert value to apps/v1/Deployment") -} - /// Creates a JSON payload suitable for [Kubernetes' Secrets](https://kubernetes.io/docs/concepts/configuration/secret/) pub fn secrets_payload( app_name: &AppName, @@ -468,6 +565,14 @@ pub fn image_pull_secret_payload( app_name: &AppName, registries_and_credentials: BTreeMap, ) -> V1Secret { + // Hashing over all registries ensures that the same secret name will be generated for the same + // registries. Thus, password or user can change and will be updated. Additionally, it will be + // idempontent to the Kubernetes API. + let mut registry_hasher = DefaultHasher::new(); + for registry in registries_and_credentials.keys() { + registry_hasher.write(registry.as_bytes()); + } + let data = ByteString( serde_json::json!({ "auths": @@ -490,8 +595,9 @@ pub fn image_pull_secret_payload( V1Secret { metadata: ObjectMeta { name: Some(format!( - "{}-image-pull-secret", - app_name.to_rfc1123_namespace_id() + "{}-image-pull-secret-{:#010x}", + app_name.to_rfc1123_namespace_id(), + registry_hasher.finish() )), namespace: Some(app_name.to_rfc1123_namespace_id()), labels: Some(BTreeMap::from([( @@ -542,8 +648,9 @@ pub fn service_payload(app_name: &AppName, service_config: &ServiceConfig) -> V1 /// See [Traefik Routers](https://docs.traefik.io/v2.0/user-guides/crd-acme/#traefik-routers) /// for more information. pub fn ingress_route_payload(app_name: &AppName, service: &DeployableService) -> IngressRoute { - let rules = service - .ingress_route() + let route = service.ingress_route(); + + let rules = route .routes() .iter() .map(|route| { @@ -606,18 +713,21 @@ pub fn ingress_route_payload(app_name: &AppName, service: &DeployableService) -> }, spec: IngressRouteSpec { routes: Some(rules), - ..Default::default() + entry_points: Some(route.entry_points().clone()), + tls: route.tls().as_ref().map(|tls| TraefikTls { + cert_resolver: Some(tls.cert_resolver.clone()), + }), }, } } -/// Creates a payload that ensures that Traefik strips out the path prefix. -/// /// See [Traefik Routers](https://docs.traefik.io/v2.0/user-guides/crd-acme/#traefik-routers) /// for more information. -pub fn middleware_payload(app_name: &AppName, service: &DeployableService) -> Vec { - service - .ingress_route() +pub fn middleware_payload( + app_name: &AppName, + ingress_route: &TraefikIngressRoute, +) -> Vec { + ingress_route .routes() .iter() .flat_map(|r| { @@ -758,7 +868,6 @@ mod tests { Vec::new(), ), &ContainerConfig::default(), - false, &None, ); @@ -837,7 +946,6 @@ mod tests { Vec::new(), ), &ContainerConfig::default(), - false, &None, ); @@ -919,7 +1027,6 @@ mod tests { Vec::new(), ), &ContainerConfig::default(), - false, &None, ); @@ -1002,7 +1109,6 @@ mod tests { Vec::new(), ), &ContainerConfig::default(), - false, &None, ); @@ -1157,15 +1263,11 @@ mod tests { #[test] fn should_create_middleware_with_default_prefix() { let app_name = AppName::master(); - let config = sc!("db", "mariadb:10.3.17"); - let service = DeployableService::new( - config, - DeploymentStrategy::RedeployAlways, - TraefikIngressRoute::with_defaults(&app_name, "db"), - Vec::new(), - ); - let payload = middleware_payload(&app_name, &service); + let payload = middleware_payload( + &app_name, + &TraefikIngressRoute::with_defaults(&app_name, "db"), + ); assert_json_diff::assert_json_include!( actual: payload, @@ -1190,15 +1292,11 @@ mod tests { #[test] fn should_create_middleware_with_default_prefix_with_name_rfc1123_app_name() { let app_name = AppName::from_str("MY-APP").unwrap(); - let config = sc!("db", "mariadb:10.3.17"); - let service = DeployableService::new( - config, - DeploymentStrategy::RedeployAlways, - TraefikIngressRoute::with_defaults(&app_name, "db"), - Vec::new(), - ); - let payload = middleware_payload(&app_name, &service); + let payload = middleware_payload( + &app_name, + &TraefikIngressRoute::with_defaults(&app_name, "db"), + ); assert_json_diff::assert_json_include!( actual: payload, @@ -1260,7 +1358,6 @@ mod tests { vec![String::from("/var/lib/data")], ), &ContainerConfig::default(), - false, &Some(HashMap::from([( &String::from("/var/lib/data"), persistent_volume_claim, @@ -1359,7 +1456,6 @@ mod tests { Vec::new(), ), &ContainerConfig::default(), - false, &None, ); @@ -1489,4 +1585,48 @@ mod tests { } ); } + + #[test] + fn create_image_pull_secrets() { + let payload = image_pull_secret_payload( + &AppName::from_str("MY-APP").unwrap(), + BTreeMap::from([( + String::from("registry.gitlab.com"), + ("oauth2", &SecUtf8::from_str("some-random-token").unwrap()), + )]), + ); + + assert_eq!( + payload, + V1Secret { + metadata: ObjectMeta { + name: Some(String::from("my-app-image-pull-secret-0x7a2952c7a89d3fd0")), + namespace: Some(String::from("my-app")), + labels: Some(BTreeMap::from([( + String::from("com.aixigo.preview.servant.app-name"), + String::from("MY-APP") + )])), + ..Default::default() + }, + immutable: Some(true), + data: Some(BTreeMap::from([( + String::from(".dockerconfigjson"), + ByteString( + serde_json::json!({ + "auths": { + "registry.gitlab.com": { + "username": "oauth2", + "password": "some-random-token" + } + } + }) + .to_string() + .into_bytes() + ) + )])), + type_: Some(String::from("kubernetes.io/dockerconfigjson")), + ..Default::default() + } + ) + } } diff --git a/api/src/infrastructure/traefik.rs b/api/src/infrastructure/traefik.rs index dd3edd3d..cc06f641 100644 --- a/api/src/infrastructure/traefik.rs +++ b/api/src/infrastructure/traefik.rs @@ -13,10 +13,18 @@ pub struct TraefikIngressRoute { } impl TraefikIngressRoute { + pub fn entry_points(&self) -> &Vec { + &self.entry_points + } + pub fn routes(&self) -> &Vec { &self.routes } + pub fn tls(&self) -> &Option { + &self.tls + } + #[cfg(test)] pub fn empty() -> Self { Self { @@ -26,14 +34,37 @@ impl TraefikIngressRoute { } } + pub fn with_app_only_defaults(app_name: &AppName) -> Self { + let mut prefixes = BTreeMap::new(); + prefixes.insert( + Value::String(String::from("prefixes")), + Value::Seq(vec![Value::String(format!("/{app_name}/",))]), + ); + + let mut middlewares = BTreeMap::new(); + middlewares.insert( + Value::String(String::from("stripPrefix")), + Value::Map(prefixes), + ); + + Self { + entry_points: Vec::new(), + routes: vec![TraefikRoute { + rule: TraefikRouterRule::path_prefix_rule([app_name.as_str()]), + middlewares: vec![TraefikMiddleware::Spec { + name: format!("{app_name}-middleware"), + spec: Value::Map(middlewares), + }], + }], + tls: None, + } + } + pub fn with_defaults(app_name: &AppName, service_name: &str) -> Self { let mut prefixes = BTreeMap::new(); prefixes.insert( Value::String(String::from("prefixes")), - Value::Seq(vec![Value::String(format!( - "/{}/{}/", - app_name, service_name - ))]), + Value::Seq(vec![Value::String(format!("/{app_name}/{service_name}/",))]), ); let mut middlewares = BTreeMap::new(); @@ -55,7 +86,6 @@ impl TraefikIngressRoute { } } - #[cfg(test)] pub fn with_rule(rule: TraefikRouterRule) -> Self { Self::with_existing_routing_rules(Vec::new(), rule, Vec::new(), None) } @@ -83,7 +113,7 @@ impl TraefikIngressRoute { } pub fn merge_with(&mut self, other: Self) { - self.entry_points.extend(other.entry_points.into_iter()); + self.entry_points.extend(other.entry_points); // FIXME: at the moment there is no handling of multiple routes which needs to be addessed // in the future when it is required. @@ -109,6 +139,47 @@ impl TraefikIngressRoute { (Some(_), Some(tls)) => Some(tls), }; } + + pub fn to_url(&self) -> Option { + let mut domain = None; + let mut path = None; + + match self.routes.first() { + Some(route) => { + let rule = &route.rule; + for m in &rule.matches { + match m { + Matcher::Host { domains } => { + domain = Some(&domains[0]); + } + Matcher::PathPrefix { paths } => { + path = Some(&paths[0]); + } + _ => {} + } + } + } + None => return None, + } + + let scheme = if self.tls.is_some() + || self + .entry_points + .iter() + .any(|entry_point| entry_point == "websecure") + { + "https" + } else { + "http" + }; + + Url::parse(&format!( + "{scheme}://{}{}", + domain?, + path.as_ref().map(|p| p.as_str()).unwrap_or_default() + )) + .ok() + } } #[derive(Clone, Debug, Eq, PartialEq)] @@ -361,7 +432,7 @@ impl Display for TraefikRouterRule { #[derive(Clone, Debug, Eq, PartialEq)] pub struct TraefikTLS { - cert_resolver: String, + pub cert_resolver: String, } #[cfg(test)] @@ -652,8 +723,7 @@ mod test { cert_resolver: String::from("letsencrypt"), }), }; - let route2 = - TraefikIngressRoute::with_defaults(&AppName::from_str("master").unwrap(), "whoami"); + let route2 = TraefikIngressRoute::with_defaults(&AppName::master(), "whoami"); route1.merge_with(route2); @@ -701,8 +771,7 @@ mod test { cert_resolver: String::from("letsencrypt"), }), }; - let mut route2 = - TraefikIngressRoute::with_defaults(&AppName::from_str("master").unwrap(), "whoami"); + let mut route2 = TraefikIngressRoute::with_defaults(&AppName::master(), "whoami"); route2.merge_with(route1); @@ -753,13 +822,13 @@ mod test { let mut route1 = TraefikIngressRoute::empty(); route1.merge_with(TraefikIngressRoute::with_defaults( - &AppName::from_str("master").unwrap(), + &AppName::master(), "test", )); assert_eq!( route1, - TraefikIngressRoute::with_defaults(&AppName::from_str("master").unwrap(), "test",) + TraefikIngressRoute::with_defaults(&AppName::master(), "test",) ); } @@ -787,4 +856,66 @@ mod test { } ); } + + mod to_url { + use super::*; + + #[test] + fn empty_route() { + assert_eq!(TraefikIngressRoute::empty().to_url(), None); + } + + #[test] + fn with_host_rule() { + let url = + TraefikIngressRoute::with_rule(TraefikRouterRule::host_rule(vec![String::from( + "example.com", + )])) + .to_url(); + + assert_eq!(url, Url::parse("http://example.com").ok()); + } + + #[test] + fn with_host_and_path_rule() { + let url = TraefikIngressRoute::with_rule( + TraefikRouterRule::from_str( + "PathPrefix(`/master/whoami/`) && Host(`prevant.example.com`)", + ) + .unwrap(), + ) + .to_url(); + + assert_eq!( + url, + Url::parse("http://prevant.example.com/master/whoami/").ok() + ); + } + + #[test] + fn with_host_rule_and_tls() { + let mut route = + TraefikIngressRoute::with_rule(TraefikRouterRule::host_rule(vec![String::from( + "example.com", + )])); + route.tls = Some(TraefikTLS { + cert_resolver: String::from("first"), + }); + let url = route.to_url(); + + assert_eq!(url, Url::parse("https://example.com").ok()); + } + + #[test] + fn with_host_rule_and_websecure_entrypoint() { + let mut route = + TraefikIngressRoute::with_rule(TraefikRouterRule::host_rule(vec![String::from( + "example.com", + )])); + route.entry_points.push(String::from("websecure")); + let url = route.to_url(); + + assert_eq!(url, Url::parse("https://example.com").ok()); + } + } } diff --git a/api/src/models/app_name.rs b/api/src/models/app_name.rs index 1f87ccf6..d04f31c9 100644 --- a/api/src/models/app_name.rs +++ b/api/src/models/app_name.rs @@ -65,6 +65,12 @@ impl std::fmt::Display for AppName { } } +impl AsRef for AppName { + fn as_ref(&self) -> &str { + self.0.as_str() + } +} + impl FromStr for AppName { type Err = AppNameError; diff --git a/api/src/models/service.rs b/api/src/models/service.rs index 9023e62d..6c36d27c 100644 --- a/api/src/models/service.rs +++ b/api/src/models/service.rs @@ -232,14 +232,6 @@ impl ServiceBuilder { self } - pub fn current_app_name(&self) -> Option<&String> { - self.app_name.as_ref() - } - - pub fn current_config(&self) -> Option<&ServiceConfig> { - self.config.as_ref() - } - pub fn started_at(mut self, started_at: DateTime) -> Self { self.started_at = Some(started_at); self diff --git a/assets/bootstrap-companions.svg b/assets/bootstrap-companions.svg new file mode 100644 index 00000000..34bcf0cf --- /dev/null +++ b/assets/bootstrap-companions.svg @@ -0,0 +1,1146 @@ + + + + + + image/svg+xml + + in-a-nutshell + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + in-a-nutshell + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/companions.md b/docs/companions.md new file mode 100644 index 00000000..15350ffb --- /dev/null +++ b/docs/companions.md @@ -0,0 +1,107 @@ +# Companions + +A companion is a service that is running inside an application + +It is possible to start containers that will be started when the client requests to create a new service. For example, if the application requires an [OpenID](https://en.wikipedia.org/wiki/OpenID_Connect) provider, it is possible to create a configuration that starts the provider for each application. Another use case might be a Kafka services that is required by the application. + +Furthermore, it is also possible to create containers for each service. For example, for each service a database container could be started. + +For these use cases following sections provide example configurations. + +## Application Wide + +If you want to include an OpenID provider for every application, you could use following configuration. + +```toml +[companions.openid] +type = 'application' +image = 'private.example.com/library/openid:latest' +env = [ 'KEY=VALUE' ] +``` + +The provided values of `serviceName` and `env` can include the [handlebars syntax](https://handlebarsjs.com/) in order to access dynamic values. + +Additionally, you could mount files that are generated from handlebars templates (example contains a properties generation): + +```toml +[companions.openid.volumes] +"/path/to/volume.properties" = """ +remote.services={{#each services~}} + {{~#if (eq type 'instance')~}} + {{name}}:{{port}}, + {{~/if~}} +{{~/each~}} +""" +``` + +Furthermore, you can provide labels through handlebars templating: + +```toml +[companions.openid.labels] +"com.github.prevant" = "bar-{{application.name}}" +``` + +### Template Variables + +The list of available handlebars variables: + +- `application`: The companion's application information + - `name`: The application name +- `services`: An array of the services of the application. Each element has following structure: + - `name`: The service name which is equivalent to the network alias + - `port`: The exposed port of the service + - `type`: The type of service. For example, `instance`, `replica`, `app-companion`, or `service-companion`. + +### Handlebar Helpers + +PREvant provides some handlebars helpers which can be used to generate more complex configuration files. See handlerbar's [block helper documentation](https://handlebarsjs.com/block_helpers.html) for more details. + +- `{{#isCompanion }}` A conditional handlerbars block helper that checks if the given service type matches any companion type. +- `isNotCompanion ` A conditional handlerbars block helper that checks if the given service type does not match any companion type. + +## Service Based + +The service-based companions works the in the same way as the application-based services. Make sure, that the `serviceName` is unique by using the handlebars templating. + +```toml +[companions.service-name] +serviceName = '{{service.name}}-db' +image = 'postgres:11' +env = [ 'KEY=VALUE' ] + +[companions.service-name.postgres.volumes] +"/path/to/volume.properties" == "…" +[companions.openid.labels] +"com.github.prevant" = "bar-{{application.name}}" +``` + + +### Template Variables + +The list of available handlebars variables: + +- `application`: The companion's application information + - `name`: The application name +- `service`: The companion's service containing following fields: + - `name`: The service name which is equivalent to the network alias + - `port`: The exposed port of the service + - `type`: The type of service. For example, `instance`, `replica`, `app-companion`, or `service-companion`. + +## Deployment Strategy + +Companions offer different deployment strategies so that a companion could be restarted or not under certain conditions. Therefore, PREvant offers following configuration flags: + +```toml +[companions.openid] +type = 'application' +image = 'private.example.com/library/openid:latest' +deploymentStrategy = 'redeploy-on-image-update' +``` + +`deploymentStrategy` offers following values and if a companion exists for an app following strategy will be applied: + +- `redeploy-always` (_default_): Re-deploys the companion every time there is a new deployment request. +- `redeploy-on-image-update`: Re-deploys the companion if there is a more rescent image available. +- `redeploy-never`: Even if there is a new deployment request the companion won't be redeployed and stays running. + +a From e1de2e8b1c534b85486267ee7d4999bd5f6b8d12 Mon Sep 17 00:00:00 2001 From: Marc Schreiber Date: Thu, 11 Jan 2024 14:14:10 +0100 Subject: [PATCH 2/6] Documentation --- README.md | 41 +- api/README.md | 118 +-- assets/bootstrap-companions.svg | 1430 ++++++++++++++++--------------- docs/companions.md | 106 ++- 4 files changed, 849 insertions(+), 846 deletions(-) diff --git a/README.md b/README.md index 029d5077..432c60c6 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ # PREvant In a Nutshell -PREvant a is Docker container that serves as an abstraction layer between continuous integration pipelines and a container orchestration platform. This abstraction serves as a reviewing platform to ensure that developers have built the features that domain expert requested. +PREvant a is Docker container that serves as an abstraction layer between continuous integration pipelines and a container orchestration platform. This abstraction serves as a reviewing platform to ensure that developers have built the features that domain expert requested. PREvant's name originates from this requirement: _Preview servant (PREvant, `prɪˈvɛnt`, it's pronounced like prevent)_ __serves__ developers to deploy previews of their application as simple as possible when their application consists of multiple microservices distributed across multiple source code repositories. These previews should __PREvant__ to do mistakes in feature development because domain experts can review changes as soon as possible. @@ -14,6 +14,40 @@ Through PREvant's web interface domain experts, managers, developers, and sales ![Access the application](assets/screenshot.png "Access the application") +## Basic Terminology + +An *application*, that PREvant manages, is a composition microservices based on +“architectural pattern that arranges an application as a collection of loosely +coupled, fine-grained services, communicating through lightweight protocols.” +([Wikipedia][wiki-microservices]) Each application has a unique name which is +the key to perform actions like creating, duplicating, modifying, or deleting +these applications via REST API or Web UI. + +In each application, PREvant manages the microservices as *services* which need +to be available in the [OCI Image Format][oci-image-spec] (a.k.a. Docker +images). At least one service needs to be available for an application and +PREvant manages following kind of services. + +- *Instance*: a service labeled as instance is a service that has been + configured explicitly when creating or updating an application. +- *Replica*: a service labeled as replica is a service that has been replicated + from another application. By default if you create an application under any + name PREvant will replicate all instances from the application *master*. + Alternatively, any other application can be specified as a source of + replication. + +Additionally, PREvant provides a way of creating service everytime it creates +an application. These services are called *companions* and there are two types +of them. + +- An application wide companion (app companion) is a unique service for the + whole application. For example, a [Kafka][kafka] instance can be stared + automatically everytime you create an application so that all service within + the application can synchronize via events. +- A companion can also be attached to a service a user wants to deploy (service + companion). For example, a [PostgreSQL][postgres] container can be started + for each to provide a dedicated database for each service. + # Usage Have a look at the examples directory. There you can find examples that deploy PREvant in different container environments: @@ -69,3 +103,8 @@ This paper is based on [the abstract](https://www.conf-micro.services/2019/paper The talk is available on [YouTube](http://www.youtube.com/watch?v=O9GxapQR5bk). Click on the image to start the playback: [![Video “PREvant: Composing Microservices into Reviewable and Testable Applications” at Microservices 2019](http://img.youtube.com/vi/O9GxapQR5bk/0.jpg)](http://www.youtube.com/watch?v=O9GxapQR5bk) + +[wiki-microservices]: https://en.wikipedia.org/wiki/Microservices +[oci-image-spec]: https://specs.opencontainers.org/image-spec/ +[kafka]: https://kafka.apache.org +[postgres]: https://www.postgresql.org diff --git a/api/README.md b/api/README.md index 6b4ca2d3..c70184ed 100644 --- a/api/README.md +++ b/api/README.md @@ -94,123 +94,7 @@ data = "LS0tLS1CRUdJTiBFTkNSWVBURUQgUF…JVkFURSBLRVktLS0tLQo=" ## Companions -It is possible to start containers that will be started when the client requests to create a new service. For example, if the application requires an [OpenID](https://en.wikipedia.org/wiki/OpenID_Connect) provider, it is possible to create a configuration that starts the provider for each application. Another use case might be a Kafka services that is required by the application. - -Furthermore, it is also possible to create containers for each service. For example, for each service a database container could be started. - -For these use cases following sections provide example configurations. - -### Application Wide - -If you want to include an OpenID provider for every application, you could use following configuration. - -```toml -[companions.openid] -type = 'application' -image = 'private.example.com/library/openid:latest' -env = [ 'KEY=VALUE' ] -``` - -The provided values of `serviceName` and `env` can include the [handlebars syntax](https://handlebarsjs.com/) in order to access dynamic values. - -Additionally, you could mount files that are generated from handlebars templates (example contains a properties generation): - -```toml -[companions.openid.volumes] -"/path/to/volume.properties" = """ -remote.services={{#each services~}} - {{~#if (eq type 'instance')~}} - {{name}}:{{port}}, - {{~/if~}} -{{~/each~}} -""" -``` - -Furthermore, you can provide labels through handlebars templating: - -```toml -[companions.openid.labels] -"com.github.prevant" = "bar-{{application.name}}" -``` - -#### Template Variables - -The list of available handlebars variables: - -- `application`: The companion's application information - - `name`: The application name -- `services`: An array of the services of the application. Each element has following structure: - - `name`: The service name which is equivalent to the network alias - - `port`: The exposed port of the service - - `type`: The type of service. For example, `instance`, `replica`, `app-companion`, or `service-companion`. - -#### Handlebar Helpers - -PREvant provides some handlebars helpers which can be used to generate more complex configuration files. See handlerbar's [block helper documentation](https://handlebarsjs.com/block_helpers.html) for more details. - -- `{{#isCompanion }}` A conditional handlerbars block helper that checks if the given service type matches any companion type. -- `isNotCompanion ` A conditional handlerbars block helper that checks if the given service type does not match any companion type. - -### Service Based - -The service-based companions works the in the same way as the application-based services. Make sure, that the `serviceName` is unique by using the handlebars templating. - -```toml -[companions.service-name] -serviceName = '{{service.name}}-db' -image = 'postgres:11' -env = [ 'KEY=VALUE' ] - -[companions.service-name.postgres.volumes] -"/path/to/volume.properties" == "…" -[companions.openid.labels] -"com.github.prevant" = "bar-{{application.name}}" -``` - - -#### Template Variables - -The list of available handlebars variables: - -- `application`: The companion's application information - - `name`: The application name -- `service`: The companion's service containing following fields: - - `name`: The service name which is equivalent to the network alias - - `port`: The exposed port of the service - - `type`: The type of service. For example, `instance`, `replica`, `app-companion`, or `service-companion`. - -### Deployment Strategy - -Companions offer different deployment strategies so that a companion could be restarted or not under certain conditions. Therefore, PREvant offers following configuration flags: - -```toml -[companions.openid] -type = 'application' -image = 'private.example.com/library/openid:latest' -deploymentStrategy = 'redeploy-on-image-update' -``` - -`deploymentStrategy` offers following values and if a companion exists for an app following strategy will be applied: - -- `redeploy-always` (_default_): Re-deploys the companion every time there is a new deployment request. -- `redeploy-on-image-update`: Re-deploys the companion if there is a more rescent image available. -- `redeploy-never`: Even if there is a new deployment request the companion won't be redeployed and stays running. - -### Storage Strategy - -Companions may have varying storage requirements and storage strategies cater to these by offering the below configuration flags: - -```toml -[companions.postgres] -type = 'application' -image = 'postgres:latest' -storageStrategy = 'mount-declared-image-volumes' -``` - -`storageStrategy` offers following values to determine how storage is managed for a companion: - -- `none` (_default_): Companion is deployed without persistent storage. -- `mount-declared-image-volumes`: Mounts the volume paths declared within the image, providing persistent storage for the companion. +See [here](../docs/companions.md) how to configure companions. ## Hooks diff --git a/assets/bootstrap-companions.svg b/assets/bootstrap-companions.svg index 34bcf0cf..2082e8b1 100644 --- a/assets/bootstrap-companions.svg +++ b/assets/bootstrap-companions.svg @@ -6,11 +6,9 @@ xmlns:svg="http://www.w3.org/2000/svg" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" - id="svg8" - viewBox="-110 0 620 198.43" version="1.1" - width="508.82001" - height="198.42999"> + viewBox="0 0 508.82 198.43" + id="svg8"> @@ -30,1084 +28,1072 @@ + height="110.61" + width="156.46001" + y="0.38" + x="0.38" + class="cls-1" /> + id="browser"> + class="cls-2"> - + + height="91.800003" + width="156.57001" + y="19.190001" + x="0.38" + class="cls-4" /> + width="92.860001" + y="7.0300002" + x="5.8400002" + class="cls-5" /> + class="cls-4" /> - + + width="41.419998" + y="40.490002" + x="108.29" + class="cls-7" /> + class="cls-6" /> + class="cls-6" /> + class="cls-6" /> + class="cls-6" /> + class="cls-6" /> + class="cls-6" /> + class="cls-6" /> + class="cls-6" /> + width="41.419998" + y="54.98" + x="108.29" + class="cls-6" /> + id="path870" + d="m 122.79,76.38 a 0.66,0.66 0 0 0 -0.17,-0.5 1.94,1.94 0 0 0 -0.64,-0.33 2.66,2.66 0 0 1 -0.7,-0.35 1.19,1.19 0 0 1 -0.37,-0.41 1.12,1.12 0 0 1 -0.12,-0.53 1.15,1.15 0 0 1 0.35,-0.85 1.25,1.25 0 0 1 0.9,-0.33 1.37,1.37 0 0 1 0.68,0.17 1.1,1.1 0 0 1 0.46,0.47 1.41,1.41 0 0 1 0.16,0.67 h -0.55 a 0.89,0.89 0 0 0 -0.19,-0.62 0.82,0.82 0 0 0 -1.07,0 0.73,0.73 0 0 0 -0.18,0.52 0.6,0.6 0 0 0 0.19,0.46 1.72,1.72 0 0 0 0.61,0.33 2.12,2.12 0 0 1 0.92,0.52 1.16,1.16 0 0 1 0.28,0.81 1.09,1.09 0 0 1 -0.35,0.85 1.31,1.31 0 0 1 -0.93,0.32 1.51,1.51 0 0 1 -0.7,-0.17 1.26,1.26 0 0 1 -0.51,-0.46 1.31,1.31 0 0 1 -0.18,-0.69 h 0.55 a 0.85,0.85 0 0 0 0.22,0.63 0.88,0.88 0 0 0 0.62,0.22 0.7,0.7 0 0 0 0.54,-0.19 0.68,0.68 0 0 0 0.18,-0.54 z" /> + id="path872" + d="m 125.15,77.54 a 1.18,1.18 0 0 1 -0.93,-0.36 1.56,1.56 0 0 1 -0.33,-1.06 v -0.39 a 1.83,1.83 0 0 1 0.32,-1.13 1.05,1.05 0 0 1 0.88,-0.41 1,1 0 0 1 0.85,0.36 1.89,1.89 0 0 1 0.28,1.13 V 76 h -1.8 v 0.08 a 1.16,1.16 0 0 0 0.19,0.75 0.69,0.69 0 0 0 0.57,0.24 0.91,0.91 0 0 0 0.41,-0.09 1,1 0 0 0 0.33,-0.28 l 0.28,0.33 a 1.19,1.19 0 0 1 -1.05,0.51 z m -0.06,-2.89 a 0.56,0.56 0 0 0 -0.49,0.22 1.25,1.25 0 0 0 -0.18,0.7 h 1.27 V 75.5 a 1.27,1.27 0 0 0 -0.16,-0.66 0.52,0.52 0 0 0 -0.44,-0.19 z" /> + id="path874" + d="m 128.19,74.75 a 0.9,0.9 0 0 0 -0.24,0 0.56,0.56 0 0 0 -0.56,0.44 v 2.31 h -0.53 v -3.25 h 0.51 v 0.33 a 0.65,0.65 0 0 1 0.6,-0.39 0.46,0.46 0 0 1 0.21,0 z" /> + id="path876" + d="m 129.69,76.68 0.61,-2.43 h 0.54 l -1,3.23 h -0.39 l -1,-3.23 h 0.53 z" /> + id="path878" + d="m 132,73.4 a 0.32,0.32 0 0 1 -0.07,0.22 0.29,0.29 0 0 1 -0.23,0.09 0.27,0.27 0 0 1 -0.22,-0.09 0.33,0.33 0 0 1 -0.08,-0.22 0.36,0.36 0 0 1 0.08,-0.23 0.27,0.27 0 0 1 0.22,-0.09 0.29,0.29 0 0 1 0.23,0.09 0.37,0.37 0 0 1 0.07,0.23 z m 0,4.08 h -0.53 V 74.25 H 132 Z" /> + id="path880" + d="m 133.92,77.09 a 0.57,0.57 0 0 0 0.42,-0.17 0.69,0.69 0 0 0 0.17,-0.46 h 0.5 a 1.07,1.07 0 0 1 -0.33,0.77 1,1 0 0 1 -0.76,0.31 1.14,1.14 0 0 1 -0.92,-0.38 1.7,1.7 0 0 1 -0.31,-1.12 v -0.36 a 1.73,1.73 0 0 1 0.31,-1.11 1.12,1.12 0 0 1 0.91,-0.38 1,1 0 0 1 0.79,0.32 1.21,1.21 0 0 1 0.31,0.85 h -0.5 a 0.83,0.83 0 0 0 -0.17,-0.53 0.54,0.54 0 0 0 -0.43,-0.18 0.59,0.59 0 0 0 -0.52,0.23 1.38,1.38 0 0 0 -0.18,0.76 v 0.41 a 1.42,1.42 0 0 0 0.17,0.8 0.62,0.62 0 0 0 0.54,0.24 z" /> + id="path882" + d="m 136.78,77.54 a 1.18,1.18 0 0 1 -0.93,-0.36 1.56,1.56 0 0 1 -0.33,-1.06 v -0.39 a 1.83,1.83 0 0 1 0.32,-1.13 1.05,1.05 0 0 1 0.88,-0.41 1,1 0 0 1 0.85,0.36 1.89,1.89 0 0 1 0.28,1.13 V 76 h -1.8 v 0.08 a 1.16,1.16 0 0 0 0.19,0.75 0.69,0.69 0 0 0 0.57,0.24 0.91,0.91 0 0 0 0.41,-0.09 1,1 0 0 0 0.33,-0.28 l 0.28,0.33 a 1.19,1.19 0 0 1 -1.05,0.51 z m -0.06,-2.89 a 0.55,0.55 0 0 0 -0.49,0.22 1.25,1.25 0 0 0 -0.18,0.7 h 1.27 V 75.5 a 1.27,1.27 0 0 0 -0.16,-0.66 0.52,0.52 0 0 0 -0.44,-0.19 z" /> + id="path884" + d="m 140,77.48 v -4.34 h 1.24 a 1.26,1.26 0 0 1 0.9,0.29 1.14,1.14 0 0 1 0.3,0.86 1,1 0 0 1 -0.15,0.54 1,1 0 0 1 -0.42,0.36 0.83,0.83 0 0 1 0.48,0.37 1.28,1.28 0 0 1 0.17,0.67 1.26,1.26 0 0 1 -0.32,0.91 1.19,1.19 0 0 1 -0.92,0.34 z m 0.55,-2.49 h 0.7 a 0.59,0.59 0 0 0 0.46,-0.19 0.72,0.72 0 0 0 0.18,-0.51 0.75,0.75 0 0 0 -0.16,-0.52 0.66,0.66 0 0 0 -0.49,-0.16 h -0.69 z m 0,0.46 V 77 h 0.75 a 0.65,0.65 0 0 0 0.49,-0.2 0.86,0.86 0 0 0 0.18,-0.58 q 0,-0.78 -0.66,-0.78 z" /> + class="cls-8" /> + class="cls-6" /> + id="path890" + d="m 122.39,89.55 a 0.69,0.69 0 0 0 -0.17,-0.5 1.76,1.76 0 0 0 -0.64,-0.34 2.68,2.68 0 0 1 -0.7,-0.34 1.18,1.18 0 0 1 -0.37,-0.41 1.15,1.15 0 0 1 -0.12,-0.53 1.09,1.09 0 0 1 0.35,-0.85 1.25,1.25 0 0 1 0.9,-0.34 1.3,1.3 0 0 1 0.68,0.18 1.16,1.16 0 0 1 0.46,0.47 1.35,1.35 0 0 1 0.16,0.67 h -0.55 a 0.93,0.93 0 0 0 -0.19,-0.63 0.81,0.81 0 0 0 -1.08,0 0.71,0.71 0 0 0 -0.18,0.52 0.63,0.63 0 0 0 0.2,0.46 1.79,1.79 0 0 0 0.61,0.33 2.08,2.08 0 0 1 0.92,0.52 1.21,1.21 0 0 1 0.28,0.81 1.12,1.12 0 0 1 -0.35,0.85 1.32,1.32 0 0 1 -0.93,0.31 1.47,1.47 0 0 1 -0.7,-0.16 1.22,1.22 0 0 1 -0.51,-0.47 1.3,1.3 0 0 1 -0.18,-0.68 h 0.55 a 0.78,0.78 0 0 0 0.84,0.85 0.74,0.74 0 0 0 0.54,-0.19 0.68,0.68 0 0 0 0.18,-0.53 z" /> + id="path892" + d="m 124.74,90.7 a 1.16,1.16 0 0 1 -0.92,-0.36 1.5,1.5 0 0 1 -0.33,-1 V 88.9 a 1.81,1.81 0 0 1 0.31,-1.13 1.06,1.06 0 0 1 0.88,-0.41 1,1 0 0 1 0.85,0.36 1.79,1.79 0 0 1 0.29,1.12 v 0.35 H 124 v 0.07 a 1.2,1.2 0 0 0 0.19,0.76 0.71,0.71 0 0 0 0.56,0.23 0.93,0.93 0 0 0 0.42,-0.09 1.12,1.12 0 0 0 0.33,-0.27 l 0.27,0.33 a 1.21,1.21 0 0 1 -1.03,0.48 z m -0.06,-2.89 a 0.55,0.55 0 0 0 -0.48,0.23 1.25,1.25 0 0 0 -0.18,0.7 h 1.27 v -0.08 a 1.25,1.25 0 0 0 -0.16,-0.65 0.54,0.54 0 0 0 -0.45,-0.2 z" /> + id="path894" + d="m 127.79,87.91 h -0.24 a 0.59,0.59 0 0 0 -0.57,0.45 v 2.3 h -0.53 V 87.42 H 127 v 0.33 a 0.65,0.65 0 0 1 0.6,-0.39 0.46,0.46 0 0 1 0.21,0 z" /> + id="path896" + d="m 129.29,89.84 0.61,-2.42 h 0.54 l -1,3.22 h -0.4 l -1,-3.22 h 0.54 z" /> + id="path898" + d="m 131.6,86.56 a 0.35,0.35 0 0 1 -0.07,0.23 0.29,0.29 0 0 1 -0.23,0.09 0.25,0.25 0 0 1 -0.22,-0.09 0.36,0.36 0 0 1 -0.08,-0.23 0.33,0.33 0 0 1 0.08,-0.22 0.26,0.26 0 0 1 0.22,-0.1 0.27,0.27 0 0 1 0.22,0.1 0.3,0.3 0 0 1 0.08,0.22 z m 0,4.08 H 131 v -3.22 h 0.53 z" /> + id="path900" + d="m 133.51,90.25 a 0.57,0.57 0 0 0 0.43,-0.16 0.71,0.71 0 0 0 0.17,-0.46 h 0.5 a 1.09,1.09 0 0 1 -0.33,0.77 1.06,1.06 0 0 1 -0.77,0.3 1.11,1.11 0 0 1 -0.91,-0.37 1.73,1.73 0 0 1 -0.32,-1.13 v -0.35 a 1.69,1.69 0 0 1 0.32,-1.11 1.12,1.12 0 0 1 0.91,-0.38 1,1 0 0 1 0.79,0.31 1.3,1.3 0 0 1 0.31,0.86 h -0.5 a 0.89,0.89 0 0 0 -0.17,-0.54 0.55,0.55 0 0 0 -0.43,-0.18 0.61,0.61 0 0 0 -0.51,0.19 1.35,1.35 0 0 0 -0.18,0.76 v 0.41 A 1.48,1.48 0 0 0 133,90 a 0.62,0.62 0 0 0 0.51,0.25 z" /> + id="path902" + d="m 136.37,90.7 a 1.16,1.16 0 0 1 -0.92,-0.36 1.5,1.5 0 0 1 -0.33,-1 V 88.9 a 1.81,1.81 0 0 1 0.31,-1.13 1.06,1.06 0 0 1 0.88,-0.41 1,1 0 0 1 0.85,0.36 1.79,1.79 0 0 1 0.29,1.12 v 0.35 h -1.8 v 0.07 a 1.2,1.2 0 0 0 0.19,0.76 0.71,0.71 0 0 0 0.56,0.23 0.93,0.93 0 0 0 0.42,-0.09 1.12,1.12 0 0 0 0.33,-0.27 l 0.27,0.33 a 1.21,1.21 0 0 1 -1.05,0.48 z m -0.06,-2.89 a 0.55,0.55 0 0 0 -0.48,0.23 1.25,1.25 0 0 0 -0.18,0.7 h 1.27 v -0.08 a 1.25,1.25 0 0 0 -0.16,-0.65 0.54,0.54 0 0 0 -0.45,-0.2 z" /> + id="path904" + d="m 142.29,89.26 a 1.68,1.68 0 0 1 -0.39,1.07 1.34,1.34 0 0 1 -1,0.37 1.23,1.23 0 0 1 -1,-0.49 2.13,2.13 0 0 1 -0.39,-1.35 v -0.78 a 2,2 0 0 1 0.4,-1.34 1.3,1.3 0 0 1 1.08,-0.5 1.25,1.25 0 0 1 1,0.38 1.65,1.65 0 0 1 0.38,1.08 h -0.55 a 1.34,1.34 0 0 0 -0.23,-0.76 0.69,0.69 0 0 0 -0.57,-0.23 0.79,0.79 0 0 0 -0.69,0.35 1.79,1.79 0 0 0 -0.24,1 v 0.79 a 1.86,1.86 0 0 0 0.23,1 0.71,0.71 0 0 0 0.65,0.36 0.78,0.78 0 0 0 0.61,-0.22 1.28,1.28 0 0 0 0.24,-0.76 z" /> + width="4.9400001" + y="85.910004" + x="112.72" + class="cls-8" /> + class="cls-6" /> + id="path910" + d="m 122.44,62.6 a 0.7,0.7 0 0 0 -0.17,-0.5 1.77,1.77 0 0 0 -0.64,-0.33 2.66,2.66 0 0 1 -0.7,-0.35 1.18,1.18 0 0 1 -0.37,-0.41 1.12,1.12 0 0 1 -0.12,-0.53 1.1,1.1 0 0 1 0.35,-0.85 1.25,1.25 0 0 1 0.9,-0.33 1.3,1.3 0 0 1 0.68,0.17 1.16,1.16 0 0 1 0.46,0.47 1.38,1.38 0 0 1 0.16,0.67 h -0.55 a 0.89,0.89 0 0 0 -0.19,-0.62 0.7,0.7 0 0 0 -0.56,-0.22 0.73,0.73 0 0 0 -0.52,0.18 0.71,0.71 0 0 0 -0.18,0.52 0.61,0.61 0 0 0 0.2,0.46 1.79,1.79 0 0 0 0.61,0.33 2.18,2.18 0 0 1 0.92,0.52 1.21,1.21 0 0 1 0.28,0.81 1.09,1.09 0 0 1 -0.35,0.85 1.33,1.33 0 0 1 -0.93,0.32 1.48,1.48 0 0 1 -0.7,-0.17 1.2,1.2 0 0 1 -0.51,-0.46 1.31,1.31 0 0 1 -0.18,-0.69 h 0.55 a 0.81,0.81 0 0 0 0.22,0.62 0.85,0.85 0 0 0 0.62,0.23 0.7,0.7 0 0 0 0.54,-0.19 0.68,0.68 0 0 0 0.18,-0.5 z" /> + id="path912" + d="m 124.79,63.76 a 1.16,1.16 0 0 1 -0.92,-0.36 1.51,1.51 0 0 1 -0.33,-1.06 V 62 a 1.83,1.83 0 0 1 0.31,-1.13 1.06,1.06 0 0 1 0.88,-0.41 1,1 0 0 1 0.85,0.36 1.82,1.82 0 0 1 0.29,1.13 v 0.34 h -1.8 v 0.08 a 1.18,1.18 0 0 0 0.19,0.75 0.68,0.68 0 0 0 0.56,0.24 0.93,0.93 0 0 0 0.42,-0.09 1.17,1.17 0 0 0 0.33,-0.28 l 0.27,0.33 a 1.21,1.21 0 0 1 -1.05,0.44 z m -0.06,-2.89 a 0.54,0.54 0 0 0 -0.48,0.22 1.25,1.25 0 0 0 -0.18,0.7 h 1.27 v -0.07 a 1.27,1.27 0 0 0 -0.16,-0.66 0.53,0.53 0 0 0 -0.45,-0.19 z" /> + id="path914" + d="m 127.84,61 a 0.9,0.9 0 0 0 -0.24,0 0.58,0.58 0 0 0 -0.57,0.44 v 2.26 h -0.53 v -3.23 h 0.5 v 0.33 a 0.64,0.64 0 0 1 0.81,-0.35 z" /> + id="path916" + d="m 129.34,62.89 0.61,-2.42 h 0.54 l -1,3.23 h -0.4 l -1,-3.23 h 0.54 z" /> + id="path918" + d="m 131.65,59.62 a 0.32,0.32 0 0 1 -0.07,0.22 0.34,0.34 0 0 1 -0.46,0 0.38,0.38 0 0 1 -0.07,-0.22 0.42,0.42 0 0 1 0.07,-0.23 0.29,0.29 0 0 1 0.23,-0.09 0.27,0.27 0 0 1 0.22,0.09 0.36,0.36 0 0 1 0.08,0.23 z m 0,4.08 h -0.53 v -3.23 h 0.53 z" /> + id="path920" + d="m 133.56,63.31 a 0.58,0.58 0 0 0 0.43,-0.17 0.69,0.69 0 0 0 0.17,-0.46 h 0.5 a 1.09,1.09 0 0 1 -1.1,1.08 1.12,1.12 0 0 1 -0.91,-0.38 1.72,1.72 0 0 1 -0.32,-1.12 V 61.9 a 1.73,1.73 0 0 1 0.32,-1.11 1.12,1.12 0 0 1 0.91,-0.38 1,1 0 0 1 0.79,0.32 1.25,1.25 0 0 1 0.31,0.85 h -0.5 A 0.89,0.89 0 0 0 134,61 a 0.54,0.54 0 0 0 -0.43,-0.17 0.59,0.59 0 0 0 -0.52,0.23 1.29,1.29 0 0 0 -0.18,0.75 v 0.42 a 1.44,1.44 0 0 0 0.17,0.8 0.6,0.6 0 0 0 0.52,0.28 z" /> + id="path922" + d="M 136.42,63.76 A 1.16,1.16 0 0 1 135.5,63.4 1.51,1.51 0 0 1 135.17,62.34 V 62 a 1.83,1.83 0 0 1 0.31,-1.13 1.06,1.06 0 0 1 0.88,-0.41 1,1 0 0 1 0.85,0.36 1.82,1.82 0 0 1 0.29,1.13 v 0.34 h -1.8 v 0.08 a 1.18,1.18 0 0 0 0.19,0.75 0.68,0.68 0 0 0 0.56,0.24 0.93,0.93 0 0 0 0.42,-0.09 1.17,1.17 0 0 0 0.33,-0.28 l 0.27,0.33 a 1.21,1.21 0 0 1 -1.05,0.44 z m -0.06,-2.89 a 0.54,0.54 0 0 0 -0.48,0.22 1.25,1.25 0 0 0 -0.18,0.7 h 1.3 v -0.07 a 1.27,1.27 0 0 0 -0.16,-0.66 0.53,0.53 0 0 0 -0.48,-0.19 z" /> + id="path924" + d="m 141.64,62.56 h -1.47 l -0.34,1.14 h -0.56 l 1.4,-4.34 h 0.47 l 1.4,4.34 H 142 Z m -1.33,-0.47 h 1.19 l -0.6,-2 z" /> + class="cls-8" /> - + + width="41.419998" + y="82.080002" + x="108.29" + class="cls-9" /> - + + x2="149.71001" + y1="68.529999" + x1="108.29" + class="cls-10" /> - + + width="41.419998" + y="40.490002" + x="57.73" + class="cls-7" /> + class="cls-6" /> + class="cls-6" /> + class="cls-6" /> + class="cls-6" /> + class="cls-6" /> + class="cls-6" /> + class="cls-6" /> + width="41.419998" + y="68.529999" + x="57.73" + class="cls-6" /> + id="path956" + d="m 72.24,76.38 a 0.67,0.67 0 0 0 -0.18,-0.5 1.87,1.87 0 0 0 -0.63,-0.33 2.75,2.75 0 0 1 -0.71,-0.35 1.16,1.16 0 0 1 -0.36,-0.41 1.12,1.12 0 0 1 -0.12,-0.53 1.15,1.15 0 0 1 0.34,-0.85 1.26,1.26 0 0 1 0.91,-0.33 1.4,1.4 0 0 1 0.68,0.17 1.16,1.16 0 0 1 0.46,0.47 1.41,1.41 0 0 1 0.16,0.67 h -0.55 a 0.89,0.89 0 0 0 -0.2,-0.62 0.68,0.68 0 0 0 -0.55,-0.22 0.7,0.7 0 0 0 -0.52,0.18 0.73,0.73 0 0 0 -0.18,0.52 0.61,0.61 0 0 0 0.2,0.46 1.63,1.63 0 0 0 0.61,0.33 2.18,2.18 0 0 1 0.92,0.52 1.2,1.2 0 0 1 0.27,0.81 1.09,1.09 0 0 1 -0.34,0.85 1.34,1.34 0 0 1 -0.94,0.32 1.54,1.54 0 0 1 -0.7,-0.17 1.27,1.27 0 0 1 -0.69,-1.15 h 0.55 a 0.78,0.78 0 0 0 0.84,0.85 0.72,0.72 0 0 0 0.55,-0.19 0.68,0.68 0 0 0 0.18,-0.5 z" /> + id="path958" + d="m 74.59,77.54 a 1.2,1.2 0 0 1 -0.93,-0.36 1.56,1.56 0 0 1 -0.32,-1.06 v -0.39 a 1.83,1.83 0 0 1 0.31,-1.13 1.06,1.06 0 0 1 0.88,-0.41 1,1 0 0 1 0.85,0.36 1.89,1.89 0 0 1 0.29,1.13 V 76 h -1.81 v 0.08 a 1.16,1.16 0 0 0 0.2,0.75 0.68,0.68 0 0 0 0.56,0.24 A 0.88,0.88 0 0 0 75,77 1.08,1.08 0 0 0 75.34,76.72 l 0.27,0.33 a 1.19,1.19 0 0 1 -1.02,0.49 z m -0.06,-2.89 a 0.56,0.56 0 0 0 -0.49,0.22 1.35,1.35 0 0 0 -0.17,0.7 h 1.27 V 75.5 a 1.25,1.25 0 0 0 -0.17,-0.66 0.5,0.5 0 0 0 -0.44,-0.19 z" /> + id="path960" + d="m 77.64,74.75 a 1,1 0 0 0 -0.24,0 0.58,0.58 0 0 0 -0.57,0.44 V 77.5 H 76.3 v -3.25 h 0.51 v 0.33 a 0.65,0.65 0 0 1 0.6,-0.39 0.41,0.41 0 0 1 0.21,0 z" /> + id="path962" + d="m 79.13,76.68 0.62,-2.43 h 0.54 l -1,3.23 h -0.4 l -1,-3.23 h 0.54 z" /> + id="path964" + d="m 81.45,73.4 a 0.33,0.33 0 0 1 -0.08,0.22 0.27,0.27 0 0 1 -0.22,0.09 0.29,0.29 0 0 1 -0.23,-0.09 0.32,0.32 0 0 1 -0.07,-0.22 0.35,0.35 0 0 1 0.07,-0.23 0.29,0.29 0 0 1 0.23,-0.09 0.27,0.27 0 0 1 0.22,0.09 0.38,0.38 0 0 1 0.08,0.23 z m 0,4.08 h -0.53 v -3.23 h 0.53 z" /> + id="path966" + d="m 83.36,77.09 a 0.54,0.54 0 0 0 0.42,-0.17 0.65,0.65 0 0 0 0.18,-0.46 h 0.5 A 1.12,1.12 0 0 1 84.12,77.23 1,1 0 0 1 83.36,77.54 1.12,1.12 0 0 1 82.45,77.16 1.65,1.65 0 0 1 82.13,76 v -0.36 a 1.73,1.73 0 0 1 0.31,-1.11 1.14,1.14 0 0 1 0.92,-0.38 1,1 0 0 1 0.79,0.32 1.31,1.31 0 0 1 0.31,0.85 H 84 A 0.83,0.83 0 0 0 83.82,74.79 0.52,0.52 0 0 0 83.4,74.61 0.61,0.61 0 0 0 82.87,74.84 1.31,1.31 0 0 0 82.7,75.6 v 0.41 a 1.34,1.34 0 0 0 0.17,0.8 0.6,0.6 0 0 0 0.49,0.28 z" /> + id="path968" + d="M 86.22,77.54 A 1.2,1.2 0 0 1 85.29,77.18 1.56,1.56 0 0 1 85,76.12 v -0.39 a 1.83,1.83 0 0 1 0.31,-1.13 1.06,1.06 0 0 1 0.88,-0.41 1,1 0 0 1 0.85,0.36 1.89,1.89 0 0 1 0.29,1.13 V 76 h -1.84 v 0.08 a 1.16,1.16 0 0 0 0.2,0.75 0.68,0.68 0 0 0 0.56,0.24 0.88,0.88 0 0 0 0.41,-0.09 A 1.08,1.08 0 0 0 87,76.7 l 0.27,0.33 a 1.19,1.19 0 0 1 -1.05,0.51 z m -0.06,-2.89 a 0.56,0.56 0 0 0 -0.49,0.22 1.35,1.35 0 0 0 -0.17,0.7 h 1.27 V 75.5 A 1.25,1.25 0 0 0 86.6,74.84 0.5,0.5 0 0 0 86.16,74.65 Z" /> + id="path970" + d="m 89.41,77.48 v -4.34 h 1.24 a 1.25,1.25 0 0 1 0.89,0.29 1.14,1.14 0 0 1 0.3,0.86 1,1 0 0 1 -0.56,0.9 0.81,0.81 0 0 1 0.47,0.37 1.2,1.2 0 0 1 0.18,0.67 1.26,1.26 0 0 1 -0.32,0.91 1.21,1.21 0 0 1 -0.92,0.34 z M 90,75 h 0.71 A 0.59,0.59 0 0 0 91.17,74.81 0.77,0.77 0 0 0 91.34,74.3 0.7,0.7 0 0 0 91.18,73.78 0.63,0.63 0 0 0 90.7,73.62 H 90 Z m 0,0.46 V 77 h 0.75 a 0.63,0.63 0 0 0 0.49,-0.2 0.81,0.81 0 0 0 0.19,-0.58 c 0,-0.52 -0.22,-0.78 -0.67,-0.78 z" /> + class="cls-8" /> + class="cls-6" /> + id="path976" + d="M 71.84,89.55 A 0.65,0.65 0 0 0 71.66,89.05 1.76,1.76 0 0 0 71.02,88.71 2.85,2.85 0 0 1 70.32,88.37 1.25,1.25 0 0 1 70,88 a 1,1 0 0 1 -0.13,-0.53 1.12,1.12 0 0 1 0.35,-0.85 1.25,1.25 0 0 1 0.9,-0.34 1.31,1.31 0 0 1 0.69,0.18 1.22,1.22 0 0 1 0.46,0.47 1.47,1.47 0 0 1 0.16,0.67 h -0.55 a 0.93,0.93 0 0 0 -0.2,-0.63 0.72,0.72 0 0 0 -0.56,-0.22 0.69,0.69 0 0 0 -0.51,0.19 0.71,0.71 0 0 0 -0.18,0.52 0.62,0.62 0 0 0 0.19,0.46 2,2 0 0 0 0.61,0.33 2,2 0 0 1 0.92,0.52 1.16,1.16 0 0 1 0.28,0.81 1.12,1.12 0 0 1 -0.34,0.85 1.35,1.35 0 0 1 -0.94,0.31 1.5,1.5 0 0 1 -0.7,-0.16 1.35,1.35 0 0 1 -0.51,-0.47 1.3,1.3 0 0 1 -0.18,-0.68 h 0.55 a 0.85,0.85 0 0 0 0.23,0.62 0.82,0.82 0 0 0 0.61,0.23 0.74,0.74 0 0 0 0.54,-0.19 0.65,0.65 0 0 0 0.15,-0.54 z" /> + id="path978" + d="m 74.19,90.7 a 1.2,1.2 0 0 1 -0.93,-0.36 1.61,1.61 0 0 1 -0.33,-1 V 88.9 a 1.81,1.81 0 0 1 0.32,-1.13 1.05,1.05 0 0 1 0.88,-0.41 1,1 0 0 1 0.85,0.36 1.86,1.86 0 0 1 0.28,1.12 v 0.35 h -1.8 v 0.07 a 1.13,1.13 0 0 0 0.2,0.76 0.69,0.69 0 0 0 0.56,0.23 0.91,0.91 0 0 0 0.74,-0.36 l 0.28,0.33 a 1.21,1.21 0 0 1 -1.05,0.48 z m -0.06,-2.89 a 0.57,0.57 0 0 0 -0.49,0.23 1.44,1.44 0 0 0 -0.18,0.7 h 1.28 v -0.08 a 1.24,1.24 0 0 0 -0.17,-0.65 0.52,0.52 0 0 0 -0.44,-0.2 z" /> + id="path980" + d="M 77.23,87.91 H 77 a 0.58,0.58 0 0 0 -0.57,0.45 v 2.3 H 75.9 v -3.24 h 0.51 v 0.33 a 0.65,0.65 0 0 1 0.6,-0.39 0.41,0.41 0 0 1 0.21,0 z" /> + id="path982" + d="m 78.73,89.84 0.61,-2.42 h 0.54 l -1,3.22 h -0.39 l -1,-3.22 h 0.54 z" /> + id="path984" + d="m 81,86.56 a 0.35,0.35 0 0 1 -0.07,0.23 0.29,0.29 0 0 1 -0.23,0.09 0.27,0.27 0 0 1 -0.22,-0.09 0.36,0.36 0 0 1 -0.08,-0.23 0.33,0.33 0 0 1 0.08,-0.22 0.27,0.27 0 0 1 0.22,-0.1 0.29,0.29 0 0 1 0.23,0.1 0.34,0.34 0 0 1 0.07,0.22 z m 0,4.08 H 80.47 V 87.42 H 81 Z" /> + id="path986" + d="m 83,90.25 a 0.57,0.57 0 0 0 0.42,-0.16 0.66,0.66 0 0 0 0.17,-0.46 h 0.5 A 1.09,1.09 0 0 1 83.76,90.4 1.06,1.06 0 0 1 83,90.7 1.14,1.14 0 0 1 82.08,90.33 1.73,1.73 0 0 1 81.77,89.2 v -0.35 a 1.69,1.69 0 0 1 0.23,-1.11 1.14,1.14 0 0 1 0.92,-0.38 1,1 0 0 1 0.78,0.31 1.25,1.25 0 0 1 0.31,0.86 h -0.5 a 0.89,0.89 0 0 0 -0.17,-0.54 0.52,0.52 0 0 0 -0.42,-0.18 0.63,0.63 0 0 0 -0.53,0.23 1.35,1.35 0 0 0 -0.17,0.76 v 0.41 a 1.49,1.49 0 0 0 0.16,0.81 0.65,0.65 0 0 0 0.62,0.23 z" /> + id="path988" + d="m 85.82,90.7 a 1.2,1.2 0 0 1 -0.93,-0.36 1.61,1.61 0 0 1 -0.33,-1 V 88.9 a 1.81,1.81 0 0 1 0.32,-1.13 1.05,1.05 0 0 1 0.88,-0.41 1,1 0 0 1 0.85,0.36 1.86,1.86 0 0 1 0.28,1.12 v 0.35 h -1.8 v 0.07 a 1.13,1.13 0 0 0 0.2,0.76 0.69,0.69 0 0 0 0.56,0.23 0.91,0.91 0 0 0 0.74,-0.36 l 0.28,0.33 a 1.21,1.21 0 0 1 -1.05,0.48 z m -0.06,-2.89 a 0.57,0.57 0 0 0 -0.49,0.23 1.44,1.44 0 0 0 -0.18,0.7 h 1.28 V 88.66 A 1.24,1.24 0 0 0 86.2,88 0.52,0.52 0 0 0 85.76,87.81 Z" /> + id="path990" + d="m 91.73,89.26 a 1.58,1.58 0 0 1 -0.39,1.07 1.32,1.32 0 0 1 -1,0.37 1.26,1.26 0 0 1 -1,-0.49 2.12,2.12 0 0 1 -0.38,-1.35 v -0.78 a 2.08,2.08 0 0 1 0.39,-1.34 1.33,1.33 0 0 1 1.08,-0.5 1.27,1.27 0 0 1 1,0.38 1.65,1.65 0 0 1 0.37,1.08 h -0.55 a 1.34,1.34 0 0 0 -0.22,-0.76 0.72,0.72 0 0 0 -0.58,-0.23 0.78,0.78 0 0 0 -0.68,0.35 1.79,1.79 0 0 0 -0.24,1 v 0.79 a 1.87,1.87 0 0 0 0.22,1 0.73,0.73 0 0 0 0.65,0.36 A 0.8,0.8 0 0 0 91,90 1.36,1.36 0 0 0 91.23,89.24 Z" /> + width="4.9400001" + y="85.910004" + x="62.16" + class="cls-8" /> + class="cls-6" /> + id="path996" + d="M 71.89,62.6 A 0.67,0.67 0 0 0 71.71,62.1 1.77,1.77 0 0 0 71.07,61.77 2.83,2.83 0 0 1 70.37,61.42 1.42,1.42 0 0 1 70,61 a 1.12,1.12 0 0 1 -0.12,-0.53 1.14,1.14 0 0 1 0.35,-0.85 1.25,1.25 0 0 1 0.9,-0.33 1.31,1.31 0 0 1 0.69,0.17 1.22,1.22 0 0 1 0.46,0.47 1.5,1.5 0 0 1 0.16,0.67 h -0.55 a 0.89,0.89 0 0 0 -0.2,-0.62 0.82,0.82 0 0 0 -1.07,0 0.71,0.71 0 0 0 -0.18,0.52 0.6,0.6 0 0 0 0.19,0.46 2,2 0 0 0 0.61,0.33 2.12,2.12 0 0 1 0.92,0.52 1.16,1.16 0 0 1 0.28,0.81 1.09,1.09 0 0 1 -0.34,0.85 1.35,1.35 0 0 1 -0.94,0.32 1.51,1.51 0 0 1 -0.7,-0.17 1.33,1.33 0 0 1 -0.51,-0.46 1.31,1.31 0 0 1 -0.18,-0.69 h 0.55 a 0.79,0.79 0 0 0 0.84,0.85 0.7,0.7 0 0 0 0.54,-0.19 0.65,0.65 0 0 0 0.19,-0.53 z" /> + id="path998" + d="M 74.24,63.76 A 1.2,1.2 0 0 1 73.31,63.4 1.62,1.62 0 0 1 73,62.34 V 62 a 1.83,1.83 0 0 1 0.32,-1.13 1.05,1.05 0 0 1 0.88,-0.41 1,1 0 0 1 0.85,0.36 1.89,1.89 0 0 1 0.28,1.13 v 0.34 h -1.8 v 0.08 a 1.12,1.12 0 0 0 0.2,0.75 0.67,0.67 0 0 0 0.56,0.24 0.91,0.91 0 0 0 0.41,-0.09 1,1 0 0 0 0.33,-0.28 l 0.28,0.33 a 1.21,1.21 0 0 1 -1.07,0.44 z m -0.06,-2.89 a 0.56,0.56 0 0 0 -0.49,0.22 1.44,1.44 0 0 0 -0.18,0.7 h 1.28 v -0.07 a 1.25,1.25 0 0 0 -0.17,-0.66 0.51,0.51 0 0 0 -0.44,-0.19 z" /> + id="path1000" + d="M 77.28,61 A 0.9,0.9 0 0 0 77,61 0.56,0.56 0 0 0 76.44,61.44 V 63.7 H 76 v -3.23 h 0.51 v 0.33 a 0.65,0.65 0 0 1 0.6,-0.39 0.56,0.56 0 0 1 0.21,0 z" /> + id="path1002" + d="m 78.78,62.89 0.61,-2.42 h 0.54 L 79,63.7 h -0.39 l -1,-3.23 h 0.54 z" /> + id="path1004" + d="m 81.09,59.62 a 0.32,0.32 0 0 1 -0.07,0.22 0.29,0.29 0 0 1 -0.23,0.09 0.27,0.27 0 0 1 -0.22,-0.09 0.33,0.33 0 0 1 -0.08,-0.22 0.36,0.36 0 0 1 0.08,-0.23 0.27,0.27 0 0 1 0.22,-0.09 0.29,0.29 0 0 1 0.23,0.09 0.35,0.35 0 0 1 0.07,0.23 z m 0,4.08 h -0.53 v -3.23 h 0.53 z" /> + id="path1006" + d="m 83,63.31 a 0.57,0.57 0 0 0 0.42,-0.17 0.64,0.64 0 0 0 0.17,-0.46 h 0.5 A 1.09,1.09 0 0 1 83,63.76 1.14,1.14 0 0 1 82.08,63.38 1.72,1.72 0 0 1 81.77,62.26 V 61.9 a 1.73,1.73 0 0 1 0.31,-1.11 1.14,1.14 0 0 1 0.92,-0.38 1,1 0 0 1 0.78,0.32 1.21,1.21 0 0 1 0.31,0.85 h -0.5 A 0.89,0.89 0 0 0 83.42,61.04 0.51,0.51 0 0 0 83,60.87 0.61,0.61 0 0 0 82.47,61.1 1.3,1.3 0 0 0 82.3,61.85 v 0.42 a 1.45,1.45 0 0 0 0.16,0.8 0.63,0.63 0 0 0 0.54,0.24 z" /> + id="path1008" + d="M 85.87,63.76 A 1.2,1.2 0 0 1 84.94,63.4 1.62,1.62 0 0 1 84.61,62.34 V 62 a 1.83,1.83 0 0 1 0.32,-1.13 1.05,1.05 0 0 1 0.88,-0.41 1,1 0 0 1 0.85,0.36 1.89,1.89 0 0 1 0.28,1.13 v 0.34 h -1.8 v 0.08 a 1.18,1.18 0 0 0 0.19,0.75 0.7,0.7 0 0 0 0.57,0.24 0.91,0.91 0 0 0 0.41,-0.09 1,1 0 0 0 0.33,-0.28 l 0.28,0.33 a 1.21,1.21 0 0 1 -1.05,0.44 z m -0.06,-2.89 a 0.56,0.56 0 0 0 -0.49,0.22 1.44,1.44 0 0 0 -0.18,0.7 h 1.28 v -0.07 a 1.25,1.25 0 0 0 -0.17,-0.66 0.51,0.51 0 0 0 -0.44,-0.19 z" /> + id="path1010" + d="m 91.08,62.56 h -1.47 l -0.33,1.14 h -0.57 l 1.41,-4.34 h 0.46 L 92,63.7 h -0.56 z m -1.32,-0.47 h 1.18 l -0.59,-2 z" /> + class="cls-8" /> - + + width="41.419998" + y="82.080002" + x="57.73" + class="cls-9" /> - + + x2="99.160004" + y1="68.529999" + x1="57.73" + class="cls-10" /> - + + width="41.419998" + y="40.490002" + x="6.4099998" + class="cls-7" /> + class="cls-6" /> + class="cls-6" /> + class="cls-6" /> + class="cls-6" /> + class="cls-6" /> + class="cls-6" /> - - + + + width="41.419998" + y="82.080002" + x="6.4099998" + class="cls-6" /> + id="path1044" + d="m 20.92,76.38 a 0.67,0.67 0 0 0 -0.18,-0.5 1.94,1.94 0 0 0 -0.64,-0.33 2.83,2.83 0 0 1 -0.7,-0.35 1.3,1.3 0 0 1 -0.37,-0.41 1.12,1.12 0 0 1 -0.12,-0.53 1.15,1.15 0 0 1 0.35,-0.85 1.25,1.25 0 0 1 0.9,-0.33 1.34,1.34 0 0 1 0.68,0.17 1.1,1.1 0 0 1 0.46,0.47 1.42,1.42 0 0 1 0.17,0.67 h -0.55 a 0.89,0.89 0 0 0 -0.2,-0.62 0.82,0.82 0 0 0 -1.07,0 0.73,0.73 0 0 0 -0.18,0.52 0.6,0.6 0 0 0 0.19,0.46 1.72,1.72 0 0 0 0.61,0.33 2.12,2.12 0 0 1 0.92,0.52 1.16,1.16 0 0 1 0.28,0.81 1.09,1.09 0 0 1 -0.35,0.85 1.31,1.31 0 0 1 -0.93,0.32 1.51,1.51 0 0 1 -0.7,-0.17 1.26,1.26 0 0 1 -0.51,-0.46 1.31,1.31 0 0 1 -0.18,-0.69 h 0.55 a 0.82,0.82 0 0 0 0.23,0.63 0.86,0.86 0 0 0 0.61,0.22 0.7,0.7 0 0 0 0.54,-0.19 0.65,0.65 0 0 0 0.19,-0.54 z" /> + id="path1046" + d="M 23.27,77.54 A 1.2,1.2 0 0 1 22.34,77.18 1.62,1.62 0 0 1 22,76.12 v -0.39 a 1.83,1.83 0 0 1 0.32,-1.13 1.05,1.05 0 0 1 0.88,-0.41 1,1 0 0 1 0.85,0.36 1.89,1.89 0 0 1 0.28,1.13 V 76 h -1.8 v 0.08 a 1.16,1.16 0 0 0 0.19,0.75 0.7,0.7 0 0 0 0.57,0.24 0.91,0.91 0 0 0 0.41,-0.09 1,1 0 0 0 0.33,-0.28 l 0.28,0.33 a 1.19,1.19 0 0 1 -1.04,0.51 z m -0.06,-2.89 a 0.56,0.56 0 0 0 -0.49,0.22 1.44,1.44 0 0 0 -0.18,0.7 h 1.28 V 75.5 a 1.25,1.25 0 0 0 -0.17,-0.66 0.51,0.51 0 0 0 -0.44,-0.19 z" /> + id="path1048" + d="m 26.31,74.75 a 0.9,0.9 0 0 0 -0.24,0 0.56,0.56 0 0 0 -0.56,0.44 V 77.5 H 25 v -3.25 h 0.51 v 0.33 a 0.65,0.65 0 0 1 0.6,-0.39 0.46,0.46 0 0 1 0.21,0 z" /> + id="path1050" + d="m 27.81,76.68 0.61,-2.43 H 29 l -1,3.23 h -0.39 l -1,-3.23 h 0.53 z" /> + id="path1052" + d="m 30.12,73.4 a 0.32,0.32 0 0 1 -0.07,0.22 0.29,0.29 0 0 1 -0.23,0.09 0.27,0.27 0 0 1 -0.22,-0.09 0.33,0.33 0 0 1 -0.08,-0.22 0.36,0.36 0 0 1 0.08,-0.23 0.27,0.27 0 0 1 0.22,-0.09 0.29,0.29 0 0 1 0.23,0.09 0.37,0.37 0 0 1 0.07,0.23 z m 0,4.08 h -0.53 v -3.23 h 0.53 z" /> + id="path1054" + d="m 32,77.09 a 0.57,0.57 0 0 0 0.42,-0.17 0.69,0.69 0 0 0 0.17,-0.46 h 0.5 A 1.07,1.07 0 0 1 32.76,77.23 1,1 0 0 1 32,77.54 1.14,1.14 0 0 1 31.08,77.16 1.7,1.7 0 0 1 30.81,76 v -0.36 a 1.73,1.73 0 0 1 0.31,-1.11 1.13,1.13 0 0 1 0.92,-0.38 1,1 0 0 1 0.78,0.32 1.21,1.21 0 0 1 0.31,0.85 h -0.5 a 0.83,0.83 0 0 0 -0.17,-0.53 0.52,0.52 0 0 0 -0.42,-0.18 0.6,0.6 0 0 0 -0.53,0.23 1.31,1.31 0 0 0 -0.17,0.76 v 0.41 a 1.43,1.43 0 0 0 0.16,0.8 0.62,0.62 0 0 0 0.5,0.28 z" /> + id="path1056" + d="M 34.9,77.54 A 1.2,1.2 0 0 1 33.97,77.18 1.62,1.62 0 0 1 33.64,76.12 V 75.73 A 1.83,1.83 0 0 1 34,74.6 1.05,1.05 0 0 1 34.88,74.19 1,1 0 0 1 35.73,74.55 1.89,1.89 0 0 1 36,75.68 V 76 h -1.8 v 0.08 a 1.16,1.16 0 0 0 0.19,0.75 0.7,0.7 0 0 0 0.57,0.24 0.91,0.91 0 0 0 0.41,-0.09 1,1 0 0 0 0.33,-0.28 l 0.28,0.33 a 1.19,1.19 0 0 1 -1.08,0.51 z m -0.06,-2.89 a 0.56,0.56 0 0 0 -0.49,0.22 1.44,1.44 0 0 0 -0.18,0.7 h 1.28 V 75.5 a 1.25,1.25 0 0 0 -0.17,-0.66 0.51,0.51 0 0 0 -0.44,-0.19 z" /> + id="path1058" + d="m 38.08,77.48 v -4.34 h 1.24 a 1.26,1.26 0 0 1 0.9,0.29 1.14,1.14 0 0 1 0.3,0.86 1,1 0 0 1 -0.15,0.54 0.93,0.93 0 0 1 -0.42,0.36 0.83,0.83 0 0 1 0.48,0.37 1.28,1.28 0 0 1 0.17,0.67 1.22,1.22 0 0 1 -0.32,0.91 1.18,1.18 0 0 1 -0.91,0.34 z M 38.63,75 h 0.7 a 0.59,0.59 0 0 0 0.46,-0.19 0.72,0.72 0 0 0 0.18,-0.51 0.75,0.75 0 0 0 -0.16,-0.52 0.66,0.66 0 0 0 -0.49,-0.16 h -0.69 z m 0,0.46 V 77 h 0.75 a 0.65,0.65 0 0 0 0.49,-0.2 0.86,0.86 0 0 0 0.18,-0.58 q 0,-0.78 -0.66,-0.78 z" /> + class="cls-8" /> + class="cls-6" /> + id="path1064" + d="m 20.51,89.55 a 0.65,0.65 0 0 0 -0.17,-0.5 1.76,1.76 0 0 0 -0.64,-0.34 2.68,2.68 0 0 1 -0.7,-0.34 1.18,1.18 0 0 1 -0.37,-0.41 1.15,1.15 0 0 1 -0.12,-0.53 1.12,1.12 0 0 1 0.35,-0.85 1.25,1.25 0 0 1 0.9,-0.34 1.28,1.28 0 0 1 0.68,0.18 1.16,1.16 0 0 1 0.46,0.47 1.35,1.35 0 0 1 0.16,0.67 h -0.55 a 0.93,0.93 0 0 0 -0.19,-0.63 0.74,0.74 0 0 0 -0.56,-0.22 0.69,0.69 0 0 0 -0.51,0.19 0.68,0.68 0 0 0 -0.19,0.52 0.63,0.63 0 0 0 0.2,0.46 1.89,1.89 0 0 0 0.61,0.33 2,2 0 0 1 0.92,0.52 1.21,1.21 0 0 1 0.28,0.81 1.12,1.12 0 0 1 -0.35,0.85 1.32,1.32 0 0 1 -0.93,0.31 1.47,1.47 0 0 1 -0.7,-0.16 1.22,1.22 0 0 1 -0.51,-0.47 1.3,1.3 0 0 1 -0.18,-0.68 H 19 a 0.78,0.78 0 0 0 0.84,0.85 0.74,0.74 0 0 0 0.54,-0.19 0.68,0.68 0 0 0 0.13,-0.5 z" /> + id="path1066" + d="m 22.87,90.7 a 1.17,1.17 0 0 1 -0.93,-0.36 1.5,1.5 0 0 1 -0.33,-1 V 88.9 a 1.81,1.81 0 0 1 0.32,-1.13 1.05,1.05 0 0 1 0.88,-0.41 1,1 0 0 1 0.84,0.36 1.79,1.79 0 0 1 0.29,1.12 v 0.35 h -1.8 v 0.07 a 1.2,1.2 0 0 0 0.19,0.76 0.72,0.72 0 0 0 0.57,0.23 0.91,0.91 0 0 0 0.41,-0.09 1,1 0 0 0 0.33,-0.27 l 0.27,0.33 a 1.18,1.18 0 0 1 -1.04,0.48 z m -0.06,-2.89 a 0.56,0.56 0 0 0 -0.49,0.23 1.25,1.25 0 0 0 -0.18,0.7 h 1.27 v -0.08 a 1.25,1.25 0 0 0 -0.16,-0.65 0.53,0.53 0 0 0 -0.44,-0.2 z" /> + id="path1068" + d="m 25.91,87.91 h -0.24 a 0.59,0.59 0 0 0 -0.57,0.45 v 2.3 h -0.52 v -3.24 h 0.51 v 0.33 a 0.65,0.65 0 0 1 0.6,-0.39 0.46,0.46 0 0 1 0.21,0 z" /> + id="path1070" + d="M 27.41,89.84 28,87.42 h 0.54 l -1,3.22 h -0.39 l -1,-3.22 h 0.54 z" /> + id="path1072" + d="m 29.72,86.56 a 0.35,0.35 0 0 1 -0.07,0.23 0.29,0.29 0 0 1 -0.23,0.09 0.27,0.27 0 0 1 -0.22,-0.09 0.36,0.36 0 0 1 -0.08,-0.23 0.33,0.33 0 0 1 0.08,-0.22 0.27,0.27 0 0 1 0.22,-0.1 0.31,0.31 0 0 1 0.23,0.1 0.34,0.34 0 0 1 0.07,0.22 z m 0,4.08 h -0.53 v -3.22 h 0.53 z" /> + id="path1074" + d="m 31.64,90.25 a 0.57,0.57 0 0 0 0.42,-0.16 0.71,0.71 0 0 0 0.17,-0.46 h 0.5 A 1.09,1.09 0 0 1 32.4,90.4 1.06,1.06 0 0 1 31.64,90.7 1.14,1.14 0 0 1 30.72,90.33 1.73,1.73 0 0 1 30.4,89.2 v -0.35 a 1.69,1.69 0 0 1 0.32,-1.11 1.12,1.12 0 0 1 0.91,-0.38 1,1 0 0 1 0.79,0.31 1.3,1.3 0 0 1 0.31,0.86 h -0.5 a 0.89,0.89 0 0 0 -0.17,-0.54 0.54,0.54 0 0 0 -0.43,-0.18 0.61,0.61 0 0 0 -0.52,0.23 1.43,1.43 0 0 0 -0.18,0.76 v 0.41 a 1.48,1.48 0 0 0 0.17,0.81 0.63,0.63 0 0 0 0.54,0.23 z" /> + id="path1076" + d="m 34.5,90.7 a 1.17,1.17 0 0 1 -0.93,-0.36 1.5,1.5 0 0 1 -0.33,-1 V 88.9 a 1.81,1.81 0 0 1 0.32,-1.13 1.05,1.05 0 0 1 0.88,-0.41 1,1 0 0 1 0.84,0.36 1.79,1.79 0 0 1 0.29,1.12 v 0.35 h -1.8 v 0.07 A 1.2,1.2 0 0 0 34,90 a 0.72,0.72 0 0 0 0.57,0.23 0.91,0.91 0 0 0 0.41,-0.09 1,1 0 0 0 0.33,-0.27 l 0.27,0.33 A 1.18,1.18 0 0 1 34.5,90.7 Z M 34.44,87.81 A 0.56,0.56 0 0 0 34,88 1.25,1.25 0 0 0 33.82,88.7 H 35 v -0.08 a 1.25,1.25 0 0 0 -0.16,-0.65 0.53,0.53 0 0 0 -0.4,-0.16 z" /> + id="path1078" + d="M 40.41,89.26 A 1.63,1.63 0 0 1 40,90.33 1.34,1.34 0 0 1 39,90.7 1.24,1.24 0 0 1 38,90.21 2.13,2.13 0 0 1 37.61,88.86 v -0.78 a 2,2 0 0 1 0.4,-1.34 1.3,1.3 0 0 1 1.08,-0.5 1.29,1.29 0 0 1 1,0.38 1.7,1.7 0 0 1 0.37,1.08 h -0.55 a 1.34,1.34 0 0 0 -0.23,-0.76 0.69,0.69 0 0 0 -0.57,-0.23 0.79,0.79 0 0 0 -0.69,0.35 1.79,1.79 0 0 0 -0.23,1 v 0.79 a 1.87,1.87 0 0 0 0.22,1 0.71,0.71 0 0 0 0.65,0.36 0.78,0.78 0 0 0 0.61,-0.22 1.28,1.28 0 0 0 0.24,-0.76 z" /> + width="4.9400001" + y="85.910004" + x="10.84" + class="cls-8" /> + class="cls-6" /> + id="path1084" + d="m 20.56,62.6 a 0.7,0.7 0 0 0 -0.17,-0.5 1.77,1.77 0 0 0 -0.64,-0.33 2.66,2.66 0 0 1 -0.7,-0.35 1.18,1.18 0 0 1 -0.37,-0.41 1.12,1.12 0 0 1 -0.12,-0.53 1.14,1.14 0 0 1 0.35,-0.85 1.25,1.25 0 0 1 0.9,-0.33 1.27,1.27 0 0 1 0.68,0.17 1.16,1.16 0 0 1 0.46,0.47 1.38,1.38 0 0 1 0.16,0.67 h -0.55 a 0.89,0.89 0 0 0 -0.19,-0.62 0.7,0.7 0 0 0 -0.56,-0.22 0.69,0.69 0 0 0 -0.51,0.18 0.68,0.68 0 0 0 -0.19,0.52 0.61,0.61 0 0 0 0.2,0.46 1.89,1.89 0 0 0 0.61,0.33 2.12,2.12 0 0 1 0.92,0.52 1.21,1.21 0 0 1 0.28,0.81 1.09,1.09 0 0 1 -0.35,0.85 1.33,1.33 0 0 1 -0.93,0.32 1.48,1.48 0 0 1 -0.7,-0.17 1.2,1.2 0 0 1 -0.51,-0.46 1.31,1.31 0 0 1 -0.18,-0.69 H 19 a 0.81,0.81 0 0 0 0.22,0.62 0.85,0.85 0 0 0 0.62,0.23 0.7,0.7 0 0 0 0.54,-0.19 0.68,0.68 0 0 0 0.18,-0.5 z" /> + id="path1086" + d="M 22.91,63.76 A 1.16,1.16 0 0 1 22,63.4 1.51,1.51 0 0 1 21.67,62.34 V 62 A 1.83,1.83 0 0 1 22,60.82 1.05,1.05 0 0 1 22.88,60.41 1,1 0 0 1 23.72,60.77 1.82,1.82 0 0 1 24,61.9 v 0.34 h -1.8 v 0.08 a 1.18,1.18 0 0 0 0.19,0.75 0.69,0.69 0 0 0 0.57,0.24 0.91,0.91 0 0 0 0.41,-0.09 1.05,1.05 0 0 0 0.33,-0.28 l 0.27,0.33 a 1.19,1.19 0 0 1 -1.06,0.49 z m -0.05,-2.89 a 0.55,0.55 0 0 0 -0.49,0.22 1.25,1.25 0 0 0 -0.18,0.7 h 1.27 V 61.72 A 1.27,1.27 0 0 0 23.3,61.06 0.52,0.52 0 0 0 22.86,60.87 Z" /> + id="path1088" + d="m 26,61 a 0.9,0.9 0 0 0 -0.24,0 0.58,0.58 0 0 0 -0.57,0.44 v 2.26 h -0.53 v -3.23 h 0.52 v 0.33 a 0.64,0.64 0 0 1 0.81,-0.35 z" /> + id="path1090" + d="m 27.46,62.89 0.61,-2.42 h 0.54 l -1,3.23 h -0.4 l -1,-3.23 h 0.54 z" /> + id="path1092" + d="m 29.77,59.62 a 0.32,0.32 0 0 1 -0.07,0.22 0.29,0.29 0 0 1 -0.23,0.09 0.27,0.27 0 0 1 -0.22,-0.09 0.33,0.33 0 0 1 -0.08,-0.22 0.36,0.36 0 0 1 0.08,-0.23 0.27,0.27 0 0 1 0.22,-0.09 0.3,0.3 0 0 1 0.23,0.09 0.35,0.35 0 0 1 0.07,0.23 z m 0,4.08 H 29.2 v -3.23 h 0.53 z" /> + id="path1094" + d="m 31.69,63.31 a 0.57,0.57 0 0 0 0.42,-0.17 0.69,0.69 0 0 0 0.17,-0.46 h 0.5 a 1.09,1.09 0 0 1 -1.09,1.08 1.14,1.14 0 0 1 -0.92,-0.38 1.72,1.72 0 0 1 -0.32,-1.12 V 61.9 a 1.73,1.73 0 0 1 0.32,-1.11 1.12,1.12 0 0 1 0.91,-0.38 1,1 0 0 1 0.79,0.32 1.25,1.25 0 0 1 0.31,0.85 h -0.5 a 0.89,0.89 0 0 0 -0.17,-0.54 0.53,0.53 0 0 0 -0.43,-0.17 0.59,0.59 0 0 0 -0.52,0.23 1.37,1.37 0 0 0 -0.18,0.75 v 0.42 a 1.44,1.44 0 0 0 0.17,0.8 0.61,0.61 0 0 0 0.54,0.24 z" /> + id="path1096" + d="M 34.54,63.76 A 1.16,1.16 0 0 1 33.62,63.4 1.51,1.51 0 0 1 33.29,62.34 V 62 a 1.83,1.83 0 0 1 0.32,-1.13 1.05,1.05 0 0 1 0.88,-0.41 1,1 0 0 1 0.84,0.36 1.82,1.82 0 0 1 0.29,1.13 v 0.34 h -1.8 v 0.08 a 1.18,1.18 0 0 0 0.19,0.75 0.69,0.69 0 0 0 0.57,0.24 0.91,0.91 0 0 0 0.41,-0.09 1.05,1.05 0 0 0 0.33,-0.28 l 0.27,0.33 a 1.19,1.19 0 0 1 -1.05,0.44 z m 0,-2.89 a 0.55,0.55 0 0 0 -0.49,0.22 1.25,1.25 0 0 0 -0.18,0.7 h 1.27 v -0.07 a 1.27,1.27 0 0 0 -0.16,-0.66 0.52,0.52 0 0 0 -0.49,-0.19 z" /> + id="path1098" + d="M 39.76,62.56 H 38.29 L 38,63.7 h -0.56 l 1.4,-4.34 h 0.47 l 1.41,4.34 H 40.1 Z m -1.33,-0.47 h 1.19 l -0.59,-2 z" /> + class="cls-8" /> - + + x2="47.830002" + y1="82.080002" + x1="6.4099998" + class="cls-10" /> + class="cls-6" /> + class="cls-6" /> + class="cls-6" /> + class="cls-6" /> + class="cls-6" /> + class="cls-6" /> + class="cls-6" /> + id="logo_git"> + class="cls-6" /> + class="cls-8" /> + id="rechner"> - - + x="8.4700003" + class="cls-11" /> + width="8.4700003" + y="5.6500001" + x="8.4700003" + class="cls-11" /> - + y="2.8199999" + x="18.360001" + class="cls-11" /> + width="1.41" + y="2.8199999" + x="21.18" + class="cls-11" /> + + + + width="1.41" + y="5.6500001" + x="24.01" + class="cls-11" /> + class="cls-11" /> + class="cls-11" /> - - + x="8.4700003" + class="cls-11" /> + width="8.4700003" + y="17.65" + x="8.4700003" + class="cls-11" /> - + y="14.83" + x="18.360001" + class="cls-11" /> + width="1.41" + y="14.83" + x="21.18" + class="cls-11" /> + + + + width="1.41" + y="17.65" + x="24.01" + class="cls-11" /> - + cx="27.540001" + class="cls-11" /> + + cy="16.950001" + cx="30.360001" + class="cls-11" /> - - + x="8.4700003" + class="cls-11" /> + width="8.4700003" + y="29.66" + x="8.4700003" + class="cls-11" /> + + + width="1.41" + y="26.83" + x="24.01" + class="cls-11" /> + cy="27.540001" + cx="21.889999" + class="cls-11" /> + cy="30.360001" + cx="24.709999" + class="cls-11" /> + cy="28.950001" + cx="33.189999" + class="cls-11" /> + cy="30.360001" + cx="19.07" + class="cls-11" /> + class="cls-11" /> + class="cls-11" /> + class="cls-6" /> in-a-nutshell + + id="path3043" + d="m 335.58761,101.21941 v 35.4456 H 198.23346 l -1.3e-4,-24.76291" + style="font-variation-settings:normal;vector-effect:none;fill:none;fill-opacity:1;stroke:#949393;stroke-width:1.22892px;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:10;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1;stop-color:#000000" /> + id="line1201" + y2="101.40668" + x2="379.18933" + y1="101.40668" + x1="57.558681" + class="cls-12" /> + id="path1203" + style="stroke-width:0.819283" + d="m 322.10184,157.48383 a 1.0978394,1.0978394 0 0 0 -0.77993,0.35486 1.5074809,1.5074809 0 0 0 -0.31976,0.98271 v 0.47575 h -0.5401 v 0.54009 h 0.5401 v 3.5818 l 0.68828,0.0312 v -3.57984 h 0.72923 v -0.54011 h -0.70389 v -0.39386 c 0,-0.54891 0.18818,-0.81891 0.57325,-0.81891 a 1.2535032,1.2535032 0 0 1 0.31197,0 v -0.56545 a 1.1469963,1.1469963 0 0 0 -0.40166,-0.0663 1.0978394,1.0978394 0 0 0 -0.0975,-0.002 z m -7.0232,0.38801 v 5.5472 h 0.70583 v -5.5472 z m 17.89337,0.40946 v 1.01585 h -0.62978 v 0.54009 h 0.61418 v 2.54061 a 1.3518172,1.3518172 0 0 0 0.20473,0.81891 0.72096914,0.72096914 0 0 0 0.61419,0.27883 1.5566379,1.5566379 0 0 0 0.47575,-0.0741 v -0.57325 a 0.81928311,0.81928311 0 0 1 -0.27102,0.0487 0.27855626,0.27855626 0 0 1 -0.27102,-0.13064 0.64723366,0.64723366 0 0 1 -0.0819,-0.36851 v -2.5562 h 0.62394 v -0.5401 h -0.62394 v -1.00025 z m 12.09271,0 v 1.01585 h -0.62979 v 0.54009 h 0.62979 v 2.5328 a 1.3436243,1.3436243 0 0 0 0.21252,0.81893 0.72096914,0.72096914 0 0 0 0.61419,0.28662 1.5402523,1.5402523 0 0 0 0.46795,-0.0741 v -0.57325 a 0.81928311,0.81928311 0 0 1 -0.27102,0.0487 0.30313475,0.30313475 0 0 1 -0.27882,-0.13064 0.72916196,0.72916196 0 0 1 -0.0722,-0.36851 v -2.5562 h 0.62199 v -0.5401 h -0.62199 v -1.00025 z m -18.17221,0.93785 a 1.3436243,1.3436243 0 0 0 -0.0975,0.004 1.4747096,1.4747096 0 0 0 -0.71363,0.16378 1.212539,1.212539 0 0 0 -0.49915,0.4504 1.0404895,1.0404895 0 0 0 -0.18913,0.6064 h 0.67269 a 0.61446233,0.61446233 0 0 1 0.19692,-0.46601 0.69639065,0.69639065 0 0 1 0.49136,-0.18133 0.75374046,0.75374046 0 0 1 0.50695,0.15599 0.7373548,0.7373548 0 0 1 0.16378,0.53229 v 0.43481 h -0.47381 a 1.7614587,1.7614587 0 0 0 -1.17182,0.37631 1.2535032,1.2535032 0 0 0 -0.43481,1.00025 1.1060322,1.1060322 0 0 0 1.19718,1.20499 1.0896466,1.0896466 0 0 0 0.91641,-0.5245 1.3600099,1.3600099 0 0 0 0.0897,0.4426 l 0.68048,-0.01 v -0.0565 a 2.4578493,2.4578493 0 0 1 -0.13064,-0.81892 v -2.08239 a 1.2780817,1.2780817 0 0 0 -0.33537,-0.91641 1.3436243,1.3436243 0 0 0 -0.86961,-0.31588 z m 15.655,0 a 1.409167,1.409167 0 0 0 -1.05484,0.48746 2.1465218,2.1465218 0 0 0 -0.40166,1.41751 v 0.4582 a 2.1874859,2.1874859 0 0 0 0.40166,1.43506 1.4419383,1.4419383 0 0 0 1.17183,0.48356 1.3272386,1.3272386 0 0 0 0.9749,-0.39387 1.3927813,1.3927813 0 0 0 0.42506,-0.9827 h -0.63759 a 0.91759708,0.91759708 0 0 1 -0.22228,0.58884 0.7373548,0.7373548 0 0 1 -0.54009,0.21253 0.81928311,0.81928311 0 0 1 -0.68048,-0.33536 1.8188085,1.8188085 0 0 1 -0.22033,-1.03146 v -0.49134 a 1.7368802,1.7368802 0 0 1 0.22813,-0.97491 0.77012613,0.77012613 0 0 1 0.67268,-0.29637 0.68819781,0.68819781 0 0 1 0.54009,0.23008 1.1469963,1.1469963 0 0 1 0.22228,0.68828 h 0.63759 a 1.6385663,1.6385663 0 0 0 -0.39191,-1.09774 1.3354315,1.3354315 0 0 0 -1.00805,-0.39386 1.409167,1.409167 0 0 0 -0.11699,-0.004 z m 12.21359,0 a 1.2698888,1.2698888 0 0 0 -0.0935,0.004 1.3436243,1.3436243 0 0 0 -1.13089,0.52449 2.2857998,2.2857998 0 0 0 -0.40166,1.44286 v 0.49915 a 1.9662794,1.9662794 0 0 0 0.41726,1.35122 1.5320594,1.5320594 0 0 0 1.18743,0.46016 1.5320594,1.5320594 0 0 0 1.34537,-0.62394 l -0.35292,-0.42506 a 1.3190458,1.3190458 0 0 1 -0.427,0.36071 1.2043461,1.2043461 0 0 1 -0.52255,0.11309 0.91759708,0.91759708 0 0 1 -0.72923,-0.30222 1.5402523,1.5402523 0 0 1 -0.24762,-0.9671 v -0.0975 h 2.31052 v -0.44261 a 2.4004995,2.4004995 0 0 0 -0.36852,-1.4331 1.2698888,1.2698888 0 0 0 -0.9866,-0.46406 z m -36.15917,0.004 a 1.212539,1.212539 0 0 0 -1.00804,0.5167 v -0.44261 l -0.65514,0.0487 v 4.09654 h 0.68048 v -2.991 a 1.2862745,1.2862745 0 0 1 0.31977,-0.44066 0.71277631,0.71277631 0 0 1 0.4582,-0.16379 0.57349817,0.57349817 0 0 1 0.48356,0.20474 1.114225,1.114225 0 0 1 0.16378,0.68828 v 2.67903 h 0.67268 v -2.73753 c -0.008,-0.97495 -0.37793,-1.45845 -1.11529,-1.45845 z m 11.89187,0.0234 a 1.5320594,1.5320594 0 0 0 -0.9632,0.33731 1.1060322,1.1060322 0 0 0 -0.36851,0.81892 1.0241039,1.0241039 0 0 0 0.11503,0.48356 1.1551892,1.1551892 0 0 0 0.33537,0.3607 3.8424378,3.8424378 0 0 0 0.77017,0.38607 2.2939927,2.2939927 0 0 1 0.63175,0.36851 0.48337703,0.48337703 0 0 1 0.16378,0.37632 0.50795553,0.50795553 0 0 1 -0.14039,0.41725 0.81928311,0.81928311 0 0 1 -0.51475,0.15599 0.75374046,0.75374046 0 0 1 -0.55764,-0.19693 0.7373548,0.7373548 0 0 1 -0.20473,-0.5401 h -0.68829 a 1.2535032,1.2535032 0 0 0 0.40167,0.94175 1.5074809,1.5074809 0 0 0 1.04899,0.36852 1.4419383,1.4419383 0 0 0 0.99831,-0.32757 1.0814537,1.0814537 0 0 0 0.33731,-0.86766 0.99133256,0.99133256 0 0 0 -0.27102,-0.71363 3.15424,3.15424 0 0 0 -1.02366,-0.57324 2.0564006,2.0564006 0 0 1 -0.59858,-0.32757 0.45879854,0.45879854 0 0 1 -0.12284,-0.33537 0.53253402,0.53253402 0 0 1 0.16378,-0.40945 0.70458347,0.70458347 0 0 1 0.47575,-0.15599 0.63084799,0.63084799 0 0 1 0.47381,0.19693 0.68000498,0.68000498 0 0 1 0.18914,0.4738 h 0.67267 a 1.2043461,1.2043461 0 0 0 -0.36851,-0.90081 1.5320594,1.5320594 0 0 0 -0.95541,-0.33731 z m -5.87866,0.0176 a 0.81928311,0.81928311 0 0 0 -0.77017,0.49134 v -0.41726 h -0.66294 v 4.09655 h 0.68048 v -2.95006 a 0.74554763,0.74554763 0 0 1 0.72923,-0.56544 1.745073,1.745073 0 0 1 0.30222,0 l -0.008,-0.6064 a 0.66361932,0.66361932 0 0 0 -0.27102,-0.0487 z m 11.86458,0 a 0.81928311,0.81928311 0 0 0 -0.77018,0.49134 v -0.41726 h -0.65708 v 4.09655 h 0.68048 v -2.95006 a 0.7373548,0.7373548 0 0 1 0.72143,-0.56544 1.843387,1.843387 0 0 1 0.31197,0 l -0.0254,-0.6064 a 0.59807666,0.59807666 0 0 0 -0.26126,-0.0487 z m 15.89287,0 a 0.81928311,0.81928311 0 0 0 -0.77018,0.49134 v -0.41726 h -0.65514 v 4.09655 h 0.68049 v -2.95006 a 0.7373548,0.7373548 0 0 1 0.72143,-0.56544 1.843387,1.843387 0 0 1 0.31002,0 l -0.0234,-0.6064 a 0.59807666,0.59807666 0 0 0 -0.26322,-0.0487 z m -14.91018,0.0331 v 2.68683 a 1.9171225,1.9171225 0 0 0 0.28663,1.13089 0.99952543,0.99952543 0 0 0 0.81891,0.38607 1.1633821,1.1633821 0 0 0 0.98271,-0.44261 l -0.0409,0.33536 h 0.63954 v -4.09654 h -0.67268 v 3.05535 a 0.81928311,0.81928311 0 0 1 -0.81892,0.54985 c -0.38506,0 -0.58104,-0.32807 -0.58104,-0.96711 v -2.63809 z m 9.71589,0 v 2.68683 a 1.9744723,1.9744723 0 0 0 0.29442,1.13089 0.99952543,0.99952543 0 0 0 0.82087,0.38607 1.1715748,1.1715748 0 0 0 0.98271,-0.44261 l 0.0156,0.33536 h 0.64734 v -4.09654 h -0.68048 v 3.05535 a 0.81928311,0.81928311 0 0 1 -0.81892,0.54985 c -0.39326,0 -0.58105,-0.32807 -0.58105,-0.96711 v -2.63809 z m 7.41707,0.49915 a 0.71277631,0.71277631 0 0 1 0.0722,0 0.66361932,0.66361932 0 0 1 0.56544,0.27102 1.5812164,1.5812164 0 0 1 0.21253,0.81893 v 0.0994 h -1.63978 a 1.7778443,1.7778443 0 0 1 0.23007,-0.89301 0.71277631,0.71277631 0 0 1 0.5596,-0.29637 z m -27.65022,1.59884 a 1.310853,1.310853 0 0 1 0.0955,0 h 0.38411 v 0.98271 a 0.81928311,0.81928311 0 0 1 -0.31002,0.36851 0.88482576,0.88482576 0 0 1 -0.46795,0.13843 0.62265516,0.62265516 0 0 1 -0.50696,-0.18718 0.75374046,0.75374046 0 0 1 -0.14038,-0.5089 0.68000498,0.68000498 0 0 1 0.27883,-0.58104 1.310853,1.310853 0 0 1 0.66682,-0.21253 z" /> + id="path1231" + style="stroke-width:0.819283" + d="m 194.23936,157.7958 a 1.5812164,1.5812164 0 0 0 -1.05094,0.42896 1.4337454,1.4337454 0 0 0 -0.44261,1.08019 1.409167,1.409167 0 0 0 0.15404,0.68048 1.4747096,1.4747096 0 0 0 0.46796,0.5245 3.4082177,3.4082177 0 0 0 0.9008,0.4426 2.1219432,2.1219432 0 0 1 0.81892,0.43481 0.81928311,0.81928311 0 0 1 0.23008,0.62979 0.88482576,0.88482576 0 0 1 -0.20473,0.59079 0.94217558,0.94217558 0 0 1 -0.69608,0.23592 0.99133256,0.99133256 0 0 1 -1.07434,-1.08798 h -0.70388 a 1.6385663,1.6385663 0 0 0 0.23787,0.87546 1.5238666,1.5238666 0 0 0 0.64734,0.59859 1.941701,1.941701 0 0 0 0.89301,0.21253 1.7123017,1.7123017 0 0 0 1.20303,-0.40946 1.4173598,1.4173598 0 0 0 0.4348,-1.08215 1.4747096,1.4747096 0 0 0 -0.35291,-1.03144 2.6462844,2.6462844 0 0 0 -1.17963,-0.67269 2.1383289,2.1383289 0 0 1 -0.81891,-0.42505 0.81928311,0.81928311 0 0 1 -0.25348,-0.58104 0.90121141,0.90121141 0 0 1 0.23787,-0.66489 0.86024726,0.86024726 0 0 1 0.65514,-0.23788 0.91759708,0.91759708 0 0 1 0.72143,0.27883 1.1715748,1.1715748 0 0 1 0.24567,0.81892 h 0.70388 a 1.892544,1.892544 0 0 0 -0.20473,-0.81892 1.5566379,1.5566379 0 0 0 -0.58884,-0.60639 1.7368802,1.7368802 0 0 0 -0.87741,-0.21253 1.5812164,1.5812164 0 0 0 -0.10335,-0.002 z m -7.73683,0.0702 a 1.6385663,1.6385663 0 0 0 -0.10724,0.006 h -1.56374 v 5.5472 h 0.70388 v -2.24618 h 0.9827 l 0.99245,2.24618 h 0.75263 v -0.0409 l -1.0802,-2.36122 a 1.4419383,1.4419383 0 0 0 0.64734,-0.58105 1.7368802,1.7368802 0 0 0 0.22034,-0.89301 1.6959161,1.6959161 0 0 0 -0.41726,-1.24592 1.6385663,1.6385663 0 0 0 -1.1309,-0.43091 z m 21.97432,0.004 a 1.6385663,1.6385663 0 0 0 -0.10724,0.002 h -1.77822 v 5.5472 h 0.70388 v -2.17209 h 1.12309 a 1.6385663,1.6385663 0 0 0 1.18743,-0.4426 1.6959161,1.6959161 0 0 0 0.42701,-1.23618 1.7204945,1.7204945 0 0 0 -0.4426,-1.23812 1.6385663,1.6385663 0 0 0 -1.11335,-0.46016 z m -19.48442,0.002 v 5.5472 h 2.99881 v -0.59859 h -2.30272 v -1.96736 h 1.97516 v -0.59663 h -1.97516 v -1.78603 h 2.26178 V 157.872 Z m 7.5789,0 v 0.59859 h 1.45846 v 4.91546 h 0.70388 v -4.91546 h 1.46626 V 157.872 Z m 6.85747,0 -1.79381,5.5472 h 0.71947 l 0.42701,-1.45079 h 1.88351 l 0.44261,1.45066 h 0.72143 l -1.80163,-5.5472 z m 7.53795,0 v 5.5472 h 0.69609 v -5.5472 z m -2.58153,0.55569 a 0.86024726,0.86024726 0 0 1 0.68828,0.30417 1.1797677,1.1797677 0 0 1 0.26127,0.81892 1.1715748,1.1715748 0 0 1 -0.25348,0.81893 0.89301859,0.89301859 0 0 1 -0.71168,0.27881 h -1.07434 v -2.21887 h 1.07434 a 0.86024726,0.86024726 0 0 1 0.0156,-0.002 z m -22.85759,0.0429 h 0.81892 c 0.64723,0 0.97491,0.35962 0.97491,1.07239 a 1.0814537,1.0814537 0 0 1 -0.26128,0.75458 0.91759708,0.91759708 0 0 1 -0.70583,0.27882 h -0.81892 z m 18.19559,0.37632 0.76238,2.52304 h -1.515 z m -3.97369,1.89325 v 0.6142 h 1.54033 l 0.0409,-0.6142 z" /> + style="stroke-width:0.819283" + id="path1249" + d="m 198.28343,91.31311 a 9.8313977,9.8313977 0 0 1 9.79044,9.8314 v 0.27855 a 9.8313977,9.8313977 0 0 1 -9.79044,9.8314 h -0.041 a 9.8313977,9.8313977 0 0 1 -9.8314,-9.8314 v -0.27855 a 9.8313977,9.8313977 0 0 1 9.8314,-9.8314 h 0.041 m 0,-2.965805 h -0.041 a 12.780817,12.780817 0 0 0 -12.75624,12.764435 v 0.27855 a 12.780817,12.780817 0 0 0 25.55344,0 v -0.27855 a 12.780817,12.780817 0 0 0 -12.7562,-12.764435 z" + class="cls-6" /> + transform="matrix(0.81928314,0,0,0.81928314,-0.15513568,19.00318)" + id="g1461"> + id="path1247" /> + id="path1251" /> + id="path1253" /> + id="path1255" /> + id="use1281" + xlink:href="#browser" + transform="matrix(0.81928314,0,0,0.81928314,368.05529,55.477665)" + height="111.37" + width="157.22" /> + transform="matrix(0.81928314,0,0,0.81928314,-0.15513568,19.00318)" + id="g1455"> + y="19.57" + width="42.73" + height="12.5" + id="rect1301" /> + id="path1303" /> + id="path1305" /> + id="path1307" /> + id="path1309" /> + id="path1311" /> + id="path1313" /> + id="path1315" /> + transform="matrix(0.81928314,0,0,0.81928314,145.67726,19.00318)" + id="g3353"> + y="40.970001" + width="7.5900002" + height="115.74" + id="rect1341" /> + y1="44.68" + x2="96.470001" + y2="156.71001" + id="line1343" /> + transform="matrix(0.81928314,0,0,0.81928314,85.050311,19.00318)" + id="g1492"> + width="42.369999" + height="51.939999" + transform="translate(284.59,53.55)" + xlink:href="#rechner" + id="use1345" /> + id="path1347" /> + id="path1349" /> + style="stroke-width:0.819283" + id="rect1423" + height="5.4891968" + width="2.6544774" + y="98.539192" + x="365.32709" + class="cls-6" /> + transform="matrix(0.81928314,0,0,0.81928314,97.830236,9.4145048)" + id="g3192"> - + + class="cls-46" + data-name="SVGID" + id="path2889-5-3" /> + transform="matrix(0.81928314,0,0,0.81928314,266.5979,18.325158)" + id="g3539"> - + style="fill:#333333" /> + style="fill:#333333" /> + style="fill:#333333" /> + style="fill:#333333" /> + style="fill:#333333" /> + style="fill:#333333" /> + + class="cls-34" + style="fill:#333333" /> + width="22.76" + height="22.68" + transform="matrix(0.81928314,0,0,0.81928314,45.69891,92.28204)" + xlink:href="#logo_git" + id="use2157" /> + d="m 93.454048,105.29231 a 3.694967,3.694967 0 1 1 3.727739,-3.69497 3.694967,3.694967 0 0 1 -3.727739,3.69497 z" + id="path2215" /> + d="m 93.454048,98.475871 a 3.1214688,3.1214688 0 1 1 -3.121468,3.121469 3.1296616,3.1296616 0 0 1 3.121468,-3.121469 m 0,-1.146996 a 4.2684652,4.2684652 0 1 0 4.268465,4.268465 4.276658,4.276658 0 0 0 -4.268465,-4.268465 z" + id="path2217" /> + d="m 124.87356,105.29231 a 3.694967,3.694967 0 1 1 3.69496,-3.69497 3.7031598,3.7031598 0 0 1 -3.69496,3.69497 z" + id="path2241" /> + d="m 124.87356,98.475871 a 3.1214688,3.1214688 0 1 1 -3.11328,3.121469 3.1296616,3.1296616 0 0 1 3.12147,-3.121469 m 0,-1.146996 a 4.2684652,4.2684652 0 1 0 4.25208,4.268465 4.276658,4.276658 0 0 0 -4.26847,-4.268465 z" + id="path2243" /> + d="m 156.29306,105.29231 a 3.694967,3.694967 0 1 1 3.69497,-3.69497 3.7031598,3.7031598 0 0 1 -3.69497,3.69497 z" + id="path2265" /> - - - - + d="m 156.29306,98.475871 a 3.1214688,3.1214688 0 1 1 -3.12146,3.121469 3.1296616,3.1296616 0 0 1 3.12146,-3.121469 m 0,-1.146996 a 4.2684652,4.2684652 0 1 0 4.26847,4.268465 4.276658,4.276658 0 0 0 -4.26847,-4.268465 z" + id="path2267" /> + transform="matrix(0.81928314,0,0,0.81928314,109.91116,34.891047)" + id="g3349"> + + + + + id="path2205" + style="fill:#333333;stroke-width:0.819283" + d="m 95.159942,156.94178 v 5.84942 h 0.670733 v -5.84942 z m 3.907412,0 v 2.1292 a 1.0650681,1.0650681 0 0 0 -0.926157,-0.47576 1.1715748,1.1715748 0 0 0 -1.023649,0.5011 2.4004995,2.4004995 0 0 0 -0.360714,1.44871 l 0.03314,0.427 a 2.3759211,2.3759211 0 0 0 0.368514,1.39997 1.1797677,1.1797677 0 0 0 1.008049,0.50109 1.0650681,1.0650681 0 0 0 0.957356,-0.52449 v 0.4426 h 0.623938 v -5.84942 z m -5.391215,0.22618 a 0.32771324,0.32771324 0 0 0 -0.253475,0.11699 0.43422005,0.43422005 0 0 0 -0.09749,0.28662 0.41783439,0.41783439 0 0 0 0.09749,0.28662 0.32771324,0.32771324 0 0 0 0.286621,0.11504 0.33590608,0.33590608 0 0 0 0.286622,-0.11504 0.41783439,0.41783439 0 0 0 0.09749,-0.28662 0.45060571,0.45060571 0 0 0 -0.09749,-0.28662 0.36867739,0.36867739 0 0 0 -0.286622,-0.11504 0.32771324,0.32771324 0 0 0 -0.03314,-0.002 z m -6.637142,0.0722 a 1.5894092,1.5894092 0 0 0 -0.10529,0.004 h -1.589092 v 5.5472 h 1.637837 a 1.5320594,1.5320594 0 0 0 1.171834,-0.42701 1.8270014,1.8270014 0 0 0 0.189131,-2.02389 1.0404895,1.0404895 0 0 0 -0.606389,-0.46601 1.4009741,1.4009741 0 0 0 0.532297,-0.46796 1.2698888,1.2698888 0 0 0 0.189131,-0.68827 1.4419383,1.4419383 0 0 0 -0.386062,-1.10554 1.5894092,1.5894092 0 0 0 -1.033397,-0.37242 z m -0.06434,0.59859 a 0.81928311,0.81928311 0 0 1 0.557644,0.20864 1.1306107,1.1306107 0 0 1 0,1.31806 0.77012613,0.77012613 0 0 1 -0.590791,0.24568 h -0.900811 l -0.0156,-1.76847 h 0.893011 a 0.81928311,0.81928311 0 0 1 0.05654,-0.004 z m 2.556196,0.83062 v 2.68684 a 1.9744723,1.9744723 0 0 0 0.294421,1.13088 0.99133256,0.99133256 0 0 0 0.818919,0.38606 1.1551892,1.1551892 0 0 0 0.984652,-0.4426 l 0.0156,0.33537 h 0.647337 v -4.09655 h -0.680491 v 3.05535 a 0.81928311,0.81928311 0 0 1 -0.818919,0.54984 c -0.393256,0 -0.581043,-0.32806 -0.581043,-0.9671 v -2.63809 z m 3.835269,0.0254 v 4.09654 h 0.670735 v -4.09654 z m 4.874517,0.4972 a 0.7373548,0.7373548 0 0 1 0.07215,0.002 0.81928311,0.81928311 0 0 1 0.754574,0.49135 v 2.0824 a 0.81928311,0.81928311 0 0 1 -0.754574,0.50694 0.72096914,0.72096914 0 0 1 -0.670734,-0.32756 1.9335081,1.9335081 0 0 1 -0.212529,-1.01585 v -0.41726 a 1.990858,1.990858 0 0 1 0.212529,-1.00025 0.7373548,0.7373548 0 0 1 0.59859,-0.32172 z m -12.19994,1.01 h 1.008051 c 0.532533,0 0.818918,0.32844 0.818918,1.00025 a 1.0568752,1.0568752 0 0 1 -0.237876,0.72923 0.81928311,0.81928311 0 0 1 -0.629788,0.26127 h -0.959305 z" /> + id="path2223" + style="fill:#333333;stroke-width:0.819283" + d="m 122.86669,156.94178 v 5.84942 h 0.68048 v -1.476 l 0.36072,-0.40946 1.09774,1.88546 h 0.81892 l -1.48965,-2.40021 1.31807,-1.72168 h -0.81892 l -1.28688,1.77043 v -3.49796 z m -11.5838,0.30222 v 5.5472 h 0.70389 v -2.17208 h 1.08214 a 1.6385663,1.6385663 0 0 0 1.19523,-0.44261 1.990858,1.990858 0 0 0 0,-2.45675 1.6385663,1.6385663 0 0 0 -1.20303,-0.47576 z m 0.70389,0.55765 h 1.08214 a 0.8684401,0.8684401 0 0 1 0.70388,0.30222 1.2207319,1.2207319 0 0 1 0.25542,0.81892 1.1306107,1.1306107 0 0 1 -0.25542,0.82086 0.89301859,0.89301859 0 0 1 -0.71168,0.27883 h -1.07434 z m 5.0227,0.78967 a 1.3436243,1.3436243 0 0 0 -0.0975,0.004 1.4747096,1.4747096 0 0 0 -0.71362,0.16379 1.212539,1.212539 0 0 0 -0.49915,0.45235 1.0404895,1.0404895 0 0 0 -0.18914,0.60444 h 0.67269 a 0.61446233,0.61446233 0 0 1 0.19693,-0.46601 0.69639065,0.69639065 0 0 1 0.49134,-0.17937 0.75374046,0.75374046 0 0 1 0.50696,0.15402 0.7373548,0.7373548 0 0 1 0.16378,0.53231 v 0.4348 h -0.51475 a 1.7614587,1.7614587 0 0 0 -1.17183,0.37632 1.2535032,1.2535032 0 0 0 -0.43481,1.00025 1.1060322,1.1060322 0 0 0 1.19523,1.20497 1.0896466,1.0896466 0 0 0 0.91837,-0.52449 1.3600099,1.3600099 0 0 0 0.0896,0.4426 l 0.72144,-0.008 v -0.0585 a 2.2776071,2.2776071 0 0 1 -0.13064,-0.81892 v -2.08044 a 1.2453103,1.2453103 0 0 0 -0.33537,-0.91837 1.3436243,1.3436243 0 0 0 -0.86961,-0.31586 z m 10.88772,0 a 1.3436243,1.3436243 0 0 0 -0.0975,0.004 1.4747096,1.4747096 0 0 0 -0.71363,0.16379 1.212539,1.212539 0 0 0 -0.49915,0.45235 1.0404895,1.0404895 0 0 0 -0.18914,0.60444 h 0.67269 a 0.61446233,0.61446233 0 0 1 0.19693,-0.46601 0.71277631,0.71277631 0 0 1 0.49135,-0.17937 0.75374046,0.75374046 0 0 1 0.50695,0.15402 0.7373548,0.7373548 0 0 1 0.16573,0.53231 v 0.4348 h -0.5011 a 1.7614587,1.7614587 0 0 0 -1.17183,0.37632 1.2535032,1.2535032 0 0 0 -0.43286,1.00025 1.1060322,1.1060322 0 0 0 1.19523,1.20497 1.0896466,1.0896466 0 0 0 0.91837,-0.52449 1.3600099,1.3600099 0 0 0 0.0896,0.4426 l 0.70389,-0.008 v -0.0585 a 2.2776071,2.2776071 0 0 1 -0.13064,-0.81892 v -2.08044 a 1.2780817,1.2780817 0 0 0 -0.33537,-0.91837 1.3436243,1.3436243 0 0 0 -0.86961,-0.31586 z m 7.60814,0 a 1.2780817,1.2780817 0 0 0 -0.0935,0.004 1.3354315,1.3354315 0 0 0 -1.12309,0.52449 2.2857998,2.2857998 0 0 0 -0.40165,1.44287 v 0.49915 a 1.9498938,1.9498938 0 0 0 0.41726,1.35121 1.5320594,1.5320594 0 0 0 1.18937,0.46015 1.5320594,1.5320594 0 0 0 1.34341,-0.65513 l -0.3529,-0.427 a 1.3190458,1.3190458 0 0 1 -0.42702,0.3607 1.212539,1.212539 0 0 1 -0.53229,0.11505 0.90940425,0.90940425 0 0 1 -0.71948,-0.30418 1.5402523,1.5402523 0 0 1 -0.24568,-0.96515 v -0.0663 h 2.30857 v -0.44261 a 2.4004995,2.4004995 0 0 0 -0.36851,-1.43311 1.2780817,1.2780817 0 0 0 -0.9944,-0.46405 z m -14.93941,0.002 a 1.4173598,1.4173598 0 0 0 -1.05681,0.4855 2.1629074,2.1629074 0 0 0 -0.40165,1.41751 v 0.46016 a 2.1874859,2.1874859 0 0 0 0.40165,1.43311 1.4583239,1.4583239 0 0 0 1.17184,0.48354 1.3600099,1.3600099 0 0 0 0.98271,-0.39385 1.4255526,1.4255526 0 0 0 0.41725,-0.98271 h -0.63953 a 0.81928311,0.81928311 0 0 1 -0.22033,0.58885 0.71277631,0.71277631 0 0 1 -0.5401,0.21447 0.81928311,0.81928311 0 0 1 -0.68048,-0.27102 1.8270014,1.8270014 0 0 1 -0.21253,-1.03145 v -0.55764 a 1.6385663,1.6385663 0 0 1 0.22033,-0.9749 0.81928311,0.81928311 0 0 1 0.67268,-0.29442 0.68819781,0.68819781 0 0 1 0.5401,0.22812 1.0896466,1.0896466 0 0 1 0.22033,0.68828 h 0.63953 a 1.6385663,1.6385663 0 0 0 -0.39386,-1.08994 1.2944673,1.2944673 0 0 0 -1.0061,-0.40166 1.4173598,1.4173598 0 0 0 -0.11503,-0.002 z m 10.74148,0.002 a 1.1961533,1.1961533 0 0 0 -0.94371,0.49135 2.5315848,2.5315848 0 0 0 -0.35292,1.45846 l -0.0311,0.46015 a 2.4578493,2.4578493 0 0 0 0.37631,1.38437 1.1879605,1.1879605 0 0 0 1.00805,0.48354 1.0732608,1.0732608 0 0 0 0.93396,-0.48354 v 0.53229 c -0.0491,0.61447 -0.3125,0.91836 -0.86961,0.91836 a 1.2862745,1.2862745 0 0 1 -0.93396,-0.41141 l -0.27688,0.46795 a 1.4501311,1.4501311 0 0 0 0.57324,0.36852 1.79423,1.79423 0 0 0 0.68829,0.15598 1.4665168,1.4665168 0 0 0 1.09775,-0.40945 1.6385663,1.6385663 0 0 0 0.3919,-1.17965 v -4.16283 h -0.61419 v 0.46016 a 1.0568752,1.0568752 0 0 0 -0.95735,-0.53425 1.1961533,1.1961533 0 0 0 -0.0897,0 z m 4.03219,0.57324 a 0.71277631,0.71277631 0 0 1 0.0722,0.002 0.66361932,0.66361932 0 0 1 0.57324,0.26907 1.5812164,1.5812164 0 0 1 0.21253,0.81891 v 0.0994 h -1.63784 a 1.6959161,1.6959161 0 0 1 0.22813,-0.89301 0.71277631,0.71277631 0 0 1 0.55179,-0.29638 z m -3.83526,0.0234 a 0.7373548,0.7373548 0 0 1 0.0722,0.002 0.81928311,0.81928311 0 0 1 0.75457,0.5089 v 2.05509 a 0.81928311,0.81928311 0 0 1 -0.76237,0.5167 0.72916196,0.72916196 0 0 1 -0.67268,-0.32756 1.941701,1.941701 0 0 1 -0.20474,-1.01585 v -0.41726 a 1.9089296,1.9089296 0 0 1 0.21253,-1.00025 0.7373548,0.7373548 0 0 1 0.60055,-0.32172 z m -3.45896,1.59884 h 0.38606 v 0.98465 a 0.87663292,0.87663292 0 0 1 -0.32952,0.34317 0.93398274,0.93398274 0 0 1 -0.47379,0.14039 0.65542648,0.65542648 0 0 1 -0.48356,-0.15598 0.81928311,0.81928311 0 0 1 -0.14038,-0.51671 0.68000498,0.68000498 0 0 1 0.27881,-0.58104 1.3763956,1.3763956 0 0 1 0.76238,-0.21448 z m -10.88772,0.0409 h 0.38411 v 0.98465 a 0.81928311,0.81928311 0 0 1 -0.35096,0.30222 0.88482576,0.88482576 0 0 1 -0.46795,0.14039 0.65542648,0.65542648 0 0 1 -0.46601,-0.11504 0.81928311,0.81928311 0 0 1 -0.14039,-0.5167 0.68000498,0.68000498 0 0 1 0.27882,-0.58105 1.3763956,1.3763956 0 0 1 0.76238,-0.21447 z" /> + id="path2249" + style="fill:#333333;stroke-width:0.819283" + d="m 157.44067,156.94178 v 5.85722 h 0.68048 v -5.85722 z m -10.51726,0.30027 a 1.941701,1.941701 0 0 0 -0.11699,0.002 h -1.32587 v 5.555 h 1.24398 a 2.0400149,2.0400149 0 0 0 1.58909,-0.60639 2.4578493,2.4578493 0 0 0 0.54985,-1.69633 v -0.93396 a 2.4578493,2.4578493 0 0 0 -0.5401,-1.71193 1.941701,1.941701 0 0 0 -1.39996,-0.60834 z m 0.0195,0.63759 a 1.2616959,1.2616959 0 0 1 0.92031,0.41336 2.0318221,2.0318221 0 0 1 0.33732,1.28687 v 0.98271 a 1.941701,1.941701 0 0 1 -0.34513,1.25372 1.3272386,1.3272386 0 0 1 -1.11333,0.42506 h -0.56545 v -4.35782 h 0.66294 a 1.2616959,1.2616959 0 0 1 0.10334,-0.004 z m 8.14629,0.69023 a 1.0896466,1.0896466 0 0 0 -0.94955,0.53425 v -0.4582 h -0.6064 v 5.73438 h 0.67269 v -1.97516 a 1.1060322,1.1060322 0 0 0 0.95736,0.46795 1.1715748,1.1715748 0 0 0 1.00024,-0.48354 2.4578493,2.4578493 0 0 0 0.35292,-1.45846 l -0.0331,-0.47575 a 2.5397776,2.5397776 0 0 0 -0.34511,-1.41752 1.1797677,1.1797677 0 0 0 -1.01585,-0.46599 1.0896466,1.0896466 0 0 0 -0.0332,-0.002 z m -3.72998,0.0215 a 1.2698888,1.2698888 0 0 0 -0.0955,0.004 1.3436243,1.3436243 0 0 0 -1.13089,0.52449 2.2857998,2.2857998 0 0 0 -0.40166,1.44287 v 0.49915 a 1.990858,1.990858 0 0 0 0.41921,1.35121 1.5320594,1.5320594 0 0 0 1.18744,0.46015 1.5320594,1.5320594 0 0 0 1.34341,-0.65513 l -0.35292,-0.427 a 1.3190458,1.3190458 0 0 1 -0.42506,0.3607 1.2043461,1.2043461 0 0 1 -0.52449,0.11505 0.88482576,0.88482576 0 0 1 -0.72144,-0.30418 1.4829025,1.4829025 0 0 1 -0.25346,-0.96515 v -0.0663 h 2.31051 v -0.43481 a 2.3595354,2.3595354 0 0 0 -0.36851,-1.44091 1.2698888,1.2698888 0 0 0 -0.9866,-0.46405 z m 9.27133,0.004 a 1.4337454,1.4337454 0 0 0 -1.14843,0.52449 2.1219432,2.1219432 0 0 0 -0.43481,1.42532 v 0.4426 a 2.0973647,2.0973647 0 0 0 0.4426,1.37656 1.4665168,1.4665168 0 0 0 1.15429,0.5089 1.4255526,1.4255526 0 0 0 1.16404,-0.52449 2.1219432,2.1219432 0 0 0 0.4348,-1.41751 v -0.43481 a 2.1465218,2.1465218 0 0 0 -0.44261,-1.39216 1.4747096,1.4747096 0 0 0 -1.16403,-0.5089 1.4337454,1.4337454 0 0 0 -0.006,0 z m 2.0629,0.0741 1.21278,4.09655 -0.13259,0.46795 a 1.1060322,1.1060322 0 0 1 -0.26907,0.48355 0.71277631,0.71277631 0 0 1 -0.46016,0.13064 h -0.15403 v 0.56544 a 1.4173598,1.4173598 0 0 0 0.35096,0.0643 0.87663292,0.87663292 0 0 0 0.63954,-0.27881 1.8188085,1.8188085 0 0 0 0.43481,-0.81893 l 1.35121,-4.71073 h -0.72143 l -0.75262,3.0066 -0.76237,-3.0066 z m -11.49997,0.4972 a 0.68819781,0.68819781 0 0 1 0.0702,0.004 0.68819781,0.68819781 0 0 1 0.57325,0.27686 1.6385663,1.6385663 0 0 1 0.20472,0.82088 v 0.0897 h -1.63783 a 1.6959161,1.6959161 0 0 1 0.23787,-0.89301 0.68819781,0.68819781 0 0 1 0.55179,-0.29833 z m 9.44292,0.004 a 0.81928311,0.81928311 0 0 1 0.68048,0.36851 1.79423,1.79423 0 0 1 0.25348,1.0061 v 0.46015 c -0.0164,0.86025 -0.32808,1.29467 -0.92616,1.29467 a 0.77012613,0.77012613 0 0 1 -0.67073,-0.36851 1.7041089,1.7041089 0 0 1 -0.25543,-1.00025 v -0.39386 a 1.7204945,1.7204945 0 0 1 0.25543,-0.9983 0.76193329,0.76193329 0 0 1 0.66293,-0.36851 z m -5.6252,0.0215 a 0.75374046,0.75374046 0 0 1 0.62005,0.32952 1.9662794,1.9662794 0 0 1 0.21253,1.02365 v 0.40165 a 1.8351941,1.8351941 0 0 1 -0.22034,1.00806 0.74554763,0.74554763 0 0 1 -0.67267,0.33537 0.81928311,0.81928311 0 0 1 -0.74483,-0.47381 v -2.13894 a 0.81928311,0.81928311 0 0 1 0.73702,-0.48355 0.75374046,0.75374046 0 0 1 0.0683,-0.002 z" /> + id="path1787" + style="stroke-width:0.819283" + d="m 278.39693,157.40779 c -0.14099,0 -0.24862,0.0401 -0.32173,0.11893 -0.0705,0.0789 -0.10529,0.17605 -0.10529,0.29053 0,0.11447 0.0348,0.20835 0.10529,0.28466 0.0731,0.0763 0.18074,0.11505 0.32173,0.11505 0.14098,0 0.24666,-0.0387 0.31976,-0.11505 0.0731,-0.0763 0.10919,-0.17019 0.10919,-0.28466 0,-0.11448 -0.036,-0.21167 -0.10919,-0.29053 -0.0731,-0.0789 -0.17878,-0.11893 -0.31976,-0.11893 z m -39.03903,0.0761 v 5.55695 h 1.9771 c 0.60571,0 1.07619,-0.1421 1.40776,-0.42701 0.33157,-0.28491 0.4972,-0.67714 0.4972,-1.17573 0,-0.32562 -0.0908,-0.60634 -0.27103,-0.84037 -0.17753,-0.23658 -0.42115,-0.3957 -0.72922,-0.47966 0.26108,-0.10938 0.4686,-0.26368 0.62004,-0.4621 0.15142,-0.20096 0.22617,-0.43076 0.22617,-0.69023 0,-0.48842 -0.1566,-0.85819 -0.4699,-1.10749 -0.31068,-0.24929 -0.77535,-0.37436 -1.39412,-0.37436 z m 14.80292,0.42701 -0.72532,0.002 v 0.9983 h -0.77017 v 0.54595 h 0.77017 v 2.56009 c 0,0.35104 0.0832,0.62228 0.24763,0.81307 0.16448,0.19079 0.40742,0.28662 0.73117,0.28662 0.18015,0 0.36706,-0.0252 0.55764,-0.0761 v -0.57329 c -0.1462,0.0305 -0.26156,0.0468 -0.34511,0.0468 -0.1697,0 -0.29023,-0.0414 -0.36072,-0.12285 -0.0705,-0.084 -0.10529,-0.20901 -0.10529,-0.37436 v -2.56009 h 0.79163 v -0.54595 h -0.79163 z m 6.60985,0 -0.72533,0.002 v 0.9983 h -0.77017 v 0.54595 h 0.77017 v 2.56009 c 0,0.35104 0.0832,0.62228 0.24763,0.81307 0.16447,0.19079 0.40938,0.28662 0.73313,0.28662 0.18014,0 0.3651,-0.0252 0.55569,-0.0761 v -0.57329 c -0.14621,0.0305 -0.26157,0.0468 -0.34512,0.0468 -0.1697,0 -0.29021,-0.0414 -0.36071,-0.12285 -0.0705,-0.084 -0.10529,-0.20901 -0.10529,-0.37436 v -2.56009 h 0.79162 v -0.54595 h -0.79162 z m -20.66015,0.17743 h 1.11138 c 0.37336,0 0.65176,0.0687 0.83452,0.20863 0.18276,0.13737 0.27493,0.35962 0.27493,0.66488 0,0.27473 -0.0999,0.49098 -0.29833,0.65123 -0.1958,0.16026 -0.45875,0.24178 -0.78772,0.24178 h -1.13478 z m 5.79092,0.74678 c -0.37334,0 -0.70589,0.0877 -0.9983,0.26322 -0.2898,0.17553 -0.516,0.42565 -0.68048,0.74872 -0.16188,0.32307 -0.24373,0.68547 -0.24373,1.08995 v 0.0507 c 0,0.64105 0.17723,1.15504 0.53231,1.54425 0.35768,0.3892 0.82362,0.58494 1.39801,0.58494 0.38118,0 0.71758,-0.0891 1.01,-0.26713 0.29502,-0.17806 0.52188,-0.42684 0.67852,-0.74482 0.15926,-0.32052 0.23788,-0.68217 0.23788,-1.08409 v -0.0487 c 0,-0.64613 -0.17852,-1.16283 -0.53619,-1.55204 -0.35507,-0.38922 -0.82103,-0.58494 -1.39802,-0.58494 z m 4.50405,0 c -0.37335,0 -0.70588,0.0877 -0.9983,0.26322 -0.28979,0.17553 -0.51794,0.42565 -0.68242,0.74872 -0.16187,0.32307 -0.24178,0.68547 -0.24178,1.08995 v 0.0507 c 0,0.64105 0.17723,1.15504 0.53229,1.54425 0.35769,0.3892 0.82365,0.58494 1.39802,0.58494 0.38117,0 0.71758,-0.0891 1.00999,-0.26713 0.29502,-0.17806 0.52188,-0.42684 0.67854,-0.74482 0.15926,-0.32052 0.23787,-0.68217 0.23787,-1.08409 v -0.0487 c 0,-0.64613 -0.1785,-1.16283 -0.53619,-1.55204 -0.35508,-0.38922 -0.82103,-0.58494 -1.39802,-0.58494 z m 6.85358,0 c -0.45689,0 -0.83392,0.11693 -1.12894,0.35096 -0.29241,0.23404 -0.4387,0.51959 -0.43871,0.85792 0,0.2035 0.0502,0.38085 0.15209,0.52839 0.10443,0.14755 0.26362,0.27338 0.4777,0.38021 0.2167,0.10684 0.508,0.19861 0.87351,0.27493 0.36813,0.0763 0.62592,0.16808 0.77213,0.27492 0.14881,0.1043 0.22422,0.25182 0.22422,0.4426 0,0.18062 -0.0845,0.32543 -0.25152,0.43481 -0.16448,0.10685 -0.38296,0.15989 -0.65708,0.15989 -0.29764,0 -0.53868,-0.0674 -0.72143,-0.20473 -0.18014,-0.13991 -0.27747,-0.32888 -0.29053,-0.56545 h -0.72338 c 0,0.24421 0.0722,0.4713 0.21838,0.68244 0.14882,0.2086 0.35506,0.37238 0.61614,0.4894 0.26109,0.11702 0.56141,0.17548 0.90082,0.17548 0.48299,0 0.87482,-0.11014 1.17768,-0.33147 0.30285,-0.22385 0.4543,-0.52026 0.4543,-0.88911 0,-0.22131 -0.0541,-0.40891 -0.16378,-0.56155 -0.10705,-0.15516 -0.27266,-0.28837 -0.4972,-0.39775 -0.22453,-0.10939 -0.51841,-0.20192 -0.88131,-0.28077 -0.3603,-0.0789 -0.61035,-0.16309 -0.74873,-0.24958 -0.13837,-0.0865 -0.20668,-0.21367 -0.20668,-0.38411 0,-0.18061 0.0735,-0.32678 0.22228,-0.43871 0.14881,-0.11193 0.35634,-0.16768 0.62004,-0.16768 0.25586,0 0.4666,0.0687 0.63368,0.20862 0.1697,0.13992 0.25348,0.30777 0.25348,0.50111 h 0.72923 c 0,-0.3765 -0.15016,-0.68645 -0.4504,-0.92811 -0.29765,-0.24167 -0.68561,-0.36266 -1.16599,-0.36266 z m 6.80093,0 c -0.43602,0 -0.77242,0.18488 -1.01,0.55374 l -0.0117,-0.47773 h -0.70583 v 4.12969 h 0.72532 v -2.93056 c 0.1697,-0.39429 0.49193,-0.59274 0.96711,-0.59274 0.12793,0 0.24716,0.01 0.35681,0.0273 v -0.65709 c -0.0731,-0.0356 -0.18073,-0.0526 -0.32171,-0.0526 z m 2.4041,0 c -0.30807,0 -0.59164,0.0557 -0.85011,0.16768 -0.25586,0.11193 -0.45887,0.26487 -0.61029,0.4582 -0.14882,0.1908 -0.22423,0.38864 -0.22423,0.59469 h 0.72923 c 0,-0.18061 0.0884,-0.33296 0.26322,-0.46015 0.17493,-0.1272 0.39082,-0.19108 0.64929,-0.19108 0.29503,0 0.51736,0.0728 0.66878,0.22032 0.15143,0.14501 0.22813,0.3421 0.22813,0.58885 v 0.32367 h -0.70583 c -0.60831,0 -1.08072,0.11964 -1.4175,0.35876 -0.3342,0.23658 -0.50111,0.57034 -0.50111,1.00025 0,0.3536 0.13341,0.64593 0.39971,0.87741 0.26892,0.22895 0.61177,0.34317 1.0295,0.34317 0.46733,0 0.87013,-0.16997 1.20693,-0.51085 0.0261,0.20859 0.0596,0.35341 0.10139,0.43481 h 0.76042 v -0.0624 c -0.0992,-0.22386 -0.14818,-0.52373 -0.14818,-0.90277 v -1.90106 c -0.0105,-0.41718 -0.15416,-0.74417 -0.43091,-0.98075 -0.27674,-0.23913 -0.66021,-0.35876 -1.14844,-0.35876 z m 4.57815,0 c -0.52477,0 -0.9353,0.1781 -1.23032,0.53424 l -0.0351,-0.45823 h -0.66099 v 5.71683 h 0.72338 v -1.9888 c 0.29502,0.31798 0.7004,0.4777 1.21473,0.4777 0.50389,0 0.9041,-0.19302 1.19913,-0.57714 0.29502,-0.38412 0.44066,-0.89057 0.44066,-1.5189 v -0.0643 c -10e-6,-0.66649 -0.14887,-1.1859 -0.44651,-1.55985 -0.29503,-0.37394 -0.69587,-0.56154 -1.20498,-0.56154 z m 4.42411,0 c -0.52477,0 -0.93336,0.1781 -1.22838,0.53424 l -0.0351,-0.4582 h -0.66293 v 5.71683 h 0.72533 v -1.9888 c 0.29503,0.31798 0.69845,0.4777 1.21278,0.4777 0.5039,0 0.90412,-0.19302 1.19913,-0.57714 0.29503,-0.38412 0.44261,-0.89057 0.44261,-1.5189 v -0.0643 c 0,-0.66649 -0.14888,-1.18591 -0.44651,-1.55985 -0.29502,-0.37394 -0.69782,-0.56154 -1.20693,-0.56154 z m 6.34857,0 c -0.52216,0 -0.94558,0.19784 -1.26932,0.59469 l -0.0234,-0.51865 h -0.68439 v 4.12969 h 0.72338 v -2.94226 c 0.0992,-0.19842 0.23778,-0.35949 0.41532,-0.4816 0.17753,-0.12212 0.38311,-0.18328 0.61808,-0.18328 0.2898,0 0.50378,0.072 0.63955,0.21447 0.13824,0.14246 0.20784,0.36141 0.21057,0.65904 v 2.73363 h 0.72533 v -2.72973 c -0.008,-0.98447 -0.4596,-1.476 -1.35512,-1.476 z m 3.88402,0 c -0.50389,0 -0.90926,0.19166 -1.21473,0.57324 -0.30545,0.37904 -0.45626,0.88896 -0.45626,1.53254 0,0.67158 0.15274,1.20184 0.45821,1.59105 0.30808,0.38921 0.70893,0.58494 1.20498,0.58494 0.50128,0 0.89763,-0.16378 1.18743,-0.4894 v 0.35486 c 0,0.35105 -0.0999,0.62365 -0.29832,0.81697 -0.19581,0.19334 -0.47036,0.29052 -0.82282,0.29052 -0.44906,0 -0.828,-0.1876 -1.13868,-0.56154 l -0.37632,0.42311 c 0.15404,0.2213 0.37638,0.3973 0.66879,0.5245 0.2924,0.12719 0.59144,0.18913 0.8969,0.18913 0.54828,0 0.98459,-0.15236 1.30833,-0.46016 0.32635,-0.3078 0.4894,-0.72926 0.4894,-1.26347 v -4.03025 h -0.66098 l -0.0351,0.4582 c -0.28981,-0.35614 -0.69389,-0.53424 -1.21083,-0.53424 z m -7.67055,0.0761 v 4.12969 h 0.72338 v -4.12969 z m -32.12696,0.505 c 0.36552,0 0.65811,0.14074 0.87742,0.42311 0.22192,0.27982 0.33342,0.64493 0.33342,1.09774 0,0.50622 -0.10892,0.90116 -0.32563,1.18353 -0.21669,0.27982 -0.50928,0.41921 -0.8774,0.41921 -0.36552,0 -0.65811,-0.13802 -0.87742,-0.41531 -0.21669,-0.27728 -0.32367,-0.6457 -0.32367,-1.10359 0,-0.5164 0.10827,-0.9121 0.32757,-1.18938 0.21931,-0.27729 0.50803,-0.41531 0.86571,-0.41531 z m 4.50405,0 c 0.36552,0 0.65811,0.14074 0.87742,0.42311 0.22192,0.27982 0.33341,0.64493 0.33341,1.09774 0,0.50622 -0.10891,0.90116 -0.32561,1.18353 -0.2167,0.27982 -0.50929,0.41921 -0.87741,0.41921 -0.36552,0 -0.65811,-0.13802 -0.87742,-0.41531 -0.21669,-0.27728 -0.32561,-0.6457 -0.32561,-1.10359 0,-0.5164 0.11021,-0.9121 0.32952,-1.18938 0.21929,-0.27729 0.50802,-0.41531 0.8657,-0.41531 z m 20.42033,0.0176 c 0.35508,0 0.63478,0.13591 0.83841,0.40556 0.20366,0.26964 0.30613,0.63612 0.30613,1.10163 0,0.52913 -0.10376,0.92949 -0.31002,1.19914 -0.20364,0.2671 -0.47948,0.39971 -0.82672,0.39971 -0.44645,0 -0.77771,-0.18701 -0.9944,-0.5635 v -1.9732 c 0.21931,-0.37904 0.54799,-0.56934 0.9866,-0.56934 z m 4.42607,0 c 0.35506,0 0.63477,0.13591 0.83841,0.40556 0.20365,0.26964 0.30417,0.63612 0.30417,1.10163 0,0.52913 -0.10181,0.92949 -0.30807,1.19914 -0.20364,0.2671 -0.47949,0.39971 -0.82671,0.39971 -0.44646,0 -0.77771,-0.18701 -0.99441,-0.5635 v -1.9732 c 0.21931,-0.37904 0.54799,-0.56934 0.98661,-0.56934 z m 10.62839,0 c 0.44384,0 0.77638,0.19844 0.9983,0.59274 v 1.88351 c -0.21408,0.40448 -0.5492,0.60834 -1.0061,0.60834 -0.35246,0 -0.62701,-0.13319 -0.82282,-0.39776 -0.19581,-0.2671 -0.29441,-0.63417 -0.29441,-1.09969 0,-0.52404 0.0999,-0.91838 0.29831,-1.18549 0.19842,-0.2671 0.47426,-0.40165 0.82672,-0.40165 z m -45.76976,1.00804 h 1.25372 c 0.7467,0 1.1192,0.33512 1.1192,1.00415 0,0.31036 -0.10249,0.55311 -0.30612,0.73118 -0.20104,0.17553 -0.47492,0.26323 -0.82478,0.26323 h -1.24202 z m 26.64021,0.5947 h 0.56739 v 0.84817 c -0.0913,0.18315 -0.23896,0.33202 -0.44261,0.4465 -0.20364,0.11447 -0.41824,0.17159 -0.64538,0.17159 -0.23759,0 -0.43352,-0.0625 -0.58494,-0.18719 -0.15143,-0.12465 -0.22618,-0.29734 -0.22618,-0.51865 0,-0.50623 0.44404,-0.76042 1.33172,-0.76042 z" /> diff --git a/docs/companions.md b/docs/companions.md index 15350ffb..5c234743 100644 --- a/docs/companions.md +++ b/docs/companions.md @@ -1,14 +1,16 @@ -# Companions +# Companion Configuration -A companion is a service that is running inside an application +Have a look at the [basic terminology](../README.md) what is a companion. For +these use cases following sections provide example configurations. -It is possible to start containers that will be started when the client requests to create a new service. For example, if the application requires an [OpenID](https://en.wikipedia.org/wiki/OpenID_Connect) provider, it is possible to create a configuration that starts the provider for each application. Another use case might be a Kafka services that is required by the application. +A simple but limited configuration of companions can be done via the +`config.toml` file for [application companions](#application-wide) and [service +companions](#service-based). More complex companions can be created via +[bootstrapping](#bootstrapping-from-the-infrastructure-backend). -Furthermore, it is also possible to create containers for each service. For example, for each service a database container could be started. +## Static Configuration -For these use cases following sections provide example configurations. - -## Application Wide +### Application Wide If you want to include an OpenID provider for every application, you could use following configuration. @@ -19,7 +21,7 @@ image = 'private.example.com/library/openid:latest' env = [ 'KEY=VALUE' ] ``` -The provided values of `serviceName` and `env` can include the [handlebars syntax](https://handlebarsjs.com/) in order to access dynamic values. +The provided values of `serviceName` and `env` can include the [handlebars syntax][handlebars] in order to access dynamic values. Additionally, you could mount files that are generated from handlebars templates (example contains a properties generation): @@ -41,7 +43,7 @@ Furthermore, you can provide labels through handlebars templating: "com.github.prevant" = "bar-{{application.name}}" ``` -### Template Variables +#### Template Variables The list of available handlebars variables: @@ -52,14 +54,14 @@ The list of available handlebars variables: - `port`: The exposed port of the service - `type`: The type of service. For example, `instance`, `replica`, `app-companion`, or `service-companion`. -### Handlebar Helpers +#### Handlebar Helpers PREvant provides some handlebars helpers which can be used to generate more complex configuration files. See handlerbar's [block helper documentation](https://handlebarsjs.com/block_helpers.html) for more details. - `{{#isCompanion }}` A conditional handlerbars block helper that checks if the given service type matches any companion type. - `isNotCompanion ` A conditional handlerbars block helper that checks if the given service type does not match any companion type. -## Service Based +### Service Based The service-based companions works the in the same way as the application-based services. Make sure, that the `serviceName` is unique by using the handlebars templating. @@ -76,7 +78,7 @@ env = [ 'KEY=VALUE' ] ``` -### Template Variables +#### Template Variables The list of available handlebars variables: @@ -87,7 +89,7 @@ The list of available handlebars variables: - `port`: The exposed port of the service - `type`: The type of service. For example, `instance`, `replica`, `app-companion`, or `service-companion`. -## Deployment Strategy +### Deployment Strategy Companions offer different deployment strategies so that a companion could be restarted or not under certain conditions. Therefore, PREvant offers following configuration flags: @@ -104,4 +106,80 @@ deploymentStrategy = 'redeploy-on-image-update' - `redeploy-on-image-update`: Re-deploys the companion if there is a more rescent image available. - `redeploy-never`: Even if there is a new deployment request the companion won't be redeployed and stays running. -a +### Storage Strategy + +Companions may have varying storage requirements and storage strategies cater to these by offering the below configuration flags: + +```toml +[companions.postgres] +type = 'application' +image = 'postgres:latest' +storageStrategy = 'mount-declared-image-volumes' +``` + +`storageStrategy` offers following values to determine how storage is managed for a companion: + +- `none` (_default_): Companion is deployed without persistent storage. +- `mount-declared-image-volumes`: Mounts the volume paths declared within the image, providing persistent storage for the companion. + +## Bootstrapping From the Infrastructure Backend + +When the [static configuration](#static-configuration) is insufficient for your +use case, then PREvant can utilize the underlying infrastructure to bootstrap +the companion configuration from the stdout of containers that are run once +within the infrastructure (depicted by following image). PREvant's static +companion configuration might be insufficient if services of the application +rely on volume sharing among services (see [#123][persistent-data-issue]). + +![](../assets/bootstrap-companions.svg "Illustration how bootstrapping of companions work") + +In the depicted images PREvant will start one or more containers on the +infrastructure backend that are expected to generate output on standard out +(stdout) that will be parsed by PREvant that needs to be native to the +underlying infrastructure. + +- When PREvant uses Kubernetes as the infrastructure runtime, the bootstrap + containers need to output [Kubernetes manifests][k8s-manifest]. + + Make sure to put out YAML that is compatible with 1.1 and 1.2 + (For example, bitnami helm charts have been adjusted in part in that regard, + see [here][zookeeper-yaml-1.2-pr] and [here][kafka-yaml-1.2-pr]) +- Docker: not yet implemented but the aim is to support [Docker + compose][docker-compose] files. + +Then before deploying these bootstrapped companions PREvant merges them with +the objects generated from the HTTP request payload (all bootstrapped +companions will be considered as application companions). Thus you can add or +overwrite configurations. For example, you can change the image used or an +environment variable. If you overwrite any configuration the companion will be +turned into an instance (as PREvant did before). + +Following configuration block depicts that an image +`registry.example.com/user/bootstrap-helm-chart:lastet` is based on a [Helm +chart][helm-chart] that generates an OpenID provider for your application with +[Keycloak][keycloak]. Additionally, it is possible to passed additional +arguments to the container than can be templated with [Handlebars][handlebars]. + +```toml +[[companions.bootstrapping.containers]] +image = "registry.example.com/user/bootstrap-helm-chart:latest" +args = [ + "--set", "keycloak.httpRelativePath=/{{application.name}}/keycloak/", + "--set", "keycloak.redirectUris[0]={{application.baseUrl}}oauth_redir" +] +``` + +The list of available handlebars variables for bootstrap container arguments: + +- `application`: The companion's application information + - `name`: The application name + - `baseUrl`: The URL that all services in the application share + +[docker-compose]: https://docs.docker.com/compose/ +[handlebars]: https://handlebarsjs.com/ +[helm-chart]: https://helm.sh/docs/topics/charts/ +[k8s-manifest]: https://kubernetes.io/docs/reference/glossary/?all=true#term-manifest +[keycloak]: https://www.keycloak.org/ +[persistent-data-issue]: https://github.com/aixigo/PREvant/issues/123 +[zookeeper-yaml-1.2-pr]: https://github.com/bitnami/charts/pull/21081 +[kafka-yaml-1.2-pr]: https://github.com/bitnami/charts/pull/21086 From ecb5d177688721940c692fb2eab877786e4e26ce Mon Sep 17 00:00:00 2001 From: Marc Schreiber Date: Fri, 12 Jan 2024 14:13:36 +0100 Subject: [PATCH 3/6] Performance Improvments This commit ensures that there are less requests to the Kubernetes API, improving the performance. Additionally, it fixes some additional TODO comments. --- api/src/apps/mod.rs | 4 +- api/src/apps/routes.rs | 16 +- api/src/config/companion.rs | 35 +-- api/src/deployment/deployment_unit.rs | 2 +- api/src/deployment/hooks.rs | 7 +- .../kubernetes/deployment_unit.rs | 17 +- .../kubernetes/infrastructure.rs | 220 +++++++++--------- 7 files changed, 148 insertions(+), 153 deletions(-) diff --git a/api/src/apps/mod.rs b/api/src/apps/mod.rs index a84a28df..f9454578 100644 --- a/api/src/apps/mod.rs +++ b/api/src/apps/mod.rs @@ -352,7 +352,7 @@ impl AppsService { pub async fn get_logs( &self, app_name: &AppName, - service_name: &String, + service_name: &str, since: &Option>, limit: usize, ) -> Result, AppsServiceError> { @@ -370,7 +370,7 @@ impl AppsService { pub async fn change_status( &self, app_name: &AppName, - service_name: &String, + service_name: &str, status: ServiceStatus, ) -> Result, AppsServiceError> { Ok(self diff --git a/api/src/apps/routes.rs b/api/src/apps/routes.rs index f026cba2..acfd93ab 100644 --- a/api/src/apps/routes.rs +++ b/api/src/apps/routes.rs @@ -189,13 +189,13 @@ async fn change_status( "//logs/?&", format = "text/plain" )] -async fn logs( +async fn logs<'r>( app_name: Result, - service_name: &str, + service_name: &'r str, since: Option, limit: Option, apps: &State>, -) -> HttpResult { +) -> HttpResult> { let app_name = app_name?; let since = match since { @@ -214,13 +214,13 @@ async fn logs( let limit = limit.unwrap_or(20_000); let log_chunk = apps - .get_logs(&app_name, &service_name.to_string(), &since, limit) + .get_logs(&app_name, service_name, &since, limit) .await?; Ok(LogsResponse { log_chunk, app_name, - service_name: service_name.to_string(), + service_name, limit, }) } @@ -265,10 +265,10 @@ fn map_join_error(err: tokio::task::JoinError) -> HttpApiError { .into() } -pub struct LogsResponse { +pub struct LogsResponse<'a> { log_chunk: Option, app_name: AppName, - service_name: String, + service_name: &'a str, limit: usize, } @@ -284,7 +284,7 @@ impl CreateAppOptions { } } -impl<'r> Responder<'r, 'static> for LogsResponse { +impl<'r> Responder<'r, 'static> for LogsResponse<'r> { fn respond_to(self, _request: &'r Request) -> Result, Status> { use std::io::Cursor; let log_chunk = match self.log_chunk { diff --git a/api/src/config/companion.rs b/api/src/config/companion.rs index 7753b98d..b2eabaa9 100644 --- a/api/src/config/companion.rs +++ b/api/src/config/companion.rs @@ -26,7 +26,7 @@ use crate::config::AppSelector; use crate::models::service::ContainerType; use crate::models::{AppName, Environment, Image, Router, ServiceConfig}; -use handlebars::Handlebars; +use handlebars::{Handlebars, RenderError}; use secstr::SecUtf8; use serde_value::Value; use std::collections::BTreeMap; @@ -204,7 +204,11 @@ impl BootstrappingContainer { &self.image } - pub fn templated_args(&self, app_name: &AppName, base_url: &Option) -> Vec { + pub fn templated_args( + &self, + app_name: &AppName, + base_url: &Option, + ) -> Result, RenderError> { let handlebars = Handlebars::new(); #[derive(Serialize)] @@ -222,16 +226,17 @@ impl BootstrappingContainer { let data = Data { application: AppData { - name: &app_name, + name: app_name, base_url, }, }; - self.args - .iter() - // TODO: handle result - .map(|arg| handlebars.render_template(&arg, &data).unwrap()) - .collect() + let mut args = Vec::with_capacity(self.args.len()); + for arg in &self.args { + args.push(handlebars.render_template(arg, &data)?); + } + + Ok(args) } } @@ -287,7 +292,7 @@ mod tests { assert_eq!(container.image, Image::from_str("busybox").unwrap()); assert_eq!( - container.templated_args(&AppName::master(), &None), + container.templated_args(&AppName::master(), &None).unwrap(), Vec::::new() ); } @@ -306,7 +311,7 @@ mod tests { assert_eq!(container.image, Image::from_str("busybox").unwrap()); assert_eq!( - container.templated_args(&AppName::master(), &None), + container.templated_args(&AppName::master(), &None).unwrap(), vec![String::from("echo"), String::from("Hello master")] ); } @@ -325,10 +330,12 @@ mod tests { assert_eq!(container.image, Image::from_str("busybox").unwrap()); assert_eq!( - container.templated_args( - &AppName::master(), - &Some(Url::parse("http://example.com").unwrap()) - ), + container + .templated_args( + &AppName::master(), + &Some(Url::parse("http://example.com").unwrap()) + ) + .unwrap(), vec![ String::from("echo"), String::from("Hello http://example.com/") diff --git a/api/src/deployment/deployment_unit.rs b/api/src/deployment/deployment_unit.rs index bc4701df..05e6307d 100644 --- a/api/src/deployment/deployment_unit.rs +++ b/api/src/deployment/deployment_unit.rs @@ -353,7 +353,7 @@ impl DeploymentUnitBuilder { self.stage.service_companions.iter() { let templated_companion = service_companion - .apply_templating_for_service_companion(&self.stage.app_name, &service)?; + .apply_templating_for_service_companion(&self.stage.app_name, service)?; service_companions.push(ServiceCompanion { templated_companion, diff --git a/api/src/deployment/hooks.rs b/api/src/deployment/hooks.rs index 8c6af8bc..9cddd854 100644 --- a/api/src/deployment/hooks.rs +++ b/api/src/deployment/hooks.rs @@ -112,10 +112,7 @@ impl<'a> Hooks<'a> { } } - fn register_configs_as_global_property( - mut context: &mut Context, - services: &[DeployableService], - ) { + fn register_configs_as_global_property(context: &mut Context, services: &[DeployableService]) { let js_configs = services .iter() .map(JsServiceConfig::from) @@ -123,7 +120,7 @@ impl<'a> Hooks<'a> { let js_configs = serde_json::to_value(js_configs).expect("Should be serializable"); let js_configs = - JsValue::from_json(&js_configs, &mut context).expect("Unable to read JSON value"); + JsValue::from_json(&js_configs, context).expect("Unable to read JSON value"); context .register_global_property("serviceConfigs", js_configs, Attribute::READONLY) diff --git a/api/src/infrastructure/kubernetes/deployment_unit.rs b/api/src/infrastructure/kubernetes/deployment_unit.rs index ac86b512..215409f1 100644 --- a/api/src/infrastructure/kubernetes/deployment_unit.rs +++ b/api/src/infrastructure/kubernetes/deployment_unit.rs @@ -13,6 +13,7 @@ use crate::{ }; use failure::Error; use futures::{AsyncBufReadExt, AsyncReadExt, StreamExt, TryStreamExt}; +use handlebars::RenderError; use k8s_openapi::{ api::{ apps::v1::{Deployment, StatefulSet}, @@ -78,14 +79,16 @@ impl K8sDeploymentUnit { let containers = bootstrapping_containers .iter() .enumerate() - .map(|(i, bc)| Container { - name: format!("bootstrap-{i}"), - image: Some(bc.image().to_string()), - image_pull_policy: Some(String::from("Always")), - args: Some(bc.templated_args(app_name, &base_url)), - ..Default::default() + .map(|(i, bc)| { + Ok(Container { + name: format!("bootstrap-{i}"), + image: Some(bc.image().to_string()), + image_pull_policy: Some(String::from("Always")), + args: Some(bc.templated_args(app_name, &base_url)?), + ..Default::default() + }) }) - .collect::>(); + .collect::, RenderError>>()?; let pod_name = format!( "{}-bootstrap-{}", diff --git a/api/src/infrastructure/kubernetes/infrastructure.rs b/api/src/infrastructure/kubernetes/infrastructure.rs index d86b4f30..03bacf8b 100644 --- a/api/src/infrastructure/kubernetes/infrastructure.rs +++ b/api/src/infrastructure/kubernetes/infrastructure.rs @@ -44,7 +44,7 @@ use crate::models::{ use async_trait::async_trait; use chrono::{DateTime, FixedOffset, Utc}; use failure::Error; -use futures::stream::{FuturesUnordered, Stream}; +use futures::stream::FuturesUnordered; use futures::StreamExt; use k8s_openapi::api::core::v1::PersistentVolumeClaim; use k8s_openapi::api::storage::v1::StorageClass; @@ -64,7 +64,6 @@ use secstr::SecUtf8; use std::collections::{BTreeMap, HashMap}; use std::convert::{From, TryFrom}; use std::net::IpAddr; -use std::pin::Pin; use std::str::FromStr; pub struct KubernetesInfrastructure { @@ -115,11 +114,11 @@ impl KubernetesInfrastructure { }) } - async fn get_deployment( + async fn get_deployment_and_pod( &self, app_name: &AppName, service_name: &str, - ) -> Result, KubernetesInfrastructureError> { + ) -> Result)>, KubernetesInfrastructureError> { let client = self.client().await?; let namespace = app_name.to_rfc1123_namespace_id(); @@ -128,59 +127,44 @@ impl KubernetesInfrastructure { ..Default::default() }; - match Api::::namespaced(client.clone(), &namespace) - .list(&p) - .await? - .into_iter() - .next() - { - Some(deployment) => Ok(Some(deployment)), - None => Ok(None), - } - } - - async fn get_pod_of_deployment( - &self, - deployment: &V1Deployment, - ) -> Result, KubernetesInfrastructureError> { - let Some(spec) = &deployment.spec else { - return Ok(None); + let client_clone = client.clone(); + let deployment = async { + Api::::namespaced(client_clone, &namespace) + .list(&p) + .await + .map(|list| list.items.into_iter().next()) + }; + let pods = async { + Api::::namespaced(client, &namespace) + .list(&Default::default()) + .await + .map(|list| list.items) }; - match Api::::namespaced( - self.client().await?, - deployment - .metadata - .namespace - .as_ref() - .expect("A namespace should be present for a deployment"), - ) - .list(&ListParams { - label_selector: spec.selector.match_labels.as_ref().map(|labels| { - labels - .iter() - .map(|(k, v)| format!("{k}={v}")) - .collect::>() - .join(",") - }), - ..Default::default() - }) - .await? - .into_iter() - .next() - { - Some(pod) => Ok(Some(pod)), - None => Ok(None), - } + let (deployment, pods) = futures::try_join!(deployment, pods)?; + + Ok(deployment.and_then(|deployment| { + let spec = deployment.spec.as_ref()?; + let matches_labels = spec.selector.match_labels.as_ref()?; + let pod = pods.into_iter().find(|pod| { + pod.metadata + .labels + .as_ref() + .map(|labels| matches_labels.iter().all(|(k, v)| labels.get(k) == Some(v))) + .unwrap_or(false) + }); + + Some((deployment, pod)) + })) } - async fn create_service_from( - &self, + fn create_service_from_deployment_and_pod( deployment: V1Deployment, + pod: Option, ) -> Result { let mut builder = ServiceBuilder::try_from(deployment.clone())?; - if let Some(pod) = self.get_pod_of_deployment(&deployment).await? { + if let Some(pod) = pod { if let Some(container) = pod.spec.as_ref().and_then(|spec| spec.containers.first()) { builder = builder.started_at( pod.status @@ -214,22 +198,49 @@ impl KubernetesInfrastructure { &self, app_name: &AppName, ) -> Result, KubernetesInfrastructureError> { - let mut futures = Api::::namespaced( - self.client().await?, - &app_name.to_rfc1123_namespace_id(), - ) - .list(&Default::default()) - .await? - .items - .into_iter() - // FIXME: this performs many network requests to inspect the IPs ip addresses. - .map(|deployment| self.create_service_from(deployment)) - .map(Box::pin) - .collect::>>>(); - - let mut services = Vec::with_capacity(futures.size_hint().0); - while let Some(service) = futures.next().await { - let service = match service { + let client = self.client().await?; + + let namespace = app_name.to_rfc1123_namespace_id(); + let list_param = Default::default(); + let client_clone = client.clone(); + let deployments = async { + Api::::namespaced(client_clone, &namespace) + .list(&list_param) + .await + }; + let pods = async { + Api::::namespaced(client, &namespace) + .list(&list_param) + .await + }; + let (deployments, mut pods) = futures::try_join!(deployments, pods)?; + + let mut services = Vec::with_capacity(deployments.items.len()); + for deployment in deployments.into_iter() { + let pod = { + let Some(spec) = deployment.spec.as_ref() else { + continue; + }; + let Some(matches_labels) = spec.selector.match_labels.as_ref() else { + continue; + }; + + match pods.items.iter().position(|pod| { + pod.metadata + .labels + .as_ref() + .map(|labels| matches_labels.iter().all(|(k, v)| labels.get(k) == Some(v))) + .unwrap_or(false) + }) { + Some(pod_position) => { + let pod = pods.items.swap_remove(pod_position); + Some(pod) + } + None => None, + } + }; + + let service = match Self::create_service_from_deployment_and_pod(deployment, pod) { Ok(service) => service, Err(e) => { debug!("Deployment does not provide required data: {:?}", e); @@ -243,19 +254,6 @@ impl KubernetesInfrastructure { Ok(services) } - async fn get_service_of_app( - &self, - app_name: &AppName, - service_name: &str, - ) -> Result, KubernetesInfrastructureError> { - let deployment = self.get_deployment(app_name, service_name).await?; - - match deployment.map(|deployment| self.create_service_from(deployment)) { - Some(service) => Ok(Some(service.await?)), - None => Ok(None), - } - } - async fn create_namespace_if_necessary( &self, app_name: &AppName, @@ -461,7 +459,7 @@ impl KubernetesInfrastructure { impl Infrastructure for KubernetesInfrastructure { async fn get_services(&self) -> Result, Error> { let client = self.client().await?; - let app_names = Api::::all(client.clone()) + let mut app_name_and_services = Api::::all(client.clone()) .list(&ListParams { label_selector: Some(APP_NAME_LABEL.to_string()), ..Default::default() @@ -478,11 +476,17 @@ impl Infrastructure for KubernetesInfrastructure { .filter_map(|ns| { AppName::from_str(ns.metadata.labels.as_ref()?.get(APP_NAME_LABEL)?).ok() }) - .collect::>(); + .map(|app_name| async { + self.get_services_of_app(&app_name) + .await + .map(|services| (app_name, services)) + }) + .map(Box::pin) + .collect::>(); let mut apps = MultiMap::new(); - for app_name in app_names { - let services = self.get_services_of_app(&app_name).await?; + while let Some(res) = app_name_and_services.next().await { + let (app_name, services) = res?; apps.insert_many(app_name, services); } @@ -518,7 +522,6 @@ impl Infrastructure for KubernetesInfrastructure { let services = self.get_services_of_app(app_name).await?; k8s_deployment_unit.filter_by_instances_and_replicas(&services); - // TODO: things like cloning data from existing deployments have to be considered for deployable_service in deployment_unit.services() { let (secret, service, deployment, ingress_route, middlewares) = self .create_payloads(app_name, deployable_service, container_config) @@ -536,13 +539,7 @@ impl Infrastructure for KubernetesInfrastructure { let deployments = k8s_deployment_unit.deploy(client, app_name).await?; let mut services = Vec::with_capacity(deployments.len()); for deployment in deployments.into_iter() { - let service = match self.create_service_from(deployment).await { - Ok(service) => service, - Err(e) => { - debug!("Deployment does not provide required data: {:?}", e); - continue; - } - }; + let service = Self::create_service_from_deployment_and_pod(deployment, None)?; services.push(service); } @@ -579,18 +576,10 @@ impl Infrastructure for KubernetesInfrastructure { let client = self.client().await?; let namespace = app_name.to_rfc1123_namespace_id(); - let deployment = match self.get_deployment(app_name, service_name).await? { - Some(deployment) => deployment, - None => { - return Ok(None); - } - }; - - let pod = match self.get_pod_of_deployment(&deployment).await? { - Some(pod) => pod, - None => { - return Ok(None); - } + let Some((_deployment, Some(pod))) = + self.get_deployment_and_pod(app_name, service_name).await? + else { + return Ok(None); }; let p = LogParams { @@ -647,26 +636,25 @@ impl Infrastructure for KubernetesInfrastructure { service_name: &str, status: ServiceStatus, ) -> Result, Error> { - let (service, replicas) = match self.get_service_of_app(app_name, service_name).await? { - Some(service) if service.status() == &status => return Ok(None), - Some(service) => match status { - ServiceStatus::Running => (service, 1), - ServiceStatus::Paused => (service, 0), - }, - None => return Ok(None), + let Some((mut deployment, pod)) = + self.get_deployment_and_pod(app_name, service_name).await? + else { + return Ok(None); }; - let mut deployment = match self.get_deployment(app_name, service_name).await? { - Some(deployment) => deployment, - None => { - return Ok(None); - } - }; + let service = Self::create_service_from_deployment_and_pod(deployment.clone(), pod)?; + if service.status() == &status { + return Ok(None); + } let Some(spec) = deployment.spec.as_mut() else { return Ok(None); }; - spec.replicas = Some(replicas); + + spec.replicas = Some(match status { + ServiceStatus::Running => 1, + ServiceStatus::Paused => 0, + }); Api::::namespaced(self.client().await?, &app_name.to_rfc1123_namespace_id()) .patch( From aab4903d9ae36a9e022a0597debefc10fa4459a3 Mon Sep 17 00:00:00 2001 From: Marc Schreiber Date: Fri, 19 Jan 2024 10:21:26 +0100 Subject: [PATCH 4/6] Do not fail if deployment cannot be parsed --- api/src/infrastructure/kubernetes/infrastructure.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/api/src/infrastructure/kubernetes/infrastructure.rs b/api/src/infrastructure/kubernetes/infrastructure.rs index 03bacf8b..a5723f80 100644 --- a/api/src/infrastructure/kubernetes/infrastructure.rs +++ b/api/src/infrastructure/kubernetes/infrastructure.rs @@ -539,8 +539,9 @@ impl Infrastructure for KubernetesInfrastructure { let deployments = k8s_deployment_unit.deploy(client, app_name).await?; let mut services = Vec::with_capacity(deployments.len()); for deployment in deployments.into_iter() { - let service = Self::create_service_from_deployment_and_pod(deployment, None)?; - services.push(service); + if let Ok(service) = Self::create_service_from_deployment_and_pod(deployment, None) { + services.push(service); + } } Ok(services) From 6f8e7cbcb19e10ef0f3f5f5caf6641d2a3e2e7cb Mon Sep 17 00:00:00 2001 From: Marc Schreiber Date: Fri, 19 Jan 2024 14:34:18 +0100 Subject: [PATCH 5/6] Documentation: include import example --- docs/companions.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/companions.md b/docs/companions.md index 5c234743..97f8b059 100644 --- a/docs/companions.md +++ b/docs/companions.md @@ -129,7 +129,9 @@ use case, then PREvant can utilize the underlying infrastructure to bootstrap the companion configuration from the stdout of containers that are run once within the infrastructure (depicted by following image). PREvant's static companion configuration might be insufficient if services of the application -rely on volume sharing among services (see [#123][persistent-data-issue]). +rely on volume sharing among services (see [#123][persistent-data-issue]) or +when operations are required to be run at the application's start up, e.g. +importing test data. ![](../assets/bootstrap-companions.svg "Illustration how bootstrapping of companions work") From c2b240d05e3e6625e810189ccc6b3d8b9cd0f8e9 Mon Sep 17 00:00:00 2001 From: Marc Schreiber Date: Tue, 23 Jan 2024 14:58:32 +0100 Subject: [PATCH 6/6] Fix typos --- README.md | 24 +++++++------- .../kubernetes/deployment_unit.rs | 2 +- .../kubernetes/infrastructure.rs | 16 ++++----- api/src/infrastructure/kubernetes/payloads.rs | 4 ++- docs/companions.md | 33 +++++++++++-------- 5 files changed, 43 insertions(+), 36 deletions(-) diff --git a/README.md b/README.md index 432c60c6..ca8fa0ed 100644 --- a/README.md +++ b/README.md @@ -16,17 +16,17 @@ Through PREvant's web interface domain experts, managers, developers, and sales ## Basic Terminology -An *application*, that PREvant manages, is a composition microservices based on -“architectural pattern that arranges an application as a collection of loosely -coupled, fine-grained services, communicating through lightweight protocols.” -([Wikipedia][wiki-microservices]) Each application has a unique name which is -the key to perform actions like creating, duplicating, modifying, or deleting -these applications via REST API or Web UI. +An *application*, that PREvant manages, is a composition of microservices based +on an “architectural pattern that arranges an application as a collection of +loosely coupled, fine-grained services, communicating through lightweight +protocols.” ([Wikipedia][wiki-microservices]) Each application has a unique +name which is the key to perform actions like creating, duplicating, modifying, +or deleting these applications via REST API or Web UI. In each application, PREvant manages the microservices as *services* which need to be available in the [OCI Image Format][oci-image-spec] (a.k.a. Docker -images). At least one service needs to be available for an application and -PREvant manages following kind of services. +images). At least one service needs to be available for an application. PREvant +manages the following kind of services: - *Instance*: a service labeled as instance is a service that has been configured explicitly when creating or updating an application. @@ -40,13 +40,13 @@ Additionally, PREvant provides a way of creating service everytime it creates an application. These services are called *companions* and there are two types of them. -- An application wide companion (app companion) is a unique service for the - whole application. For example, a [Kafka][kafka] instance can be stared - automatically everytime you create an application so that all service within +- An application wide companion (app companion) is an unique service for the + whole application. For example, a [Kafka][kafka] instance can be started + automatically everytime you create an application so that all services within the application can synchronize via events. - A companion can also be attached to a service a user wants to deploy (service companion). For example, a [PostgreSQL][postgres] container can be started - for each to provide a dedicated database for each service. + for each service to provide a dedicated database for it. # Usage diff --git a/api/src/infrastructure/kubernetes/deployment_unit.rs b/api/src/infrastructure/kubernetes/deployment_unit.rs index 215409f1..b3d87d03 100644 --- a/api/src/infrastructure/kubernetes/deployment_unit.rs +++ b/api/src/infrastructure/kubernetes/deployment_unit.rs @@ -900,7 +900,7 @@ mod tests { } #[tokio::test] - async fn parse_unit_from_deploymen_stdout() { + async fn parse_unit_from_deployment_stdout() { let unit = parse_unit( r#" apiVersion: apps/v1 diff --git a/api/src/infrastructure/kubernetes/infrastructure.rs b/api/src/infrastructure/kubernetes/infrastructure.rs index a5723f80..503bacea 100644 --- a/api/src/infrastructure/kubernetes/infrastructure.rs +++ b/api/src/infrastructure/kubernetes/infrastructure.rs @@ -313,7 +313,7 @@ impl KubernetesInfrastructure { async fn create_payloads( &self, app_name: &AppName, - deployabel_service: &DeployableService, + deployable_service: &DeployableService, container_config: &ContainerConfig, ) -> Result< ( @@ -325,23 +325,23 @@ impl KubernetesInfrastructure { ), KubernetesInfrastructureError, > { - let secret = deployabel_service + let secret = deployable_service .files() - .map(|files| secrets_payload(app_name, deployabel_service, files)); + .map(|files| secrets_payload(app_name, deployable_service, files)); - let service = service_payload(app_name, deployabel_service); + let service = service_payload(app_name, deployable_service); let deployment = deployment_payload( app_name, - deployabel_service, + deployable_service, container_config, &self - .create_persistent_volume_claim(app_name, deployabel_service) + .create_persistent_volume_claim(app_name, deployable_service) .await?, ); - let ingress_route = ingress_route_payload(app_name, deployabel_service); - let middlewares = middleware_payload(app_name, deployabel_service.ingress_route()); + let ingress_route = ingress_route_payload(app_name, deployable_service); + let middlewares = middleware_payload(app_name, deployable_service.ingress_route()); Ok((secret, service, deployment, ingress_route, middlewares)) } diff --git a/api/src/infrastructure/kubernetes/payloads.rs b/api/src/infrastructure/kubernetes/payloads.rs index ff343405..7a4b567d 100644 --- a/api/src/infrastructure/kubernetes/payloads.rs +++ b/api/src/infrastructure/kubernetes/payloads.rs @@ -278,7 +278,9 @@ pub fn convert_k8s_ingress_to_traefik_ingress( .and_then(|service| service.port.as_ref()) .and_then(|port| port.number) .map(|p| p as u16) - // TODO: how to get the if missing + // TODO: for now it is okay to assume that if the port is missing, port 80 is a + // good default. However, in the future there should be some better error + // handling. .unwrap_or(80), ), }], diff --git a/docs/companions.md b/docs/companions.md index 97f8b059..94860bb1 100644 --- a/docs/companions.md +++ b/docs/companions.md @@ -1,9 +1,10 @@ # Companion Configuration -Have a look at the [basic terminology](../README.md) what is a companion. For -these use cases following sections provide example configurations. +Have a look at the [basic terminology](../README.md) to understand what a +companion is. For these use cases the following sections provide example +configurations. -A simple but limited configuration of companions can be done via the +A simple, but limited configuration of companions can be done via the `config.toml` file for [application companions](#application-wide) and [service companions](#service-based). More complex companions can be created via [bootstrapping](#bootstrapping-from-the-infrastructure-backend). @@ -12,7 +13,8 @@ companions](#service-based). More complex companions can be created via ### Application Wide -If you want to include an OpenID provider for every application, you could use following configuration. +If you want to include an OpenID provider for every application, you could use +the following configuration: ```toml [companions.openid] @@ -49,7 +51,8 @@ The list of available handlebars variables: - `application`: The companion's application information - `name`: The application name -- `services`: An array of the services of the application. Each element has following structure: +- `services`: An array of the services of the application. Each element has the + following structure: - `name`: The service name which is equivalent to the network alias - `port`: The exposed port of the service - `type`: The type of service. For example, `instance`, `replica`, `app-companion`, or `service-companion`. @@ -63,7 +66,9 @@ PREvant provides some handlebars helpers which can be used to generate more comp ### Service Based -The service-based companions works the in the same way as the application-based services. Make sure, that the `serviceName` is unique by using the handlebars templating. +The service-based companions work the in the same way as the application-based +services. Make sure, that the `serviceName` is unique by using handlebars +templating. ```toml [companions.service-name] @@ -84,7 +89,7 @@ The list of available handlebars variables: - `application`: The companion's application information - `name`: The application name -- `service`: The companion's service containing following fields: +- `service`: The companion's service containing the following fields: - `name`: The service name which is equivalent to the network alias - `port`: The exposed port of the service - `type`: The type of service. For example, `instance`, `replica`, `app-companion`, or `service-companion`. @@ -100,7 +105,7 @@ image = 'private.example.com/library/openid:latest' deploymentStrategy = 'redeploy-on-image-update' ``` -`deploymentStrategy` offers following values and if a companion exists for an app following strategy will be applied: +`deploymentStrategy` offers the following values and if a companion exists for an app following strategy will be applied: - `redeploy-always` (_default_): Re-deploys the companion every time there is a new deployment request. - `redeploy-on-image-update`: Re-deploys the companion if there is a more rescent image available. @@ -117,7 +122,7 @@ image = 'postgres:latest' storageStrategy = 'mount-declared-image-volumes' ``` -`storageStrategy` offers following values to determine how storage is managed for a companion: +`storageStrategy` offers the following values to determine how storage is managed for a companion: - `none` (_default_): Companion is deployed without persistent storage. - `mount-declared-image-volumes`: Mounts the volume paths declared within the image, providing persistent storage for the companion. @@ -127,7 +132,7 @@ storageStrategy = 'mount-declared-image-volumes' When the [static configuration](#static-configuration) is insufficient for your use case, then PREvant can utilize the underlying infrastructure to bootstrap the companion configuration from the stdout of containers that are run once -within the infrastructure (depicted by following image). PREvant's static +within the infrastructure (depicted by the following image). PREvant's static companion configuration might be insufficient if services of the application rely on volume sharing among services (see [#123][persistent-data-issue]) or when operations are required to be run at the application's start up, e.g. @@ -143,7 +148,7 @@ underlying infrastructure. - When PREvant uses Kubernetes as the infrastructure runtime, the bootstrap containers need to output [Kubernetes manifests][k8s-manifest]. - Make sure to put out YAML that is compatible with 1.1 and 1.2 + Make sure to output YAML that is compatible with 1.1 and 1.2 (For example, bitnami helm charts have been adjusted in part in that regard, see [here][zookeeper-yaml-1.2-pr] and [here][kafka-yaml-1.2-pr]) - Docker: not yet implemented but the aim is to support [Docker @@ -156,11 +161,11 @@ overwrite configurations. For example, you can change the image used or an environment variable. If you overwrite any configuration the companion will be turned into an instance (as PREvant did before). -Following configuration block depicts that an image +The following configuration block depicts that an image `registry.example.com/user/bootstrap-helm-chart:lastet` is based on a [Helm chart][helm-chart] that generates an OpenID provider for your application with -[Keycloak][keycloak]. Additionally, it is possible to passed additional -arguments to the container than can be templated with [Handlebars][handlebars]. +[Keycloak][keycloak]. Additionally, it is possible to pass additional arguments +to the container than can be templated with [Handlebars][handlebars]. ```toml [[companions.bootstrapping.containers]]