Merge branch 'main' into chore/version-bumps

stackabletech · Dec 17, 2024 · 3545cb1 · 3545cb1
2 parents 401c19f + 25e6854
commit 3545cb1
Show file tree

Hide file tree

Showing 15 changed files with 128 additions and 26 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -14,6 +14,7 @@ on:
       - "renovate/**"
     tags:
       - '[0-9][0-9].[0-9]+.[0-9]+'
+      - '[0-9][0-9].[0-9]+.[0-9]+-rc[0-9]+'
   pull_request:
   merge_group:
   schedule:
@@ -25,7 +26,7 @@ env:
   CARGO_TERM_COLOR: always
   CARGO_INCREMENTAL: '0'
   CARGO_PROFILE_DEV_DEBUG: '0'
-  RUST_TOOLCHAIN_VERSION: "1.81.0"
+  RUST_TOOLCHAIN_VERSION: "1.82.0"
   RUSTFLAGS: "-D warnings"
   RUSTDOCFLAGS: "-D warnings"
   RUST_LOG: "info"
@@ -344,9 +345,22 @@ jobs:
         with:
           crate: cargo-edit
           bin: cargo-set-version
-      - name: Update version if PR
-        if: ${{ github.event_name == 'pull_request' }}
-        run: cargo set-version --offline --workspace 0.0.0-pr${{ github.event.pull_request.number }}
+      - name: Update version if PR against main branch
+        if: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.ref == 'main' }}
+        env:
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+        run: |
+          PR_VERSION="0.0.0-pr${PR_NUMBER}"
+          cargo set-version --offline --workspace "$PR_VERSION"
+      - name: Update version if PR against non-main branch
+        # For PRs to be merged against a release branch, use the version that has already been set in the calling script.
+        if: ${{ github.event_name == 'pull_request' && startsWith(github.event.pull_request.base.ref, 'release-') }}
+        env:
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+        run: |
+          MANIFEST_VERSION=$(cargo metadata --format-version 1 --no-deps | jq -r '.packages[0].version')
+          PR_VERSION="${MANIFEST_VERSION}-pr${PR_NUMBER}"
+          cargo set-version --offline --workspace "$PR_VERSION"
 
       # Recreate charts and publish charts and docker image. The "-e" is needed as we want to override the
       # default value in the makefile if called from this action, but not otherwise (i.e. when called locally).
@@ -410,9 +424,22 @@ jobs:
         with:
           crate: cargo-edit
           bin: cargo-set-version
-      - name: Update version if PR
-        if: ${{ github.event_name == 'pull_request' }}
-        run: cargo set-version --offline --workspace 0.0.0-pr${{ github.event.pull_request.number }}
+      - name: Update version if PR against main branch
+        if: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.ref == 'main' }}
+        env:
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+        run: |
+          PR_VERSION="0.0.0-pr${PR_NUMBER}"
+          cargo set-version --offline --workspace "$PR_VERSION"
+      - name: Update version if PR against non-main branch
+        # For PRs to be merged against a release branch, use the version that has already been set in the calling script.
+        if: ${{ github.event_name == 'pull_request' && startsWith(github.event.pull_request.base.ref, 'release-') }}
+        env:
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+        run: |
+          MANIFEST_VERSION=$(cargo metadata --format-version 1 --no-deps | jq -r '.packages[0].version')
+          PR_VERSION="${MANIFEST_VERSION}-pr${PR_NUMBER}"
+          cargo set-version --offline --workspace "$PR_VERSION"
       - name: Build manifest list
         run: |
           # Creating manifest list

diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml
@@ -21,12 +21,16 @@ on:
     inputs:
       test-platform:
         description: |
-          The test platform to run on (kind doesn't support `arm64`)
+          The test platform to run on
         required: true
         type: choice
         options:
-          - kind-1.31.0
-          - kind-1.30.3
+          - kind-1.31.2
+          - kind-1.30.6
+          - rke2-1.31.2
+          - rke2-1.30.6
+          - k3s-1.31.2
+          - k3s-1.30.6
           - aks-1.29
           - aks-1.28
           - aks-1.27
@@ -41,7 +45,8 @@ on:
           - okd-4.13
       test-architecture:
         description: |
-          The architecture the tests will run on
+          The architecture the tests will run on. Consult the run-integration-test action README for
+          more details on supported architectures for each distribution
         required: true
         type: choice
         options:
@@ -81,7 +86,7 @@ jobs:
 
       - name: Run Integration Test
         id: test
-        uses: stackabletech/actions/run-integration-test@5b66858af3597c4ea34f9b33664b8034a1d28427 # v0.3.0
+        uses: stackabletech/actions/run-integration-test@95626907fd8cae2f17c862ae40bcab1aa5a2c854 # v0.5.0
         with:
           test-platform: ${{ env.TEST_PLATFORM }}-${{ env.TEST_ARCHITECTURE }}
           test-run: ${{ env.TEST_RUN }}

diff --git a/.github/workflows/pr_pre-commit.yaml b/.github/workflows/pr_pre-commit.yaml
@@ -6,7 +6,7 @@ on:
 
 env:
   CARGO_TERM_COLOR: always
-  RUST_TOOLCHAIN_VERSION: "1.81.0"
+  RUST_TOOLCHAIN_VERSION: "1.82.0"
   HADOLINT_VERSION: "v2.12.0"
   PYTHON_VERSION: "3.12"
 

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -4,6 +4,11 @@ All notable changes to this project will be documented in this file.
 
 ## [Unreleased]
 
+### Added
+
+- The lifetime of auto generated TLS certificates is now configurable with the role and roleGroup
+  config property `requestedSecretLifetime`. This helps reducing frequent Pod restarts ([#722]).
+
 ### Fixed
 
 - Fix OIDC endpoint construction in case the `rootPath` does not have a trailing slash ([#718]).
@@ -13,6 +18,7 @@ All notable changes to this project will be documented in this file.
 
 [#717]: https://github.com/stackabletech/nifi-operator/pull/717
 [#718]: https://github.com/stackabletech/nifi-operator/pull/718
+[#722]: https://github.com/stackabletech/nifi-operator/pull/722
 
 ## [24.11.0] - 2024-11-18
 

diff --git a/Cargo.toml b/Cargo.toml
@@ -32,6 +32,6 @@ tracing = "0.1"
 url = { version = "2.5.2" }
 xml-rs = "0.8"
 
-# [patch."https://github.com/stackabletech/operator-rs.git"]
-# stackable-operator = { git = "https://github.com/stackabletech//operator-rs.git", branch = "main" }
+#[patch."https://github.com/stackabletech/operator-rs.git"]
+#stackable-operator = { git = "https://github.com/stackabletech//operator-rs.git", branch = "main" }
 # stackable-operator = { path = "../operator-rs/crates/stackable-operator" }
diff --git a/deny.toml b/deny.toml
@@ -9,6 +9,27 @@ targets = [
 
 [advisories]
 yanked = "deny"
+ignore = [
+    # https://rustsec.org/advisories/RUSTSEC-2023-0071
+    # "rsa" crate: Marvin Attack: potential key recovery through timing sidechannel
+    #
+    # No patch is yet available, however work is underway to migrate to a fully constant-time implementation
+    # So we need to accept this, as of SDP 24.11 we are not using the rsa crate to create certificates used in production
+    # setups.
+    #
+    # TODO: Remove after https://github.com/RustCrypto/RSA/pull/394 is merged
+    "RUSTSEC-2023-0071",
+
+    # https://rustsec.org/advisories/RUSTSEC-2024-0384
+    # "instant" is unmaintained
+    #
+    # The upstream "kube" crate also silenced this in https://github.com/kube-rs/kube/commit/4f1e889f265da8f19f03f60683569cae1a154fda
+    # They/we are actively working on migrating kube from backoff to backon, which removes the transitive dependency on
+    # instant, in https://github.com/kube-rs/kube/pull/1652.
+    #
+    # TODO: Remove after https://github.com/kube-rs/kube/pull/1652 is merged
+    "RUSTSEC-2024-0384",
+]
 
 [bans]
 multiple-versions = "allow"
@@ -26,6 +47,7 @@ allow = [
     "LicenseRef-webpki",
     "MIT",
     "MPL-2.0",
+    "OpenSSL", # Needed for the ring and/or aws-lc-sys crate. See https://github.com/stackabletech/operator-templating/pull/464 for details
     "Unicode-3.0",
     "Unicode-DFS-2016",
     "Zlib",

diff --git a/deploy/helm/nifi-operator/crds/crds.yaml b/deploy/helm/nifi-operator/crds/crds.yaml
@@ -358,6 +358,10 @@ spec:
                               nullable: true
                               type: boolean
                           type: object
+                        requestedSecretLifetime:
+                          description: Request secret (currently only autoTls certificates) lifetime from the secret operator, e.g. `7d`, or `30d`. Please note that this can be shortened by the `maxCertificateLifetime` setting on the SecretClass issuing the TLS certificate.
+                          nullable: true
+                          type: string
                         resources:
                           default:
                             cpu:
@@ -815,6 +819,10 @@ spec:
                                     nullable: true
                                     type: boolean
                                 type: object
+                              requestedSecretLifetime:
+                                description: Request secret (currently only autoTls certificates) lifetime from the secret operator, e.g. `7d`, or `30d`. Please note that this can be shortened by the `maxCertificateLifetime` setting on the SecretClass issuing the TLS certificate.
+                                nullable: true
+                                type: string
                               resources:
                                 default:
                                   cpu:

diff --git a/rust-toolchain.toml b/rust-toolchain.toml
@@ -1,3 +1,3 @@
 # DO NOT EDIT, this file is generated by operator-templating
 [toolchain]
-channel = "1.81.0"
+channel = "1.82.0"
diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs
@@ -411,10 +411,16 @@ pub struct NifiConfig {
     /// Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details.
     #[fragment_attrs(serde(default))]
     pub graceful_shutdown_timeout: Option<Duration>,
+
+    /// Request secret (currently only autoTls certificates) lifetime from the secret operator, e.g. `7d`, or `30d`.
+    /// Please note that this can be shortened by the `maxCertificateLifetime` setting on the SecretClass issuing the TLS certificate.
+    #[fragment_attrs(serde(default))]
+    pub requested_secret_lifetime: Option<Duration>,
 }
 
 impl NifiConfig {
-    pub const NIFI_SENSITIVE_PROPS_KEY: &'static str = "NIFI_SENSITIVE_PROPS_KEY";
+    // Auto TLS certificate lifetime
+    const DEFAULT_NODE_SECRET_LIFETIME: Duration = Duration::from_days_unchecked(1);
 
     pub fn default_config(cluster_name: &str, role: &NifiRole) -> NifiConfigFragment {
         NifiConfigFragment {
@@ -458,6 +464,7 @@ impl NifiConfig {
             },
             affinity: get_affinity(cluster_name, role),
             graceful_shutdown_timeout: Some(DEFAULT_NODE_GRACEFUL_SHUTDOWN_TIMEOUT),
+            requested_secret_lifetime: Some(Self::DEFAULT_NODE_SECRET_LIFETIME),
         }
     }
 }

diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs
@@ -112,6 +112,9 @@ pub struct Ctx {
 #[strum_discriminants(derive(IntoStaticStr))]
 #[allow(clippy::enum_variant_names)]
 pub enum Error {
+    #[snafu(display("missing secret lifetime"))]
+    MissingSecretLifetime,
+
     #[snafu(display("NifiCluster object is invalid"))]
     InvalidNifiCluster {
         source: error_boundary::InvalidObject,
@@ -1253,6 +1256,9 @@ async fn build_node_rolegroup_statefulset(
         .context(MetadataBuildSnafu)?
         .build();
 
+    let requested_secret_lifetime = merged_config
+        .requested_secret_lifetime
+        .context(MissingSecretLifetimeSnafu)?;
     let nifi_cluster_name = nifi.name_any();
     pod_builder
         .metadata(metadata)
@@ -1301,6 +1307,7 @@ async fn build_node_rolegroup_statefulset(
                     &build_reporting_task_service_name(&nifi_cluster_name),
                 ],
                 SecretFormat::TlsPkcs12,
+                &requested_secret_lifetime,
             )
             .context(SecuritySnafu)?,
         )

diff --git a/rust/operator-binary/src/reporting_task/mod.rs b/rust/operator-binary/src/reporting_task/mod.rs
@@ -24,10 +24,15 @@
 //!
 use std::collections::BTreeMap;
 
+use crate::security::{
+    authentication::{NifiAuthenticationConfig, STACKABLE_ADMIN_USERNAME},
+    build_tls_volume,
+};
 use snafu::{OptionExt, ResultExt, Snafu};
 use stackable_nifi_crd::{
     NifiCluster, NifiRole, APP_NAME, HTTPS_PORT, HTTPS_PORT_NAME, METRICS_PORT,
 };
+use stackable_operator::time::Duration;
 use stackable_operator::{
     builder::{
         self,
@@ -50,11 +55,6 @@ use stackable_operator::{
     utils::cluster_info::KubernetesClusterInfo,
 };
 
-use crate::security::{
-    authentication::{NifiAuthenticationConfig, STACKABLE_ADMIN_USERNAME},
-    build_tls_volume,
-};
-
 use super::controller::{build_recommended_labels, NIFI_UID};
 
 const REPORTING_TASK_CERT_VOLUME_NAME: &str = "tls";
@@ -359,6 +359,10 @@ fn build_reporting_task_job(
                 REPORTING_TASK_CERT_VOLUME_NAME,
                 vec![],
                 SecretFormat::TlsPem,
+                // The certificate is only used for the REST API call, so a short lifetime is sufficient.
+                // There is no correct way to configure this job since it's an implementation detail.
+                // Also it will be dropped when support for 1.x is removed.
+                &Duration::from_days_unchecked(1),
             )
             .context(SecretVolumeBuildFailureSnafu)?,
         )

diff --git a/rust/operator-binary/src/security/mod.rs b/rust/operator-binary/src/security/mod.rs
@@ -1,6 +1,7 @@
 use snafu::{ResultExt, Snafu};
 use stackable_nifi_crd::NifiCluster;
 use stackable_operator::client::Client;
+use stackable_operator::time::Duration;
 use stackable_operator::{builder::pod::volume::SecretFormat, k8s_openapi::api::core::v1::Volume};
 
 pub mod authentication;
@@ -42,6 +43,14 @@ pub fn build_tls_volume(
     volume_name: &str,
     service_scopes: Vec<&str>,
     secret_format: SecretFormat,
+    requested_secret_lifetime: &Duration,
 ) -> Result<Volume> {
-    tls::build_tls_volume(nifi, volume_name, service_scopes, secret_format).context(TlsSnafu)
+    tls::build_tls_volume(
+        nifi,
+        volume_name,
+        service_scopes,
+        secret_format,
+        requested_secret_lifetime,
+    )
+    .context(TlsSnafu)
 }
diff --git a/rust/operator-binary/src/security/tls.rs b/rust/operator-binary/src/security/tls.rs
@@ -1,12 +1,12 @@
+use crate::security::authentication::STACKABLE_TLS_STORE_PASSWORD;
 use snafu::{ResultExt, Snafu};
 use stackable_nifi_crd::NifiCluster;
+use stackable_operator::time::Duration;
 use stackable_operator::{
     builder::pod::volume::{SecretFormat, SecretOperatorVolumeSourceBuilder, VolumeBuilder},
     k8s_openapi::api::core::v1::Volume,
 };
 
-use crate::security::authentication::STACKABLE_TLS_STORE_PASSWORD;
-
 pub const KEYSTORE_VOLUME_NAME: &str = "keystore";
 pub const KEYSTORE_NIFI_CONTAINER_MOUNT: &str = "/stackable/keystore";
 pub const TRUSTSTORE_VOLUME_NAME: &str = "truststore";
@@ -26,6 +26,7 @@ pub(crate) fn build_tls_volume(
     volume_name: &str,
     service_scopes: Vec<&str>,
     secret_format: SecretFormat,
+    requested_secret_lifetime: &Duration,
 ) -> Result<Volume> {
     let mut secret_volume_source_builder =
         SecretOperatorVolumeSourceBuilder::new(nifi.server_tls_secret_class());
@@ -44,6 +45,7 @@ pub(crate) fn build_tls_volume(
                 .with_node_scope()
                 .with_pod_scope()
                 .with_format(secret_format)
+                .with_auto_tls_cert_lifetime(*requested_secret_lifetime)
                 .build()
                 .context(TlsCertSecretClassVolumeBuildSnafu)?,
         )

diff --git a/scripts/docs_templating.sh b/scripts/docs_templating.sh
@@ -21,7 +21,7 @@ fi
 if ! command -v jinja2 &> /dev/null
 then
   echo "jinja2 could not be found. Use 'pip install jinja2-cli' to install it."
-  exit
+  exit 1
 fi
 
 # Check if templating vars file exists

diff --git a/tests/infrastructure.yaml b/tests/infrastructure.yaml
@@ -0,0 +1,5 @@
+cluster-ttl: 6h
+instance-size: medium
+disk: 100
+nodes: 3
+parallelism: 1
-Original file line number
+Diff line change
@@ Expand Up / @@ -21,7 +21,7 @@ fi @@
     if ! command -v jinja2 &> /dev/null
     then
       echo "jinja2 could not be found. Use 'pip install jinja2-cli' to install it."
-      exit
+      exit 1
     fi
     # Check if templating vars file exists
@@ Expand Down @@