Add unit tests, e2e test, modify readOnlyRootFilesystem to be *bool (…

…so we can set it to true by default later), add verification that MCAC is disabled when used with readOnlyRootFilesystem

.github/workflows/kindIntegTest.yml

@@ -155,15 +155,13 @@ jobs:
     strategy:
       matrix:
         version:
-          - "4.1.4"
+          - "4.1.5"
         integration_test:
           # Single worker tests:
           - additional_serviceoptions
           - additional_volumes
           # - delete_node_terminated_container # This does not test any operator behavior
           - podspec_simple
-          # - smoke_test_oss # Converted to test_all_the_things, see below job
-          # - smoke_test_dse # Converted to test_all_the_things, see below job
           # - terminate
           # - timeout_prestop_termination
           # - upgrade_operator # See kind_311_tests job, Only works for 3.11 right now
@@ -200,10 +198,17 @@ jobs:
           - scale_up
           - scale_up_stop_resume
           - seed_selection
+          - smoke_test_read_only_fs
           #- config_fql # OSS only
           - decommission_dc
         # - stop_resume_scale_up # Odd insufficient CPU issues in kind+GHA
-      # let other tests continue to run
+        include:
+          - version: 4.1.5
+            serverImage: michaelburman290/cass-management-api:4.1.5-ubi8 # Modified version of cass-management-api
+            serverType: cassandra
+            integration_test: "smoke_test_read_only_fs"
+
+        # let other tests continue to run
       # even if one fails
       fail-fast: false
     runs-on: ubuntu-latest

.github/workflows/workflow-integration-tests.yaml

@@ -176,7 +176,6 @@ jobs:
           - additional_volumes
           # - delete_node_terminated_container # This does not test any operator behavior
           - podspec_simple
-          # - smoke_test_oss # Converted to test_all_the_things, see below job
           # - smoke_test_dse # Converted to test_all_the_things, see below job
           # - terminate # test_all_things
           # - timeout_prestop_termination # This is testing a Kubernetes behavior, not interesting to us

apis/cassandra/v1beta1/cassandradatacenter_types.go

@@ -271,7 +271,7 @@ type CassandraDatacenterSpec struct {
 
 	// ReadOnlyRootFilesystem makes the cassandra container to be run with a read-only root filesystem. Currently only functional when used with the
 	// new k8ssandra-client config builder (Cassandra 4.1 and newer and HCD)
-	ReadOnlyRootFilesystem bool `json:"readOnlyRootFilesystem,omitempty"`
+	ReadOnlyRootFilesystem *bool `json:"readOnlyRootFilesystem,omitempty"`
 }
 
 type NetworkingConfig struct {

config/manager/image_config.yaml

@@ -3,7 +3,7 @@ kind: ImageConfig
 metadata:
   name: image-config
 images:
-  system-logger: "k8ssandra/system-logger:latest"
+  system-logger: "k8ssandra/system-logger:v1.22.0-dev.e8bef6b-20240716"
   config-builder: "datastax/cass-config-builder:1.0-ubi8"
   k8ssandra-client: "k8ssandra/k8ssandra-client:v0.4.0"
   # cassandra:

config/manager/kustomization.yaml

@@ -14,4 +14,4 @@ kind: Kustomization
 images:
 - name: controller
   newName: k8ssandra/cass-operator
-  newTag: latest
+  newTag: v1.22.0-dev.e8bef6b-20240716

pkg/reconciliation/construct_podtemplatespec.go

@@ -662,6 +662,7 @@ func buildContainers(dc *api.CassandraDatacenter, baseTemplate *corev1.PodTempla
 		{Name: "NODE_NAME", ValueFrom: selectorFromFieldPath("spec.nodeName")},
 		{Name: "DS_LICENSE", Value: "accept"},
 		{Name: "USE_MGMT_API", Value: "true"},
+		{Name: "MGMT_API_NO_KEEP_ALIVE", Value: "true"},
 		{Name: "MGMT_API_EXPLICIT_START", Value: "true"},
 	}
 
@@ -679,6 +680,10 @@ func buildContainers(dc *api.CassandraDatacenter, baseTemplate *corev1.PodTempla
 		envDefaults = append(envDefaults, corev1.EnvVar{Name: "HCD_AUTO_CONF_OFF", Value: "all"})
 	}
 
+	if readOnlyFs(dc) {
+		envDefaults = append(envDefaults, corev1.EnvVar{Name: "$MGMT_API_DISABLE_MCAC", Value: "true"})
+	}
+
 	cassContainer.Env = combineEnvSlices(envDefaults, cassContainer.Env)
 
 	// Combine ports
@@ -801,7 +806,7 @@ func buildContainers(dc *api.CassandraDatacenter, baseTemplate *corev1.PodTempla
 }
 
 func readOnlyFs(dc *api.CassandraDatacenter) bool {
-	return dc.Spec.ReadOnlyRootFilesystem && dc.UseClientImage()
+	return dc.Spec.ReadOnlyRootFilesystem != nil && *dc.Spec.ReadOnlyRootFilesystem && dc.UseClientImage()
 }
 
 func buildPodTemplateSpec(dc *api.CassandraDatacenter, rack api.Rack, addLegacyInternodeMount bool) (*corev1.PodTemplateSpec, error) {

pkg/reconciliation/construct_podtemplatespec_test.go

@@ -10,6 +10,7 @@ import (
 	"testing"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/utils/ptr"
 
 	"k8s.io/apimachinery/pkg/api/resource"
 
@@ -433,6 +434,7 @@ func TestCassandraContainerEnvVars(t *testing.T) {
 	nodeNameEnvVar := corev1.EnvVar{Name: "NODE_NAME", ValueFrom: selectorFromFieldPath("spec.nodeName")}
 	useMgmtApiEnvVar := corev1.EnvVar{Name: "USE_MGMT_API", Value: "true"}
 	explicitStartEnvVar := corev1.EnvVar{Name: "MGMT_API_EXPLICIT_START", Value: "true"}
+	noKeepAliveEnvVar := corev1.EnvVar{Name: "MGMT_API_NO_KEEP_ALIVE", Value: "true"}
 
 	templateSpec := &corev1.PodTemplateSpec{}
 	dc := &api.CassandraDatacenter{
@@ -459,6 +461,7 @@ func TestCassandraContainerEnvVars(t *testing.T) {
 	assert.True(envVarsContains(cassContainer.Env, nodeNameEnvVar))
 	assert.True(envVarsContains(cassContainer.Env, useMgmtApiEnvVar))
 	assert.True(envVarsContains(cassContainer.Env, explicitStartEnvVar))
+	assert.True(envVarsContains(cassContainer.Env, noKeepAliveEnvVar))
 }
 
 func TestHCDContainerEnvVars(t *testing.T) {
@@ -468,6 +471,7 @@ func TestHCDContainerEnvVars(t *testing.T) {
 	useMgmtApiEnvVar := corev1.EnvVar{Name: "USE_MGMT_API", Value: "true"}
 	explicitStartEnvVar := corev1.EnvVar{Name: "MGMT_API_EXPLICIT_START", Value: "true"}
 	hcdAutoConf := corev1.EnvVar{Name: "HCD_AUTO_CONF_OFF", Value: "all"}
+	noKeepAliveEnvVar := corev1.EnvVar{Name: "MGMT_API_NO_KEEP_ALIVE", Value: "true"}
 
 	templateSpec := &corev1.PodTemplateSpec{}
 	dc := &api.CassandraDatacenter{
@@ -494,6 +498,7 @@ func TestHCDContainerEnvVars(t *testing.T) {
 	assert.True(envVarsContains(cassContainer.Env, nodeNameEnvVar))
 	assert.True(envVarsContains(cassContainer.Env, useMgmtApiEnvVar))
 	assert.True(envVarsContains(cassContainer.Env, explicitStartEnvVar))
+	assert.True(envVarsContains(cassContainer.Env, noKeepAliveEnvVar))
 	assert.True(envVarsContains(cassContainer.Env, hcdAutoConf))
 }
 
@@ -503,6 +508,7 @@ func TestDSEContainerEnvVars(t *testing.T) {
 	nodeNameEnvVar := corev1.EnvVar{Name: "NODE_NAME", ValueFrom: selectorFromFieldPath("spec.nodeName")}
 	useMgmtApiEnvVar := corev1.EnvVar{Name: "USE_MGMT_API", Value: "true"}
 	explicitStartEnvVar := corev1.EnvVar{Name: "MGMT_API_EXPLICIT_START", Value: "true"}
+	noKeepAliveEnvVar := corev1.EnvVar{Name: "MGMT_API_NO_KEEP_ALIVE", Value: "true"}
 	dseExplicitStartEnvVar := corev1.EnvVar{Name: "DSE_MGMT_EXPLICIT_START", Value: "true"}
 	dseAutoConf := corev1.EnvVar{Name: "DSE_AUTO_CONF_OFF", Value: "all"}
 
@@ -531,6 +537,7 @@ func TestDSEContainerEnvVars(t *testing.T) {
 	assert.True(envVarsContains(cassContainer.Env, nodeNameEnvVar))
 	assert.True(envVarsContains(cassContainer.Env, useMgmtApiEnvVar))
 	assert.True(envVarsContains(cassContainer.Env, explicitStartEnvVar))
+	assert.True(envVarsContains(cassContainer.Env, noKeepAliveEnvVar))
 	assert.True(envVarsContains(cassContainer.Env, dseAutoConf))
 	assert.True(envVarsContains(cassContainer.Env, dseExplicitStartEnvVar))
 }
@@ -1943,3 +1950,59 @@ func TestServiceAccountPrecedence(t *testing.T) {
 		assert.Equal(test.accountName, pds.Spec.ServiceAccountName)
 	}
 }
+
+func TestReadOnlyRootFilesystemVolumeChanges(t *testing.T) {
+	assert := assert.New(t)
+	dc := &api.CassandraDatacenter{
+		Spec: api.CassandraDatacenterSpec{
+			ClusterName:            "bob",
+			ServerType:             "cassandra",
+			ServerVersion:          "4.1.5",
+			ReadOnlyRootFilesystem: ptr.To[bool](true),
+			Racks: []api.Rack{
+				{
+					Name: "r1",
+				},
+			},
+		},
+	}
+
+	podTemplateSpec, err := buildPodTemplateSpec(dc, dc.Spec.Racks[0], false)
+	assert.NoError(err, "failed to build PodTemplateSpec")
+
+	containers := podTemplateSpec.Spec.Containers
+	assert.NotNil(containers, "Unexpected containers containers received")
+	assert.NoError(err, "Unexpected error encountered")
+
+	assert.Len(containers, 2, "Unexpected number of containers containers returned")
+	assert.Equal("cassandra", containers[0].Name)
+	assert.Equal(ptr.To[bool](true), containers[0].SecurityContext.ReadOnlyRootFilesystem)
+
+	assert.True(reflect.DeepEqual(containers[0].VolumeMounts,
+		[]corev1.VolumeMount{
+			{
+				Name:      "tmp",
+				MountPath: "/tmp",
+			},
+			{
+				Name:      "etc-cassandra",
+				MountPath: "/etc/cassandra",
+			},
+			{
+				Name:      "server-logs",
+				MountPath: "/var/log/cassandra",
+			},
+			{
+				Name:      "server-data",
+				MountPath: "/var/lib/cassandra",
+			},
+			{
+				Name:      "server-config",
+				MountPath: "/config",
+			},
+		}), fmt.Sprintf("Unexpected volume mounts for the cassandra container: %v", containers[0].VolumeMounts))
+
+	// TODO Verify MCAC is disabled since it will fail with ReadOnlyRootFilesystem
+	mcacDisabled := corev1.EnvVar{Name: "$MGMT_API_DISABLE_MCAC", Value: "true"}
+	assert.True(envVarsContains(containers[0].Env, mcacDisabled))
+}

tests/smoke_test_dse/smoke_test_dse_suite_test.go → tests/smoke_test_read_only_fs/smoke_test_read_only_fs.go

@@ -1,29 +1,27 @@
 // Copyright DataStax, Inc.
 // Please see the included license file for details.
 
-package smoke_test_dse
+package smoke_test_read_only_fs
 
 import (
 	"fmt"
 	"testing"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
-	corev1 "k8s.io/api/core/v1"
 
 	"github.com/k8ssandra/cass-operator/tests/kustomize"
 	ginkgo_util "github.com/k8ssandra/cass-operator/tests/util/ginkgo"
 	"github.com/k8ssandra/cass-operator/tests/util/kubectl"
 )
 
 var (
-	testName   = "Smoke test of basic functionality for one-node DSE cluster."
-	namespace  = "test-smoke-test-dse"
-	dcName     = "dc2"
-	dcYaml     = "../testdata/smoke-test-dse.yaml"
-	dcResource = fmt.Sprintf("CassandraDatacenter/%s", dcName)
-	dcLabel    = fmt.Sprintf("cassandra.datastax.com/datacenter=%s", dcName)
-	ns         = ginkgo_util.NewWrapper(testName, namespace)
+	testName  = "Smoke test of basic functionality for readOnlyRootFilesystem"
+	namespace = "test-smoke-test-read-only-fs"
+	dcName    = "dc1"
+	dcYaml    = "../testdata/default-single-rack-single-node-dc-with-readonly-fs.yaml"
+	dcLabel   = fmt.Sprintf("cassandra.datastax.com/datacenter=%s", dcName)
+	ns        = ginkgo_util.NewWrapper(testName, namespace)
 )
 
 func TestLifecycle(t *testing.T) {
@@ -68,20 +66,6 @@ var _ = Describe(testName, func() {
 			ns.WaitForDatacenterReady(dcName)
 			ns.ExpectDoneReconciling(dcName)
 
-			step = "scale up to 2 nodes"
-			json = "{\"spec\": {\"size\": 2}}"
-			k = kubectl.PatchMerge(dcResource, json)
-			ns.ExecAndLog(step, k)
-
-			ns.WaitForDatacenterCondition(dcName, "ScalingUp", string(corev1.ConditionTrue))
-			ns.WaitForDatacenterOperatorProgress(dcName, "Updating", 60)
-			ns.WaitForDatacenterCondition(dcName, "ScalingUp", string(corev1.ConditionFalse))
-
-			// Ensure that when 'ScaleUp' becomes 'false' that our pods are in fact up and running
-			Expect(len(ns.GetDatacenterReadyPodNames(dcName))).To(Equal(2))
-
-			ns.WaitForDatacenterReady(dcName)
-
 			step = "deleting the dc"
 			k = kubectl.DeleteFromFiles(dcYaml)
 			ns.ExecAndLog(step, k)

tests/testdata/default-single-rack-single-node-dc-with-readonly-fs.yaml

@@ -0,0 +1,27 @@
+apiVersion: cassandra.datastax.com/v1beta1
+kind: CassandraDatacenter
+metadata:
+  name: dc1
+spec:
+  clusterName: cluster1
+  serverType: cassandra
+  serverVersion: "4.1.5"
+  serverImage: michaelburman290/cass-management-api:4.1.5-ubi8
+  managementApiAuth:
+    insecure: {}
+  readOnlyRootFilesystem: true
+  size: 1
+  storageConfig:
+      cassandraDataVolumeClaimSpec:
+        storageClassName: standard
+        accessModes:
+          - ReadWriteOnce
+        resources:
+          requests:
+            storage: 1Gi
+  racks:
+    - name: r1
+  config:
+    jvm-server-options:
+      initial_heap_size: "512m"
+      max_heap_size: "512m"

tests/testdata/default-two-rack-two-node-dc.yaml

@@ -9,7 +9,6 @@ spec:
   serverVersion: "4.0.10"
   managementApiAuth:
     insecure: {}
-  readOnlyRootFilesystem: true
   size: 2
   storageConfig:
       cassandraDataVolumeClaimSpec: