From e20b1fc5c3a9a246979ddf5980b8f8eeff588c89 Mon Sep 17 00:00:00 2001 From: Michael Burman Date: Tue, 16 Jul 2024 16:22:35 +0300 Subject: [PATCH] Add unit tests, e2e test, modify readOnlyRootFilesystem to be *bool (so we can set it to true by default later), add verification that MCAC is disabled when used with readOnlyRootFilesystem --- .github/workflows/kindIntegTest.yml | 13 ++-- .../workflows/workflow-integration-tests.yaml | 1 - .../v1beta1/cassandradatacenter_types.go | 2 +- .../v1beta1/zz_generated.deepcopy.go | 5 ++ config/manager/image_config.yaml | 2 +- config/manager/kustomization.yaml | 2 +- .../construct_podtemplatespec.go | 7 ++- .../construct_podtemplatespec_test.go | 63 +++++++++++++++++++ .../smoke_test_read_only_fs.go} | 30 +++------ ...-rack-single-node-dc-with-readonly-fs.yaml | 27 ++++++++ .../default-two-rack-two-node-dc.yaml | 1 - 11 files changed, 120 insertions(+), 33 deletions(-) rename tests/{smoke_test_dse/smoke_test_dse_suite_test.go => smoke_test_read_only_fs/smoke_test_read_only_fs.go} (71%) create mode 100644 tests/testdata/default-single-rack-single-node-dc-with-readonly-fs.yaml diff --git a/.github/workflows/kindIntegTest.yml b/.github/workflows/kindIntegTest.yml index 4e5bffc4..29b0d847 100644 --- a/.github/workflows/kindIntegTest.yml +++ b/.github/workflows/kindIntegTest.yml @@ -155,15 +155,13 @@ jobs: strategy: matrix: version: - - "4.1.4" + - "4.1.5" integration_test: # Single worker tests: - additional_serviceoptions - additional_volumes # - delete_node_terminated_container # This does not test any operator behavior - podspec_simple - # - smoke_test_oss # Converted to test_all_the_things, see below job - # - smoke_test_dse # Converted to test_all_the_things, see below job # - terminate # - timeout_prestop_termination # - upgrade_operator # See kind_311_tests job, Only works for 3.11 right now @@ -200,10 +198,17 @@ jobs: - scale_up - scale_up_stop_resume - seed_selection + - smoke_test_read_only_fs #- config_fql # OSS only - decommission_dc # - stop_resume_scale_up # Odd insufficient CPU issues in kind+GHA - # let other tests continue to run + include: + - version: 4.1.5 + serverImage: michaelburman290/cass-management-api:4.1.5-ubi8 # Modified version of cass-management-api + serverType: cassandra + integration_test: "smoke_test_read_only_fs" + + # let other tests continue to run # even if one fails fail-fast: false runs-on: ubuntu-latest diff --git a/.github/workflows/workflow-integration-tests.yaml b/.github/workflows/workflow-integration-tests.yaml index 4f3d3333..bb36d1e7 100644 --- a/.github/workflows/workflow-integration-tests.yaml +++ b/.github/workflows/workflow-integration-tests.yaml @@ -176,7 +176,6 @@ jobs: - additional_volumes # - delete_node_terminated_container # This does not test any operator behavior - podspec_simple - # - smoke_test_oss # Converted to test_all_the_things, see below job # - smoke_test_dse # Converted to test_all_the_things, see below job # - terminate # test_all_things # - timeout_prestop_termination # This is testing a Kubernetes behavior, not interesting to us diff --git a/apis/cassandra/v1beta1/cassandradatacenter_types.go b/apis/cassandra/v1beta1/cassandradatacenter_types.go index 8f6539a9..ec9aa20e 100644 --- a/apis/cassandra/v1beta1/cassandradatacenter_types.go +++ b/apis/cassandra/v1beta1/cassandradatacenter_types.go @@ -271,7 +271,7 @@ type CassandraDatacenterSpec struct { // ReadOnlyRootFilesystem makes the cassandra container to be run with a read-only root filesystem. Currently only functional when used with the // new k8ssandra-client config builder (Cassandra 4.1 and newer and HCD) - ReadOnlyRootFilesystem bool `json:"readOnlyRootFilesystem,omitempty"` + ReadOnlyRootFilesystem *bool `json:"readOnlyRootFilesystem,omitempty"` } type NetworkingConfig struct { diff --git a/apis/cassandra/v1beta1/zz_generated.deepcopy.go b/apis/cassandra/v1beta1/zz_generated.deepcopy.go index ad4af351..6d091ed4 100644 --- a/apis/cassandra/v1beta1/zz_generated.deepcopy.go +++ b/apis/cassandra/v1beta1/zz_generated.deepcopy.go @@ -362,6 +362,11 @@ func (in *CassandraDatacenterSpec) DeepCopyInto(out *CassandraDatacenterSpec) { *out = new(int32) **out = **in } + if in.ReadOnlyRootFilesystem != nil { + in, out := &in.ReadOnlyRootFilesystem, &out.ReadOnlyRootFilesystem + *out = new(bool) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CassandraDatacenterSpec. diff --git a/config/manager/image_config.yaml b/config/manager/image_config.yaml index 2d2de737..34904fb8 100644 --- a/config/manager/image_config.yaml +++ b/config/manager/image_config.yaml @@ -3,7 +3,7 @@ kind: ImageConfig metadata: name: image-config images: - system-logger: "k8ssandra/system-logger:latest" + system-logger: "k8ssandra/system-logger:v1.22.0-dev.e8bef6b-20240716" config-builder: "datastax/cass-config-builder:1.0-ubi8" k8ssandra-client: "k8ssandra/k8ssandra-client:v0.4.0" # cassandra: diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index 7621efda..b3dfc913 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -14,4 +14,4 @@ kind: Kustomization images: - name: controller newName: k8ssandra/cass-operator - newTag: latest + newTag: v1.22.0-dev.e8bef6b-20240716 diff --git a/pkg/reconciliation/construct_podtemplatespec.go b/pkg/reconciliation/construct_podtemplatespec.go index 181f2186..84822762 100644 --- a/pkg/reconciliation/construct_podtemplatespec.go +++ b/pkg/reconciliation/construct_podtemplatespec.go @@ -662,6 +662,7 @@ func buildContainers(dc *api.CassandraDatacenter, baseTemplate *corev1.PodTempla {Name: "NODE_NAME", ValueFrom: selectorFromFieldPath("spec.nodeName")}, {Name: "DS_LICENSE", Value: "accept"}, {Name: "USE_MGMT_API", Value: "true"}, + {Name: "MGMT_API_NO_KEEP_ALIVE", Value: "true"}, {Name: "MGMT_API_EXPLICIT_START", Value: "true"}, } @@ -679,6 +680,10 @@ func buildContainers(dc *api.CassandraDatacenter, baseTemplate *corev1.PodTempla envDefaults = append(envDefaults, corev1.EnvVar{Name: "HCD_AUTO_CONF_OFF", Value: "all"}) } + if readOnlyFs(dc) { + envDefaults = append(envDefaults, corev1.EnvVar{Name: "$MGMT_API_DISABLE_MCAC", Value: "true"}) + } + cassContainer.Env = combineEnvSlices(envDefaults, cassContainer.Env) // Combine ports @@ -801,7 +806,7 @@ func buildContainers(dc *api.CassandraDatacenter, baseTemplate *corev1.PodTempla } func readOnlyFs(dc *api.CassandraDatacenter) bool { - return dc.Spec.ReadOnlyRootFilesystem && dc.UseClientImage() + return dc.Spec.ReadOnlyRootFilesystem != nil && *dc.Spec.ReadOnlyRootFilesystem && dc.UseClientImage() } func buildPodTemplateSpec(dc *api.CassandraDatacenter, rack api.Rack, addLegacyInternodeMount bool) (*corev1.PodTemplateSpec, error) { diff --git a/pkg/reconciliation/construct_podtemplatespec_test.go b/pkg/reconciliation/construct_podtemplatespec_test.go index f8926f7e..711f3cd0 100644 --- a/pkg/reconciliation/construct_podtemplatespec_test.go +++ b/pkg/reconciliation/construct_podtemplatespec_test.go @@ -10,6 +10,7 @@ import ( "testing" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" "k8s.io/apimachinery/pkg/api/resource" @@ -433,6 +434,7 @@ func TestCassandraContainerEnvVars(t *testing.T) { nodeNameEnvVar := corev1.EnvVar{Name: "NODE_NAME", ValueFrom: selectorFromFieldPath("spec.nodeName")} useMgmtApiEnvVar := corev1.EnvVar{Name: "USE_MGMT_API", Value: "true"} explicitStartEnvVar := corev1.EnvVar{Name: "MGMT_API_EXPLICIT_START", Value: "true"} + noKeepAliveEnvVar := corev1.EnvVar{Name: "MGMT_API_NO_KEEP_ALIVE", Value: "true"} templateSpec := &corev1.PodTemplateSpec{} dc := &api.CassandraDatacenter{ @@ -459,6 +461,7 @@ func TestCassandraContainerEnvVars(t *testing.T) { assert.True(envVarsContains(cassContainer.Env, nodeNameEnvVar)) assert.True(envVarsContains(cassContainer.Env, useMgmtApiEnvVar)) assert.True(envVarsContains(cassContainer.Env, explicitStartEnvVar)) + assert.True(envVarsContains(cassContainer.Env, noKeepAliveEnvVar)) } func TestHCDContainerEnvVars(t *testing.T) { @@ -468,6 +471,7 @@ func TestHCDContainerEnvVars(t *testing.T) { useMgmtApiEnvVar := corev1.EnvVar{Name: "USE_MGMT_API", Value: "true"} explicitStartEnvVar := corev1.EnvVar{Name: "MGMT_API_EXPLICIT_START", Value: "true"} hcdAutoConf := corev1.EnvVar{Name: "HCD_AUTO_CONF_OFF", Value: "all"} + noKeepAliveEnvVar := corev1.EnvVar{Name: "MGMT_API_NO_KEEP_ALIVE", Value: "true"} templateSpec := &corev1.PodTemplateSpec{} dc := &api.CassandraDatacenter{ @@ -494,6 +498,7 @@ func TestHCDContainerEnvVars(t *testing.T) { assert.True(envVarsContains(cassContainer.Env, nodeNameEnvVar)) assert.True(envVarsContains(cassContainer.Env, useMgmtApiEnvVar)) assert.True(envVarsContains(cassContainer.Env, explicitStartEnvVar)) + assert.True(envVarsContains(cassContainer.Env, noKeepAliveEnvVar)) assert.True(envVarsContains(cassContainer.Env, hcdAutoConf)) } @@ -503,6 +508,7 @@ func TestDSEContainerEnvVars(t *testing.T) { nodeNameEnvVar := corev1.EnvVar{Name: "NODE_NAME", ValueFrom: selectorFromFieldPath("spec.nodeName")} useMgmtApiEnvVar := corev1.EnvVar{Name: "USE_MGMT_API", Value: "true"} explicitStartEnvVar := corev1.EnvVar{Name: "MGMT_API_EXPLICIT_START", Value: "true"} + noKeepAliveEnvVar := corev1.EnvVar{Name: "MGMT_API_NO_KEEP_ALIVE", Value: "true"} dseExplicitStartEnvVar := corev1.EnvVar{Name: "DSE_MGMT_EXPLICIT_START", Value: "true"} dseAutoConf := corev1.EnvVar{Name: "DSE_AUTO_CONF_OFF", Value: "all"} @@ -531,6 +537,7 @@ func TestDSEContainerEnvVars(t *testing.T) { assert.True(envVarsContains(cassContainer.Env, nodeNameEnvVar)) assert.True(envVarsContains(cassContainer.Env, useMgmtApiEnvVar)) assert.True(envVarsContains(cassContainer.Env, explicitStartEnvVar)) + assert.True(envVarsContains(cassContainer.Env, noKeepAliveEnvVar)) assert.True(envVarsContains(cassContainer.Env, dseAutoConf)) assert.True(envVarsContains(cassContainer.Env, dseExplicitStartEnvVar)) } @@ -1943,3 +1950,59 @@ func TestServiceAccountPrecedence(t *testing.T) { assert.Equal(test.accountName, pds.Spec.ServiceAccountName) } } + +func TestReadOnlyRootFilesystemVolumeChanges(t *testing.T) { + assert := assert.New(t) + dc := &api.CassandraDatacenter{ + Spec: api.CassandraDatacenterSpec{ + ClusterName: "bob", + ServerType: "cassandra", + ServerVersion: "4.1.5", + ReadOnlyRootFilesystem: ptr.To[bool](true), + Racks: []api.Rack{ + { + Name: "r1", + }, + }, + }, + } + + podTemplateSpec, err := buildPodTemplateSpec(dc, dc.Spec.Racks[0], false) + assert.NoError(err, "failed to build PodTemplateSpec") + + containers := podTemplateSpec.Spec.Containers + assert.NotNil(containers, "Unexpected containers containers received") + assert.NoError(err, "Unexpected error encountered") + + assert.Len(containers, 2, "Unexpected number of containers containers returned") + assert.Equal("cassandra", containers[0].Name) + assert.Equal(ptr.To[bool](true), containers[0].SecurityContext.ReadOnlyRootFilesystem) + + assert.True(reflect.DeepEqual(containers[0].VolumeMounts, + []corev1.VolumeMount{ + { + Name: "tmp", + MountPath: "/tmp", + }, + { + Name: "etc-cassandra", + MountPath: "/etc/cassandra", + }, + { + Name: "server-logs", + MountPath: "/var/log/cassandra", + }, + { + Name: "server-data", + MountPath: "/var/lib/cassandra", + }, + { + Name: "server-config", + MountPath: "/config", + }, + }), fmt.Sprintf("Unexpected volume mounts for the cassandra container: %v", containers[0].VolumeMounts)) + + // TODO Verify MCAC is disabled since it will fail with ReadOnlyRootFilesystem + mcacDisabled := corev1.EnvVar{Name: "$MGMT_API_DISABLE_MCAC", Value: "true"} + assert.True(envVarsContains(containers[0].Env, mcacDisabled)) +} diff --git a/tests/smoke_test_dse/smoke_test_dse_suite_test.go b/tests/smoke_test_read_only_fs/smoke_test_read_only_fs.go similarity index 71% rename from tests/smoke_test_dse/smoke_test_dse_suite_test.go rename to tests/smoke_test_read_only_fs/smoke_test_read_only_fs.go index 1049c664..c1fa3de8 100644 --- a/tests/smoke_test_dse/smoke_test_dse_suite_test.go +++ b/tests/smoke_test_read_only_fs/smoke_test_read_only_fs.go @@ -1,7 +1,7 @@ // Copyright DataStax, Inc. // Please see the included license file for details. -package smoke_test_dse +package smoke_test_read_only_fs import ( "fmt" @@ -9,7 +9,6 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - corev1 "k8s.io/api/core/v1" "github.com/k8ssandra/cass-operator/tests/kustomize" ginkgo_util "github.com/k8ssandra/cass-operator/tests/util/ginkgo" @@ -17,13 +16,12 @@ import ( ) var ( - testName = "Smoke test of basic functionality for one-node DSE cluster." - namespace = "test-smoke-test-dse" - dcName = "dc2" - dcYaml = "../testdata/smoke-test-dse.yaml" - dcResource = fmt.Sprintf("CassandraDatacenter/%s", dcName) - dcLabel = fmt.Sprintf("cassandra.datastax.com/datacenter=%s", dcName) - ns = ginkgo_util.NewWrapper(testName, namespace) + testName = "Smoke test of basic functionality for readOnlyRootFilesystem" + namespace = "test-smoke-test-read-only-fs" + dcName = "dc1" + dcYaml = "../testdata/default-single-rack-single-node-dc-with-readonly-fs.yaml" + dcLabel = fmt.Sprintf("cassandra.datastax.com/datacenter=%s", dcName) + ns = ginkgo_util.NewWrapper(testName, namespace) ) func TestLifecycle(t *testing.T) { @@ -68,20 +66,6 @@ var _ = Describe(testName, func() { ns.WaitForDatacenterReady(dcName) ns.ExpectDoneReconciling(dcName) - step = "scale up to 2 nodes" - json = "{\"spec\": {\"size\": 2}}" - k = kubectl.PatchMerge(dcResource, json) - ns.ExecAndLog(step, k) - - ns.WaitForDatacenterCondition(dcName, "ScalingUp", string(corev1.ConditionTrue)) - ns.WaitForDatacenterOperatorProgress(dcName, "Updating", 60) - ns.WaitForDatacenterCondition(dcName, "ScalingUp", string(corev1.ConditionFalse)) - - // Ensure that when 'ScaleUp' becomes 'false' that our pods are in fact up and running - Expect(len(ns.GetDatacenterReadyPodNames(dcName))).To(Equal(2)) - - ns.WaitForDatacenterReady(dcName) - step = "deleting the dc" k = kubectl.DeleteFromFiles(dcYaml) ns.ExecAndLog(step, k) diff --git a/tests/testdata/default-single-rack-single-node-dc-with-readonly-fs.yaml b/tests/testdata/default-single-rack-single-node-dc-with-readonly-fs.yaml new file mode 100644 index 00000000..8a734a74 --- /dev/null +++ b/tests/testdata/default-single-rack-single-node-dc-with-readonly-fs.yaml @@ -0,0 +1,27 @@ +apiVersion: cassandra.datastax.com/v1beta1 +kind: CassandraDatacenter +metadata: + name: dc1 +spec: + clusterName: cluster1 + serverType: cassandra + serverVersion: "4.1.5" + serverImage: michaelburman290/cass-management-api:4.1.5-ubi8 + managementApiAuth: + insecure: {} + readOnlyRootFilesystem: true + size: 1 + storageConfig: + cassandraDataVolumeClaimSpec: + storageClassName: standard + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + racks: + - name: r1 + config: + jvm-server-options: + initial_heap_size: "512m" + max_heap_size: "512m" diff --git a/tests/testdata/default-two-rack-two-node-dc.yaml b/tests/testdata/default-two-rack-two-node-dc.yaml index 8ed229a1..97817e4c 100644 --- a/tests/testdata/default-two-rack-two-node-dc.yaml +++ b/tests/testdata/default-two-rack-two-node-dc.yaml @@ -9,7 +9,6 @@ spec: serverVersion: "4.0.10" managementApiAuth: insecure: {} - readOnlyRootFilesystem: true size: 2 storageConfig: cassandraDataVolumeClaimSpec: