Skip to content

Commit

Permalink
Add unit tests, e2e test, modify readOnlyRootFilesystem to be *bool (…
Browse files Browse the repository at this point in the history
…so we can set it to true by default later), add verification that MCAC is disabled when used with readOnlyRootFilesystem
  • Loading branch information
burmanm committed Jul 19, 2024
1 parent bb8e733 commit e20b1fc
Show file tree
Hide file tree
Showing 11 changed files with 120 additions and 33 deletions.
13 changes: 9 additions & 4 deletions .github/workflows/kindIntegTest.yml
Original file line number Diff line number Diff line change
Expand Up @@ -155,15 +155,13 @@ jobs:
strategy:
matrix:
version:
- "4.1.4"
- "4.1.5"
integration_test:
# Single worker tests:
- additional_serviceoptions
- additional_volumes
# - delete_node_terminated_container # This does not test any operator behavior
- podspec_simple
# - smoke_test_oss # Converted to test_all_the_things, see below job
# - smoke_test_dse # Converted to test_all_the_things, see below job
# - terminate
# - timeout_prestop_termination
# - upgrade_operator # See kind_311_tests job, Only works for 3.11 right now
Expand Down Expand Up @@ -200,10 +198,17 @@ jobs:
- scale_up
- scale_up_stop_resume
- seed_selection
- smoke_test_read_only_fs
#- config_fql # OSS only
- decommission_dc
# - stop_resume_scale_up # Odd insufficient CPU issues in kind+GHA
# let other tests continue to run
include:
- version: 4.1.5
serverImage: michaelburman290/cass-management-api:4.1.5-ubi8 # Modified version of cass-management-api
serverType: cassandra
integration_test: "smoke_test_read_only_fs"

# let other tests continue to run
# even if one fails
fail-fast: false
runs-on: ubuntu-latest
Expand Down
1 change: 0 additions & 1 deletion .github/workflows/workflow-integration-tests.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -176,7 +176,6 @@ jobs:
- additional_volumes
# - delete_node_terminated_container # This does not test any operator behavior
- podspec_simple
# - smoke_test_oss # Converted to test_all_the_things, see below job
# - smoke_test_dse # Converted to test_all_the_things, see below job
# - terminate # test_all_things
# - timeout_prestop_termination # This is testing a Kubernetes behavior, not interesting to us
Expand Down
2 changes: 1 addition & 1 deletion apis/cassandra/v1beta1/cassandradatacenter_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -271,7 +271,7 @@ type CassandraDatacenterSpec struct {

// ReadOnlyRootFilesystem makes the cassandra container to be run with a read-only root filesystem. Currently only functional when used with the
// new k8ssandra-client config builder (Cassandra 4.1 and newer and HCD)
ReadOnlyRootFilesystem bool `json:"readOnlyRootFilesystem,omitempty"`
ReadOnlyRootFilesystem *bool `json:"readOnlyRootFilesystem,omitempty"`
}

type NetworkingConfig struct {
Expand Down
5 changes: 5 additions & 0 deletions apis/cassandra/v1beta1/zz_generated.deepcopy.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion config/manager/image_config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ kind: ImageConfig
metadata:
name: image-config
images:
system-logger: "k8ssandra/system-logger:latest"
system-logger: "k8ssandra/system-logger:v1.22.0-dev.e8bef6b-20240716"
config-builder: "datastax/cass-config-builder:1.0-ubi8"
k8ssandra-client: "k8ssandra/k8ssandra-client:v0.4.0"
# cassandra:
Expand Down
2 changes: 1 addition & 1 deletion config/manager/kustomization.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -14,4 +14,4 @@ kind: Kustomization
images:
- name: controller
newName: k8ssandra/cass-operator
newTag: latest
newTag: v1.22.0-dev.e8bef6b-20240716
7 changes: 6 additions & 1 deletion pkg/reconciliation/construct_podtemplatespec.go
Original file line number Diff line number Diff line change
Expand Up @@ -662,6 +662,7 @@ func buildContainers(dc *api.CassandraDatacenter, baseTemplate *corev1.PodTempla
{Name: "NODE_NAME", ValueFrom: selectorFromFieldPath("spec.nodeName")},
{Name: "DS_LICENSE", Value: "accept"},
{Name: "USE_MGMT_API", Value: "true"},
{Name: "MGMT_API_NO_KEEP_ALIVE", Value: "true"},
{Name: "MGMT_API_EXPLICIT_START", Value: "true"},
}

Expand All @@ -679,6 +680,10 @@ func buildContainers(dc *api.CassandraDatacenter, baseTemplate *corev1.PodTempla
envDefaults = append(envDefaults, corev1.EnvVar{Name: "HCD_AUTO_CONF_OFF", Value: "all"})
}

if readOnlyFs(dc) {
envDefaults = append(envDefaults, corev1.EnvVar{Name: "$MGMT_API_DISABLE_MCAC", Value: "true"})
}

cassContainer.Env = combineEnvSlices(envDefaults, cassContainer.Env)

// Combine ports
Expand Down Expand Up @@ -801,7 +806,7 @@ func buildContainers(dc *api.CassandraDatacenter, baseTemplate *corev1.PodTempla
}

func readOnlyFs(dc *api.CassandraDatacenter) bool {
return dc.Spec.ReadOnlyRootFilesystem && dc.UseClientImage()
return dc.Spec.ReadOnlyRootFilesystem != nil && *dc.Spec.ReadOnlyRootFilesystem && dc.UseClientImage()
}

func buildPodTemplateSpec(dc *api.CassandraDatacenter, rack api.Rack, addLegacyInternodeMount bool) (*corev1.PodTemplateSpec, error) {
Expand Down
63 changes: 63 additions & 0 deletions pkg/reconciliation/construct_podtemplatespec_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
"testing"

metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/utils/ptr"

"k8s.io/apimachinery/pkg/api/resource"

Expand Down Expand Up @@ -433,6 +434,7 @@ func TestCassandraContainerEnvVars(t *testing.T) {
nodeNameEnvVar := corev1.EnvVar{Name: "NODE_NAME", ValueFrom: selectorFromFieldPath("spec.nodeName")}
useMgmtApiEnvVar := corev1.EnvVar{Name: "USE_MGMT_API", Value: "true"}
explicitStartEnvVar := corev1.EnvVar{Name: "MGMT_API_EXPLICIT_START", Value: "true"}
noKeepAliveEnvVar := corev1.EnvVar{Name: "MGMT_API_NO_KEEP_ALIVE", Value: "true"}

templateSpec := &corev1.PodTemplateSpec{}
dc := &api.CassandraDatacenter{
Expand All @@ -459,6 +461,7 @@ func TestCassandraContainerEnvVars(t *testing.T) {
assert.True(envVarsContains(cassContainer.Env, nodeNameEnvVar))
assert.True(envVarsContains(cassContainer.Env, useMgmtApiEnvVar))
assert.True(envVarsContains(cassContainer.Env, explicitStartEnvVar))
assert.True(envVarsContains(cassContainer.Env, noKeepAliveEnvVar))
}

func TestHCDContainerEnvVars(t *testing.T) {
Expand All @@ -468,6 +471,7 @@ func TestHCDContainerEnvVars(t *testing.T) {
useMgmtApiEnvVar := corev1.EnvVar{Name: "USE_MGMT_API", Value: "true"}
explicitStartEnvVar := corev1.EnvVar{Name: "MGMT_API_EXPLICIT_START", Value: "true"}
hcdAutoConf := corev1.EnvVar{Name: "HCD_AUTO_CONF_OFF", Value: "all"}
noKeepAliveEnvVar := corev1.EnvVar{Name: "MGMT_API_NO_KEEP_ALIVE", Value: "true"}

templateSpec := &corev1.PodTemplateSpec{}
dc := &api.CassandraDatacenter{
Expand All @@ -494,6 +498,7 @@ func TestHCDContainerEnvVars(t *testing.T) {
assert.True(envVarsContains(cassContainer.Env, nodeNameEnvVar))
assert.True(envVarsContains(cassContainer.Env, useMgmtApiEnvVar))
assert.True(envVarsContains(cassContainer.Env, explicitStartEnvVar))
assert.True(envVarsContains(cassContainer.Env, noKeepAliveEnvVar))
assert.True(envVarsContains(cassContainer.Env, hcdAutoConf))
}

Expand All @@ -503,6 +508,7 @@ func TestDSEContainerEnvVars(t *testing.T) {
nodeNameEnvVar := corev1.EnvVar{Name: "NODE_NAME", ValueFrom: selectorFromFieldPath("spec.nodeName")}
useMgmtApiEnvVar := corev1.EnvVar{Name: "USE_MGMT_API", Value: "true"}
explicitStartEnvVar := corev1.EnvVar{Name: "MGMT_API_EXPLICIT_START", Value: "true"}
noKeepAliveEnvVar := corev1.EnvVar{Name: "MGMT_API_NO_KEEP_ALIVE", Value: "true"}
dseExplicitStartEnvVar := corev1.EnvVar{Name: "DSE_MGMT_EXPLICIT_START", Value: "true"}
dseAutoConf := corev1.EnvVar{Name: "DSE_AUTO_CONF_OFF", Value: "all"}

Expand Down Expand Up @@ -531,6 +537,7 @@ func TestDSEContainerEnvVars(t *testing.T) {
assert.True(envVarsContains(cassContainer.Env, nodeNameEnvVar))
assert.True(envVarsContains(cassContainer.Env, useMgmtApiEnvVar))
assert.True(envVarsContains(cassContainer.Env, explicitStartEnvVar))
assert.True(envVarsContains(cassContainer.Env, noKeepAliveEnvVar))
assert.True(envVarsContains(cassContainer.Env, dseAutoConf))
assert.True(envVarsContains(cassContainer.Env, dseExplicitStartEnvVar))
}
Expand Down Expand Up @@ -1943,3 +1950,59 @@ func TestServiceAccountPrecedence(t *testing.T) {
assert.Equal(test.accountName, pds.Spec.ServiceAccountName)
}
}

func TestReadOnlyRootFilesystemVolumeChanges(t *testing.T) {
assert := assert.New(t)
dc := &api.CassandraDatacenter{
Spec: api.CassandraDatacenterSpec{
ClusterName: "bob",
ServerType: "cassandra",
ServerVersion: "4.1.5",
ReadOnlyRootFilesystem: ptr.To[bool](true),
Racks: []api.Rack{
{
Name: "r1",
},
},
},
}

podTemplateSpec, err := buildPodTemplateSpec(dc, dc.Spec.Racks[0], false)
assert.NoError(err, "failed to build PodTemplateSpec")

containers := podTemplateSpec.Spec.Containers
assert.NotNil(containers, "Unexpected containers containers received")
assert.NoError(err, "Unexpected error encountered")

assert.Len(containers, 2, "Unexpected number of containers containers returned")
assert.Equal("cassandra", containers[0].Name)
assert.Equal(ptr.To[bool](true), containers[0].SecurityContext.ReadOnlyRootFilesystem)

assert.True(reflect.DeepEqual(containers[0].VolumeMounts,
[]corev1.VolumeMount{
{
Name: "tmp",
MountPath: "/tmp",
},
{
Name: "etc-cassandra",
MountPath: "/etc/cassandra",
},
{
Name: "server-logs",
MountPath: "/var/log/cassandra",
},
{
Name: "server-data",
MountPath: "/var/lib/cassandra",
},
{
Name: "server-config",
MountPath: "/config",
},
}), fmt.Sprintf("Unexpected volume mounts for the cassandra container: %v", containers[0].VolumeMounts))

// TODO Verify MCAC is disabled since it will fail with ReadOnlyRootFilesystem
mcacDisabled := corev1.EnvVar{Name: "$MGMT_API_DISABLE_MCAC", Value: "true"}
assert.True(envVarsContains(containers[0].Env, mcacDisabled))
}
Original file line number Diff line number Diff line change
@@ -1,29 +1,27 @@
// Copyright DataStax, Inc.
// Please see the included license file for details.

package smoke_test_dse
package smoke_test_read_only_fs

import (
"fmt"
"testing"

. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
corev1 "k8s.io/api/core/v1"

"github.com/k8ssandra/cass-operator/tests/kustomize"
ginkgo_util "github.com/k8ssandra/cass-operator/tests/util/ginkgo"
"github.com/k8ssandra/cass-operator/tests/util/kubectl"
)

var (
testName = "Smoke test of basic functionality for one-node DSE cluster."
namespace = "test-smoke-test-dse"
dcName = "dc2"
dcYaml = "../testdata/smoke-test-dse.yaml"
dcResource = fmt.Sprintf("CassandraDatacenter/%s", dcName)
dcLabel = fmt.Sprintf("cassandra.datastax.com/datacenter=%s", dcName)
ns = ginkgo_util.NewWrapper(testName, namespace)
testName = "Smoke test of basic functionality for readOnlyRootFilesystem"
namespace = "test-smoke-test-read-only-fs"
dcName = "dc1"
dcYaml = "../testdata/default-single-rack-single-node-dc-with-readonly-fs.yaml"
dcLabel = fmt.Sprintf("cassandra.datastax.com/datacenter=%s", dcName)
ns = ginkgo_util.NewWrapper(testName, namespace)
)

func TestLifecycle(t *testing.T) {
Expand Down Expand Up @@ -68,20 +66,6 @@ var _ = Describe(testName, func() {
ns.WaitForDatacenterReady(dcName)
ns.ExpectDoneReconciling(dcName)

step = "scale up to 2 nodes"
json = "{\"spec\": {\"size\": 2}}"
k = kubectl.PatchMerge(dcResource, json)
ns.ExecAndLog(step, k)

ns.WaitForDatacenterCondition(dcName, "ScalingUp", string(corev1.ConditionTrue))
ns.WaitForDatacenterOperatorProgress(dcName, "Updating", 60)
ns.WaitForDatacenterCondition(dcName, "ScalingUp", string(corev1.ConditionFalse))

// Ensure that when 'ScaleUp' becomes 'false' that our pods are in fact up and running
Expect(len(ns.GetDatacenterReadyPodNames(dcName))).To(Equal(2))

ns.WaitForDatacenterReady(dcName)

step = "deleting the dc"
k = kubectl.DeleteFromFiles(dcYaml)
ns.ExecAndLog(step, k)
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
apiVersion: cassandra.datastax.com/v1beta1
kind: CassandraDatacenter
metadata:
name: dc1
spec:
clusterName: cluster1
serverType: cassandra
serverVersion: "4.1.5"
serverImage: michaelburman290/cass-management-api:4.1.5-ubi8
managementApiAuth:
insecure: {}
readOnlyRootFilesystem: true
size: 1
storageConfig:
cassandraDataVolumeClaimSpec:
storageClassName: standard
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 1Gi
racks:
- name: r1
config:
jvm-server-options:
initial_heap_size: "512m"
max_heap_size: "512m"
1 change: 0 additions & 1 deletion tests/testdata/default-two-rack-two-node-dc.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ spec:
serverVersion: "4.0.10"
managementApiAuth:
insecure: {}
readOnlyRootFilesystem: true
size: 2
storageConfig:
cassandraDataVolumeClaimSpec:
Expand Down

0 comments on commit e20b1fc

Please sign in to comment.