Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

ReadOnlyRootFilesystem support #661

Merged
merged 6 commits into from
Jul 19, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 9 additions & 4 deletions .github/workflows/kindIntegTest.yml
Original file line number Diff line number Diff line change
Expand Up @@ -155,15 +155,13 @@ jobs:
strategy:
matrix:
version:
- "4.1.4"
- "4.1.5"
integration_test:
# Single worker tests:
- additional_serviceoptions
- additional_volumes
# - delete_node_terminated_container # This does not test any operator behavior
- podspec_simple
# - smoke_test_oss # Converted to test_all_the_things, see below job
# - smoke_test_dse # Converted to test_all_the_things, see below job
# - terminate
# - timeout_prestop_termination
# - upgrade_operator # See kind_311_tests job, Only works for 3.11 right now
Expand Down Expand Up @@ -200,10 +198,17 @@ jobs:
- scale_up
- scale_up_stop_resume
- seed_selection
- smoke_test_read_only_fs
#- config_fql # OSS only
- decommission_dc
# - stop_resume_scale_up # Odd insufficient CPU issues in kind+GHA
# let other tests continue to run
include:
- version: 4.1.5
serverImage: michaelburman290/cass-management-api:4.1.5-ubi8 # Modified version of cass-management-api
serverType: cassandra
integration_test: "smoke_test_read_only_fs"

# let other tests continue to run
# even if one fails
fail-fast: false
runs-on: ubuntu-latest
Expand Down
1 change: 0 additions & 1 deletion .github/workflows/workflow-integration-tests.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -176,7 +176,6 @@ jobs:
- additional_volumes
# - delete_node_terminated_container # This does not test any operator behavior
- podspec_simple
# - smoke_test_oss # Converted to test_all_the_things, see below job
# - smoke_test_dse # Converted to test_all_the_things, see below job
# - terminate # test_all_things
# - timeout_prestop_termination # This is testing a Kubernetes behavior, not interesting to us
Expand Down
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ Changelog for Cass Operator, new PRs should update the `main / unreleased` secti
* [FEATURE] [#646](https://github.com/k8ssandra/cass-operator/issues/646) Allow starting multiple parallel pods if they have already previously bootstrapped and not planned for replacement. Set annotation ``cassandra.datastax.com/allow-parallel-starts: true`` to enable this feature.
* [ENHANCEMENT] [#648](https://github.com/k8ssandra/cass-operator/issues/648) Make MinReadySeconds configurable value in the Spec.
* [ENHANCEMENT] [#184](https://github.com/k8ssandra/cass-operator/issues/349) Add CassandraDatacenter.Status fields as metrics also
* [ENHANCEMENT] [#199](https://github.com/k8ssandra/cass-operator/issues/199) If .spec.readOnlyRootFilesystem is set, run the cassandra container with readOnlyRootFilesystem. Also, modify the default SecurityContext to mention runAsNonRoot: true

## v1.21.1

Expand Down
4 changes: 4 additions & 0 deletions apis/cassandra/v1beta1/cassandradatacenter_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -268,6 +268,10 @@ type CassandraDatacenterSpec struct {
// MinReadySeconds sets the minimum number of seconds for which a newly created pod should be ready without any of its containers crashing, for it to be considered available. Defaults to 5 seconds and is set in the StatefulSet spec.
// Setting to 0 might cause multiple Cassandra pods to restart at the same time despite PodDisruptionBudget settings.
MinReadySeconds *int32 `json:"minReadySeconds,omitempty"`

// ReadOnlyRootFilesystem makes the cassandra container to be run with a read-only root filesystem. Currently only functional when used with the
// new k8ssandra-client config builder (Cassandra 4.1 and newer and HCD)
ReadOnlyRootFilesystem *bool `json:"readOnlyRootFilesystem,omitempty"`
}

type NetworkingConfig struct {
Expand Down
5 changes: 5 additions & 0 deletions apis/cassandra/v1beta1/zz_generated.deepcopy.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Original file line number Diff line number Diff line change
Expand Up @@ -8820,6 +8820,11 @@ spec:
- name
type: object
type: array
readOnlyRootFilesystem:
description: |-
ReadOnlyRootFilesystem makes the cassandra container to be run with a read-only root filesystem. Currently only functional when used with the
new k8ssandra-client config builder (Cassandra 4.1 and newer and HCD)
type: boolean
replaceNodes:
description: Deprecated Use CassandraTask replacenode to achieve correct
node replacement. A list of pod names that need to be replaced.
Expand Down
55 changes: 51 additions & 4 deletions pkg/reconciliation/construct_podtemplatespec.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import (
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/intstr"
"k8s.io/utils/ptr"
)

const (
Expand Down Expand Up @@ -303,8 +304,27 @@ func addVolumes(dc *api.CassandraDatacenter, baseTemplate *corev1.PodTemplateSpe
EmptyDir: &corev1.EmptyDirVolumeSource{},
},
}

volumeDefaults := []corev1.Volume{vServerConfig, vServerLogs}

if readOnlyFs(dc) {
tmp := corev1.Volume{
Name: "tmp",
VolumeSource: corev1.VolumeSource{
EmptyDir: &corev1.EmptyDirVolumeSource{},
},
}

etcCass := corev1.Volume{
Name: "etc-cassandra",
VolumeSource: corev1.VolumeSource{
EmptyDir: &corev1.EmptyDirVolumeSource{},
},
}

volumeDefaults = append(volumeDefaults, tmp, etcCass)
}

if dc.UseClientImage() {
vBaseConfig := corev1.Volume{
Name: "server-config-base",
Expand Down Expand Up @@ -435,7 +455,7 @@ func buildInitContainers(dc *api.CassandraDatacenter, rackName string, baseTempl

configMounts = append(configMounts, configBaseMount)

// Similar to k8ssandra 1.x, use config-container if use new config-builder replacement
// Similar to k8ssandra 1.x, use config-container if we use k8ssandra-client to build configs
if configContainerIndex < 0 {
configContainer = &corev1.Container{
Name: ServerBaseConfigContainerName,
Expand Down Expand Up @@ -629,13 +649,20 @@ func buildContainers(dc *api.CassandraDatacenter, baseTemplate *corev1.PodTempla
}
}

if readOnlyFs(dc) {
cassContainer.SecurityContext = &corev1.SecurityContext{
ReadOnlyRootFilesystem: ptr.To[bool](true),
}
}

// Combine env vars

envDefaults := []corev1.EnvVar{
{Name: "POD_NAME", ValueFrom: selectorFromFieldPath("metadata.name")},
{Name: "NODE_NAME", ValueFrom: selectorFromFieldPath("spec.nodeName")},
{Name: "DS_LICENSE", Value: "accept"},
{Name: "USE_MGMT_API", Value: "true"},
{Name: "MGMT_API_NO_KEEP_ALIVE", Value: "true"},
{Name: "MGMT_API_EXPLICIT_START", Value: "true"},
}

Expand All @@ -653,6 +680,10 @@ func buildContainers(dc *api.CassandraDatacenter, baseTemplate *corev1.PodTempla
envDefaults = append(envDefaults, corev1.EnvVar{Name: "HCD_AUTO_CONF_OFF", Value: "all"})
}

if readOnlyFs(dc) {
envDefaults = append(envDefaults, corev1.EnvVar{Name: "MGMT_API_DISABLE_MCAC", Value: "true"})
}

cassContainer.Env = combineEnvSlices(envDefaults, cassContainer.Env)

// Combine ports
Expand Down Expand Up @@ -706,6 +737,17 @@ func buildContainers(dc *api.CassandraDatacenter, baseTemplate *corev1.PodTempla
}
}

if readOnlyFs(dc) {
cassContainer.VolumeMounts = append(cassContainer.VolumeMounts, corev1.VolumeMount{
Name: "tmp",
MountPath: "/tmp",
})
cassContainer.VolumeMounts = append(cassContainer.VolumeMounts, corev1.VolumeMount{
Name: "etc-cassandra",
MountPath: "/etc/cassandra",
})
}

volumeMounts = combineVolumeMountSlices(volumeMounts, cassContainer.VolumeMounts)
cassContainer.VolumeMounts = combineVolumeMountSlices(volumeMounts, generateStorageConfigVolumesMount(dc))

Expand Down Expand Up @@ -763,6 +805,10 @@ func buildContainers(dc *api.CassandraDatacenter, baseTemplate *corev1.PodTempla
return nil
}

func readOnlyFs(dc *api.CassandraDatacenter) bool {
return dc.Spec.ReadOnlyRootFilesystem != nil && *dc.Spec.ReadOnlyRootFilesystem && dc.UseClientImage()
}

func buildPodTemplateSpec(dc *api.CassandraDatacenter, rack api.Rack, addLegacyInternodeMount bool) (*corev1.PodTemplateSpec, error) {

baseTemplate := dc.Spec.PodTemplateSpec.DeepCopy()
Expand Down Expand Up @@ -795,9 +841,10 @@ func buildPodTemplateSpec(dc *api.CassandraDatacenter, rack api.Rack, addLegacyI
if baseTemplate.Spec.SecurityContext == nil {
var userID int64 = 999
baseTemplate.Spec.SecurityContext = &corev1.PodSecurityContext{
RunAsUser: &userID,
RunAsGroup: &userID,
FSGroup: &userID,
RunAsUser: &userID,
RunAsGroup: &userID,
FSGroup: &userID,
RunAsNonRoot: ptr.To[bool](true),
}
}

Expand Down
63 changes: 63 additions & 0 deletions pkg/reconciliation/construct_podtemplatespec_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
"testing"

metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/utils/ptr"

"k8s.io/apimachinery/pkg/api/resource"

Expand Down Expand Up @@ -433,6 +434,7 @@ func TestCassandraContainerEnvVars(t *testing.T) {
nodeNameEnvVar := corev1.EnvVar{Name: "NODE_NAME", ValueFrom: selectorFromFieldPath("spec.nodeName")}
useMgmtApiEnvVar := corev1.EnvVar{Name: "USE_MGMT_API", Value: "true"}
explicitStartEnvVar := corev1.EnvVar{Name: "MGMT_API_EXPLICIT_START", Value: "true"}
noKeepAliveEnvVar := corev1.EnvVar{Name: "MGMT_API_NO_KEEP_ALIVE", Value: "true"}

templateSpec := &corev1.PodTemplateSpec{}
dc := &api.CassandraDatacenter{
Expand All @@ -459,6 +461,7 @@ func TestCassandraContainerEnvVars(t *testing.T) {
assert.True(envVarsContains(cassContainer.Env, nodeNameEnvVar))
assert.True(envVarsContains(cassContainer.Env, useMgmtApiEnvVar))
assert.True(envVarsContains(cassContainer.Env, explicitStartEnvVar))
assert.True(envVarsContains(cassContainer.Env, noKeepAliveEnvVar))
}

func TestHCDContainerEnvVars(t *testing.T) {
Expand All @@ -468,6 +471,7 @@ func TestHCDContainerEnvVars(t *testing.T) {
useMgmtApiEnvVar := corev1.EnvVar{Name: "USE_MGMT_API", Value: "true"}
explicitStartEnvVar := corev1.EnvVar{Name: "MGMT_API_EXPLICIT_START", Value: "true"}
hcdAutoConf := corev1.EnvVar{Name: "HCD_AUTO_CONF_OFF", Value: "all"}
noKeepAliveEnvVar := corev1.EnvVar{Name: "MGMT_API_NO_KEEP_ALIVE", Value: "true"}

templateSpec := &corev1.PodTemplateSpec{}
dc := &api.CassandraDatacenter{
Expand All @@ -494,6 +498,7 @@ func TestHCDContainerEnvVars(t *testing.T) {
assert.True(envVarsContains(cassContainer.Env, nodeNameEnvVar))
assert.True(envVarsContains(cassContainer.Env, useMgmtApiEnvVar))
assert.True(envVarsContains(cassContainer.Env, explicitStartEnvVar))
assert.True(envVarsContains(cassContainer.Env, noKeepAliveEnvVar))
assert.True(envVarsContains(cassContainer.Env, hcdAutoConf))
}

Expand All @@ -503,6 +508,7 @@ func TestDSEContainerEnvVars(t *testing.T) {
nodeNameEnvVar := corev1.EnvVar{Name: "NODE_NAME", ValueFrom: selectorFromFieldPath("spec.nodeName")}
useMgmtApiEnvVar := corev1.EnvVar{Name: "USE_MGMT_API", Value: "true"}
explicitStartEnvVar := corev1.EnvVar{Name: "MGMT_API_EXPLICIT_START", Value: "true"}
noKeepAliveEnvVar := corev1.EnvVar{Name: "MGMT_API_NO_KEEP_ALIVE", Value: "true"}
dseExplicitStartEnvVar := corev1.EnvVar{Name: "DSE_MGMT_EXPLICIT_START", Value: "true"}
dseAutoConf := corev1.EnvVar{Name: "DSE_AUTO_CONF_OFF", Value: "all"}

Expand Down Expand Up @@ -531,6 +537,7 @@ func TestDSEContainerEnvVars(t *testing.T) {
assert.True(envVarsContains(cassContainer.Env, nodeNameEnvVar))
assert.True(envVarsContains(cassContainer.Env, useMgmtApiEnvVar))
assert.True(envVarsContains(cassContainer.Env, explicitStartEnvVar))
assert.True(envVarsContains(cassContainer.Env, noKeepAliveEnvVar))
assert.True(envVarsContains(cassContainer.Env, dseAutoConf))
assert.True(envVarsContains(cassContainer.Env, dseExplicitStartEnvVar))
}
Expand Down Expand Up @@ -1943,3 +1950,59 @@ func TestServiceAccountPrecedence(t *testing.T) {
assert.Equal(test.accountName, pds.Spec.ServiceAccountName)
}
}

func TestReadOnlyRootFilesystemVolumeChanges(t *testing.T) {
assert := assert.New(t)
dc := &api.CassandraDatacenter{
Spec: api.CassandraDatacenterSpec{
ClusterName: "bob",
ServerType: "cassandra",
ServerVersion: "4.1.5",
ReadOnlyRootFilesystem: ptr.To[bool](true),
Racks: []api.Rack{
{
Name: "r1",
},
},
},
}

podTemplateSpec, err := buildPodTemplateSpec(dc, dc.Spec.Racks[0], false)
assert.NoError(err, "failed to build PodTemplateSpec")

containers := podTemplateSpec.Spec.Containers
assert.NotNil(containers, "Unexpected containers containers received")
assert.NoError(err, "Unexpected error encountered")

assert.Len(containers, 2, "Unexpected number of containers containers returned")
assert.Equal("cassandra", containers[0].Name)
assert.Equal(ptr.To[bool](true), containers[0].SecurityContext.ReadOnlyRootFilesystem)

assert.True(reflect.DeepEqual(containers[0].VolumeMounts,
[]corev1.VolumeMount{
{
Name: "tmp",
MountPath: "/tmp",
},
{
Name: "etc-cassandra",
MountPath: "/etc/cassandra",
},
{
Name: "server-logs",
MountPath: "/var/log/cassandra",
},
{
Name: "server-data",
MountPath: "/var/lib/cassandra",
},
{
Name: "server-config",
MountPath: "/config",
},
}), fmt.Sprintf("Unexpected volume mounts for the cassandra container: %v", containers[0].VolumeMounts))

// TODO Verify MCAC is disabled since it will fail with ReadOnlyRootFilesystem
mcacDisabled := corev1.EnvVar{Name: "MGMT_API_DISABLE_MCAC", Value: "true"}
assert.True(envVarsContains(containers[0].Env, mcacDisabled))
}
7 changes: 4 additions & 3 deletions pkg/reconciliation/construct_statefulset_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -436,9 +436,10 @@ func Test_newStatefulSetForCassandraPodSecurityContext(t *testing.T) {
}

defaultSecurityContext := &corev1.PodSecurityContext{
RunAsUser: ptr.To(int64(999)),
RunAsGroup: ptr.To(int64(999)),
FSGroup: ptr.To(int64(999)),
RunAsUser: ptr.To(int64(999)),
RunAsGroup: ptr.To(int64(999)),
FSGroup: ptr.To(int64(999)),
RunAsNonRoot: ptr.To[bool](true),
}

tests := []struct {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,29 +1,27 @@
// Copyright DataStax, Inc.
// Please see the included license file for details.

package smoke_test_dse
package smoke_test_read_only_fs

import (
"fmt"
"testing"

. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
corev1 "k8s.io/api/core/v1"

"github.com/k8ssandra/cass-operator/tests/kustomize"
ginkgo_util "github.com/k8ssandra/cass-operator/tests/util/ginkgo"
"github.com/k8ssandra/cass-operator/tests/util/kubectl"
)

var (
testName = "Smoke test of basic functionality for one-node DSE cluster."
namespace = "test-smoke-test-dse"
dcName = "dc2"
dcYaml = "../testdata/smoke-test-dse.yaml"
dcResource = fmt.Sprintf("CassandraDatacenter/%s", dcName)
dcLabel = fmt.Sprintf("cassandra.datastax.com/datacenter=%s", dcName)
ns = ginkgo_util.NewWrapper(testName, namespace)
testName = "Smoke test of basic functionality for readOnlyRootFilesystem"
namespace = "test-smoke-test-read-only-fs"
dcName = "dc1"
dcYaml = "../testdata/default-single-rack-single-node-dc-with-readonly-fs.yaml"
dcLabel = fmt.Sprintf("cassandra.datastax.com/datacenter=%s", dcName)
ns = ginkgo_util.NewWrapper(testName, namespace)
)

func TestLifecycle(t *testing.T) {
Expand Down Expand Up @@ -68,20 +66,6 @@ var _ = Describe(testName, func() {
ns.WaitForDatacenterReady(dcName)
ns.ExpectDoneReconciling(dcName)

step = "scale up to 2 nodes"
json = "{\"spec\": {\"size\": 2}}"
k = kubectl.PatchMerge(dcResource, json)
ns.ExecAndLog(step, k)

ns.WaitForDatacenterCondition(dcName, "ScalingUp", string(corev1.ConditionTrue))
ns.WaitForDatacenterOperatorProgress(dcName, "Updating", 60)
ns.WaitForDatacenterCondition(dcName, "ScalingUp", string(corev1.ConditionFalse))

// Ensure that when 'ScaleUp' becomes 'false' that our pods are in fact up and running
Expect(len(ns.GetDatacenterReadyPodNames(dcName))).To(Equal(2))

ns.WaitForDatacenterReady(dcName)

step = "deleting the dc"
k = kubectl.DeleteFromFiles(dcYaml)
ns.ExecAndLog(step, k)
Expand Down
Loading
Loading