Skip to content

Commit

Permalink
change config dir location
Browse files Browse the repository at this point in the history
Before this change, the driver failed to work on Bottlerocket because it
tries to write files into the host directory /etc/amazon/efs. On
Bottlerocket that directory is not writable by the container.

We want to move the host mount point to /var/amazon/efs, but we need to
support backward compatibility with hosts that may already have configs
written to /etc/amazon/efs.

To solve this, we mount both host paths and check for the presence of
config files at container runtime. If we find files in the old location,
we symlink to that location, otherwise we symlink to the new location.
  • Loading branch information
webern committed Jan 7, 2021
1 parent aad2f30 commit 6d4b96d
Show file tree
Hide file tree
Showing 6 changed files with 367 additions and 5 deletions.
2 changes: 1 addition & 1 deletion Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ RUN yum install amazon-efs-utils-1.26-3.amzn2.noarch -y
# client certificate, in the same config directory can be persisted to host with a host path volume.
# Otherwise creating a host path volume for that directory will clean up everything inside at the first time.
# Those static files need to be copied back to the config directory when the driver starts up.
RUN cp -r /etc/amazon/efs /etc/amazon/efs-static-files
RUN mv /etc/amazon/efs /etc/amazon/efs-static-files

COPY --from=builder /go/src/github.com/kubernetes-sigs/aws-efs-csi-driver/bin/aws-efs-csi-driver /bin/aws-efs-csi-driver
COPY THIRD-PARTY /
Expand Down
13 changes: 11 additions & 2 deletions cmd/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,11 +26,15 @@ import (
"github.com/kubernetes-sigs/aws-efs-csi-driver/pkg/driver"
)

// etcAmazonEfs is the non-negotiable directory that the mount.efs will use for config files. We will create a symlink here.
const etcAmazonEfs = "/etc/amazon/efs"

func main() {
var (
endpoint = flag.String("endpoint", "unix://tmp/csi.sock", "CSI Endpoint")
version = flag.Bool("version", false, "Print the version and exit")
efsUtilsCfgDirPath = flag.String("efs-utils-config-dir-path", "/etc/amazon/efs/", "The path to efs-utils config directory")
efsUtilsCfgDirPath = flag.String("efs-utils-config-dir-path", "/var/amazon/efs", "The preferred path for the efs-utils config directory. efs-utils-config-legacy-dir-path will be used if it is not empty, otherwise efs-utils-config-dir-path will be used.")
efsUtilsCfgLegacyDirPath = flag.String("efs-utils-config-legacy-dir-path", "/etc/amazon/efs-legacy", "The path to the legacy efs-utils config directory mounted from the host path /etc/amazon/efs")
efsUtilsStaticFilesPath = flag.String("efs-utils-static-files-path", "/etc/amazon/efs-static-files/", "The path to efs-utils static files directory")
volMetricsOptIn = flag.Bool("vol-metrics-opt-in", false, "Opt in to emit volume metrics")
volMetricsRefreshPeriod = flag.Float64("vol-metrics-refresh-period", 240, "Refresh period for volume metrics in minutes")
Expand All @@ -50,7 +54,12 @@ func main() {
os.Exit(0)
}

drv := driver.NewDriver(*endpoint, *efsUtilsCfgDirPath, *efsUtilsStaticFilesPath, *volMetricsOptIn, *volMetricsRefreshPeriod, *volMetricsFsRateLimit, *deleteAccessPointRootDir)
// chose which configuration directory we will use and create a symlink to it
err := driver.InitConfigDir(*efsUtilsCfgLegacyDirPath, *efsUtilsCfgDirPath, etcAmazonEfs)
if err != nil {
klog.Fatalln(err)
}
drv := driver.NewDriver(*endpoint, etcAmazonEfs, *efsUtilsStaticFilesPath, *volMetricsOptIn, *volMetricsRefreshPeriod, *volMetricsFsRateLimit, *deleteAccessPointRootDir)
if err := drv.Run(); err != nil {
klog.Fatalln(err)
}
Expand Down
8 changes: 7 additions & 1 deletion deploy/kubernetes/base/node.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,9 @@ spec:
- name: efs-state-dir
mountPath: /var/run/efs
- name: efs-utils-config
mountPath: /etc/amazon/efs
mountPath: /var/amazon/efs
- name: efs-utils-config-legacy
mountPath: /etc/amazon/efs-legacy
ports:
- containerPort: 9809
name: healthz
Expand Down Expand Up @@ -104,6 +106,10 @@ spec:
path: /var/run/efs
type: DirectoryOrCreate
- name: efs-utils-config
hostPath:
path: /var/amazon/efs
type: DirectoryOrCreate
- name: efs-utils-config-legacy
hostPath:
path: /etc/amazon/efs
type: DirectoryOrCreate
Expand Down
8 changes: 7 additions & 1 deletion helm/templates/daemonset.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,9 @@ spec:
- name: efs-state-dir
mountPath: /var/run/efs
- name: efs-utils-config
mountPath: /etc/amazon/efs
mountPath: /var/amazon/efs
- name: efs-utils-config-legacy
mountPath: /etc/amazon/efs-legacy
ports:
- name: healthz
containerPort: 9809
Expand Down Expand Up @@ -136,6 +138,10 @@ spec:
path: /var/run/efs
type: DirectoryOrCreate
- name: efs-utils-config
hostPath:
path: /var/amazon/efs
type: DirectoryOrCreate
- name: efs-utils-config-legacy
hostPath:
path: /etc/amazon/efs
type: DirectoryOrCreate
99 changes: 99 additions & 0 deletions pkg/driver/config_dir.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
/*
Copyright 2019 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package driver

import (
"fmt"
"k8s.io/klog"
"os"
"path"
)

// InitConfigDir decides which of two mounted directories will be used to store driver config files. It creates a
// symlink at etcAmazonEfs to the chosen location (legacyDir or preferredDir). If neither of these locations is present
// (i.e. when the user does not need to durably store configs and thus does not mount host directories), an empty
// directory will be created at etcAmazonEfs.
//
// - legacyDir is the path to a config directory where previous versions of this driver may have written config
// files. In previous versions of this driver, a host path that was not writeable on Bottlerocket hosts
// was being used, so we introduce preferredDir.
//
// - preferredDir is the path to config directory that we will use so long as we do not find files in legacyDir.
//
// - etcAmazonEfs is the path where the symlink will be written. In practice, this will always be /etc/amazon/efs, but
// we take it as an input so the function can be tested.
// Examples:
// On a host that has EFS mounts created by an earlier version of this driver, InitConfigDir will detect a conf file in
// legacyDir and write a symlink at etcAmazonEfs pointing to legacyDir.
//
// On a host that does not have pre-existing legacy EFS mount configs, InitConfigDir will detect no files in legacyDir
// and will create a symlink at etcAmazonEfs pointing to preferredDir.
//
// For a use case in which durable storage of the configs is not required, and no host volumes are mounted, the function
// creates an empty directory at etcAmazonEfs.
//
// If a symlink already existing at etcAmazonEfs, InitConfigDir does nothing. If something other than a symlink exists
// at etcAmazonEfs, InitConfigDir returns an error.
func InitConfigDir(legacyDir, preferredDir, etcAmazonEfs string) error {

// if there is already a symlink or directory in place, we have nothing to do
if _, err := os.Stat(etcAmazonEfs); err == nil {
klog.Infof("Symlink or directory exists at '%s', no need to create one", etcAmazonEfs)
return nil
}

// if neither legacyDir nor preferredDir exist, create an empty directory.
if _, err := os.Stat(preferredDir); err != nil {
if _, err := os.Stat(legacyDir); err != nil {
klog.Infof("Mounted directories do not exist, creating directory at '%s'", etcAmazonEfs)
if err := os.MkdirAll(etcAmazonEfs, 0755); err != nil {
return fmt.Errorf(
"unable to create directory at '%s': %s",
etcAmazonEfs,
err.Error())
}
return nil
}
}

// check if a conf file exists in the legacy directory and symlink to the directory if so
existingConfFile := path.Join(legacyDir, "efs-utils.conf")
if _, err := os.Stat(existingConfFile); err == nil {
if err = os.Symlink(legacyDir, etcAmazonEfs); err != nil {
return fmt.Errorf(
"unable to create symlink from '%s' to '%s': %s",
etcAmazonEfs,
legacyDir,
err.Error())
}
klog.Infof("Pre-existing config files are being used from '%s'", legacyDir)
return nil
}

klog.Infof("Creating symlink from '%s' to '%s'", etcAmazonEfs, preferredDir)

// create a symlink to the config directory
if err := os.Symlink(preferredDir, etcAmazonEfs); err != nil {
return fmt.Errorf(
"unable to create symlink from '%s' to '%s': %s",
etcAmazonEfs,
preferredDir,
err.Error())
}

return nil
}
Loading

0 comments on commit 6d4b96d

Please sign in to comment.