From 8fde866c9a0a326f58f466c5253a0c519561ebd9 Mon Sep 17 00:00:00 2001 From: Bartosz Ciesielczyk Date: Fri, 13 Oct 2023 15:32:52 +0200 Subject: [PATCH 1/6] remove possible nil pointer dereference in mkdir --- service/mount.go | 83 ++++++++++++++++++++++++------------------- service/mount_test.go | 71 ++++++++++++++++++++++++++++++++++++ 2 files changed, 117 insertions(+), 37 deletions(-) create mode 100644 service/mount_test.go diff --git a/service/mount.go b/service/mount.go index f55a143f..356b3211 100644 --- a/service/mount.go +++ b/service/mount.go @@ -16,7 +16,9 @@ package service import ( "context" + "errors" "fmt" + "io/fs" "os" "os/exec" "path/filepath" @@ -66,7 +68,7 @@ func stagePublishNFS(ctx context.Context, req *csi.NodeStageVolumeRequest, expor if len(mnts) != 0 { for _, m := range mnts { - //Idempotency check + // Idempotency check for _, exportPathURL := range exportPaths { if m.Device == exportPathURL { if m.Path == stagingTargetPath { @@ -76,7 +78,7 @@ func stagePublishNFS(ctx context.Context, req *csi.NodeStageVolumeRequest, expor } return status.Error(codes.InvalidArgument, utils.GetMessageWithRunID(rid, "Staging target path: %s is already mounted to export path: %s with conflicting access modes", stagingTargetPath, exportPathURL)) } - //It is possible that a different export path URL is used to mount stage target path + // It is possible that a different export path URL is used to mount stage target path continue } } @@ -85,7 +87,7 @@ func stagePublishNFS(ctx context.Context, req *csi.NodeStageVolumeRequest, expor log.Debugf("Stage - Mount flags for NFS: %s", mntFlags) - var mountOverride = false + mountOverride := false for _, flag := range mntFlags { if strings.Contains(flag, "vers") { mountOverride = true @@ -93,7 +95,7 @@ func stagePublishNFS(ctx context.Context, req *csi.NodeStageVolumeRequest, expor } } - //if nfsv4 specified or mount options is provided, mount as is + // if nfsv4 specified or mount options is provided, mount as is if nfsv4 || mountOverride { nfsv4 = false for _, exportPathURL := range exportPaths { @@ -105,7 +107,7 @@ func stagePublishNFS(ctx context.Context, req *csi.NodeStageVolumeRequest, expor } } - //Try remount as nfsv3 only if NAS server supports nfs v3 + // Try remount as nfsv3 only if NAS server supports nfs v3 if nfsv3 && !nfsv4 && !mountOverride { mntFlags = append(mntFlags, "vers=3") for _, exportPathURL := range exportPaths { @@ -152,7 +154,7 @@ func publishNFS(ctx context.Context, req *csi.NodePublishVolumeRequest, exportPa mntVol := volCap.GetMount() mntFlags := mntVol.GetMountFlags() rwoArray = append(rwoArray, mntFlags...) - //Check if stage target mount exists + // Check if stage target mount exists var stageExportPathURL string stageMountExists := false for _, exportPathURL := range exportPaths { @@ -178,7 +180,7 @@ func publishNFS(ctx context.Context, req *csi.NodePublishVolumeRequest, exportPa if len(mnts) != 0 { for _, m := range mnts { - //Idempotency check + // Idempotency check if m.Device == stageExportPathURL { if m.Path == targetPath { if utils.ArrayContains(m.Opts, rwo) { @@ -192,7 +194,7 @@ func publishNFS(ctx context.Context, req *csi.NodePublishVolumeRequest, exportPa if accMode.Mode == csi.VolumeCapability_AccessMode_SINGLE_NODE_SINGLE_WRITER || (accMode.Mode == csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER && !allowRWOmultiPodAccess) { return status.Error(codes.InvalidArgument, utils.GetMessageWithRunID(rid, "Export path: %s is already mounted to different target path: %s", stageExportPathURL, m.Path)) } - //For multi-node access modes and when allowRWOmultiPodAccess is true for single-node access, target mount will be executed + // For multi-node access modes and when allowRWOmultiPodAccess is true for single-node access, target mount will be executed continue } } @@ -201,7 +203,7 @@ func publishNFS(ctx context.Context, req *csi.NodePublishVolumeRequest, exportPa log.Debugf("Publish - Mount flags for NFS: %s", mntFlags) - var mountOverride = false + mountOverride := false for _, flag := range mntFlags { if strings.Contains(flag, "vers") { mountOverride = true @@ -209,8 +211,8 @@ func publishNFS(ctx context.Context, req *csi.NodePublishVolumeRequest, exportPa } } - //if nfsv4 specified or mount options is provided, mount as is - //Proceeding to perform bind mount to target path + // if nfsv4 specified or mount options is provided, mount as is + // Proceeding to perform bind mount to target path if nfsv4 || mountOverride { nfsv4 = false err = gofsutil.BindMount(ctx, stagingTargetPath, targetPath, rwoArray...) @@ -237,7 +239,7 @@ func stageVolume(ctx context.Context, req *csi.NodeStageVolumeRequest, stagingPa volCap := req.GetVolumeCapability() id := req.GetVolumeId() mntVol := volCap.GetMount() - ro := false //since only SINGLE_NODE_WRITER is supported + ro := false // since only SINGLE_NODE_WRITER is supported // make sure device is valid sysDevice, err := GetDevice(ctx, symlinkPath) @@ -327,7 +329,7 @@ func stageVolume(ctx context.Context, req *csi.NodeStageVolumeRequest, stagingPa return status.Error(codes.InvalidArgument, utils.GetMessageWithRunID(rid, "access mode conflicts with existing mounts")) } } - //It is ok if the device is mounted elsewhere - could be targetPath. If not this will be caught during NodePublish + // It is ok if the device is mounted elsewhere - could be targetPath. If not this will be caught during NodePublish } if !mounted { return status.Error(codes.Internal, utils.GetMessageWithRunID(rid, "device already in use and mounted elsewhere. Cannot do private mount")) @@ -338,7 +340,6 @@ func stageVolume(ctx context.Context, req *csi.NodeStageVolumeRequest, stagingPa } func publishVolume(ctx context.Context, req *csi.NodePublishVolumeRequest, targetPath, symlinkPath, chroot string, allowRWOmultiPodAccess bool) error { - rid, log := utils.GetRunidAndLogger(ctx) stagingPath := req.GetStagingTargetPath() id := req.GetVolumeId() @@ -379,7 +380,7 @@ func publishVolume(ctx context.Context, req *csi.NodePublishVolumeRequest, targe } else if m.Path == stagingPath || m.Path == chroot+stagingPath { continue } else if accMode.GetMode() == csi.VolumeCapability_AccessMode_SINGLE_NODE_SINGLE_WRITER || (accMode.GetMode() == csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER && !allowRWOmultiPodAccess) { - //Device has been mounted aleady to another target + // Device has been mounted aleady to another target return status.Error(codes.Internal, utils.GetMessageWithRunID(rid, "device already in use and mounted elsewhere")) } } @@ -393,7 +394,7 @@ func publishVolume(ctx context.Context, req *csi.NodePublishVolumeRequest, targe if len(pathMnts) > 0 { for _, m := range pathMnts { if !(m.Source == sysDevice.FullPath || m.Device == sysDevice.FullPath) { - //target is mounted by some other device + // target is mounted by some other device return status.Error(codes.Internal, utils.GetMessageWithRunID(rid, "Target is mounted using a different device %s", m.Device)) } } @@ -458,7 +459,7 @@ func unpublishVolume(ctx context.Context, req *csi.NodeUnpublishVolumeRequest) e return nil } - //It is alright if the device is mounted elsewhere - could be staging mount + // It is alright if the device is mounted elsewhere - could be staging mount } devicePath := targetMount.Device @@ -475,7 +476,7 @@ func unpublishVolume(ctx context.Context, req *csi.NodeUnpublishVolumeRequest) e return nil } - //Get existing mounts + // Get existing mounts mnts, err := gofsutil.GetMounts(ctx) if err != nil { return status.Error(codes.Internal, utils.GetMessageWithRunID(rid, "could not reliably determine existing mount status: %s", err.Error())) @@ -489,7 +490,7 @@ func unpublishVolume(ctx context.Context, req *csi.NodeUnpublishVolumeRequest) e break } } else if (m.Source == ubuntuNodeRoot+sysDevice.RealDev || m.Source == sysDevice.RealDev) && m.Device == "udev" { - //For Ubuntu mounts + // For Ubuntu mounts if m.Path == target { tgtMnt = true break @@ -532,7 +533,7 @@ func unstageVolume(ctx context.Context, req *csi.NodeUnstageVolumeRequest, devic stageMount, err := getTargetMount(ctx, stagingTarget) - //Make sure volume is not mounted elsewhere + // Make sure volume is not mounted elsewhere devMnts := make([]gofsutil.Info, 0) symlinkPath, devicePath, err := gofsutil.WWNToDevicePathX(ctx, deviceWWN) @@ -567,7 +568,7 @@ func unstageVolume(ctx context.Context, req *csi.NodeUnstageVolumeRequest, devic return true, devicePath, nil } - //Get existing mounts + // Get existing mounts mnts, err := gofsutil.GetMounts(ctx) if err != nil { return lastUnmounted, "", status.Error(codes.Internal, utils.GetMessageWithRunID(rid, "could not reliably determine existing mount status: %s", err.Error())) @@ -611,7 +612,7 @@ func unpublishNFS(ctx context.Context, targetPath, arrayID string, exportPaths [ ctx, log, rid := GetRunidLog(ctx) ctx, log = setArrayIDContext(ctx, arrayID) - //Get existing mounts + // Get existing mounts mnts, err := gofsutil.GetMounts(ctx) if err != nil { return status.Error(codes.Internal, utils.GetMessageWithRunID(rid, "could not reliably determine existing mount status: %v", err)) @@ -628,7 +629,7 @@ func unpublishNFS(ctx context.Context, targetPath, arrayID string, exportPaths [ } } if !mountExists { - //Path is mounted but with some other NFS Share + // Path is mounted but with some other NFS Share return status.Error(codes.Unknown, utils.GetMessageWithRunID(rid, "Path: %s mounted by different NFS Share with export path: %s", targetPath, m.Device)) } } @@ -637,7 +638,7 @@ func unpublishNFS(ctx context.Context, targetPath, arrayID string, exportPaths [ } } if !mountExists { - //Idempotent case + // Idempotent case log.Debugf("Path: %s not mounted", targetPath) return nil } @@ -662,7 +663,7 @@ func getDevMounts(ctx context.Context, sysDevice *Device) ([]gofsutil.Info, erro } else if (m.Source == ubuntuNodeRoot+sysDevice.RealDev || m.Source == sysDevice.RealDev) && m.Device == "udev" { devMnts = append(devMnts, m) } else { - //Find the multipath device mapper from the device obtained + // Find the multipath device mapper from the device obtained mpDevName := strings.TrimPrefix(sysDevice.RealDev, "/dev/") filename := fmt.Sprintf("/sys/devices/virtual/block/%s/dm/name", mpDevName) if name, err := os.ReadFile(filepath.Clean(filename)); err != nil { @@ -708,7 +709,7 @@ func createDirIfNotExist(ctx context.Context, path, arrayID string) error { tgtStat, err := os.Stat(path) if err != nil { if os.IsNotExist(err) { - //Create target directory if it doesnt exist + // Create target directory if it doesnt exist _, err := mkdir(ctx, path) if err != nil { return status.Error(codes.FailedPrecondition, utils.GetMessageWithRunID(rid, "Could not create path: %s. Error: %v", path, err)) @@ -730,18 +731,25 @@ func createDirIfNotExist(ctx context.Context, path, arrayID string) error { func mkdir(ctx context.Context, path string) (bool, error) { log := utils.GetRunidLogger(ctx) st, err := os.Stat(path) - if os.IsNotExist(err) { - if err := os.MkdirAll(path, 0750); err != nil { - log.WithField("dir", path).WithError(err).Error("Unable to create dir") - return false, err + if err == nil { + if !st.IsDir() { + return false, fmt.Errorf("existing path is not a directory") } - log.WithField("path", path).Debug("created directory") - return true, nil + return false, nil } - if !st.IsDir() { - return false, fmt.Errorf("existing path is not a directory") + if !errors.Is(err, fs.ErrNotExist) { + log.WithField("dir", path).WithError(err).Error("Unable to stat dir") + return false, err } - return false, nil + + // Case when there is error and the error is fs.ErrNotExists. + if err := os.MkdirAll(path, 0o750); err != nil { + log.WithField("dir", path).WithError(err).Error("Unable to create dir") + return false, err + } + + log.WithField("path", path).Debug("created directory") + return true, nil } // mkfile creates a file specified by the path @@ -750,7 +758,7 @@ func mkfile(ctx context.Context, path string) (bool, error) { log := utils.GetRunidLogger(ctx) st, err := os.Stat(path) if os.IsNotExist(err) { - file, err := os.OpenFile(filepath.Clean(path), os.O_CREATE, 0600) + file, err := os.OpenFile(filepath.Clean(path), os.O_CREATE, 0o600) if err != nil { log.WithField("path", path).WithError( err).Error("Unable to create file") @@ -811,7 +819,8 @@ func GetDevice(ctx context.Context, path string) (*Device, error) { func unmountStagingMount( ctx context.Context, dev *Device, - target, chroot string) (bool, error) { + target, chroot string, +) (bool, error) { log := utils.GetRunidLogger(ctx) lastUnmounted := false diff --git a/service/mount_test.go b/service/mount_test.go new file mode 100644 index 00000000..4762b93e --- /dev/null +++ b/service/mount_test.go @@ -0,0 +1,71 @@ +package service + +import ( + "context" + "os" + "testing" + + "github.com/dell/csi-unity/service/utils" + "github.com/stretchr/testify/assert" +) + +// Split TestMkdir into separate tests to test each case +func TestMkdirCreateDir(t *testing.T) { + log := utils.GetLogger() + ctx := context.Background() + entry := log.WithField(utils.RUNID, "1111") + ctx = context.WithValue(ctx, utils.UnityLogger, entry) + + // Make temp directory to use for testing + basepath, err := os.MkdirTemp("/tmp", "*") + defer os.RemoveAll(basepath) + path := basepath + "/test" + + // Test creating a directory + created, err := mkdir(ctx, path) + assert.NoError(t, err) + assert.True(t, created) +} + +func TestMkdirExistingDir(t *testing.T) { + log := utils.GetLogger() + ctx := context.Background() + entry := log.WithField(utils.RUNID, "1111") + ctx = context.WithValue(ctx, utils.UnityLogger, entry) + + // Make temp directory to use for testing + basepath, err := os.MkdirTemp("/tmp", "*") + defer os.RemoveAll(basepath) + + path := basepath + "/test" + + // Pre-create the directory + err = os.Mkdir(path, 0o755) + assert.NoError(t, err) + + // Test creating an existing directory + created, err := mkdir(ctx, path) + assert.NoError(t, err) + assert.False(t, created) +} + +func TestMkdirExistingFile(t *testing.T) { + log := utils.GetLogger() + ctx := context.Background() + entry := log.WithField(utils.RUNID, "1111") + ctx = context.WithValue(ctx, utils.UnityLogger, entry) + + // Make temp directory to use for testing + basepath, err := os.MkdirTemp("/tmp", "*") + defer os.RemoveAll(basepath) + + path := basepath + "/file" + file, err := os.Create(path) + assert.NoError(t, err) + file.Close() + + // Test creating a directory with an existing file path + created, err := mkdir(ctx, path) + assert.Error(t, err) + assert.False(t, created) +} From 981d9631b7b4b5694db61c5387d1183c5efaf53c Mon Sep 17 00:00:00 2001 From: Bartosz Ciesielczyk Date: Mon, 23 Oct 2023 16:10:55 +0200 Subject: [PATCH 2/6] update copyright notice --- service/mount.go | 2 +- service/mount_test.go | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/service/mount.go b/service/mount.go index 356b3211..ab02226f 100644 --- a/service/mount.go +++ b/service/mount.go @@ -1,5 +1,5 @@ /* - Copyright © 2019 Dell Inc. or its subsidiaries. All Rights Reserved. + Copyright © 2019-2023 Dell Inc. or its subsidiaries. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/service/mount_test.go b/service/mount_test.go index 4762b93e..67cf520b 100644 --- a/service/mount_test.go +++ b/service/mount_test.go @@ -1,3 +1,17 @@ +/* + Copyright © 2019-2023 Dell Inc. or its subsidiaries. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + package service import ( From 0cd38addc97016d1eea3e82ca814908cd63dadae Mon Sep 17 00:00:00 2001 From: Alik Saring Date: Fri, 3 Nov 2023 08:36:39 -0400 Subject: [PATCH 3/6] upgrade ubi image --- build.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/build.sh b/build.sh index 736142b9..9ea11d35 100644 --- a/build.sh +++ b/build.sh @@ -28,9 +28,10 @@ function git_version { echo Target Version=$VERSION } + function build_image { - # Tag corresponding to digest sha256:630cf7bdef807f048cadfe7180d6c27eb3aaa99323ffc3628811da230ed3322a for ubi9 micro is 9.2-13 - bash build_ubi_micro.sh registry.access.redhat.com/ubi9/ubi-micro@sha256:630cf7bdef807f048cadfe7180d6c27eb3aaa99323ffc3628811da230ed3322a + # Tag corresponding to digest sha256:d14ac3ae12148f838511d08261e1569fb2a54da4c54a817aea7f16c1c9078f0b for ubi9 micro is 9.2-15 + bash build_ubi_micro.sh registry.redhat.io/ubi9/ubi-micro@sha256:d14ac3ae12148f838511d08261e1569fb2a54da4c54a817aea7f16c1c9078f0b echo $BUILDCMD build -t ${IMAGE_NAME}:${IMAGE_TAG} . (cd .. && $BUILDCMD build -t ${IMAGE_NAME}:${IMAGE_TAG} --build-arg GOPROXY=$GOPROXY -f csi-unity/Dockerfile.podman . --format=docker) echo $BUILDCMD tag ${IMAGE_NAME}:${IMAGE_TAG} ${IMAGE_REPO}/${IMAGE_REPO_NAMESPACE}/${IMAGE_NAME}:${IMAGE_TAG} From 10b41c21f2d7d594abe4a65404a63aad2374a80a Mon Sep 17 00:00:00 2001 From: Alik Saring Date: Tue, 7 Nov 2023 10:29:38 -0500 Subject: [PATCH 4/6] ubi9.3-6 image --- build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.sh b/build.sh index 9ea11d35..8f64dfe8 100644 --- a/build.sh +++ b/build.sh @@ -30,8 +30,8 @@ function git_version { function build_image { - # Tag corresponding to digest sha256:d14ac3ae12148f838511d08261e1569fb2a54da4c54a817aea7f16c1c9078f0b for ubi9 micro is 9.2-15 - bash build_ubi_micro.sh registry.redhat.io/ubi9/ubi-micro@sha256:d14ac3ae12148f838511d08261e1569fb2a54da4c54a817aea7f16c1c9078f0b + # Tag corresponding to digest sha256:b7d8fd2840e92e8adc68414e13d859763ef33c3d4c4e27f910e939c00d642c29 for ubi9 micro is 9.3-6 + bash build_ubi_micro.sh registry.access.redhat.com/ubi9/ubi-micro@sha256:b7d8fd2840e92e8adc68414e13d859763ef33c3d4c4e27f910e939c00d642c29 echo $BUILDCMD build -t ${IMAGE_NAME}:${IMAGE_TAG} . (cd .. && $BUILDCMD build -t ${IMAGE_NAME}:${IMAGE_TAG} --build-arg GOPROXY=$GOPROXY -f csi-unity/Dockerfile.podman . --format=docker) echo $BUILDCMD tag ${IMAGE_NAME}:${IMAGE_TAG} ${IMAGE_REPO}/${IMAGE_REPO_NAMESPACE}/${IMAGE_NAME}:${IMAGE_TAG} From 4bd9537fbb6eefd1bfd0c5dc337e336842c1da18 Mon Sep 17 00:00:00 2001 From: Alik Saring Date: Wed, 8 Nov 2023 08:44:38 -0500 Subject: [PATCH 5/6] add common csm ubi image --- Makefile | 14 ++++++++++---- build.sh | 22 +++++++++++++--------- 2 files changed, 23 insertions(+), 13 deletions(-) diff --git a/Makefile b/Makefile index 3c2cef5d..13533d8a 100644 --- a/Makefile +++ b/Makefile @@ -26,15 +26,21 @@ integration-test: unit-test: ( cd test/unit-test; sh run.sh ) +.PHONY: download-csm-common +download-csm-common: + curl -O -L https://raw.githubusercontent.com/dell/csm/main/config/csm-common.mk + # # Docker-related tasks # # Generates the docker container (but does not push) -podman-build: go-build - sh build.sh +podman-build: download-csm-common go-build + $(eval include csm-common.mk) + sh build.sh --baseubi $(DEFAULT_BASEIMAGE) -podman-push: go-build - sh build.sh -p +podman-push: download-csm-common go-build + $(eval include csm-common.mk) + sh build.sh --baseubi $(DEFAULT_BASEIMAGE) --push # # Docker-related tasks diff --git a/build.sh b/build.sh index 8f64dfe8..a9a53ac0 100644 --- a/build.sh +++ b/build.sh @@ -30,8 +30,8 @@ function git_version { function build_image { - # Tag corresponding to digest sha256:b7d8fd2840e92e8adc68414e13d859763ef33c3d4c4e27f910e939c00d642c29 for ubi9 micro is 9.3-6 - bash build_ubi_micro.sh registry.access.redhat.com/ubi9/ubi-micro@sha256:b7d8fd2840e92e8adc68414e13d859763ef33c3d4c4e27f910e939c00d642c29 + echo ${BASE_UBI_IMAGE} + bash build_ubi_micro.sh ${BASE_UBI_IMAGE} echo $BUILDCMD build -t ${IMAGE_NAME}:${IMAGE_TAG} . (cd .. && $BUILDCMD build -t ${IMAGE_NAME}:${IMAGE_TAG} --build-arg GOPROXY=$GOPROXY -f csi-unity/Dockerfile.podman . --format=docker) echo $BUILDCMD tag ${IMAGE_NAME}:${IMAGE_TAG} ${IMAGE_REPO}/${IMAGE_REPO_NAMESPACE}/${IMAGE_NAME}:${IMAGE_TAG} @@ -53,13 +53,17 @@ IMAGE_REPO_NAMESPACE=csi-unity IMAGE_TAG=${VERSION} # Read options -while getopts 'ph' flag; do - case "${flag}" in - p) PUSH_IMAGE='true' ;; - h) git_version - exit 0 ;; - *) git_version - exit 0 ;; +for param in $*; do + case $param in + "--baseubi") + shift + BASE_UBI_IMAGE=$1 + shift + ;; + "--push") + shift + PUSH_IMAGE='true' + ;; esac done From c44c5cc96cd291790ccaab6e2835704518daa0be Mon Sep 17 00:00:00 2001 From: Alik Saring Date: Wed, 8 Nov 2023 10:31:10 -0500 Subject: [PATCH 6/6] making sure spacing and tap are proper --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 13533d8a..b86bcef4 100644 --- a/Makefile +++ b/Makefile @@ -35,11 +35,11 @@ download-csm-common: # # Generates the docker container (but does not push) podman-build: download-csm-common go-build - $(eval include csm-common.mk) + $(eval include csm-common.mk) sh build.sh --baseubi $(DEFAULT_BASEIMAGE) podman-push: download-csm-common go-build - $(eval include csm-common.mk) + $(eval include csm-common.mk) sh build.sh --baseubi $(DEFAULT_BASEIMAGE) --push #