[experiment] bump sdk to 0.58.0-rc1

frameworks/kafka/build.gradle

@@ -19,7 +19,7 @@ ext {
     junitVer = "4.12"
     systemRulesVer = "1.16.0"
     mockitoVer = "2.27.0"
-    dcosSDKVer = "0.57.3"
+    dcosSDKVer = "0.58.0-rc1"
 }
 
 dependencies {

frameworks/kafka/versions.sh

@@ -13,4 +13,4 @@
 # instructions at:
 # https://wiki.mesosphere.com/display/ENGINEERING/Uploading+an+asset+to+production
 export TEMPLATE_KAFKA_VERSION="2.12-2.4.0"
-export TEMPLATE_DCOS_SDK_VERSION="0.57.3"
+export TEMPLATE_DCOS_SDK_VERSION="0.58.0-rc1"

test.sh

@@ -84,8 +84,8 @@ else
   pytest_m="sanity and not azure"
 fi
 gradle_cache="${DCOS_COMMONS_DIRECTORY}/.gradle_cache"
-ssh_path="${HOME}/.ssh/ccm.pem"
-ssh_user="core"
+ssh_path="${HOME}/.ssh/id_rsa"
+ssh_user="centos"
 aws_credentials_path="${HOME}/.aws/credentials"
 enterprise="true"
 headless="false"

testing/sdk_cmd.py

@@ -24,7 +24,7 @@
 log = logging.getLogger(__name__)
 
 DEFAULT_TIMEOUT_SECONDS = 30 * 60
-SSH_USERNAME = os.environ.get("DCOS_SSH_USERNAME", "core")
+SSH_USERNAME = os.environ.get("DCOS_SSH_USERNAME", "centos")
 SSH_KEY_FILE = os.environ.get("DCOS_SSH_KEY_FILE", "")
 
 # Silence this warning. We expect certs to be self-signed:
@@ -36,15 +36,15 @@
 
 
 def service_request(
-        method: str,
-        service_name: str,
-        service_path: str,
-        retry: bool = True,
-        raise_on_error: bool = True,
-        log_args: bool = True,
-        log_response: bool = False,
-        timeout_seconds: int = 60,
-        **kwargs: Any,
+    method: str,
+    service_name: str,
+    service_path: str,
+    retry: bool = True,
+    raise_on_error: bool = True,
+    log_args: bool = True,
+    log_response: bool = False,
+    timeout_seconds: int = 60,
+    **kwargs: Any,
 ) -> requests.Response:
     """Used to query a service running on the cluster. See `cluster_request()` for arg meanings.
     : param service_name: The name of the service, e.g. 'marathon' or 'hello-world'
@@ -184,11 +184,7 @@ def svc_cli(
     return rc, stdout, stderr
 
 
-def run_cli(
-    cmd: str,
-    print_output: bool = True,
-    check: bool = False,
-) -> Tuple[int, str, str]:
+def run_cli(cmd: str, print_output: bool = True, check: bool = False,) -> Tuple[int, str, str]:
     """Runs the command with `dcos` as the prefix to the shell command
     and returns a tuple containing exit code, stdout, and stderr.
 
@@ -201,10 +197,7 @@ def run_cli(
 
 
 def _run_cmd(
-    cmd: str,
-    print_output: bool,
-    check: bool,
-    timeout_seconds: Optional[int] = None,
+    cmd: str, print_output: bool, check: bool, timeout_seconds: Optional[int] = None,
 ) -> Tuple[int, str, str]:
     result = subprocess.run(
         [cmd],
@@ -311,7 +304,9 @@ def kill_task_with_pattern(pattern: str, user: str, agent_host: Optional[str] =
     return rc == 0
 
 
-def master_ssh(cmd: str, timeout_seconds: int = 60, print_output: bool = True, check: bool = False) -> Tuple[int, str, str]:
+def master_ssh(
+    cmd: str, timeout_seconds: int = 60, print_output: bool = True, check: bool = False
+) -> Tuple[int, str, str]:
     """
     Runs the provided command on the cluster leader, using ssh.
     Returns the exit code, stdout, and stderr as three separate values.
@@ -321,7 +316,11 @@ def master_ssh(cmd: str, timeout_seconds: int = 60, print_output: bool = True, c
 
 
 def agent_ssh(
-    agent_host: str, cmd: str, timeout_seconds: int = 60, print_output: bool = True, check: bool = False
+    agent_host: str,
+    cmd: str,
+    timeout_seconds: int = 60,
+    print_output: bool = True,
+    check: bool = False,
 ) -> Tuple[int, str, str]:
     """
     Runs the provided command on the specified agent host, using ssh.
@@ -332,7 +331,11 @@ def agent_ssh(
 
 
 def master_scp(
-    file_content: str, remote_path: str, timeout_seconds: int = 60, print_output: bool = True, check: bool = False
+    file_content: str,
+    remote_path: str,
+    timeout_seconds: int = 60,
+    print_output: bool = True,
+    check: bool = False,
 ) -> int:
     """
     Writes the provided input path to the specified path on cluster leader, using scp.
@@ -360,7 +363,9 @@ def agent_scp(
     return _scp(file_content, remote_path, agent_host, timeout_seconds, print_output, check)
 
 
-def _ssh(cmd: str, host: str, timeout_seconds: int, print_output: bool, check: bool) -> Tuple[int, str, str]:
+def _ssh(
+    cmd: str, host: str, timeout_seconds: int, print_output: bool, check: bool
+) -> Tuple[int, str, str]:
     common_args = " ".join(
         [
             # -oBatchMode=yes: Don't prompt for password if keyfile doesn't work.
@@ -388,11 +393,7 @@ def _ssh(cmd: str, host: str, timeout_seconds: int, print_output: bool, check: b
         ]
     )
 
-    nested_args = " ".join(
-        [
-            common_args
-        ]
-    )
+    nested_args = " ".join([common_args])
 
     if os.environ.get("DCOS_SSH_DIRECT", ""):
         # Direct SSH access to the node:
@@ -429,7 +430,7 @@ def _scp(
             # -i <identity_file>: The identity file to use for login
             "-i {}".format("/root/.ssh/id_rsa"),
             # verbose
-            "-vvv"
+            "-vvv",
         ]
     )
 

testing/sdk_security.py

@@ -394,6 +394,25 @@ def openssl_ciphers() -> Set[str]:
     )
 
 
+def grant_marathon_root_user() -> None:
+    # This grants dcos_marathon to launch tasks as the root user.
+    log.info("Granting root permissions to dcos_marathon")
+    permissions = [
+        {
+            "user": "dcos_marathon",
+            "acl": "dcos:mesos:master:task:user:root",
+            "description": "Service dcos_marathon may register with the Mesos master with user root",
+            "action": "create",
+        }
+    ]
+
+    for permission in permissions:
+        _grant(
+            permission["user"], permission["acl"], permission["description"], permission["action"]
+        )
+    log.info("Permission setup completed for dcos_marathon")
+
+
 def is_cipher_enabled(
     service_name: str, task_name: str, cipher: str, endpoint: str, openssl_timeout: str = "1"
 ) -> bool:

tools/build_package.sh

@@ -1,45 +1,53 @@
 #!/usr/bin/env bash
+# Optional envvars:
+#   REPO_ROOT_DIR: path to root of source repository (default: parent directory of this file)
+#   REPO_NAME: name of the source repository (default: directory name of REPO_ROOT_DIR)
+#   UNIVERSE_DIR: path to universe packaging (default: </absolute/framework/path>/universe/)
+
+
+# Script exits if some command returns non-zero value
+set -e
 
-set -e -x
 
 user_usage() {
     # This script is generally called by an upstream 'build.sh' which would be invoked directly by users.
     # This function returns the syntax expected to be used by that upstream 'build.sh'
     echo "Syntax: build.sh [-h|--help] [aws|local|.dcos]"
 }
 
+
 dev_usage() {
     # Called when a syntax error appears to be an error on the part of the developer.
-    echo "Developer syntax: build_package.sh <framework-name> </abs/path/to/framework> [-a 'path1' -a 'path2' ...] [aws|local|.dcos]"
+    echo "Developer syntax: build_package.sh <framework-name> </abs/path/to/framework> [-v] [-a 'path1' -a 'path2' ...] [aws|local|.dcos]"
 }
 
-# Optional envvars:
-#   REPO_ROOT_DIR: path to root of source repository (default: parent directory of this file)
-#   REPO_NAME: name of the source repository (default: directory name of REPO_ROOT_DIR)
-#   UNIVERSE_DIR: path to universe packaging (default: </absolute/framework/path>/universe/)
 
 if [ $# -lt 3 ]; then
     dev_usage
     exit 1
 fi
 
+
 # required args:
 FRAMEWORK_NAME=$1
 shift
 FRAMEWORK_DIR=$1
 shift
 
-echo "Building $FRAMEWORK_NAME in $FRAMEWORK_DIR:"
+
+echo "Building $FRAMEWORK_NAME package in $FRAMEWORK_DIR:"
 
 # optional args, currently just used for providing paths to service artifacts:
 custom_artifacts=
-while getopts 'a:' opt; do
+while getopts 'va:' opt; do
     case $opt in
-        a)
-            custom_artifacts="$custom_artifacts $OPTARG"
-            ;;
-        \?)
-            dev_usage
+        v) echo "Verbose mode enabled"
+           VERBOSE=true
+           set -x
+           ;;
+        a) custom_artifacts="$custom_artifacts $OPTARG"
+           ;;
+        \?) dev_usage
             exit 1
             ;;
     esac
@@ -53,6 +61,10 @@ case $1 in
         publish_method="aws"
         shift
         ;;
+    azure)
+        publish_method="azure"
+        shift
+        ;;
     local)
         publish_method="local"
         shift
@@ -77,7 +89,10 @@ export REPO_NAME=${REPO_NAME:=$(basename $REPO_ROOT_DIR)} # default to name of R
 
 UNIVERSE_DIR=${UNIVERSE_DIR:=${FRAMEWORK_DIR}/universe} # default to 'universe' directory in framework dir
 echo "- Universe:  $UNIVERSE_DIR"
-echo "- Artifacts:$custom_artifacts"
+echo "- Artifacts:"
+for cus_art in $custom_artifacts; do
+  echo "  - $cus_art"
+done
 echo "- Publish:   $publish_method"
 echo "---"
 
@@ -96,6 +111,10 @@ case "$publish_method" in
         echo "Uploading to S3"
         PUBLISH_SCRIPT=${TOOLS_DIR}/publish_aws.py
         ;;
+    azure)
+        echo "Uploading to Azure blob storage"
+        PUBLISH_SCRIPT=${TOOLS_DIR}/publish_azure.py
+        ;;
     .dcos)
         echo "Uploading .dcos files to S3"
         PUBLISH_SCRIPT=${TOOLS_DIR}/publish_dcos_file.py
@@ -106,13 +125,23 @@ case "$publish_method" in
         echo "Use one of the following additional arguments to get something that runs on a cluster:"
         echo "- 'local': Host the build in a local HTTP server."
         echo "- 'aws':   Upload the build to S3."
+        echo "- 'azure':   Upload the build to blob storage."
         echo "- '.dcos': Upload the build as a .dcos file to S3."
         ;;
 esac
 
 PACKAGE_VERSION=${1:-"stub-universe"}
 
+
+# Launch Publisher script if is defined
 if [ -n "$PUBLISH_SCRIPT" ]; then
     # All the scripts use the same argument format:
-    $PUBLISH_SCRIPT "${FRAMEWORK_NAME}" "${PACKAGE_VERSION}" "${UNIVERSE_DIR}" ${custom_artifacts}
+    publisher_log_file="/tmp/$(basename ${PUBLISH_SCRIPT}).log"
+    echo "Logs of publisher script in $publisher_log_file"
+    if [ $VERBOSE ]; then
+        $PUBLISH_SCRIPT "${FRAMEWORK_NAME}" "${PACKAGE_VERSION}" "${UNIVERSE_DIR}" ${custom_artifacts} | tee $publisher_log_file
+    else 
+        $PUBLISH_SCRIPT "${FRAMEWORK_NAME}" "${PACKAGE_VERSION}" "${UNIVERSE_DIR}" ${custom_artifacts} &> $publisher_log_file
+    fi
 fi
+echo "Package Building Successful"