Add parameter to specify ES plugins

With this commit we add a new command line parameter
--elasticsearch-plugins to Night Rally. It will pass this parameter to
Rally but also check whether we've specified "x-pack:security" as a
plugin and will make the necessary adjustments (change expected cluster
health and adjust the client options).

Relates elastic#21

night_rally.py

@@ -145,8 +145,9 @@ def __init__(self, revision, effective_start_date, target_host, root_dir, config
 
 
 class ReleaseCommand(BaseCommand):
-    def __init__(self, effective_start_date, target_host, root_dir, distribution_version, configuration_name, tag):
+    def __init__(self, effective_start_date, target_host, plugins, root_dir, distribution_version, configuration_name, tag):
         super().__init__(effective_start_date, target_host, root_dir)
+        self.plugins = plugins
         self.configuration_name = configuration_name
         self.pipeline = "from-distribution"
         self.distribution_version = distribution_version
@@ -167,6 +168,12 @@ def command_line(self, track, challenge, car):
               "--user-tag=\"{9}\"".format(self.distribution_version, self.ts, track, challenge, car,
                                           self.report_path(track, challenge, car), self.pipeline,
                                           self.target_host, self.configuration_name, self.tag(), RALLY_BINARY)
+        if self.plugins:
+            cmd += " --elasticsearch-plugins=\"%s\"" % self.plugins
+            if "x-pack:security" in self.plugins:
+                cmd += " --cluster-health=yellow " \
+                       "--client-options=\"use_ssl:true,verify_certs:false,basic_auth_user:'rally',basic_auth_password:'rally-password'\""
+
         return cmd
 
     def tag(self):
@@ -352,6 +359,10 @@ def parse_args():
         "--target-host",
         help="The Elasticsearch node that should be targeted",
         required=True)
+    parser.add_argument(
+        "--elasticsearch-plugins",
+        help="Elasticsearch plugins to install for the benchmark (default: None)",
+        default=None)
     parser.add_argument(
         "--fixtures",
         help="A comma-separated list of fixtures that have been run",
@@ -391,8 +402,15 @@ def main():
     adhoc_mode = args.mode == "adhoc"
     nightly_mode = args.mode == "nightly"
 
+    plugins = args.elasticsearch_plugins
+
     tag = args.tag
-    release_tag = "env:ear" if "encryption-at-rest" in args.fixtures else "env:bare"
+    if "encryption-at-rest" in args.fixtures:
+        release_tag = "env:ear"
+    elif "x-pack:security" in plugins:
+        release_tag = "env:x-pack"
+    else:
+        release_tag = "env:bare"
     docker_benchmark = args.release.startswith("Docker ")
     release = args.release.replace("Docker ", "")
 
@@ -403,20 +421,26 @@ def main():
         # use always the same name for release comparison benchmarks
         env_name = sanitize(args.mode)
         if docker_benchmark:
+            if plugins:
+                raise RuntimeError("User specified plugins [%s] but this is not supported for Docker benchmarks." % plugins)
             logger.info("Running Docker release benchmarks for release [%s] against [%s]." % (release, args.target_host))
             command = DockerCommand(args.effective_start_date, args.target_host, root_dir, release, env_name)
             tag = command.tag()
         else:
             logger.info("Running release benchmarks for release [%s] against [%s] (release tag is [%s])."
                         % (release, args.target_host, release_tag))
-            command = ReleaseCommand(args.effective_start_date, args.target_host, root_dir, release, env_name, release_tag)
+            command = ReleaseCommand(args.effective_start_date, args.target_host, plugins, root_dir, release, env_name, release_tag)
             tag = command.tag()
     elif adhoc_mode:
+        if plugins:
+            raise RuntimeError("User specified plugins [%s] but this is not supported for adhoc benchmarks." % plugins)
         logger.info("Running adhoc benchmarks for revision [%s] against [%s]." % (args.revision, args.target_host))
         # copy data from templates directory to our dedicated output directory
         env_name = sanitize(args.release)
         command = AdHocCommand(args.revision, args.effective_start_date, args.target_host, root_dir, env_name, args.tag, args.override_src_dir)
     else:
+        if plugins:
+            raise RuntimeError("User specified plugins [%s] but this is not supported for nightly benchmarks." % plugins)
         logger.info("Running nightly benchmarks against [%s]." % args.target_host)
         env_name = NightlyCommand.CONFIG_NAME
         command = NightlyCommand(args.effective_start_date, args.target_host, root_dir, args.override_src_dir)

night_rally.sh

@@ -43,6 +43,7 @@ RELEASE="master"
 REVISION="latest"
 TARGET_HOST="localhost:9200"
 TAG=""
+PLUGINS=""
 
 
 for i in "$@"
@@ -88,6 +89,10 @@ case ${i} in
     TARGET_HOST="${i#*=}"
     shift # past argument=value
     ;;
+    --elasticsearch-plugins=*)
+    PLUGINS="${i#*=}"
+    shift # past argument=value
+    ;;
     *)
     echo "unknown command line option passed to night_rally"
     exit 1
@@ -164,7 +169,7 @@ fi
 #****************************
 set +e
 # Avoid failing before we transferred all results. Usually only a single benchmark trial run fails but lots of other succeed.
-python3 ${NIGHT_RALLY_HOME}/night_rally.py --target-host=${TARGET_HOST} --effective-start-date="${START_DATE}" ${NIGHT_RALLY_OVERRIDE} --mode=${MODE} ${NIGHT_RALLY_DRY_RUN} --fixtures="${FIXTURES}" --revision="${REVISION}" --release="${RELEASE}" --tag="${TAG}"
+python3 ${NIGHT_RALLY_HOME}/night_rally.py --target-host=${TARGET_HOST} --elasticsearch-plugins="${PLUGINS}" --effective-start-date="${START_DATE}" ${NIGHT_RALLY_OVERRIDE} --mode=${MODE} ${NIGHT_RALLY_DRY_RUN} --fixtures="${FIXTURES}" --revision="${REVISION}" --release="${RELEASE}" --tag="${TAG}"
 exit_code=$?
 
 echo "Killing any lingering Rally processes"

tests.py

@@ -162,7 +162,7 @@ def test_run_adhoc_benchmark(self):
             system_call.calls
         )
 
-    def test_run_release_benchmark(self):
+    def test_run_release_benchmark_without_plugins(self):
         system_call = RecordingSystemCall(return_value=False)
 
         tracks = [
@@ -181,7 +181,7 @@ def test_run_release_benchmark(self):
             }
         ]
         start_date = datetime.datetime(2016, 1, 1)
-        cmd = night_rally.ReleaseCommand(start_date, "localhost", "/rally_root", "5.3.0", "release", tag="env:bare")
+        cmd = night_rally.ReleaseCommand(start_date, "localhost", None, "/rally_root", "5.3.0", "release", tag="env:bare")
         night_rally.run_rally(tracks, cmd, system=system_call)
         self.assertEqual(2, len(system_call.calls))
         self.assertEqual(
@@ -202,6 +202,50 @@ def test_run_release_benchmark(self):
             system_call.calls
         )
 
+    def test_run_release_benchmark_with_plugins(self):
+        system_call = RecordingSystemCall(return_value=False)
+
+        tracks = [
+            {
+                "track": "geonames",
+                "combinations": [
+                    {
+                        "challenge": "append-no-conflicts",
+                        "car": "defaults"
+                    },
+                    {
+                        "challenge": "append-no-conflicts",
+                        "car": "4gheap"
+                    }
+                ]
+            }
+        ]
+        start_date = datetime.datetime(2016, 1, 1)
+        cmd = night_rally.ReleaseCommand(start_date, "localhost", "x-pack:security,monitoring", "/rally_root", "5.3.0", "release",
+                                         tag="env:x-pack")
+        night_rally.run_rally(tracks, cmd, system=system_call)
+        self.assertEqual(2, len(system_call.calls))
+        self.assertEqual(
+            [
+                "rally --skip-update --configuration-name=release --target-host=localhost --pipeline=from-distribution --quiet "
+                "--distribution-version=5.3.0 --effective-start-date \"2016-01-01 00:00:00\" --track=geonames "
+                "--challenge=append-no-conflicts --car=defaults --report-format=csv "
+                "--report-file=/rally_root/reports/rally/2016-01-01-00-00-00/geonames/append-no-conflicts/defaults/report.csv "
+                "--user-tag=\"env:x-pack\" --elasticsearch-plugins=\"x-pack:security,monitoring\" --cluster-health=yellow "
+                "--client-options=\"use_ssl:true,verify_certs:false,basic_auth_user:'rally',basic_auth_password:'rally-password'\"",
+
+                "rally --skip-update --configuration-name=release --target-host=localhost --pipeline=from-distribution --quiet "
+                "--distribution-version=5.3.0 --effective-start-date \"2016-01-01 00:00:00\" --track=geonames "
+                "--challenge=append-no-conflicts --car=4gheap --report-format=csv "
+                "--report-file=/rally_root/reports/rally/2016-01-01-00-00-00/geonames/append-no-conflicts/4gheap/report.csv "
+                "--user-tag=\"env:x-pack\" --elasticsearch-plugins=\"x-pack:security,monitoring\" --cluster-health=yellow "
+                "--client-options=\"use_ssl:true,verify_certs:false,basic_auth_user:'rally',basic_auth_password:'rally-password'\"",
+            ]
+            ,
+            system_call.calls
+        )
+
+
     def test_run_docker_benchmark(self):
         system_call = RecordingSystemCall(return_value=False)