Skip to content

Commit

Permalink
Sense for VirtualBox and $HOME when deciding to turn on vagrant testi…
Browse files Browse the repository at this point in the history
…ng. (elastic#24636)

We're using Vagrant in more places now than before. This commit includes a plugin that verifies
the Vagrant and Virtualbox installations for projects that depend on them. This shared code
should fix up the errors we've seen from CI builds relating to the new Kerberos fixture.
  • Loading branch information
jbaiera authored Jun 2, 2017
1 parent e024c67 commit 4ed0abe
Show file tree
Hide file tree
Showing 5 changed files with 153 additions and 82 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,127 @@
package org.elasticsearch.gradle.vagrant

import org.gradle.api.GradleException
import org.gradle.api.InvalidUserDataException
import org.gradle.api.Plugin
import org.gradle.api.Project
import org.gradle.process.ExecResult
import org.gradle.process.internal.ExecException

/**
* Global configuration for if Vagrant tasks are supported in this
* build environment.
*/
class VagrantSupportPlugin implements Plugin<Project> {

@Override
void apply(Project project) {
if (project.rootProject.ext.has('vagrantEnvChecksDone') == false) {
Map vagrantInstallation = getVagrantInstallation(project)
Map virtualBoxInstallation = getVirtualBoxInstallation(project)

project.rootProject.ext.vagrantInstallation = vagrantInstallation
project.rootProject.ext.virtualBoxInstallation = virtualBoxInstallation
project.rootProject.ext.vagrantSupported = vagrantInstallation.supported && virtualBoxInstallation.supported
project.rootProject.ext.vagrantEnvChecksDone = true

// Finding that HOME needs to be set when performing vagrant updates
String homeLocation = System.getenv("HOME")
if (project.rootProject.ext.vagrantSupported && homeLocation == null) {
throw new GradleException("Could not locate \$HOME environment variable. Vagrant is enabled " +
"and requires \$HOME to be set to function properly.")
}
}

addVerifyInstallationTasks(project)
}

private Map getVagrantInstallation(Project project) {
try {
ByteArrayOutputStream pipe = new ByteArrayOutputStream()
ExecResult runResult = project.exec {
commandLine 'vagrant', '--version'
standardOutput pipe
ignoreExitValue true
}
String version = pipe.toString().trim()
if (runResult.exitValue == 0) {
if (version ==~ /Vagrant 1\.(8\.[6-9]|9\.[0-9])+/) {
return [ 'supported' : true ]
} else {
return [ 'supported' : false,
'info' : "Illegal version of vagrant [${version}]. Need [Vagrant 1.8.6+]" ]
}
} else {
return [ 'supported' : false,
'info' : "Could not read installed vagrant version:\n" + version ]
}
} catch (ExecException e) {
// Exec still throws this if it cannot find the command, regardless if ignoreExitValue is set.
// Swallow error. Vagrant isn't installed. Don't halt the build here.
return [ 'supported' : false, 'info' : "Could not find vagrant: " + e.message ]
}
}

private Map getVirtualBoxInstallation(Project project) {
try {
ByteArrayOutputStream pipe = new ByteArrayOutputStream()
ExecResult runResult = project.exec {
commandLine 'vboxmanage', '--version'
standardOutput = pipe
ignoreExitValue true
}
String version = pipe.toString().trim()
if (runResult.exitValue == 0) {
try {
String[] versions = version.split('\\.')
int major = Integer.parseInt(versions[0])
int minor = Integer.parseInt(versions[1])
if ((major < 5) || (major == 5 && minor < 1)) {
return [ 'supported' : false,
'info' : "Illegal version of virtualbox [${version}]. Need [5.1+]" ]
} else {
return [ 'supported' : true ]
}
} catch (NumberFormatException | ArrayIndexOutOfBoundsException e) {
return [ 'supported' : false,
'info' : "Unable to parse version of virtualbox [${version}]. Required [5.1+]" ]
}
} else {
return [ 'supported': false, 'info': "Could not read installed virtualbox version:\n" + version ]
}
} catch (ExecException e) {
// Exec still throws this if it cannot find the command, regardless if ignoreExitValue is set.
// Swallow error. VirtualBox isn't installed. Don't halt the build here.
return [ 'supported' : false, 'info' : "Could not find virtualbox: " + e.message ]
}
}

private void addVerifyInstallationTasks(Project project) {
createCheckVagrantVersionTask(project)
createCheckVirtualBoxVersionTask(project)
}

private void createCheckVagrantVersionTask(Project project) {
project.tasks.create('vagrantCheckVersion') {
description 'Check the Vagrant version'
group 'Verification'
doLast {
if (project.rootProject.vagrantInstallation.supported == false) {
throw new InvalidUserDataException(project.rootProject.vagrantInstallation.info)
}
}
}
}

private void createCheckVirtualBoxVersionTask(Project project) {
project.tasks.create('virtualboxCheckVersion') {
description 'Check the Virtualbox version'
group 'Verification'
doLast {
if (project.rootProject.virtualBoxInstallation.supported == false) {
throw new InvalidUserDataException(project.rootProject.virtualBoxInstallation.info)
}
}
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -227,43 +227,6 @@ class VagrantTestPlugin implements Plugin<Project> {
vagrantSetUpTask.dependsOn copyBatsTests, copyBatsUtils, copyBatsArchives, createVersionFile, createUpgradeFromFile
}

private static void createCheckVagrantVersionTask(Project project) {
project.tasks.create('vagrantCheckVersion', Exec) {
description 'Check the Vagrant version'
group 'Verification'
commandLine 'vagrant', '--version'
standardOutput = new ByteArrayOutputStream()
doLast {
String version = standardOutput.toString().trim()
if ((version ==~ /Vagrant 1\.(8\.[6-9]|9\.[0-9])+/) == false) {
throw new InvalidUserDataException("Illegal version of vagrant [${version}]. Need [Vagrant 1.8.6+]")
}
}
}
}

private static void createCheckVirtualBoxVersionTask(Project project) {
project.tasks.create('virtualboxCheckVersion', Exec) {
description 'Check the Virtualbox version'
group 'Verification'
commandLine 'vboxmanage', '--version'
standardOutput = new ByteArrayOutputStream()
doLast {
String version = standardOutput.toString().trim()
try {
String[] versions = version.split('\\.')
int major = Integer.parseInt(versions[0])
int minor = Integer.parseInt(versions[1])
if ((major < 5) || (major == 5 && minor < 1)) {
throw new InvalidUserDataException("Illegal version of virtualbox [${version}]. Need [5.1+]")
}
} catch (NumberFormatException | ArrayIndexOutOfBoundsException e) {
throw new InvalidUserDataException("Unable to parse version of virtualbox [${version}]. Required [5.1+]", e)
}
}
}
}

private static void createPackagingTestTask(Project project) {
project.tasks.create('packagingTest') {
group 'Verification'
Expand Down Expand Up @@ -291,8 +254,6 @@ class VagrantTestPlugin implements Plugin<Project> {
createCleanTask(project)
createStopTask(project)
createSmokeTestTask(project)
createCheckVagrantVersionTask(project)
createCheckVirtualBoxVersionTask(project)
createPrepareVagrantTestEnvTask(project)
createPackagingTestTask(project)
createPlatformTestTask(project)
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
implementation-class=org.elasticsearch.gradle.vagrant.VagrantSupportPlugin
67 changes: 24 additions & 43 deletions plugins/repository-hdfs/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,8 @@ esplugin {
classname 'org.elasticsearch.repositories.hdfs.HdfsPlugin'
}

apply plugin: 'elasticsearch.vagrantsupport'

versions << [
'hadoop2': '2.7.1'
]
Expand Down Expand Up @@ -83,6 +85,7 @@ task krb5kdcUpdate(type: org.elasticsearch.gradle.vagrant.VagrantCommandTask) {
subcommand 'update'
boxName box
environmentVars vagrantEnvVars
dependsOn "vagrantCheckVersion", "virtualboxCheckVersion"
}

task krb5kdcFixture(type: org.elasticsearch.gradle.test.VagrantFixture) {
Expand Down Expand Up @@ -125,7 +128,7 @@ task secureHdfsFixture(type: org.elasticsearch.gradle.test.AntFixture) {
"${keytabPath}"
}

boolean fixtureSupported = false;
boolean fixtureSupported = false
if (Os.isFamily(Os.FAMILY_WINDOWS)) {
// hdfs fixture will not start without hadoop native libraries on windows
String nativePath = System.getenv("HADOOP_HOME")
Expand Down Expand Up @@ -153,60 +156,38 @@ if (fixtureSupported) {
integTestRunner.systemProperty 'tests.rest.suite', 'hdfs_repository/10_basic'
}

boolean secureFixtureSupported = false;
boolean secureFixtureSupported = false
if (fixtureSupported) {
// Only do secure fixture support if the regular fixture is supported,
// and if vagrant is installed. The ignoreExitValue on exec only matters
// in cases where the command can be found and successfully started. In
// situations where the vagrant command isn't able to be started at all
// (it's not installed) then Gradle still throws ExecException.
ByteArrayOutputStream pipe = new ByteArrayOutputStream()
try {
ExecResult runResult = exec {
commandLine 'vagrant', '--version'
standardOutput pipe
ignoreExitValue true
}
String output = pipe.toString().trim()
if (runResult.exitValue == 0) {
secureFixtureSupported = (output ==~ /Vagrant 1\.(8\.[6-9]|9\.[0-9])+/)
} else {
logger.warn("Could not read installed vagrant version:\n" + output)
}
} catch (org.gradle.process.internal.ExecException e) {
logger.warn("Could not find vagrant: " + e.message)
// Swallow error. Vagrant isn't installed. Leave secure fixture support off.
}
secureFixtureSupported = project.rootProject.vagrantSupported
}

// Create a Integration Test suite just for security based tests
if (secureFixtureSupported && false) { // This fails due to a vagrant configuration issue - remove the false check to re-enable
// This must execute before the afterEvaluate block from integTestSecure
project.afterEvaluate {
Path elasticsearchKT = project(':test:fixtures:krb5kdc-fixture').buildDir.toPath().resolve("keytabs").resolve("elasticsearch.keytab").toAbsolutePath()
Path krb5conf = project(':test:fixtures:krb5kdc-fixture').buildDir.toPath().resolve("conf").resolve("krb5.conf").toAbsolutePath()

project.integTestSecureCluster.dependsOn(project.bundlePlugin)
project.integTestSecure.clusterConfig.plugin(project.path)
project.integTestSecure.clusterConfig.extraConfigFile("repository-hdfs/krb5.keytab", "${elasticsearchKT}")
project.integTestSecure.clusterConfig.jvmArgs = "-Xms" + System.getProperty('tests.heap.size', '512m') +
" " + "-Xmx" + System.getProperty('tests.heap.size', '512m') +
" " + "-Djava.security.krb5.conf=${krb5conf}" +
" " + System.getProperty('tests.jvm.argline', '')
}
// This must execute before the afterEvaluate block from integTestSecure
project.afterEvaluate {
Path elasticsearchKT = project(':test:fixtures:krb5kdc-fixture').buildDir.toPath().resolve("keytabs").resolve("elasticsearch.keytab").toAbsolutePath()
Path krb5conf = project(':test:fixtures:krb5kdc-fixture').buildDir.toPath().resolve("conf").resolve("krb5.conf").toAbsolutePath()

project.integTestSecureCluster.dependsOn(project.bundlePlugin)
project.integTestSecure.clusterConfig.plugin(project.path)
project.integTestSecure.clusterConfig.extraConfigFile("repository-hdfs/krb5.keytab", "${elasticsearchKT}")
project.integTestSecure.clusterConfig.jvmArgs = "-Xms" + System.getProperty('tests.heap.size', '512m') +
" " + "-Xmx" + System.getProperty('tests.heap.size', '512m') +
" " + "-Djava.security.krb5.conf=${krb5conf}" +
" " + System.getProperty('tests.jvm.argline', '')
}

RestIntegTestTask integTestSecure = project.tasks.create('integTestSecure', RestIntegTestTask.class) {
description = "Runs rest tests against an elasticsearch cluster with HDFS secured by MIT Kerberos."
}
RestIntegTestTask integTestSecure = project.tasks.create('integTestSecure', RestIntegTestTask.class) {
description = "Runs rest tests against an elasticsearch cluster with HDFS secured by MIT Kerberos."
}

integTestSecure.mustRunAfter(project.integTest)
if (secureFixtureSupported) {
project.check.dependsOn(integTestSecure)

// Fixture dependencies
integTestSecureCluster.dependsOn secureHdfsFixture, krb5kdcFixture
integTestSecureRunner.systemProperty 'tests.rest.suite', 'secure_hdfs_repository'
} else {
logger.warn("secured hdfsFixture is unsupported, please install Vagrant 1.8.6+ to enable")
integTestSecure.enabled = false
}

thirdPartyAudit.excludes = [
Expand Down
1 change: 1 addition & 0 deletions qa/vagrant/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
* under the License.
*/

apply plugin: 'elasticsearch.vagrantsupport'
apply plugin: 'elasticsearch.vagrant'

List<String> plugins = []
Expand Down

0 comments on commit 4ed0abe

Please sign in to comment.