Skip to content

Commit

Permalink
Improvement
Browse files Browse the repository at this point in the history
- Modified the API to use label strings from Constants.groovy
- Added setDescription function

Signed-off-by: Shubham Verma <shubhamv.sv@gmail.com>
  • Loading branch information
VermaSh committed Apr 10, 2018
1 parent 3a4834e commit c655d37
Show file tree
Hide file tree
Showing 4 changed files with 267 additions and 12 deletions.
116 changes: 116 additions & 0 deletions Jenkins_jobs/CreateNewNode.groovy
Original file line number Diff line number Diff line change
@@ -0,0 +1,116 @@
@Library('NodeHelper') _
import hudson.slaves.CommandLauncher;
import hudson.plugins.sshslaves.SSHLauncher
import hudson.model.Node.Mode;
import hudson.plugins.sshslaves.verifiers.NonVerifyingKeyVerificationStrategy;
import jenkins.model.Jenkins;
import hudson.model.Computer;
import org.jenkinsci.plugins.scriptsecurity.scripts.ScriptApproval;
import hudson.model.Slave;
import hudson.slaves.JNLPLauncher;

clones = [:]
List<String> newMachines = new ArrayList<String>();

node {
stage('jenkins made me do it') {
def nodeHelper = new NodeHelper();

String[] machines = params.machineNames.split(",")
String[] machineIPs = params.machineIPs.split(",")
String[] labels = params.labelStrings.split(",")

def launcher
String remoteFS
String newMachineLabels
String os
String newMachineName

for (int index = 0; index < machineIPs.length; index++) {

if (nodeHelper.getComputer(machines[index]) == null) {
if (Integer.parseInt(machineIPs[index].split("\\.")[0]) == 10) {
launcher = new CommandLauncher(Constants.SSH_COMMAND + "${machineIPs[index]} " + Constants.WGET_SLAVE_JAR);
remoteFS = Constants.REMOTE_FS;
} else if (machines[index].contains("win")) {
launcher = new JNLPLauncher("", "", new jenkins.slaves.RemotingWorkDirSettings(false, "", "remoting", false));
remoteFS = Constants.WIN_REMOTE_FS;
} else {
launcher = new SSHLauncher(machines[index], 22, Constants.SSH_CREDENTIAL_ID, null, null, null, null, null, null, null, new NonVerifyingKeyVerificationStrategy());
remoteFS = Constants.REMOTE_FS;
}

newMachineLabels = labels[index%labels.length]

newMachineName = nodeHelper.addNewNode(
machines[index],
machineIPs[index],
remoteFS,
1, // Number of executers
Mode.EXCLUSIVE,
newMachineLabels.toLowerCase(),
launcher
);

// This part is to approve the script used to add a 10. machine
def scripts = ScriptApproval.get()
def scriptSet = scripts.getPendingScripts()
def iterator = scriptSet.iterator()
if (launcher.getClass().toString().contains("slaves.CommandLauncher")) {
for (ScriptApproval.PendingScript script : scriptSet) {
// TODO: Search for entire string instead of a part
if (script.script.contains("ssh -C -i /data/jenkins/J9_Build/home/.ssh/j9build.key j9build@" + machineIPs[index])) {
println "Script Approved"
scripts.approveScript(script.getHash());
}
}
(Jenkins.getInstance().getComputer(newMachineName)).connect(false);
}

println "Machine ${newMachineName} was added with following labels ${newMachineLabels}";

// This will be used to get the names of machines that need to have description updated
newMachines.add(newMachineName);
} else {
echo "Machine(${machines[index]}) already exists."
}
}

}
}

/* Iterates over the machine names and updated their descriptin with CPU count,
* total memory and total space
*/
for (String machine : newMachines) {
String machineName = machine
clones[machineName] = {
node (machineName) {
stage('Add_Labels') {
echo "NODE_NAME = ${env.NODE_NAME}"

Slave slave = (Slave)Jenkins.getInstance().getNode(env.NODE_NAME);
NodeHelper nodeHelper = new NodeHelper();

String newMachineLabels = nodeHelper.constructLabels(env.NODE_NAME);
for (String label : slave.getLabelString().split(" ")) {
/* Here we check if the machine one of the above three labels
* So that we can add them back to the machine.
*/
if (label.contains(Constants.IGNORE_LABELS[0])
|| label.contains(Constants.IGNORE_LABELS[1])
|| label.contains(Constants.IGNORE_LABELS[2])) {
newMachineLabels += " " + label;
}
}

newMachineLabels = nodeHelper.addLabel(env.NODE_NAME, newMachineLabels);

println "Machine ${env.NODE_NAME} was added with following labels ${newMachineLabels}";
}
}
}
}


parallel clones
41 changes: 41 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
## Files
### API Functions (NodeHelper.groovy)
The NodeHelper API contains helper functions to query basic machine stats in real time, add new machines, update/overwrite labels.

* Get CPU count ```String getCpuCount(String computerName)```
* Get Installed memory ```Tuple getMemory(String computerName)```
* Get OS information ```Tuple getOsInfo(String computerName)```
Expand All @@ -20,9 +21,49 @@ The NodeHelper API contains helper functions to query basic machine stats in rea

### Space Monitoring (WorkspaceInfo.groovy)
Iterates over online nodes on Jenkins and prints the contents of the workspace directory along with the space they occupy

* The computers it iterates over can be limited by input parameter, ```projectLabel```
* As of now, it only works for linux, aix, and mac

### Update Machine Identifiers (UpdateMachineIdentifiers.groovy)
Used to update machine labels and description

* The computers it iterates over can be limited by input parameter, ```projectLabel```
* The job expects 5 input parameters
* ```boolean initialRun```
* This is used to add the project or any other required labels on the machine
* ```String labels```
* Labels you would like to be added to the machine. Seperate the labels for a single machine with spaces. Use "," if doing batch update. Or if they will be same across the machines supply only one set
* ```String machineNames```
* Can either enter a list of names or a single name. For list seperate them with ","
* ```boolean updateDescription```
* If this is set true, the job will update labels and update description
* ```String projectlabel```
* This limits which machines will be touched
* Use Cases:
* Update labels:
* Objective: add default, os, arch, and kernel lables
* Procedure: initial_run is checked off and only the machine name(s) is supplied
* Append labels:
* Objective: want to add a custom label.
* Procedure: supply labels and machine names
* Update description:
* It adds CPU count, Disk space and installed RAM to the description
* Side note: this is implemented here instead of in CreateNewNode job because sometimes the machine may not have the needed information available in time for the script to udpate the description
* Procedure: have ```updateDescription``` parameter checked

### Create New Node (CreateNewNode.groovy)
Used to create new nodes with any basic labels

* This job expects 3 parameters
* ```String machineNames```
* Host names of the machine(s), seperated by ```,```
* ```String machineIPs```
* ip address of the machine(s), seperated by ```,```
* ```String labelStrings```
* Labels you would like to be added to the machine. Seperate the labels for a single machine with spaces. Use ```,``` if doing batch update


## How-to

### Setup
Expand Down
97 changes: 85 additions & 12 deletions src/NodeHelper.groovy
Original file line number Diff line number Diff line change
Expand Up @@ -228,8 +228,16 @@ class NodeHelper {
def osInfo = getOsInfo(computer.getName());
String osVersion = osInfo.get(1);

ret = "sw.os." + osInfo.get(0) + osVersion;
ret += " sw.os." + osInfo.get(0);
ret = String.format(
Constants.OS_LABEL_FORMAT,
Constants.OS_LABEL_PREFIX,
osInfo.get(0),osVersion);
ret += ' ';
ret += String.format(
Constants.GENERIC_LABEL_FORMAT,
Constants.OS_LABEL_PREFIX,
osInfo.get(0));

}

return ret.toLowerCase();
Expand All @@ -248,7 +256,10 @@ class NodeHelper {

Computer computer = getComputer(computerName);
if (computer != null) {
ret = "hw.endian." + getEndian(computer.getName());
ret = String.format(
Constants.GENERIC_LABEL_FORMAT,
Constants.ENDIAN_LABEL_PREFIX,
getEndian(computer.getName()));
}

return ret.toLowerCase();
Expand All @@ -275,7 +286,10 @@ class NodeHelper {
break;
}

ret = "hw.platform." + ret;
ret = String.format(
Constants.GENERIC_LABEL_FORMAT,
Constants.PLATFORM_LABEL_PREFIX,
ret);
}

return ret.toLowerCase();
Expand Down Expand Up @@ -330,7 +344,10 @@ class NodeHelper {
ret = "INVALID_ARCH";
break;
}
ret = "hw.arch." + ret;
ret = String.format(
Constants.GENERIC_LABEL_FORMAT,
Constants.ARCH_LABEL_PREFIX,
ret);
}


Expand All @@ -343,7 +360,10 @@ class NodeHelper {
Computer computer = getComputer(computerName);
if (computer != null) {
def kernelInfo = getOsKernelInfo(computer.getName());
ret = "sw.os." + kernelInfo.get(0);
ret = String.format(
Constants.GENERIC_LABEL_FORMAT,
Constants.OS_LABEL_PREFIX,
kernelInfo.get(0));
}

return ret;
Expand All @@ -354,7 +374,11 @@ class NodeHelper {

Computer computer = getComputer(computerName);
if (computer != null) {
ret = "hw.hypervisor.";// TODO: finish implementation, get something
// TODO: finish implementation, get something
// ret = String.format(
// Constants.GENERIC_LABEL_FORMAT,
// Constants.HYPERVISOR_LABEL_PREFIX,
// kernelInfo.get(0));
}

return ret.toLowerCase();
Expand Down Expand Up @@ -525,7 +549,11 @@ class NodeHelper {
/* As of now, cases 1-3 should work with
* parseRedHatOsInfoString
*/
osInfo = parseRedHatOsInfoString(cmdResult);
if (cmdResult.contains("Fedora")) {
osInfo = parseFedoraOsInfoString(cmdResult);
} else {
osInfo = parseRedHatOsInfoString(cmdResult);
}
break;
case 4:
osInfo = parseSuseOsInfoString(cmdResult);
Expand Down Expand Up @@ -633,6 +661,29 @@ class NodeHelper {
return new Tuple(retOsName,retOsVersion);
}

private Tuple parseFedoraOsInfoString(String rawValue) {
String retOsName = "parseFedoraOsInfoString:INVALID_INPUT";
String retOsVersion = "parseFedoraOsInfoString:INVALID_INPUT";

/* Sample raw values
* Fedora release 24 (twenty-four)
*/

if (rawValue.length() > 0) {
rawValue = rawValue.trim();

String[] rawValueSplit = rawValue.split(" ");

retOsName = rawValueSplit[0];
if (rawValueSplit[1].equals("release")) {
retOsVersion = rawValueSplit[2];
}

}

return new Tuple(retOsName,retOsVersion);
}

private Tuple parseOsInfoString(String rawValue) {
String retOsName = "parseOsInfoString:INVALID_INPUT";
String retOsVersion = "parseOsInfoString:INVALID_INPUT";
Expand Down Expand Up @@ -807,6 +858,26 @@ class NodeHelper {
return ret;
}

/**
* Sets the machine description from jenkins
*
* @param compterName computer whose location is needed
* @param description the new updated description
*
* @return machine description as string
*/
public String setDescription(String computerName, String description) {
String ret = "setDescription:COMPUTER_NOT_FOUND";

Computer computer = getComputer(computerName);
if (computer != null) {
computer.getNode().setNodeDescription(description);
ret = getDescription(computerName);
}

return ret;
}

/**
* Gets cpu count via exec on the computer passed
* in.
Expand Down Expand Up @@ -1095,7 +1166,7 @@ class NodeHelper {
*/
humanReadable = humanReadable/1000;

return String.format("%d%sB", (Math.rint(humanReadable)).intValue(), pre);
return String.format("%d %sB", (Math.rint(humanReadable)).intValue(), pre);
}

/**
Expand All @@ -1115,9 +1186,11 @@ class NodeHelper {
* the 2
*/
ret = Jenkins.getInstance().getComputer(computerName);
if (ret == null) {
// tries to search for computer without the domain
ret = Jenkins.getInstance().getComputer(computerName.substring(0,computerName.indexOf(".")));
if (ret == null) { // tries to search for computer without the domain
int endIndex = computerName.indexOf(".");
if ( endIndex > 0) {
ret = Jenkins.getInstance().getComputer(computerName.substring(0,endIndex));
}
}
}

Expand Down
25 changes: 25 additions & 0 deletions vars/Constants.groovy
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
class Constants {
static final String[] IGNORE_LABELS = ['.ignore'];
static final String REMOTE_FS = "/home/jenkins";
static final String WIN_REMOTE_FS = "C:\\Users\\jenkins";

// This the key that'll be used for SSHLauncher in CreateNewNode
static final String SSH_CREDENTIAL_ID = "";

static final String SLAVE_JAR_LOCATION = "<jenkins_URL/>/jnlpJars/slave.jar";
static final String WGET_SLAVE_JAR = "\"wget -q --no-check-certificate -O slave.jar ${SLAVE_JAR_LOCATION} ; java -jar slave.jar\"";
static final String SSH_COMMAND = "ssh -C -i ${SSH_KEY_LOCATION} <userName/>@";
static final String SSH_KEY_LOCATION = "";

static final String JENKINS_URL = "https://ci.adoptopenjdk.net/";

static final String OS_LABEL_FORMAT = "%s.%s.%s";
static final String GENERIC_LABEL_FORMAT = "%s.%s";
static final String OS_LABEL_PREFIX = "sw.os";
static final String ENDIAN_LABEL_PREFIX = "hw.endian";
static final String PLATFORM_LABEL_PREFIX = "hw.platform";
static final String ARCH_LABEL_PREFIX = "hw.arch";
static final String KERNEL_LABEL_PREFIX = "hw.kernel";
static final String HYPERVISOR_LABEL_PREFIX = "hw.hypervisor";
}

0 comments on commit c655d37

Please sign in to comment.