Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[JENKINS-60434] Control pausing pipeline builds during quieting down period #932

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
Open

[JENKINS-60434] Control pausing pipeline builds during quieting down period #932

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
3b5ac36
adding logic to control pausing and eviction of running buidls during…
TueDissingWork Sep 6, 2024
44aecd6
adding logic to adhere to config
TueDissingWork Sep 10, 2024
dc6f304
adding tests
TueDissingWork Sep 10, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
97 changes: 65 additions & 32 deletions plugin/src/main/java/org/jenkinsci/plugins/workflow/cps/CpsThreadGroup.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,43 +24,13 @@

package org.jenkinsci.plugins.workflow.cps;

import com.cloudbees.groovy.cps.Continuable;
import com.cloudbees.groovy.cps.Outcome;
import com.google.common.util.concurrent.Futures;
import com.thoughtworks.xstream.XStream;
import com.thoughtworks.xstream.converters.Converter;
import com.thoughtworks.xstream.converters.MarshallingContext;
import com.thoughtworks.xstream.converters.UnmarshallingContext;
import com.thoughtworks.xstream.io.HierarchicalStreamReader;
import com.thoughtworks.xstream.io.HierarchicalStreamWriter;
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
import groovy.lang.Closure;
import groovy.lang.GroovyShell;
import groovy.lang.Script;
import hudson.ExtensionList;
import hudson.Functions;
import hudson.Main;
import hudson.Util;
import hudson.model.Result;
import hudson.util.XStream2;
import jenkins.model.Jenkins;
import jenkins.util.Timer;
import org.jenkinsci.plugins.workflow.actions.ErrorAction;
import org.jenkinsci.plugins.workflow.cps.persistence.PersistIn;
import org.jenkinsci.plugins.workflow.cps.persistence.PersistenceContext;
import org.jenkinsci.plugins.workflow.graph.FlowNode;
import org.jenkinsci.plugins.workflow.steps.FlowInterruptedException;
import org.jenkinsci.plugins.workflow.support.pickles.serialization.RiverWriter;

import edu.umd.cs.findbugs.annotations.NonNull;
import java.io.File;
import java.io.IOException;
import java.io.Serializable;
import java.nio.file.Files;
import java.nio.file.StandardCopyOption;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
Expand All @@ -76,13 +46,46 @@
import java.util.logging.Level;
import java.util.logging.Logger;

import edu.umd.cs.findbugs.annotations.CheckForNull;
import org.jenkinsci.plugins.workflow.actions.ErrorAction;
import org.jenkinsci.plugins.workflow.cps.config.CPSConfiguration;
import org.jenkinsci.plugins.workflow.cps.persistence.PersistIn;
import org.jenkinsci.plugins.workflow.cps.persistence.PersistenceContext;
import org.jenkinsci.plugins.workflow.graph.FlowNode;
import org.jenkinsci.plugins.workflow.pickles.Pickle;
import org.jenkinsci.plugins.workflow.pickles.PickleFactory;
import org.jenkinsci.plugins.workflow.steps.FlowInterruptedException;
import org.jenkinsci.plugins.workflow.support.concurrent.WithThreadName;
import org.jenkinsci.plugins.workflow.support.pickles.SingleTypedPickleFactory;
import org.jenkinsci.plugins.workflow.support.pickles.serialization.RiverWriter;
import org.jenkinsci.plugins.workflow.support.storage.FlowNodeStorage;

import com.cloudbees.groovy.cps.Continuable;
import com.cloudbees.groovy.cps.Outcome;
import com.google.common.util.concurrent.Futures;
import com.thoughtworks.xstream.XStream;
import com.thoughtworks.xstream.converters.Converter;
import com.thoughtworks.xstream.converters.MarshallingContext;
import com.thoughtworks.xstream.converters.UnmarshallingContext;
import com.thoughtworks.xstream.io.HierarchicalStreamReader;
import com.thoughtworks.xstream.io.HierarchicalStreamWriter;

import edu.umd.cs.findbugs.annotations.CheckForNull;
import edu.umd.cs.findbugs.annotations.NonNull;
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
import groovy.lang.Closure;
import groovy.lang.GroovyShell;
import groovy.lang.Script;
import hudson.ExtensionList;
import hudson.Functions;
import hudson.Main;
import hudson.Util;
import hudson.model.Result;
import hudson.util.XStream2;
import jenkins.model.CauseOfInterruption;
import jenkins.model.Jenkins;
import jenkins.model.CauseOfInterruption.UserInterruption;
import jenkins.util.Timer;

/**
* List of {@link CpsThread}s that form a single {@link CpsFlowExecution}.
*
Expand Down Expand Up @@ -303,10 +306,11 @@ public Void call() throws Exception {
LOGGER.log(Level.WARNING, null, e);
}
}
if (paused.get() || j == null || (execution != null && j.isQuietingDown())) {
if (CPSConfiguration.get().isPipelinesPausingWhenQueitingDown() && (paused.get() || j == null || (execution != null && j.isQuietingDown()))) {
if (j != null && j.isQuietingDown() && execution != null && pausedByQuietMode.compareAndSet(false, true)) {
try {
execution.getOwner().getListener().getLogger().println("Pausing (Preparing for shutdown)");
//runner.awaitTermination(10, TimeUnit.SECONDS);
} catch (IOException e) {
LOGGER.log(Level.WARNING, null, e);
}
Expand All @@ -326,6 +330,35 @@ public void run() {
saveProgramIfPossible(true);
f.complete(null);
return null;
} else {
// Do not pause build during quieting down period

// Should we start a timer?
if (CPSConfiguration.get().isForcefullyStopBuldsAfterTimeout() ) {

// Should likely not start multiple threads
Timer.get().schedule(new Runnable() {
@Override
public void run() {
if (j.isQuietingDown()) {
// still quieting down, let's stop this build
try {
execution.interrupt(Result.ABORTED, new ShutdownInterruption());
} catch (IOException e) {
LOGGER.log(Level.WARNING, null, e);
} catch (InterruptedException interruptException) {
interruptException.printStackTrace();
LOGGER.log(Level.WARNING, null, interruptException);
}
} else {
scheduleRun();
}
}
}, Main.isUnitTest ? 1 : CPSConfiguration.get().getBuildTerminationTimeoutMinutes() * 60,
TimeUnit.SECONDS);

}

}

boolean stillRunnable = run();
Expand Down
12 changes: 12 additions & 0 deletions plugin/src/main/java/org/jenkinsci/plugins/workflow/cps/ShutdownInterruption.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
package org.jenkinsci.plugins.workflow.cps;

import jenkins.model.CauseOfInterruption;

public class ShutdownInterruption extends CauseOfInterruption {

@Override
public String getShortDescription() {
return "Jenkins needs to terminate the execusion of this builds";
}

}
36 changes: 34 additions & 2 deletions plugin/src/main/java/org/jenkinsci/plugins/workflow/cps/config/CPSConfiguration.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,12 +24,13 @@

package org.jenkinsci.plugins.workflow.cps.config;

import org.jenkinsci.Symbol;

import edu.umd.cs.findbugs.annotations.NonNull;
import hudson.Extension;
import hudson.ExtensionList;
import jenkins.model.GlobalConfiguration;
import jenkins.model.GlobalConfigurationCategory;
import org.jenkinsci.Symbol;

@Symbol("cps")
@Extension
Expand All @@ -39,7 +40,10 @@ public class CPSConfiguration extends GlobalConfiguration {
* Whether to show the sandbox checkbox in jobs to users without Jenkins.ADMINISTER
*/
private boolean hideSandbox;

private boolean pipelinesPausingWhenQueitingDown = true;
private boolean forcefullyStopBuldsAfterTimeout = false;
private int buildTerminationTimeoutMinutes;

public CPSConfiguration() {
load();
}
Expand All @@ -52,6 +56,33 @@ public void setHideSandbox(boolean hideSandbox) {
this.hideSandbox = hideSandbox;
save();
}

public boolean isPipelinesPausingWhenQueitingDown() {
return pipelinesPausingWhenQueitingDown;
}

public void setPipelinesPausingWhenQueitingDown(boolean enabled) {
this.pipelinesPausingWhenQueitingDown = enabled;
save();
}

public boolean isForcefullyStopBuldsAfterTimeout() {
return forcefullyStopBuldsAfterTimeout;
}

public void setForcefullyStopBuldsAfterTimeout(boolean stop) {
this.forcefullyStopBuldsAfterTimeout = stop;
save();
}

public int getBuildTerminationTimeoutMinutes() {
return buildTerminationTimeoutMinutes;
}

public void setBuildTerminationTimeoutMinutes(int delay) {
this.buildTerminationTimeoutMinutes = delay;
save();
}

@NonNull
@Override
Expand All @@ -62,4 +93,5 @@ public GlobalConfigurationCategory getCategory() {
public static CPSConfiguration get() {
return ExtensionList.lookupSingleton(CPSConfiguration.class);
}

}
15 changes: 14 additions & 1 deletion ...rc/main/resources/org/jenkinsci/plugins/workflow/cps/config/CPSConfiguration/config.jelly
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,10 +24,23 @@ THE SOFTWARE.
-->

<?jelly escape-by-default='true'?>
<j:jelly xmlns:j="jelly:core" xmlns:f="/lib/form">
<j:jelly xmlns:j="jelly:core" xmlns:f="/lib/form" >
<f:section title="${%Pipeline Sandbox}">
<f:entry field="hideSandbox" title="${%Hide Sandbox checkbox in pipeline jobs}">
<f:checkbox/>
</f:entry>
</f:section>

<f:section title="${%Pipeline Pausing}">
<f:entry title="${%Pause pipelines in quiet mode}" description="${%Control pipeline behaviour when Jenkins is entering quiet mode}">
<f:checkbox name="pipelinesPausingWhenQueitingDown" checked="${instance.isPipelinesPausingWhenQueitingDown()}"/>
</f:entry>
<f:entry title="${%Force stopping bulid after timeout}" description="${%Should Jenkins stop any builds still running after the specified timeout ignored when pausing is enabled?}">
<f:checkbox name="forcefullyStopBuldsAfterTimeout" checked="${instance.isForcefullyStopBuldsAfterTimeout()}"/>
</f:entry>
<f:entry title="${%Build termination delay}" description="${%The time in minutes to wait for buidls to finish, before they are forcefully stopped, when Jenkins is in quiet mode.}">
<f:textbox name="buildTerminationTimeoutMinutes" default="${instance.getBuildTerminationTimeoutMinutes()}"/>
</f:entry>
</f:section>

</j:jelly>
32 changes: 32 additions & 0 deletions plugin/src/test/java/org/jenkinsci/plugins/workflow/cps/CpsFlowExecutionTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,7 @@
import org.jenkinsci.plugins.scriptsecurity.sandbox.whitelists.Whitelisted;
import org.jenkinsci.plugins.workflow.cps.CpsFlowExecution.TimingFlowNodeStorage;
import org.jenkinsci.plugins.workflow.cps.GroovySourceFileAllowlist.DefaultAllowlist;
import org.jenkinsci.plugins.workflow.cps.config.CPSConfiguration;
import org.jenkinsci.plugins.workflow.flow.FlowExecution;
import org.jenkinsci.plugins.workflow.flow.FlowExecutionList;
import org.jenkinsci.plugins.workflow.flow.FlowExecutionOwner;
Expand Down Expand Up @@ -440,6 +441,37 @@ private static List<String> stepNames(ListenableFuture<List<StepExecution>> exec
r.assertBuildStatusSuccess(r.waitForCompletion(b));
});
}

@Test public void doNotPauseBuildsDuringQuietDown() throws Throwable {
sessions.then(r -> {
CPSConfiguration.get().setPipelinesPausingWhenQueitingDown(false);
WorkflowJob p = r.createProject(WorkflowJob.class);
p.setDefinition(new CpsFlowDefinition("semaphore 'wait'; echo 'I am done'", true));
WorkflowRun b = p.scheduleBuild2(0).waitForStart();
SemaphoreStep.waitForStart("wait/1", b);
r.jenkins.doQuietDown(true, 0, "Jenkins config change", false);
r.assertLogNotContains("Pausing (Preparing for shutdown)", b);
SemaphoreStep.success("wait/1", null);
r.waitForMessage("I am done", b);
});
}

@Test public void abortRunningBuildsDuringQuietDown() throws Throwable {
sessions.then(r -> {
CPSConfiguration.get().setPipelinesPausingWhenQueitingDown(false);
CPSConfiguration.get().setForcefullyStopBuldsAfterTimeout(true);
CPSConfiguration.get().setBuildTerminationTimeoutMinutes(1); // code overwrite this to 1 second
WorkflowJob p = r.createProject(WorkflowJob.class);
p.setDefinition(new CpsFlowDefinition("semaphore 'wait'; echo 'I am done'", true));
WorkflowRun b = p.scheduleBuild2(0).waitForStart();
SemaphoreStep.waitForStart("wait/1", b);
r.jenkins.doQuietDown(true, 0, "Jenkins config change", false);
r.assertLogNotContains("Pausing (Preparing for shutdown)", b);
SemaphoreStep.success("wait/1", null);
r.waitForMessage("Jenkins needs to terminate the execusion of this builds", b);
});
}

public static final class SlowToResume extends Step {
@DataBoundConstructor public SlowToResume() {}
@Override public StepExecution start(StepContext context) throws Exception {
Expand Down
Loading