Skip to content

Discard idle instances during shutdown #1125

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
fcojfernandez merged 3 commits into from
Aug 26, 2025
Merged

Discard idle instances during shutdown #1125

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
7cf2d09
Discard spare instances during shutdown
jglick Jul 30, 2025
e4ac68f
Introduce `terminateSpareInstances` option https://github.com/jenkins…
jglick Aug 1, 2025
f41ca8a
`terminateSpareInstances` → `terminateIdleDuringShutdown`
jglick Aug 4, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion src/main/java/hudson/plugins/ec2/EC2AbstractSlave.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.logging.Level;
import java.util.logging.Logger;
Expand Down Expand Up @@ -757,7 +758,7 @@ public static Instance getInstance(String instanceId, EC2Cloud cloud) {
/**
* Terminates the instance in EC2.
*/
public abstract void terminate();
public abstract Future<?> terminate();
Copy link
Copy Markdown
Member

@fcojfernandez fcojfernandez Aug 26, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I've checked potential external impacts and it seems that only https://github.com/jenkinsci/ec2-cloud-axis-plugin/blob/4d1a8263cfe04320b311fe26ae613d1805073aa2/src/main/java/hudson/plugins/ec2/Utils.java#L14 would be using this method. Seems safe


void stop() {
try {
Expand Down
8 changes: 6 additions & 2 deletions src/main/java/hudson/plugins/ec2/EC2OndemandSlave.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@
import java.io.IOException;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.Future;
import java.util.logging.Level;
import java.util.logging.Logger;
import jenkins.model.Jenkins;
Expand Down Expand Up @@ -503,11 +505,11 @@ public EC2OndemandSlave(String instanceId) throws FormException, IOException {
* Terminates the instance in EC2.
*/
@Override
public void terminate() {
public Future<?> terminate() {
if (terminateScheduled.getCount() == 0) {
synchronized (terminateScheduled) {
if (terminateScheduled.getCount() == 0) {
Computer.threadPoolForRemoting.submit(() -> {
Future<?> f = Computer.threadPoolForRemoting.submit(() -> {
try {
if (!isAlive(true)) {
/*
Expand All @@ -533,9 +535,11 @@ public void terminate() {
}
});
terminateScheduled.reset();
return f;
}
}
}
return CompletableFuture.completedFuture(null);
}

@Override
Expand Down
8 changes: 6 additions & 2 deletions src/main/java/hudson/plugins/ec2/EC2SpotSlave.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@
import java.io.IOException;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.Future;
import java.util.logging.Level;
import java.util.logging.Logger;
import jenkins.model.Jenkins;
Expand Down Expand Up @@ -182,11 +184,11 @@ protected boolean isAlive(boolean force) {
* Cancel the spot request for the instance. Terminate the instance if it is up. Remove the agent from Jenkins.
*/
@Override
public void terminate() {
public Future<?> terminate() {
if (terminateScheduled.getCount() == 0) {
synchronized (terminateScheduled) {
if (terminateScheduled.getCount() == 0) {
Computer.threadPoolForRemoting.submit(() -> {
Future<?> f = Computer.threadPoolForRemoting.submit(() -> {
try {
// Cancel the spot request
Ec2Client ec2 = getCloud().connect();
Expand Down Expand Up @@ -246,9 +248,11 @@ public void terminate() {
}
});
terminateScheduled.reset();
return f;
}
}
}
return CompletableFuture.completedFuture(null);
}

/**
Expand Down
11 changes: 11 additions & 0 deletions src/main/java/hudson/plugins/ec2/SlaveTemplate.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -186,6 +186,8 @@ public class SlaveTemplate implements Describable<SlaveTemplate> {

public final String idleTerminationMinutes;

private boolean terminateIdleDuringShutdown;

public final String iamInstanceProfile;

public final boolean deleteRootOnTermination;
Expand Down Expand Up @@ -1674,6 +1676,15 @@ public String getidleTerminationMinutes() {
return idleTerminationMinutes;
}

public boolean getTerminateIdleDuringShutdown() {
return terminateIdleDuringShutdown;
}

@DataBoundSetter
public void setTerminateIdleDuringShutdown(boolean terminateIdleDuringShutdown) {
this.terminateIdleDuringShutdown = terminateIdleDuringShutdown;
}

public Set<LabelAtom> getLabelSet() {
if (labelSet == null) {
labelSet = Label.parse(labels);
Expand Down
48 changes: 40 additions & 8 deletions src/main/java/hudson/plugins/ec2/util/MinimumInstanceChecker.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,17 +2,24 @@

import edu.umd.cs.findbugs.annotations.NonNull;
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
import hudson.init.Terminator;
import hudson.model.Computer;
import hudson.model.Label;
import hudson.model.Queue;
import hudson.plugins.ec2.EC2AbstractSlave;
import hudson.plugins.ec2.EC2Cloud;
import hudson.plugins.ec2.EC2Computer;
import hudson.plugins.ec2.SlaveTemplate;
import java.time.Clock;
import java.time.LocalDateTime;
import java.time.LocalTime;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Objects;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.logging.Logger;
import java.util.stream.Stream;
import jenkins.model.Jenkins;
import org.kohsuke.accmod.Restricted;
Expand All @@ -21,14 +28,17 @@
@Restricted(NoExternalUse.class)
public class MinimumInstanceChecker {

private static final Logger LOGGER = Logger.getLogger(MinimumInstanceChecker.class.getName());

@SuppressFBWarnings(value = "MS_SHOULD_BE_FINAL", justification = "Needs to be overridden from tests")
public static Clock clock = Clock.systemDefaultZone();

private static Stream<Computer> agentsForTemplate(@NonNull SlaveTemplate agentTemplate) {
private static Stream<EC2Computer> agentsForTemplate(@NonNull SlaveTemplate agentTemplate) {
return Arrays.stream(Jenkins.get().getComputers())
.filter(EC2Computer.class::isInstance)
.map(EC2Computer.class::cast)
.filter(computer -> {
SlaveTemplate computerTemplate = ((EC2Computer) computer).getSlaveTemplate();
SlaveTemplate computerTemplate = computer.getSlaveTemplate();
return computerTemplate != null
&& Objects.equals(computerTemplate.description, agentTemplate.description);
});
Expand All @@ -38,16 +48,16 @@ public static int countCurrentNumberOfAgents(@NonNull SlaveTemplate agentTemplat
return (int) agentsForTemplate(agentTemplate).count();
}

private static Stream<EC2Computer> idleAgents(@NonNull SlaveTemplate agentTemplate) {
return agentsForTemplate(agentTemplate).filter(Computer::isIdle);
}

public static int countCurrentNumberOfSpareAgents(@NonNull SlaveTemplate agentTemplate) {
return (int) agentsForTemplate(agentTemplate)
.filter(computer -> computer.countBusy() == 0)
.filter(Computer::isOnline)
.count();
return (int) idleAgents(agentTemplate).filter(Computer::isOnline).count();
}

public static int countCurrentNumberOfProvisioningAgents(@NonNull SlaveTemplate agentTemplate) {
return (int) agentsForTemplate(agentTemplate)
.filter(computer -> computer.countBusy() == 0)
return (int) idleAgents(agentTemplate)
.filter(Computer::isOffline)
.filter(Computer::isConnecting)
.count();
Expand Down Expand Up @@ -142,4 +152,26 @@ public static boolean minimumInstancesActive(
}
return false;
}

@Terminator
public static void discardIdleInstances() throws Exception {
LOGGER.fine("Looking for idle instances to discard");
List<Future<?>> futures = new ArrayList<>();
Jenkins.get().clouds.stream()
.filter(EC2Cloud.class::isInstance)
.map(EC2Cloud.class::cast)
.forEach(cloud -> cloud.getTemplates().stream()
.filter(SlaveTemplate::getTerminateIdleDuringShutdown)
.forEach(agentTemplate -> idleAgents(agentTemplate).forEach(computer -> {
EC2AbstractSlave agent = computer.getNode();
if (agent != null) {
LOGGER.info(() -> "discarding idle instance " + agent.getInstanceId());
futures.add(agent.terminate());
}
})));
// Must wait; otherwise task could run too late during shutdown, leading to NoClassDefFoundError.
for (Future<?> future : futures) {
future.get(5, TimeUnit.SECONDS);
}
}
}
4 changes: 4 additions & 0 deletions src/main/resources/hudson/plugins/ec2/SlaveTemplate/config.jelly
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,10 @@ THE SOFTWARE.
<f:textbox default="30" />
</f:entry>

<f:entry title="${%Terminate idle agents during shutdown}" field="terminateIdleDuringShutdown">
<f:checkbox />
</f:entry>

<f:entry title="${%Init script}" field="initScript">
<f:textarea />
</f:entry>
Expand Down
5 changes: 5 additions & 0 deletions src/main/resources/hudson/plugins/ec2/SlaveTemplate/help-terminateIdleDuringShutdown.html
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
<div>
Delete any idle agents and terminate their instances when the controller stops.
This is especially useful in conjunction with the <b>Minimum number of instances</b> or <b>Minimum number of spare instances</b> options,
when setting <b>Idle termination time</b> to zero.
</div>
11 changes: 7 additions & 4 deletions src/test/java/hudson/plugins/ec2/EC2AbstractSlaveTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@
import hudson.model.Node;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.Future;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.jvnet.hudson.test.JenkinsRule;
Expand Down Expand Up @@ -58,9 +60,8 @@ void testGetLaunchTimeoutInMillisShouldNotOverflow() throws Exception {
EC2AbstractSlave.DEFAULT_ENCLAVE_ENABLED) {

@Override
public void terminate() {
// To change body of implemented methods use File | Settings |
// File Templates.
public Future<?> terminate() {
return CompletableFuture.completedFuture(null);
}

@Override
Expand Down Expand Up @@ -159,7 +160,9 @@ void testMaxUsesBackwardCompat() throws Exception {
EC2AbstractSlave.DEFAULT_METADATA_SUPPORTED,
EC2AbstractSlave.DEFAULT_ENCLAVE_ENABLED) {
@Override
public void terminate() {}
public Future<?> terminate() {
return CompletableFuture.completedFuture(null);
}

@Override
public String getEc2Type() {
Expand Down
13 changes: 10 additions & 3 deletions src/test/java/hudson/plugins/ec2/EC2RetentionStrategyTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,8 @@
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.logging.Level;
Expand Down Expand Up @@ -251,7 +253,9 @@ private EC2Computer computerWithIdleTime(
EC2AbstractSlave.DEFAULT_METADATA_SUPPORTED,
EC2AbstractSlave.DEFAULT_ENCLAVE_ENABLED) {
@Override
public void terminate() {}
public Future<?> terminate() {
return CompletableFuture.completedFuture(null);
}

@Override
public String getEc2Type() {
Expand Down Expand Up @@ -396,7 +400,9 @@ private EC2Computer computerWithUpTime(
EC2AbstractSlave.DEFAULT_METADATA_SUPPORTED,
EC2AbstractSlave.DEFAULT_ENCLAVE_ENABLED) {
@Override
public void terminate() {}
public Future<?> terminate() {
return CompletableFuture.completedFuture(null);
}

@Override
public String getEc2Type() {
Expand Down Expand Up @@ -562,8 +568,9 @@ private EC2Computer computerWithUsageLimit(final int usageLimit) throws Exceptio
EC2AbstractSlave.DEFAULT_METADATA_SUPPORTED,
EC2AbstractSlave.DEFAULT_ENCLAVE_ENABLED) {
@Override
public void terminate() {
public Future<?> terminate() {
terminateCalled.set(true);
return CompletableFuture.completedFuture(null);
}

@Override
Expand Down
6 changes: 5 additions & 1 deletion src/test/java/hudson/plugins/ec2/MockEC2Computer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@
import hudson.model.Node;
import java.util.ArrayList;
import java.util.Collections;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.Future;
import software.amazon.awssdk.core.exception.SdkException;
import software.amazon.awssdk.services.ec2.model.InstanceType;

Expand Down Expand Up @@ -55,7 +57,9 @@ public static MockEC2Computer createComputer(String suffix) throws Exception {
EC2AbstractSlave.DEFAULT_METADATA_SUPPORTED,
EC2AbstractSlave.DEFAULT_ENCLAVE_ENABLED) {
@Override
public void terminate() {}
public Future<?> terminate() {
return CompletableFuture.completedFuture(null);
}

@Override
public String getEc2Type() {
Expand Down
6 changes: 5 additions & 1 deletion src/test/java/hudson/plugins/ec2/ssh/InitScriptExecutionTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@
import java.io.PrintStream;
import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.logging.Level;
import org.apache.sshd.client.SshClient;
Expand Down Expand Up @@ -223,7 +225,9 @@ private EC2AbstractSlave getMockNodeTemplate(String initScript) throws Exception
EC2AbstractSlave.DEFAULT_METADATA_HOPS_LIMIT,
EC2AbstractSlave.DEFAULT_METADATA_SUPPORTED) {
@Override
public void terminate() {}
public Future<?> terminate() {
return CompletableFuture.completedFuture(null);
}

@Override
public String getEc2Type() {
Expand Down