diff --git a/src/main/java/hudson/plugins/ec2/EC2AbstractSlave.java b/src/main/java/hudson/plugins/ec2/EC2AbstractSlave.java index c2173cc70..ce72ea86c 100644 --- a/src/main/java/hudson/plugins/ec2/EC2AbstractSlave.java +++ b/src/main/java/hudson/plugins/ec2/EC2AbstractSlave.java @@ -44,6 +44,7 @@ import java.util.HashSet; import java.util.LinkedList; import java.util.List; +import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.logging.Level; import java.util.logging.Logger; @@ -757,7 +758,7 @@ public static Instance getInstance(String instanceId, EC2Cloud cloud) { /** * Terminates the instance in EC2. */ - public abstract void terminate(); + public abstract Future terminate(); void stop() { try { diff --git a/src/main/java/hudson/plugins/ec2/EC2OndemandSlave.java b/src/main/java/hudson/plugins/ec2/EC2OndemandSlave.java index cd605b1ec..1d8b51b57 100644 --- a/src/main/java/hudson/plugins/ec2/EC2OndemandSlave.java +++ b/src/main/java/hudson/plugins/ec2/EC2OndemandSlave.java @@ -12,6 +12,8 @@ import java.io.IOException; import java.util.Collections; import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Future; import java.util.logging.Level; import java.util.logging.Logger; import jenkins.model.Jenkins; @@ -503,11 +505,11 @@ public EC2OndemandSlave(String instanceId) throws FormException, IOException { * Terminates the instance in EC2. */ @Override - public void terminate() { + public Future terminate() { if (terminateScheduled.getCount() == 0) { synchronized (terminateScheduled) { if (terminateScheduled.getCount() == 0) { - Computer.threadPoolForRemoting.submit(() -> { + Future f = Computer.threadPoolForRemoting.submit(() -> { try { if (!isAlive(true)) { /* @@ -533,9 +535,11 @@ public void terminate() { } }); terminateScheduled.reset(); + return f; } } } + return CompletableFuture.completedFuture(null); } @Override diff --git a/src/main/java/hudson/plugins/ec2/EC2SpotSlave.java b/src/main/java/hudson/plugins/ec2/EC2SpotSlave.java index 787e80c86..7256dfe57 100644 --- a/src/main/java/hudson/plugins/ec2/EC2SpotSlave.java +++ b/src/main/java/hudson/plugins/ec2/EC2SpotSlave.java @@ -11,6 +11,8 @@ import java.io.IOException; import java.util.Collections; import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Future; import java.util.logging.Level; import java.util.logging.Logger; import jenkins.model.Jenkins; @@ -182,11 +184,11 @@ protected boolean isAlive(boolean force) { * Cancel the spot request for the instance. Terminate the instance if it is up. Remove the agent from Jenkins. */ @Override - public void terminate() { + public Future terminate() { if (terminateScheduled.getCount() == 0) { synchronized (terminateScheduled) { if (terminateScheduled.getCount() == 0) { - Computer.threadPoolForRemoting.submit(() -> { + Future f = Computer.threadPoolForRemoting.submit(() -> { try { // Cancel the spot request Ec2Client ec2 = getCloud().connect(); @@ -246,9 +248,11 @@ public void terminate() { } }); terminateScheduled.reset(); + return f; } } } + return CompletableFuture.completedFuture(null); } /** diff --git a/src/main/java/hudson/plugins/ec2/SlaveTemplate.java b/src/main/java/hudson/plugins/ec2/SlaveTemplate.java index 51891379b..08e76779e 100644 --- a/src/main/java/hudson/plugins/ec2/SlaveTemplate.java +++ b/src/main/java/hudson/plugins/ec2/SlaveTemplate.java @@ -186,6 +186,8 @@ public class SlaveTemplate implements Describable { public final String idleTerminationMinutes; + private boolean terminateIdleDuringShutdown; + public final String iamInstanceProfile; public final boolean deleteRootOnTermination; @@ -1674,6 +1676,15 @@ public String getidleTerminationMinutes() { return idleTerminationMinutes; } + public boolean getTerminateIdleDuringShutdown() { + return terminateIdleDuringShutdown; + } + + @DataBoundSetter + public void setTerminateIdleDuringShutdown(boolean terminateIdleDuringShutdown) { + this.terminateIdleDuringShutdown = terminateIdleDuringShutdown; + } + public Set getLabelSet() { if (labelSet == null) { labelSet = Label.parse(labels); diff --git a/src/main/java/hudson/plugins/ec2/util/MinimumInstanceChecker.java b/src/main/java/hudson/plugins/ec2/util/MinimumInstanceChecker.java index ff3ec2167..21c116978 100644 --- a/src/main/java/hudson/plugins/ec2/util/MinimumInstanceChecker.java +++ b/src/main/java/hudson/plugins/ec2/util/MinimumInstanceChecker.java @@ -2,17 +2,24 @@ import edu.umd.cs.findbugs.annotations.NonNull; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import hudson.init.Terminator; import hudson.model.Computer; import hudson.model.Label; import hudson.model.Queue; +import hudson.plugins.ec2.EC2AbstractSlave; import hudson.plugins.ec2.EC2Cloud; import hudson.plugins.ec2.EC2Computer; import hudson.plugins.ec2.SlaveTemplate; import java.time.Clock; import java.time.LocalDateTime; import java.time.LocalTime; +import java.util.ArrayList; import java.util.Arrays; +import java.util.List; import java.util.Objects; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.logging.Logger; import java.util.stream.Stream; import jenkins.model.Jenkins; import org.kohsuke.accmod.Restricted; @@ -21,14 +28,17 @@ @Restricted(NoExternalUse.class) public class MinimumInstanceChecker { + private static final Logger LOGGER = Logger.getLogger(MinimumInstanceChecker.class.getName()); + @SuppressFBWarnings(value = "MS_SHOULD_BE_FINAL", justification = "Needs to be overridden from tests") public static Clock clock = Clock.systemDefaultZone(); - private static Stream agentsForTemplate(@NonNull SlaveTemplate agentTemplate) { + private static Stream agentsForTemplate(@NonNull SlaveTemplate agentTemplate) { return Arrays.stream(Jenkins.get().getComputers()) .filter(EC2Computer.class::isInstance) + .map(EC2Computer.class::cast) .filter(computer -> { - SlaveTemplate computerTemplate = ((EC2Computer) computer).getSlaveTemplate(); + SlaveTemplate computerTemplate = computer.getSlaveTemplate(); return computerTemplate != null && Objects.equals(computerTemplate.description, agentTemplate.description); }); @@ -38,16 +48,16 @@ public static int countCurrentNumberOfAgents(@NonNull SlaveTemplate agentTemplat return (int) agentsForTemplate(agentTemplate).count(); } + private static Stream idleAgents(@NonNull SlaveTemplate agentTemplate) { + return agentsForTemplate(agentTemplate).filter(Computer::isIdle); + } + public static int countCurrentNumberOfSpareAgents(@NonNull SlaveTemplate agentTemplate) { - return (int) agentsForTemplate(agentTemplate) - .filter(computer -> computer.countBusy() == 0) - .filter(Computer::isOnline) - .count(); + return (int) idleAgents(agentTemplate).filter(Computer::isOnline).count(); } public static int countCurrentNumberOfProvisioningAgents(@NonNull SlaveTemplate agentTemplate) { - return (int) agentsForTemplate(agentTemplate) - .filter(computer -> computer.countBusy() == 0) + return (int) idleAgents(agentTemplate) .filter(Computer::isOffline) .filter(Computer::isConnecting) .count(); @@ -142,4 +152,26 @@ public static boolean minimumInstancesActive( } return false; } + + @Terminator + public static void discardIdleInstances() throws Exception { + LOGGER.fine("Looking for idle instances to discard"); + List> futures = new ArrayList<>(); + Jenkins.get().clouds.stream() + .filter(EC2Cloud.class::isInstance) + .map(EC2Cloud.class::cast) + .forEach(cloud -> cloud.getTemplates().stream() + .filter(SlaveTemplate::getTerminateIdleDuringShutdown) + .forEach(agentTemplate -> idleAgents(agentTemplate).forEach(computer -> { + EC2AbstractSlave agent = computer.getNode(); + if (agent != null) { + LOGGER.info(() -> "discarding idle instance " + agent.getInstanceId()); + futures.add(agent.terminate()); + } + }))); + // Must wait; otherwise task could run too late during shutdown, leading to NoClassDefFoundError. + for (Future future : futures) { + future.get(5, TimeUnit.SECONDS); + } + } } diff --git a/src/main/resources/hudson/plugins/ec2/SlaveTemplate/config.jelly b/src/main/resources/hudson/plugins/ec2/SlaveTemplate/config.jelly index d8fdfe429..80f2658ab 100644 --- a/src/main/resources/hudson/plugins/ec2/SlaveTemplate/config.jelly +++ b/src/main/resources/hudson/plugins/ec2/SlaveTemplate/config.jelly @@ -106,6 +106,10 @@ THE SOFTWARE. + + + + diff --git a/src/main/resources/hudson/plugins/ec2/SlaveTemplate/help-terminateIdleDuringShutdown.html b/src/main/resources/hudson/plugins/ec2/SlaveTemplate/help-terminateIdleDuringShutdown.html new file mode 100644 index 000000000..5043e55ed --- /dev/null +++ b/src/main/resources/hudson/plugins/ec2/SlaveTemplate/help-terminateIdleDuringShutdown.html @@ -0,0 +1,5 @@ +
+ Delete any idle agents and terminate their instances when the controller stops. + This is especially useful in conjunction with the Minimum number of instances or Minimum number of spare instances options, + when setting Idle termination time to zero. +
diff --git a/src/test/java/hudson/plugins/ec2/EC2AbstractSlaveTest.java b/src/test/java/hudson/plugins/ec2/EC2AbstractSlaveTest.java index 1ad00f217..2ddf79d50 100644 --- a/src/test/java/hudson/plugins/ec2/EC2AbstractSlaveTest.java +++ b/src/test/java/hudson/plugins/ec2/EC2AbstractSlaveTest.java @@ -5,6 +5,8 @@ import hudson.model.Node; import java.util.ArrayList; import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Future; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.jvnet.hudson.test.JenkinsRule; @@ -58,9 +60,8 @@ void testGetLaunchTimeoutInMillisShouldNotOverflow() throws Exception { EC2AbstractSlave.DEFAULT_ENCLAVE_ENABLED) { @Override - public void terminate() { - // To change body of implemented methods use File | Settings | - // File Templates. + public Future terminate() { + return CompletableFuture.completedFuture(null); } @Override @@ -159,7 +160,9 @@ void testMaxUsesBackwardCompat() throws Exception { EC2AbstractSlave.DEFAULT_METADATA_SUPPORTED, EC2AbstractSlave.DEFAULT_ENCLAVE_ENABLED) { @Override - public void terminate() {} + public Future terminate() { + return CompletableFuture.completedFuture(null); + } @Override public String getEc2Type() { diff --git a/src/test/java/hudson/plugins/ec2/EC2RetentionStrategyTest.java b/src/test/java/hudson/plugins/ec2/EC2RetentionStrategyTest.java index b2bb01647..ae93103d9 100644 --- a/src/test/java/hudson/plugins/ec2/EC2RetentionStrategyTest.java +++ b/src/test/java/hudson/plugins/ec2/EC2RetentionStrategyTest.java @@ -34,6 +34,8 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.logging.Level; @@ -251,7 +253,9 @@ private EC2Computer computerWithIdleTime( EC2AbstractSlave.DEFAULT_METADATA_SUPPORTED, EC2AbstractSlave.DEFAULT_ENCLAVE_ENABLED) { @Override - public void terminate() {} + public Future terminate() { + return CompletableFuture.completedFuture(null); + } @Override public String getEc2Type() { @@ -396,7 +400,9 @@ private EC2Computer computerWithUpTime( EC2AbstractSlave.DEFAULT_METADATA_SUPPORTED, EC2AbstractSlave.DEFAULT_ENCLAVE_ENABLED) { @Override - public void terminate() {} + public Future terminate() { + return CompletableFuture.completedFuture(null); + } @Override public String getEc2Type() { @@ -562,8 +568,9 @@ private EC2Computer computerWithUsageLimit(final int usageLimit) throws Exceptio EC2AbstractSlave.DEFAULT_METADATA_SUPPORTED, EC2AbstractSlave.DEFAULT_ENCLAVE_ENABLED) { @Override - public void terminate() { + public Future terminate() { terminateCalled.set(true); + return CompletableFuture.completedFuture(null); } @Override diff --git a/src/test/java/hudson/plugins/ec2/MockEC2Computer.java b/src/test/java/hudson/plugins/ec2/MockEC2Computer.java index 502d7bd2c..fdc00745b 100644 --- a/src/test/java/hudson/plugins/ec2/MockEC2Computer.java +++ b/src/test/java/hudson/plugins/ec2/MockEC2Computer.java @@ -3,6 +3,8 @@ import hudson.model.Node; import java.util.ArrayList; import java.util.Collections; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Future; import software.amazon.awssdk.core.exception.SdkException; import software.amazon.awssdk.services.ec2.model.InstanceType; @@ -55,7 +57,9 @@ public static MockEC2Computer createComputer(String suffix) throws Exception { EC2AbstractSlave.DEFAULT_METADATA_SUPPORTED, EC2AbstractSlave.DEFAULT_ENCLAVE_ENABLED) { @Override - public void terminate() {} + public Future terminate() { + return CompletableFuture.completedFuture(null); + } @Override public String getEc2Type() { diff --git a/src/test/java/hudson/plugins/ec2/ssh/InitScriptExecutionTest.java b/src/test/java/hudson/plugins/ec2/ssh/InitScriptExecutionTest.java index 8ae1a54c8..91d842b33 100644 --- a/src/test/java/hudson/plugins/ec2/ssh/InitScriptExecutionTest.java +++ b/src/test/java/hudson/plugins/ec2/ssh/InitScriptExecutionTest.java @@ -16,6 +16,8 @@ import java.io.PrintStream; import java.io.PrintWriter; import java.util.ArrayList; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.logging.Level; import org.apache.sshd.client.SshClient; @@ -223,7 +225,9 @@ private EC2AbstractSlave getMockNodeTemplate(String initScript) throws Exception EC2AbstractSlave.DEFAULT_METADATA_HOPS_LIMIT, EC2AbstractSlave.DEFAULT_METADATA_SUPPORTED) { @Override - public void terminate() {} + public Future terminate() { + return CompletableFuture.completedFuture(null); + } @Override public String getEc2Type() {