From 414521e30a8d18710b26ccc3009be9f29ba26785 Mon Sep 17 00:00:00 2001 From: Pierson Yieh Date: Tue, 14 Sep 2021 16:36:05 -0700 Subject: [PATCH 1/3] Add filters for Security Group and Subnet ID when counting existing EC2 instances --- src/main/java/hudson/plugins/ec2/EC2Cloud.java | 12 ++++++++++++ src/main/java/hudson/plugins/ec2/SlaveTemplate.java | 11 ++++++----- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/src/main/java/hudson/plugins/ec2/EC2Cloud.java b/src/main/java/hudson/plugins/ec2/EC2Cloud.java index ee181abd7..2bac2d90b 100644 --- a/src/main/java/hudson/plugins/ec2/EC2Cloud.java +++ b/src/main/java/hudson/plugins/ec2/EC2Cloud.java @@ -646,6 +646,18 @@ private List getGenericFilters(String jenkinsServerUrl, SlaveTemplate te } } } + // Add Security group filters + if (template.getSecurityGroupSet() != null && template.getSecurityGroupSet().size() > 0) { + for (String sg: template.getSecurityGroupSet()) { + filters.add(new Filter("group-name").withValues(sg)); + } + } + // Add Subnet filters + if (template.getSubnetId() != null && !template.getSubnetId().isEmpty()) { + for (String s: template.getSubnetId().split(SlaveTemplate.EC2_RESOURCE_ID_DELIMETERS)) { + filters.add(new Filter("subnet-id").withValues(s)); + } + } } return filters; } diff --git a/src/main/java/hudson/plugins/ec2/SlaveTemplate.java b/src/main/java/hudson/plugins/ec2/SlaveTemplate.java index eca05077a..349434573 100644 --- a/src/main/java/hudson/plugins/ec2/SlaveTemplate.java +++ b/src/main/java/hudson/plugins/ec2/SlaveTemplate.java @@ -17,10 +17,6 @@ * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ package hudson.plugins.ec2; -import static hudson.plugins.ec2.EC2AbstractSlave.DEFAULT_METADATA_ENDPOINT_ENABLED; -import static hudson.plugins.ec2.EC2AbstractSlave.DEFAULT_METADATA_TOKENS_REQUIRED; -import static hudson.plugins.ec2.EC2AbstractSlave.DEFAULT_METADATA_HOPS_LIMIT; - import com.amazonaws.AmazonClientException; import com.amazonaws.AmazonServiceException; import com.amazonaws.auth.AWSCredentialsProvider; @@ -133,6 +129,10 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +import static hudson.plugins.ec2.EC2AbstractSlave.DEFAULT_METADATA_ENDPOINT_ENABLED; +import static hudson.plugins.ec2.EC2AbstractSlave.DEFAULT_METADATA_HOPS_LIMIT; +import static hudson.plugins.ec2.EC2AbstractSlave.DEFAULT_METADATA_TOKENS_REQUIRED; + /** * Template of {@link EC2AbstractSlave} to launch. * @@ -141,7 +141,7 @@ public class SlaveTemplate implements Describable { private static final Logger LOGGER = Logger.getLogger(SlaveTemplate.class.getName()); - private static final String EC2_RESOURCE_ID_DELIMETERS = "[\\s,;]+"; + static final String EC2_RESOURCE_ID_DELIMETERS = "[\\s,;]+"; public String ami; @@ -254,6 +254,7 @@ public class SlaveTemplate implements Describable { @CheckForNull private List amiFilters; + /* * Necessary to handle reading from old configurations. The UnixData object is created in readResolve() */ From adaa91760cdce5e7c5bfb2316eadb7f6766be4ef Mon Sep 17 00:00:00 2001 From: Pierson Yieh Date: Tue, 14 Sep 2021 17:25:29 -0700 Subject: [PATCH 2/3] Add IAM instance profile to filter --- src/main/java/hudson/plugins/ec2/EC2Cloud.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/main/java/hudson/plugins/ec2/EC2Cloud.java b/src/main/java/hudson/plugins/ec2/EC2Cloud.java index 2bac2d90b..fcc6c0ea3 100644 --- a/src/main/java/hudson/plugins/ec2/EC2Cloud.java +++ b/src/main/java/hudson/plugins/ec2/EC2Cloud.java @@ -658,6 +658,12 @@ private List getGenericFilters(String jenkinsServerUrl, SlaveTemplate te filters.add(new Filter("subnet-id").withValues(s)); } } + + // Add IAM Instance profile filters + if (template.getIamInstanceProfile() != null && !template.getIamInstanceProfile().isEmpty()) { + filters.add(new Filter("iam-instance-profile.arn").withValues(template.getIamInstanceProfile())); + } + } return filters; } From 2c6259c2cbef1e03067e3b2abdb6e7ceacc1667a Mon Sep 17 00:00:00 2001 From: Pierson Yieh Date: Thu, 16 Sep 2021 17:49:33 -0700 Subject: [PATCH 3/3] Method to get VPC based on slavetemplate subnet to add as filter when counting --- .../java/hudson/plugins/ec2/EC2Cloud.java | 30 ++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/src/main/java/hudson/plugins/ec2/EC2Cloud.java b/src/main/java/hudson/plugins/ec2/EC2Cloud.java index fcc6c0ea3..6e36787c4 100644 --- a/src/main/java/hudson/plugins/ec2/EC2Cloud.java +++ b/src/main/java/hudson/plugins/ec2/EC2Cloud.java @@ -471,7 +471,7 @@ public HttpResponse doProvision(@QueryParameter String template) throws ServletE * * @param template If left null, then all instances are counted. */ - private int countCurrentEC2Slaves(SlaveTemplate template) throws AmazonClientException { + private int countCurrentEC2Slaves(SlaveTemplate template, List addonFilters) throws AmazonClientException { String jenkinsServerUrl = JenkinsLocationConfiguration.get().getUrl(); if (jenkinsServerUrl == null) { @@ -487,6 +487,8 @@ private int countCurrentEC2Slaves(SlaveTemplate template) throws AmazonClientExc String description = template != null ? template.description : null; List filters = getGenericFilters(jenkinsServerUrl, template); + if (addonFilters != null) + filters.addAll(addonFilters); filters.add(new Filter("instance-state-name").withValues("running", "pending", "stopping")); DescribeInstancesRequest dir = new DescribeInstancesRequest().withFilters(filters); DescribeInstancesResult result = null; @@ -510,6 +512,10 @@ private int countCurrentEC2Slaves(SlaveTemplate template) throws AmazonClientExc return n; } + int countCurrentEC2Slaves(SlaveTemplate template) throws AmazonClientException { + return countCurrentEC2Slaves(template, null); + } + /** * Counts the number of EC2 Spot instances that can be used with the specified image and a template. Also removes any * nodes associated with canceled requests. @@ -692,6 +698,11 @@ private boolean isEc2ProvisionedAmiSlave(List tags, String description) { * Returns the maximum number of possible agents that can be created. */ private int getPossibleNewSlavesCount(SlaveTemplate template) throws AmazonClientException { + List vpcFilters = new ArrayList<>(); + for (String vpc: getTemplateVpcs(template)) { + vpcFilters.add(new Filter("vpc-id").withValues(vpc)); + } + int estimatedTotalSlaves = countCurrentEC2Slaves(null); int estimatedAmiSlaves = countCurrentEC2Slaves(template); @@ -703,6 +714,23 @@ private int getPossibleNewSlavesCount(SlaveTemplate template) throws AmazonClien return Math.min(availableAmiSlaves, availableTotalSlaves); } + /* Gets the VPCs of the subnets configured for a given Slave Template */ + private Set getTemplateVpcs(SlaveTemplate template) throws AmazonClientException { + Set vpcs = new HashSet<>(); + if (template != null && template.getSubnetId() != null && !template.getSubnetId().isEmpty()) { + DescribeSubnetsRequest dsr = new DescribeSubnetsRequest().withSubnetIds(template.getSubnetId().split(SlaveTemplate.EC2_RESOURCE_ID_DELIMETERS)); + DescribeSubnetsResult result = null; + do { + result = connect().describeSubnets(dsr); + dsr.setNextToken(result.getNextToken()); + for (Subnet subnet: result.getSubnets()) { + vpcs.add(subnet.getVpcId()); + } + } while (result.getNextToken() != null); + } + return vpcs; + } + /** * Obtains a agent whose AMI matches the AMI of the given template, and that also has requiredLabel (if requiredLabel is non-null) * forceCreateNew specifies that the creation of a new agent is required. Otherwise, an existing matching agent may be re-used