From b8fe32b58e03d1edd43dac9539a1f45eaac466ee Mon Sep 17 00:00:00 2001 From: Lew Gordon Date: Tue, 20 Mar 2018 10:53:53 -0400 Subject: [PATCH 01/11] INFRA-3043 Adding ability to assume role. --- .../java/hudson/plugins/s3/ClientHelper.java | 20 +++++-- .../hudson/plugins/s3/S3ArtifactsAction.java | 24 ++++---- .../hudson/plugins/s3/S3BucketPublisher.java | 46 +++++++++----- .../java/hudson/plugins/s3/S3Profile.java | 60 +++++++++++-------- .../s3/callable/S3BaseUploadCallable.java | 4 +- .../plugins/s3/callable/S3Callable.java | 10 ++-- .../s3/callable/S3DownloadCallable.java | 4 +- .../plugins/s3/callable/S3GzipCallable.java | 4 +- .../plugins/s3/callable/S3UploadCallable.java | 6 +- .../plugins/s3/S3BucketPublisher/global.jelly | 3 + src/test/java/hudson/plugins/s3/S3Test.java | 2 +- 11 files changed, 110 insertions(+), 73 deletions(-) diff --git a/src/main/java/hudson/plugins/s3/ClientHelper.java b/src/main/java/hudson/plugins/s3/ClientHelper.java index 8251d2d2..a9398cef 100644 --- a/src/main/java/hudson/plugins/s3/ClientHelper.java +++ b/src/main/java/hudson/plugins/s3/ClientHelper.java @@ -1,7 +1,10 @@ package hudson.plugins.s3; import com.amazonaws.ClientConfiguration; +import com.amazonaws.auth.AWSStaticCredentialsProvider; import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.auth.STSAssumeRoleSessionCredentialsProvider; +import com.amazonaws.auth.profile.internal.ProfileAssumeRoleCredentialsProvider; import com.amazonaws.regions.Region; import com.amazonaws.regions.RegionUtils; import com.amazonaws.regions.Regions; @@ -19,17 +22,24 @@ public class ClientHelper { "hudson.plugins.s3.DEFAULT_AMAZON_S3_REGION", com.amazonaws.services.s3.model.Region.US_Standard.toAWSRegion().getName()); - public static AmazonS3Client createClient(String accessKey, String secretKey, boolean useRole, String region, ProxyConfiguration proxy) + public static AmazonS3 createClient(String accessKey, String secretKey, boolean useRole, String assumeRole, String region, ProxyConfiguration proxy) { Region awsRegion = getRegionFromString(region); ClientConfiguration clientConfiguration = getClientConfiguration(proxy, awsRegion); - final AmazonS3Client client; - if (useRole) { - client = new AmazonS3Client(clientConfiguration); + final AmazonS3 client; + if (assumeRole != null) { + client = AmazonS3Client.builder() + .withCredentials(new STSAssumeRoleSessionCredentialsProvider.Builder(assumeRole, "jenkins-s3-plugin").build()) + .build(); + } else if (useRole) { + client = AmazonS3Client.builder() + .withClientConfiguration(clientConfiguration).build(); } else { - client = new AmazonS3Client(new BasicAWSCredentials(accessKey, secretKey), clientConfiguration); + client = AmazonS3Client.builder() + .withCredentials(new AWSStaticCredentialsProvider(new BasicAWSCredentials(accessKey, secretKey))) + .withClientConfiguration(clientConfiguration).build(); } client.setRegion(awsRegion); diff --git a/src/main/java/hudson/plugins/s3/S3ArtifactsAction.java b/src/main/java/hudson/plugins/s3/S3ArtifactsAction.java index 1f4608af..a9af3c05 100644 --- a/src/main/java/hudson/plugins/s3/S3ArtifactsAction.java +++ b/src/main/java/hudson/plugins/s3/S3ArtifactsAction.java @@ -1,25 +1,23 @@ package hudson.plugins.s3; -import java.io.File; -import java.io.IOException; - -import java.util.Date; -import java.util.List; - -import javax.servlet.ServletException; +import static javax.servlet.http.HttpServletResponse.SC_NOT_FOUND; +import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3Client; import com.amazonaws.services.s3.model.GeneratePresignedUrlRequest; import com.amazonaws.services.s3.model.ResponseHeaderOverrides; -import jenkins.model.RunAction2; import org.kohsuke.stapler.StaplerRequest; import org.kohsuke.stapler.StaplerResponse; - -import hudson.model.Run; import org.kohsuke.stapler.export.Exported; import org.kohsuke.stapler.export.ExportedBean; +import hudson.model.Run; +import jenkins.model.RunAction2; -import static javax.servlet.http.HttpServletResponse.SC_NOT_FOUND; +import java.io.File; +import java.io.IOException; +import java.util.Date; +import java.util.List; +import javax.servlet.ServletException; @ExportedBean public class S3ArtifactsAction implements RunAction2 { @@ -77,7 +75,7 @@ public void doDownload(final StaplerRequest request, final StaplerResponse respo for (FingerprintRecord record : artifacts) { if (record.getArtifact().getName().equals(artifact)) { final S3Profile s3 = S3BucketPublisher.getProfile(profile); - final AmazonS3Client client = s3.getClient(record.getArtifact().getRegion()); + final AmazonS3 client = s3.getClient(record.getArtifact().getRegion()); final String url = getDownloadURL(client, s3.getSignedUrlExpirySeconds(), build, record); response.sendRedirect2(url); return; @@ -94,7 +92,7 @@ public void doDownload(final StaplerRequest request, final StaplerResponse respo * download and there's no need for the user to have credentials to * access S3. */ - private String getDownloadURL(AmazonS3Client client, int signedUrlExpirySeconds, Run run, FingerprintRecord record) { + private String getDownloadURL(AmazonS3 client, int signedUrlExpirySeconds, Run run, FingerprintRecord record) { final Destination dest = Destination.newFromRun(run, record.getArtifact()); final GeneratePresignedUrlRequest request = new GeneratePresignedUrlRequest(dest.bucketName, dest.objectName); request.setExpiration(new Date(System.currentTimeMillis() + signedUrlExpirySeconds*1000)); diff --git a/src/main/java/hudson/plugins/s3/S3BucketPublisher.java b/src/main/java/hudson/plugins/s3/S3BucketPublisher.java index e2a1000b..f27d1b1a 100644 --- a/src/main/java/hudson/plugins/s3/S3BucketPublisher.java +++ b/src/main/java/hudson/plugins/s3/S3BucketPublisher.java @@ -1,13 +1,29 @@ package hudson.plugins.s3; -import com.amazonaws.AmazonClientException; -import com.amazonaws.regions.Region; -import com.amazonaws.services.s3.AmazonS3Client; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; import com.google.common.collect.Maps; -import hudson.*; -import hudson.model.*; + +import com.amazonaws.AmazonClientException; +import com.amazonaws.regions.Region; +import com.amazonaws.services.s3.AmazonS3; +import net.sf.json.JSONArray; +import net.sf.json.JSONObject; +import org.apache.commons.lang.StringUtils; +import org.jenkinsci.Symbol; +import org.kohsuke.stapler.DataBoundConstructor; +import org.kohsuke.stapler.StaplerRequest; +import org.kohsuke.stapler.StaplerResponse; +import hudson.Extension; +import hudson.FilePath; +import hudson.Launcher; +import hudson.Util; +import hudson.model.AbstractProject; +import hudson.model.Action; +import hudson.model.Fingerprint; +import hudson.model.Result; +import hudson.model.Run; +import hudson.model.TaskListener; import hudson.model.listeners.RunListener; import hudson.tasks.BuildStepDescriptor; import hudson.tasks.BuildStepMonitor; @@ -19,20 +35,17 @@ import hudson.util.ListBoxModel; import jenkins.model.Jenkins; import jenkins.tasks.SimpleBuildStep; -import net.sf.json.JSONArray; -import net.sf.json.JSONObject; -import org.apache.commons.lang.StringUtils; -import org.jenkinsci.Symbol; -import org.kohsuke.stapler.DataBoundConstructor; -import org.kohsuke.stapler.StaplerRequest; -import org.kohsuke.stapler.StaplerResponse; -import javax.annotation.Nonnull; import java.io.IOException; import java.io.PrintStream; -import java.util.*; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import java.util.logging.Level; import java.util.logging.Logger; +import javax.annotation.Nonnull; public final class S3BucketPublisher extends Recorder implements SimpleBuildStep { @@ -485,6 +498,7 @@ public FormValidation doLoginCheck(final StaplerRequest req, StaplerResponse rsp final boolean couldBeValidated = !name.isEmpty() && !accessKey.isEmpty() && !secretKey.isEmpty(); final boolean useRole = Boolean.parseBoolean(useIAMCredential); + final String assumeRole = req.getParameter("assumeRole"); if (!couldBeValidated) { if (name.isEmpty()) @@ -501,8 +515,8 @@ public FormValidation doLoginCheck(final StaplerRequest req, StaplerResponse rsp } final String defaultRegion = ClientHelper.DEFAULT_AMAZON_S3_REGION_NAME; - final AmazonS3Client client = ClientHelper.createClient( - accessKey, secretKey, useRole, defaultRegion, Jenkins.getActiveInstance().proxy); + final AmazonS3 client = ClientHelper.createClient( + accessKey, secretKey, useRole, assumeRole, defaultRegion, Jenkins.getActiveInstance().proxy); try { client.listBuckets(); diff --git a/src/main/java/hudson/plugins/s3/S3Profile.java b/src/main/java/hudson/plugins/s3/S3Profile.java index 95074592..30373b9a 100644 --- a/src/main/java/hudson/plugins/s3/S3Profile.java +++ b/src/main/java/hudson/plugins/s3/S3Profile.java @@ -1,30 +1,34 @@ package hudson.plugins.s3; -import hudson.FilePath; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.concurrent.Callable; -import java.util.concurrent.TimeUnit; - -import hudson.ProxyConfiguration; -import hudson.plugins.s3.callable.*; -import jenkins.model.Jenkins; -import org.apache.commons.io.FilenameUtils; -import org.kohsuke.stapler.DataBoundConstructor; +import com.google.common.collect.Lists; +import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3Client; import com.amazonaws.services.s3.model.DeleteObjectRequest; import com.amazonaws.services.s3.model.GetObjectRequest; import com.amazonaws.services.s3.model.ListObjectsRequest; import com.amazonaws.services.s3.model.ObjectListing; import com.amazonaws.services.s3.model.S3ObjectSummary; -import com.google.common.collect.Lists; - +import org.apache.commons.io.FilenameUtils; +import org.kohsuke.stapler.DataBoundConstructor; +import hudson.FilePath; +import hudson.ProxyConfiguration; import hudson.model.Run; +import hudson.plugins.s3.callable.MasterSlaveCallable; +import hudson.plugins.s3.callable.S3CleanupUploadCallable; +import hudson.plugins.s3.callable.S3DownloadCallable; +import hudson.plugins.s3.callable.S3GzipCallable; +import hudson.plugins.s3.callable.S3UploadCallable; +import hudson.plugins.s3.callable.S3WaitUploadCallable; import hudson.util.Secret; +import jenkins.model.Jenkins; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Callable; +import java.util.concurrent.TimeUnit; public class S3Profile { private final String name; @@ -37,12 +41,14 @@ public class S3Profile { private final boolean keepStructure; private final boolean useRole; + private final String assumeRole; private final int signedUrlExpirySeconds; @DataBoundConstructor - public S3Profile(String name, String accessKey, String secretKey, boolean useRole, int signedUrlExpirySeconds, String maxUploadRetries, String uploadRetryTime, String maxDownloadRetries, String downloadRetryTime, boolean keepStructure) { + public S3Profile(String name, String accessKey, String secretKey, boolean useRole, String assumeRole, int signedUrlExpirySeconds, String maxUploadRetries, String uploadRetryTime, String maxDownloadRetries, String downloadRetryTime, boolean keepStructure) { this.name = name; this.useRole = useRole; + this.assumeRole = assumeRole; this.maxUploadRetries = parseWithDefault(maxUploadRetries, 5); this.uploadRetryTime = parseWithDefault(uploadRetryTime, 5); this.maxDownloadRetries = parseWithDefault(maxDownloadRetries, 5); @@ -103,6 +109,10 @@ public final boolean getUseRole() { return this.useRole; } + public final String getAssumeRole() { + return assumeRole; + } + public boolean isUseRole() { return useRole; } @@ -111,8 +121,8 @@ public int getSignedUrlExpirySeconds() { return signedUrlExpirySeconds; } - public AmazonS3Client getClient(String region) { - return ClientHelper.createClient(accessKey, Secret.toString(secretKey), useRole, region, getProxy()); + public AmazonS3 getClient(String region) { + return ClientHelper.createClient(accessKey, Secret.toString(secretKey), useRole, assumeRole, region, getProxy()); } public List upload(Run run, @@ -145,10 +155,10 @@ public List upload(Run run, final MasterSlaveCallable upload; if (gzipFiles) { - upload = new S3GzipCallable(accessKey, secretKey, useRole, dest, userMetadata, - storageClass, selregion, useServerSideEncryption, getProxy()); + upload = new S3GzipCallable(accessKey, secretKey, useRole, assumeRole, dest, + userMetadata, storageClass, selregion, useServerSideEncryption, getProxy()); } else { - upload = new S3UploadCallable(accessKey, secretKey, useRole, dest, userMetadata, + upload = new S3UploadCallable(accessKey, secretKey, useRole, assumeRole, dest, userMetadata, storageClass, selregion, useServerSideEncryption, getProxy()); } @@ -197,7 +207,7 @@ private T invoke(boolean uploadFromSlave, FilePath filePath, MasterSlaveCall } public List list(Run build, String bucket) { - final AmazonS3Client s3client = getClient(ClientHelper.DEFAULT_AMAZON_S3_REGION_NAME); + final AmazonS3 s3client = getClient(ClientHelper.DEFAULT_AMAZON_S3_REGION_NAME); final String buildName = build.getDisplayName(); final int buildID = build.getNumber(); @@ -241,7 +251,7 @@ public List downloadAll(Run build, fingerprints.add(repeat(maxDownloadRetries, downloadRetryTime, dest, new Callable() { @Override public FingerprintRecord call() throws IOException, InterruptedException { - final String md5 = target.act(new S3DownloadCallable(accessKey, secretKey, useRole, dest, artifact.getRegion(), getProxy())); + final String md5 = target.act(new S3DownloadCallable(accessKey, secretKey, useRole, assumeRole, dest, artifact.getRegion(), getProxy())); return new FingerprintRecord(true, dest.bucketName, target.getName(), artifact.getRegion(), md5); } })); @@ -281,7 +291,7 @@ private FilePath getFilePath(FilePath targetDir, boolean flatten, String fullNam public void delete(Run run, FingerprintRecord record) { final Destination dest = Destination.newFromRun(run, record.getArtifact()); final DeleteObjectRequest req = new DeleteObjectRequest(dest.bucketName, dest.objectName); - final AmazonS3Client client = getClient(record.getArtifact().getRegion()); + final AmazonS3 client = getClient(record.getArtifact().getRegion()); client.deleteObject(req); } diff --git a/src/main/java/hudson/plugins/s3/callable/S3BaseUploadCallable.java b/src/main/java/hudson/plugins/s3/callable/S3BaseUploadCallable.java index 76e72cc5..7cde64f2 100644 --- a/src/main/java/hudson/plugins/s3/callable/S3BaseUploadCallable.java +++ b/src/main/java/hudson/plugins/s3/callable/S3BaseUploadCallable.java @@ -24,9 +24,9 @@ public abstract class S3BaseUploadCallable extends S3Callable { public S3BaseUploadCallable(String accessKey, Secret secretKey, boolean useRole, - Destination dest, Map userMetadata, String storageClass, String selregion, + String assumeRole, Destination dest, Map userMetadata, String storageClass, String selregion, boolean useServerSideEncryption, ProxyConfiguration proxy) { - super(accessKey, secretKey, useRole, selregion, proxy); + super(accessKey, secretKey, useRole, assumeRole, selregion, proxy); this.dest = dest; this.storageClass = storageClass; this.userMetadata = userMetadata; diff --git a/src/main/java/hudson/plugins/s3/callable/S3Callable.java b/src/main/java/hudson/plugins/s3/callable/S3Callable.java index 0916366a..e845ff2d 100644 --- a/src/main/java/hudson/plugins/s3/callable/S3Callable.java +++ b/src/main/java/hudson/plugins/s3/callable/S3Callable.java @@ -16,15 +16,17 @@ abstract class S3Callable implements FileCallable { private final String accessKey; private final Secret secretKey; private final boolean useRole; + private final String assumeRole; private final String region; private final ProxyConfiguration proxy; private static transient HashMap transferManagers = new HashMap<>(); - S3Callable(String accessKey, Secret secretKey, boolean useRole, String region, ProxyConfiguration proxy) { + S3Callable(String accessKey, Secret secretKey, boolean useRole, String assumeRole, String region, ProxyConfiguration proxy) { this.accessKey = accessKey; this.secretKey = secretKey; this.useRole = useRole; + this.assumeRole = assumeRole; this.region = region; this.proxy = proxy; } @@ -32,7 +34,7 @@ abstract class S3Callable implements FileCallable { protected synchronized TransferManager getTransferManager() { final String uniqueKey = getUniqueKey(); if (transferManagers.get(uniqueKey) == null) { - final AmazonS3 client = ClientHelper.createClient(accessKey, Secret.toString(secretKey), useRole, region, proxy); + final AmazonS3 client = ClientHelper.createClient(accessKey, Secret.toString(secretKey), useRole, assumeRole, region, proxy); transferManagers.put(uniqueKey, new TransferManager(client)); } @@ -45,6 +47,6 @@ public void checkRoles(RoleChecker roleChecker) throws SecurityException { } private String getUniqueKey() { - return region + '_' + secretKey + '_' + accessKey + '_' + useRole; + return region + '_' + secretKey + '_' + accessKey + '_' + useRole + '_' + assumeRole; } -} \ No newline at end of file +} diff --git a/src/main/java/hudson/plugins/s3/callable/S3DownloadCallable.java b/src/main/java/hudson/plugins/s3/callable/S3DownloadCallable.java index 82964882..4619e3e6 100644 --- a/src/main/java/hudson/plugins/s3/callable/S3DownloadCallable.java +++ b/src/main/java/hudson/plugins/s3/callable/S3DownloadCallable.java @@ -16,9 +16,9 @@ public final class S3DownloadCallable extends S3Callable private static final long serialVersionUID = 1L; private final Destination dest; - public S3DownloadCallable(String accessKey, Secret secretKey, boolean useRole, Destination dest, String region, ProxyConfiguration proxy) + public S3DownloadCallable(String accessKey, Secret secretKey, boolean useRole, String assumeRole, Destination dest, String region, ProxyConfiguration proxy) { - super(accessKey, secretKey, useRole, region, proxy); + super(accessKey, secretKey, useRole, assumeRole, region, proxy); this.dest = dest; } diff --git a/src/main/java/hudson/plugins/s3/callable/S3GzipCallable.java b/src/main/java/hudson/plugins/s3/callable/S3GzipCallable.java index 735831e0..25c503f2 100644 --- a/src/main/java/hudson/plugins/s3/callable/S3GzipCallable.java +++ b/src/main/java/hudson/plugins/s3/callable/S3GzipCallable.java @@ -17,8 +17,8 @@ import java.util.zip.GZIPOutputStream; public final class S3GzipCallable extends S3BaseUploadCallable implements MasterSlaveCallable { - public S3GzipCallable(String accessKey, Secret secretKey, boolean useRole, Destination dest, Map userMetadata, String storageClass, String selregion, boolean useServerSideEncryption, ProxyConfiguration proxy) { - super(accessKey, secretKey, useRole, dest, userMetadata, storageClass, selregion, useServerSideEncryption, proxy); + public S3GzipCallable(String accessKey, Secret secretKey, boolean useRole, String assumeRole, Destination dest, Map userMetadata, String storageClass, String selregion, boolean useServerSideEncryption, ProxyConfiguration proxy) { + super(accessKey, secretKey, useRole, assumeRole, dest, userMetadata, storageClass, selregion, useServerSideEncryption, proxy); } // Return a File containing the gzipped contents of the input file. diff --git a/src/main/java/hudson/plugins/s3/callable/S3UploadCallable.java b/src/main/java/hudson/plugins/s3/callable/S3UploadCallable.java index 064aea57..3339c32e 100644 --- a/src/main/java/hudson/plugins/s3/callable/S3UploadCallable.java +++ b/src/main/java/hudson/plugins/s3/callable/S3UploadCallable.java @@ -14,8 +14,8 @@ public final class S3UploadCallable extends S3BaseUploadCallable implements MasterSlaveCallable { private static final long serialVersionUID = 1L; - public S3UploadCallable(String accessKey, Secret secretKey, boolean useRole, Destination dest, Map userMetadata, String storageClass, String selregion, boolean useServerSideEncryption, ProxyConfiguration proxy) { - super(accessKey, secretKey, useRole, dest, userMetadata, storageClass, selregion, useServerSideEncryption, proxy); + public S3UploadCallable(String accessKey, Secret secretKey, boolean useRole, String assumeRole, Destination dest, Map userMetadata, String storageClass, String selregion, boolean useServerSideEncryption, ProxyConfiguration proxy) { + super(accessKey, secretKey, useRole, assumeRole, dest, userMetadata, storageClass, selregion, useServerSideEncryption, proxy); } /** @@ -29,4 +29,4 @@ public String invoke(FilePath file) throws IOException, InterruptedException { return MD5.generateFromFile(file); } -} \ No newline at end of file +} diff --git a/src/main/resources/hudson/plugins/s3/S3BucketPublisher/global.jelly b/src/main/resources/hudson/plugins/s3/S3BucketPublisher/global.jelly index 86dcfb42..ab4d6b23 100644 --- a/src/main/resources/hudson/plugins/s3/S3BucketPublisher/global.jelly +++ b/src/main/resources/hudson/plugins/s3/S3BucketPublisher/global.jelly @@ -11,6 +11,9 @@ + + + Date: Tue, 20 Mar 2018 10:54:56 -0400 Subject: [PATCH 02/11] INFRA-3043 Adding ci.jenkinsfile --- ci.jenkinsfile | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 ci.jenkinsfile diff --git a/ci.jenkinsfile b/ci.jenkinsfile new file mode 100644 index 00000000..b23e8ac3 --- /dev/null +++ b/ci.jenkinsfile @@ -0,0 +1,10 @@ +@Library('pipeline-library@master') +import com.genesys.jenkins.Service + +node('dev_mesos') { + final serviceBuild = new com.genesys.jenkins.Service() + checkout scm + + serviceBuild.runMavenGoals('clean verify') +} + From a82549ae4581358e4e54321ef2e6e55812df33da Mon Sep 17 00:00:00 2001 From: Lew Gordon Date: Tue, 20 Mar 2018 11:00:54 -0400 Subject: [PATCH 03/11] INFRA-3043 Adding some documentation. --- .../resources/hudson/plugins/s3/S3BucketPublisher/global.jelly | 2 +- src/main/webapp/help-assumeRole.html | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 src/main/webapp/help-assumeRole.html diff --git a/src/main/resources/hudson/plugins/s3/S3BucketPublisher/global.jelly b/src/main/resources/hudson/plugins/s3/S3BucketPublisher/global.jelly index ab4d6b23..1a4d6cb5 100644 --- a/src/main/resources/hudson/plugins/s3/S3BucketPublisher/global.jelly +++ b/src/main/resources/hudson/plugins/s3/S3BucketPublisher/global.jelly @@ -11,7 +11,7 @@ - + diff --git a/src/main/webapp/help-assumeRole.html b/src/main/webapp/help-assumeRole.html new file mode 100644 index 00000000..ddb0abb3 --- /dev/null +++ b/src/main/webapp/help-assumeRole.html @@ -0,0 +1 @@ +
If this field is set, when doing any S3 operations the Jenkins master will try to assume the role given. This overrides using an IAM role or Access Key/Secret Key
From b3e000464c5ac359bc634298b770917fa5e8153d Mon Sep 17 00:00:00 2001 From: Lew Gordon Date: Tue, 20 Mar 2018 11:31:12 -0400 Subject: [PATCH 04/11] INFRA-3043 Revert to mutable client since immutable client needs fixes elsewhere. --- src/main/java/hudson/plugins/s3/ClientHelper.java | 11 +++-------- .../java/hudson/plugins/s3/S3ArtifactsAction.java | 1 - src/main/java/hudson/plugins/s3/S3Profile.java | 1 - 3 files changed, 3 insertions(+), 10 deletions(-) diff --git a/src/main/java/hudson/plugins/s3/ClientHelper.java b/src/main/java/hudson/plugins/s3/ClientHelper.java index a9398cef..dcc25967 100644 --- a/src/main/java/hudson/plugins/s3/ClientHelper.java +++ b/src/main/java/hudson/plugins/s3/ClientHelper.java @@ -30,16 +30,11 @@ public static AmazonS3 createClient(String accessKey, String secretKey, boolean final AmazonS3 client; if (assumeRole != null) { - client = AmazonS3Client.builder() - .withCredentials(new STSAssumeRoleSessionCredentialsProvider.Builder(assumeRole, "jenkins-s3-plugin").build()) - .build(); + client = new AmazonS3Client(new STSAssumeRoleSessionCredentialsProvider.Builder(assumeRole, "jenkins-s3-plugin").build()); } else if (useRole) { - client = AmazonS3Client.builder() - .withClientConfiguration(clientConfiguration).build(); + client = new AmazonS3Client(clientConfiguration); } else { - client = AmazonS3Client.builder() - .withCredentials(new AWSStaticCredentialsProvider(new BasicAWSCredentials(accessKey, secretKey))) - .withClientConfiguration(clientConfiguration).build(); + client = new AmazonS3Client(new BasicAWSCredentials(accessKey, secretKey), clientConfiguration); } client.setRegion(awsRegion); diff --git a/src/main/java/hudson/plugins/s3/S3ArtifactsAction.java b/src/main/java/hudson/plugins/s3/S3ArtifactsAction.java index a9af3c05..1a49c4bb 100644 --- a/src/main/java/hudson/plugins/s3/S3ArtifactsAction.java +++ b/src/main/java/hudson/plugins/s3/S3ArtifactsAction.java @@ -3,7 +3,6 @@ import static javax.servlet.http.HttpServletResponse.SC_NOT_FOUND; import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.AmazonS3Client; import com.amazonaws.services.s3.model.GeneratePresignedUrlRequest; import com.amazonaws.services.s3.model.ResponseHeaderOverrides; import org.kohsuke.stapler.StaplerRequest; diff --git a/src/main/java/hudson/plugins/s3/S3Profile.java b/src/main/java/hudson/plugins/s3/S3Profile.java index 30373b9a..dc8ba315 100644 --- a/src/main/java/hudson/plugins/s3/S3Profile.java +++ b/src/main/java/hudson/plugins/s3/S3Profile.java @@ -3,7 +3,6 @@ import com.google.common.collect.Lists; import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.AmazonS3Client; import com.amazonaws.services.s3.model.DeleteObjectRequest; import com.amazonaws.services.s3.model.GetObjectRequest; import com.amazonaws.services.s3.model.ListObjectsRequest; From b3b548da8a91eb09366bf909386f950892a7fe8e Mon Sep 17 00:00:00 2001 From: Lew Gordon Date: Tue, 20 Mar 2018 13:01:39 -0400 Subject: [PATCH 05/11] INFRA-3043 Adding assume role validation. --- .../java/hudson/plugins/s3/ClientHelper.java | 67 ++++++++++++++++--- .../hudson/plugins/s3/S3BucketPublisher.java | 16 +++++ .../plugins/s3/S3BucketPublisher/global.jelly | 2 +- 3 files changed, 73 insertions(+), 12 deletions(-) diff --git a/src/main/java/hudson/plugins/s3/ClientHelper.java b/src/main/java/hudson/plugins/s3/ClientHelper.java index dcc25967..9f42644b 100644 --- a/src/main/java/hudson/plugins/s3/ClientHelper.java +++ b/src/main/java/hudson/plugins/s3/ClientHelper.java @@ -1,19 +1,19 @@ package hudson.plugins.s3; import com.amazonaws.ClientConfiguration; +import com.amazonaws.auth.AWSCredentialsProvider; import com.amazonaws.auth.AWSStaticCredentialsProvider; import com.amazonaws.auth.BasicAWSCredentials; import com.amazonaws.auth.STSAssumeRoleSessionCredentialsProvider; -import com.amazonaws.auth.profile.internal.ProfileAssumeRoleCredentialsProvider; import com.amazonaws.regions.Region; import com.amazonaws.regions.RegionUtils; import com.amazonaws.regions.Regions; import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3Client; +import com.amazonaws.services.s3.AmazonS3ClientBuilder; import hudson.ProxyConfiguration; import java.util.regex.Pattern; - import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -22,23 +22,68 @@ public class ClientHelper { "hudson.plugins.s3.DEFAULT_AMAZON_S3_REGION", com.amazonaws.services.s3.model.Region.US_Standard.toAWSRegion().getName()); - public static AmazonS3 createClient(String accessKey, String secretKey, boolean useRole, String assumeRole, String region, ProxyConfiguration proxy) - { - Region awsRegion = getRegionFromString(region); + public static class Builder { + private final ProxyConfiguration proxyConfiguration; + private final String region; + + public Builder(String region, ProxyConfiguration proxyConfiguration) { + this.region = region; + this.proxyConfiguration = proxyConfiguration; + } + + public AmazonS3 build(String accessKey, String secretKey) { + return this.buildClient(new AWSStaticCredentialsProvider(new BasicAWSCredentials(accessKey, secretKey))); + } + + public AmazonS3 build(String assumeRole) { + return this.buildClient(new STSAssumeRoleSessionCredentialsProvider.Builder(assumeRole, "jenkins-s3-plugin").build()); + } + + public AmazonS3 build() { + return this.buildClient(null); + } + + private AmazonS3 buildClient(@Nullable AWSCredentialsProvider awsCredentialsProvider) { + Region awsRegion = getRegionFromString(region); - ClientConfiguration clientConfiguration = getClientConfiguration(proxy, awsRegion); + ClientConfiguration clientConfiguration = getClientConfiguration(proxyConfiguration, awsRegion); + + final AmazonS3 client; + if (awsCredentialsProvider != null) { + client = new AmazonS3Client(awsCredentialsProvider, clientConfiguration); + } else { + client = new AmazonS3Client(clientConfiguration); + } + + client.setRegion(awsRegion); + + return client; + } + } + + /** + * @deprecated use {@link ClientHelper.Builder} instead + */ + @Deprecated + public static AmazonS3 createClient( + String accessKey, + String secretKey, + boolean useRole, + String assumeRole, + String region, + ProxyConfiguration proxy + ) { + Builder builder = new Builder(region, proxy); final AmazonS3 client; if (assumeRole != null) { - client = new AmazonS3Client(new STSAssumeRoleSessionCredentialsProvider.Builder(assumeRole, "jenkins-s3-plugin").build()); + client = builder.build(assumeRole); } else if (useRole) { - client = new AmazonS3Client(clientConfiguration); + client = builder.build(); } else { - client = new AmazonS3Client(new BasicAWSCredentials(accessKey, secretKey), clientConfiguration); + client = builder.build(accessKey, secretKey); } - client.setRegion(awsRegion); - return client; } diff --git a/src/main/java/hudson/plugins/s3/S3BucketPublisher.java b/src/main/java/hudson/plugins/s3/S3BucketPublisher.java index f27d1b1a..6689be8c 100644 --- a/src/main/java/hudson/plugins/s3/S3BucketPublisher.java +++ b/src/main/java/hudson/plugins/s3/S3BucketPublisher.java @@ -12,6 +12,7 @@ import org.apache.commons.lang.StringUtils; import org.jenkinsci.Symbol; import org.kohsuke.stapler.DataBoundConstructor; +import org.kohsuke.stapler.QueryParameter; import org.kohsuke.stapler.StaplerRequest; import org.kohsuke.stapler.StaplerResponse; import hudson.Extension; @@ -46,6 +47,7 @@ import java.util.logging.Level; import java.util.logging.Logger; import javax.annotation.Nonnull; +import javax.servlet.ServletException; public final class S3BucketPublisher extends Recorder implements SimpleBuildStep { @@ -489,6 +491,20 @@ public Result[] getPluginFailureResultConstraints() { return pluginFailureResultConstraints; } + @SuppressWarnings("unused") + public FormValidation doCheckAssumeRole(@QueryParameter String value) throws IOException, ServletException { + final String defaultRegion = ClientHelper.DEFAULT_AMAZON_S3_REGION_NAME; + final AmazonS3 client = new ClientHelper.Builder(defaultRegion, Jenkins.getActiveInstance().proxy).build(value); + + try { + client.listBuckets(); + } catch (AmazonClientException e) { + LOGGER.log(Level.SEVERE, e.getMessage(), e); + return FormValidation.error("Cannot list buckets from S3: " + e.getMessage()); + } + return FormValidation.ok("Successfully assumed role: " + value); + } + @SuppressWarnings("unused") public FormValidation doLoginCheck(final StaplerRequest req, StaplerResponse rsp) { final String name = Util.fixNull(req.getParameter("name")); diff --git a/src/main/resources/hudson/plugins/s3/S3BucketPublisher/global.jelly b/src/main/resources/hudson/plugins/s3/S3BucketPublisher/global.jelly index 1a4d6cb5..d15c5384 100644 --- a/src/main/resources/hudson/plugins/s3/S3BucketPublisher/global.jelly +++ b/src/main/resources/hudson/plugins/s3/S3BucketPublisher/global.jelly @@ -11,7 +11,7 @@ - + From b85e3c675ad9a2a51d400bfe62a00e961f94a13d Mon Sep 17 00:00:00 2001 From: Lew Gordon Date: Wed, 28 Mar 2018 16:55:11 -0400 Subject: [PATCH 06/11] NO-JIRA archive hpi --- ci.jenkinsfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ci.jenkinsfile b/ci.jenkinsfile index b23e8ac3..f52d1ee8 100644 --- a/ci.jenkinsfile +++ b/ci.jenkinsfile @@ -6,5 +6,7 @@ node('dev_mesos') { checkout scm serviceBuild.runMavenGoals('clean verify') + + archiveArtifacts 'target/s3.hpi' } From 2df05cab474845b8fdb7cceba88c975b96c00e17 Mon Sep 17 00:00:00 2001 From: Lew Gordon Date: Tue, 24 Apr 2018 16:38:21 -0400 Subject: [PATCH 07/11] INFRA-3120 Check if not empty --- src/main/java/hudson/plugins/s3/ClientHelper.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/hudson/plugins/s3/ClientHelper.java b/src/main/java/hudson/plugins/s3/ClientHelper.java index 9f42644b..f94cf645 100644 --- a/src/main/java/hudson/plugins/s3/ClientHelper.java +++ b/src/main/java/hudson/plugins/s3/ClientHelper.java @@ -11,6 +11,7 @@ import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3Client; import com.amazonaws.services.s3.AmazonS3ClientBuilder; +import org.apache.commons.lang.StringUtils; import hudson.ProxyConfiguration; import java.util.regex.Pattern; @@ -76,7 +77,7 @@ public static AmazonS3 createClient( Builder builder = new Builder(region, proxy); final AmazonS3 client; - if (assumeRole != null) { + if (StringUtils.isNotEmpty(assumeRole)) { client = builder.build(assumeRole); } else if (useRole) { client = builder.build(); From 5aafd84c8db6d7856ca4714c0f635ba628f33b58 Mon Sep 17 00:00:00 2001 From: Lew Gordon Date: Tue, 24 Apr 2018 17:05:28 -0400 Subject: [PATCH 08/11] INFRA-3120 Fix form validation --- src/main/java/hudson/plugins/s3/S3BucketPublisher.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/java/hudson/plugins/s3/S3BucketPublisher.java b/src/main/java/hudson/plugins/s3/S3BucketPublisher.java index 6689be8c..5d646100 100644 --- a/src/main/java/hudson/plugins/s3/S3BucketPublisher.java +++ b/src/main/java/hudson/plugins/s3/S3BucketPublisher.java @@ -493,6 +493,10 @@ public Result[] getPluginFailureResultConstraints() { @SuppressWarnings("unused") public FormValidation doCheckAssumeRole(@QueryParameter String value) throws IOException, ServletException { + if(StringUtils.isEmpty(value)) { + return FormValidation.ok(); + } + final String defaultRegion = ClientHelper.DEFAULT_AMAZON_S3_REGION_NAME; final AmazonS3 client = new ClientHelper.Builder(defaultRegion, Jenkins.getActiveInstance().proxy).build(value); From 2d7831c0bdf4cd67cfe3f2411317e49fa7624786 Mon Sep 17 00:00:00 2001 From: Lew Gordon Date: Wed, 15 Apr 2020 13:27:44 -0400 Subject: [PATCH 09/11] Validate role assumption, when using an assumed role. --- .../hudson/plugins/s3/S3BucketPublisher.java | 18 ++++++++++++------ .../plugins/s3/S3BucketPublisher/global.jelly | 4 ++-- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/src/main/java/hudson/plugins/s3/S3BucketPublisher.java b/src/main/java/hudson/plugins/s3/S3BucketPublisher.java index a36d1a9e..ba1495c9 100644 --- a/src/main/java/hudson/plugins/s3/S3BucketPublisher.java +++ b/src/main/java/hudson/plugins/s3/S3BucketPublisher.java @@ -511,14 +511,13 @@ public Result[] getPluginFailureResultConstraints() { return pluginFailureResultConstraints.clone(); } - @SuppressWarnings("unused") - public FormValidation doCheckAssumeRole(@QueryParameter String value) throws IOException, ServletException { - if(StringUtils.isEmpty(value)) { + private FormValidation doCheckAssumeRole(String assumeRole) { + if(StringUtils.isEmpty(assumeRole)) { return FormValidation.ok(); } final String defaultRegion = ClientHelper.DEFAULT_AMAZON_S3_REGION_NAME; - final AmazonS3 client = new ClientHelper.Builder(defaultRegion, Jenkins.getActiveInstance().proxy).build(value); + final AmazonS3 client = new ClientHelper.Builder(defaultRegion, Jenkins.get().proxy).build(assumeRole); try { client.listBuckets(); @@ -526,17 +525,19 @@ public FormValidation doCheckAssumeRole(@QueryParameter String value) throws IOE LOGGER.log(Level.SEVERE, e.getMessage(), e); return FormValidation.error("Cannot list buckets from S3: " + e.getMessage()); } - return FormValidation.ok("Successfully assumed role: " + value); + return FormValidation.ok("Successfully assumed role: " + assumeRole); } @SuppressWarnings("unused") @RequirePOST public FormValidation doLoginCheck(@QueryParameter String name, @QueryParameter String accessKey, - @QueryParameter Secret secretKey, @QueryParameter String assumeRole, @QueryParameter boolean useRole) { + @QueryParameter Secret secretKey, @QueryParameter String assumeRole, + @QueryParameter boolean useRole) { Jenkins.get().checkPermission(Jenkins.ADMINISTER); final String checkedName = Util.fixNull(name); final String checkedAccessKey = Util.fixNull(accessKey); + final String checkedAssumeRole = Util.fixNull(assumeRole); final String checkedSecretKey = secretKey != null ? secretKey.getPlainText() : ""; final boolean couldBeValidated = !checkedName.isEmpty() && !checkedAccessKey.isEmpty() && !checkedSecretKey.isEmpty(); @@ -550,6 +551,11 @@ public FormValidation doLoginCheck(@QueryParameter String name, @QueryParameter return FormValidation.ok(); } + if (!checkedAssumeRole.isEmpty()) { + return doCheckAssumeRole(checkedAssumeRole); + } + + if (checkedAccessKey.isEmpty()) { return FormValidation.ok("Please, enter accessKey"); } diff --git a/src/main/resources/hudson/plugins/s3/S3BucketPublisher/global.jelly b/src/main/resources/hudson/plugins/s3/S3BucketPublisher/global.jelly index 58311c9a..12cbf800 100644 --- a/src/main/resources/hudson/plugins/s3/S3BucketPublisher/global.jelly +++ b/src/main/resources/hudson/plugins/s3/S3BucketPublisher/global.jelly @@ -12,7 +12,7 @@ - + @@ -22,7 +22,7 @@ + method="loginCheck" with="name,useRole,accessKey,secretKey,assumeRole" /> From 552500b73af1892a8ab01fe874efac8ebaa66bbb Mon Sep 17 00:00:00 2001 From: Lew Gordon Date: Wed, 15 Apr 2020 14:12:15 -0400 Subject: [PATCH 10/11] Remove unneeded Jenkinsfile. --- ci.jenkinsfile | 12 ------------ 1 file changed, 12 deletions(-) delete mode 100644 ci.jenkinsfile diff --git a/ci.jenkinsfile b/ci.jenkinsfile deleted file mode 100644 index f52d1ee8..00000000 --- a/ci.jenkinsfile +++ /dev/null @@ -1,12 +0,0 @@ -@Library('pipeline-library@master') -import com.genesys.jenkins.Service - -node('dev_mesos') { - final serviceBuild = new com.genesys.jenkins.Service() - checkout scm - - serviceBuild.runMavenGoals('clean verify') - - archiveArtifacts 'target/s3.hpi' -} - From d6212f020240d0c7a021dbc2c3baf821a9acedcb Mon Sep 17 00:00:00 2001 From: Anthony Green Date: Wed, 13 May 2020 15:18:48 +0100 Subject: [PATCH 11/11] provide for objects to have canned ACL applied --- src/main/java/hudson/plugins/s3/Entry.java | 17 ++++++++++++++++- .../hudson/plugins/s3/S3BucketPublisher.java | 2 +- src/main/java/hudson/plugins/s3/S3Profile.java | 5 +++-- src/main/java/hudson/plugins/s3/Uploads.java | 7 ++++++- .../s3/callable/S3BaseUploadCallable.java | 12 +++++++++++- .../plugins/s3/callable/S3GzipCallable.java | 8 +++++--- .../plugins/s3/callable/S3UploadCallable.java | 8 +++++--- .../hudson/plugins/s3/Entry/config.jelly | 3 +++ .../hudson/plugins/s3/Entry/help-cannedACL.html | 3 +++ src/test/java/hudson/plugins/s3/S3Test.java | 3 ++- 10 files changed, 55 insertions(+), 13 deletions(-) create mode 100644 src/main/resources/hudson/plugins/s3/Entry/help-cannedACL.html diff --git a/src/main/java/hudson/plugins/s3/Entry.java b/src/main/java/hudson/plugins/s3/Entry.java index cecc5fd2..05a22a2e 100644 --- a/src/main/java/hudson/plugins/s3/Entry.java +++ b/src/main/java/hudson/plugins/s3/Entry.java @@ -3,6 +3,7 @@ import com.amazonaws.regions.Region; import com.amazonaws.regions.RegionUtils; import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.model.CannedAccessControlList; import org.kohsuke.stapler.DataBoundConstructor; import hudson.Extension; import hudson.model.Describable; @@ -64,6 +65,11 @@ public final class Entry implements Describable { */ public boolean useServerSideEncryption; + /** + * Use this canned ACL when uploading + */ + public String cannedACL; + /** * Flatten directories */ @@ -93,7 +99,7 @@ public final class Entry implements Describable { @DataBoundConstructor public Entry(String bucket, String sourceFile, String excludedFile, String storageClass, String selectedRegion, boolean noUploadOnFailure, boolean uploadFromSlave, boolean managedArtifacts, - boolean useServerSideEncryption, boolean flatten, boolean gzipFiles, boolean keepForever, + boolean useServerSideEncryption, String cannedACL, boolean flatten, boolean gzipFiles, boolean keepForever, boolean showDirectlyInBrowser, List userMetadata) { this.bucket = bucket; this.sourceFile = sourceFile; @@ -104,6 +110,7 @@ public Entry(String bucket, String sourceFile, String excludedFile, String stora this.uploadFromSlave = uploadFromSlave; this.managedArtifacts = managedArtifacts; this.useServerSideEncryption = useServerSideEncryption; + this.cannedACL = cannedACL; this.flatten = flatten; this.gzipFiles = gzipFiles; this.keepForever = keepForever; @@ -134,6 +141,14 @@ public ListBoxModel doFillStorageClassItems() { return model; } + public ListBoxModel doFillCannedACLItems() { + final ListBoxModel model = new ListBoxModel(); + for (CannedAccessControlList cannedAccessControlList : CannedAccessControlList.values()) { + model.add(cannedAccessControlList.toString(), cannedAccessControlList.name()); + } + return model; + } + public ListBoxModel doFillSelectedRegionItems() { final ListBoxModel model = new ListBoxModel(); for (Region r : regions) { diff --git a/src/main/java/hudson/plugins/s3/S3BucketPublisher.java b/src/main/java/hudson/plugins/s3/S3BucketPublisher.java index ba1495c9..7edff43e 100644 --- a/src/main/java/hudson/plugins/s3/S3BucketPublisher.java +++ b/src/main/java/hudson/plugins/s3/S3BucketPublisher.java @@ -291,7 +291,7 @@ public void perform(@Nonnull Run run, @Nonnull FilePath ws, @Nonnull Launc final Map escapedMetadata = buildMetadata(envVars, entry); final List records = Lists.newArrayList(); - final List fingerprints = profile.upload(run, bucket, paths, filenames, escapedMetadata, storageClass, selRegion, entry.uploadFromSlave, entry.managedArtifacts, entry.useServerSideEncryption, entry.gzipFiles); + final List fingerprints = profile.upload(run, bucket, paths, filenames, escapedMetadata, storageClass, selRegion, entry.uploadFromSlave, entry.managedArtifacts, entry.useServerSideEncryption, entry.cannedACL, entry.gzipFiles); for (FingerprintRecord fingerprintRecord : fingerprints) { records.add(fingerprintRecord); diff --git a/src/main/java/hudson/plugins/s3/S3Profile.java b/src/main/java/hudson/plugins/s3/S3Profile.java index 84485f6a..d5bd599a 100644 --- a/src/main/java/hudson/plugins/s3/S3Profile.java +++ b/src/main/java/hudson/plugins/s3/S3Profile.java @@ -134,6 +134,7 @@ public List upload(Run run, final boolean uploadFromSlave, final boolean managedArtifacts, final boolean useServerSideEncryption, + final String cannedACL, final boolean gzipFiles) throws IOException, InterruptedException { final List fingerprints = new ArrayList<>(fileNames.size()); @@ -155,10 +156,10 @@ public List upload(Run run, final MasterSlaveCallable upload; if (gzipFiles) { upload = new S3GzipCallable(accessKey, secretKey, useRole, assumeRole, dest, - userMetadata, storageClass, selregion, useServerSideEncryption, getProxy()); + userMetadata, storageClass, selregion, useServerSideEncryption, cannedACL, getProxy()); } else { upload = new S3UploadCallable(accessKey, secretKey, useRole, assumeRole, dest, userMetadata, - storageClass, selregion, useServerSideEncryption, getProxy()); + storageClass, selregion, useServerSideEncryption, cannedACL, getProxy()); } final FingerprintRecord fingerprintRecord = repeat(maxUploadRetries, uploadRetryTime, dest, new Callable() { diff --git a/src/main/java/hudson/plugins/s3/Uploads.java b/src/main/java/hudson/plugins/s3/Uploads.java index c7d6a02a..d28e4c10 100644 --- a/src/main/java/hudson/plugins/s3/Uploads.java +++ b/src/main/java/hudson/plugins/s3/Uploads.java @@ -5,6 +5,7 @@ import com.amazonaws.services.s3.model.PutObjectRequest; import com.amazonaws.services.s3.transfer.TransferManager; import com.amazonaws.services.s3.transfer.Upload; +import com.amazonaws.services.s3.model.CannedAccessControlList; import hudson.FilePath; import java.io.IOException; @@ -21,9 +22,13 @@ private Uploads() {} private final transient HashMap startedUploads = new HashMap<>(); private final transient HashMap openedStreams = new HashMap<>(); - public Upload startUploading(TransferManager manager, FilePath file, InputStream inputsStream, String bucketName, String objectName, ObjectMetadata metadata) throws AmazonServiceException { + public Upload startUploading(TransferManager manager, FilePath file, InputStream inputsStream, String bucketName, String objectName, ObjectMetadata metadata, CannedAccessControlList cannedAcl) throws AmazonServiceException { final PutObjectRequest request = new PutObjectRequest(bucketName, objectName, inputsStream, metadata); + if (!cannedAcl.equals(CannedAccessControlList.Private)) { + request.setCannedAcl(cannedAcl); + } + // Set the buffer size (ReadLimit) equal to the multipart upload size, // allowing us to resend data if the connection breaks. request.getRequestClientOptions().setReadLimit(MULTIPART_UPLOAD_THRESHOLD); diff --git a/src/main/java/hudson/plugins/s3/callable/S3BaseUploadCallable.java b/src/main/java/hudson/plugins/s3/callable/S3BaseUploadCallable.java index 7cde64f2..1313f0a8 100644 --- a/src/main/java/hudson/plugins/s3/callable/S3BaseUploadCallable.java +++ b/src/main/java/hudson/plugins/s3/callable/S3BaseUploadCallable.java @@ -2,6 +2,7 @@ import com.amazonaws.services.s3.internal.Mimetypes; import com.amazonaws.services.s3.model.ObjectMetadata; +import com.amazonaws.services.s3.model.CannedAccessControlList; import hudson.FilePath; import hudson.ProxyConfiguration; import hudson.plugins.s3.Destination; @@ -21,16 +22,18 @@ public abstract class S3BaseUploadCallable extends S3Callable { private final String storageClass; private final Map userMetadata; private final boolean useServerSideEncryption; + private final String cannedACLName; public S3BaseUploadCallable(String accessKey, Secret secretKey, boolean useRole, String assumeRole, Destination dest, Map userMetadata, String storageClass, String selregion, - boolean useServerSideEncryption, ProxyConfiguration proxy) { + boolean useServerSideEncryption, String cannedACLName, ProxyConfiguration proxy) { super(accessKey, secretKey, useRole, assumeRole, selregion, proxy); this.dest = dest; this.storageClass = storageClass; this.userMetadata = userMetadata; this.useServerSideEncryption = useServerSideEncryption; + this.cannedACLName = cannedACLName; } /** @@ -89,4 +92,11 @@ protected ObjectMetadata buildMetadata(FilePath filePath) throws IOException, In public Destination getDest() { return dest; } + + public CannedAccessControlList getCannedAcl() { + if (cannedACLName == null) { + return CannedAccessControlList.Private; + } + return CannedAccessControlList.valueOf(cannedACLName); + } } diff --git a/src/main/java/hudson/plugins/s3/callable/S3GzipCallable.java b/src/main/java/hudson/plugins/s3/callable/S3GzipCallable.java index b2dac036..74a18e2c 100644 --- a/src/main/java/hudson/plugins/s3/callable/S3GzipCallable.java +++ b/src/main/java/hudson/plugins/s3/callable/S3GzipCallable.java @@ -4,6 +4,7 @@ import com.amazonaws.services.s3.transfer.Upload; import com.amazonaws.event.ProgressListener; import com.amazonaws.event.ProgressEvent; +import com.amazonaws.services.s3.model.CannedAccessControlList; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import hudson.FilePath; import hudson.ProxyConfiguration; @@ -23,8 +24,8 @@ import java.util.zip.GZIPOutputStream; public final class S3GzipCallable extends S3BaseUploadCallable implements MasterSlaveCallable { - public S3GzipCallable(String accessKey, Secret secretKey, boolean useRole, String assumeRole, Destination dest, Map userMetadata, String storageClass, String selregion, boolean useServerSideEncryption, ProxyConfiguration proxy) { - super(accessKey, secretKey, useRole, assumeRole, dest, userMetadata, storageClass, selregion, useServerSideEncryption, proxy); + public S3GzipCallable(String accessKey, Secret secretKey, boolean useRole, String assumeRole, Destination dest, Map userMetadata, String storageClass, String selregion, boolean useServerSideEncryption, String cannedACL, ProxyConfiguration proxy) { + super(accessKey, secretKey, useRole, assumeRole, dest, userMetadata, storageClass, selregion, useServerSideEncryption, cannedACL, proxy); } // Return a File containing the gzipped contents of the input file. @@ -72,11 +73,12 @@ public String invoke(FilePath file) throws IOException, InterruptedException { Upload upload = null; try (final InputStream gzippedStream = new FileInputStream(localFile)) { + final CannedAccessControlList cannedAcl = getCannedAcl(); final ObjectMetadata metadata = buildMetadata(file); metadata.setContentEncoding("gzip"); metadata.setContentLength(localFile.length()); - upload = Uploads.getInstance().startUploading(getTransferManager(), file, gzippedStream, getDest().bucketName, getDest().objectName, metadata); + upload = Uploads.getInstance().startUploading(getTransferManager(), file, gzippedStream, getDest().bucketName, getDest().objectName, metadata, cannedAcl); String md5 = MD5.generateFromFile(localFile); diff --git a/src/main/java/hudson/plugins/s3/callable/S3UploadCallable.java b/src/main/java/hudson/plugins/s3/callable/S3UploadCallable.java index 3339c32e..cefb11f7 100644 --- a/src/main/java/hudson/plugins/s3/callable/S3UploadCallable.java +++ b/src/main/java/hudson/plugins/s3/callable/S3UploadCallable.java @@ -1,6 +1,7 @@ package hudson.plugins.s3.callable; import com.amazonaws.services.s3.model.ObjectMetadata; +import com.amazonaws.services.s3.model.CannedAccessControlList; import hudson.FilePath; import hudson.ProxyConfiguration; import hudson.plugins.s3.Destination; @@ -14,8 +15,8 @@ public final class S3UploadCallable extends S3BaseUploadCallable implements MasterSlaveCallable { private static final long serialVersionUID = 1L; - public S3UploadCallable(String accessKey, Secret secretKey, boolean useRole, String assumeRole, Destination dest, Map userMetadata, String storageClass, String selregion, boolean useServerSideEncryption, ProxyConfiguration proxy) { - super(accessKey, secretKey, useRole, assumeRole, dest, userMetadata, storageClass, selregion, useServerSideEncryption, proxy); + public S3UploadCallable(String accessKey, Secret secretKey, boolean useRole, String assumeRole, Destination dest, Map userMetadata, String storageClass, String selregion, boolean useServerSideEncryption, String cannedACL, ProxyConfiguration proxy) { + super(accessKey, secretKey, useRole, assumeRole, dest, userMetadata, storageClass, selregion, useServerSideEncryption, cannedACL, proxy); } /** @@ -24,8 +25,9 @@ public S3UploadCallable(String accessKey, Secret secretKey, boolean useRole, Str @Override public String invoke(FilePath file) throws IOException, InterruptedException { final ObjectMetadata metadata = buildMetadata(file); + final CannedAccessControlList cannedAcl = getCannedAcl(); - Uploads.getInstance().startUploading(getTransferManager(), file, file.read(), getDest().bucketName, getDest().objectName, metadata); + Uploads.getInstance().startUploading(getTransferManager(), file, file.read(), getDest().bucketName, getDest().objectName, metadata, cannedAcl); return MD5.generateFromFile(file); } diff --git a/src/main/resources/hudson/plugins/s3/Entry/config.jelly b/src/main/resources/hudson/plugins/s3/Entry/config.jelly index 7046a7ed..bf462b3d 100644 --- a/src/main/resources/hudson/plugins/s3/Entry/config.jelly +++ b/src/main/resources/hudson/plugins/s3/Entry/config.jelly @@ -28,6 +28,9 @@ + + + diff --git a/src/main/resources/hudson/plugins/s3/Entry/help-cannedACL.html b/src/main/resources/hudson/plugins/s3/Entry/help-cannedACL.html new file mode 100644 index 00000000..863939a4 --- /dev/null +++ b/src/main/resources/hudson/plugins/s3/Entry/help-cannedACL.html @@ -0,0 +1,3 @@ +
+ Choose a canned ACL for uploaded objects. See possible values here. AWS default value for new objects is private. Setting any other value requires to be granted PutObjectAcl. +
diff --git a/src/test/java/hudson/plugins/s3/S3Test.java b/src/test/java/hudson/plugins/s3/S3Test.java index d0fcf01b..510fa1da 100644 --- a/src/test/java/hudson/plugins/s3/S3Test.java +++ b/src/test/java/hudson/plugins/s3/S3Test.java @@ -102,7 +102,7 @@ public void dontSetBuildResultTest() throws Exception { } private Entry entryForFile(String fileName) { - return new Entry("bucket", fileName, "", "", "", false, false, true, false, false, false, false, false, null); + return new Entry("bucket", fileName, "", "", "", false, false, true, false, "", false, false, false, false, null); } private Builder stepCreatingFile(String fileName) { @@ -134,6 +134,7 @@ private S3Profile mockS3Profile(String profileName) throws IOException, Interrup Mockito.anyBoolean(), Mockito.anyBoolean(), Mockito.anyBoolean(), + Mockito.anyString(), Mockito.anyBoolean() )).thenReturn(newArrayList(new FingerprintRecord(true, "bucket", "path", "eu-west-1", "xxxx"))); return profile;