From 30535d89a7553b1457102d9b0b5b7865adba16f2 Mon Sep 17 00:00:00 2001 From: Emanuele Zattin Date: Fri, 4 Jul 2014 15:34:56 +0200 Subject: [PATCH 1/2] Add support for redirects cleaning up code --- src/main/java/hudson/plugins/s3/Redirect.java | 7 +++++ .../hudson/plugins/s3/S3BucketPublisher.java | 26 ++++++++++++++++--- .../java/hudson/plugins/s3/S3Profile.java | 5 ++-- .../plugins/s3/callable/S3UploadCallable.java | 13 ++++++++-- .../plugins/s3/S3BucketPublisher/config.jelly | 21 +++++++++++++++ src/main/webapp/help-redirect-bucket.html | 3 +++ .../webapp/help-redirect-destination.html | 3 +++ src/main/webapp/help-redirect-source.html | 3 +++ 8 files changed, 74 insertions(+), 7 deletions(-) create mode 100644 src/main/java/hudson/plugins/s3/Redirect.java create mode 100644 src/main/webapp/help-redirect-bucket.html create mode 100644 src/main/webapp/help-redirect-destination.html create mode 100644 src/main/webapp/help-redirect-source.html diff --git a/src/main/java/hudson/plugins/s3/Redirect.java b/src/main/java/hudson/plugins/s3/Redirect.java new file mode 100644 index 00000000..980020b1 --- /dev/null +++ b/src/main/java/hudson/plugins/s3/Redirect.java @@ -0,0 +1,7 @@ +package hudson.plugins.s3; + +public final class Redirect { + public String bucket; + public String source; + public String destination; +} diff --git a/src/main/java/hudson/plugins/s3/S3BucketPublisher.java b/src/main/java/hudson/plugins/s3/S3BucketPublisher.java index 273c871c..dc1063a7 100644 --- a/src/main/java/hudson/plugins/s3/S3BucketPublisher.java +++ b/src/main/java/hudson/plugins/s3/S3BucketPublisher.java @@ -30,8 +30,10 @@ import java.io.PrintStream; import java.util.ArrayList; import java.util.Collection; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.logging.Level; import java.util.logging.Logger; @@ -44,8 +46,10 @@ public final class S3BucketPublisher extends Recorder implements Describable entries; - + private boolean dontWaitForConcurrentBuildCompletion; + + private List redirects = new ArrayList(); /** * User metadata key/value pairs to tag the upload with. @@ -112,6 +116,14 @@ public static S3Profile getProfile(String profileName) { return null; } + public List getRedirects() { + return redirects; + } + + public void setRedirects(List redirects) { + this.redirects = redirects; + } + @Override public Collection getProjectActions(AbstractProject project) { return ImmutableList.of(new S3ArtifactsProjectAction(project)); @@ -174,12 +186,20 @@ public boolean perform(AbstractBuild build, Util.replaceMacro(metadataPair.value, envVars)) ); } + Set escapedRedirects = new HashSet(); + for (Redirect redirect : redirects) { + Redirect escapedRedirect = new Redirect(); + escapedRedirect.bucket = Util.replaceMacro(redirect.bucket, envVars); + escapedRedirect.source = Util.replaceMacro(redirect.source, envVars); + escapedRedirect.destination = Util.replaceMacro(redirect.destination, envVars); + escapedRedirects.add(escapedRedirect); + } List records = Lists.newArrayList(); for (FilePath src : paths) { - log(listener.getLogger(), "bucket=" + bucket + ", file=" + src.getName() + " region=" + selRegion + ", upload from slave=" + entry.uploadFromSlave + " managed="+ entry.managedArtifacts + " , server encryption "+entry.useServerSideEncryption); - records.add(profile.upload(build, listener, bucket, src, searchPathLength, escapedUserMetadata, storageClass, selRegion, entry.uploadFromSlave, entry.managedArtifacts, entry.useServerSideEncryption, entry.flatten)); + log(listener.getLogger(), String.format("bucket=%s, file=%s region=%s, upload from slave=%s managed=%s , server encryption %s", bucket, src.getName(), selRegion, entry.uploadFromSlave, entry.managedArtifacts, entry.useServerSideEncryption)); + records.add(profile.upload(build, listener, bucket, src, searchPathLength, escapedUserMetadata, escapedRedirects, storageClass, selRegion, entry.uploadFromSlave, entry.managedArtifacts, entry.useServerSideEncryption, entry.flatten)); } if (entry.managedArtifacts) { artifacts.addAll(records); diff --git a/src/main/java/hudson/plugins/s3/S3Profile.java b/src/main/java/hudson/plugins/s3/S3Profile.java index 651322b3..4c0ba8ad 100644 --- a/src/main/java/hudson/plugins/s3/S3Profile.java +++ b/src/main/java/hudson/plugins/s3/S3Profile.java @@ -9,6 +9,7 @@ import java.net.URL; import java.util.Date; import java.util.List; +import java.util.Set; import java.util.regex.Pattern; import jenkins.model.Jenkins; @@ -150,7 +151,7 @@ public void check() throws Exception { getClient().listBuckets(); } - public FingerprintRecord upload(AbstractBuild build, final BuildListener listener, String bucketName, FilePath filePath, int searchPathLength, List userMetadata, + public FingerprintRecord upload(AbstractBuild build, final BuildListener listener, String bucketName, FilePath filePath, int searchPathLength, List userMetadata, Set redirects, String storageClass, String selregion, boolean uploadFromSlave, boolean managedArtifacts,boolean useServerSideEncryption, boolean flatten) throws IOException, InterruptedException { if (filePath.isDirectory()) { throw new IOException(filePath + " is a directory"); @@ -174,7 +175,7 @@ public FingerprintRecord upload(AbstractBuild build, final BuildListener li while (true) { try { - S3UploadCallable callable = new S3UploadCallable(produced, getClient(), dest, userMetadata, storageClass, selregion,useServerSideEncryption); + S3UploadCallable callable = new S3UploadCallable(produced, getClient(), dest, userMetadata, redirects, storageClass, selregion, useServerSideEncryption); if (uploadFromSlave) { return filePath.act(callable); } else { diff --git a/src/main/java/hudson/plugins/s3/callable/S3UploadCallable.java b/src/main/java/hudson/plugins/s3/callable/S3UploadCallable.java index 4503a4a3..8fe8cb18 100644 --- a/src/main/java/hudson/plugins/s3/callable/S3UploadCallable.java +++ b/src/main/java/hudson/plugins/s3/callable/S3UploadCallable.java @@ -6,6 +6,7 @@ import hudson.plugins.s3.Destination; import hudson.plugins.s3.FingerprintRecord; import hudson.plugins.s3.MetadataPair; +import hudson.plugins.s3.Redirect; import hudson.remoting.VirtualChannel; import hudson.util.Secret; @@ -16,6 +17,7 @@ import java.text.SimpleDateFormat; import java.util.Date; import java.util.List; +import java.util.Set; import com.amazonaws.regions.Region; import com.amazonaws.regions.RegionUtils; @@ -30,16 +32,18 @@ public class S3UploadCallable extends AbstractS3Callable implements FileCallable private final Destination dest; private final String storageClass; private final List userMetadata; + private final Set redirects; private final String selregion; private final boolean produced; private final boolean useServerSideEncryption; - public S3UploadCallable(boolean produced, AmazonS3Client client, Destination dest, List userMetadata, String storageClass, - String selregion, boolean useServerSideEncryption) { + public S3UploadCallable(boolean produced, AmazonS3Client client, Destination dest, List userMetadata, Set redirects, + String storageClass, String selregion, boolean useServerSideEncryption) { super(client); this.dest = dest; this.storageClass = storageClass; this.userMetadata = userMetadata; + this.redirects = redirects; this.selregion = selregion; this.produced = produced; this.useServerSideEncryption = useServerSideEncryption; @@ -108,6 +112,11 @@ public FingerprintRecord invoke(FilePath file) throws IOException, InterruptedEx final PutObjectRequest request = new PutObjectRequest(dest.bucketName, dest.objectName, localFile) .withMetadata(buildMetadata(file)); final PutObjectResult result = getClient().putObject(request); + + for (Redirect redirect : redirects) { + getClient().setObjectRedirectLocation(redirect.bucket, redirect.source, redirect.destination); + } + return new FingerprintRecord(produced, dest.bucketName, file.getName(), result.getETag()); } finally { if (os != null) { diff --git a/src/main/resources/hudson/plugins/s3/S3BucketPublisher/config.jelly b/src/main/resources/hudson/plugins/s3/S3BucketPublisher/config.jelly index 84edb628..c69a8882 100644 --- a/src/main/resources/hudson/plugins/s3/S3BucketPublisher/config.jelly +++ b/src/main/resources/hudson/plugins/s3/S3BucketPublisher/config.jelly @@ -28,4 +28,25 @@ + + + + + + + + + + + + + + +
+ +
+ +
+ + diff --git a/src/main/webapp/help-redirect-bucket.html b/src/main/webapp/help-redirect-bucket.html new file mode 100644 index 00000000..268b7a7e --- /dev/null +++ b/src/main/webapp/help-redirect-bucket.html @@ -0,0 +1,3 @@ +
+ The bucket in which the source file is located. +
\ No newline at end of file diff --git a/src/main/webapp/help-redirect-destination.html b/src/main/webapp/help-redirect-destination.html new file mode 100644 index 00000000..a90b2bc8 --- /dev/null +++ b/src/main/webapp/help-redirect-destination.html @@ -0,0 +1,3 @@ +
+ Destination of the redirect. +
\ No newline at end of file diff --git a/src/main/webapp/help-redirect-source.html b/src/main/webapp/help-redirect-source.html new file mode 100644 index 00000000..ddaf99b9 --- /dev/null +++ b/src/main/webapp/help-redirect-source.html @@ -0,0 +1,3 @@ +
+ The key (path in the bucket) of the file to be redirected. +
\ No newline at end of file From b20cbfc843faaed079630dd73add572d0b99ed38 Mon Sep 17 00:00:00 2001 From: Ray Huang Date: Fri, 13 Feb 2015 16:26:28 -0800 Subject: [PATCH 2/2] Rebased and fixed redirect functionality Rebased off master and added the following enhancements over the original patch: - fixed bug where redirect gets uploaded extra times - fixed bug where redirect does not get created (if key does not already exist) - added clientside validation --- src/main/java/hudson/plugins/s3/Redirect.java | 79 ++++++++++++++++++- .../hudson/plugins/s3/S3BucketPublisher.java | 24 ++++-- .../java/hudson/plugins/s3/S3Profile.java | 5 +- .../plugins/s3/callable/S3UploadCallable.java | 11 +-- .../hudson/plugins/s3/Redirect/config.jelly | 11 +++ .../plugins/s3/Redirect/help-bucket.html} | 0 .../hudson/plugins/s3/Redirect/help-key.html} | 0 .../s3/Redirect/help-redirectLocation.html | 3 + .../plugins/s3/S3BucketPublisher/config.jelly | 25 ++---- .../webapp/help-redirect-destination.html | 3 - 10 files changed, 115 insertions(+), 46 deletions(-) create mode 100644 src/main/resources/hudson/plugins/s3/Redirect/config.jelly rename src/main/{webapp/help-redirect-bucket.html => resources/hudson/plugins/s3/Redirect/help-bucket.html} (100%) rename src/main/{webapp/help-redirect-source.html => resources/hudson/plugins/s3/Redirect/help-key.html} (100%) create mode 100644 src/main/resources/hudson/plugins/s3/Redirect/help-redirectLocation.html delete mode 100644 src/main/webapp/help-redirect-destination.html diff --git a/src/main/java/hudson/plugins/s3/Redirect.java b/src/main/java/hudson/plugins/s3/Redirect.java index 980020b1..0dbc1cef 100644 --- a/src/main/java/hudson/plugins/s3/Redirect.java +++ b/src/main/java/hudson/plugins/s3/Redirect.java @@ -1,7 +1,78 @@ package hudson.plugins.s3; -public final class Redirect { - public String bucket; - public String source; - public String destination; +import hudson.Extension; +import hudson.model.AbstractDescribableImpl; +import hudson.model.Descriptor; +import hudson.util.FormValidation; +import org.kohsuke.stapler.DataBoundConstructor; +import org.kohsuke.stapler.QueryParameter; + +import java.net.URI; +import java.net.URISyntaxException; + +public final class Redirect extends AbstractDescribableImpl { + public final String bucket; + public final String key; + public final String redirectLocation; + + public String getKey() { + return this.key; + } + + public String getRedirectLocation() { + return this.redirectLocation; + } + + @DataBoundConstructor + public Redirect(String bucket, String key, String redirectLocation) { + this.bucket = bucket; + this.key = key; + this.redirectLocation = redirectLocation; + } + + @Extension + public static class DescriptorImpl extends Descriptor { + public String getDisplayName() { + return "S3 Redirect"; + } + + public FormValidation doCheckKey(@QueryParameter String value) { + if (value.isEmpty()) { + return FormValidation.warning("This field cannot be empty."); + } + + if (value.charAt(0) == '/') { + return FormValidation.error("Key should not have a leading slash `/`."); + } + + return FormValidation.ok(); + } + + + /** + * A valid redirect is a URI that is either: + * 1. has a HTTP or HTTPS Scheme + * 2. does not have a scheme and points to a key in the same bucket. + */ + public FormValidation doCheckRedirectLocation(@QueryParameter String value) { + if (value.isEmpty()) { + return FormValidation.warning("This field cannot be empty."); + } + + try { + URI uri = new URI(value); + + if ("http".equals(uri.getScheme()) + || "https".equals(uri.getScheme()) + || ((uri.getScheme() == null) && (uri.getPath().charAt(0) == '/'))) { + return FormValidation.ok(); + } else { + return FormValidation.error( + "RedirectLocation must refer to `http`, `https` or to a key in the same bucket (beginning with '/')."); + } + } catch (URISyntaxException e) { + return FormValidation.error("Not a valid URI."); + } + } + } } diff --git a/src/main/java/hudson/plugins/s3/S3BucketPublisher.java b/src/main/java/hudson/plugins/s3/S3BucketPublisher.java index dc1063a7..a5f04400 100644 --- a/src/main/java/hudson/plugins/s3/S3BucketPublisher.java +++ b/src/main/java/hudson/plugins/s3/S3BucketPublisher.java @@ -1,6 +1,7 @@ package hudson.plugins.s3; import com.amazonaws.regions.Regions; +import com.amazonaws.services.s3.model.PutObjectRequest; import hudson.Extension; import hudson.FilePath; import hudson.Launcher; @@ -49,7 +50,7 @@ public final class S3BucketPublisher extends Recorder implements Describable redirects = new ArrayList(); + private List redirects; /** * User metadata key/value pairs to tag the upload with. @@ -57,7 +58,7 @@ public final class S3BucketPublisher extends Recorder implements Describable userMetadata; @DataBoundConstructor - public S3BucketPublisher(String profileName, List entries, List userMetadata, + public S3BucketPublisher(String profileName, List entries, List userMetadata, List redirects, boolean dontWaitForConcurrentBuildCompletion) { if (profileName == null) { // defaults to the first one @@ -68,7 +69,8 @@ public S3BucketPublisher(String profileName, List entries, List(); this.userMetadata = userMetadata; @@ -188,10 +190,10 @@ public boolean perform(AbstractBuild build, } Set escapedRedirects = new HashSet(); for (Redirect redirect : redirects) { - Redirect escapedRedirect = new Redirect(); - escapedRedirect.bucket = Util.replaceMacro(redirect.bucket, envVars); - escapedRedirect.source = Util.replaceMacro(redirect.source, envVars); - escapedRedirect.destination = Util.replaceMacro(redirect.destination, envVars); + Redirect escapedRedirect = new Redirect( + Util.replaceMacro(redirect.bucket, envVars), + Util.replaceMacro(redirect.key, envVars), + Util.replaceMacro(redirect.redirectLocation, envVars)); escapedRedirects.add(escapedRedirect); } @@ -199,8 +201,14 @@ public boolean perform(AbstractBuild build, for (FilePath src : paths) { log(listener.getLogger(), String.format("bucket=%s, file=%s region=%s, upload from slave=%s managed=%s , server encryption %s", bucket, src.getName(), selRegion, entry.uploadFromSlave, entry.managedArtifacts, entry.useServerSideEncryption)); - records.add(profile.upload(build, listener, bucket, src, searchPathLength, escapedUserMetadata, escapedRedirects, storageClass, selRegion, entry.uploadFromSlave, entry.managedArtifacts, entry.useServerSideEncryption, entry.flatten)); + records.add(profile.upload(build, listener, bucket, src, searchPathLength, escapedUserMetadata, storageClass, selRegion, entry.uploadFromSlave, entry.managedArtifacts, entry.useServerSideEncryption, entry.flatten)); } + + for (Redirect r : escapedRedirects) { + log(listener.getLogger(), String.format("S3 redirect: bucket=%s, source=%s, redirect=%s", r.bucket, r.key, r.redirectLocation)); + profile.getClient().putObject(new PutObjectRequest(r.bucket, r.key, r.redirectLocation)); + } + if (entry.managedArtifacts) { artifacts.addAll(records); diff --git a/src/main/java/hudson/plugins/s3/S3Profile.java b/src/main/java/hudson/plugins/s3/S3Profile.java index 4c0ba8ad..26f9ddbf 100644 --- a/src/main/java/hudson/plugins/s3/S3Profile.java +++ b/src/main/java/hudson/plugins/s3/S3Profile.java @@ -9,7 +9,6 @@ import java.net.URL; import java.util.Date; import java.util.List; -import java.util.Set; import java.util.regex.Pattern; import jenkins.model.Jenkins; @@ -151,7 +150,7 @@ public void check() throws Exception { getClient().listBuckets(); } - public FingerprintRecord upload(AbstractBuild build, final BuildListener listener, String bucketName, FilePath filePath, int searchPathLength, List userMetadata, Set redirects, + public FingerprintRecord upload(AbstractBuild build, final BuildListener listener, String bucketName, FilePath filePath, int searchPathLength, List userMetadata, String storageClass, String selregion, boolean uploadFromSlave, boolean managedArtifacts,boolean useServerSideEncryption, boolean flatten) throws IOException, InterruptedException { if (filePath.isDirectory()) { throw new IOException(filePath + " is a directory"); @@ -175,7 +174,7 @@ public FingerprintRecord upload(AbstractBuild build, final BuildListener li while (true) { try { - S3UploadCallable callable = new S3UploadCallable(produced, getClient(), dest, userMetadata, redirects, storageClass, selregion, useServerSideEncryption); + S3UploadCallable callable = new S3UploadCallable(produced, getClient(), dest, userMetadata, storageClass, selregion, useServerSideEncryption); if (uploadFromSlave) { return filePath.act(callable); } else { diff --git a/src/main/java/hudson/plugins/s3/callable/S3UploadCallable.java b/src/main/java/hudson/plugins/s3/callable/S3UploadCallable.java index 8fe8cb18..b9f69cef 100644 --- a/src/main/java/hudson/plugins/s3/callable/S3UploadCallable.java +++ b/src/main/java/hudson/plugins/s3/callable/S3UploadCallable.java @@ -6,9 +6,7 @@ import hudson.plugins.s3.Destination; import hudson.plugins.s3.FingerprintRecord; import hudson.plugins.s3.MetadataPair; -import hudson.plugins.s3.Redirect; import hudson.remoting.VirtualChannel; -import hudson.util.Secret; import java.io.File; import java.io.FileOutputStream; @@ -17,7 +15,6 @@ import java.text.SimpleDateFormat; import java.util.Date; import java.util.List; -import java.util.Set; import com.amazonaws.regions.Region; import com.amazonaws.regions.RegionUtils; @@ -32,18 +29,16 @@ public class S3UploadCallable extends AbstractS3Callable implements FileCallable private final Destination dest; private final String storageClass; private final List userMetadata; - private final Set redirects; private final String selregion; private final boolean produced; private final boolean useServerSideEncryption; - public S3UploadCallable(boolean produced, AmazonS3Client client, Destination dest, List userMetadata, Set redirects, + public S3UploadCallable(boolean produced, AmazonS3Client client, Destination dest, List userMetadata, String storageClass, String selregion, boolean useServerSideEncryption) { super(client); this.dest = dest; this.storageClass = storageClass; this.userMetadata = userMetadata; - this.redirects = redirects; this.selregion = selregion; this.produced = produced; this.useServerSideEncryption = useServerSideEncryption; @@ -113,10 +108,6 @@ public FingerprintRecord invoke(FilePath file) throws IOException, InterruptedEx .withMetadata(buildMetadata(file)); final PutObjectResult result = getClient().putObject(request); - for (Redirect redirect : redirects) { - getClient().setObjectRedirectLocation(redirect.bucket, redirect.source, redirect.destination); - } - return new FingerprintRecord(produced, dest.bucketName, file.getName(), result.getETag()); } finally { if (os != null) { diff --git a/src/main/resources/hudson/plugins/s3/Redirect/config.jelly b/src/main/resources/hudson/plugins/s3/Redirect/config.jelly new file mode 100644 index 00000000..b368aedf --- /dev/null +++ b/src/main/resources/hudson/plugins/s3/Redirect/config.jelly @@ -0,0 +1,11 @@ + + + + + + + + + + + diff --git a/src/main/webapp/help-redirect-bucket.html b/src/main/resources/hudson/plugins/s3/Redirect/help-bucket.html similarity index 100% rename from src/main/webapp/help-redirect-bucket.html rename to src/main/resources/hudson/plugins/s3/Redirect/help-bucket.html diff --git a/src/main/webapp/help-redirect-source.html b/src/main/resources/hudson/plugins/s3/Redirect/help-key.html similarity index 100% rename from src/main/webapp/help-redirect-source.html rename to src/main/resources/hudson/plugins/s3/Redirect/help-key.html diff --git a/src/main/resources/hudson/plugins/s3/Redirect/help-redirectLocation.html b/src/main/resources/hudson/plugins/s3/Redirect/help-redirectLocation.html new file mode 100644 index 00000000..26d38024 --- /dev/null +++ b/src/main/resources/hudson/plugins/s3/Redirect/help-redirectLocation.html @@ -0,0 +1,3 @@ +
+ Destination of the redirect. It must be prefixed with 'http://' or 'https://' or '/' +
\ No newline at end of file diff --git a/src/main/resources/hudson/plugins/s3/S3BucketPublisher/config.jelly b/src/main/resources/hudson/plugins/s3/S3BucketPublisher/config.jelly index c69a8882..90585ec2 100644 --- a/src/main/resources/hudson/plugins/s3/S3BucketPublisher/config.jelly +++ b/src/main/resources/hudson/plugins/s3/S3BucketPublisher/config.jelly @@ -30,23 +30,12 @@ - - - - - - - - - - - - -
- -
- -
- + + +
+ +
+ + diff --git a/src/main/webapp/help-redirect-destination.html b/src/main/webapp/help-redirect-destination.html deleted file mode 100644 index a90b2bc8..00000000 --- a/src/main/webapp/help-redirect-destination.html +++ /dev/null @@ -1,3 +0,0 @@ -
- Destination of the redirect. -
\ No newline at end of file