diff --git a/src/main/java/hudson/plugins/s3/S3BucketPublisher.java b/src/main/java/hudson/plugins/s3/S3BucketPublisher.java index 2195d8be..5dc352b3 100644 --- a/src/main/java/hudson/plugins/s3/S3BucketPublisher.java +++ b/src/main/java/hudson/plugins/s3/S3BucketPublisher.java @@ -46,6 +46,7 @@ import org.kohsuke.stapler.interceptor.RequirePOST; import software.amazon.awssdk.core.exception.SdkException; import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.model.ChecksumAlgorithm; import java.io.IOException; import java.io.PrintStream; @@ -53,6 +54,7 @@ import java.util.Collection; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.concurrent.CopyOnWriteArrayList; import java.util.logging.Level; @@ -69,6 +71,7 @@ public final class S3BucketPublisher extends Recorder implements SimpleBuildStep private boolean dontWaitForConcurrentBuildCompletion; private boolean dontSetBuildResultOnFailure; private int uploadTimeout = 30; // default 30 mins + private ChecksumAlgorithm checksumAlgorithm = ChecksumAlgorithm.CRC32; // SDK's default /** * In-memory representation of console log level. @@ -248,6 +251,23 @@ public void setUploadTimeout(int uploadTimeout) { this.uploadTimeout = Math.max(uploadTimeout, Uploads.MIN_UPLOAD_TIMEOUT); } + @DataBoundSetter + public void setChecksumAlgorithm(String checksumAlgorithm) { + if (checksumAlgorithm == null || checksumAlgorithm.isBlank()) { + this.checksumAlgorithm = ChecksumAlgorithm.CRC32; + return; + } + String normalized = checksumAlgorithm.toUpperCase(Locale.ROOT); + ChecksumAlgorithm algo = ChecksumAlgorithm.fromValue(normalized); + if (algo == ChecksumAlgorithm.UNKNOWN_TO_SDK_VERSION) { + throw new IllegalArgumentException("Unsupported checksum algorithm: " + checksumAlgorithm); + } else if (algo == ChecksumAlgorithm.CRC64_NVME) { + throw new UnsupportedOperationException("Checksum algorithm '" + checksumAlgorithm + "' requires AWS CRT dependency, which is currently unavailable." + + "\nUse another algorithm (CRC32, CRC32C, SHA1, SHA256)."); + } + this.checksumAlgorithm = algo; + } + private void log(final PrintStream logger, final String message) { log(Level.INFO, logger, message); } @@ -333,7 +353,7 @@ public void perform(@NonNull Run run, @NonNull FilePath ws, @NonNull Launc final Map escapedMetadata = buildMetadata(envVars, entry); final List records = Lists.newArrayList(); - final List fingerprints = profile.upload(run, bucket, paths, filenames, escapedMetadata, storageClass, selRegion, entry.uploadFromSlave, entry.managedArtifacts, entry.useServerSideEncryption, entry.gzipFiles, uploadTimeout); + final List fingerprints = profile.upload(run, bucket, paths, filenames, escapedMetadata, storageClass, selRegion, entry.uploadFromSlave, entry.managedArtifacts, entry.useServerSideEncryption, entry.gzipFiles, checksumAlgorithm, uploadTimeout); for (FingerprintRecord fingerprintRecord : fingerprints) { records.add(fingerprintRecord); diff --git a/src/main/java/hudson/plugins/s3/S3Profile.java b/src/main/java/hudson/plugins/s3/S3Profile.java index 2c9a52f1..f03aa0e4 100644 --- a/src/main/java/hudson/plugins/s3/S3Profile.java +++ b/src/main/java/hudson/plugins/s3/S3Profile.java @@ -16,6 +16,7 @@ import org.kohsuke.stapler.DataBoundConstructor; import org.kohsuke.stapler.DataBoundSetter; import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.ChecksumAlgorithm; import software.amazon.awssdk.services.s3.model.DeleteObjectRequest; import software.amazon.awssdk.services.s3.model.GetObjectRequest; import software.amazon.awssdk.services.s3.model.ListObjectsV2Request; @@ -163,6 +164,23 @@ public List upload(Run run, final boolean useServerSideEncryption, final boolean gzipFiles, final int uploadTimeout) throws IOException, InterruptedException { + return upload(run, bucketName, filePaths, fileNames, userMetadata, storageClass, selregion, + uploadFromSlave, managedArtifacts, useServerSideEncryption, gzipFiles, null, uploadTimeout); + } + + public List upload(Run run, + final String bucketName, + final List filePaths, + final List fileNames, + final Map userMetadata, + final String storageClass, + final String selregion, + final boolean uploadFromSlave, + final boolean managedArtifacts, + final boolean useServerSideEncryption, + final boolean gzipFiles, + final ChecksumAlgorithm checksumAlgorithm, + final int uploadTimeout) throws IOException, InterruptedException { final List fingerprints = new ArrayList<>(fileNames.size()); try { @@ -183,10 +201,10 @@ public List upload(Run run, final MasterSlaveCallable upload; if (gzipFiles) { upload = new S3GzipCallable(accessKey, secretKey, useRole, dest, userMetadata, - storageClass, selregion, useServerSideEncryption, getProxy(), usePathStyle); + storageClass, selregion, useServerSideEncryption, getProxy(), usePathStyle, checksumAlgorithm); } else { upload = new S3UploadCallable(accessKey, secretKey, useRole, dest, userMetadata, - storageClass, selregion, useServerSideEncryption, getProxy(), usePathStyle); + storageClass, selregion, useServerSideEncryption, getProxy(), usePathStyle, checksumAlgorithm); } final FingerprintRecord fingerprintRecord = repeat(maxUploadRetries, uploadRetryTime, dest, () -> { diff --git a/src/main/java/hudson/plugins/s3/callable/S3BaseUploadCallable.java b/src/main/java/hudson/plugins/s3/callable/S3BaseUploadCallable.java index 4d72a4f8..8a5fccfe 100644 --- a/src/main/java/hudson/plugins/s3/callable/S3BaseUploadCallable.java +++ b/src/main/java/hudson/plugins/s3/callable/S3BaseUploadCallable.java @@ -7,6 +7,7 @@ import hudson.plugins.s3.Destination; import hudson.remoting.VirtualChannel; import hudson.util.Secret; +import software.amazon.awssdk.services.s3.model.ChecksumAlgorithm; import software.amazon.awssdk.services.s3.model.PutObjectRequest; import java.io.File; @@ -15,6 +16,7 @@ import java.text.SimpleDateFormat; import java.util.Date; import java.util.Map; +import java.util.Objects; import java.util.function.Consumer; public abstract class S3BaseUploadCallable extends S3Callable { @@ -23,16 +25,23 @@ public abstract class S3BaseUploadCallable extends S3Callable { private final String storageClass; private final Map userMetadata; private final boolean useServerSideEncryption; - + private final ChecksumAlgorithm checksumAlgorithm; public S3BaseUploadCallable(String accessKey, Secret secretKey, boolean useRole, Destination dest, Map userMetadata, String storageClass, String selregion, boolean useServerSideEncryption, ProxyConfiguration proxy, boolean usePathStyle) { + this(accessKey, secretKey, useRole, dest, userMetadata, storageClass, selregion, useServerSideEncryption, proxy, usePathStyle, null); + } + + public S3BaseUploadCallable(String accessKey, Secret secretKey, boolean useRole, + Destination dest, Map userMetadata, String storageClass, String selregion, + boolean useServerSideEncryption, ProxyConfiguration proxy, boolean usePathStyle, ChecksumAlgorithm checksumAlgorithm) { super(accessKey, secretKey, useRole, selregion, proxy, usePathStyle); this.dest = dest; this.storageClass = storageClass; this.userMetadata = userMetadata; this.useServerSideEncryption = useServerSideEncryption; + this.checksumAlgorithm = checksumAlgorithm; } /** @@ -59,6 +68,7 @@ protected Uploads.Metadata buildMetadata(FilePath filePath) throws IOException, if (useServerSideEncryption) { metadata.sseCustomerAlgorithm("AES256"); } + metadata.checksumAlgorithm(Objects.requireNonNullElse(checksumAlgorithm, ChecksumAlgorithm.CRC32)); }; Uploads.Metadata metadata = new Uploads.Metadata(builder); metadata.setContentLength(contentLength); diff --git a/src/main/java/hudson/plugins/s3/callable/S3GzipCallable.java b/src/main/java/hudson/plugins/s3/callable/S3GzipCallable.java index 68fc0eba..66eefc45 100644 --- a/src/main/java/hudson/plugins/s3/callable/S3GzipCallable.java +++ b/src/main/java/hudson/plugins/s3/callable/S3GzipCallable.java @@ -8,6 +8,7 @@ import hudson.plugins.s3.Uploads; import hudson.util.Secret; import org.apache.commons.io.IOUtils; +import software.amazon.awssdk.services.s3.model.ChecksumAlgorithm; import software.amazon.awssdk.transfer.s3.model.Upload; import software.amazon.awssdk.transfer.s3.progress.TransferListener; @@ -23,7 +24,11 @@ public final class S3GzipCallable extends S3BaseUploadCallable implements MasterSlaveCallable { public S3GzipCallable(String accessKey, Secret secretKey, boolean useRole, Destination dest, Map userMetadata, String storageClass, String selregion, boolean useServerSideEncryption, ProxyConfiguration proxy, boolean usePathStyle) { - super(accessKey, secretKey, useRole, dest, userMetadata, storageClass, selregion, useServerSideEncryption, proxy, usePathStyle); + this(accessKey, secretKey, useRole, dest, userMetadata, storageClass, selregion, useServerSideEncryption, proxy, usePathStyle, null); + } + + public S3GzipCallable(String accessKey, Secret secretKey, boolean useRole, Destination dest, Map userMetadata, String storageClass, String selregion, boolean useServerSideEncryption, ProxyConfiguration proxy, boolean usePathStyle, ChecksumAlgorithm checksumAlgorithm) { + super(accessKey, secretKey, useRole, dest, userMetadata, storageClass, selregion, useServerSideEncryption, proxy, usePathStyle, checksumAlgorithm); } // Return a File containing the gzipped contents of the input file. diff --git a/src/main/java/hudson/plugins/s3/callable/S3UploadCallable.java b/src/main/java/hudson/plugins/s3/callable/S3UploadCallable.java index ad78426a..545633ea 100644 --- a/src/main/java/hudson/plugins/s3/callable/S3UploadCallable.java +++ b/src/main/java/hudson/plugins/s3/callable/S3UploadCallable.java @@ -6,6 +6,7 @@ import hudson.plugins.s3.MD5; import hudson.plugins.s3.Uploads; import hudson.util.Secret; +import software.amazon.awssdk.services.s3.model.ChecksumAlgorithm; import java.io.IOException; import java.util.Map; @@ -14,7 +15,11 @@ public final class S3UploadCallable extends S3BaseUploadCallable implements Mast private static final long serialVersionUID = 1L; public S3UploadCallable(String accessKey, Secret secretKey, boolean useRole, Destination dest, Map userMetadata, String storageClass, String selregion, boolean useServerSideEncryption, ProxyConfiguration proxy, boolean usePathStyle) { - super(accessKey, secretKey, useRole, dest, userMetadata, storageClass, selregion, useServerSideEncryption, proxy, usePathStyle); + this(accessKey, secretKey, useRole, dest, userMetadata, storageClass, selregion, useServerSideEncryption, proxy, usePathStyle, null); + } + + public S3UploadCallable(String accessKey, Secret secretKey, boolean useRole, Destination dest, Map userMetadata, String storageClass, String selregion, boolean useServerSideEncryption, ProxyConfiguration proxy, boolean usePathStyle, ChecksumAlgorithm checksumAlgorithm) { + super(accessKey, secretKey, useRole, dest, userMetadata, storageClass, selregion, useServerSideEncryption, proxy, usePathStyle, checksumAlgorithm); } /** diff --git a/src/test/java/hudson/plugins/s3/S3Test.java b/src/test/java/hudson/plugins/s3/S3Test.java index daa49f51..ad734fb5 100644 --- a/src/test/java/hudson/plugins/s3/S3Test.java +++ b/src/test/java/hudson/plugins/s3/S3Test.java @@ -19,6 +19,7 @@ import org.jvnet.hudson.test.JenkinsRule; import org.jvnet.hudson.test.junit.jupiter.WithJenkins; import org.mockito.Mockito; +import software.amazon.awssdk.services.s3.model.ChecksumAlgorithm; import java.io.IOException; import java.util.Collections; @@ -133,6 +134,7 @@ private S3Profile mockS3Profile(String profileName) throws IOException, Interrup Mockito.anyBoolean(), Mockito.anyBoolean(), Mockito.anyBoolean(), + Mockito.any(ChecksumAlgorithm.class), Mockito.anyInt() )).thenReturn(newArrayList(new FingerprintRecord(true, "bucket", "path", "eu-west-1", "xxxx"))); return profile;