Added MpuObjectSize and ChecksumValue parameters to Write-S3Object to support precalculated checkums for multipart uploads

Author: jnunn-aws

modules/AWSPowerShell/Cmdlets/S3/Advanced/Write-S3Object-Cmdlet.cs

@@ -48,6 +48,7 @@ namespace Amazon.PowerShell.Cmdlets.S3
     public class WriteS3ObjectCmdlet : AmazonS3ClientCmdlet, IExecutor
     {
         const string ParamSet_FromLocalFile = "UploadSingleFile";
+        const string ParamSet_FromLocalFileChecksum = "UploadSingleFileChecksum";
         const string ParamSet_FromContent = "UploadFromContent";
         const string ParamSet_FromLocalFolder = "UploadFolder";
         const string ParamSet_FromStream = "UploadFromStream";
@@ -117,6 +118,7 @@ public class WriteS3ObjectCmdlet : AmazonS3ClientCmdlet, IExecutor
         /// and the object key can be inferred from the filename value supplied to the -File parameter.
         /// </summary>
         [Parameter(Position = 1, ParameterSetName = ParamSet_FromLocalFile, ValueFromPipelineByPropertyName = true)]
+        [Parameter(Position = 1, ParameterSetName = ParamSet_FromLocalFileChecksum, ValueFromPipelineByPropertyName = true)]
         [Parameter(Position = 1, ParameterSetName = ParamSet_FromContent, Mandatory = true, ValueFromPipelineByPropertyName = true)]
         [Parameter(Position = 1, ParameterSetName = ParamSet_FromStream, Mandatory = true, ValueFromPipelineByPropertyName = true)]
         [Amazon.PowerShell.Common.AWSRequiredParameter(ParameterSets = new[] { ParamSet_FromContent, ParamSet_FromStream })]
@@ -128,6 +130,7 @@ public class WriteS3ObjectCmdlet : AmazonS3ClientCmdlet, IExecutor
         /// The full path to the local file to be uploaded.
         /// </summary>
         [Parameter(Position = 2, ParameterSetName = ParamSet_FromLocalFile, Mandatory = true, ValueFromPipelineByPropertyName = true)]
+        [Parameter(Position = 2, ParameterSetName = ParamSet_FromLocalFileChecksum, Mandatory = true, ValueFromPipelineByPropertyName = true)]
         [Amazon.PowerShell.Common.AWSRequiredParameter]
         public System.String File { get; set; }
         #endregion
@@ -401,10 +404,36 @@ public class WriteS3ObjectCmdlet : AmazonS3ClientCmdlet, IExecutor
         [Parameter(ValueFromPipelineByPropertyName = true, ParameterSetName = ParamSet_FromLocalFile)]
         [Parameter(ValueFromPipelineByPropertyName = true, ParameterSetName = ParamSet_FromContent)]
         [Parameter(ValueFromPipelineByPropertyName = true, ParameterSetName = ParamSet_FromStream)]
+        [Parameter(ValueFromPipelineByPropertyName = true, ParameterSetName = ParamSet_FromLocalFileChecksum, Mandatory = true)]
+        [Amazon.PowerShell.Common.AWSRequiredParameter(ParameterSets = new[] { ParamSet_FromLocalFileChecksum })]
         [AWSConstantClassSource("Amazon.S3.ChecksumAlgorithm")]
         public ChecksumAlgorithm ChecksumAlgorithm { get; set; }
         #endregion
 
+        #region Parameter ChecksumValue
+        /// <summary>
+        /// The checksum of the object base64 encoded with the alorithm specified in the <code>ChecksumAlgorithm</code> parameter. This checksum is only present if the checksum was uploaded
+        /// with the object. When you use an API operation on an object that was uploaded using multipart uploads, this value may not be a direct checksum value of the full object. 
+        /// Instead, it's a calculation based on the checksum values of each individual part. For more information about how checksums are calculated
+        /// with multipart uploads, see <a href=\"https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html#large-object-checksums\">
+        /// Checking object integrity</a> in the <i>Amazon S3 User Guide</i>."
+        /// </summary>
+        [Parameter(ValueFromPipelineByPropertyName = true, ParameterSetName = ParamSet_FromLocalFileChecksum)]
+        public String ChecksumValue { get; set; }
+        #endregion
+
+        #region Parameter MpuObjectSize
+        /// <summary>
+        /// The expected total object size of the multipart upload request. If there's a mismatch
+        /// between the specified object size value and the actual object size value, it results in an
+        /// <code>HTTP 400 InvalidRequest</code> error. This value is ignored if the operation is not a
+        /// multipart upload.
+        /// </summary>
+        [Parameter(ValueFromPipelineByPropertyName = true, ParameterSetName = ParamSet_FromLocalFile)]
+        [Parameter(ValueFromPipelineByPropertyName = true, ParameterSetName = ParamSet_FromLocalFileChecksum)]
+        public long? MpuObjectSize { get; set; }
+        #endregion
+
         #region Parameter RequestPayer
         /// <summary>
         /// <para>
@@ -498,7 +527,7 @@ protected override void ProcessRecord()
                 base.UserAgentAddition = AmazonS3Helper.GetCleanKeyUserAgentAdditionString(this.Key, context.Key);
             }
 
-            if (this.ParameterSetName == ParamSet_FromLocalFile)
+            if (this.ParameterSetName == ParamSet_FromLocalFile || this.ParameterSetName == ParamSet_FromLocalFileChecksum)
             {
                 context.File = PSHelpers.PSPathToAbsolute(this.SessionState.Path, this.File.Trim());
                 if (string.IsNullOrEmpty(context.Key))
@@ -576,6 +605,15 @@ protected override void ProcessRecord()
                 context.ChecksumAlgorithm = this.ChecksumAlgorithm;
             }
 
+            if (!string.IsNullOrEmpty(this.ChecksumValue))
+                context.ChecksumValue = this.ChecksumValue;
+
+            if (this.MpuObjectSize != null)
+            {
+                long mpuObjectSize = this.MpuObjectSize.Value;
+                context.MpuObjectSize = mpuObjectSize;
+            }
+
             if (this.PartSize != null)
             {
                 if (this.PartSize.FileSizeInBytes < MinPartSize || this.PartSize.FileSizeInBytes > MaxPartSize)
@@ -725,6 +763,35 @@ private CmdletOutput UploadFileToS3(ExecutorContext context)
                     request.TagSet = new List<Tag>(cmdletContext.TagSet);
                 if (cmdletContext.ChecksumAlgorithm != null)
                     request.ChecksumAlgorithm = cmdletContext.ChecksumAlgorithm;
+
+                if (!string.IsNullOrEmpty(cmdletContext.ChecksumValue))
+                {
+                    switch (cmdletContext.ChecksumAlgorithm.Value)
+                    {
+                        case "CRC32":
+                            request.ChecksumCRC32 = cmdletContext.ChecksumValue;
+                            break;
+                        case "CRC32C":
+                            request.ChecksumCRC32C = cmdletContext.ChecksumValue;
+                            break;
+                        case "CRC64NVME":
+                            request.ChecksumCRC64NVME = cmdletContext.ChecksumValue;
+                            break;
+                        case "SHA1":
+                            request.ChecksumSHA1 = cmdletContext.ChecksumValue;
+                            break;
+                        case "SHA256":
+                            request.ChecksumSHA256 = cmdletContext.ChecksumValue;
+                            break;
+                    }
+                }
+
+                if (cmdletContext.MpuObjectSize != null)
+                {
+                    long mpuObjectSize = cmdletContext.MpuObjectSize.Value;
+                    request.MpuObjectSize = mpuObjectSize;
+                }
+
                 if (!string.IsNullOrEmpty(cmdletContext.IfNoneMatch))
                     request.IfNoneMatch = cmdletContext.IfNoneMatch;
 
@@ -878,6 +945,8 @@ internal class CmdletContext : ExecutorContext
             public string ServerSideEncryptionCustomerProvidedKeyMD5 { get; set; }
 
             public ChecksumAlgorithm ChecksumAlgorithm { get; set; }
+            public String ChecksumValue { get; set; }
+            public long? MpuObjectSize { get; set; }
 
             public Hashtable Metadata { get; set; }
             public Hashtable Headers { get; set; }

tests/S3/S3.Tests.ps1

@@ -184,10 +184,12 @@ Describe -Tag "Smoke" "S3" {
         BeforeAll {
             $script:bucketName = "pstest-" + [DateTime]::Now.ToFileTime()
             New-S3Bucket -BucketName $script:bucketName
+            $testFilePath = Join-Path -Path (Split-Path -Path (Get-Location)) -ChildPath 'foo.txt'
         }
 
         AfterAll {
             $script:bucketName | Remove-S3Bucket -Force -DeleteBucketContent
+            Remove-Item $testFilePath -ErrorAction 'SilentlyContinue'
         }
 
         It "Put and get an object with a SHA1 checksum" {
@@ -222,5 +224,17 @@ Describe -Tag "Smoke" "S3" {
             { Read-S3Object -BucketName $script:bucketName -Key "key1" -File "temp\key1.txt" } | Should -Throw
             { Read-S3Object -BucketName $script:bucketName -Key "key2" -File "temp\key2.txt" } | Should -Throw
         }
+
+        It "Can write objects with a precalcuated checksum value" {
+            Set-Content -Path $testFilePath -Value "Hello world!"
+            $checksumValue = 'vjV9Yg=='
+            Write-S3Object -BucketName $script:bucketName -Key foo.txt -File $testFilePath -ChecksumAlgorithm CRC32C -ChecksumValue $checksumValue
+        }
+
+        It "Throws when writing objects with an invalid checksum value" {
+            Set-Content -Path $testFilePath -Value "Hello world!"
+            $checksumValue = 'abc123'
+            {Write-S3Object -BucketName $script:bucketName -Key foo.txt -File $testFilePath -ChecksumAlgorithm CRC32C -ChecksumValue $checksumValue} | Should -Throw '* is invalid.'
+        }
     }
 }