Updated multipart_upload.sh

Author: Hasshim Ali

README.md

@@ -1,2 +1,2 @@
 # scripts
-my scripts
+My personal scripts. Feel free to download, edit and use them.

aws_s3_multipart_upload.sh

@@ -1,32 +1,58 @@
 #!/bin/bash
+# Created by Hasshim Ali
+# Version 1.0
 # This script is used to split files and upload them to S3 using the multipart upload.
 # 1. Execute the script, set the number of files to be splitted.
 # 2. Provide the filename.
+############################################################################
 
 split_upload () {
-    # split the file into chunks
-    echo "Spliiting file...."
+    # Split the file into chunks
+    echo "Spliiting file ${filename} into $numinput parts."
     split -n $numinput $filename $filename-
+    ls $filename-* | sort > files.txt
+    cat files.txt
 
-    # this will generate an uplaod id
+    # This will generate the key and an upload id
     aws s3api create-multipart-upload --bucket dips-multipart-test --key $filename > key_uploadId.json
     uploadId=$(sed -n '/UploadId/ s/.*\: //p' key_uploadId.json | sed 's/\(\"\|\"\)//g')
     echo " "
-    echo "UploadID is ${uploadId}"
+    echo "aws s3api UploadID: ${uploadId}"
 
-    # Upload each part
+    # Perform file part count
     echo " "
     totalFiles=$(ls -l ${filename}-* | wc -l)
     i=1
     letter=a
 
-    while [ $i <= $totalFiles ] 
+    # Create list.json file.
+    echo '{"Parts":[' > list.json
+
+    # Loop through the filelist and upload
+    while [ $i -le $totalFiles ];
     do
-        echo "Uploading ${filename} part ${i}"
-        aws s3api upload-part --bucket dips-multipart-test --key $filename --part-number $i --body $filename-a$letter --upload-id $uploadId
-        filecount+=1
+        bodyName=$(sed -n ${i}p files.txt)
+        echo "Uploading ${bodyName} part ${i}"
+        ETag=$(aws s3api upload-part --bucket dips-multipart-test --key ${filename} --part-number ${i} --body ${bodyName} --upload-id ${uploadId} | sed -n '/ETag/ s/.*\: //p')
+
+        if [ $i -eq $totalFiles ]; then
+            echo '{ "PartNumber": '${i}', "ETag": '${ETag}'}]}' >> list.json
+        else
+            echo '{ "PartNumber": '${i}', "ETag": '${ETag}'},' >> list.json
+        fi
+
+        ((i++))
         letter=$(echo "$letter" | tr "0-9a-z" "1-9a-z_")
+        echo " "
     done
+
+    # Get the parts
+    aws s3api list-parts --bucket dips-multipart-test --key $filename --upload-id $uploadId > parts.json
+
+    # After parts uploaded. Compile Etag and part number into one json file and upload.
+    echo " "
+    echo "Completing ${filename} multipart-upload"
+    aws s3api complete-multipart-upload --multipart-upload file://list.json --bucket dips-multipart-test --key $filename --upload-id $uploadId
 }
 
 ############################# MAIN MENU FUNCTION ########################################
@@ -45,7 +71,7 @@ mainmenu () {
     echo ""
     echo "Press x to exit the script"
     echo ""
-    read -n 2 -p "Input Selection: " filename
+    read -p "Input Selection: " filename
     echo ""
 
     if [ "$filename" = "x" ]; then 
@@ -55,10 +81,4 @@ mainmenu () {
     fi
 }
 
-mainmenu
-
-# # Get all the ETags and part numbers
-# aws s3api list-parts --bucket dips-multipart-test --key <key_name_from_step_2> --upload-id $uploadId
-
-# # After parts uploaded. Compile Etag and part number into one json file and upload.
-# aws s3api complete-multipart-upload --multipart-upload file://<jsonfilename> --bucket dips-multipart-test --key <key_name_from_step_2> --upload-id $uploadId
+mainmenu