Merge pull request #45 from causecode/move-files-to-new-cdn-changes

Move files to new CDN changes

Author: Shivam Pandey

README.md

@@ -1,4 +1,4 @@
-# File-Uploader Plugin (Latest 4.0.3)
+# File-Uploader Plugin (Latest 4.0.4)
 
 [![Maintainability](https://api.codeclimate.com/v1/badges/13bfee73c29ecd2ea4b2/maintainability)](https://codeclimate.com/github/causecode/grails-file-uploader/maintainability)
 [![Test Coverage](https://api.codeclimate.com/v1/badges/13bfee73c29ecd2ea4b2/test_coverage)](https://codeclimate.com/github/causecode/grails-file-uploader/test_coverage)

build.gradle

@@ -15,7 +15,7 @@ buildscript {
     }
 }
 
-version "4.0.3"
+version "4.0.4"
 group "com.causecode.plugins"
 
 apply plugin: "idea"

changelog.md

@@ -1,5 +1,7 @@
 # ChangeLog
 
+## Version 4.0.4 [24-04-2019]
+
 ## Version 4.0.3 [28-03-2019]
 
 ### Fixed

grails-app/services/com/causecode/fileuploader/FileUploaderService.groovy

@@ -414,6 +414,46 @@ class FileUploaderService {
         return Holders.flatConfig["fileuploader.groups.${fileGroup}.makePublic"] ? true : false
     }
 
+    /**
+     * This methods move all the instances of {@link UFile} which are not local to the destination {@link CDNProvider}.
+     * The migration from one CDN to another follows below steps.
+     * 1. A copy of {@link File} from the source CDN {@link UFile} instance.
+     * 2. Upload it to the destination CDN.
+     * 3. Update the respective {@link UFile} instance.
+     * 4. Creates an instance of {@link UFileMoveHistory} which contains the {@link UFile} move history.
+     *
+     * @param toCDNProvider {@link CDNProvider} - Target destination CDN provider.
+     * @param makePublic {@link boolean} - All the URLs either public or signed.
+     * @return {@link boolean} Based on the successful move.
+     */
+    boolean moveToNewCDN(CDNProvider toCDNProvider, boolean makePublic = false) {
+        if (!toCDNProvider) {
+            log.debug 'Please provide the target CDN provider name.'
+
+            return false
+        }
+
+        log.debug "Migration from source CDN to target ${toCDNProvider.name()} has been started..."
+
+        List<UFile> uFileList, uFilesUploadFailuresList = []
+        int offset = 0
+
+        while ((uFileList =  UFile.createCriteria().list(max: 500, offset: offset) {
+            ne('type', CDNProvider.LOCAL)
+        }).size()) {
+            uFilesUploadFailuresList.addAll(moveFilesToCDN(uFileList, toCDNProvider, makePublic))
+
+            log.debug "Moved ${uFileList.size()} files to new CDN and failed count: ${uFilesUploadFailuresList.size()}"
+
+            offset += 500
+        }
+
+        log.debug "Successfully moved files to new ${toCDNProvider.toString()} CDN and failed to upload total files" +
+                ": ${uFilesUploadFailuresList.size()}"
+
+        return true
+    }
+
     /**
      * Moves all UFiles stored at any CDN provider to the given CDN provider. Does not touch UFiles stored locally.
      * Needs to be executed only once.

src/test/groovy/com/causecode/fileuploader/FileUploaderServiceSpec.groovy

@@ -5,6 +5,7 @@ import com.causecode.fileuploader.util.checksum.Algorithm
 import com.causecode.fileuploader.util.checksum.exceptions.DuplicateFileException
 import grails.buildtestdata.BuildDataTest
 import grails.buildtestdata.mixin.Build
+import grails.gorm.CriteriaBuilder
 import grails.testing.services.ServiceUnitTest
 import grails.util.Holders
 import groovy.json.JsonBuilder
@@ -36,6 +37,23 @@ class FileUploaderServiceSpec extends BaseFileUploaderServiceSpecSetup implement
         service.providerService = providerService
     }
 
+    @SuppressWarnings('MapAsMethodParameter')
+    void mockCreateCriteria(List<Object> resultList) {
+        GroovyMock(UFile, global: true)
+        CriteriaBuilder criteriaBuilder = Mock()
+
+        criteriaBuilder.list(_, _) >> { Map params, Closure closure ->
+            JsonBuilder jsonBuilder = new JsonBuilder()
+            jsonBuilder closure
+
+            return resultList
+        } >> {
+            return []
+        }
+
+        UFile.createCriteria() >> criteriaBuilder
+    }
+
     void "test isPublicGroup for various file groups"() {
         expect: 'Following conditions should pass'
         service.isPublicGroup('user') == true
@@ -864,4 +882,95 @@ class FileUploaderServiceSpec extends BaseFileUploaderServiceSpecSetup implement
         StorageConfigurationException exception = thrown(StorageConfigurationException)
         exception.message == 'Container name not defined in the Config. Please define one.'
     }
+
+    void "test moveToNewCDN method for successfully moving the files to the target CDN provider"() {
+        given: 'An instance of UFile'
+        UFile uFileOne = UFile.build(path: '/tmp/test1.txt', provider: CDNProvider.GOOGLE, fileGroup: 'testGoogle')
+        UFile uFileTwo = UFile.build(path: '/tmp/test2.txt', provider: CDNProvider.GOOGLE, fileGroup: 'testGoogle')
+
+        and: 'Mocked the createCriteria method to return the UFiles list'
+        mockCreateCriteria([uFileOne, uFileTwo])
+
+        and: 'Mocked the moveFilesToCDN method to move the files to source CDN provider successfully'
+        service.metaClass.static.moveFilesToCDN = { List<UFile> list, CDNProvider provider, boolean makePublic ->
+            list.each { UFile file ->
+                file.provider = CDNProvider.AMAZON
+                file.save()
+            }
+        }
+
+        when: 'moveToNewCDN method is called'
+        assert uFileOne.provider == CDNProvider.GOOGLE
+        assert uFileTwo.provider == CDNProvider.GOOGLE
+
+        boolean status = service.moveToNewCDN(CDNProvider.AMAZON, false)
+
+        then: 'it should return true and all the UFile providers must be changed to target CDN'
+        status
+        uFileOne.provider == CDNProvider.AMAZON
+        uFileTwo.provider == CDNProvider.AMAZON
+    }
+
+    void "test moveToNewCDN method for the failure case when we move files to target CDN"() {
+        given: 'An instance of UFile'
+        UFile uFileOne = UFile.build(path: '/tmp/test1.txt', provider: CDNProvider.GOOGLE, fileGroup: 'testGoogle')
+        UFile uFileTwo = UFile.build(path: '/tmp/test2.txt', provider: CDNProvider.GOOGLE, fileGroup: 'testGoogle')
+
+        and: 'Mocked the createCriteria method to return the UFiles list'
+        mockCreateCriteria([uFileOne, uFileTwo])
+
+        and: 'Mocked the moveFilesToCDN method to return the list of failed documents'
+        service.metaClass.static.moveFilesToCDN = { List<UFile> list, CDNProvider provider, boolean makePublic ->
+            list.each { UFile file ->
+                UFileMoveHistory.build(ufile: file, fromCDN: file.provider, toCDN: provider, status: MoveStatus.FAILURE,
+                        details: 'failed to move.')
+               return false
+            }
+
+            return list
+        }
+
+        when: 'moveToNewCDN method is called'
+        boolean status = service.moveToNewCDN(CDNProvider.AMAZON, false)
+
+        then: 'it should return true and all the UFile providers must not be changed'
+        status
+        uFileOne.provider == CDNProvider.GOOGLE
+        uFileTwo.provider == CDNProvider.GOOGLE
+        UFileMoveHistory.count() == 2
+    }
+
+    void "test moveToNewCDN method for amazon when exception occurs in a file URL"() {
+        given: 'An instance of UFile and File'
+        UFile uFileInstance = UFile.build(path: '/tmp/test.txt', provider: CDNProvider.GOOGLE, fileGroup: 'testGoogle')
+        File fileInstance = getFileInstance('/tmp/test.txt')
+
+        and: 'Mocked method'
+        service.metaClass.getFileFromURL = { String url, String filename ->
+            throw new IOException('Error getting file from URL')
+        }
+
+        and: 'Mocked Amazon provider instance'
+        mockGetProviderInstance(CDNProvider.AMAZON.name())
+
+        and: 'Mocked the uploadFile method of amazon'
+        mockUploadFileMethod(true)
+
+        when: 'moveFilesToCDN method is called'
+        service.moveToNewCDN(CDNProvider.AMAZON, true)
+
+        then: 'File won\'t be moved'
+        uFileInstance.provider == CDNProvider.GOOGLE
+
+        cleanup:
+        fileInstance.delete()
+    }
+
+    void "test moveToNewCDN method is called with null CDN provider name"() {
+        when: 'moveToNewCDN method is called'
+        boolean status = service.moveToNewCDN(null, false)
+
+        then: 'it should return false'
+        !status
+    }
 }