BCDA-1347: The CCLF Cleanup functionality does not report errors prop…

…erly (#290)
CMSgov · Jun 11, 2019 · 45cd063 · 45cd063
1 parent 50262ce
commit 45cd063
Show file tree

Hide file tree

Showing 2 changed files with 21 additions and 7 deletions.
diff --git a/bcda/cclf/cclf.go b/bcda/cclf/cclf.go
@@ -404,7 +404,11 @@ func ImportCCLFDirectory(filePath string) (success, failure, skipped int, err er
 		}
 		cclf0.imported = cclf8 != nil && cclf8.imported && cclf9 != nil && cclf9.imported
 	}
-	cleanupCCLF(cclfmap)
+
+	err = cleanupCCLF(cclfmap)
+	if err != nil {
+		log.Error(err)
+	}
 
 	if failure > 0 {
 		err = errors.New("one or more files failed to import correctly")
@@ -573,7 +577,8 @@ func DeleteDirectoryContents(dirToDelete string) (filesDeleted int, err error) {
 	return len(files), nil
 }
 
-func cleanupCCLF(cclfmap map[string][]*cclfFileMetadata) {
+func cleanupCCLF(cclfmap map[string][]*cclfFileMetadata) error {
+	errCount := 0
 	for _, cclflist := range cclfmap {
 		for _, cclf := range cclflist {
 			fmt.Printf("Cleaning up file %s.\n", cclf)
@@ -585,8 +590,10 @@ func cleanupCCLF(cclfmap map[string][]*cclfFileMetadata) {
 				if int(elapsed) > deleteThresholdHr {
 					err := os.Rename(cclf.filePath, newpath)
 					if err != nil {
-						fmt.Printf("File %s failed to cleanup properly.\n", cclf)
-						log.Errorf("File %s failed to cleanup properly", cclf)
+						errCount++
+						errMsg := fmt.Sprintf("File %s failed to clean up properly: %v", cclf, err)
+						fmt.Println(errMsg)
+						log.Error(errMsg)
 					} else {
 						fmt.Printf("File %s never ingested, moved to the pending deletion dir.\n", cclf)
 						log.Infof("File %s never ingested, moved to the pending deletion dir", cclf)
@@ -596,15 +603,21 @@ func cleanupCCLF(cclfmap map[string][]*cclfFileMetadata) {
 				// move the successful files to the deletion dir
 				err := os.Rename(cclf.filePath, newpath)
 				if err != nil {
-					fmt.Printf("File %s failed to cleanup properly.\n", cclf)
-					log.Errorf("File %s failed to cleanup properly", cclf)
+					errCount++
+					errMsg := fmt.Sprintf("File %s failed to clean up properly: %v", cclf, err)
+					fmt.Println(errMsg)
+					log.Error(errMsg)
 				} else {
 					fmt.Printf("File %s successfully ingested, moved to the pending deletion dir.\n", cclf)
 					log.Infof("File %s successfully ingested, moved to the pending deletion dir", cclf)
 				}
 			}
 		}
 	}
+	if errCount > 0 {
+		return fmt.Errorf("%d files could not be cleaned up", errCount)
+	}
+	return nil
 }
 
 func (m cclfFileMetadata) String() string {

diff --git a/bcda/cclf/cclf_test.go b/bcda/cclf/cclf_test.go
@@ -547,7 +547,8 @@ func (s *CCLFTestSuite) TestCleanupCCLF() {
 		imported:  true,
 	}
 	cclfmap["A0001_18"] = []*cclfFileMetadata{cclf0metadata, cclf8metadata, cclf9metadata}
-	cleanupCCLF(cclfmap)
+	err := cleanupCCLF(cclfmap)
+	assert.Nil(err)
 
 	files, err := ioutil.ReadDir(os.Getenv("PENDING_DELETION_DIR"))
 	if err != nil {