Parwinder/bcda-8627-archive cleanup job migration

bcda/bcdacli/cli.go

@@ -13,7 +13,6 @@ import (
 	"path"
 	"path/filepath"
 	"regexp"
-	"strconv"
 	"strings"
 	"syscall"
 	"time"
@@ -39,7 +38,6 @@ import (
 	"github.com/CMSgov/bcda-app/conf"
 	"github.com/CMSgov/bcda-app/log"
 	"github.com/CMSgov/bcda-app/optout"
-	"github.com/sirupsen/logrus"
 
 	"github.com/pborman/uuid"
 	"github.com/pkg/errors"
@@ -74,7 +72,6 @@ func setUpApp() *cli.App {
 		fmt.Println("Error converting FILE_ARCHIVE_THRESHOLD_HR to uint", err)
 	}
 	var acoName, acoCMSID, acoID, accessToken, acoSize, filePath, fileSource, s3Endpoint, assumeRoleArn, environment, groupID, groupName, ips, fileType, alrFile string
-	var thresholdHr int
 	var httpPort, httpsPort int
 	app.Commands = []cli.Command{
 		{
@@ -286,75 +283,6 @@ func setUpApp() *cli.App {
 				return nil
 			},
 		},
-		{
-			Name:     "archive-job-files",
-			Category: "Cleanup",
-			Usage:    "Update job statuses and move files to an inaccessible location",
-			Flags: []cli.Flag{
-				cli.IntFlag{
-					Name:        "threshold",
-					Value:       24,
-					Usage:       constants.CliArchDesc,
-					EnvVar:      "ARCHIVE_THRESHOLD_HR",
-					Destination: &thresholdHr,
-				},
-			},
-			Action: func(c *cli.Context) error {
-				cutoff := time.Now().Add(-time.Hour * time.Duration(thresholdHr))
-				return archiveExpiring(cutoff)
-			},
-		},
-		{
-			Name:     constants.CleanupArchArg,
-			Category: "Cleanup",
-			Usage:    constants.CliRemoveArchDesc,
-			Flags: []cli.Flag{
-				cli.IntFlag{
-					Name:        "threshold",
-					Usage:       constants.CliArchDesc,
-					Destination: &thresholdHr,
-				},
-			},
-			Action: func(c *cli.Context) error {
-				cutoff := time.Now().Add(-time.Hour * time.Duration(thresholdHr))
-				return cleanupJob(cutoff, models.JobStatusArchived, models.JobStatusExpired,
-					conf.GetEnv("FHIR_ARCHIVE_DIR"), conf.GetEnv("FHIR_STAGING_DIR"))
-			},
-		},
-		{
-			Name:     "cleanup-failed",
-			Category: "Cleanup",
-			Usage:    constants.CliRemoveArchDesc,
-			Flags: []cli.Flag{
-				cli.IntFlag{
-					Name:        "threshold",
-					Usage:       constants.CliArchDesc,
-					Destination: &thresholdHr,
-				},
-			},
-			Action: func(c *cli.Context) error {
-				cutoff := time.Now().Add(-(time.Hour * time.Duration(thresholdHr)))
-				return cleanupJob(cutoff, models.JobStatusFailed, models.JobStatusFailedExpired,
-					conf.GetEnv("FHIR_STAGING_DIR"), conf.GetEnv("FHIR_PAYLOAD_DIR"))
-			},
-		},
-		{
-			Name:     "cleanup-cancelled",
-			Category: "Cleanup",
-			Usage:    constants.CliRemoveArchDesc,
-			Flags: []cli.Flag{
-				cli.IntFlag{
-					Name:        "threshold",
-					Usage:       constants.CliRemoveArchDesc,
-					Destination: &thresholdHr,
-				},
-			},
-			Action: func(c *cli.Context) error {
-				cutoff := time.Now().Add(-(time.Hour * time.Duration(thresholdHr)))
-				return cleanupJob(cutoff, models.JobStatusCancelled, models.JobStatusCancelledExpired,
-					conf.GetEnv("FHIR_STAGING_DIR"), conf.GetEnv("FHIR_PAYLOAD_DIR"))
-			},
-		},
 		{
 			Name:     "import-cclf-directory",
 			Category: constants.CliDataImpCategory,
@@ -748,90 +676,6 @@ func revokeAccessToken(accessToken string) error {
 	return auth.GetProvider().RevokeAccessToken(accessToken)
 }
 
-func archiveExpiring(maxDate time.Time) error {
-	log.API.Info("Archiving expiring job files...")
-
-	jobs, err := r.GetJobsByUpdateTimeAndStatus(context.Background(),
-		time.Time{}, maxDate, models.JobStatusCompleted)
-	if err != nil {
-		log.API.Error(err)
-		return err
-	}
-
-	var lastJobError error
-	for _, j := range jobs {
-		id := j.ID
-		jobPayloadDir := fmt.Sprintf("%s/%d", conf.GetEnv("FHIR_PAYLOAD_DIR"), id)
-		_, err = os.Stat(jobPayloadDir)
-		jobPayloadDirExist := err == nil
-		jobArchiveDir := fmt.Sprintf("%s/%d", conf.GetEnv("FHIR_ARCHIVE_DIR"), id)
-
-		if jobPayloadDirExist {
-			err = os.Rename(jobPayloadDir, jobArchiveDir)
-			if err != nil {
-				log.API.Error(err)
-				lastJobError = err
-				continue
-			}
-		}
-
-		j.Status = models.JobStatusArchived
-		err = r.UpdateJob(context.Background(), *j)
-		if err != nil {
-			log.API.Error(err)
-			lastJobError = err
-		}
-	}
-
-	return lastJobError
-}
-
-func cleanupJob(maxDate time.Time, currentStatus, newStatus models.JobStatus, rootDirsToClean ...string) error {
-	jobs, err := r.GetJobsByUpdateTimeAndStatus(context.Background(),
-		time.Time{}, maxDate, currentStatus)
-	if err != nil {
-		return err
-	}
-
-	if len(jobs) == 0 {
-		log.API.Infof("No %s job files to clean", currentStatus)
-		return nil
-	}
-
-	for _, job := range jobs {
-		if err := cleanupJobData(job.ID, rootDirsToClean...); err != nil {
-			log.API.Errorf("Unable to cleanup directories %s", err)
-			continue
-		}
-
-		job.Status = newStatus
-		err = r.UpdateJob(context.Background(), *job)
-		if err != nil {
-			log.API.Errorf("Failed to update job status to %s %s", newStatus, err)
-			continue
-		}
-
-		log.API.WithFields(logrus.Fields{
-			"job_began":     job.CreatedAt,
-			"files_removed": time.Now(),
-			"job_id":        job.ID,
-		}).Infof("Files cleaned from %s and job status set to %s", rootDirsToClean, newStatus)
-	}
-
-	return nil
-}
-
-func cleanupJobData(jobID uint, rootDirs ...string) error {
-	for _, rootDirToClean := range rootDirs {
-		dir := filepath.Join(rootDirToClean, strconv.FormatUint(uint64(jobID), 10))
-		if err := os.RemoveAll(dir); err != nil {
-			return fmt.Errorf("unable to remove %s because %s", dir, err)
-		}
-	}
-
-	return nil
-}
-
 func setDenylistState(cmsID string, td *models.Termination) error {
 	aco, err := r.GetACOByCMSID(context.Background(), cmsID)
 	if err != nil {