diff --git a/nextflow/configs/profiles/sumner2.config b/nextflow/configs/profiles/sumner2.config index cceacfc..83bd708 100644 --- a/nextflow/configs/profiles/sumner2.config +++ b/nextflow/configs/profiles/sumner2.config @@ -121,6 +121,7 @@ params { globus_remote_endpoint = "70850914-722d-11e7-aa01-22000bf2d287" // JAX T2 storage endpoint globus_remote_folder = "/tier2/vkumar/" dropbox_prefix = "labdropbox:\"/KumarLab's shared workspace/VideoData/MDS_Tests/\"" + rclone_config = "/projects/kumar-lab/meta/secrets/tokens/rclone_dropbox.conf" } apptainer { diff --git a/nextflow/modules/remote_io.nf b/nextflow/modules/remote_io.nf index a0a4bae..836fec9 100644 --- a/nextflow/modules/remote_io.nf +++ b/nextflow/modules/remote_io.nf @@ -51,6 +51,7 @@ process FILTER_UNPROCESSED_DROPBOX { input: path test_files val dropbox_prefix + path rclone_config output: path "unprocessed_files.txt", emit: unprocessed_files @@ -62,7 +63,7 @@ process FILTER_UNPROCESSED_DROPBOX { touch unprocessed_files.txt while read test_file; do test_pose=\${test_file/.*}_pose_est_v6.h5 - rclone ls ${dropbox_prefix}\${test_pose} > /dev/null 2>&1 + rclone ls --config=${rclone_config} ${dropbox_prefix}\${test_pose} > /dev/null 2>&1 if [[ \$? != 0 ]]; then echo \$test_file >> unprocessed_files.txt fi @@ -122,6 +123,7 @@ process GET_DATA_FROM_DROPBOX { input: path files_to_transfer val dropbox_prefix + path rclone_config output: path "fetched_files.txt", emit: remote_files @@ -129,7 +131,7 @@ process GET_DATA_FROM_DROPBOX { script: """ echo ${dropbox_prefix} - rclone copy --transfers=1 --include-from ${files_to_transfer} ${dropbox_prefix} retrieved_files/. + rclone copy --config=${rclone_config} --transfers=1 --include-from ${files_to_transfer} ${dropbox_prefix} retrieved_files/. find \$(pwd)/retrieved_files/ -type f > fetched_files.txt """ } @@ -142,9 +144,10 @@ process PUT_DATA_TO_DROPBOX { path file_to_upload tuple path(result_file), val(publish_filename) val dropbox_prefix + path rclone_config script: """ - rclone copy --transfers=1 ${result_file} ${dropbox_prefix}/${publish_filename} + rclone copy --config=${rclone_config} --transfers=1 ${result_file} ${dropbox_prefix}/${publish_filename} """ } diff --git a/nextflow/workflows/io.nf b/nextflow/workflows/io.nf index 693fb2b..e203e5e 100644 --- a/nextflow/workflows/io.nf +++ b/nextflow/workflows/io.nf @@ -24,8 +24,8 @@ workflow PREPARE_DATA { if (location == "local") { file_batch = FILTER_LOCAL_BATCH(all_valid_files, params.ignore_invalid_inputs, params.filter_processed, params.pubdir).process_filelist } else if (location == "dropbox") { - in_video_list = FILTER_UNPROCESSED_DROPBOX(all_valid_files, params.dropbox_prefix).unprocessed_files - file_batch = GET_DATA_FROM_DROPBOX(in_video_list, params.dropbox_prefix).remote_files + in_video_list = FILTER_UNPROCESSED_DROPBOX(all_valid_files, params.dropbox_prefix, params.rclone_config).unprocessed_files + file_batch = GET_DATA_FROM_DROPBOX(in_video_list, params.dropbox_prefix, params.rclone_config).remote_files } else if (location == "globus") { CHECK_GLOBUS_AUTH() in_video_list = FILTER_UNPROCESSED_GLOBUS(params.globus_remote_endpoint, all_valid_files).unprocessed_files