Add NCI run methods using parallel and PBS

README

@@ -0,0 +1,16 @@
+## How to add L2C1 LEDAPS output from ESPA to a remote datacube 
+
+run ./ledaps_lasrc_pbs_tasker.sh to create PBS tasks output to qsub_scripts
+
+Tasks in qsub_scripts can be executed by qsub <qsub task within qsub-scripts>
+
+ledaps_lasrc_prepare.sh provides the template for qsub scripts
+
+usgs_l2c1.yaml is the dataset type which can be added to datacube by using:
+ datacube --config_file <config_file> product add usgs_l2c1.yaml
+
+Output yamls can be added to the datacube by using:
+ datacube --config_file <config_file> dataset add yamls\*yaml
+
+
+

ledaps_lasrc_pbs_tasker.sh

@@ -0,0 +1,2 @@
+for i in `ls -1 /g/data/v10/projects/ARD_interoperability/L2/`; do if [[ $i != *":"* ]]; then  mkdir -p /g/data/v10/AGDCv2/indexed_datasets/ledaps_lasrc/opendatacubepipelines.ledapslasrc/yamls/$i ; fi; done
+for i in `ls /g/data/v10/AGDCv2/indexed_datasets/ledaps_lasrc/opendatacubepipelines.ledapslasrc/yamls/`; do cp ledaps_lasrc_prepare.sh /g/data/v10/AGDCv2/indexed_datasets/ledaps_lasrc/opendatacubepipelines.ledapslasrc/qsub_scripts/$i.qsub; sed -i -e "s/TARGET/$i/g" "/g/data/v10/AGDCv2/indexed_datasets/ledaps_lasrc/opendatacubepipelines.ledapslasrc/qsub_scripts/$i.qsub"; echo 'qsub /g/data/v10/AGDCv2/indexed_datasets/ledaps_lasrc/opendatacubepipelines.ledapslasrc/qsub_scripts/'$i'.qsub'; done

ledaps_lasrc_prepare.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+#
+#PBS -P v10
+#PBS -q normal
+#PBS -l walltime=2:00:00,ncpus=8,mem=64GB
+#PBS -l wd
+#PBS -me
+
+module use /g/data/v10/public/modules/modulefiles
+module use /g/data/v10/private/modules/modulefiles/
+module load gaip/dev-sen2redo
+#module load agdc-py3-prod/1.5.1
+module load parallel
+
+find /g/data/v10/projects/ARD_interoperability/L2/TARGET -name *tar.gz | parallel --jobs 8 "python ls_usgs_l2_prepare.py {} --output /g/data/v10/AGDCv2/indexed_datasets/ledaps_lasrc/opendatacubepipelines.ledapslasrc/yamls/TARGET --no-checksum --date 1/1/1999"