From c461f41e4a453d2ecaddbcc55f5c1fea7759c15a Mon Sep 17 00:00:00 2001
From: Simon Oliver <sao547@vdi-n1.novalocal>
Date: Fri, 15 Dec 2017 11:10:59 +1100
Subject: [PATCH] Add NCI run methods using parallel and PBS

---
 README                     | 16 ++++++++++++++++
 ledaps_lasrc_pbs_tasker.sh |  2 ++
 ledaps_lasrc_prepare.sh    | 15 +++++++++++++++
 3 files changed, 33 insertions(+)
 create mode 100644 README
 create mode 100755 ledaps_lasrc_pbs_tasker.sh
 create mode 100644 ledaps_lasrc_prepare.sh

diff --git a/README b/README
new file mode 100644
index 0000000..086c1c4
--- /dev/null
+++ b/README
@@ -0,0 +1,16 @@
+## How to add L2C1 LEDAPS output from ESPA to a remote datacube 
+
+run ./ledaps_lasrc_pbs_tasker.sh to create PBS tasks output to qsub_scripts
+
+Tasks in qsub_scripts can be executed by qsub <qsub task within qsub-scripts>
+
+ledaps_lasrc_prepare.sh provides the template for qsub scripts
+
+usgs_l2c1.yaml is the dataset type which can be added to datacube by using:
+ datacube --config_file <config_file> product add usgs_l2c1.yaml
+
+Output yamls can be added to the datacube by using:
+ datacube --config_file <config_file> dataset add yamls\*yaml
+
+
+
diff --git a/ledaps_lasrc_pbs_tasker.sh b/ledaps_lasrc_pbs_tasker.sh
new file mode 100755
index 0000000..3ee12cf
--- /dev/null
+++ b/ledaps_lasrc_pbs_tasker.sh
@@ -0,0 +1,2 @@
+for i in `ls -1 /g/data/v10/projects/ARD_interoperability/L2/`; do if [[ $i != *":"* ]]; then  mkdir -p /g/data/v10/AGDCv2/indexed_datasets/ledaps_lasrc/opendatacubepipelines.ledapslasrc/yamls/$i ; fi; done
+for i in `ls /g/data/v10/AGDCv2/indexed_datasets/ledaps_lasrc/opendatacubepipelines.ledapslasrc/yamls/`; do cp ledaps_lasrc_prepare.sh /g/data/v10/AGDCv2/indexed_datasets/ledaps_lasrc/opendatacubepipelines.ledapslasrc/qsub_scripts/$i.qsub; sed -i -e "s/TARGET/$i/g" "/g/data/v10/AGDCv2/indexed_datasets/ledaps_lasrc/opendatacubepipelines.ledapslasrc/qsub_scripts/$i.qsub"; echo 'qsub /g/data/v10/AGDCv2/indexed_datasets/ledaps_lasrc/opendatacubepipelines.ledapslasrc/qsub_scripts/'$i'.qsub'; done
diff --git a/ledaps_lasrc_prepare.sh b/ledaps_lasrc_prepare.sh
new file mode 100644
index 0000000..f6a2d0b
--- /dev/null
+++ b/ledaps_lasrc_prepare.sh
@@ -0,0 +1,15 @@
+#!/bin/bash
+#
+#PBS -P v10
+#PBS -q normal
+#PBS -l walltime=2:00:00,ncpus=8,mem=64GB
+#PBS -l wd
+#PBS -me
+
+module use /g/data/v10/public/modules/modulefiles
+module use /g/data/v10/private/modules/modulefiles/
+module load gaip/dev-sen2redo
+#module load agdc-py3-prod/1.5.1
+module load parallel
+
+find /g/data/v10/projects/ARD_interoperability/L2/TARGET -name *tar.gz | parallel --jobs 8 "python ls_usgs_l2_prepare.py {} --output /g/data/v10/AGDCv2/indexed_datasets/ledaps_lasrc/opendatacubepipelines.ledapslasrc/yamls/TARGET --no-checksum --date 1/1/1999"