diff --git a/spark-2.0.1-install b/spark-2.0.1-install new file mode 100644 index 00000000..dc8faaf5 --- /dev/null +++ b/spark-2.0.1-install @@ -0,0 +1,50 @@ +#!/usr/bin/env bash + +############################################### +# Installing spark +# +# by James Hetherington, 2016 +# + +APPNAME=${APPNAME:-spark} +VERSION=${VERSION:-2.0.1} +# We do not need Hadoop. But the built jars "binaries" contain a Hadoop version. +HADOOPVERSION=${HADOOPVERSION:-2.7} +INSTALL_ZONE=/home/ucgajhe/software/ +# INSTALL will be to /shared/ucl/apps/ +INSTALL_PREFIX=${INSTALL_PREFIX:-$INSTALL_ZONE/$APPNAME/$VERSION} +# TEMPDIR will be on /dev/shm +TEMP_ZONE=/home/ucgajhe/software/shm + +SHA256=${SHA256:-3d017807650f41377118a736e2f2298cd0146a593e7243a28c2ed72a88b9a043} +SRC_ARCHIVE=${SRC_ARCHIVE:-http://mirror.catn.com/pub/apache/$APPNAME/$APPNAME-$VERSION/$APPNAME-$VERSION-bin-hadoop${HADOOPVERSION}.tgz} + +set -e + +export PATH=$INSTALL_PREFIX/bin:$PATH + +echo Install prefix : $INSTALL_PREFIX + +mkdir -p $TEMP_ZONE/$APPNAME +temp_dir=`mktemp -d -p $TEMP_ZONE/$APPNAME` + +cd $temp_dir + +wget -O ${APPNAME}-${VERSION}.tgz $SRC_ARCHIVE + +CHECKSUM=`sha256sum ${APPNAME}-${VERSION}.tgz | awk '{print $1}'` + +if [ "$SHA256" == "$CHECKSUM" ] +then + mkdir -p $INSTALL_PREFIX + cd $INSTALL_PREFIX + tar -xzvf $temp_dir/${APPNAME}-${VERSION}.tgz + mv spark-${VERSION}-bin-hadoop${HADOOPVERSION}/* . + rmdir spark-${VERSION}-bin-hadoop${HADOOPVERSION} + wget -O bin/sparkrun https://raw.githubusercontent.com/jamespjh/rcps-buildscripts/master/spark-files/sparkrun + chmod a+x bin/sparkrun +else + echo "Hash mismatch." + echo "Expected: $SHA256" + echo "Got: $CHECKSUM" +fi diff --git a/spark-files/sparkrun b/spark-files/sparkrun new file mode 100644 index 00000000..d4706739 --- /dev/null +++ b/spark-files/sparkrun @@ -0,0 +1,36 @@ +# A gerun-inspired wrapper for running spark jobs on UCL Clusters +# Mostly based on a script by Jonathan Dursi +# from this post: http://www.dursi.ca/spark-in-hpc-clusters/ + +# Make a per-job output directory +OUTPUT_DIR="$(pwd)/run.${JOB_ID}" +mkdir -p "$OUTPUT_DIR" + +# Get our list of allocated nodes and how many +nodes=($( sort -u <"$TMPDIR/machines" | sed -e 's/$/.data.legion.ucl.ac.uk/' )) +nnodes=${#nodes[@]} +last=$(( nnodes - 1 )) + +export SPARK_LOCAL_DIRS="$TMPDIR" + +ssh "${nodes[0]}" "module load java; cd ${SPARK_HOME}; ./sbin/start-master.sh" +sparkmaster="spark://${nodes[0]}:7077" + +# Start the spark workers on all nodes +for i in $( seq 0 $last ) +do + ssh "${nodes[$i]}" "cd ${SPARK_HOME}; module load java; export SPARK_LOCAL_DIRS=\"$TMPDIR\"; nohup spark-class org.apache.spark.deploy.worker.Worker ${sparkmaster} &> ${OUTPUT_DIR}/nohup-${nodes[$i]}.out" & +done + +# Submit the script to the Spark cluster + +spark-submit --master "${sparkmaster}" "$@" + +# Stop the Spark master and kill all the Spark processes to clean up +for i in $( seq 0 $last ) +do + ssh "${nodes[$i]}" "module load java; cd ${SPARK_HOME}; ./sbin/stop-slaves.sh" & +done + +ssh "${nodes[0]}" "module load java; cd ${SPARK_HOME}; ./sbin/stop-master.sh" & +wait