From 52db2f779f6b0e6551c6370833ac3eeae3ed0ce1 Mon Sep 17 00:00:00 2001 From: Alexey Egorov Date: Thu, 12 Nov 2020 15:18:08 +0100 Subject: [PATCH 1/2] updated build script to be used on mac (requires GNU version of grep from homebrew) --- build/build.sh | 51 ++++++++++++++++++++++++++++++++------------------ 1 file changed, 33 insertions(+), 18 deletions(-) diff --git a/build/build.sh b/build/build.sh index 10034e0..741ef40 100755 --- a/build/build.sh +++ b/build/build.sh @@ -6,16 +6,31 @@ # -- Variables --------------------------------------------------------------------------------------------------------- # ---------------------------------------------------------------------------------------------------------------------- +# check if 'mac' is passed as an argument and use 'ggrep' then +# 'ggrep' (GNU version of grep for Mac) can be installed from homebrew +if [[ $1 == "mac" ]] +then + grep="ggrep" +else + grep="grep" +fi + +grep_origin="$(${grep} --version)" +if [[ $grep_origin != *"GNU"* ]]; then + echo "Using wrong grep: ${grep_origin}. GNU version is required. You can install it e.g. with homebrew on Mac." + exit +fi + BUILD_DATE="$(date -u +'%Y-%m-%d')" -SHOULD_BUILD_BASE="$(grep -m 1 build_base build.yml | grep -o -P '(?<=").*(?=")')" -SHOULD_BUILD_SPARK="$(grep -m 1 build_spark build.yml | grep -o -P '(?<=").*(?=")')" -SHOULD_BUILD_JUPYTERLAB="$(grep -m 1 build_jupyter build.yml | grep -o -P '(?<=").*(?=")')" +SHOULD_BUILD_BASE="$(${grep} -m 1 build_base build.yml | ${grep} -o -P '(?<=").*(?=")')" +SHOULD_BUILD_SPARK="$(${grep} -m 1 build_spark build.yml | ${grep} -o -P '(?<=").*(?=")')" +SHOULD_BUILD_JUPYTERLAB="$(${grep} -m 1 build_jupyter build.yml | ${grep} -o -P '(?<=").*(?=")')" -SCALA_VERSION="$(grep -m 1 scala build.yml | grep -o -P '(?<=").*(?=")')" -SPARK_VERSION="$(grep -m 1 spark build.yml | grep -o -P '(?<=").*(?=")')" -HADOOP_VERSION="$(grep -m 1 hadoop build.yml | grep -o -P '(?<=").*(?=")')" -JUPYTERLAB_VERSION="$(grep -m 1 jupyterlab build.yml | grep -o -P '(?<=").*(?=")')" +SCALA_VERSION="$(${grep} -m 1 scala build.yml | ${grep} -o -P '(?<=").*(?=")')" +SPARK_VERSION="$(${grep} -m 1 spark build.yml | ${grep} -o -P '(?<=").*(?=")')" +HADOOP_VERSION="$(${grep} -m 1 hadoop build.yml | ${grep} -o -P '(?<=").*(?=")')" +JUPYTERLAB_VERSION="$(${grep} -m 1 jupyterlab build.yml | ${grep} -o -P '(?<=").*(?=")')" # ---------------------------------------------------------------------------------------------------------------------- # -- Functions---------------------------------------------------------------------------------------------------------- @@ -25,7 +40,7 @@ function cleanContainers() { if [[ "${SHOULD_BUILD_JUPYTERLAB}" == "true" ]] then - container="$(docker ps -a | grep 'jupyterlab' | awk '{print $1}')" + container="$(docker ps -a | ${grep} 'jupyterlab' | awk '{print $1}')" docker stop "${container}" docker rm "${container}" fi @@ -33,19 +48,19 @@ function cleanContainers() { if [[ "${SHOULD_BUILD_SPARK}" == "true" ]] then - container="$(docker ps -a | grep 'spark-worker' -m 1 | awk '{print $1}')" + container="$(docker ps -a | ${grep} 'spark-worker' -m 1 | awk '{print $1}')" while [ -n "${container}" ]; do docker stop "${container}" docker rm "${container}" - container="$(docker ps -a | grep 'spark-worker' -m 1 | awk '{print $1}')" + container="$(docker ps -a | ${grep} 'spark-worker' -m 1 | awk '{print $1}')" done - container="$(docker ps -a | grep 'spark-master' | awk '{print $1}')" + container="$(docker ps -a | ${grep} 'spark-master' | awk '{print $1}')" docker stop "${container}" docker rm "${container}" - container="$(docker ps -a | grep 'spark-base' | awk '{print $1}')" + container="$(docker ps -a | ${grep} 'spark-base' | awk '{print $1}')" docker stop "${container}" docker rm "${container}" @@ -53,7 +68,7 @@ function cleanContainers() { if [[ "${SHOULD_BUILD_BASE}" == "true" ]] then - container="$(docker ps -a | grep 'base' | awk '{print $1}')" + container="$(docker ps -a | ${grep} 'base' | awk '{print $1}')" docker stop "${container}" docker rm "${container}" fi @@ -64,19 +79,19 @@ function cleanImages() { if [[ "${SHOULD_BUILD_JUPYTERLAB}" == "true" ]] then - docker rmi -f "$(docker images | grep -m 1 'jupyterlab' | awk '{print $3}')" + docker rmi -f "$(docker images | ${grep} -m 1 'jupyterlab' | awk '{print $3}')" fi if [[ "${SHOULD_BUILD_SPARK}" == "true" ]] then - docker rmi -f "$(docker images | grep -m 1 'spark-worker' | awk '{print $3}')" - docker rmi -f "$(docker images | grep -m 1 'spark-master' | awk '{print $3}')" - docker rmi -f "$(docker images | grep -m 1 'spark-base' | awk '{print $3}')" + docker rmi -f "$(docker images | ${grep} -m 1 'spark-worker' | awk '{print $3}')" + docker rmi -f "$(docker images | ${grep} -m 1 'spark-master' | awk '{print $3}')" + docker rmi -f "$(docker images | ${grep} -m 1 'spark-base' | awk '{print $3}')" fi if [[ "${SHOULD_BUILD_BASE}" == "true" ]] then - docker rmi -f "$(docker images | grep -m 1 'base' | awk '{print $3}')" + docker rmi -f "$(docker images | ${grep} -m 1 'base' | awk '{print $3}')" fi } From 1c83443c0a213cb880dda9466e17b3accf339d7b Mon Sep 17 00:00:00 2001 From: Alexey Egorov Date: Thu, 12 Nov 2020 15:27:00 +0100 Subject: [PATCH 2/2] update README description for building on Mac OSX --- README.md | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 9ede8c7..cce01eb 100644 --- a/README.md +++ b/README.md @@ -61,7 +61,7 @@ docker-compose up ### Build from your local machine -> **Note**: Local build is currently only supported on Linux OS distributions. +> **Note**: Local build is currently only supported on Linux OS and Mac OSX distributions. 1. Download the source code or clone the repository; 2. Move to the build directory; @@ -87,6 +87,24 @@ docker-compose up 7. Run Apache Spark code using the provided Jupyter [notebooks](build/workspace/) with Scala, PySpark and SparkR examples; 8. Stop the cluster by typing `ctrl+c`. +#### Mac OSX + +Mac OSX comes with the preinstalled BSD version of `grep`. +In order to be able to build these Docker images, you will need to install GNU version of grep with Homebrew: + +``` +brew install grep +``` + +> **Note**: From now on you can use the GNU version of grep by calling ``ggrep``. + +Now you can run the steps described above, but add an argument in step 5: + +```bash +chmod +x build.sh ; ./build.sh mac +``` + + ## Tech Stack - Infrastructure