Downgrade R in Docker image (#685)

* Downgrade R to 3.4.3 * Upgrade Python2 version to fix libffi-related pip install failure * Bump docker image version * Provide version for base docker when building R / python images * Make pyenv work with latest conda * Install R dependencies using conda
palantir · Jun 2, 2020 · 80c23bb · 80c23bb
1 parent 3760767
commit 80c23bb
Show file tree

Hide file tree

Showing 5 changed files with 31 additions and 28 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -2,7 +2,7 @@ version: 2
 
 defaults: &defaults
   docker:
-    - image: palantirtechnologies/circle-spark-base:0.2.2
+    - image: palantirtechnologies/circle-spark-base:0.2.3
   resource_class: xlarge
   environment: &defaults-environment
     TERM: dumb
@@ -129,7 +129,7 @@ jobs:
     <<: *defaults
     # Some part of the maven setup fails if there's no R, so we need to use the R image here
     docker:
-      - image: palantirtechnologies/circle-spark-r:0.2.2
+      - image: palantirtechnologies/circle-spark-r:0.2.3
     steps:
       # Saves us from recompiling every time...
       - restore_cache:
@@ -296,7 +296,7 @@ jobs:
     # depends on build-sbt, but we only need the assembly jars
     <<: *defaults
     docker:
-      - image: palantirtechnologies/circle-spark-python:0.2.2
+      - image: palantirtechnologies/circle-spark-python:0.2.3
     parallelism: 2
     steps:
       - *checkout-code
@@ -321,7 +321,7 @@ jobs:
     # depends on build-sbt, but we only need the assembly jars
     <<: *defaults
     docker:
-      - image: palantirtechnologies/circle-spark-r:0.2.2
+      - image: palantirtechnologies/circle-spark-r:0.2.3
     steps:
       - *checkout-code
       - attach_workspace:
@@ -434,7 +434,7 @@ jobs:
     <<: *defaults
     # Some part of the maven setup fails if there's no R, so we need to use the R image here
     docker:
-      - image: palantirtechnologies/circle-spark-r:0.2.2
+      - image: palantirtechnologies/circle-spark-r:0.2.3
     steps:
       - *checkout-code
       - restore_cache:
@@ -454,7 +454,7 @@ jobs:
   deploy-gradle:
     <<: *defaults
     docker:
-      - image: palantirtechnologies/circle-spark-r:0.2.2
+      - image: palantirtechnologies/circle-spark-r:0.2.3
     steps:
       - *checkout-code
       - *restore-gradle-wrapper-cache
@@ -466,7 +466,7 @@ jobs:
     <<: *defaults
     # Some part of the maven setup fails if there's no R, so we need to use the R image here
     docker:
-      - image: palantirtechnologies/circle-spark-r:0.2.2
+      - image: palantirtechnologies/circle-spark-r:0.2.3
     steps:
       # This cache contains the whole project after version was set and mvn package was called
       # Restoring first (and instead of checkout) as mvn versions:set mutates real source code...

diff --git a/dev/docker-images/Makefile b/dev/docker-images/Makefile
@@ -17,7 +17,7 @@
 
 .PHONY: all publish base python r
 
-VERSION=0.2.2
+VERSION=0.2.3
 BASE_IMAGE_NAME = "palantirtechnologies/circle-spark-base:${VERSION}"
 PYTHON_IMAGE_NAME = "palantirtechnologies/circle-spark-python:${VERSION}"
 R_IMAGE_NAME = "palantirtechnologies/circle-spark-r:${VERSION}"

diff --git a/dev/docker-images/base/Dockerfile b/dev/docker-images/base/Dockerfile
@@ -111,7 +111,7 @@ WORKDIR $CIRCLE_HOME
 # Install miniconda, we are using it to test conda support and a bunch of tests expect CONDA_BIN to be set
 ENV CONDA_ROOT=$CIRCLE_HOME/miniconda
 ENV CONDA_BIN=$CIRCLE_HOME/miniconda/bin/conda
-ENV MINICONDA2_VERSION=4.5.11
+ENV MINICONDA2_VERSION=4.7.12.1
 
 RUN curl -sO https://repo.anaconda.com/miniconda/Miniconda2-${MINICONDA2_VERSION}-Linux-x86_64.sh \
   && bash Miniconda2-${MINICONDA2_VERSION}-Linux-x86_64.sh -b -p ${CONDA_ROOT} \

diff --git a/dev/docker-images/python/Dockerfile b/dev/docker-images/python/Dockerfile
@@ -15,27 +15,29 @@
 # limitations under the License.
 #
 
-FROM palantirtechnologies/circle-spark-base
+FROM palantirtechnologies/circle-spark-base:0.2.3
 
 # Install pyenv
-ENV PATH="$CIRCLE_HOME/.pyenv/bin:$PATH"
-RUN curl -L https://github.com/pyenv/pyenv-installer/raw/master/bin/pyenv-installer | bash \
-  && cat >>.bashrc <<<'eval "$($HOME/.pyenv/bin/pyenv init -)"' \
-  && cat >>.bashrc <<<'eval "$($HOME/.pyenv/bin/pyenv virtualenv-init -)"'
+RUN curl -L https://github.com/pyenv/pyenv-installer/raw/master/bin/pyenv-installer | bash
+ENV PATH="$CIRCLE_HOME/.pyenv/shims:$CIRCLE_HOME/.pyenv/bin:$PATH"
 
 # Must install numpy 1.11 or else a bunch of tests break due to different output formatting on e.g. nparray
 # A version I've tested earlier that I know it breaks with is 1.14.1
 RUN mkdir -p $(pyenv root)/versions \
   && ln -s $CONDA_ROOT $(pyenv root)/versions/our-miniconda \
-  && $CONDA_BIN create -y -n python2 -c anaconda -c conda-forge python==2.7.15 numpy=1.14.0 pyarrow==0.12.1 pandas nomkl \
+  && $CONDA_BIN create -y -n python2 -c anaconda -c conda-forge python==2.7.18 numpy=1.14.0 pyarrow==0.12.1 pandas nomkl \
   && $CONDA_BIN create -y -n python3 -c anaconda -c conda-forge python=3.6 numpy=1.14.0 pyarrow==0.12.1 pandas nomkl \
   && $CONDA_BIN clean --all
 
 RUN pyenv global our-miniconda/envs/python2 our-miniconda/envs/python3 \
   && pyenv rehash
 
-# Expose pyenv globally
-ENV PATH=$CIRCLE_HOME/.pyenv/shims:$PATH
-
 RUN PYENV_VERSION=our-miniconda/envs/python2 $CIRCLE_HOME/.pyenv/shims/pip install unishark "unittest-xml-reporting<3"
 RUN PYENV_VERSION=our-miniconda/envs/python3 $CIRCLE_HOME/.pyenv/shims/pip install unishark unittest-xml-reporting
+
+# Conda 'activate' should take precedence over pyenv's (pyenv/pyenv-virtualenv#270).
+# In run-pip-tests, we do 'source activate /path'. Conda's activate is fine with that, but not pyenv's.
+RUN mkdir $CIRCLE_HOME/.bin
+RUN ln -s $CONDA_ROOT/bin/activate $CIRCLE_HOME/.bin/activate
+RUN ln -s $CONDA_ROOT/bin/deactivate $CIRCLE_HOME/.bin/deactivate
+ENV PATH="$CIRCLE_HOME/.bin:$PATH"
diff --git a/dev/docker-images/r/Dockerfile b/dev/docker-images/r/Dockerfile
@@ -15,16 +15,17 @@
 # limitations under the License.
 #
 
-FROM palantirtechnologies/circle-spark-base
+FROM palantirtechnologies/circle-spark-base:0.2.3
 
-USER root
-
-### Install R
-RUN apt-get update \
-  && apt-get install r-base r-base-dev qpdf \
-  && rm -rf /var/lib/apt/lists/* \
-  && chmod 777 /usr/local/lib/R/site-library \
-  && /usr/lib/R/bin/R -e "install.packages(c('devtools'), repos='http://cran.us.r-project.org', lib='/usr/local/lib/R/site-library'); devtools::install_github('r-lib/[email protected]', lib='/usr/local/lib/R/site-library'); install.packages(c('knitr', 'rmarkdown', 'e1071', 'survival', 'roxygen2', 'lintr'), repos='http://cran.us.r-project.org', lib='/usr/local/lib/R/site-library')"
-ENV R_HOME=/usr/lib/R
+# Install R and SparkR dependencies
+RUN $CONDA_BIN install --yes --channel r \
+    r-base=3.4.3 r-knitr r-rmarkdown r-e1071 r-survival r-testthat r-lintr r-roxygen2 r-devtools
 
+# Install qpdf (used for SparkR documentation, required in R tests)
+USER root
+RUN apt-get update && apt-get install qpdf
 USER circleci
+
+# Add conda-installed R to path
+ENV PATH="$CONDA_ROOT/bin:$PATH"
+ENV R_HOME=$CONDA_ROOT/lib/R