Enable CI backed by GitLab CI and CircleCI

.ci/README.md

@@ -0,0 +1,9 @@
+# Continuous Integration (CI)
+
+We support several implementations of CI. All these implementations rely on
+[docker](https://docker.com) in some way. This directory contains bits which
+are shared between these CI implementations. The relevant docker files can be
+found in `/docker/`.
+
+* [CircleCI](https://circleci.com) is configured in `/.circleci/`.
+* [GitLab CI](https://gitlab.com) is configured in `/.gitlab-ci.yml`.

.ci/build-docker.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+
+# This script gets called from CI to build several flavours of docker images
+# which contain Sage.
+
+# ****************************************************************************
+#       Copyright (C) 2018 Julian Rüth <[email protected]>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#                  http://www.gnu.org/licenses/
+# ****************************************************************************
+set -ex
+
+[[ -z "$DOCKER_TAG" ]] && DOCKER_TAG=none
+[[ "$DOCKER_TAG" = "master" ]] && DOCKER_TAG=latest
+
+. .ci/setup-make-parallelity.sh
+
+# We speed up the build process by copying built artifacts from ARTIFACT_BASE
+# during docker build. See /docker/Dockerfile for more details.
+ARTIFACT_BASE=${ARTIFACT_BASE:-sagemath/sagemath-dev:latest}
+
+# Seed our cache with $ARTIFACT_BASE if it exists
+docker pull $ARTIFACT_BASE || true
+
+function docker_build {
+    time docker build -f docker/Dockerfile --build-arg "MAKE=${MAKE}" --build-arg ARTIFACT_BASE=$ARTIFACT_BASE $@
+}
+
+# We use a multi-stage build /docker/Dockerfile. For the caching to be
+# effective, we populate the cache by building the make-all target. (Just
+# building the last target is not enough as intermediate targets would be
+# discarded from the cache and therefore the caching would fail for our actual
+# builds below.)
+docker_build --pull --tag make-all --target make-all .
+
+# Build the release image without build artifacts.
+DOCKER_IMAGE_CLI=${DOCKER_USER:-sagemath}/sagemath:$DOCKER_TAG
+docker_build --target sagemath --tag "$DOCKER_IMAGE_CLI" .
+# Build the developer image with the build artifacts intact.
+# Note: It's important to build the dev image last because it might be tagged as ARTIFACT_BASE.
+DOCKER_IMAGE_DEV=${DOCKER_USER:-sagemath}/sagemath-dev:$DOCKER_TAG
+docker_build --target sagemath-dev --tag "$DOCKER_IMAGE_DEV" .

.ci/protect-secrets.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+# This script protects all environment variables that start with "SECRET_".
+# It puts them in a temporary file. The name of the variable contains the path
+# of that file.  This filename can then safely be used in `cat` even if `set
+# -x` has been turned on. Also you can run "export" to understand the
+# environment without danger.
+# Be careful, however, not to use this like the following:
+# docker login $DOCKER_USER $(cat $SECRET_DOCKER_PASS)
+# as this would expose the password if `set -x` has been turned on.
+
+# ****************************************************************************
+#       Copyright (C) 2018 Julian Rüth <[email protected]>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#                  http://www.gnu.org/licenses/
+# ****************************************************************************
+
+set -eo pipefail
+set +x
+
+function encrypt {
+    RET=`mktemp`
+    eval " echo \$$1" > "$RET"
+    echo $RET
+}
+
+for name in `awk 'END { for (name in ENVIRON) { print name; } }' < /dev/null`; do
+case "$name" in 
+  SECRET_*)
+    export $name="$(encrypt $name)"
+    echo "Protected $name"
+    ;;
+esac
+done
+
+unset encrypt

.ci/pull-gitlab.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+
+# This script gets called from CI to pull the Sage docker images that were
+# built during the "build" phase to pull all the connected docker daemon
+# (likely a docker-in-docker.)
+# This script expects a single parameter, the base name of the docker image
+# such as sagemath or sagemath-dev.
+# The variable $DOCKER_IMAGE is set to the full name of the pulled image;
+# source this script to use it.
+
+# ****************************************************************************
+#       Copyright (C) 2018 Julian Rüth <[email protected]>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#                  http://www.gnu.org/licenses/
+# ****************************************************************************
+
+set -ex
+
+[[ -z "$DOCKER_TAG" ]] && (echo "Can not pull untagged build."; exit 0)
+[[ "$DOCKER_TAG" = "master" ]] && DOCKER_TAG=latest
+
+# Pull the built images from the gitlab registry and give them the original
+# names they had after built.
+# Note that "set -x" prints the $CI_BUILD_TOKEN here but GitLab removes it
+# automatically from the log output.
+docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $CI_REGISTRY
+docker pull $CI_REGISTRY_IMAGE/$1:$DOCKER_TAG
+DOCKER_IMAGE="${DOCKER_USER:-sagemath}/$1:$DOCKER_TAG"
+docker tag $CI_REGISTRY_IMAGE/$1:$DOCKER_TAG $DOCKER_IMAGE

.ci/push-dockerhub.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+# This script gets called from CI to push our docker images to
+# $DOCKER_USER/sagemath* on the Docker Hub.
+# This script expects a single parameter, the base name of the docker image
+# such as sagemath or sagemath-dev.
+
+# ****************************************************************************
+#       Copyright (C) 2018 Julian Rüth <[email protected]>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#                  http://www.gnu.org/licenses/
+# ****************************************************************************
+
+set -ex
+
+[[ -z "$DOCKER_TAG" ]] && (echo "Can not push untagged build."; exit 0)
+[[ "$DOCKER_TAG" = "master" ]] && DOCKER_TAG=latest
+
+# Push the built images to the docker hub (and fail silently if
+# DOCKER_USER/SECRET_DOCKER_PASS have not been configured.)
+if [[ -z "$DOCKER_USER" || -z "$SECRET_DOCKER_PASS" ]]; then
+    echo "DOCKER_USER/SECRET_DOCKER_PASS variables have not been configured in your Continuous Integration setup. Not pushing built images to Docker Hub."
+else
+  cat "$SECRET_DOCKER_PASS" | docker login -u $DOCKER_USER --password-stdin
+  docker push ${DOCKER_USER:-sagemath}/$1:$DOCKER_TAG
+fi

.ci/push-gitlab.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+set -ex
+
+# This script gets called from CI to push our docker images to registry
+# configured in GitLab. (Mostly, so we can pull them again to push them to the
+# Docker Hub.)
+# This script expects a single parameter, the base name of the docker image
+# such as sagemath or sagemath-dev.
+
+# ****************************************************************************
+#       Copyright (C) 2018 Julian Rüth <[email protected]>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#                  http://www.gnu.org/licenses/
+# ****************************************************************************
+
+[[ -z "$DOCKER_TAG" ]] && (echo "Can not push untagged build."; exit 0)
+[[ "$DOCKER_TAG" = "master" ]] && DOCKER_TAG=latest
+
+# Note that "set -x" prints the $CI_BUILD_TOKEN here but GitLab removes it
+# automatically from the log output.
+docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $CI_REGISTRY
+docker tag ${DOCKER_USER:-sagemath}/$1:$DOCKER_TAG $CI_REGISTRY_IMAGE/$1:$DOCKER_TAG
+docker push $CI_REGISTRY_IMAGE/$1:$DOCKER_TAG

.ci/setup-make-parallelity.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+# ****************************************************************************
+#       Copyright (C) 2018 Julian Rüth <[email protected]>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#                  http://www.gnu.org/licenses/
+# ****************************************************************************
+set -ex
+
+# Determine the number of threads that can run simultaneously on this system
+# (we might not have nproc available.)
+# Note that this value is incorrect for some CI providers (notably CircleCI:
+# https://circleci.com/docs/2.0/configuration-reference/#resource_class) which
+# provision fewer vCPUs than shown in /proc/cpuinfo. Also, setting this value
+# too high can lead to RAM being insufficient, so it's best to set this
+# variable manually in your CI configuration.
+[[ -z "$NTHREADS" ]] && NTHREADS=`grep -E '^processor' /proc/cpuinfo | wc -l` || true
+# Set -j and -l for make (though -l is probably stripped by Sage)
+[[ -z "$MAKEOPTS" ]] && MAKEOPTS="-j $NTHREADS -l $((NTHREADS-1)).8" || true
+# Not all parts of Sage seem to honor MAKEOPTS, so the current way of telling
+# the system which concurrency to use, seems to be setting $MAKE.
+[[ -z "$MAKE" ]] && MAKE="make $MAKEOPTS" || true

.ci/test-cli.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+# This script gets called from CI to run minimal tests on the sagemath image.
+
+# Usage: ./test-cli.sh sage-cli-image
+
+# ****************************************************************************
+#       Copyright (C) 2018 Julian Rüth <[email protected]>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#                  http://www.gnu.org/licenses/
+# ****************************************************************************
+set -exo pipefail
+
+echo "Checking that Sage starts and can calculate 1+1…"
+# Calculate 1+1 (remove startup messages and leading & trailing whitespace)
+TWO=`docker run "$1" sage -c "'print(1+1)'" | tail -1 | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//'`
+[[ "x$TWO" = "x2" ]]

.ci/test-dev.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+# This script gets called from CI to run minimal tests on the sagemath-dev image.
+# This script expects a single argument, the full name of the docker image to
+# test.
+
+# ****************************************************************************
+#       Copyright (C) 2018 Julian Rüth <[email protected]>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#                  http://www.gnu.org/licenses/
+# ****************************************************************************
+set -exo pipefail
+
+IMAGE="$1"
+
+. .ci/setup-make-parallelity.sh
+
+# Usage: timed_run limit args
+# Runs $IMAGE with args and check that it terminates with a zero exit code in at most limit seconds.
+function timed_run {
+    START=`date +%s`
+    docker run -e MAKE="$MAKE" "$IMAGE" "$2"
+    END=`date +%s`
+    TOTAL=$((END-START))
+    echo "Checking that \"$2\" was fast…"
+    [[ $TOTAL -lt $1 ]]
+}
+
+timed_run 60 true # runs make build
+# TODO: Can't we get this faster than that?
+timed_run 300 make # runs make build and then make

.ci/test-jupyter.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+# This script gets called from CI to run minimal tests on the sagemath-jupyter
+# image.
+
+# Usage: ./test-jupyter.sh sage-jupyter-image [host]
+
+# ****************************************************************************
+#       Copyright (C) 2018 Julian Rüth <[email protected]>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#                  http://www.gnu.org/licenses/
+# ****************************************************************************
+set -ex
+
+docker run --name sage-jupyter -p 8888:8888 -d "$1" "sage -n jupyter --no-browser --ip='*' --port=8888"
+echo "Checking that the Jupyter notebook is running…"
+sleep 10 # giving the server some time to start
+docker logs sage-jupyter
+wget --retry-connrefused --tries=10 --wait=3 "http://${2:-localhost}:8888"

.circleci/config.yml

@@ -0,0 +1,41 @@
+# This file configures automatic builds of Sage on [CircleCI](https://circleci.com).
+# To make the build time not too excessive, we seed the build cache with
+# sagemath/sagemath-dev:latest. When basic SPKGs change, this does not help much
+# and the full build might exceed CircleCI's limits for open source projcets (five
+# hours on 2 vCPUs as of early 2018.)
+# You might want to try to build locally or with GitLab CI, see
+# `.gitlab-ci.yml` for more options.
+# Note that we do not use workflows because it was not clear whether the docker
+# images sizes would not exceed the size limits of CircleCI workspaces. Also,
+# workflows would not make things faster. We do not get more machines for free
+# and the frequent loading of docker images probably exceeds the cost of the
+# actual tests.
+
+version: 2
+jobs:
+  # As https://circleci.com/docs/2.0/docker-layer-caching/ is a paid feature,
+  # we do build & test & release in one step.
+  build:
+    machine: true
+    steps:
+      - checkout
+      - run:
+          # The docker commands sometimes take a while to produce output
+          no_output_timeout: 30m
+          command: |
+            # CircleCI has no mechanism to hide secret variables.
+            # Therefore we roll our own to protect $SECRET_* variables.
+            . .ci/protect-secrets.sh
+
+            export DOCKER_TAG=${CIRCLE_TAG:-$CIRCLE_BRANCH}
+            # Build docker images
+            # TODO: Change this line to sagemath/sagemath-dev:latest
+            export ARTIFACT_BASE=saraedum/sagemath-dev:gitlabci
+            . .ci/build-docker.sh
+            # Test that the images work
+            . .ci/test-dev.sh $DOCKER_IMAGE_DEV
+            . .ci/test-cli.sh $DOCKER_IMAGE_CLI
+            . .ci/test-jupyter.sh $DOCKER_IMAGE_CLI localhost
+            # Push docker images to dockerhub if a dockerhub user has been configured
+            . .ci/push-dockerhub.sh sagemath-dev
+            . .ci/push-dockerhub.sh sagemath

.dockerignore

@@ -0,0 +1 @@
+.gitignore

.gitignore

@@ -81,3 +81,9 @@ $RECYCLE.BIN/
 ###########
 .ipynb_checkpoints
 Untitled*.ipynb
+
+#############################
+# GitLab CI generated files #
+#############################
+gitlab-build-docker.log
+