diff --git a/.ci/README.md b/.ci/README.md new file mode 100644 index 00000000000..e2b165cc518 --- /dev/null +++ b/.ci/README.md @@ -0,0 +1,9 @@ +# Continuous Integration (CI) + +We support several implementations of CI. All these implementations rely on +[docker](https://docker.com) in some way. This directory contains bits which +are shared between these CI implementations. The relevant docker files can be +found in `/docker/`. + +* [CircleCI](https://circleci.com) is configured in `/.circleci/`. +* [GitLab CI](https://gitlab.com) is configured in `/.gitlab-ci.yml`. diff --git a/.ci/build-docker.sh b/.ci/build-docker.sh new file mode 100755 index 00000000000..11651e5dd38 --- /dev/null +++ b/.ci/build-docker.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +# This script gets called from CI to build several flavours of docker images +# which contain Sage. + +# **************************************************************************** +# Copyright (C) 2018 Julian Rüth +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# http://www.gnu.org/licenses/ +# **************************************************************************** +set -ex + +[[ -z "$DOCKER_TAG" ]] && DOCKER_TAG=none +[[ "$DOCKER_TAG" = "master" ]] && DOCKER_TAG=latest + +. .ci/setup-make-parallelity.sh + +# We speed up the build process by copying built artifacts from ARTIFACT_BASE +# during docker build. See /docker/Dockerfile for more details. +ARTIFACT_BASE=${ARTIFACT_BASE:-sagemath/sagemath-dev:latest} + +# Seed our cache with $ARTIFACT_BASE if it exists +docker pull $ARTIFACT_BASE || true + +function docker_build { + time docker build -f docker/Dockerfile --build-arg "MAKE=${MAKE}" --build-arg ARTIFACT_BASE=$ARTIFACT_BASE $@ +} + +# We use a multi-stage build /docker/Dockerfile. For the caching to be +# effective, we populate the cache by building the make-all target. (Just +# building the last target is not enough as intermediate targets would be +# discarded from the cache and therefore the caching would fail for our actual +# builds below.) +docker_build --pull --tag make-all --target make-all . + +# Build the release image without build artifacts. +DOCKER_IMAGE_CLI=${DOCKER_USER:-sagemath}/sagemath:$DOCKER_TAG +docker_build --target sagemath --tag "$DOCKER_IMAGE_CLI" . +# Build the developer image with the build artifacts intact. +# Note: It's important to build the dev image last because it might be tagged as ARTIFACT_BASE. +DOCKER_IMAGE_DEV=${DOCKER_USER:-sagemath}/sagemath-dev:$DOCKER_TAG +docker_build --target sagemath-dev --tag "$DOCKER_IMAGE_DEV" . diff --git a/.ci/protect-secrets.sh b/.ci/protect-secrets.sh new file mode 100755 index 00000000000..a3ae9239988 --- /dev/null +++ b/.ci/protect-secrets.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +# This script protects all environment variables that start with "SECRET_". +# It puts them in a temporary file. The name of the variable contains the path +# of that file. This filename can then safely be used in `cat` even if `set +# -x` has been turned on. Also you can run "export" to understand the +# environment without danger. +# Be careful, however, not to use this like the following: +# docker login $DOCKER_USER $(cat $SECRET_DOCKER_PASS) +# as this would expose the password if `set -x` has been turned on. + +# **************************************************************************** +# Copyright (C) 2018 Julian Rüth +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# http://www.gnu.org/licenses/ +# **************************************************************************** + +set -eo pipefail +set +x + +function encrypt { + RET=`mktemp` + eval " echo \$$1" > "$RET" + echo $RET +} + +for name in `awk 'END { for (name in ENVIRON) { print name; } }' < /dev/null`; do +case "$name" in + SECRET_*) + export $name="$(encrypt $name)" + echo "Protected $name" + ;; +esac +done + +unset encrypt diff --git a/.ci/pull-gitlab.sh b/.ci/pull-gitlab.sh new file mode 100755 index 00000000000..01bb10cdd08 --- /dev/null +++ b/.ci/pull-gitlab.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +# This script gets called from CI to pull the Sage docker images that were +# built during the "build" phase to pull all the connected docker daemon +# (likely a docker-in-docker.) +# This script expects a single parameter, the base name of the docker image +# such as sagemath or sagemath-dev. +# The variable $DOCKER_IMAGE is set to the full name of the pulled image; +# source this script to use it. + +# **************************************************************************** +# Copyright (C) 2018 Julian Rüth +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# http://www.gnu.org/licenses/ +# **************************************************************************** + +set -ex + +[[ -z "$DOCKER_TAG" ]] && (echo "Can not pull untagged build."; exit 0) +[[ "$DOCKER_TAG" = "master" ]] && DOCKER_TAG=latest + +# Pull the built images from the gitlab registry and give them the original +# names they had after built. +# Note that "set -x" prints the $CI_BUILD_TOKEN here but GitLab removes it +# automatically from the log output. +docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $CI_REGISTRY +docker pull $CI_REGISTRY_IMAGE/$1:$DOCKER_TAG +DOCKER_IMAGE="${DOCKER_USER:-sagemath}/$1:$DOCKER_TAG" +docker tag $CI_REGISTRY_IMAGE/$1:$DOCKER_TAG $DOCKER_IMAGE diff --git a/.ci/push-dockerhub.sh b/.ci/push-dockerhub.sh new file mode 100755 index 00000000000..ca2c4906eec --- /dev/null +++ b/.ci/push-dockerhub.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +# This script gets called from CI to push our docker images to +# $DOCKER_USER/sagemath* on the Docker Hub. +# This script expects a single parameter, the base name of the docker image +# such as sagemath or sagemath-dev. + +# **************************************************************************** +# Copyright (C) 2018 Julian Rüth +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# http://www.gnu.org/licenses/ +# **************************************************************************** + +set -ex + +[[ -z "$DOCKER_TAG" ]] && (echo "Can not push untagged build."; exit 0) +[[ "$DOCKER_TAG" = "master" ]] && DOCKER_TAG=latest + +# Push the built images to the docker hub (and fail silently if +# DOCKER_USER/SECRET_DOCKER_PASS have not been configured.) +if [[ -z "$DOCKER_USER" || -z "$SECRET_DOCKER_PASS" ]]; then + echo "DOCKER_USER/SECRET_DOCKER_PASS variables have not been configured in your Continuous Integration setup. Not pushing built images to Docker Hub." +else + cat "$SECRET_DOCKER_PASS" | docker login -u $DOCKER_USER --password-stdin + docker push ${DOCKER_USER:-sagemath}/$1:$DOCKER_TAG +fi diff --git a/.ci/push-gitlab.sh b/.ci/push-gitlab.sh new file mode 100755 index 00000000000..9d98dca7e49 --- /dev/null +++ b/.ci/push-gitlab.sh @@ -0,0 +1,27 @@ +#!/bin/bash +set -ex + +# This script gets called from CI to push our docker images to registry +# configured in GitLab. (Mostly, so we can pull them again to push them to the +# Docker Hub.) +# This script expects a single parameter, the base name of the docker image +# such as sagemath or sagemath-dev. + +# **************************************************************************** +# Copyright (C) 2018 Julian Rüth +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# http://www.gnu.org/licenses/ +# **************************************************************************** + +[[ -z "$DOCKER_TAG" ]] && (echo "Can not push untagged build."; exit 0) +[[ "$DOCKER_TAG" = "master" ]] && DOCKER_TAG=latest + +# Note that "set -x" prints the $CI_BUILD_TOKEN here but GitLab removes it +# automatically from the log output. +docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $CI_REGISTRY +docker tag ${DOCKER_USER:-sagemath}/$1:$DOCKER_TAG $CI_REGISTRY_IMAGE/$1:$DOCKER_TAG +docker push $CI_REGISTRY_IMAGE/$1:$DOCKER_TAG diff --git a/.ci/setup-make-parallelity.sh b/.ci/setup-make-parallelity.sh new file mode 100644 index 00000000000..12f3a252179 --- /dev/null +++ b/.ci/setup-make-parallelity.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +# **************************************************************************** +# Copyright (C) 2018 Julian Rüth +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# http://www.gnu.org/licenses/ +# **************************************************************************** +set -ex + +# Determine the number of threads that can run simultaneously on this system +# (we might not have nproc available.) +# Note that this value is incorrect for some CI providers (notably CircleCI: +# https://circleci.com/docs/2.0/configuration-reference/#resource_class) which +# provision fewer vCPUs than shown in /proc/cpuinfo. Also, setting this value +# too high can lead to RAM being insufficient, so it's best to set this +# variable manually in your CI configuration. +[[ -z "$NTHREADS" ]] && NTHREADS=`grep -E '^processor' /proc/cpuinfo | wc -l` || true +# Set -j and -l for make (though -l is probably stripped by Sage) +[[ -z "$MAKEOPTS" ]] && MAKEOPTS="-j $NTHREADS -l $((NTHREADS-1)).8" || true +# Not all parts of Sage seem to honor MAKEOPTS, so the current way of telling +# the system which concurrency to use, seems to be setting $MAKE. +[[ -z "$MAKE" ]] && MAKE="make $MAKEOPTS" || true diff --git a/.ci/test-cli.sh b/.ci/test-cli.sh new file mode 100755 index 00000000000..a9e8c059ffe --- /dev/null +++ b/.ci/test-cli.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +# This script gets called from CI to run minimal tests on the sagemath image. + +# Usage: ./test-cli.sh sage-cli-image + +# **************************************************************************** +# Copyright (C) 2018 Julian Rüth +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# http://www.gnu.org/licenses/ +# **************************************************************************** +set -exo pipefail + +echo "Checking that Sage starts and can calculate 1+1…" +# Calculate 1+1 (remove startup messages and leading & trailing whitespace) +TWO=`docker run "$1" sage -c "'print(1+1)'" | tail -1 | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//'` +[[ "x$TWO" = "x2" ]] diff --git a/.ci/test-dev.sh b/.ci/test-dev.sh new file mode 100755 index 00000000000..60b59d6a5ca --- /dev/null +++ b/.ci/test-dev.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +# This script gets called from CI to run minimal tests on the sagemath-dev image. +# This script expects a single argument, the full name of the docker image to +# test. + +# **************************************************************************** +# Copyright (C) 2018 Julian Rüth +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# http://www.gnu.org/licenses/ +# **************************************************************************** +set -exo pipefail + +IMAGE="$1" + +. .ci/setup-make-parallelity.sh + +# Usage: timed_run limit args +# Runs $IMAGE with args and check that it terminates with a zero exit code in at most limit seconds. +function timed_run { + START=`date +%s` + docker run -e MAKE="$MAKE" "$IMAGE" "$2" + END=`date +%s` + TOTAL=$((END-START)) + echo "Checking that \"$2\" was fast…" + [[ $TOTAL -lt $1 ]] +} + +timed_run 60 true # runs make build +# TODO: Can't we get this faster than that? +timed_run 300 make # runs make build and then make diff --git a/.ci/test-jupyter.sh b/.ci/test-jupyter.sh new file mode 100755 index 00000000000..982f06fbeac --- /dev/null +++ b/.ci/test-jupyter.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +# This script gets called from CI to run minimal tests on the sagemath-jupyter +# image. + +# Usage: ./test-jupyter.sh sage-jupyter-image [host] + +# **************************************************************************** +# Copyright (C) 2018 Julian Rüth +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# http://www.gnu.org/licenses/ +# **************************************************************************** +set -ex + +docker run --name sage-jupyter -p 8888:8888 -d "$1" "sage -n jupyter --no-browser --ip='*' --port=8888" +echo "Checking that the Jupyter notebook is running…" +sleep 10 # giving the server some time to start +docker logs sage-jupyter +wget --retry-connrefused --tries=10 --wait=3 "http://${2:-localhost}:8888" diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 00000000000..13f8b567f5f --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,41 @@ +# This file configures automatic builds of Sage on [CircleCI](https://circleci.com). +# To make the build time not too excessive, we seed the build cache with +# sagemath/sagemath-dev:latest. When basic SPKGs change, this does not help much +# and the full build might exceed CircleCI's limits for open source projcets (five +# hours on 2 vCPUs as of early 2018.) +# You might want to try to build locally or with GitLab CI, see +# `.gitlab-ci.yml` for more options. +# Note that we do not use workflows because it was not clear whether the docker +# images sizes would not exceed the size limits of CircleCI workspaces. Also, +# workflows would not make things faster. We do not get more machines for free +# and the frequent loading of docker images probably exceeds the cost of the +# actual tests. + +version: 2 +jobs: + # As https://circleci.com/docs/2.0/docker-layer-caching/ is a paid feature, + # we do build & test & release in one step. + build: + machine: true + steps: + - checkout + - run: + # The docker commands sometimes take a while to produce output + no_output_timeout: 30m + command: | + # CircleCI has no mechanism to hide secret variables. + # Therefore we roll our own to protect $SECRET_* variables. + . .ci/protect-secrets.sh + + export DOCKER_TAG=${CIRCLE_TAG:-$CIRCLE_BRANCH} + # Build docker images + # TODO: Change this line to sagemath/sagemath-dev:latest + export ARTIFACT_BASE=saraedum/sagemath-dev:gitlabci + . .ci/build-docker.sh + # Test that the images work + . .ci/test-dev.sh $DOCKER_IMAGE_DEV + . .ci/test-cli.sh $DOCKER_IMAGE_CLI + . .ci/test-jupyter.sh $DOCKER_IMAGE_CLI localhost + # Push docker images to dockerhub if a dockerhub user has been configured + . .ci/push-dockerhub.sh sagemath-dev + . .ci/push-dockerhub.sh sagemath diff --git a/.dockerignore b/.dockerignore new file mode 120000 index 00000000000..3e4e48b0b5f --- /dev/null +++ b/.dockerignore @@ -0,0 +1 @@ +.gitignore \ No newline at end of file diff --git a/.gitignore b/.gitignore index 101d349690c..498b92c73b9 100644 --- a/.gitignore +++ b/.gitignore @@ -81,3 +81,9 @@ $RECYCLE.BIN/ ########### .ipynb_checkpoints Untitled*.ipynb + +############################# +# GitLab CI generated files # +############################# +gitlab-build-docker.log + diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 00000000000..e256dd1effe --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,99 @@ +# This file configures automatic builds of Sage on [GitLab](https://gitlab.com). +# To make the build time not too excessive, we seed the build cache with +# sagemath/sagemath-dev:latest. When basic SPKGs changed, this does not help +# much and the full build might the set time limit in GitLab. You can increase +# that limit in Settings → CI/CD. +# You can also provision your own private more powerful runner in the same +# place; or set up your favourite cloud service to provide an on-demand +# autoscale runner. + +image: docker:latest + +stages: + - build + - test + - release + +variables: + DOCKER_TAG: $CI_COMMIT_REF_SLUG + # Builds are very I/O intensive; make sure we have a fast file system. + DOCKER_DRIVER: overlay2 + # TODO: Change this line to sagemath/sagemath-dev:latest + ARTIFACT_BASE: saraedum/sagemath-dev:gitlabci + +before_script: + # GitLab has no mechanism yet to hide secret variables: https://gitlab.com/gitlab-org/gitlab-ce/issues/13784 + # So we roll our own which protects all variables that start with SECRET_ + - . .ci/protect-secrets.sh + +# We use docker-in-docker to build our docker images. It can be faster to +# expose your outer docker daemon by mounting /var/run/docker.sock to +# /var/run/docker.sock and setting DOCKER_HOST in Settings -> CI/CD -> Secret +# variable to unix:///var/run/docker.sock +services: +- docker:dind + +# Build Sage and its documentation. +# The build starts from the build artifacts of ARTIFACT_BASE which is usually +# much faster than building from a clean checkout of Sage. +build-from-latest: &build + stage: build + artifacts: + when: always + paths: + - gitlab-build-docker.log + expire_in: 1 month + script: + # The output of the build can get larger than gitlab.com's limit; only print the first 3MB. + - . .ci/build-docker.sh | tee gitlab-build-docker.log | head -c 3m + - . .ci/push-gitlab.sh sagemath-dev + - . .ci/push-gitlab.sh sagemath + except: + - master + +# Build Sage and its documentation from a clean checkout of Sage. +# Note that this takes a very long time. You probably want to run this on your +# own gitlab-runner and increase the standard GitLab time limit for CI runs. +build-from-clean: + << : *build + variables: + ARTIFACT_BASE: "source-clean" + only: + - master + +test-dev: + stage: test + script: + - . .ci/pull-gitlab.sh sagemath-dev + - . .ci/test-dev.sh "$DOCKER_IMAGE" + +test-cli: + stage: test + script: + - . .ci/pull-gitlab.sh sagemath + - . .ci/test-cli.sh "$DOCKER_IMAGE" + +test-jupyter: + stage: test + script: + # Force usage of docker-in-docker (and don't start docker on a bind-mounted + # /var/run/docker.sock set through a proivate GitLab CI variable) so that + # the -p flag to docker run works as expected. + - export DOCKER_HOST='tcp://docker:2375' + - apk update + - apk add wget + - . .ci/pull-gitlab.sh sagemath + - . .ci/test-jupyter.sh "$DOCKER_IMAGE" docker + +# Pushes the built images to Docker Hub if the Settings -> CI/CD -> Secret +# variables DOCKER_USER and SECRET_DOCKER_PASS have been set up. +dockerhub: + stage: release + only: + - branches + - tags + script: + - . .ci/pull-gitlab.sh sagemath-dev + - . .ci/push-dockerhub.sh sagemath-dev + - . .ci/pull-gitlab.sh sagemath + - . .ci/push-dockerhub.sh sagemath diff --git a/Makefile b/Makefile index 92457458fe5..d2dfc71256d 100644 --- a/Makefile +++ b/Makefile @@ -68,6 +68,7 @@ misc-clean: rm -f build/make/Makefile build/make/Makefile-auto rm -f .BUILDSTART +# TODO: What is a "bdist"? A binary distribution? bdist-clean: clean $(MAKE) misc-clean @@ -89,6 +90,31 @@ maintainer-clean: distclean bootstrap-clean micro_release: bdist-clean sagelib-clean @echo "Stripping binaries ..." LC_ALL=C find local/lib local/bin -type f -exec strip '{}' ';' 2>&1 | grep -v "File format not recognized" | grep -v "File truncated" || true + @echo "Removing .py files that have a corresponding .pyc or .pyo file as they are not needed when running code with CPython..." + find local/lib/python* -name '*.py' | while IFS= read -r fname; do [ -e "$${fname}c" -o -e "$${fname}o" ] && rm "$$fname"; done || true + @echo "Removing sphinx artifacts..." + rm -rf local/share/doc/sage/doctrees local/share/doc/sage/inventory + @echo "Removing unnecessary files & directories - make will not be functional afterwards anymore" + @# We need src/sage/ for introspection with "??" + @# We need src/sage/bin/ for the scripts that invoke Sage + @# We need sage, the script to start Sage + @# We need local/, the dependencies and the built Sage library itself. + @# We keep VERSION.txt. + @# We keep COPYING.txt so we ship a license with this distribution. + find . -name . -o -prune ! -name src ! -name sage ! -name local ! -name VERSION.txt ! -name COPYING.txt ! -name build -exec rm -rf \{\} \; + cd src && find . -name . -o -prune ! -name sage ! -name bin -exec rm -rf \{\} \; + +# Leaves everything that is needed to make the next "make" fast but removes +# all the cheap build artifacts that can be quickly regenerated. +fast-rebuild-clean: misc-clean bdist-clean + rm -rf upstream/ + rm -rf src/build/temp.* + # Without site-packages/sage sage does not start but copying/compiling + # them from src/build is very fast. + rm -rf local/lib/python*/site-packages/sage + # The .py files in src/build are restored from src/sage without their + # mtimes changed. + find src/build -name '*.py' -exec rm \{\} \; TESTALL = ./sage -t --all PTESTALL = ./sage -t -p --all diff --git a/configure.ac b/configure.ac index 5c61a89704e..2d86803c6c9 100644 --- a/configure.ac +++ b/configure.ac @@ -285,9 +285,9 @@ fi AC_CHECK_PROG(found_latex, latex, yes, no) if test x$found_latex != xyes then - AC_MSG_WARN([You do not have 'latex', which is recommended, but not]) - AC_MSG_WARN([required. Latex is only really used for building pdf]) - AC_MSG_WARN([documents and for %latex mode in the AC_PACKAGE_NAME notebook.]) + AC_MSG_NOTICE([You do not have 'latex', which is recommended, but not]) + AC_MSG_NOTICE([required. Latex is only really used for building pdf]) + AC_MSG_NOTICE([documents and for %latex mode in the AC_PACKAGE_NAME notebook.]) fi # Check that perl is available, with version 5.8.0 or later. diff --git a/docker/.gitignore b/docker/.gitignore new file mode 100644 index 00000000000..a38152cb5ee --- /dev/null +++ b/docker/.gitignore @@ -0,0 +1,2 @@ +# Stores the commit that was used to create a sagemath-dev image +commit diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 00000000000..503f9eb579f --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,176 @@ +################################################################################ +# SageMath images for Docker # +################################################################################ +# This is a description of the layout of this Dockerfile; for details on the # +# created docker images, see the README.md please. # +# # +# This Dockerfile builds sagemath (for end-users) and sagemath-dev (for # +# developers) it consists of lots of intermediate targets, mostly to shrink # +# the resulting images but also to make this hopefully easier to maintain. # +# The aims of this Dockerfile are: # +# (1) Make it build in reasonable time. # +# (2) It should be self-contained and work on its own, i.e., just by invoking # +# docker build without any external orchestration script. # +# # +# The idea to achieve (1) is to reuse the build artifacts from the latest # +# master build. This is slightly against the philosophy of a Dockerfile (which # +# should produce perfectly reproducible outputs) but building Sage from scratch# +# just takes too long at the moment to do this all the time. ARTIFACT_BASE # +# controls which build artifacts are used. You probably want to set this to # +# sagemath/sagemath-dev:latest which takes the latest build from the official # +# master branch. The default is source-clean which builds Sage from scratch. # +# If you want to understand how this works, have a look at source-from-context # +# which merges ARTIFACT_BASE with the context, i.e., the contents of the sage # +# source directory. # +################################################################################ + + +ARG ARTIFACT_BASE=source-clean + +################################################################################ +# Image containing the run-time dependencies for Sage # +################################################################################ +FROM ubuntu:xenial as run-time-dependencies +LABEL maintainer="Erik M. Bray , Julian Rüth " +# Set sane defaults for common environment variables. +ENV LC_ALL C.UTF-8 +ENV LANG C.UTF-8 +ENV SHELL /bin/bash +# Create symlinks for sage and sagemath - we copy a built sage to the target of these symlinks later. +ARG SAGE_ROOT=/home/sage/sage +RUN ln -s "$SAGE_ROOT/sage" /usr/bin/sage +RUN ln -s /usr/bin/sage /usr/bin/sagemath +# Sage needs the fortran libraries at run-time because we do not build gfortran +# with Sage but use the system's. +# We also install sudo for the sage user, see below. +RUN apt-get update -qq \ + && apt-get install -y --no-install-recommends gfortran sudo \ + && apt-get clean \ + && rm -r /var/lib/apt/lists/* +# Sage refuses to build as root, so we add a "sage" user that can sudo without a password. +# We also want this user at runtime as some commands in sage know about the user that was used during build. +ARG HOME=/home/sage +RUN adduser --quiet --shell /bin/bash --gecos "Sage user,101,," --disabled-password --home "$HOME" sage \ + && echo "sage ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers.d/01-sage \ + && chmod 0440 /etc/sudoers.d/01-sage +# Run everything from now on as the sage user in sage's home +USER sage +ENV HOME $HOME +WORKDIR $HOME + +################################################################################ +# Image containing everything so that a make in a clone of the Sage repository # +# completes without errors # +################################################################################ +FROM run-time-dependencies as build-time-dependencies +# Install the build time dependencies & git +RUN sudo apt-get update -qq \ + && sudo apt-get install -y wget build-essential automake m4 dpkg-dev sudo python libssl-dev git \ + && sudo apt-get clean \ + && sudo rm -r /var/lib/apt/lists/* + +################################################################################ +# Image with an empty git repository in $SAGE_ROOT. # +################################################################################ +FROM build-time-dependencies as source-clean +ARG SAGE_ROOT=/home/sage/sage +RUN mkdir -p "$SAGE_ROOT" +WORKDIR $SAGE_ROOT +RUN git init +RUN git remote add trac git@trac.sagemath.org:sage.git + +################################################################################ +# Image with the build context added, i.e., the directory from which `docker # +# build` has been called in a separate directory so we can copy files from # +# there. # +# This blows up the size of this docker image significantly, but we only use # +# this image to create artifacts for our final image. # +# Warning: If you set ARTIFACT_BASE to something else than source-clean, the # +# build is not going to use the build-time-dependencies target but rely on # +# whatever tools are installed in ARTIFACT_BASE. # +################################################################################ +FROM $ARTIFACT_BASE as source-from-context +WORKDIR $HOME +COPY --chown=sage:sage . sage-context +# Checkout the commit that is in $HOME/sage-context +ARG SAGE_ROOT=/home/sage/sage +WORKDIR $SAGE_ROOT +RUN git fetch "$HOME/sage-context" HEAD \ + && git reset FETCH_HEAD \ + && git checkout -f FETCH_HEAD \ + && git log -1 --format=%H > docker/commit \ + && rm -rf .git +# Copy over all the untracked/staged/unstaged changes from sage-context +WORKDIR $HOME/sage-context +RUN if git status --porcelain | read CHANGES; then \ + git -c user.name=docker-build -c user.email=docker-build@sage.invalid stash -u \ + && git stash show -p > "$HOME"/sage-context.patch; \ + else \ + touch "$HOME"/sage-context.patch; \ + fi +WORKDIR $SAGE_ROOT +RUN patch -p1 < "$HOME"/sage-context.patch + +################################################################################ +# Image with a built sage but without sage's documentation. # +################################################################################ +FROM source-from-context as make-build +# Make sure that the results runs on most CPUs. +ENV SAGE_FAT_BINARY yes +# Just to be sure Sage doesn't try to build its own GCC (even though +# it shouldn't with a recent GCC package from the system and with gfortran) +ENV SAGE_INSTALL_GCC no +# Make everything in the build use multiple cores (this causes trouble for some packages outside Sage but it still seems to be they way Sage is doing this.) +ARG MAKE="make -j2" +ENV MAKE $MAKE +RUN make build + +################################################################################ +# Image with a full build of sage and its documentation. # +################################################################################ +FROM make-build as make-all +RUN make + +################################################################################ +# Image with a full build of sage and its documentation but everything # +# stripped that can be quickly rebuild by make. # +################################################################################ +FROM make-all as make-fast-rebuild-clean +RUN make fast-rebuild-clean + +################################################################################ +# A releasable (relatively small, but still huge) image of this build with all # +# the build artifacts intact so developers can make changes and rebuild # +# quickly # +################################################################################ +FROM source-clean as sagemath-dev +ARG SAGE_ROOT=/home/sage/sage +COPY --chown=sage:sage --from=make-fast-rebuild-clean $SAGE_ROOT $SAGE_ROOT +COPY ./docker/entrypoint-dev.sh /usr/local/bin/sage-entrypoint +ENTRYPOINT ["/usr/local/bin/sage-entrypoint"] +CMD ["bash"] + +################################################################################ +# Image with a full build of sage, ready to release, i.e., with stripped # +# binaries and some extras to run the jupyter notebook. # +################################################################################ +FROM make-all as make-release +RUN sage -pip install terminado "notebook>=5" "ipykernel>=4.6" +RUN sage -i gap_jupyter singular_jupyter pari_jupyter +RUN make micro_release + +################################################################################ +# A releasable (relatively small) image which contains a copy of sage without # +# temporary build artifacts which is set up to start the command line # +# interface if no parameters are passed in. # +################################################################################ +FROM run-time-dependencies as sagemath +ARG SAGE_ROOT=/home/sage/sage +COPY --chown=sage:sage --from=make-release $SAGE_ROOT/ $SAGE_ROOT/ +# Put scripts to start gap, gp, maxima, ... in /usr/bin +WORKDIR $SAGE_ROOT +RUN sudo sage --nodotsage -c "install_scripts('/usr/bin')" +COPY ./docker/entrypoint.sh /usr/local/bin/sage-entrypoint +ENTRYPOINT ["/usr/local/bin/sage-entrypoint"] +EXPOSE 8888 +CMD ["sage"] diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 00000000000..9b653689031 --- /dev/null +++ b/docker/README.md @@ -0,0 +1,51 @@ +[![License: GPL v3](https://img.shields.io/badge/License-GPL%20v3-blue.svg)](https://github.com/sagemath/sage/COPYING.txt) [![Maintained](https://img.shields.io/maintenance/yes/2018.svg)](https://github.com/sagemath/sage/commits/master) [![CircleCI](https://circleci.com/gh/saraedum/sage.svg?style=svg)](https://circleci.com/gh/saraedum/sage) [![GitLab CI](https://gitlab.com/saraedum/sage/badges/gitlabci/pipeline.svg)](https://gitlab.com/saraedum/sage/commits/gitlabci) + +# Supported tags + +* `latest` — the stable `master` branch +* `x.x.x` — all stable releases of Sage are tagged with their version number. +* `develop` — the current development version of Sage which gets merged into the `master` branch when a new version of Sage is released + +# What is SageMath + +SageMath is a free open-source mathematics software system licensed under the GPL. It builds on top of many existing open-source packages: NumPy, SciPy, matplotlib, Sympy, Maxima, GAP, FLINT, R and many more. Access their combined power through a common, Python-based language or directly via interfaces or wrappers. + +**Mission**: *Creating a viable free open source alternative to Magma, Maple, Mathematica and Matlab.* + +# What's in this image + +There are several flavours of this image. + +* [`sagemath/sagemath`![image size](https://img.shields.io/microbadger/image-size/sagemath/sagemath.svg)](https://hub.docker.com/sagemath/sagemath) contains everything necessary to run Sage on the command line. Run it with: + ``` + docker run -it sagemath/sagemath:latest + ``` + You can start a graphical [Jupyter Notebook](https://jupyter.org) at http://localhost:8888 instead. To use the notebook, follow the instructions printed when you run: + ``` + docker run -p8888:8888 sagemath/sagemath:latest "sage -n jupyter --no-browser --ip='*' --port=8888" + ``` +* [`sagemath/sagemath-dev`![image size](https://img.shields.io/microbadger/image-size/sagemath/sagemath-dev.svg)](https://hub.docker.com/sagemath/sagemath-dev) contains all the build artifacts to rebuild Sage quickly. This version is probably only relevant for Sage developers. Run this image with: + ``` + docker run -it sagemath/sagemath-dev:latest + ``` + This triggers a rebuild and drops you in a shell afterwards. Note that the git repository has been emptied to save space. If you want to use git, fetch from your git repository with `git fetch trac` and go to the commit that was used to create this image with + ``` + git reset $(cat docker/commit) + ``` + +# How to build your own SageMath images + +Run `docker build -f docker/Dockerfile --target TARGET .` in the Sage repository with `TARGET` one of `sage`, `sage-jupyter`, or `dev`. + +# How these images get updated + +Every push to our [github repository](https://github.com/sagemath/sage) triggers a build in [CircleCI](https://circleci.com) which builds and pushes the docker images. +A push to master also triggers a "build" on our [Docker Hub](https://hub.docker.com) repositories. The latter build is mostly disabled by the `hooks/` and only updates the `README.md`. + +Every push to our [GitLab repository](https://gitlab.com/sagemath/sage) triggers a pipeline in GitLab CI. This build also pushes images to Docker Hub. + +Have a look at `.circleci/` and `.gitlab-ci.yml` if you want to setup either continuous integration service for your own fork of the SageMath repository. + +# License + +The whole Sage software distribution is licensed under the General Public License, version 3. More details can be found in our [COPYING.txt](https://github.com/sagemath/sage/blob/master/COPYING.txt) diff --git a/docker/entrypoint-dev.sh b/docker/entrypoint-dev.sh new file mode 100755 index 00000000000..67299a5a92d --- /dev/null +++ b/docker/entrypoint-dev.sh @@ -0,0 +1,4 @@ +#!/bin/bash +set -e +make build +exec "$@" diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh new file mode 100755 index 00000000000..692b5c83390 --- /dev/null +++ b/docker/entrypoint.sh @@ -0,0 +1,2 @@ +#!/bin/bash +exec sage -sh -c "$*" diff --git a/docker/hooks/build b/docker/hooks/build new file mode 100644 index 00000000000..b23e55619b2 --- /dev/null +++ b/docker/hooks/build @@ -0,0 +1 @@ +#!/bin/true diff --git a/docker/hooks/push b/docker/hooks/push new file mode 100644 index 00000000000..b23e55619b2 --- /dev/null +++ b/docker/hooks/push @@ -0,0 +1 @@ +#!/bin/true diff --git a/src/sage_setup/docbuild/__init__.py b/src/sage_setup/docbuild/__init__.py index b8f79c2269a..bf48bf57cdf 100644 --- a/src/sage_setup/docbuild/__init__.py +++ b/src/sage_setup/docbuild/__init__.py @@ -911,7 +911,12 @@ def get_new_and_updated_modules(self): except ImportError as err: logger.error("Warning: Could not import %s %s", module_name, err) raise - newtime = os.path.getmtime(sys.modules[module_name].__file__) + + module_filename = sys.modules[module_name].__file__ + if (module_filename.endswith('.pyc') or module_filename.endswith('.pyo')): + source_filename = module_filename[:-1] + if (os.path.exists(source_filename)): module_filename = source_filename + newtime = os.path.getmtime(module_filename) if newtime > mtime: updated_modules.append(module_name)