From 38f3236fc08b87d0cccc0a8af972d0679f0317de Mon Sep 17 00:00:00 2001 From: Gregory Arndt Date: Mon, 16 Nov 2015 12:01:15 -0600 Subject: [PATCH 01/74] Bug 1226413 - Allow task images to be built upon push r=wcosta --HG-- extra : commitid : A6frApCneR2 extra : rebase_source : 28cac7f5eef2f9c71db699fbd179bd6cc16d447d extra : amend_source : 8464b96cc2f7aacd9076f8b2e3e4bb9a1af856f2 --- testing/docker/image_builder/Dockerfile | 34 ++++++++++++++++++ testing/docker/image_builder/REGISTRY | 1 + testing/docker/image_builder/VERSION | 1 + .../docker/image_builder/bin/build_image.sh | 35 +++++++++++++++++++ 4 files changed, 71 insertions(+) create mode 100644 testing/docker/image_builder/Dockerfile create mode 100644 testing/docker/image_builder/REGISTRY create mode 100644 testing/docker/image_builder/VERSION create mode 100755 testing/docker/image_builder/bin/build_image.sh diff --git a/testing/docker/image_builder/Dockerfile b/testing/docker/image_builder/Dockerfile new file mode 100644 index 000000000..2c1365862 --- /dev/null +++ b/testing/docker/image_builder/Dockerfile @@ -0,0 +1,34 @@ +FROM ubuntu:14.04 + +WORKDIR /home/worker/bin + +RUN apt-get update && apt-get install -y apt-transport-https +RUN sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 36A1D7869245C8950F966E92D8576A8BA88D21E9 && \ + sudo sh -c "echo deb https://get.docker.io/ubuntu docker main\ + > /etc/apt/sources.list.d/docker.list" +RUN apt-get update && apt-get install -y \ + lxc-docker-1.6.1 \ + curl \ + wget \ + git \ + mercurial \ + tar \ + zip \ + unzip \ + vim \ + sudo \ + ca-certificates \ + build-essential + +ENV NODE_VERSION v0.12.4 +RUN cd /usr/local/ && \ + curl https://nodejs.org/dist/$NODE_VERSION/node-$NODE_VERSION-linux-x64.tar.gz | tar -xz --strip-components 1 && \ + node -v + +RUN npm install -g taskcluster-vcs@2.3.11 + +ADD bin /home/worker/bin +RUN chmod +x /home/worker/bin/* + +# Set a default command useful for debugging +CMD ["/bin/bash", "--login"] diff --git a/testing/docker/image_builder/REGISTRY b/testing/docker/image_builder/REGISTRY new file mode 100644 index 000000000..cb1e1bb48 --- /dev/null +++ b/testing/docker/image_builder/REGISTRY @@ -0,0 +1 @@ +taskcluster diff --git a/testing/docker/image_builder/VERSION b/testing/docker/image_builder/VERSION new file mode 100644 index 000000000..b1e80bb24 --- /dev/null +++ b/testing/docker/image_builder/VERSION @@ -0,0 +1 @@ +0.1.3 diff --git a/testing/docker/image_builder/bin/build_image.sh b/testing/docker/image_builder/bin/build_image.sh new file mode 100755 index 000000000..c9f1f9e70 --- /dev/null +++ b/testing/docker/image_builder/bin/build_image.sh @@ -0,0 +1,35 @@ +#!/bin/bash -vex + +# Set bash options to exit immediately if a pipeline exists non-zero, expand +# print a trace of commands, and make output verbose (print shell input as it's +# read) +# See https://www.gnu.org/software/bash/manual/html_node/The-Set-Builtin.html +set -x -e -v + +# Prefix errors with taskcluster error prefix so that they are parsed by Treeherder +raise_error() { + echo + echo "[taskcluster:error] Error: $1" + exit 1 +} + +# Ensure that the PROJECT is specified so the image can be indexed +test -n "$PROJECT" || raise_error "Project must be provided." +test -n "$HASH" || raise_error "Context Hash must be provided." + +mkdir /artifacts + +if [ ! -z "$CONTEXT_URL" ]; then + mkdir /context + curl -L "$CONTEXT_URL" | tar -xz --strip-components 1 -C /context + CONTEXT_PATH=/context +else + tc-vcs checkout /home/worker/workspace/src $BASE_REPOSITORY $HEAD_REPOSITORY $HEAD_REV $HEAD_REF + CONTEXT_PATH=/home/worker/workspace/src/$CONTEXT_PATH +fi + +test -d $CONTEXT_PATH || raise_error "Context Path $CONTEXT_PATH does not exist." +test -f "$CONTEXT_PATH/Dockerfile" || raise_error "Dockerfile must be present in $CONTEXT_PATH." + +docker build -t $PROJECT:$HASH $CONTEXT_PATH +docker save $PROJECT:$HASH > /artifacts/image.tar From f82285b56fbd53db7a72f9bdd63e3d6f34c27fcd Mon Sep 17 00:00:00 2001 From: Gregory Arndt Date: Fri, 22 Apr 2016 06:43:24 -0500 Subject: [PATCH 02/74] Bug 1266719 - Add retry and timeout options when downloading image context r=wcosta MozReview-Commit-ID: BTpHZa4PX0G --HG-- extra : rebase_source : 4e8487755acf37a4eef8192061abf00262d2407b --- testing/docker/image_builder/VERSION | 2 +- testing/docker/image_builder/bin/build_image.sh | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/testing/docker/image_builder/VERSION b/testing/docker/image_builder/VERSION index b1e80bb24..845639eef 100644 --- a/testing/docker/image_builder/VERSION +++ b/testing/docker/image_builder/VERSION @@ -1 +1 @@ -0.1.3 +0.1.4 diff --git a/testing/docker/image_builder/bin/build_image.sh b/testing/docker/image_builder/bin/build_image.sh index c9f1f9e70..ef34b7edd 100755 --- a/testing/docker/image_builder/bin/build_image.sh +++ b/testing/docker/image_builder/bin/build_image.sh @@ -9,7 +9,7 @@ set -x -e -v # Prefix errors with taskcluster error prefix so that they are parsed by Treeherder raise_error() { echo - echo "[taskcluster:error] Error: $1" + echo "[taskcluster-image-build:error] $1" exit 1 } @@ -21,7 +21,9 @@ mkdir /artifacts if [ ! -z "$CONTEXT_URL" ]; then mkdir /context - curl -L "$CONTEXT_URL" | tar -xz --strip-components 1 -C /context + if ! curl -L --retry 5 --connect-timeout 30 "$CONTEXT_URL" | tar -xz --strip-components 1 -C /context; then + raise_error "Error downloading image context from decision task." + fi CONTEXT_PATH=/context else tc-vcs checkout /home/worker/workspace/src $BASE_REPOSITORY $HEAD_REPOSITORY $HEAD_REV $HEAD_REF From acbe39d1b14d16114a3123fe02cb78bad81d06d5 Mon Sep 17 00:00:00 2001 From: Gregory Arndt Date: Tue, 24 May 2016 08:57:18 -0500 Subject: [PATCH 03/74] Bug 1275276 - Fail if image context cannot be downloaded r=jhford MozReview-Commit-ID: DGOfH3MF2Sb --HG-- extra : rebase_source : 0de27a48615b7abef4fa52e701a8e9d554c0cb00 --- testing/docker/image_builder/VERSION | 2 +- testing/docker/image_builder/bin/build_image.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/testing/docker/image_builder/VERSION b/testing/docker/image_builder/VERSION index 845639eef..9faa1b7a7 100644 --- a/testing/docker/image_builder/VERSION +++ b/testing/docker/image_builder/VERSION @@ -1 +1 @@ -0.1.4 +0.1.5 diff --git a/testing/docker/image_builder/bin/build_image.sh b/testing/docker/image_builder/bin/build_image.sh index ef34b7edd..9095ff79a 100755 --- a/testing/docker/image_builder/bin/build_image.sh +++ b/testing/docker/image_builder/bin/build_image.sh @@ -21,7 +21,7 @@ mkdir /artifacts if [ ! -z "$CONTEXT_URL" ]; then mkdir /context - if ! curl -L --retry 5 --connect-timeout 30 "$CONTEXT_URL" | tar -xz --strip-components 1 -C /context; then + if ! curl -L --retry 5 --connect-timeout 30 --fail "$CONTEXT_URL" | tar -xz --strip-components 1 -C /context; then raise_error "Error downloading image context from decision task." fi CONTEXT_PATH=/context From 7b35ba1d452f2af1cc795bbc37bbb1aab396d9c1 Mon Sep 17 00:00:00 2001 From: CuriousLearner Date: Thu, 20 Oct 2016 18:25:34 +0530 Subject: [PATCH 04/74] Bug 1302763 - Move docker images out of testing/docker into taskcluster/docker; r=dustin MozReview-Commit-ID: 7v1uCDB5qoN --HG-- rename : testing/docker/README.md => taskcluster/docker/README.md rename : testing/docker/REGISTRY => taskcluster/docker/REGISTRY rename : testing/docker/android-gradle-build/README.md => taskcluster/docker/android-gradle-build/README.md rename : testing/docker/image_builder/REGISTRY => taskcluster/docker/android-gradle-build/REGISTRY rename : testing/docker/android-gradle-build/VERSION => taskcluster/docker/android-gradle-build/VERSION rename : testing/docker/android-gradle-build/bin/after.sh => taskcluster/docker/android-gradle-build/bin/after.sh rename : testing/docker/android-gradle-build/bin/before.sh => taskcluster/docker/android-gradle-build/bin/before.sh rename : testing/docker/android-gradle-build/bin/build.sh => taskcluster/docker/android-gradle-build/bin/build.sh rename : testing/docker/desktop-build/bin/checkout-script.sh => taskcluster/docker/android-gradle-build/bin/checkout-script.sh rename : testing/docker/android-gradle-build/bin/checkout-sources.sh => taskcluster/docker/android-gradle-build/bin/checkout-sources.sh rename : testing/docker/desktop-build/buildprops.json => taskcluster/docker/android-gradle-build/buildprops.json rename : testing/docker/desktop-test/dot-files/config/pip/pip.conf => taskcluster/docker/android-gradle-build/dot-config/pip/pip.conf rename : testing/docker/desktop-build/oauth.txt => taskcluster/docker/android-gradle-build/oauth.txt rename : testing/docker/base-build/Dockerfile => taskcluster/docker/base-build/Dockerfile rename : testing/docker/base-build/VERSION => taskcluster/docker/base-build/VERSION rename : testing/docker/base-build/system-setup.sh => taskcluster/docker/base-build/system-setup.sh rename : testing/docker/base-test/Dockerfile => taskcluster/docker/base-test/Dockerfile rename : testing/docker/image_builder/REGISTRY => taskcluster/docker/base-test/REGISTRY rename : testing/docker/base-test/VERSION => taskcluster/docker/base-test/VERSION rename : testing/docker/base-test/sources.list => taskcluster/docker/base-test/sources.list rename : testing/docker/image_builder/REGISTRY => taskcluster/docker/builder/REGISTRY rename : testing/docker/centos6-build-upd/Dockerfile => taskcluster/docker/centos6-build-upd/Dockerfile rename : testing/docker/image_builder/REGISTRY => taskcluster/docker/centos6-build-upd/REGISTRY rename : testing/docker/centos6-build-upd/VERSION => taskcluster/docker/centos6-build-upd/VERSION rename : testing/docker/centos6-build/Dockerfile => taskcluster/docker/centos6-build/Dockerfile rename : testing/docker/image_builder/REGISTRY => taskcluster/docker/centos6-build/REGISTRY rename : testing/docker/decision/VERSION => taskcluster/docker/centos6-build/VERSION rename : testing/docker/centos6-build/hgrc => taskcluster/docker/centos6-build/hgrc rename : testing/docker/centos6-build/system-setup.sh => taskcluster/docker/centos6-build/system-setup.sh rename : testing/docker/decision/README.md => taskcluster/docker/decision/README.md rename : testing/docker/image_builder/REGISTRY => taskcluster/docker/decision/REGISTRY rename : testing/docker/decision/VERSION => taskcluster/docker/decision/VERSION rename : testing/docker/decision/system-setup.sh => taskcluster/docker/decision/system-setup.sh rename : testing/docker/desktop-build/bin/build.sh => taskcluster/docker/desktop-build/bin/build.sh rename : testing/docker/desktop-build/bin/checkout-script.sh => taskcluster/docker/desktop-build/bin/checkout-script.sh rename : testing/docker/desktop-build/bin/checkout-sources.sh => taskcluster/docker/desktop-build/bin/checkout-sources.sh rename : testing/docker/desktop-build/buildprops.json => taskcluster/docker/desktop-build/buildprops.json rename : testing/docker/desktop-test/dot-files/config/pip/pip.conf => taskcluster/docker/desktop-build/dot-config/pip/pip.conf rename : testing/docker/desktop-build/oauth.txt => taskcluster/docker/desktop-build/oauth.txt rename : testing/docker/desktop-test/apport => taskcluster/docker/desktop-test/apport rename : testing/docker/desktop-test/buildprops.json => taskcluster/docker/desktop-test/buildprops.json rename : testing/docker/desktop1604-test/deja-dup-monitor.desktop => taskcluster/docker/desktop-test/deja-dup-monitor.desktop rename : testing/docker/desktop-test/dot-files/config/pip/pip.conf => taskcluster/docker/desktop-test/dot-files/config/pip/pip.conf rename : testing/docker/desktop-test/dot-files/config/user-dirs.dirs => taskcluster/docker/desktop-test/dot-files/config/user-dirs.dirs rename : testing/docker/tester/dot-config/user-dirs.locale => taskcluster/docker/desktop-test/dot-files/config/user-dirs.locale rename : testing/docker/tester/dot-pulse/default.pa => taskcluster/docker/desktop-test/dot-files/pulse/default.pa rename : testing/docker/desktop1604-test/fonts.conf => taskcluster/docker/desktop-test/fonts.conf rename : testing/docker/desktop-test/jockey-gtk.desktop => taskcluster/docker/desktop-test/jockey-gtk.desktop rename : testing/docker/desktop-test/motd => taskcluster/docker/desktop-test/motd rename : testing/docker/desktop-test/release-upgrades => taskcluster/docker/desktop-test/release-upgrades rename : testing/docker/desktop-test/taskcluster-interactive-shell => taskcluster/docker/desktop-test/taskcluster-interactive-shell rename : testing/docker/desktop1604-test/tc-vcs-config.yml => taskcluster/docker/desktop-test/tc-vcs-config.yml rename : testing/docker/tester/tester.env => taskcluster/docker/desktop-test/tester.env rename : testing/docker/desktop-test/apport => taskcluster/docker/desktop1604-test/apport rename : testing/docker/desktop1604-test/bin/run-wizard => taskcluster/docker/desktop1604-test/bin/run-wizard rename : testing/docker/desktop-test/buildprops.json => taskcluster/docker/desktop1604-test/buildprops.json rename : testing/docker/desktop1604-test/deja-dup-monitor.desktop => taskcluster/docker/desktop1604-test/deja-dup-monitor.desktop rename : testing/docker/desktop-test/dot-files/config/pip/pip.conf => taskcluster/docker/desktop1604-test/dot-files/config/pip/pip.conf rename : testing/docker/desktop-test/dot-files/config/user-dirs.dirs => taskcluster/docker/desktop1604-test/dot-files/config/user-dirs.dirs rename : testing/docker/tester/dot-config/user-dirs.locale => taskcluster/docker/desktop1604-test/dot-files/config/user-dirs.locale rename : testing/docker/tester/dot-pulse/default.pa => taskcluster/docker/desktop1604-test/dot-files/pulse/default.pa rename : testing/docker/desktop1604-test/fonts.conf => taskcluster/docker/desktop1604-test/fonts.conf rename : testing/docker/desktop-test/jockey-gtk.desktop => taskcluster/docker/desktop1604-test/jockey-gtk.desktop rename : testing/docker/desktop-test/motd => taskcluster/docker/desktop1604-test/motd rename : testing/docker/desktop-test/release-upgrades => taskcluster/docker/desktop1604-test/release-upgrades rename : testing/docker/desktop1604-test/taskcluster-interactive-shell => taskcluster/docker/desktop1604-test/taskcluster-interactive-shell rename : testing/docker/desktop1604-test/tc-vcs-config.yml => taskcluster/docker/desktop1604-test/tc-vcs-config.yml rename : testing/docker/tester/tester.env => taskcluster/docker/desktop1604-test/tester.env rename : testing/docker/image_builder/Dockerfile => taskcluster/docker/image_builder/Dockerfile rename : testing/docker/image_builder/REGISTRY => taskcluster/docker/image_builder/REGISTRY rename : testing/docker/image_builder/VERSION => taskcluster/docker/image_builder/VERSION rename : testing/docker/image_builder/bin/build_image.sh => taskcluster/docker/image_builder/bin/build_image.sh rename : testing/docker/lint/system-setup.sh => taskcluster/docker/lint/system-setup.sh rename : testing/docker/recipes/centos6-build-system-setup.sh => taskcluster/docker/recipes/centos6-build-system-setup.sh rename : testing/docker/recipes/common.sh => taskcluster/docker/recipes/common.sh rename : testing/docker/recipes/install-mercurial.sh => taskcluster/docker/recipes/install-mercurial.sh rename : testing/docker/recipes/run-task => taskcluster/docker/recipes/run-task rename : testing/docker/recipes/tooltool.py => taskcluster/docker/recipes/tooltool.py rename : testing/docker/recipes/ubuntu1204-test-system-setup.sh => taskcluster/docker/recipes/ubuntu1204-test-system-setup.sh rename : testing/docker/recipes/ubuntu1604-test-system-setup.sh => taskcluster/docker/recipes/ubuntu1604-test-system-setup.sh rename : testing/docker/recipes/xvfb.sh => taskcluster/docker/recipes/xvfb.sh rename : testing/docker/rust-build/Dockerfile => taskcluster/docker/rust-build/Dockerfile rename : testing/docker/rust-build/README.md => taskcluster/docker/rust-build/README.md rename : testing/docker/rust-build/REGISTRY => taskcluster/docker/rust-build/REGISTRY rename : testing/docker/rust-build/VERSION => taskcluster/docker/rust-build/VERSION rename : testing/docker/rust-build/build_cargo.sh => taskcluster/docker/rust-build/build_cargo.sh rename : testing/docker/rust-build/build_rust.sh => taskcluster/docker/rust-build/build_rust.sh rename : testing/docker/rust-build/build_rust_mac.sh => taskcluster/docker/rust-build/build_rust_mac.sh rename : testing/docker/rust-build/fetch_cargo.sh => taskcluster/docker/rust-build/fetch_cargo.sh rename : testing/docker/rust-build/fetch_rust.sh => taskcluster/docker/rust-build/fetch_rust.sh rename : testing/docker/rust-build/package_rust.sh => taskcluster/docker/rust-build/package_rust.sh rename : testing/docker/rust-build/repack_rust.py => taskcluster/docker/rust-build/repack_rust.py rename : testing/docker/rust-build/task.json => taskcluster/docker/rust-build/task.json rename : testing/docker/rust-build/tcbuild.py => taskcluster/docker/rust-build/tcbuild.py rename : testing/docker/rust-build/upload_rust.sh => taskcluster/docker/rust-build/upload_rust.sh rename : testing/docker/tester/Dockerfile => taskcluster/docker/tester/Dockerfile rename : testing/docker/image_builder/REGISTRY => taskcluster/docker/tester/REGISTRY rename : testing/docker/tester/VERSION => taskcluster/docker/tester/VERSION rename : testing/docker/tester/bin/test.sh => taskcluster/docker/tester/bin/test.sh rename : testing/docker/desktop-test/dot-files/config/pip/pip.conf => taskcluster/docker/tester/dot-config/pip/pip.conf rename : testing/docker/tester/dot-config/user-dirs.dirs => taskcluster/docker/tester/dot-config/user-dirs.dirs rename : testing/docker/tester/dot-config/user-dirs.locale => taskcluster/docker/tester/dot-config/user-dirs.locale rename : testing/docker/tester/dot-pulse/default.pa => taskcluster/docker/tester/dot-pulse/default.pa rename : testing/docker/desktop1604-test/tc-vcs-config.yml => taskcluster/docker/tester/tc-vcs-config.yml rename : testing/docker/tester/tester.env => taskcluster/docker/tester/tester.env extra : rebase_source : 94c16b2ea7c5824e70637e7fb64a4f5dc3a02837 --- .../image_builder => taskcluster/docker/image-builder}/Dockerfile | 0 .../image_builder => taskcluster/docker/image-builder}/REGISTRY | 0 .../image_builder => taskcluster/docker/image-builder}/VERSION | 0 .../docker/image-builder}/bin/build_image.sh | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename {testing/docker/image_builder => taskcluster/docker/image-builder}/Dockerfile (100%) rename {testing/docker/image_builder => taskcluster/docker/image-builder}/REGISTRY (100%) rename {testing/docker/image_builder => taskcluster/docker/image-builder}/VERSION (100%) rename {testing/docker/image_builder => taskcluster/docker/image-builder}/bin/build_image.sh (100%) diff --git a/testing/docker/image_builder/Dockerfile b/taskcluster/docker/image-builder/Dockerfile similarity index 100% rename from testing/docker/image_builder/Dockerfile rename to taskcluster/docker/image-builder/Dockerfile diff --git a/testing/docker/image_builder/REGISTRY b/taskcluster/docker/image-builder/REGISTRY similarity index 100% rename from testing/docker/image_builder/REGISTRY rename to taskcluster/docker/image-builder/REGISTRY diff --git a/testing/docker/image_builder/VERSION b/taskcluster/docker/image-builder/VERSION similarity index 100% rename from testing/docker/image_builder/VERSION rename to taskcluster/docker/image-builder/VERSION diff --git a/testing/docker/image_builder/bin/build_image.sh b/taskcluster/docker/image-builder/bin/build_image.sh similarity index 100% rename from testing/docker/image_builder/bin/build_image.sh rename to taskcluster/docker/image-builder/bin/build_image.sh From 5c08844492b72c8e18ab7cc0a567b6afb1f1d6c4 Mon Sep 17 00:00:00 2001 From: Wes Kocher Date: Thu, 20 Oct 2016 15:59:47 -0700 Subject: [PATCH 05/74] Backed out changeset d15798b73b9e (bug 1302763) for taskcluster build issues a=backout CLOSED TREE --HG-- rename : taskcluster/docker/README.md => testing/docker/README.md rename : taskcluster/docker/REGISTRY => testing/docker/REGISTRY rename : taskcluster/docker/android-gradle-build/README.md => testing/docker/android-gradle-build/README.md rename : taskcluster/docker/android-gradle-build/VERSION => testing/docker/android-gradle-build/VERSION rename : taskcluster/docker/android-gradle-build/bin/after.sh => testing/docker/android-gradle-build/bin/after.sh rename : taskcluster/docker/android-gradle-build/bin/before.sh => testing/docker/android-gradle-build/bin/before.sh rename : taskcluster/docker/android-gradle-build/bin/build.sh => testing/docker/android-gradle-build/bin/build.sh rename : taskcluster/docker/android-gradle-build/bin/checkout-sources.sh => testing/docker/android-gradle-build/bin/checkout-sources.sh rename : taskcluster/docker/base-build/Dockerfile => testing/docker/base-build/Dockerfile rename : taskcluster/docker/base-build/VERSION => testing/docker/base-build/VERSION rename : taskcluster/docker/base-build/system-setup.sh => testing/docker/base-build/system-setup.sh rename : taskcluster/docker/base-test/Dockerfile => testing/docker/base-test/Dockerfile rename : taskcluster/docker/base-test/VERSION => testing/docker/base-test/VERSION rename : taskcluster/docker/base-test/sources.list => testing/docker/base-test/sources.list rename : taskcluster/docker/centos6-build-upd/Dockerfile => testing/docker/centos6-build-upd/Dockerfile rename : taskcluster/docker/centos6-build-upd/VERSION => testing/docker/centos6-build-upd/VERSION rename : taskcluster/docker/centos6-build/Dockerfile => testing/docker/centos6-build/Dockerfile rename : taskcluster/docker/centos6-build/hgrc => testing/docker/centos6-build/hgrc rename : taskcluster/docker/centos6-build/system-setup.sh => testing/docker/centos6-build/system-setup.sh rename : taskcluster/docker/decision/README.md => testing/docker/decision/README.md rename : taskcluster/docker/decision/VERSION => testing/docker/decision/VERSION rename : taskcluster/docker/decision/system-setup.sh => testing/docker/decision/system-setup.sh rename : taskcluster/docker/desktop-build/bin/build.sh => testing/docker/desktop-build/bin/build.sh rename : taskcluster/docker/desktop-build/bin/checkout-script.sh => testing/docker/desktop-build/bin/checkout-script.sh rename : taskcluster/docker/desktop-build/bin/checkout-sources.sh => testing/docker/desktop-build/bin/checkout-sources.sh rename : taskcluster/docker/desktop-build/buildprops.json => testing/docker/desktop-build/buildprops.json rename : taskcluster/docker/desktop-build/oauth.txt => testing/docker/desktop-build/oauth.txt rename : taskcluster/docker/desktop1604-test/apport => testing/docker/desktop-test/apport rename : taskcluster/docker/desktop1604-test/buildprops.json => testing/docker/desktop-test/buildprops.json rename : taskcluster/docker/tester/dot-config/pip/pip.conf => testing/docker/desktop-test/dot-files/config/pip/pip.conf rename : taskcluster/docker/desktop1604-test/dot-files/config/user-dirs.dirs => testing/docker/desktop-test/dot-files/config/user-dirs.dirs rename : taskcluster/docker/desktop1604-test/jockey-gtk.desktop => testing/docker/desktop-test/jockey-gtk.desktop rename : taskcluster/docker/desktop1604-test/motd => testing/docker/desktop-test/motd rename : taskcluster/docker/desktop1604-test/release-upgrades => testing/docker/desktop-test/release-upgrades rename : taskcluster/docker/desktop-test/taskcluster-interactive-shell => testing/docker/desktop-test/taskcluster-interactive-shell rename : taskcluster/docker/desktop1604-test/bin/run-wizard => testing/docker/desktop1604-test/bin/run-wizard rename : taskcluster/docker/desktop1604-test/deja-dup-monitor.desktop => testing/docker/desktop1604-test/deja-dup-monitor.desktop rename : taskcluster/docker/desktop1604-test/fonts.conf => testing/docker/desktop1604-test/fonts.conf rename : taskcluster/docker/desktop1604-test/taskcluster-interactive-shell => testing/docker/desktop1604-test/taskcluster-interactive-shell rename : taskcluster/docker/tester/tc-vcs-config.yml => testing/docker/desktop1604-test/tc-vcs-config.yml rename : taskcluster/docker/image_builder/Dockerfile => testing/docker/image_builder/Dockerfile rename : taskcluster/docker/tester/REGISTRY => testing/docker/image_builder/REGISTRY rename : taskcluster/docker/image_builder/VERSION => testing/docker/image_builder/VERSION rename : taskcluster/docker/image_builder/bin/build_image.sh => testing/docker/image_builder/bin/build_image.sh rename : taskcluster/docker/lint/system-setup.sh => testing/docker/lint/system-setup.sh rename : taskcluster/docker/recipes/centos6-build-system-setup.sh => testing/docker/recipes/centos6-build-system-setup.sh rename : taskcluster/docker/recipes/common.sh => testing/docker/recipes/common.sh rename : taskcluster/docker/recipes/install-mercurial.sh => testing/docker/recipes/install-mercurial.sh rename : taskcluster/docker/recipes/run-task => testing/docker/recipes/run-task rename : taskcluster/docker/recipes/tooltool.py => testing/docker/recipes/tooltool.py rename : taskcluster/docker/recipes/ubuntu1204-test-system-setup.sh => testing/docker/recipes/ubuntu1204-test-system-setup.sh rename : taskcluster/docker/recipes/ubuntu1604-test-system-setup.sh => testing/docker/recipes/ubuntu1604-test-system-setup.sh rename : taskcluster/docker/recipes/xvfb.sh => testing/docker/recipes/xvfb.sh rename : taskcluster/docker/rust-build/Dockerfile => testing/docker/rust-build/Dockerfile rename : taskcluster/docker/rust-build/README.md => testing/docker/rust-build/README.md rename : taskcluster/docker/rust-build/REGISTRY => testing/docker/rust-build/REGISTRY rename : taskcluster/docker/rust-build/VERSION => testing/docker/rust-build/VERSION rename : taskcluster/docker/rust-build/build_cargo.sh => testing/docker/rust-build/build_cargo.sh rename : taskcluster/docker/rust-build/build_rust.sh => testing/docker/rust-build/build_rust.sh rename : taskcluster/docker/rust-build/build_rust_mac.sh => testing/docker/rust-build/build_rust_mac.sh rename : taskcluster/docker/rust-build/fetch_cargo.sh => testing/docker/rust-build/fetch_cargo.sh rename : taskcluster/docker/rust-build/fetch_rust.sh => testing/docker/rust-build/fetch_rust.sh rename : taskcluster/docker/rust-build/package_rust.sh => testing/docker/rust-build/package_rust.sh rename : taskcluster/docker/rust-build/repack_rust.py => testing/docker/rust-build/repack_rust.py rename : taskcluster/docker/rust-build/task.json => testing/docker/rust-build/task.json rename : taskcluster/docker/rust-build/tcbuild.py => testing/docker/rust-build/tcbuild.py rename : taskcluster/docker/rust-build/upload_rust.sh => testing/docker/rust-build/upload_rust.sh rename : taskcluster/docker/tester/Dockerfile => testing/docker/tester/Dockerfile rename : taskcluster/docker/tester/VERSION => testing/docker/tester/VERSION rename : taskcluster/docker/tester/bin/test.sh => testing/docker/tester/bin/test.sh rename : taskcluster/docker/tester/dot-config/user-dirs.dirs => testing/docker/tester/dot-config/user-dirs.dirs rename : taskcluster/docker/tester/dot-config/user-dirs.locale => testing/docker/tester/dot-config/user-dirs.locale rename : taskcluster/docker/tester/dot-pulse/default.pa => testing/docker/tester/dot-pulse/default.pa rename : taskcluster/docker/tester/tester.env => testing/docker/tester/tester.env --- .../image-builder => testing/docker/image_builder}/Dockerfile | 0 .../image-builder => testing/docker/image_builder}/REGISTRY | 0 .../docker/image-builder => testing/docker/image_builder}/VERSION | 0 .../docker/image_builder}/bin/build_image.sh | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename {taskcluster/docker/image-builder => testing/docker/image_builder}/Dockerfile (100%) rename {taskcluster/docker/image-builder => testing/docker/image_builder}/REGISTRY (100%) rename {taskcluster/docker/image-builder => testing/docker/image_builder}/VERSION (100%) rename {taskcluster/docker/image-builder => testing/docker/image_builder}/bin/build_image.sh (100%) diff --git a/taskcluster/docker/image-builder/Dockerfile b/testing/docker/image_builder/Dockerfile similarity index 100% rename from taskcluster/docker/image-builder/Dockerfile rename to testing/docker/image_builder/Dockerfile diff --git a/taskcluster/docker/image-builder/REGISTRY b/testing/docker/image_builder/REGISTRY similarity index 100% rename from taskcluster/docker/image-builder/REGISTRY rename to testing/docker/image_builder/REGISTRY diff --git a/taskcluster/docker/image-builder/VERSION b/testing/docker/image_builder/VERSION similarity index 100% rename from taskcluster/docker/image-builder/VERSION rename to testing/docker/image_builder/VERSION diff --git a/taskcluster/docker/image-builder/bin/build_image.sh b/testing/docker/image_builder/bin/build_image.sh similarity index 100% rename from taskcluster/docker/image-builder/bin/build_image.sh rename to testing/docker/image_builder/bin/build_image.sh From e1f52a252fc5bdff0eabbbd1a8b9b2c702b82e95 Mon Sep 17 00:00:00 2001 From: Jonas Finnemann Jensen Date: Mon, 7 Nov 2016 11:26:27 -0800 Subject: [PATCH 06/74] Bug 1316183 - Compress docker images with zstd. r=dustin * Compress docker images with zstd * Removed need for context.tar from decision task * Index images by level rather than project MozReview-Commit-ID: 4RL4QXNWmpd --HG-- extra : rebase_source : 677d8030a15af3288866a70fc648a10b22c396a3 --- testing/docker/image_builder/Dockerfile | 74 ++++++++++--------- testing/docker/image_builder/VERSION | 2 +- .../docker/image_builder/bin/build_image.sh | 37 ---------- testing/docker/image_builder/build-image.sh | 59 +++++++++++++++ testing/docker/image_builder/setup.sh | 53 +++++++++++++ 5 files changed, 153 insertions(+), 72 deletions(-) delete mode 100755 testing/docker/image_builder/bin/build_image.sh create mode 100755 testing/docker/image_builder/build-image.sh create mode 100644 testing/docker/image_builder/setup.sh diff --git a/testing/docker/image_builder/Dockerfile b/testing/docker/image_builder/Dockerfile index 2c1365862..9acbafaab 100644 --- a/testing/docker/image_builder/Dockerfile +++ b/testing/docker/image_builder/Dockerfile @@ -1,34 +1,40 @@ -FROM ubuntu:14.04 - -WORKDIR /home/worker/bin - -RUN apt-get update && apt-get install -y apt-transport-https -RUN sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 36A1D7869245C8950F966E92D8576A8BA88D21E9 && \ - sudo sh -c "echo deb https://get.docker.io/ubuntu docker main\ - > /etc/apt/sources.list.d/docker.list" -RUN apt-get update && apt-get install -y \ - lxc-docker-1.6.1 \ - curl \ - wget \ - git \ - mercurial \ - tar \ - zip \ - unzip \ - vim \ - sudo \ - ca-certificates \ - build-essential - -ENV NODE_VERSION v0.12.4 -RUN cd /usr/local/ && \ - curl https://nodejs.org/dist/$NODE_VERSION/node-$NODE_VERSION-linux-x64.tar.gz | tar -xz --strip-components 1 && \ - node -v - -RUN npm install -g taskcluster-vcs@2.3.11 - -ADD bin /home/worker/bin -RUN chmod +x /home/worker/bin/* - -# Set a default command useful for debugging -CMD ["/bin/bash", "--login"] +FROM ubuntu:16.04 + +# %include testing/docker/recipes/tooltool.py +ADD topsrcdir/testing/docker/recipes/tooltool.py /setup/tooltool.py + +# %include testing/docker/recipes/common.sh +ADD topsrcdir/testing/docker/recipes/common.sh /setup/common.sh + +# %include testing/docker/recipes/install-mercurial.sh +ADD topsrcdir/testing/docker/recipes/install-mercurial.sh /setup/install-mercurial.sh + +# %include testing/mozharness/external_tools/robustcheckout.py +ADD topsrcdir/testing/mozharness/external_tools/robustcheckout.py /usr/local/mercurial/robustcheckout.py + +# %include testing/docker/recipes/run-task +ADD topsrcdir/testing/docker/recipes/run-task /usr/local/bin/run-task + +# Add and run setup script +ADD build-image.sh /usr/local/bin/build-image.sh +ADD setup.sh /setup/setup.sh +RUN bash /setup/setup.sh + +# Setup a workspace that won't use AUFS +VOLUME /home/worker/workspace + +# Set variable normally configured at login, by the shells parent process, these +# are taken from GNU su manual +ENV HOME /home/worker +ENV SHELL /bin/bash +ENV USER worker +ENV LOGNAME worker +ENV HOSTNAME taskcluster-worker +ENV LC_ALL C + +# Create worker user +RUN useradd -d /home/worker -s /bin/bash -m worker + +# Set some sane defaults +WORKDIR /home/worker/ +CMD build-image.sh diff --git a/testing/docker/image_builder/VERSION b/testing/docker/image_builder/VERSION index 9faa1b7a7..3eefcb9dd 100644 --- a/testing/docker/image_builder/VERSION +++ b/testing/docker/image_builder/VERSION @@ -1 +1 @@ -0.1.5 +1.0.0 diff --git a/testing/docker/image_builder/bin/build_image.sh b/testing/docker/image_builder/bin/build_image.sh deleted file mode 100755 index 9095ff79a..000000000 --- a/testing/docker/image_builder/bin/build_image.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/bash -vex - -# Set bash options to exit immediately if a pipeline exists non-zero, expand -# print a trace of commands, and make output verbose (print shell input as it's -# read) -# See https://www.gnu.org/software/bash/manual/html_node/The-Set-Builtin.html -set -x -e -v - -# Prefix errors with taskcluster error prefix so that they are parsed by Treeherder -raise_error() { - echo - echo "[taskcluster-image-build:error] $1" - exit 1 -} - -# Ensure that the PROJECT is specified so the image can be indexed -test -n "$PROJECT" || raise_error "Project must be provided." -test -n "$HASH" || raise_error "Context Hash must be provided." - -mkdir /artifacts - -if [ ! -z "$CONTEXT_URL" ]; then - mkdir /context - if ! curl -L --retry 5 --connect-timeout 30 --fail "$CONTEXT_URL" | tar -xz --strip-components 1 -C /context; then - raise_error "Error downloading image context from decision task." - fi - CONTEXT_PATH=/context -else - tc-vcs checkout /home/worker/workspace/src $BASE_REPOSITORY $HEAD_REPOSITORY $HEAD_REV $HEAD_REF - CONTEXT_PATH=/home/worker/workspace/src/$CONTEXT_PATH -fi - -test -d $CONTEXT_PATH || raise_error "Context Path $CONTEXT_PATH does not exist." -test -f "$CONTEXT_PATH/Dockerfile" || raise_error "Dockerfile must be present in $CONTEXT_PATH." - -docker build -t $PROJECT:$HASH $CONTEXT_PATH -docker save $PROJECT:$HASH > /artifacts/image.tar diff --git a/testing/docker/image_builder/build-image.sh b/testing/docker/image_builder/build-image.sh new file mode 100755 index 000000000..25e0d6a28 --- /dev/null +++ b/testing/docker/image_builder/build-image.sh @@ -0,0 +1,59 @@ +#!/bin/bash -vex + +# Set bash options to exit immediately if a pipeline exists non-zero, expand +# print a trace of commands, and make output verbose (print shell input as it's +# read) +# See https://www.gnu.org/software/bash/manual/html_node/The-Set-Builtin.html +set -x -e -v + +# Prefix errors with taskcluster error prefix so that they are parsed by Treeherder +raise_error() { + echo + echo "[taskcluster-image-build:error] $1" + exit 1 +} + +# Ensure that the PROJECT is specified so the image can be indexed +test -n "$PROJECT" || raise_error "PROJECT must be provided." +test -n "$HASH" || raise_error "Context HASH must be provided." +test -n "$IMAGE_NAME" || raise_error "IMAGE_NAME must be provided." + +# Create artifact folder +mkdir -p /home/worker/workspace/artifacts + +# Construct a CONTEXT_FILE +CONTEXT_FILE=/home/worker/workspace/context.tar + +# Run ./mach taskcluster-build-image with --context-only to build context +run-task \ + --chown-recursive "/home/worker/workspace" \ + --vcs-checkout "/home/worker/checkouts/gecko" \ + -- \ + /home/worker/checkouts/gecko/mach taskcluster-build-image \ + --context-only "$CONTEXT_FILE" \ + "$IMAGE_NAME" +test -f "$CONTEXT_FILE" || raise_error "Context file wasn't created" + +# Post context tar-ball to docker daemon +# This interacts directly with the docker remote API, see: +# https://docs.docker.com/engine/reference/api/docker_remote_api_v1.18/ +curl -s \ + -X POST \ + --header 'Content-Type: application/tar' \ + --data-binary "@$CONTEXT_FILE" \ + --unix-socket /var/run/docker.sock "http:/build?t=$IMAGE_NAME:$HASH" \ + | tee /tmp/docker-build.log \ + | jq -r '.status + .progress, .stream[:-1], .error | select(. != null)' + +# Exit non-zero if there is error entries in the log +if cat /tmp/docker-build.log | jq -se 'add | .error' > /dev/null; then + raise_error "Image build failed: `cat /tmp/docker-build.log | jq -rse 'add | .error'`"; +fi + +# Get image from docker daemon +# This interacts directly with the docker remote API, see: +# https://docs.docker.com/engine/reference/api/docker_remote_api_v1.18/ +curl -s \ + -X GET \ + --unix-socket /var/run/docker.sock "http:/images/$IMAGE_NAME:$HASH/get" \ + | zstd -3 -c -o /home/worker/workspace/artifacts/image.tar.zst diff --git a/testing/docker/image_builder/setup.sh b/testing/docker/image_builder/setup.sh new file mode 100644 index 000000000..1a2d13503 --- /dev/null +++ b/testing/docker/image_builder/setup.sh @@ -0,0 +1,53 @@ +#!/bin/bash -vex +set -v -e -x + +export DEBIAN_FRONTEND=noninteractive + +# Update apt-get lists +apt-get update -y + +# Install dependencies +apt-get install -y \ + curl \ + tar \ + jq \ + python \ + build-essential # Only needed for zstd installation, will be removed later + +# Install mercurial +. /setup/common.sh +. /setup/install-mercurial.sh + +# Install build-image.sh script +chmod +x /usr/local/bin/build-image.sh +chmod +x /usr/local/bin/run-task + +# Create workspace +mkdir -p /home/worker/workspace + +# Install zstd 1.1.1 +cd /setup +tooltool_fetch < Date: Thu, 17 Nov 2016 14:58:13 -0800 Subject: [PATCH 07/74] Bug 1318466 - Fix image_builder to fail if docker socket returns 404. r=dustin MozReview-Commit-ID: sgjfYiAqtT --HG-- extra : rebase_source : a8433a408d774225a04245f4127e15dd8e4530cf --- testing/docker/image_builder/VERSION | 2 +- testing/docker/image_builder/build-image.sh | 18 +++++++++++------- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/testing/docker/image_builder/VERSION b/testing/docker/image_builder/VERSION index 3eefcb9dd..9084fa2f7 100644 --- a/testing/docker/image_builder/VERSION +++ b/testing/docker/image_builder/VERSION @@ -1 +1 @@ -1.0.0 +1.1.0 diff --git a/testing/docker/image_builder/build-image.sh b/testing/docker/image_builder/build-image.sh index 25e0d6a28..6286bd7b5 100755 --- a/testing/docker/image_builder/build-image.sh +++ b/testing/docker/image_builder/build-image.sh @@ -37,23 +37,27 @@ test -f "$CONTEXT_FILE" || raise_error "Context file wasn't created" # Post context tar-ball to docker daemon # This interacts directly with the docker remote API, see: # https://docs.docker.com/engine/reference/api/docker_remote_api_v1.18/ -curl -s \ +curl -s --fail \ -X POST \ --header 'Content-Type: application/tar' \ --data-binary "@$CONTEXT_FILE" \ --unix-socket /var/run/docker.sock "http:/build?t=$IMAGE_NAME:$HASH" \ | tee /tmp/docker-build.log \ - | jq -r '.status + .progress, .stream[:-1], .error | select(. != null)' + | jq -jr '(.status + .progress, .error | select(. != null) + "\n"), .stream | select(. != null)' # Exit non-zero if there is error entries in the log if cat /tmp/docker-build.log | jq -se 'add | .error' > /dev/null; then raise_error "Image build failed: `cat /tmp/docker-build.log | jq -rse 'add | .error'`"; fi -# Get image from docker daemon +# Get image from docker daemon (try up to 10 times) # This interacts directly with the docker remote API, see: # https://docs.docker.com/engine/reference/api/docker_remote_api_v1.18/ -curl -s \ - -X GET \ - --unix-socket /var/run/docker.sock "http:/images/$IMAGE_NAME:$HASH/get" \ - | zstd -3 -c -o /home/worker/workspace/artifacts/image.tar.zst +count=0 +while ! curl -s --fail -X GET \ + --unix-socket /var/run/docker.sock "http:/images/$IMAGE_NAME:$HASH/get" \ + | zstd -3 -c -o /home/worker/workspace/artifacts/image.tar.zst; do + ((c++)) && ((c==10)) && echo 'Failed to get image from docker' && exit 1; + echo 'Waiting for image to be ready'; + sleep 5; +done From 39bc99c367ba7454f782f49426a55e4c5a2ec6da Mon Sep 17 00:00:00 2001 From: Jonas Finnemann Jensen Date: Tue, 22 Nov 2016 14:13:15 -0800 Subject: [PATCH 08/74] Bug 1319449 - Set -o pipefail and other robustness improvements for image_builder. r=dustin MozReview-Commit-ID: 5oIdvcrScRt --HG-- extra : rebase_source : 6b6dd2dc3b3e8465127d33fb428877f68537ad5b --- testing/docker/image_builder/VERSION | 2 +- testing/docker/image_builder/build-image.sh | 20 ++++++++++++++++++-- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/testing/docker/image_builder/VERSION b/testing/docker/image_builder/VERSION index 9084fa2f7..26aaba0e8 100644 --- a/testing/docker/image_builder/VERSION +++ b/testing/docker/image_builder/VERSION @@ -1 +1 @@ -1.1.0 +1.2.0 diff --git a/testing/docker/image_builder/build-image.sh b/testing/docker/image_builder/build-image.sh index 6286bd7b5..1df0e470f 100755 --- a/testing/docker/image_builder/build-image.sh +++ b/testing/docker/image_builder/build-image.sh @@ -4,7 +4,7 @@ # print a trace of commands, and make output verbose (print shell input as it's # read) # See https://www.gnu.org/software/bash/manual/html_node/The-Set-Builtin.html -set -x -e -v +set -x -e -v -o pipefail # Prefix errors with taskcluster error prefix so that they are parsed by Treeherder raise_error() { @@ -50,14 +50,30 @@ if cat /tmp/docker-build.log | jq -se 'add | .error' > /dev/null; then raise_error "Image build failed: `cat /tmp/docker-build.log | jq -rse 'add | .error'`"; fi +# Sanity check that image was built successfully +if ! cat /tmp/docker-build.log | tail -n 1 | jq -r '.stream' | grep '^Successfully built' > /dev/null; then + echo 'docker-build.log for debugging:'; + cat /tmp/docker-build.log | tail -n 50; + raise_error "Image build log didn't with 'Successfully built'"; +fi + # Get image from docker daemon (try up to 10 times) # This interacts directly with the docker remote API, see: # https://docs.docker.com/engine/reference/api/docker_remote_api_v1.18/ +IMAGE_FILE=/home/worker/workspace/image.tar count=0 while ! curl -s --fail -X GET \ --unix-socket /var/run/docker.sock "http:/images/$IMAGE_NAME:$HASH/get" \ - | zstd -3 -c -o /home/worker/workspace/artifacts/image.tar.zst; do + -o "$IMAGE_FILE"; do ((c++)) && ((c==10)) && echo 'Failed to get image from docker' && exit 1; echo 'Waiting for image to be ready'; sleep 5; done + +# Test that image was exported +if [ ! -s "$IMAGE_FILE" ]; then + raise_error "Failed to export docker image"; +fi + +# Compress image with zst +zstd -3 -c -o /home/worker/workspace/artifacts/image.tar.zst "$IMAGE_FILE" From f2173ae8ec62f1f7ad340afd1670980d8247fbeb Mon Sep 17 00:00:00 2001 From: CuriousLearner Date: Thu, 20 Oct 2016 18:25:34 +0530 Subject: [PATCH 09/74] Bug 1302763 - Move docker images out of testing/docker into taskcluster/docker; r=CuriousLearner MozReview-Commit-ID: 7v1uCDB5qoN --HG-- rename : testing/docker/README.md => taskcluster/docker/README.md rename : testing/docker/REGISTRY => taskcluster/docker/REGISTRY rename : testing/docker/android-gradle-build/Dockerfile => taskcluster/docker/android-gradle-build/Dockerfile rename : testing/docker/android-gradle-build/README.md => taskcluster/docker/android-gradle-build/README.md rename : testing/docker/android-gradle-build/REGISTRY => taskcluster/docker/android-gradle-build/REGISTRY rename : testing/docker/android-gradle-build/VERSION => taskcluster/docker/android-gradle-build/VERSION rename : testing/docker/android-gradle-build/bin/after.sh => taskcluster/docker/android-gradle-build/bin/after.sh rename : testing/docker/android-gradle-build/bin/before.sh => taskcluster/docker/android-gradle-build/bin/before.sh rename : testing/docker/android-gradle-build/bin/build.sh => taskcluster/docker/android-gradle-build/bin/build.sh rename : testing/docker/android-gradle-build/bin/checkout-script.sh => taskcluster/docker/android-gradle-build/bin/checkout-script.sh rename : testing/docker/android-gradle-build/bin/checkout-sources.sh => taskcluster/docker/android-gradle-build/bin/checkout-sources.sh rename : testing/docker/android-gradle-build/buildprops.json => taskcluster/docker/android-gradle-build/buildprops.json rename : testing/docker/android-gradle-build/dot-config/pip/pip.conf => taskcluster/docker/android-gradle-build/dot-config/pip/pip.conf rename : testing/docker/android-gradle-build/oauth.txt => taskcluster/docker/android-gradle-build/oauth.txt rename : testing/docker/base-build/Dockerfile => taskcluster/docker/base-build/Dockerfile rename : testing/docker/base-build/VERSION => taskcluster/docker/base-build/VERSION rename : testing/docker/base-build/system-setup.sh => taskcluster/docker/base-build/system-setup.sh rename : testing/docker/base-test/Dockerfile => taskcluster/docker/base-test/Dockerfile rename : testing/docker/base-test/REGISTRY => taskcluster/docker/base-test/REGISTRY rename : testing/docker/base-test/VERSION => taskcluster/docker/base-test/VERSION rename : testing/docker/base-test/sources.list => taskcluster/docker/base-test/sources.list rename : testing/docker/centos6-build-upd/Dockerfile => taskcluster/docker/centos6-build-upd/Dockerfile rename : testing/docker/centos6-build-upd/REGISTRY => taskcluster/docker/centos6-build-upd/REGISTRY rename : testing/docker/centos6-build-upd/VERSION => taskcluster/docker/centos6-build-upd/VERSION rename : testing/docker/centos6-build/Dockerfile => taskcluster/docker/centos6-build/Dockerfile rename : testing/docker/centos6-build/REGISTRY => taskcluster/docker/centos6-build/REGISTRY rename : testing/docker/centos6-build/VERSION => taskcluster/docker/centos6-build/VERSION rename : testing/docker/centos6-build/hgrc => taskcluster/docker/centos6-build/hgrc rename : testing/docker/centos6-build/system-setup.sh => taskcluster/docker/centos6-build/system-setup.sh rename : testing/docker/decision/Dockerfile => taskcluster/docker/decision/Dockerfile rename : testing/docker/decision/README.md => taskcluster/docker/decision/README.md rename : testing/docker/decision/REGISTRY => taskcluster/docker/decision/REGISTRY rename : testing/docker/decision/VERSION => taskcluster/docker/decision/VERSION rename : testing/docker/decision/system-setup.sh => taskcluster/docker/decision/system-setup.sh rename : testing/docker/desktop-build/Dockerfile => taskcluster/docker/desktop-build/Dockerfile rename : testing/docker/desktop-build/bin/build.sh => taskcluster/docker/desktop-build/bin/build.sh rename : testing/docker/desktop-build/bin/checkout-script.sh => taskcluster/docker/desktop-build/bin/checkout-script.sh rename : testing/docker/desktop-build/bin/checkout-sources.sh => taskcluster/docker/desktop-build/bin/checkout-sources.sh rename : testing/docker/desktop-build/buildprops.json => taskcluster/docker/desktop-build/buildprops.json rename : testing/docker/desktop-build/dot-config/pip/pip.conf => taskcluster/docker/desktop-build/dot-config/pip/pip.conf rename : testing/docker/desktop-build/oauth.txt => taskcluster/docker/desktop-build/oauth.txt rename : testing/docker/desktop-test/Dockerfile => taskcluster/docker/desktop-test/Dockerfile rename : testing/docker/desktop-test/apport => taskcluster/docker/desktop-test/apport rename : testing/docker/desktop-test/buildprops.json => taskcluster/docker/desktop-test/buildprops.json rename : testing/docker/desktop-test/deja-dup-monitor.desktop => taskcluster/docker/desktop-test/deja-dup-monitor.desktop rename : testing/docker/desktop-test/dot-files/config/pip/pip.conf => taskcluster/docker/desktop-test/dot-files/config/pip/pip.conf rename : testing/docker/desktop-test/dot-files/config/user-dirs.dirs => taskcluster/docker/desktop-test/dot-files/config/user-dirs.dirs rename : testing/docker/desktop-test/dot-files/config/user-dirs.locale => taskcluster/docker/desktop-test/dot-files/config/user-dirs.locale rename : testing/docker/desktop-test/dot-files/pulse/default.pa => taskcluster/docker/desktop-test/dot-files/pulse/default.pa rename : testing/docker/desktop-test/fonts.conf => taskcluster/docker/desktop-test/fonts.conf rename : testing/docker/desktop-test/jockey-gtk.desktop => taskcluster/docker/desktop-test/jockey-gtk.desktop rename : testing/docker/desktop-test/motd => taskcluster/docker/desktop-test/motd rename : testing/docker/desktop-test/release-upgrades => taskcluster/docker/desktop-test/release-upgrades rename : testing/docker/desktop-test/taskcluster-interactive-shell => taskcluster/docker/desktop-test/taskcluster-interactive-shell rename : testing/docker/desktop-test/tc-vcs-config.yml => taskcluster/docker/desktop-test/tc-vcs-config.yml rename : testing/docker/desktop-test/tester.env => taskcluster/docker/desktop-test/tester.env rename : testing/docker/desktop1604-test/Dockerfile => taskcluster/docker/desktop1604-test/Dockerfile rename : testing/docker/desktop1604-test/apport => taskcluster/docker/desktop1604-test/apport rename : testing/docker/desktop1604-test/buildprops.json => taskcluster/docker/desktop1604-test/buildprops.json rename : testing/docker/desktop1604-test/deja-dup-monitor.desktop => taskcluster/docker/desktop1604-test/deja-dup-monitor.desktop rename : testing/docker/desktop1604-test/dot-files/config/pip/pip.conf => taskcluster/docker/desktop1604-test/dot-files/config/pip/pip.conf rename : testing/docker/desktop1604-test/dot-files/config/user-dirs.dirs => taskcluster/docker/desktop1604-test/dot-files/config/user-dirs.dirs rename : testing/docker/desktop1604-test/dot-files/config/user-dirs.locale => taskcluster/docker/desktop1604-test/dot-files/config/user-dirs.locale rename : testing/docker/desktop1604-test/dot-files/pulse/default.pa => taskcluster/docker/desktop1604-test/dot-files/pulse/default.pa rename : testing/docker/desktop1604-test/fonts.conf => taskcluster/docker/desktop1604-test/fonts.conf rename : testing/docker/desktop1604-test/jockey-gtk.desktop => taskcluster/docker/desktop1604-test/jockey-gtk.desktop rename : testing/docker/desktop1604-test/motd => taskcluster/docker/desktop1604-test/motd rename : testing/docker/desktop1604-test/release-upgrades => taskcluster/docker/desktop1604-test/release-upgrades rename : testing/docker/desktop1604-test/taskcluster-interactive-shell => taskcluster/docker/desktop1604-test/taskcluster-interactive-shell rename : testing/docker/desktop1604-test/tc-vcs-config.yml => taskcluster/docker/desktop1604-test/tc-vcs-config.yml rename : testing/docker/desktop1604-test/tester.env => taskcluster/docker/desktop1604-test/tester.env rename : testing/docker/image_builder/Dockerfile => taskcluster/docker/image_builder/Dockerfile rename : testing/docker/image_builder/REGISTRY => taskcluster/docker/image_builder/REGISTRY rename : testing/docker/image_builder/VERSION => taskcluster/docker/image_builder/VERSION rename : testing/docker/image_builder/build-image.sh => taskcluster/docker/image_builder/build-image.sh rename : testing/docker/image_builder/setup.sh => taskcluster/docker/image_builder/setup.sh rename : testing/docker/lint/Dockerfile => taskcluster/docker/lint/Dockerfile rename : testing/docker/lint/system-setup.sh => taskcluster/docker/lint/system-setup.sh rename : testing/docker/recipes/centos6-build-system-setup.sh => taskcluster/docker/recipes/centos6-build-system-setup.sh rename : testing/docker/recipes/common.sh => taskcluster/docker/recipes/common.sh rename : testing/docker/recipes/install-mercurial.sh => taskcluster/docker/recipes/install-mercurial.sh rename : testing/docker/recipes/run-task => taskcluster/docker/recipes/run-task rename : testing/docker/recipes/tooltool.py => taskcluster/docker/recipes/tooltool.py rename : testing/docker/recipes/ubuntu1204-test-system-setup.sh => taskcluster/docker/recipes/ubuntu1204-test-system-setup.sh rename : testing/docker/recipes/ubuntu1604-test-system-setup.sh => taskcluster/docker/recipes/ubuntu1604-test-system-setup.sh rename : testing/docker/recipes/xvfb.sh => taskcluster/docker/recipes/xvfb.sh rename : testing/docker/rust-build/Dockerfile => taskcluster/docker/rust-build/Dockerfile rename : testing/docker/rust-build/README.md => taskcluster/docker/rust-build/README.md rename : testing/docker/rust-build/REGISTRY => taskcluster/docker/rust-build/REGISTRY rename : testing/docker/rust-build/VERSION => taskcluster/docker/rust-build/VERSION rename : testing/docker/rust-build/build_cargo.sh => taskcluster/docker/rust-build/build_cargo.sh rename : testing/docker/rust-build/build_rust.sh => taskcluster/docker/rust-build/build_rust.sh rename : testing/docker/rust-build/build_rust_mac.sh => taskcluster/docker/rust-build/build_rust_mac.sh rename : testing/docker/rust-build/fetch_cargo.sh => taskcluster/docker/rust-build/fetch_cargo.sh rename : testing/docker/rust-build/fetch_rust.sh => taskcluster/docker/rust-build/fetch_rust.sh rename : testing/docker/rust-build/package_rust.sh => taskcluster/docker/rust-build/package_rust.sh rename : testing/docker/rust-build/repack_rust.py => taskcluster/docker/rust-build/repack_rust.py rename : testing/docker/rust-build/splat_rust.py => taskcluster/docker/rust-build/splat_rust.py rename : testing/docker/rust-build/task.json => taskcluster/docker/rust-build/task.json rename : testing/docker/rust-build/tcbuild.py => taskcluster/docker/rust-build/tcbuild.py rename : testing/docker/rust-build/upload_rust.sh => taskcluster/docker/rust-build/upload_rust.sh rename : testing/docker/tester/Dockerfile => taskcluster/docker/tester/Dockerfile rename : testing/docker/tester/REGISTRY => taskcluster/docker/tester/REGISTRY rename : testing/docker/tester/VERSION => taskcluster/docker/tester/VERSION rename : testing/docker/tester/bin/test.sh => taskcluster/docker/tester/bin/test.sh rename : testing/docker/tester/dot-config/pip/pip.conf => taskcluster/docker/tester/dot-config/pip/pip.conf rename : testing/docker/tester/dot-config/user-dirs.dirs => taskcluster/docker/tester/dot-config/user-dirs.dirs rename : testing/docker/tester/dot-config/user-dirs.locale => taskcluster/docker/tester/dot-config/user-dirs.locale rename : testing/docker/tester/dot-pulse/default.pa => taskcluster/docker/tester/dot-pulse/default.pa rename : testing/docker/tester/tc-vcs-config.yml => taskcluster/docker/tester/tc-vcs-config.yml rename : testing/docker/tester/tester.env => taskcluster/docker/tester/tester.env rename : testing/docker/upload-symbols/Dockerfile => taskcluster/docker/upload-symbols/Dockerfile rename : testing/docker/upload-symbols/README.md => taskcluster/docker/upload-symbols/README.md rename : testing/docker/upload-symbols/bin/checkout-script.sh => taskcluster/docker/upload-symbols/bin/checkout-script.sh rename : testing/docker/upload-symbols/bin/upload.sh => taskcluster/docker/upload-symbols/bin/upload.sh rename : testing/docker/upload-symbols/test_exports.sh => taskcluster/docker/upload-symbols/test_exports.sh extra : rebase_source : 2cf145867ff1f14f8e64b20e88e93fabaff3e756 extra : source : 50adff295bf00ea0d9d7426e745acf3635e7dba5 --- .../docker/image-builder}/Dockerfile | 20 +++++++++---------- .../docker/image-builder}/REGISTRY | 0 .../docker/image-builder}/VERSION | 0 .../docker/image-builder}/build-image.sh | 0 .../docker/image-builder}/setup.sh | 0 5 files changed, 10 insertions(+), 10 deletions(-) rename {testing/docker/image_builder => taskcluster/docker/image-builder}/Dockerfile (51%) rename {testing/docker/image_builder => taskcluster/docker/image-builder}/REGISTRY (100%) rename {testing/docker/image_builder => taskcluster/docker/image-builder}/VERSION (100%) rename {testing/docker/image_builder => taskcluster/docker/image-builder}/build-image.sh (100%) rename {testing/docker/image_builder => taskcluster/docker/image-builder}/setup.sh (100%) diff --git a/testing/docker/image_builder/Dockerfile b/taskcluster/docker/image-builder/Dockerfile similarity index 51% rename from testing/docker/image_builder/Dockerfile rename to taskcluster/docker/image-builder/Dockerfile index 9acbafaab..41c718c8c 100644 --- a/testing/docker/image_builder/Dockerfile +++ b/taskcluster/docker/image-builder/Dockerfile @@ -1,19 +1,19 @@ FROM ubuntu:16.04 -# %include testing/docker/recipes/tooltool.py -ADD topsrcdir/testing/docker/recipes/tooltool.py /setup/tooltool.py +# %include taskcluster/docker/recipes/tooltool.py +ADD topsrcdir/taskcluster/docker/recipes/tooltool.py /setup/tooltool.py -# %include testing/docker/recipes/common.sh -ADD topsrcdir/testing/docker/recipes/common.sh /setup/common.sh +# %include taskcluster/docker/recipes/common.sh +ADD topsrcdir/taskcluster/docker/recipes/common.sh /setup/common.sh -# %include testing/docker/recipes/install-mercurial.sh -ADD topsrcdir/testing/docker/recipes/install-mercurial.sh /setup/install-mercurial.sh +# %include taskcluster/docker/recipes/install-mercurial.sh +ADD topsrcdir/taskcluster/docker/recipes/install-mercurial.sh /setup/install-mercurial.sh -# %include testing/mozharness/external_tools/robustcheckout.py -ADD topsrcdir/testing/mozharness/external_tools/robustcheckout.py /usr/local/mercurial/robustcheckout.py +# %include taskcluster/mozharness/external_tools/robustcheckout.py +ADD topsrcdir/taskcluster/mozharness/external_tools/robustcheckout.py /usr/local/mercurial/robustcheckout.py -# %include testing/docker/recipes/run-task -ADD topsrcdir/testing/docker/recipes/run-task /usr/local/bin/run-task +# %include taskcluster/docker/recipes/run-task +ADD topsrcdir/taskcluster/docker/recipes/run-task /usr/local/bin/run-task # Add and run setup script ADD build-image.sh /usr/local/bin/build-image.sh diff --git a/testing/docker/image_builder/REGISTRY b/taskcluster/docker/image-builder/REGISTRY similarity index 100% rename from testing/docker/image_builder/REGISTRY rename to taskcluster/docker/image-builder/REGISTRY diff --git a/testing/docker/image_builder/VERSION b/taskcluster/docker/image-builder/VERSION similarity index 100% rename from testing/docker/image_builder/VERSION rename to taskcluster/docker/image-builder/VERSION diff --git a/testing/docker/image_builder/build-image.sh b/taskcluster/docker/image-builder/build-image.sh similarity index 100% rename from testing/docker/image_builder/build-image.sh rename to taskcluster/docker/image-builder/build-image.sh diff --git a/testing/docker/image_builder/setup.sh b/taskcluster/docker/image-builder/setup.sh similarity index 100% rename from testing/docker/image_builder/setup.sh rename to taskcluster/docker/image-builder/setup.sh From b81397c7a645c755ac1979691bddc7e44c451ca6 Mon Sep 17 00:00:00 2001 From: Sebastian Hengst Date: Mon, 19 Dec 2016 21:15:07 +0100 Subject: [PATCH 10/74] Backed out changeset fe7303de56be (bug 1302763) for failing webdriver tests on Linux x64 debug (geckodriver.manifest fetch failed). r=backout --HG-- rename : taskcluster/docker/README.md => testing/docker/README.md rename : taskcluster/docker/REGISTRY => testing/docker/REGISTRY rename : taskcluster/docker/android-gradle-build/Dockerfile => testing/docker/android-gradle-build/Dockerfile rename : taskcluster/docker/android-gradle-build/README.md => testing/docker/android-gradle-build/README.md rename : taskcluster/docker/android-gradle-build/REGISTRY => testing/docker/android-gradle-build/REGISTRY rename : taskcluster/docker/android-gradle-build/VERSION => testing/docker/android-gradle-build/VERSION rename : taskcluster/docker/android-gradle-build/bin/after.sh => testing/docker/android-gradle-build/bin/after.sh rename : taskcluster/docker/android-gradle-build/bin/before.sh => testing/docker/android-gradle-build/bin/before.sh rename : taskcluster/docker/android-gradle-build/bin/build.sh => testing/docker/android-gradle-build/bin/build.sh rename : taskcluster/docker/android-gradle-build/bin/checkout-script.sh => testing/docker/android-gradle-build/bin/checkout-script.sh rename : taskcluster/docker/android-gradle-build/bin/checkout-sources.sh => testing/docker/android-gradle-build/bin/checkout-sources.sh rename : taskcluster/docker/android-gradle-build/buildprops.json => testing/docker/android-gradle-build/buildprops.json rename : taskcluster/docker/android-gradle-build/dot-config/pip/pip.conf => testing/docker/android-gradle-build/dot-config/pip/pip.conf rename : taskcluster/docker/android-gradle-build/oauth.txt => testing/docker/android-gradle-build/oauth.txt rename : taskcluster/docker/base-build/Dockerfile => testing/docker/base-build/Dockerfile rename : taskcluster/docker/base-build/VERSION => testing/docker/base-build/VERSION rename : taskcluster/docker/base-build/system-setup.sh => testing/docker/base-build/system-setup.sh rename : taskcluster/docker/base-test/Dockerfile => testing/docker/base-test/Dockerfile rename : taskcluster/docker/base-test/REGISTRY => testing/docker/base-test/REGISTRY rename : taskcluster/docker/base-test/VERSION => testing/docker/base-test/VERSION rename : taskcluster/docker/base-test/sources.list => testing/docker/base-test/sources.list rename : taskcluster/docker/centos6-build-upd/Dockerfile => testing/docker/centos6-build-upd/Dockerfile rename : taskcluster/docker/centos6-build-upd/REGISTRY => testing/docker/centos6-build-upd/REGISTRY rename : taskcluster/docker/centos6-build-upd/VERSION => testing/docker/centos6-build-upd/VERSION rename : taskcluster/docker/centos6-build/Dockerfile => testing/docker/centos6-build/Dockerfile rename : taskcluster/docker/centos6-build/REGISTRY => testing/docker/centos6-build/REGISTRY rename : taskcluster/docker/centos6-build/VERSION => testing/docker/centos6-build/VERSION rename : taskcluster/docker/centos6-build/hgrc => testing/docker/centos6-build/hgrc rename : taskcluster/docker/centos6-build/system-setup.sh => testing/docker/centos6-build/system-setup.sh rename : taskcluster/docker/decision/Dockerfile => testing/docker/decision/Dockerfile rename : taskcluster/docker/decision/README.md => testing/docker/decision/README.md rename : taskcluster/docker/decision/REGISTRY => testing/docker/decision/REGISTRY rename : taskcluster/docker/decision/VERSION => testing/docker/decision/VERSION rename : taskcluster/docker/decision/system-setup.sh => testing/docker/decision/system-setup.sh rename : taskcluster/docker/desktop-build/Dockerfile => testing/docker/desktop-build/Dockerfile rename : taskcluster/docker/desktop-build/bin/build.sh => testing/docker/desktop-build/bin/build.sh rename : taskcluster/docker/desktop-build/bin/checkout-script.sh => testing/docker/desktop-build/bin/checkout-script.sh rename : taskcluster/docker/desktop-build/bin/checkout-sources.sh => testing/docker/desktop-build/bin/checkout-sources.sh rename : taskcluster/docker/desktop-build/buildprops.json => testing/docker/desktop-build/buildprops.json rename : taskcluster/docker/desktop-build/dot-config/pip/pip.conf => testing/docker/desktop-build/dot-config/pip/pip.conf rename : taskcluster/docker/desktop-build/oauth.txt => testing/docker/desktop-build/oauth.txt rename : taskcluster/docker/desktop-test/Dockerfile => testing/docker/desktop-test/Dockerfile rename : taskcluster/docker/desktop-test/apport => testing/docker/desktop-test/apport rename : taskcluster/docker/desktop-test/buildprops.json => testing/docker/desktop-test/buildprops.json rename : taskcluster/docker/desktop-test/deja-dup-monitor.desktop => testing/docker/desktop-test/deja-dup-monitor.desktop rename : taskcluster/docker/desktop-test/dot-files/config/pip/pip.conf => testing/docker/desktop-test/dot-files/config/pip/pip.conf rename : taskcluster/docker/desktop-test/dot-files/config/user-dirs.dirs => testing/docker/desktop-test/dot-files/config/user-dirs.dirs rename : taskcluster/docker/desktop-test/dot-files/config/user-dirs.locale => testing/docker/desktop-test/dot-files/config/user-dirs.locale rename : taskcluster/docker/desktop-test/dot-files/pulse/default.pa => testing/docker/desktop-test/dot-files/pulse/default.pa rename : taskcluster/docker/desktop-test/fonts.conf => testing/docker/desktop-test/fonts.conf rename : taskcluster/docker/desktop-test/jockey-gtk.desktop => testing/docker/desktop-test/jockey-gtk.desktop rename : taskcluster/docker/desktop-test/motd => testing/docker/desktop-test/motd rename : taskcluster/docker/desktop-test/release-upgrades => testing/docker/desktop-test/release-upgrades rename : taskcluster/docker/desktop-test/taskcluster-interactive-shell => testing/docker/desktop-test/taskcluster-interactive-shell rename : taskcluster/docker/desktop-test/tc-vcs-config.yml => testing/docker/desktop-test/tc-vcs-config.yml rename : taskcluster/docker/desktop-test/tester.env => testing/docker/desktop-test/tester.env rename : taskcluster/docker/desktop1604-test/Dockerfile => testing/docker/desktop1604-test/Dockerfile rename : taskcluster/docker/desktop1604-test/apport => testing/docker/desktop1604-test/apport rename : taskcluster/docker/desktop1604-test/buildprops.json => testing/docker/desktop1604-test/buildprops.json rename : taskcluster/docker/desktop1604-test/deja-dup-monitor.desktop => testing/docker/desktop1604-test/deja-dup-monitor.desktop rename : taskcluster/docker/desktop1604-test/dot-files/config/pip/pip.conf => testing/docker/desktop1604-test/dot-files/config/pip/pip.conf rename : taskcluster/docker/desktop1604-test/dot-files/config/user-dirs.dirs => testing/docker/desktop1604-test/dot-files/config/user-dirs.dirs rename : taskcluster/docker/desktop1604-test/dot-files/config/user-dirs.locale => testing/docker/desktop1604-test/dot-files/config/user-dirs.locale rename : taskcluster/docker/desktop1604-test/dot-files/pulse/default.pa => testing/docker/desktop1604-test/dot-files/pulse/default.pa rename : taskcluster/docker/desktop1604-test/fonts.conf => testing/docker/desktop1604-test/fonts.conf rename : taskcluster/docker/desktop1604-test/jockey-gtk.desktop => testing/docker/desktop1604-test/jockey-gtk.desktop rename : taskcluster/docker/desktop1604-test/motd => testing/docker/desktop1604-test/motd rename : taskcluster/docker/desktop1604-test/release-upgrades => testing/docker/desktop1604-test/release-upgrades rename : taskcluster/docker/desktop1604-test/taskcluster-interactive-shell => testing/docker/desktop1604-test/taskcluster-interactive-shell rename : taskcluster/docker/desktop1604-test/tc-vcs-config.yml => testing/docker/desktop1604-test/tc-vcs-config.yml rename : taskcluster/docker/desktop1604-test/tester.env => testing/docker/desktop1604-test/tester.env rename : taskcluster/docker/image_builder/Dockerfile => testing/docker/image_builder/Dockerfile rename : taskcluster/docker/image_builder/REGISTRY => testing/docker/image_builder/REGISTRY rename : taskcluster/docker/image_builder/VERSION => testing/docker/image_builder/VERSION rename : taskcluster/docker/image_builder/build-image.sh => testing/docker/image_builder/build-image.sh rename : taskcluster/docker/image_builder/setup.sh => testing/docker/image_builder/setup.sh rename : taskcluster/docker/lint/Dockerfile => testing/docker/lint/Dockerfile rename : taskcluster/docker/lint/system-setup.sh => testing/docker/lint/system-setup.sh rename : taskcluster/docker/recipes/centos6-build-system-setup.sh => testing/docker/recipes/centos6-build-system-setup.sh rename : taskcluster/docker/recipes/common.sh => testing/docker/recipes/common.sh rename : taskcluster/docker/recipes/install-mercurial.sh => testing/docker/recipes/install-mercurial.sh rename : taskcluster/docker/recipes/run-task => testing/docker/recipes/run-task rename : taskcluster/docker/recipes/tooltool.py => testing/docker/recipes/tooltool.py rename : taskcluster/docker/recipes/ubuntu1204-test-system-setup.sh => testing/docker/recipes/ubuntu1204-test-system-setup.sh rename : taskcluster/docker/recipes/ubuntu1604-test-system-setup.sh => testing/docker/recipes/ubuntu1604-test-system-setup.sh rename : taskcluster/docker/recipes/xvfb.sh => testing/docker/recipes/xvfb.sh rename : taskcluster/docker/rust-build/Dockerfile => testing/docker/rust-build/Dockerfile rename : taskcluster/docker/rust-build/README.md => testing/docker/rust-build/README.md rename : taskcluster/docker/rust-build/REGISTRY => testing/docker/rust-build/REGISTRY rename : taskcluster/docker/rust-build/VERSION => testing/docker/rust-build/VERSION rename : taskcluster/docker/rust-build/build_cargo.sh => testing/docker/rust-build/build_cargo.sh rename : taskcluster/docker/rust-build/build_rust.sh => testing/docker/rust-build/build_rust.sh rename : taskcluster/docker/rust-build/build_rust_mac.sh => testing/docker/rust-build/build_rust_mac.sh rename : taskcluster/docker/rust-build/fetch_cargo.sh => testing/docker/rust-build/fetch_cargo.sh rename : taskcluster/docker/rust-build/fetch_rust.sh => testing/docker/rust-build/fetch_rust.sh rename : taskcluster/docker/rust-build/package_rust.sh => testing/docker/rust-build/package_rust.sh rename : taskcluster/docker/rust-build/repack_rust.py => testing/docker/rust-build/repack_rust.py rename : taskcluster/docker/rust-build/splat_rust.py => testing/docker/rust-build/splat_rust.py rename : taskcluster/docker/rust-build/task.json => testing/docker/rust-build/task.json rename : taskcluster/docker/rust-build/tcbuild.py => testing/docker/rust-build/tcbuild.py rename : taskcluster/docker/rust-build/upload_rust.sh => testing/docker/rust-build/upload_rust.sh rename : taskcluster/docker/tester/Dockerfile => testing/docker/tester/Dockerfile rename : taskcluster/docker/tester/REGISTRY => testing/docker/tester/REGISTRY rename : taskcluster/docker/tester/VERSION => testing/docker/tester/VERSION rename : taskcluster/docker/tester/bin/test.sh => testing/docker/tester/bin/test.sh rename : taskcluster/docker/tester/dot-config/pip/pip.conf => testing/docker/tester/dot-config/pip/pip.conf rename : taskcluster/docker/tester/dot-config/user-dirs.dirs => testing/docker/tester/dot-config/user-dirs.dirs rename : taskcluster/docker/tester/dot-config/user-dirs.locale => testing/docker/tester/dot-config/user-dirs.locale rename : taskcluster/docker/tester/dot-pulse/default.pa => testing/docker/tester/dot-pulse/default.pa rename : taskcluster/docker/tester/tc-vcs-config.yml => testing/docker/tester/tc-vcs-config.yml rename : taskcluster/docker/tester/tester.env => testing/docker/tester/tester.env rename : taskcluster/docker/upload-symbols/Dockerfile => testing/docker/upload-symbols/Dockerfile rename : taskcluster/docker/upload-symbols/README.md => testing/docker/upload-symbols/README.md rename : taskcluster/docker/upload-symbols/bin/checkout-script.sh => testing/docker/upload-symbols/bin/checkout-script.sh rename : taskcluster/docker/upload-symbols/bin/upload.sh => testing/docker/upload-symbols/bin/upload.sh rename : taskcluster/docker/upload-symbols/test_exports.sh => testing/docker/upload-symbols/test_exports.sh --- .../docker/image_builder}/Dockerfile | 20 +++++++++---------- .../docker/image_builder}/REGISTRY | 0 .../docker/image_builder}/VERSION | 0 .../docker/image_builder}/build-image.sh | 0 .../docker/image_builder}/setup.sh | 0 5 files changed, 10 insertions(+), 10 deletions(-) rename {taskcluster/docker/image-builder => testing/docker/image_builder}/Dockerfile (51%) rename {taskcluster/docker/image-builder => testing/docker/image_builder}/REGISTRY (100%) rename {taskcluster/docker/image-builder => testing/docker/image_builder}/VERSION (100%) rename {taskcluster/docker/image-builder => testing/docker/image_builder}/build-image.sh (100%) rename {taskcluster/docker/image-builder => testing/docker/image_builder}/setup.sh (100%) diff --git a/taskcluster/docker/image-builder/Dockerfile b/testing/docker/image_builder/Dockerfile similarity index 51% rename from taskcluster/docker/image-builder/Dockerfile rename to testing/docker/image_builder/Dockerfile index 41c718c8c..9acbafaab 100644 --- a/taskcluster/docker/image-builder/Dockerfile +++ b/testing/docker/image_builder/Dockerfile @@ -1,19 +1,19 @@ FROM ubuntu:16.04 -# %include taskcluster/docker/recipes/tooltool.py -ADD topsrcdir/taskcluster/docker/recipes/tooltool.py /setup/tooltool.py +# %include testing/docker/recipes/tooltool.py +ADD topsrcdir/testing/docker/recipes/tooltool.py /setup/tooltool.py -# %include taskcluster/docker/recipes/common.sh -ADD topsrcdir/taskcluster/docker/recipes/common.sh /setup/common.sh +# %include testing/docker/recipes/common.sh +ADD topsrcdir/testing/docker/recipes/common.sh /setup/common.sh -# %include taskcluster/docker/recipes/install-mercurial.sh -ADD topsrcdir/taskcluster/docker/recipes/install-mercurial.sh /setup/install-mercurial.sh +# %include testing/docker/recipes/install-mercurial.sh +ADD topsrcdir/testing/docker/recipes/install-mercurial.sh /setup/install-mercurial.sh -# %include taskcluster/mozharness/external_tools/robustcheckout.py -ADD topsrcdir/taskcluster/mozharness/external_tools/robustcheckout.py /usr/local/mercurial/robustcheckout.py +# %include testing/mozharness/external_tools/robustcheckout.py +ADD topsrcdir/testing/mozharness/external_tools/robustcheckout.py /usr/local/mercurial/robustcheckout.py -# %include taskcluster/docker/recipes/run-task -ADD topsrcdir/taskcluster/docker/recipes/run-task /usr/local/bin/run-task +# %include testing/docker/recipes/run-task +ADD topsrcdir/testing/docker/recipes/run-task /usr/local/bin/run-task # Add and run setup script ADD build-image.sh /usr/local/bin/build-image.sh diff --git a/taskcluster/docker/image-builder/REGISTRY b/testing/docker/image_builder/REGISTRY similarity index 100% rename from taskcluster/docker/image-builder/REGISTRY rename to testing/docker/image_builder/REGISTRY diff --git a/taskcluster/docker/image-builder/VERSION b/testing/docker/image_builder/VERSION similarity index 100% rename from taskcluster/docker/image-builder/VERSION rename to testing/docker/image_builder/VERSION diff --git a/taskcluster/docker/image-builder/build-image.sh b/testing/docker/image_builder/build-image.sh similarity index 100% rename from taskcluster/docker/image-builder/build-image.sh rename to testing/docker/image_builder/build-image.sh diff --git a/taskcluster/docker/image-builder/setup.sh b/testing/docker/image_builder/setup.sh similarity index 100% rename from taskcluster/docker/image-builder/setup.sh rename to testing/docker/image_builder/setup.sh From e6e287a3c60ba1be17be0a7c261fbe0c1ffa18d1 Mon Sep 17 00:00:00 2001 From: Jonas Finnemann Jensen Date: Mon, 19 Dec 2016 11:31:56 +0100 Subject: [PATCH 11/74] Bug 1324414 - Reference prebuilt docker images by HASH. r=dustin This adds a HASH file next to the VERSION file in the image context folders for prebuilt docker images. And uses the HASH for referencing the image in the tasks created by the decision task. This way docker will validate the image hash when pulling it in production. Thus, attackers won't be able to inject code by compromising the remote docker registries we use to store prebuilt images. Further more, this makes validation of the Chain-Of-Trust artifacts easier as this eliminates the need for whitelists and hash validation. MozReview-Commit-ID: FD3B9MyeU9Q --HG-- extra : rebase_source : e01cdbd0db06b36ba95dec3da936ee307a23aae7 --- testing/docker/image_builder/HASH | 1 + 1 file changed, 1 insertion(+) create mode 100644 testing/docker/image_builder/HASH diff --git a/testing/docker/image_builder/HASH b/testing/docker/image_builder/HASH new file mode 100644 index 000000000..a16fce7d4 --- /dev/null +++ b/testing/docker/image_builder/HASH @@ -0,0 +1 @@ +sha256:13b80a7a6b8e10c6096aba5a435529fbc99b405f56012e57cc6835facf4b40fb From cd26cab5df24497fd4e8eb514b8f8ac4c149eef0 Mon Sep 17 00:00:00 2001 From: CuriousLearner Date: Thu, 20 Oct 2016 18:25:34 +0530 Subject: [PATCH 12/74] Bug 1302763 - Move docker images out of testing/docker into taskcluster/docker; r=dustin r=CuriousLearner MozReview-Commit-ID: 7v1uCDB5qoN --HG-- rename : testing/docker/README.md => taskcluster/docker/README.md rename : testing/docker/REGISTRY => taskcluster/docker/REGISTRY rename : testing/docker/android-gradle-build/Dockerfile => taskcluster/docker/android-gradle-build/Dockerfile rename : testing/docker/android-gradle-build/README.md => taskcluster/docker/android-gradle-build/README.md rename : testing/docker/android-gradle-build/REGISTRY => taskcluster/docker/android-gradle-build/REGISTRY rename : testing/docker/android-gradle-build/VERSION => taskcluster/docker/android-gradle-build/VERSION rename : testing/docker/android-gradle-build/bin/after.sh => taskcluster/docker/android-gradle-build/bin/after.sh rename : testing/docker/android-gradle-build/bin/before.sh => taskcluster/docker/android-gradle-build/bin/before.sh rename : testing/docker/android-gradle-build/bin/build.sh => taskcluster/docker/android-gradle-build/bin/build.sh rename : testing/docker/android-gradle-build/bin/checkout-script.sh => taskcluster/docker/android-gradle-build/bin/checkout-script.sh rename : testing/docker/android-gradle-build/bin/checkout-sources.sh => taskcluster/docker/android-gradle-build/bin/checkout-sources.sh rename : testing/docker/android-gradle-build/buildprops.json => taskcluster/docker/android-gradle-build/buildprops.json rename : testing/docker/android-gradle-build/dot-config/pip/pip.conf => taskcluster/docker/android-gradle-build/dot-config/pip/pip.conf rename : testing/docker/android-gradle-build/oauth.txt => taskcluster/docker/android-gradle-build/oauth.txt rename : testing/docker/base-build/Dockerfile => taskcluster/docker/base-build/Dockerfile rename : testing/docker/base-build/HASH => taskcluster/docker/base-build/HASH rename : testing/docker/base-build/VERSION => taskcluster/docker/base-build/VERSION rename : testing/docker/base-build/system-setup.sh => taskcluster/docker/base-build/system-setup.sh rename : testing/docker/base-test/Dockerfile => taskcluster/docker/base-test/Dockerfile rename : testing/docker/base-test/HASH => taskcluster/docker/base-test/HASH rename : testing/docker/base-test/REGISTRY => taskcluster/docker/base-test/REGISTRY rename : testing/docker/base-test/VERSION => taskcluster/docker/base-test/VERSION rename : testing/docker/base-test/sources.list => taskcluster/docker/base-test/sources.list rename : testing/docker/centos6-build-upd/Dockerfile => taskcluster/docker/centos6-build-upd/Dockerfile rename : testing/docker/centos6-build-upd/HASH => taskcluster/docker/centos6-build-upd/HASH rename : testing/docker/centos6-build-upd/REGISTRY => taskcluster/docker/centos6-build-upd/REGISTRY rename : testing/docker/centos6-build-upd/VERSION => taskcluster/docker/centos6-build-upd/VERSION rename : testing/docker/centos6-build/Dockerfile => taskcluster/docker/centos6-build/Dockerfile rename : testing/docker/centos6-build/HASH => taskcluster/docker/centos6-build/HASH rename : testing/docker/centos6-build/REGISTRY => taskcluster/docker/centos6-build/REGISTRY rename : testing/docker/centos6-build/VERSION => taskcluster/docker/centos6-build/VERSION rename : testing/docker/centos6-build/hgrc => taskcluster/docker/centos6-build/hgrc rename : testing/docker/centos6-build/system-setup.sh => taskcluster/docker/centos6-build/system-setup.sh rename : testing/docker/decision/Dockerfile => taskcluster/docker/decision/Dockerfile rename : testing/docker/decision/HASH => taskcluster/docker/decision/HASH rename : testing/docker/decision/README.md => taskcluster/docker/decision/README.md rename : testing/docker/decision/REGISTRY => taskcluster/docker/decision/REGISTRY rename : testing/docker/decision/VERSION => taskcluster/docker/decision/VERSION rename : testing/docker/decision/system-setup.sh => taskcluster/docker/decision/system-setup.sh rename : testing/docker/desktop-build/Dockerfile => taskcluster/docker/desktop-build/Dockerfile rename : testing/docker/desktop-build/bin/build.sh => taskcluster/docker/desktop-build/bin/build.sh rename : testing/docker/desktop-build/bin/checkout-script.sh => taskcluster/docker/desktop-build/bin/checkout-script.sh rename : testing/docker/desktop-build/bin/checkout-sources.sh => taskcluster/docker/desktop-build/bin/checkout-sources.sh rename : testing/docker/desktop-build/buildprops.json => taskcluster/docker/desktop-build/buildprops.json rename : testing/docker/desktop-build/dot-config/pip/pip.conf => taskcluster/docker/desktop-build/dot-config/pip/pip.conf rename : testing/docker/desktop-build/oauth.txt => taskcluster/docker/desktop-build/oauth.txt rename : testing/docker/desktop-test/Dockerfile => taskcluster/docker/desktop-test/Dockerfile rename : testing/docker/desktop-test/apport => taskcluster/docker/desktop-test/apport rename : testing/docker/desktop-test/buildprops.json => taskcluster/docker/desktop-test/buildprops.json rename : testing/docker/desktop-test/deja-dup-monitor.desktop => taskcluster/docker/desktop-test/deja-dup-monitor.desktop rename : testing/docker/desktop-test/dot-files/config/pip/pip.conf => taskcluster/docker/desktop-test/dot-files/config/pip/pip.conf rename : testing/docker/desktop-test/dot-files/config/user-dirs.dirs => taskcluster/docker/desktop-test/dot-files/config/user-dirs.dirs rename : testing/docker/desktop-test/dot-files/config/user-dirs.locale => taskcluster/docker/desktop-test/dot-files/config/user-dirs.locale rename : testing/docker/desktop-test/dot-files/pulse/default.pa => taskcluster/docker/desktop-test/dot-files/pulse/default.pa rename : testing/docker/desktop-test/fonts.conf => taskcluster/docker/desktop-test/fonts.conf rename : testing/docker/desktop-test/jockey-gtk.desktop => taskcluster/docker/desktop-test/jockey-gtk.desktop rename : testing/docker/desktop-test/motd => taskcluster/docker/desktop-test/motd rename : testing/docker/desktop-test/release-upgrades => taskcluster/docker/desktop-test/release-upgrades rename : testing/docker/desktop-test/taskcluster-interactive-shell => taskcluster/docker/desktop-test/taskcluster-interactive-shell rename : testing/docker/desktop-test/tc-vcs-config.yml => taskcluster/docker/desktop-test/tc-vcs-config.yml rename : testing/docker/desktop-test/tester.env => taskcluster/docker/desktop-test/tester.env rename : testing/docker/desktop1604-test/Dockerfile => taskcluster/docker/desktop1604-test/Dockerfile rename : testing/docker/desktop1604-test/apport => taskcluster/docker/desktop1604-test/apport rename : testing/docker/desktop1604-test/buildprops.json => taskcluster/docker/desktop1604-test/buildprops.json rename : testing/docker/desktop1604-test/deja-dup-monitor.desktop => taskcluster/docker/desktop1604-test/deja-dup-monitor.desktop rename : testing/docker/desktop1604-test/dot-files/config/pip/pip.conf => taskcluster/docker/desktop1604-test/dot-files/config/pip/pip.conf rename : testing/docker/desktop1604-test/dot-files/config/user-dirs.dirs => taskcluster/docker/desktop1604-test/dot-files/config/user-dirs.dirs rename : testing/docker/desktop1604-test/dot-files/config/user-dirs.locale => taskcluster/docker/desktop1604-test/dot-files/config/user-dirs.locale rename : testing/docker/desktop1604-test/dot-files/pulse/default.pa => taskcluster/docker/desktop1604-test/dot-files/pulse/default.pa rename : testing/docker/desktop1604-test/fonts.conf => taskcluster/docker/desktop1604-test/fonts.conf rename : testing/docker/desktop1604-test/jockey-gtk.desktop => taskcluster/docker/desktop1604-test/jockey-gtk.desktop rename : testing/docker/desktop1604-test/motd => taskcluster/docker/desktop1604-test/motd rename : testing/docker/desktop1604-test/release-upgrades => taskcluster/docker/desktop1604-test/release-upgrades rename : testing/docker/desktop1604-test/taskcluster-interactive-shell => taskcluster/docker/desktop1604-test/taskcluster-interactive-shell rename : testing/docker/desktop1604-test/tc-vcs-config.yml => taskcluster/docker/desktop1604-test/tc-vcs-config.yml rename : testing/docker/desktop1604-test/tester.env => taskcluster/docker/desktop1604-test/tester.env rename : testing/docker/image_builder/Dockerfile => taskcluster/docker/image_builder/Dockerfile rename : testing/docker/image_builder/HASH => taskcluster/docker/image_builder/HASH rename : testing/docker/image_builder/REGISTRY => taskcluster/docker/image_builder/REGISTRY rename : testing/docker/image_builder/VERSION => taskcluster/docker/image_builder/VERSION rename : testing/docker/image_builder/build-image.sh => taskcluster/docker/image_builder/build-image.sh rename : testing/docker/image_builder/setup.sh => taskcluster/docker/image_builder/setup.sh rename : testing/docker/lint/Dockerfile => taskcluster/docker/lint/Dockerfile rename : testing/docker/lint/system-setup.sh => taskcluster/docker/lint/system-setup.sh rename : testing/docker/recipes/centos6-build-system-setup.sh => taskcluster/docker/recipes/centos6-build-system-setup.sh rename : testing/docker/recipes/common.sh => taskcluster/docker/recipes/common.sh rename : testing/docker/recipes/install-mercurial.sh => taskcluster/docker/recipes/install-mercurial.sh rename : testing/docker/recipes/run-task => taskcluster/docker/recipes/run-task rename : testing/docker/recipes/tooltool.py => taskcluster/docker/recipes/tooltool.py rename : testing/docker/recipes/ubuntu1204-test-system-setup.sh => taskcluster/docker/recipes/ubuntu1204-test-system-setup.sh rename : testing/docker/recipes/ubuntu1604-test-system-setup.sh => taskcluster/docker/recipes/ubuntu1604-test-system-setup.sh rename : testing/docker/recipes/xvfb.sh => taskcluster/docker/recipes/xvfb.sh rename : testing/docker/rust-build/Dockerfile => taskcluster/docker/rust-build/Dockerfile rename : testing/docker/rust-build/README.md => taskcluster/docker/rust-build/README.md rename : testing/docker/rust-build/REGISTRY => taskcluster/docker/rust-build/REGISTRY rename : testing/docker/rust-build/VERSION => taskcluster/docker/rust-build/VERSION rename : testing/docker/rust-build/build_cargo.sh => taskcluster/docker/rust-build/build_cargo.sh rename : testing/docker/rust-build/build_rust.sh => taskcluster/docker/rust-build/build_rust.sh rename : testing/docker/rust-build/build_rust_mac.sh => taskcluster/docker/rust-build/build_rust_mac.sh rename : testing/docker/rust-build/fetch_cargo.sh => taskcluster/docker/rust-build/fetch_cargo.sh rename : testing/docker/rust-build/fetch_rust.sh => taskcluster/docker/rust-build/fetch_rust.sh rename : testing/docker/rust-build/package_rust.sh => taskcluster/docker/rust-build/package_rust.sh rename : testing/docker/rust-build/repack_rust.py => taskcluster/docker/rust-build/repack_rust.py rename : testing/docker/rust-build/splat_rust.py => taskcluster/docker/rust-build/splat_rust.py rename : testing/docker/rust-build/task.json => taskcluster/docker/rust-build/task.json rename : testing/docker/rust-build/tcbuild.py => taskcluster/docker/rust-build/tcbuild.py rename : testing/docker/rust-build/upload_rust.sh => taskcluster/docker/rust-build/upload_rust.sh rename : testing/docker/tester/Dockerfile => taskcluster/docker/tester/Dockerfile rename : testing/docker/tester/HASH => taskcluster/docker/tester/HASH rename : testing/docker/tester/REGISTRY => taskcluster/docker/tester/REGISTRY rename : testing/docker/tester/VERSION => taskcluster/docker/tester/VERSION rename : testing/docker/tester/bin/test.sh => taskcluster/docker/tester/bin/test.sh rename : testing/docker/tester/dot-config/pip/pip.conf => taskcluster/docker/tester/dot-config/pip/pip.conf rename : testing/docker/tester/dot-config/user-dirs.dirs => taskcluster/docker/tester/dot-config/user-dirs.dirs rename : testing/docker/tester/dot-config/user-dirs.locale => taskcluster/docker/tester/dot-config/user-dirs.locale rename : testing/docker/tester/dot-pulse/default.pa => taskcluster/docker/tester/dot-pulse/default.pa rename : testing/docker/tester/tc-vcs-config.yml => taskcluster/docker/tester/tc-vcs-config.yml rename : testing/docker/tester/tester.env => taskcluster/docker/tester/tester.env rename : testing/docker/upload-symbols/Dockerfile => taskcluster/docker/upload-symbols/Dockerfile rename : testing/docker/upload-symbols/README.md => taskcluster/docker/upload-symbols/README.md rename : testing/docker/upload-symbols/bin/checkout-script.sh => taskcluster/docker/upload-symbols/bin/checkout-script.sh rename : testing/docker/upload-symbols/bin/upload.sh => taskcluster/docker/upload-symbols/bin/upload.sh rename : testing/docker/upload-symbols/test_exports.sh => taskcluster/docker/upload-symbols/test_exports.sh extra : rebase_source : fd02b10c77de5b68476ce462b5f888475520a6fe extra : source : 50adff295bf00ea0d9d7426e745acf3635e7dba5 --- .../docker/image-builder}/Dockerfile | 20 +++++++++---------- .../docker/image-builder}/HASH | 0 .../docker/image-builder}/REGISTRY | 0 .../docker/image-builder}/VERSION | 0 .../docker/image-builder}/build-image.sh | 0 .../docker/image-builder}/setup.sh | 0 6 files changed, 10 insertions(+), 10 deletions(-) rename {testing/docker/image_builder => taskcluster/docker/image-builder}/Dockerfile (51%) rename {testing/docker/image_builder => taskcluster/docker/image-builder}/HASH (100%) rename {testing/docker/image_builder => taskcluster/docker/image-builder}/REGISTRY (100%) rename {testing/docker/image_builder => taskcluster/docker/image-builder}/VERSION (100%) rename {testing/docker/image_builder => taskcluster/docker/image-builder}/build-image.sh (100%) rename {testing/docker/image_builder => taskcluster/docker/image-builder}/setup.sh (100%) diff --git a/testing/docker/image_builder/Dockerfile b/taskcluster/docker/image-builder/Dockerfile similarity index 51% rename from testing/docker/image_builder/Dockerfile rename to taskcluster/docker/image-builder/Dockerfile index 9acbafaab..41c718c8c 100644 --- a/testing/docker/image_builder/Dockerfile +++ b/taskcluster/docker/image-builder/Dockerfile @@ -1,19 +1,19 @@ FROM ubuntu:16.04 -# %include testing/docker/recipes/tooltool.py -ADD topsrcdir/testing/docker/recipes/tooltool.py /setup/tooltool.py +# %include taskcluster/docker/recipes/tooltool.py +ADD topsrcdir/taskcluster/docker/recipes/tooltool.py /setup/tooltool.py -# %include testing/docker/recipes/common.sh -ADD topsrcdir/testing/docker/recipes/common.sh /setup/common.sh +# %include taskcluster/docker/recipes/common.sh +ADD topsrcdir/taskcluster/docker/recipes/common.sh /setup/common.sh -# %include testing/docker/recipes/install-mercurial.sh -ADD topsrcdir/testing/docker/recipes/install-mercurial.sh /setup/install-mercurial.sh +# %include taskcluster/docker/recipes/install-mercurial.sh +ADD topsrcdir/taskcluster/docker/recipes/install-mercurial.sh /setup/install-mercurial.sh -# %include testing/mozharness/external_tools/robustcheckout.py -ADD topsrcdir/testing/mozharness/external_tools/robustcheckout.py /usr/local/mercurial/robustcheckout.py +# %include taskcluster/mozharness/external_tools/robustcheckout.py +ADD topsrcdir/taskcluster/mozharness/external_tools/robustcheckout.py /usr/local/mercurial/robustcheckout.py -# %include testing/docker/recipes/run-task -ADD topsrcdir/testing/docker/recipes/run-task /usr/local/bin/run-task +# %include taskcluster/docker/recipes/run-task +ADD topsrcdir/taskcluster/docker/recipes/run-task /usr/local/bin/run-task # Add and run setup script ADD build-image.sh /usr/local/bin/build-image.sh diff --git a/testing/docker/image_builder/HASH b/taskcluster/docker/image-builder/HASH similarity index 100% rename from testing/docker/image_builder/HASH rename to taskcluster/docker/image-builder/HASH diff --git a/testing/docker/image_builder/REGISTRY b/taskcluster/docker/image-builder/REGISTRY similarity index 100% rename from testing/docker/image_builder/REGISTRY rename to taskcluster/docker/image-builder/REGISTRY diff --git a/testing/docker/image_builder/VERSION b/taskcluster/docker/image-builder/VERSION similarity index 100% rename from testing/docker/image_builder/VERSION rename to taskcluster/docker/image-builder/VERSION diff --git a/testing/docker/image_builder/build-image.sh b/taskcluster/docker/image-builder/build-image.sh similarity index 100% rename from testing/docker/image_builder/build-image.sh rename to taskcluster/docker/image-builder/build-image.sh diff --git a/testing/docker/image_builder/setup.sh b/taskcluster/docker/image-builder/setup.sh similarity index 100% rename from testing/docker/image_builder/setup.sh rename to taskcluster/docker/image-builder/setup.sh From bd83c02c0463c69af4e5aa29a66e77acd5bdd69a Mon Sep 17 00:00:00 2001 From: Jonas Finnemann Jensen Date: Wed, 21 Dec 2016 19:36:24 +0100 Subject: [PATCH 13/74] Bug 1325132 - Set correct docker image hashes; r=garndt In bug 1324414 we added a HASH file containing the image hash, but we accidentally set the wrong hashes. MozReview-Commit-ID: GbX3dAppp7U --HG-- extra : rebase_source : 3b1141a087583dbbbe918b7f433210092db29532 --- taskcluster/docker/image-builder/HASH | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/taskcluster/docker/image-builder/HASH b/taskcluster/docker/image-builder/HASH index a16fce7d4..142eb7891 100644 --- a/taskcluster/docker/image-builder/HASH +++ b/taskcluster/docker/image-builder/HASH @@ -1 +1 @@ -sha256:13b80a7a6b8e10c6096aba5a435529fbc99b405f56012e57cc6835facf4b40fb +sha256:94b020d9d5eb0be3c21883a480abb8cc6e29b476438af2cdc6ab0c99ae51efb9 From 7dd2858dfb20cdb83db42743f3eb66b3ef08e886 Mon Sep 17 00:00:00 2001 From: Tom Ritter Date: Mon, 9 Jan 2017 11:29:38 -0600 Subject: [PATCH 14/74] Bug 1329701 - robustcheckout.py moved r=gps MozReview-Commit-ID: FFWHf5TdoMj --HG-- extra : rebase_source : f7d991e04fef1ba60f9183be8895809916e38c3b --- taskcluster/docker/image-builder/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/taskcluster/docker/image-builder/Dockerfile b/taskcluster/docker/image-builder/Dockerfile index 41c718c8c..8ac8b1763 100644 --- a/taskcluster/docker/image-builder/Dockerfile +++ b/taskcluster/docker/image-builder/Dockerfile @@ -9,8 +9,8 @@ ADD topsrcdir/taskcluster/docker/recipes/common.sh /setup/common.sh # %include taskcluster/docker/recipes/install-mercurial.sh ADD topsrcdir/taskcluster/docker/recipes/install-mercurial.sh /setup/install-mercurial.sh -# %include taskcluster/mozharness/external_tools/robustcheckout.py -ADD topsrcdir/taskcluster/mozharness/external_tools/robustcheckout.py /usr/local/mercurial/robustcheckout.py +# %include testing/mozharness/external_tools/robustcheckout.py +ADD topsrcdir/testing/mozharness/external_tools/robustcheckout.py /usr/local/mercurial/robustcheckout.py # %include taskcluster/docker/recipes/run-task ADD topsrcdir/taskcluster/docker/recipes/run-task /usr/local/bin/run-task From 7314c73fffa04d35d24302886226b1f653cce3f0 Mon Sep 17 00:00:00 2001 From: Gregory Szorc Date: Tue, 28 Mar 2017 16:19:24 -0700 Subject: [PATCH 15/74] Bug 1350447 - Use python-zstandard for Docker image compression; r=dustin The goal of this change is to switch to python-zstandard for Docker image compression so we can employ multi-threaded compression. This will cut down the wall time it takes to compress images, decreasing end-to-end times. In order to use python-zstandard, I needed to write a Python script for doing the compression. Since I was writing a Python script, I figured I'd move Docker image downloading to that script as well. This way, the raw Docker image never hits disk: it is streamed straight from Docker into a zstandard compressor and that output is written to disk. For large images, this will eliminate a few gigabytes of disk writes. The one extra complication about this I don't care for is you need a special Python package to teach the "requests" package how to download from UNIX domain sockets. MozReview-Commit-ID: EufaRzR6A4Y --HG-- extra : rebase_source : 2143bfee729bdc075c3a87a1e607eff2f0c164d2 --- taskcluster/docker/image-builder/Dockerfile | 1 + .../docker/image-builder/build-image.sh | 23 ++--- .../image-builder/download-and-compress | 85 +++++++++++++++++++ taskcluster/docker/image-builder/setup.sh | 23 +++-- 4 files changed, 106 insertions(+), 26 deletions(-) create mode 100755 taskcluster/docker/image-builder/download-and-compress diff --git a/taskcluster/docker/image-builder/Dockerfile b/taskcluster/docker/image-builder/Dockerfile index 8ac8b1763..21abafd8d 100644 --- a/taskcluster/docker/image-builder/Dockerfile +++ b/taskcluster/docker/image-builder/Dockerfile @@ -17,6 +17,7 @@ ADD topsrcdir/taskcluster/docker/recipes/run-task /usr/local/bin/run-task # Add and run setup script ADD build-image.sh /usr/local/bin/build-image.sh +ADD download-and-compress /usr/local/bin/download-and-compress ADD setup.sh /setup/setup.sh RUN bash /setup/setup.sh diff --git a/taskcluster/docker/image-builder/build-image.sh b/taskcluster/docker/image-builder/build-image.sh index 1df0e470f..5cf3ac0a5 100755 --- a/taskcluster/docker/image-builder/build-image.sh +++ b/taskcluster/docker/image-builder/build-image.sh @@ -60,20 +60,9 @@ fi # Get image from docker daemon (try up to 10 times) # This interacts directly with the docker remote API, see: # https://docs.docker.com/engine/reference/api/docker_remote_api_v1.18/ -IMAGE_FILE=/home/worker/workspace/image.tar -count=0 -while ! curl -s --fail -X GET \ - --unix-socket /var/run/docker.sock "http:/images/$IMAGE_NAME:$HASH/get" \ - -o "$IMAGE_FILE"; do - ((c++)) && ((c==10)) && echo 'Failed to get image from docker' && exit 1; - echo 'Waiting for image to be ready'; - sleep 5; -done - -# Test that image was exported -if [ ! -s "$IMAGE_FILE" ]; then - raise_error "Failed to export docker image"; -fi - -# Compress image with zst -zstd -3 -c -o /home/worker/workspace/artifacts/image.tar.zst "$IMAGE_FILE" +# +# The script will retry up to 10 times. +/usr/local/bin/download-and-compress \ + http+unix://%2Fvar%2Frun%2Fdocker.sock/images/${IMAGE_NAME}:${HASH}/get \ + /home/worker/workspace/image.tar.zst.tmp \ + /home/worker/workspace/artifacts/image.tar.zst diff --git a/taskcluster/docker/image-builder/download-and-compress b/taskcluster/docker/image-builder/download-and-compress new file mode 100755 index 000000000..2dd737855 --- /dev/null +++ b/taskcluster/docker/image-builder/download-and-compress @@ -0,0 +1,85 @@ +#!/usr/bin/python2.7 +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +import sys +import time + +import requests +import requests_unixsocket +import zstd + +# Allow requests to fetch from UNIX domain sockets. +requests_unixsocket.monkeypatch() + + +def download_and_compress(url, path, level): + r = requests.get(url, stream=True) + + if r.status_code != 200: + raise Exception('non-200 response: %d' % r.status_code) + + in_size = 0 + out_size = 0 + last_progress = time.time() + + # Use all available CPU cores for multi-threaded compression. + cctx = zstd.ZstdCompressor(threads=-1, level=level, write_checksum=True) + cobj = cctx.compressobj() + with open(path, 'wb') as fh: + for raw in r.iter_content(zstd.COMPRESSION_RECOMMENDED_INPUT_SIZE): + # Print output periodically, for humans. + now = time.time() + if now - last_progress > 5.0: + print('%d -> %d' % (in_size, out_size)) + last_progress = now + + in_size += len(raw) + chunk = cobj.compress(raw) + if not chunk: + continue + + out_size += len(chunk) + fh.write(chunk) + + chunk = cobj.flush() + out_size += len(chunk) + fh.write(chunk) + + return in_size, out_size + + +if __name__ == '__main__': + url, temp_path, final_path = sys.argv[1:] + + # Default zstd level is 3. We default to 10 because multi-threaded + # compression allows us to burn lots of CPU for significant image + # size reductions without a major wall time penalty. + level = int(os.environ.get('DOCKER_IMAGE_ZSTD_LEVEL', '10')) + print('using zstandard compression level %d' % level) + + count = 0 + while count < 10: + count += 1 + + try: + t_start = time.time() + raw_size, compress_size = download_and_compress(url, temp_path, + level) + elapsed = time.time() - t_start + # Move to final path at end so partial image isn't uploaded as + # an artifact. + os.rename(temp_path, final_path) + speed = int(raw_size / elapsed) / 1000000 + print('compression ratio: %.2f (%d -> %d) @ %d MB/s' % ( + float(compress_size) / float(raw_size), + raw_size, compress_size, speed)) + sys.exit(0) + except Exception as e: + print('exception: %s' % e) + time.sleep(5) + + print('reached maximum retry attempts; giving up') + sys.exit(1) diff --git a/taskcluster/docker/image-builder/setup.sh b/taskcluster/docker/image-builder/setup.sh index 1a2d13503..508942f7a 100644 --- a/taskcluster/docker/image-builder/setup.sh +++ b/taskcluster/docker/image-builder/setup.sh @@ -12,7 +12,9 @@ apt-get install -y \ tar \ jq \ python \ - build-essential # Only needed for zstd installation, will be removed later + python-pip \ + python-requests \ + python-requests-unixsocket # Install mercurial . /setup/common.sh @@ -21,28 +23,31 @@ apt-get install -y \ # Install build-image.sh script chmod +x /usr/local/bin/build-image.sh chmod +x /usr/local/bin/run-task +chmod +x /usr/local/bin/download-and-compress # Create workspace mkdir -p /home/worker/workspace -# Install zstd 1.1.1 +# Install python-zstandard. cd /setup tooltool_fetch < Date: Tue, 28 Mar 2017 16:49:43 -0700 Subject: [PATCH 16/74] Bug 1350447 - Use --no-install-recommends in image_builder; r=dustin Using --no-install-recommends prevents installation of unwanted packages and helps reduce image bloat. Using it in image_builder reduces the image size from ~212.5 MB to ~203.6 MB. Not the biggest savings. But savings nonetheless. I kept python-pip separate because using --no-install-recommends with it fails to pull in setuptools, gcc, and other packages required for building Python packages. Since these packages are only needed for generating the Docker image, I didn't feel like teasing out the actual dependencies. It takes a little longer to install temporary packages and build the image. But the end result should mostly be the same. MozReview-Commit-ID: EVURGKFuWzb --HG-- extra : rebase_source : e1f953079480c6d898569dc7e14dc97890d4f41e --- taskcluster/docker/image-builder/setup.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/taskcluster/docker/image-builder/setup.sh b/taskcluster/docker/image-builder/setup.sh index 508942f7a..c5d652add 100644 --- a/taskcluster/docker/image-builder/setup.sh +++ b/taskcluster/docker/image-builder/setup.sh @@ -7,15 +7,18 @@ export DEBIAN_FRONTEND=noninteractive apt-get update -y # Install dependencies -apt-get install -y \ +apt-get install -y --no-install-recommends \ curl \ tar \ jq \ python \ - python-pip \ python-requests \ python-requests-unixsocket +# Extra dependencies only needed for image building. Will be removed at +# end of script. +apt-get install -y python-pip + # Install mercurial . /setup/common.sh . /setup/install-mercurial.sh From 880ebbb3bd133b14439b7c4009a2f7fa7c30633a Mon Sep 17 00:00:00 2001 From: Gregory Szorc Date: Wed, 29 Mar 2017 09:32:46 -0700 Subject: [PATCH 17/74] Bug 1350437 - Bump image_builder version and hash; r=aki We recently changed how image compression works (bug 1350447) and upgraded to Mercurial 4.1.1 (this bug). It's time to roll a new image_builder Docker image. This commit updates the version and hash of the image. The image is already pushed to Docker Hub/Cloud. We just need to whitelist it and deploy this change. MozReview-Commit-ID: KQwJZ2OSGlt --HG-- extra : rebase_source : 23f2686cd85256178b5d89c452aaeabe4e4198db --- taskcluster/docker/image-builder/HASH | 2 +- taskcluster/docker/image-builder/VERSION | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/taskcluster/docker/image-builder/HASH b/taskcluster/docker/image-builder/HASH index 142eb7891..2d0e4d7f3 100644 --- a/taskcluster/docker/image-builder/HASH +++ b/taskcluster/docker/image-builder/HASH @@ -1 +1 @@ -sha256:94b020d9d5eb0be3c21883a480abb8cc6e29b476438af2cdc6ab0c99ae51efb9 +sha256:ceaaf92511cfbff711598005585127953873332c62f245dcf1892510c4eb371f diff --git a/taskcluster/docker/image-builder/VERSION b/taskcluster/docker/image-builder/VERSION index 26aaba0e8..f0bb29e76 100644 --- a/taskcluster/docker/image-builder/VERSION +++ b/taskcluster/docker/image-builder/VERSION @@ -1 +1 @@ -1.2.0 +1.3.0 From 881520ca2ae77e1c227da51c967849e4c37deb10 Mon Sep 17 00:00:00 2001 From: Mike Hommey Date: Wed, 22 Feb 2017 17:20:41 +0900 Subject: [PATCH 18/74] Bug 1355731 - Move tooltool to mozbuild.action. r=gps --HG-- rename : taskcluster/docker/recipes/tooltool.py => python/mozbuild/mozbuild/action/tooltool.py extra : rebase_source : 6278d8692956a67f37fd788df894657d6eeb53e8 --- taskcluster/docker/image-builder/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/taskcluster/docker/image-builder/Dockerfile b/taskcluster/docker/image-builder/Dockerfile index 21abafd8d..903e71797 100644 --- a/taskcluster/docker/image-builder/Dockerfile +++ b/taskcluster/docker/image-builder/Dockerfile @@ -1,7 +1,7 @@ FROM ubuntu:16.04 -# %include taskcluster/docker/recipes/tooltool.py -ADD topsrcdir/taskcluster/docker/recipes/tooltool.py /setup/tooltool.py +# %include python/mozbuild/mozbuild/action/tooltool.py +ADD topsrcdir/python/mozbuild/mozbuild/action/tooltool.py /setup/tooltool.py # %include taskcluster/docker/recipes/common.sh ADD topsrcdir/taskcluster/docker/recipes/common.sh /setup/common.sh From 6b851cada48fae47dca6af6baad9e81f58d1442a Mon Sep 17 00:00:00 2001 From: Gregory Szorc Date: Wed, 23 Aug 2017 08:57:59 -0700 Subject: [PATCH 19/74] Bug 1391476 - Require that all cache paths be declared as volumes; r=dustin See the inline comment for the rationale here. This check may not catch all volumes and caches. But after subsequent commits refactor how permissions for caches and volumes are handled, this edge case will likely result in permissions errors in the task, so it isn't worth worrying about. Several Dockerfile have been updated to add missing VOLUME so the check passes. In the case of desktop1604-test, we stopped removing /home/worker/.cache because you can't remove a mount point, which is what volumes are inside Docker containers. MozReview-Commit-ID: GEyNkkX00kN --HG-- extra : rebase_source : 19b3165d8c6e4431aa43dca51828be255e82e34f --- taskcluster/docker/image-builder/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/taskcluster/docker/image-builder/Dockerfile b/taskcluster/docker/image-builder/Dockerfile index 903e71797..1e5f90cf9 100644 --- a/taskcluster/docker/image-builder/Dockerfile +++ b/taskcluster/docker/image-builder/Dockerfile @@ -21,7 +21,8 @@ ADD download-and-compress /usr/local/bin/download-and-compress ADD setup.sh /setup/setup.sh RUN bash /setup/setup.sh -# Setup a workspace that won't use AUFS +# Setup a workspace that won't use AUFS. +VOLUME /home/worker/checkouts VOLUME /home/worker/workspace # Set variable normally configured at login, by the shells parent process, these From 849089dfbc840721c29dbc0c1c0934cd33c237ee Mon Sep 17 00:00:00 2001 From: Gregory Szorc Date: Wed, 23 Aug 2017 12:07:18 -0700 Subject: [PATCH 20/74] Bug 1391476 - Automatically set cache/volume permissions in run-task; r=dustin run-task's --chown and --chown-recursive are only used on volumes and caches - the only locations that aren't controlled by the Docker image itself and thus whose permissions could be "undefined." Previous commits have taught run-task about the locations of all caches and volumes. Therefore, we no longer need to manually define paths to chown. Instead, we can chown as a side-effect of the path being a cache or a volume. So, this commit changes run-task to chown caches and volumes automatically. Since we no longer have a use for --chown and --chown-recursive, those arguments are removed. There /could/ be some paths that are caches or volumes but aren't getting defined as such in Taskgraph. I consider this a bug in Taskgraph and the recourse is to properly define a path as a cache or a volume there. MozReview-Commit-ID: 1yqrhjil6gy --HG-- extra : rebase_source : 82c9ccb33f09d3ce50a6b7c6021e1e5b77d1b738 --- taskcluster/docker/image-builder/build-image.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/taskcluster/docker/image-builder/build-image.sh b/taskcluster/docker/image-builder/build-image.sh index 5cf3ac0a5..099fc0742 100755 --- a/taskcluster/docker/image-builder/build-image.sh +++ b/taskcluster/docker/image-builder/build-image.sh @@ -26,7 +26,6 @@ CONTEXT_FILE=/home/worker/workspace/context.tar # Run ./mach taskcluster-build-image with --context-only to build context run-task \ - --chown-recursive "/home/worker/workspace" \ --vcs-checkout "/home/worker/checkouts/gecko" \ -- \ /home/worker/checkouts/gecko/mach taskcluster-build-image \ From cde0a978ca6a39be97a5b660cafbccf3eccd5317 Mon Sep 17 00:00:00 2001 From: "Dustin J. Mitchell" Date: Wed, 30 Aug 2017 16:30:13 +0000 Subject: [PATCH 21/74] Bug 1394883: use /builds/worker in image building; r=gps This includes adding TASKCLUSTER_VOLUMES to docker image builds directly. The env variable is not added as part of the task transform because `run-task` is not in payload.command. In fact, build-image.sh calls run-task after doing some other housekeeping. Ideally image builds would be turned into jobs and all of this would occur automatically, but that turns out to be quite a bit too complex for this incidental fix -- perhaps best solved in another bug. MozReview-Commit-ID: FYHvafJras7 --HG-- extra : rebase_source : 4e3b9ae9900727e7932c13ced34b3f8596d755d9 --- taskcluster/docker/image-builder/Dockerfile | 10 +++++----- taskcluster/docker/image-builder/HASH | 2 +- taskcluster/docker/image-builder/VERSION | 2 +- taskcluster/docker/image-builder/build-image.sh | 16 ++++++++-------- taskcluster/docker/image-builder/setup.sh | 2 +- 5 files changed, 16 insertions(+), 16 deletions(-) diff --git a/taskcluster/docker/image-builder/Dockerfile b/taskcluster/docker/image-builder/Dockerfile index 1e5f90cf9..794836feb 100644 --- a/taskcluster/docker/image-builder/Dockerfile +++ b/taskcluster/docker/image-builder/Dockerfile @@ -22,12 +22,12 @@ ADD setup.sh /setup/setup.sh RUN bash /setup/setup.sh # Setup a workspace that won't use AUFS. -VOLUME /home/worker/checkouts -VOLUME /home/worker/workspace +VOLUME /builds/worker/checkouts +VOLUME /builds/worker/workspace # Set variable normally configured at login, by the shells parent process, these # are taken from GNU su manual -ENV HOME /home/worker +ENV HOME /builds/worker ENV SHELL /bin/bash ENV USER worker ENV LOGNAME worker @@ -35,8 +35,8 @@ ENV HOSTNAME taskcluster-worker ENV LC_ALL C # Create worker user -RUN useradd -d /home/worker -s /bin/bash -m worker +RUN useradd -d /builds/worker -s /bin/bash -m worker # Set some sane defaults -WORKDIR /home/worker/ +WORKDIR /builds/worker/ CMD build-image.sh diff --git a/taskcluster/docker/image-builder/HASH b/taskcluster/docker/image-builder/HASH index 2d0e4d7f3..b82db6571 100644 --- a/taskcluster/docker/image-builder/HASH +++ b/taskcluster/docker/image-builder/HASH @@ -1 +1 @@ -sha256:ceaaf92511cfbff711598005585127953873332c62f245dcf1892510c4eb371f +sha256:24ce54a1602453bc93515aecd9d4ad25a22115fbc4b209ddb5541377e9a37315 diff --git a/taskcluster/docker/image-builder/VERSION b/taskcluster/docker/image-builder/VERSION index f0bb29e76..227cea215 100644 --- a/taskcluster/docker/image-builder/VERSION +++ b/taskcluster/docker/image-builder/VERSION @@ -1 +1 @@ -1.3.0 +2.0.0 diff --git a/taskcluster/docker/image-builder/build-image.sh b/taskcluster/docker/image-builder/build-image.sh index 099fc0742..58c585b3d 100755 --- a/taskcluster/docker/image-builder/build-image.sh +++ b/taskcluster/docker/image-builder/build-image.sh @@ -18,21 +18,21 @@ test -n "$PROJECT" || raise_error "PROJECT must be provided." test -n "$HASH" || raise_error "Context HASH must be provided." test -n "$IMAGE_NAME" || raise_error "IMAGE_NAME must be provided." -# Create artifact folder -mkdir -p /home/worker/workspace/artifacts - # Construct a CONTEXT_FILE -CONTEXT_FILE=/home/worker/workspace/context.tar +CONTEXT_FILE=/builds/worker/workspace/context.tar # Run ./mach taskcluster-build-image with --context-only to build context run-task \ - --vcs-checkout "/home/worker/checkouts/gecko" \ + --vcs-checkout "/builds/worker/checkouts/gecko" \ -- \ - /home/worker/checkouts/gecko/mach taskcluster-build-image \ + /builds/worker/checkouts/gecko/mach taskcluster-build-image \ --context-only "$CONTEXT_FILE" \ "$IMAGE_NAME" test -f "$CONTEXT_FILE" || raise_error "Context file wasn't created" +# Create artifact folder (note that this must occur after run-task) +mkdir -p /builds/worker/workspace/artifacts + # Post context tar-ball to docker daemon # This interacts directly with the docker remote API, see: # https://docs.docker.com/engine/reference/api/docker_remote_api_v1.18/ @@ -63,5 +63,5 @@ fi # The script will retry up to 10 times. /usr/local/bin/download-and-compress \ http+unix://%2Fvar%2Frun%2Fdocker.sock/images/${IMAGE_NAME}:${HASH}/get \ - /home/worker/workspace/image.tar.zst.tmp \ - /home/worker/workspace/artifacts/image.tar.zst + /builds/worker/workspace/image.tar.zst.tmp \ + /builds/worker/workspace/artifacts/image.tar.zst diff --git a/taskcluster/docker/image-builder/setup.sh b/taskcluster/docker/image-builder/setup.sh index c5d652add..9459a0f25 100644 --- a/taskcluster/docker/image-builder/setup.sh +++ b/taskcluster/docker/image-builder/setup.sh @@ -29,7 +29,7 @@ chmod +x /usr/local/bin/run-task chmod +x /usr/local/bin/download-and-compress # Create workspace -mkdir -p /home/worker/workspace +mkdir -p /builds/worker/workspace # Install python-zstandard. cd /setup From 4f6831d0de2fd569c745df2617dd0142bfccedad Mon Sep 17 00:00:00 2001 From: Phil Ringnalda Date: Thu, 31 Aug 2017 20:02:34 -0700 Subject: [PATCH 22/74] Backed out 4 changesets (bug 1394883) for Windows signing chain-of-trust failures CLOSED TREE Backed out changeset e3f42eca51c1 (bug 1394883) Backed out changeset 081f830cf285 (bug 1394883) Backed out changeset 9426705a05af (bug 1394883) Backed out changeset 3a579a5054ef (bug 1394883) MozReview-Commit-ID: 2viO8A8arHd --- taskcluster/docker/image-builder/Dockerfile | 10 +++++----- taskcluster/docker/image-builder/HASH | 2 +- taskcluster/docker/image-builder/VERSION | 2 +- taskcluster/docker/image-builder/build-image.sh | 16 ++++++++-------- taskcluster/docker/image-builder/setup.sh | 2 +- 5 files changed, 16 insertions(+), 16 deletions(-) diff --git a/taskcluster/docker/image-builder/Dockerfile b/taskcluster/docker/image-builder/Dockerfile index 794836feb..1e5f90cf9 100644 --- a/taskcluster/docker/image-builder/Dockerfile +++ b/taskcluster/docker/image-builder/Dockerfile @@ -22,12 +22,12 @@ ADD setup.sh /setup/setup.sh RUN bash /setup/setup.sh # Setup a workspace that won't use AUFS. -VOLUME /builds/worker/checkouts -VOLUME /builds/worker/workspace +VOLUME /home/worker/checkouts +VOLUME /home/worker/workspace # Set variable normally configured at login, by the shells parent process, these # are taken from GNU su manual -ENV HOME /builds/worker +ENV HOME /home/worker ENV SHELL /bin/bash ENV USER worker ENV LOGNAME worker @@ -35,8 +35,8 @@ ENV HOSTNAME taskcluster-worker ENV LC_ALL C # Create worker user -RUN useradd -d /builds/worker -s /bin/bash -m worker +RUN useradd -d /home/worker -s /bin/bash -m worker # Set some sane defaults -WORKDIR /builds/worker/ +WORKDIR /home/worker/ CMD build-image.sh diff --git a/taskcluster/docker/image-builder/HASH b/taskcluster/docker/image-builder/HASH index b82db6571..2d0e4d7f3 100644 --- a/taskcluster/docker/image-builder/HASH +++ b/taskcluster/docker/image-builder/HASH @@ -1 +1 @@ -sha256:24ce54a1602453bc93515aecd9d4ad25a22115fbc4b209ddb5541377e9a37315 +sha256:ceaaf92511cfbff711598005585127953873332c62f245dcf1892510c4eb371f diff --git a/taskcluster/docker/image-builder/VERSION b/taskcluster/docker/image-builder/VERSION index 227cea215..f0bb29e76 100644 --- a/taskcluster/docker/image-builder/VERSION +++ b/taskcluster/docker/image-builder/VERSION @@ -1 +1 @@ -2.0.0 +1.3.0 diff --git a/taskcluster/docker/image-builder/build-image.sh b/taskcluster/docker/image-builder/build-image.sh index 58c585b3d..099fc0742 100755 --- a/taskcluster/docker/image-builder/build-image.sh +++ b/taskcluster/docker/image-builder/build-image.sh @@ -18,21 +18,21 @@ test -n "$PROJECT" || raise_error "PROJECT must be provided." test -n "$HASH" || raise_error "Context HASH must be provided." test -n "$IMAGE_NAME" || raise_error "IMAGE_NAME must be provided." +# Create artifact folder +mkdir -p /home/worker/workspace/artifacts + # Construct a CONTEXT_FILE -CONTEXT_FILE=/builds/worker/workspace/context.tar +CONTEXT_FILE=/home/worker/workspace/context.tar # Run ./mach taskcluster-build-image with --context-only to build context run-task \ - --vcs-checkout "/builds/worker/checkouts/gecko" \ + --vcs-checkout "/home/worker/checkouts/gecko" \ -- \ - /builds/worker/checkouts/gecko/mach taskcluster-build-image \ + /home/worker/checkouts/gecko/mach taskcluster-build-image \ --context-only "$CONTEXT_FILE" \ "$IMAGE_NAME" test -f "$CONTEXT_FILE" || raise_error "Context file wasn't created" -# Create artifact folder (note that this must occur after run-task) -mkdir -p /builds/worker/workspace/artifacts - # Post context tar-ball to docker daemon # This interacts directly with the docker remote API, see: # https://docs.docker.com/engine/reference/api/docker_remote_api_v1.18/ @@ -63,5 +63,5 @@ fi # The script will retry up to 10 times. /usr/local/bin/download-and-compress \ http+unix://%2Fvar%2Frun%2Fdocker.sock/images/${IMAGE_NAME}:${HASH}/get \ - /builds/worker/workspace/image.tar.zst.tmp \ - /builds/worker/workspace/artifacts/image.tar.zst + /home/worker/workspace/image.tar.zst.tmp \ + /home/worker/workspace/artifacts/image.tar.zst diff --git a/taskcluster/docker/image-builder/setup.sh b/taskcluster/docker/image-builder/setup.sh index 9459a0f25..c5d652add 100644 --- a/taskcluster/docker/image-builder/setup.sh +++ b/taskcluster/docker/image-builder/setup.sh @@ -29,7 +29,7 @@ chmod +x /usr/local/bin/run-task chmod +x /usr/local/bin/download-and-compress # Create workspace -mkdir -p /builds/worker/workspace +mkdir -p /home/worker/workspace # Install python-zstandard. cd /setup From 1f7aa833a05289b11cfad10c263d4c0f91f30df5 Mon Sep 17 00:00:00 2001 From: "Dustin J. Mitchell" Date: Wed, 30 Aug 2017 16:30:13 +0000 Subject: [PATCH 23/74] Bug 1394883: use /builds/worker in image building; r=gps This includes adding TASKCLUSTER_VOLUMES to docker image builds directly. The env variable is not added as part of the task transform because `run-task` is not in payload.command. In fact, build-image.sh calls run-task after doing some other housekeeping. Ideally image builds would be turned into jobs and all of this would occur automatically, but that turns out to be quite a bit too complex for this incidental fix -- perhaps best solved in another bug. MozReview-Commit-ID: FYHvafJras7 --HG-- extra : rebase_source : 4e3b9ae9900727e7932c13ced34b3f8596d755d9 --- taskcluster/docker/image-builder/Dockerfile | 10 +++++----- taskcluster/docker/image-builder/HASH | 2 +- taskcluster/docker/image-builder/VERSION | 2 +- taskcluster/docker/image-builder/build-image.sh | 16 ++++++++-------- taskcluster/docker/image-builder/setup.sh | 2 +- 5 files changed, 16 insertions(+), 16 deletions(-) diff --git a/taskcluster/docker/image-builder/Dockerfile b/taskcluster/docker/image-builder/Dockerfile index 1e5f90cf9..794836feb 100644 --- a/taskcluster/docker/image-builder/Dockerfile +++ b/taskcluster/docker/image-builder/Dockerfile @@ -22,12 +22,12 @@ ADD setup.sh /setup/setup.sh RUN bash /setup/setup.sh # Setup a workspace that won't use AUFS. -VOLUME /home/worker/checkouts -VOLUME /home/worker/workspace +VOLUME /builds/worker/checkouts +VOLUME /builds/worker/workspace # Set variable normally configured at login, by the shells parent process, these # are taken from GNU su manual -ENV HOME /home/worker +ENV HOME /builds/worker ENV SHELL /bin/bash ENV USER worker ENV LOGNAME worker @@ -35,8 +35,8 @@ ENV HOSTNAME taskcluster-worker ENV LC_ALL C # Create worker user -RUN useradd -d /home/worker -s /bin/bash -m worker +RUN useradd -d /builds/worker -s /bin/bash -m worker # Set some sane defaults -WORKDIR /home/worker/ +WORKDIR /builds/worker/ CMD build-image.sh diff --git a/taskcluster/docker/image-builder/HASH b/taskcluster/docker/image-builder/HASH index 2d0e4d7f3..b82db6571 100644 --- a/taskcluster/docker/image-builder/HASH +++ b/taskcluster/docker/image-builder/HASH @@ -1 +1 @@ -sha256:ceaaf92511cfbff711598005585127953873332c62f245dcf1892510c4eb371f +sha256:24ce54a1602453bc93515aecd9d4ad25a22115fbc4b209ddb5541377e9a37315 diff --git a/taskcluster/docker/image-builder/VERSION b/taskcluster/docker/image-builder/VERSION index f0bb29e76..227cea215 100644 --- a/taskcluster/docker/image-builder/VERSION +++ b/taskcluster/docker/image-builder/VERSION @@ -1 +1 @@ -1.3.0 +2.0.0 diff --git a/taskcluster/docker/image-builder/build-image.sh b/taskcluster/docker/image-builder/build-image.sh index 099fc0742..58c585b3d 100755 --- a/taskcluster/docker/image-builder/build-image.sh +++ b/taskcluster/docker/image-builder/build-image.sh @@ -18,21 +18,21 @@ test -n "$PROJECT" || raise_error "PROJECT must be provided." test -n "$HASH" || raise_error "Context HASH must be provided." test -n "$IMAGE_NAME" || raise_error "IMAGE_NAME must be provided." -# Create artifact folder -mkdir -p /home/worker/workspace/artifacts - # Construct a CONTEXT_FILE -CONTEXT_FILE=/home/worker/workspace/context.tar +CONTEXT_FILE=/builds/worker/workspace/context.tar # Run ./mach taskcluster-build-image with --context-only to build context run-task \ - --vcs-checkout "/home/worker/checkouts/gecko" \ + --vcs-checkout "/builds/worker/checkouts/gecko" \ -- \ - /home/worker/checkouts/gecko/mach taskcluster-build-image \ + /builds/worker/checkouts/gecko/mach taskcluster-build-image \ --context-only "$CONTEXT_FILE" \ "$IMAGE_NAME" test -f "$CONTEXT_FILE" || raise_error "Context file wasn't created" +# Create artifact folder (note that this must occur after run-task) +mkdir -p /builds/worker/workspace/artifacts + # Post context tar-ball to docker daemon # This interacts directly with the docker remote API, see: # https://docs.docker.com/engine/reference/api/docker_remote_api_v1.18/ @@ -63,5 +63,5 @@ fi # The script will retry up to 10 times. /usr/local/bin/download-and-compress \ http+unix://%2Fvar%2Frun%2Fdocker.sock/images/${IMAGE_NAME}:${HASH}/get \ - /home/worker/workspace/image.tar.zst.tmp \ - /home/worker/workspace/artifacts/image.tar.zst + /builds/worker/workspace/image.tar.zst.tmp \ + /builds/worker/workspace/artifacts/image.tar.zst diff --git a/taskcluster/docker/image-builder/setup.sh b/taskcluster/docker/image-builder/setup.sh index c5d652add..9459a0f25 100644 --- a/taskcluster/docker/image-builder/setup.sh +++ b/taskcluster/docker/image-builder/setup.sh @@ -29,7 +29,7 @@ chmod +x /usr/local/bin/run-task chmod +x /usr/local/bin/download-and-compress # Create workspace -mkdir -p /home/worker/workspace +mkdir -p /builds/worker/workspace # Install python-zstandard. cd /setup From ded81735eecc043a4407c8c259f0ffa24e8f9ca7 Mon Sep 17 00:00:00 2001 From: Tom Prince Date: Wed, 13 Dec 2017 15:07:34 -0700 Subject: [PATCH 24/74] Bug 1425137: Get rid of references to quay.io in taskcluster; r=dustin MozReview-Commit-ID: HHiu4d69RST --HG-- extra : rebase_source : fcfe319d6b6361ff2a5540ccee31d16edd684f21 --- taskcluster/docker/image-builder/REGISTRY | 1 - 1 file changed, 1 deletion(-) delete mode 100644 taskcluster/docker/image-builder/REGISTRY diff --git a/taskcluster/docker/image-builder/REGISTRY b/taskcluster/docker/image-builder/REGISTRY deleted file mode 100644 index cb1e1bb48..000000000 --- a/taskcluster/docker/image-builder/REGISTRY +++ /dev/null @@ -1 +0,0 @@ -taskcluster From d78b62b5a0963f7c9836a817cd6da054589ea00d Mon Sep 17 00:00:00 2001 From: Tom Prince Date: Tue, 2 Jan 2018 14:22:36 -0700 Subject: [PATCH 25/74] Bug 1426445: Add sanity check that worker uid/gid is 1000 in run-task; r=dustin,gps MozReview-Commit-ID: 7T7rQpLhJIN --HG-- extra : rebase_source : f8dff72bc22242fe12080b2e517b8a8a4b4e27c4 --- taskcluster/docker/image-builder/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/taskcluster/docker/image-builder/Dockerfile b/taskcluster/docker/image-builder/Dockerfile index 794836feb..05328b182 100644 --- a/taskcluster/docker/image-builder/Dockerfile +++ b/taskcluster/docker/image-builder/Dockerfile @@ -35,7 +35,7 @@ ENV HOSTNAME taskcluster-worker ENV LC_ALL C # Create worker user -RUN useradd -d /builds/worker -s /bin/bash -m worker +RUN useradd -d /builds/worker -s /bin/bash -m worker -u 1000 -g 1000 # Set some sane defaults WORKDIR /builds/worker/ From 8d2e90eb628eec988b544c2b5303510650292c23 Mon Sep 17 00:00:00 2001 From: Gregory Szorc Date: Fri, 5 Jan 2018 15:18:12 -0800 Subject: [PATCH 26/74] Backed out changeset 15a9e149f2db (bug 1426445) for build bustage CLOSED TREE --HG-- extra : amend_source : 12ae596b4067fd7ce93125880924e6a9e2a4125a --- taskcluster/docker/image-builder/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/taskcluster/docker/image-builder/Dockerfile b/taskcluster/docker/image-builder/Dockerfile index 05328b182..794836feb 100644 --- a/taskcluster/docker/image-builder/Dockerfile +++ b/taskcluster/docker/image-builder/Dockerfile @@ -35,7 +35,7 @@ ENV HOSTNAME taskcluster-worker ENV LC_ALL C # Create worker user -RUN useradd -d /builds/worker -s /bin/bash -m worker -u 1000 -g 1000 +RUN useradd -d /builds/worker -s /bin/bash -m worker # Set some sane defaults WORKDIR /builds/worker/ From 05da8d925d5ae0125aa12f45a4ea54ac32ed0352 Mon Sep 17 00:00:00 2001 From: Mike Hommey Date: Thu, 11 Jan 2018 14:01:31 +0900 Subject: [PATCH 27/74] Bug 1429669 - Separate mercurial conf from install-mercurial.sh. r=dustin The install-mercurial.sh script currently installs a global mercurial configuration after installing mercurial manually. In order to share that configuration with docker images installing a mercurial package through packages tasks, we move it to a separate file. We however keep the part setting web.cacerts in install-mercurial.sh, since it uses a path depending on what kind of environment the script is run. Moreover, the instructions that come with mercurial to build Debian packages come with web.cacerts set to the right path already, so it's not needed in that case anyways. At the same time, use multiple files in /etc/mercurial/hgrc.d/ instead of a single /etc/mercurial/hgrc file. --HG-- extra : rebase_source : 8140d8243cf012489025afe058f467c72224c891 --- taskcluster/docker/image-builder/Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/taskcluster/docker/image-builder/Dockerfile b/taskcluster/docker/image-builder/Dockerfile index 794836feb..be081d4d4 100644 --- a/taskcluster/docker/image-builder/Dockerfile +++ b/taskcluster/docker/image-builder/Dockerfile @@ -12,6 +12,9 @@ ADD topsrcdir/taskcluster/docker/recipes/install-mercurial.sh /setup/install-mer # %include testing/mozharness/external_tools/robustcheckout.py ADD topsrcdir/testing/mozharness/external_tools/robustcheckout.py /usr/local/mercurial/robustcheckout.py +# %include taskcluster/docker/recipes/hgrc +COPY topsrcdir/taskcluster/docker/recipes/hgrc /etc/mercurial/hgrc.d/mozilla.rc + # %include taskcluster/docker/recipes/run-task ADD topsrcdir/taskcluster/docker/recipes/run-task /usr/local/bin/run-task From d3b8f272283bc55d260fc44fc2bd5e1568ac229a Mon Sep 17 00:00:00 2001 From: Mike Hommey Date: Fri, 12 Jan 2018 10:45:35 +0900 Subject: [PATCH 28/74] Partially backout changeset 22587e6ef7ca from bug 1429669 on a CLOSED TREE. r=me Because Ubuntu 16.04 changed in a way that busts gl3 tests and we can't update the desktop1604-test image anymore. --HG-- extra : amend_source : bfa07f9f77990dd6915b8c92d218227436bc6fc4 --- taskcluster/docker/image-builder/Dockerfile | 3 --- 1 file changed, 3 deletions(-) diff --git a/taskcluster/docker/image-builder/Dockerfile b/taskcluster/docker/image-builder/Dockerfile index be081d4d4..794836feb 100644 --- a/taskcluster/docker/image-builder/Dockerfile +++ b/taskcluster/docker/image-builder/Dockerfile @@ -12,9 +12,6 @@ ADD topsrcdir/taskcluster/docker/recipes/install-mercurial.sh /setup/install-mer # %include testing/mozharness/external_tools/robustcheckout.py ADD topsrcdir/testing/mozharness/external_tools/robustcheckout.py /usr/local/mercurial/robustcheckout.py -# %include taskcluster/docker/recipes/hgrc -COPY topsrcdir/taskcluster/docker/recipes/hgrc /etc/mercurial/hgrc.d/mozilla.rc - # %include taskcluster/docker/recipes/run-task ADD topsrcdir/taskcluster/docker/recipes/run-task /usr/local/bin/run-task From 77f61a26dc9edd258697aa8d9cb2b045e7baa4b9 Mon Sep 17 00:00:00 2001 From: Mike Hommey Date: Thu, 11 Jan 2018 17:50:00 +0900 Subject: [PATCH 29/74] Bug 1430037 - Use the in-tree image builder image to build docker images. r=dustin The image builder image we use to build docker images is updated manually, and not necessarily when changes occur in tree that should be reflected by a new image builder image. For instance, its run-task is currently outdated. Not enough that it's actually a problem, but it could rapidly become a problem. There is also a lot of friction when trying to make changes in how docker images are built, and while last time I tried, I ended up not being able to do the changes I wanted to make because the docker version on the host is too old, but this is already the second time I've been trying to make things better and hit a wall because the the image builder is essentially fixed in stone on the docker hub. So with this change, we make all the docker images use the in-tree image builder image, except itself, obviously. That one uses the last version that was uploaded. We may want to update it at some point, but not doing so will only impact building the image builder image itself, not the other ones. --HG-- extra : rebase_source : 73e8fc51ea53af1e647fc1d5093c67d614dd009e --- taskcluster/docker/image-builder/HASH | 1 - taskcluster/docker/image-builder/VERSION | 1 - 2 files changed, 2 deletions(-) delete mode 100644 taskcluster/docker/image-builder/HASH delete mode 100644 taskcluster/docker/image-builder/VERSION diff --git a/taskcluster/docker/image-builder/HASH b/taskcluster/docker/image-builder/HASH deleted file mode 100644 index b82db6571..000000000 --- a/taskcluster/docker/image-builder/HASH +++ /dev/null @@ -1 +0,0 @@ -sha256:24ce54a1602453bc93515aecd9d4ad25a22115fbc4b209ddb5541377e9a37315 diff --git a/taskcluster/docker/image-builder/VERSION b/taskcluster/docker/image-builder/VERSION deleted file mode 100644 index 227cea215..000000000 --- a/taskcluster/docker/image-builder/VERSION +++ /dev/null @@ -1 +0,0 @@ -2.0.0 From 1f764a01a05e1b0e56aa66dd3d8e56dedbc31cce Mon Sep 17 00:00:00 2001 From: Mike Hommey Date: Thu, 11 Jan 2018 17:03:09 +0900 Subject: [PATCH 30/74] Bug 1430037 - Use sparse checkouts for docker image builds. r=dustin In many cases, building docker images starts on machines that don't have a cached checkout, and it often takes forever to get a full clone. It used to be worsened when 3 jobs could run at the same time because the worker would start up clean, and 3 jobs would be doing a mercurial clone at the same time, thrashing I/O, but that part is fortunately fixed. It is still, however, appreciable not to waste time in the mercurial clone part of image creation. --HG-- extra : rebase_source : bbe8b001849e59bb655bb0e9766a6071ad38a52c --- taskcluster/docker/image-builder/build-image.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/taskcluster/docker/image-builder/build-image.sh b/taskcluster/docker/image-builder/build-image.sh index 58c585b3d..917539b9e 100755 --- a/taskcluster/docker/image-builder/build-image.sh +++ b/taskcluster/docker/image-builder/build-image.sh @@ -24,6 +24,7 @@ CONTEXT_FILE=/builds/worker/workspace/context.tar # Run ./mach taskcluster-build-image with --context-only to build context run-task \ --vcs-checkout "/builds/worker/checkouts/gecko" \ + --sparse-profile build/sparse-profiles/docker-image \ -- \ /builds/worker/checkouts/gecko/mach taskcluster-build-image \ --context-only "$CONTEXT_FILE" \ From b7b8410b53ab34b203bc7b35d3be9253dbe7687d Mon Sep 17 00:00:00 2001 From: shindli Date: Wed, 17 Jan 2018 04:30:00 +0200 Subject: [PATCH 31/74] Backed out 2 changesets (bug 1430037) for bustage on beta r=glandium Backed out changeset 8b80471c1d79 (bug 1430037) Backed out changeset 800640c0dc3f (bug 1430037) --- taskcluster/docker/image-builder/HASH | 1 + taskcluster/docker/image-builder/VERSION | 1 + taskcluster/docker/image-builder/build-image.sh | 1 - 3 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 taskcluster/docker/image-builder/HASH create mode 100644 taskcluster/docker/image-builder/VERSION diff --git a/taskcluster/docker/image-builder/HASH b/taskcluster/docker/image-builder/HASH new file mode 100644 index 000000000..b82db6571 --- /dev/null +++ b/taskcluster/docker/image-builder/HASH @@ -0,0 +1 @@ +sha256:24ce54a1602453bc93515aecd9d4ad25a22115fbc4b209ddb5541377e9a37315 diff --git a/taskcluster/docker/image-builder/VERSION b/taskcluster/docker/image-builder/VERSION new file mode 100644 index 000000000..227cea215 --- /dev/null +++ b/taskcluster/docker/image-builder/VERSION @@ -0,0 +1 @@ +2.0.0 diff --git a/taskcluster/docker/image-builder/build-image.sh b/taskcluster/docker/image-builder/build-image.sh index 917539b9e..58c585b3d 100755 --- a/taskcluster/docker/image-builder/build-image.sh +++ b/taskcluster/docker/image-builder/build-image.sh @@ -24,7 +24,6 @@ CONTEXT_FILE=/builds/worker/workspace/context.tar # Run ./mach taskcluster-build-image with --context-only to build context run-task \ --vcs-checkout "/builds/worker/checkouts/gecko" \ - --sparse-profile build/sparse-profiles/docker-image \ -- \ /builds/worker/checkouts/gecko/mach taskcluster-build-image \ --context-only "$CONTEXT_FILE" \ From 687ca52fafc476f6dd9ac97a6ab163fe3fe0fb8a Mon Sep 17 00:00:00 2001 From: Mike Hommey Date: Thu, 11 Jan 2018 17:50:00 +0900 Subject: [PATCH 32/74] Bug 1430037 - Use the in-tree image builder image to build docker images. r=dustin The image builder image we use to build docker images is updated manually, and not necessarily when changes occur in tree that should be reflected by a new image builder image. For instance, its run-task is currently outdated. Not enough that it's actually a problem, but it could rapidly become a problem. There is also a lot of friction when trying to make changes in how docker images are built, and while last time I tried, I ended up not being able to do the changes I wanted to make because the docker version on the host is too old, but this is already the second time I've been trying to make things better and hit a wall because the the image builder is essentially fixed in stone on the docker hub. So with this change, we make all the docker images use the in-tree image builder image, except itself, obviously. That one uses the last version that was uploaded. We may want to update it at some point, but not doing so will only impact building the image builder image itself, not the other ones. --HG-- extra : rebase_source : 978cf033732cbbbb277d206dec69660175b82afa --- taskcluster/docker/image-builder/HASH | 1 - taskcluster/docker/image-builder/VERSION | 1 - 2 files changed, 2 deletions(-) delete mode 100644 taskcluster/docker/image-builder/HASH delete mode 100644 taskcluster/docker/image-builder/VERSION diff --git a/taskcluster/docker/image-builder/HASH b/taskcluster/docker/image-builder/HASH deleted file mode 100644 index b82db6571..000000000 --- a/taskcluster/docker/image-builder/HASH +++ /dev/null @@ -1 +0,0 @@ -sha256:24ce54a1602453bc93515aecd9d4ad25a22115fbc4b209ddb5541377e9a37315 diff --git a/taskcluster/docker/image-builder/VERSION b/taskcluster/docker/image-builder/VERSION deleted file mode 100644 index 227cea215..000000000 --- a/taskcluster/docker/image-builder/VERSION +++ /dev/null @@ -1 +0,0 @@ -2.0.0 From 4e0c9f036d869707916f482acd3b4c782c82abee Mon Sep 17 00:00:00 2001 From: Mike Hommey Date: Thu, 11 Jan 2018 17:03:09 +0900 Subject: [PATCH 33/74] Bug 1430037 - Use sparse checkouts for docker image builds. r=dustin In many cases, building docker images starts on machines that don't have a cached checkout, and it often takes forever to get a full clone. It used to be worsened when 3 jobs could run at the same time because the worker would start up clean, and 3 jobs would be doing a mercurial clone at the same time, thrashing I/O, but that part is fortunately fixed. It is still, however, appreciable not to waste time in the mercurial clone part of image creation. --HG-- extra : rebase_source : 8c76bc91e1d5102f68c43e1050d61971fef32e9f --- taskcluster/docker/image-builder/build-image.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/taskcluster/docker/image-builder/build-image.sh b/taskcluster/docker/image-builder/build-image.sh index 58c585b3d..917539b9e 100755 --- a/taskcluster/docker/image-builder/build-image.sh +++ b/taskcluster/docker/image-builder/build-image.sh @@ -24,6 +24,7 @@ CONTEXT_FILE=/builds/worker/workspace/context.tar # Run ./mach taskcluster-build-image with --context-only to build context run-task \ --vcs-checkout "/builds/worker/checkouts/gecko" \ + --sparse-profile build/sparse-profiles/docker-image \ -- \ /builds/worker/checkouts/gecko/mach taskcluster-build-image \ --context-only "$CONTEXT_FILE" \ From 8582d0a1a510d3fe909a2f38248b78ed89c77ab5 Mon Sep 17 00:00:00 2001 From: Mike Hommey Date: Thu, 18 Jan 2018 17:04:50 +0900 Subject: [PATCH 34/74] Bug 1429996 - Revert partial backout of changeset 22587e6ef7ca from bug 1429669. r=me --- taskcluster/docker/image-builder/Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/taskcluster/docker/image-builder/Dockerfile b/taskcluster/docker/image-builder/Dockerfile index 794836feb..be081d4d4 100644 --- a/taskcluster/docker/image-builder/Dockerfile +++ b/taskcluster/docker/image-builder/Dockerfile @@ -12,6 +12,9 @@ ADD topsrcdir/taskcluster/docker/recipes/install-mercurial.sh /setup/install-mer # %include testing/mozharness/external_tools/robustcheckout.py ADD topsrcdir/testing/mozharness/external_tools/robustcheckout.py /usr/local/mercurial/robustcheckout.py +# %include taskcluster/docker/recipes/hgrc +COPY topsrcdir/taskcluster/docker/recipes/hgrc /etc/mercurial/hgrc.d/mozilla.rc + # %include taskcluster/docker/recipes/run-task ADD topsrcdir/taskcluster/docker/recipes/run-task /usr/local/bin/run-task From 1cde08cd4a2c5455c005255e7a63db80bc1fee5c Mon Sep 17 00:00:00 2001 From: Simon Fraser Date: Wed, 24 Jan 2018 12:59:11 +0000 Subject: [PATCH 35/74] Bug 1432768 Linting compliance for image builder scripts r=Callek MozReview-Commit-ID: 3TmtiB47Z5W --HG-- extra : rebase_source : 7e21151be1125affdcabf2aa4876aea9a97575fd --- taskcluster/docker/image-builder/build-image.sh | 10 +++++----- taskcluster/docker/image-builder/setup.sh | 5 ++++- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/taskcluster/docker/image-builder/build-image.sh b/taskcluster/docker/image-builder/build-image.sh index 917539b9e..de6383a62 100755 --- a/taskcluster/docker/image-builder/build-image.sh +++ b/taskcluster/docker/image-builder/build-image.sh @@ -46,14 +46,14 @@ curl -s --fail \ | jq -jr '(.status + .progress, .error | select(. != null) + "\n"), .stream | select(. != null)' # Exit non-zero if there is error entries in the log -if cat /tmp/docker-build.log | jq -se 'add | .error' > /dev/null; then - raise_error "Image build failed: `cat /tmp/docker-build.log | jq -rse 'add | .error'`"; +if result=$(jq -se 'add | .error' /tmp/docker-build.log); then + raise_error "Image build failed: ${result}"; fi # Sanity check that image was built successfully -if ! cat /tmp/docker-build.log | tail -n 1 | jq -r '.stream' | grep '^Successfully built' > /dev/null; then +if ! tail -n 1 /tmp/docker-build.log | jq -r '.stream' | grep '^Successfully built' > /dev/null; then echo 'docker-build.log for debugging:'; - cat /tmp/docker-build.log | tail -n 50; + tail -n 50 /tmp/docker-build.log; raise_error "Image build log didn't with 'Successfully built'"; fi @@ -63,6 +63,6 @@ fi # # The script will retry up to 10 times. /usr/local/bin/download-and-compress \ - http+unix://%2Fvar%2Frun%2Fdocker.sock/images/${IMAGE_NAME}:${HASH}/get \ + "http+unix://%2Fvar%2Frun%2Fdocker.sock/images/${IMAGE_NAME}:${HASH}/get "\ /builds/worker/workspace/image.tar.zst.tmp \ /builds/worker/workspace/artifacts/image.tar.zst diff --git a/taskcluster/docker/image-builder/setup.sh b/taskcluster/docker/image-builder/setup.sh index 9459a0f25..70a6c5ce2 100644 --- a/taskcluster/docker/image-builder/setup.sh +++ b/taskcluster/docker/image-builder/setup.sh @@ -20,7 +20,9 @@ apt-get install -y --no-install-recommends \ apt-get install -y python-pip # Install mercurial +# shellcheck disable=SC1091 . /setup/common.sh +# shellcheck disable=SC1091 . /setup/install-mercurial.sh # Install build-image.sh script @@ -32,6 +34,7 @@ chmod +x /usr/local/bin/download-and-compress mkdir -p /builds/worker/workspace # Install python-zstandard. +( cd /setup tooltool_fetch < Date: Wed, 24 Jan 2018 16:56:10 +0200 Subject: [PATCH 36/74] Backed out changeset d0dc56c1e423 (bug 1432768) for bustage taskcluster-image-build:error on a CLOSED TREE --- taskcluster/docker/image-builder/build-image.sh | 10 +++++----- taskcluster/docker/image-builder/setup.sh | 5 +---- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/taskcluster/docker/image-builder/build-image.sh b/taskcluster/docker/image-builder/build-image.sh index de6383a62..917539b9e 100755 --- a/taskcluster/docker/image-builder/build-image.sh +++ b/taskcluster/docker/image-builder/build-image.sh @@ -46,14 +46,14 @@ curl -s --fail \ | jq -jr '(.status + .progress, .error | select(. != null) + "\n"), .stream | select(. != null)' # Exit non-zero if there is error entries in the log -if result=$(jq -se 'add | .error' /tmp/docker-build.log); then - raise_error "Image build failed: ${result}"; +if cat /tmp/docker-build.log | jq -se 'add | .error' > /dev/null; then + raise_error "Image build failed: `cat /tmp/docker-build.log | jq -rse 'add | .error'`"; fi # Sanity check that image was built successfully -if ! tail -n 1 /tmp/docker-build.log | jq -r '.stream' | grep '^Successfully built' > /dev/null; then +if ! cat /tmp/docker-build.log | tail -n 1 | jq -r '.stream' | grep '^Successfully built' > /dev/null; then echo 'docker-build.log for debugging:'; - tail -n 50 /tmp/docker-build.log; + cat /tmp/docker-build.log | tail -n 50; raise_error "Image build log didn't with 'Successfully built'"; fi @@ -63,6 +63,6 @@ fi # # The script will retry up to 10 times. /usr/local/bin/download-and-compress \ - "http+unix://%2Fvar%2Frun%2Fdocker.sock/images/${IMAGE_NAME}:${HASH}/get "\ + http+unix://%2Fvar%2Frun%2Fdocker.sock/images/${IMAGE_NAME}:${HASH}/get \ /builds/worker/workspace/image.tar.zst.tmp \ /builds/worker/workspace/artifacts/image.tar.zst diff --git a/taskcluster/docker/image-builder/setup.sh b/taskcluster/docker/image-builder/setup.sh index 70a6c5ce2..9459a0f25 100644 --- a/taskcluster/docker/image-builder/setup.sh +++ b/taskcluster/docker/image-builder/setup.sh @@ -20,9 +20,7 @@ apt-get install -y --no-install-recommends \ apt-get install -y python-pip # Install mercurial -# shellcheck disable=SC1091 . /setup/common.sh -# shellcheck disable=SC1091 . /setup/install-mercurial.sh # Install build-image.sh script @@ -34,7 +32,6 @@ chmod +x /usr/local/bin/download-and-compress mkdir -p /builds/worker/workspace # Install python-zstandard. -( cd /setup tooltool_fetch < Date: Wed, 24 Jan 2018 12:59:11 +0000 Subject: [PATCH 37/74] Bug 1432768 Linting compliance for image builder scripts r=Callek MozReview-Commit-ID: 3TmtiB47Z5W --HG-- extra : rebase_source : c66da9713e1732be6f5a6ba26d6b2a18ca17fa34 --- taskcluster/docker/image-builder/build-image.sh | 10 ++++++---- taskcluster/docker/image-builder/setup.sh | 5 ++++- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/taskcluster/docker/image-builder/build-image.sh b/taskcluster/docker/image-builder/build-image.sh index 917539b9e..c73bcc294 100755 --- a/taskcluster/docker/image-builder/build-image.sh +++ b/taskcluster/docker/image-builder/build-image.sh @@ -46,14 +46,14 @@ curl -s --fail \ | jq -jr '(.status + .progress, .error | select(. != null) + "\n"), .stream | select(. != null)' # Exit non-zero if there is error entries in the log -if cat /tmp/docker-build.log | jq -se 'add | .error' > /dev/null; then - raise_error "Image build failed: `cat /tmp/docker-build.log | jq -rse 'add | .error'`"; +if result=$(jq -se 'add | .error' /tmp/docker-build.log); then + raise_error "Image build failed: ${result}"; fi # Sanity check that image was built successfully -if ! cat /tmp/docker-build.log | tail -n 1 | jq -r '.stream' | grep '^Successfully built' > /dev/null; then +if ! tail -n 1 /tmp/docker-build.log | jq -r '.stream' | grep '^Successfully built' > /dev/null; then echo 'docker-build.log for debugging:'; - cat /tmp/docker-build.log | tail -n 50; + tail -n 50 /tmp/docker-build.log; raise_error "Image build log didn't with 'Successfully built'"; fi @@ -62,6 +62,8 @@ fi # https://docs.docker.com/engine/reference/api/docker_remote_api_v1.18/ # # The script will retry up to 10 times. +# Disable quoting error until fixing the / escaping +# shellcheck disable=SC2086 /usr/local/bin/download-and-compress \ http+unix://%2Fvar%2Frun%2Fdocker.sock/images/${IMAGE_NAME}:${HASH}/get \ /builds/worker/workspace/image.tar.zst.tmp \ diff --git a/taskcluster/docker/image-builder/setup.sh b/taskcluster/docker/image-builder/setup.sh index 9459a0f25..70a6c5ce2 100644 --- a/taskcluster/docker/image-builder/setup.sh +++ b/taskcluster/docker/image-builder/setup.sh @@ -20,7 +20,9 @@ apt-get install -y --no-install-recommends \ apt-get install -y python-pip # Install mercurial +# shellcheck disable=SC1091 . /setup/common.sh +# shellcheck disable=SC1091 . /setup/install-mercurial.sh # Install build-image.sh script @@ -32,6 +34,7 @@ chmod +x /usr/local/bin/download-and-compress mkdir -p /builds/worker/workspace # Install python-zstandard. +( cd /setup tooltool_fetch < Date: Thu, 25 Jan 2018 13:36:47 +0900 Subject: [PATCH 38/74] Bug 1432390 - Make `mach taskcluster-build-image` talk directly to the docker socket in the image builder. r=dustin Now that `mach taskcluster-build-image` can, we can avoid all the manual handling based on curl and jq in the image builder. An additional advantage on relying on `mach taskcluster-build-image` doing more is that less changes to the build-image.sh script will be necessary, and thus less updates of the image builder docker image. --HG-- extra : rebase_source : dd174d60675e41e4391894f28235c674c1840829 --- .../docker/image-builder/build-image.sh | 38 +++++-------------- taskcluster/docker/image-builder/setup.sh | 4 +- 2 files changed, 10 insertions(+), 32 deletions(-) diff --git a/taskcluster/docker/image-builder/build-image.sh b/taskcluster/docker/image-builder/build-image.sh index c73bcc294..05845f3ea 100755 --- a/taskcluster/docker/image-builder/build-image.sh +++ b/taskcluster/docker/image-builder/build-image.sh @@ -18,45 +18,25 @@ test -n "$PROJECT" || raise_error "PROJECT must be provided." test -n "$HASH" || raise_error "Context HASH must be provided." test -n "$IMAGE_NAME" || raise_error "IMAGE_NAME must be provided." -# Construct a CONTEXT_FILE -CONTEXT_FILE=/builds/worker/workspace/context.tar - -# Run ./mach taskcluster-build-image with --context-only to build context +# The docker socket is mounted by the taskcluster worker in a way that prevents +# us changing its permissions to allow the worker user to access it. Create a +# proxy socket that the worker user can use. +export DOCKER_SOCKET=/var/run/docker.proxy +socat UNIX-LISTEN:$DOCKER_SOCKET,fork,group=worker,mode=0775 UNIX-CLIENT:/var/run/docker.sock /dev/null; then - echo 'docker-build.log for debugging:'; - tail -n 50 /tmp/docker-build.log; - raise_error "Image build log didn't with 'Successfully built'"; -fi - # Get image from docker daemon (try up to 10 times) # This interacts directly with the docker remote API, see: # https://docs.docker.com/engine/reference/api/docker_remote_api_v1.18/ diff --git a/taskcluster/docker/image-builder/setup.sh b/taskcluster/docker/image-builder/setup.sh index 70a6c5ce2..bbf05149e 100644 --- a/taskcluster/docker/image-builder/setup.sh +++ b/taskcluster/docker/image-builder/setup.sh @@ -8,9 +8,7 @@ apt-get update -y # Install dependencies apt-get install -y --no-install-recommends \ - curl \ - tar \ - jq \ + socat \ python \ python-requests \ python-requests-unixsocket From 6528026c903a03fb7d3099a6ebd9727881a7a46c Mon Sep 17 00:00:00 2001 From: Mike Hommey Date: Thu, 25 Jan 2018 11:07:54 +0900 Subject: [PATCH 39/74] Bug 1432390 - Allow to derive docker images from other in-tree images. r=dustin So far, the best we've been able to do is to upload an image to the docker hub, and point an image's Dockerfile's FROM to the version uploaded onto the hub. That is a cumbersome process, and makes the use of "layered" docker images painful. This change allows to declare a parent docker image in the taskcluster/ci/docker-image/kind.yml definitions, which will be automatically loaded before building the image. The Dockerfile can then reference the image, using the DOCKER_IMAGE_PARENT argument, which will contain the full image name:tag. Some details are left off, for now, such as VOLUMEs. At this point, VOLUMEs should all be defined in leaf docker images. --HG-- extra : rebase_source : 221cff0ca5a91d694ff5c3626fe707c15ba45e23 --- taskcluster/docker/image-builder/build-image.sh | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/taskcluster/docker/image-builder/build-image.sh b/taskcluster/docker/image-builder/build-image.sh index 05845f3ea..d99a414ba 100755 --- a/taskcluster/docker/image-builder/build-image.sh +++ b/taskcluster/docker/image-builder/build-image.sh @@ -25,14 +25,24 @@ export DOCKER_SOCKET=/var/run/docker.proxy socat UNIX-LISTEN:$DOCKER_SOCKET,fork,group=worker,mode=0775 UNIX-CLIENT:/var/run/docker.sock Date: Mon, 29 Jan 2018 15:27:04 +0000 Subject: [PATCH 40/74] Bug 1432627 Add shellcheck linter to CI r=ahal MozReview-Commit-ID: 2IYUsXunfab --HG-- extra : rebase_source : a40376ffc66e4a35d30e7e02a6ec208b1f52b021 --- taskcluster/docker/image-builder/build-image.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/taskcluster/docker/image-builder/build-image.sh b/taskcluster/docker/image-builder/build-image.sh index d99a414ba..7da8da97c 100755 --- a/taskcluster/docker/image-builder/build-image.sh +++ b/taskcluster/docker/image-builder/build-image.sh @@ -23,6 +23,8 @@ test -n "$IMAGE_NAME" || raise_error "IMAGE_NAME must be provided." # proxy socket that the worker user can use. export DOCKER_SOCKET=/var/run/docker.proxy socat UNIX-LISTEN:$DOCKER_SOCKET,fork,group=worker,mode=0775 UNIX-CLIENT:/var/run/docker.sock Date: Mon, 29 Jan 2018 19:09:55 +0200 Subject: [PATCH 41/74] Backed out changeset 85794e045326 (bug 1432627) for shell lint failure in /builds/worker/checkouts/gecko/taskcluster/docker/firefox-snap/runme.sh:50:95 on a CLOSED TREE --- taskcluster/docker/image-builder/build-image.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/taskcluster/docker/image-builder/build-image.sh b/taskcluster/docker/image-builder/build-image.sh index 7da8da97c..d99a414ba 100755 --- a/taskcluster/docker/image-builder/build-image.sh +++ b/taskcluster/docker/image-builder/build-image.sh @@ -23,8 +23,6 @@ test -n "$IMAGE_NAME" || raise_error "IMAGE_NAME must be provided." # proxy socket that the worker user can use. export DOCKER_SOCKET=/var/run/docker.proxy socat UNIX-LISTEN:$DOCKER_SOCKET,fork,group=worker,mode=0775 UNIX-CLIENT:/var/run/docker.sock Date: Mon, 29 Jan 2018 15:27:04 +0000 Subject: [PATCH 42/74] Bug 1432627 Add shellcheck linter to CI r=ahal MozReview-Commit-ID: 2IYUsXunfab --HG-- extra : rebase_source : 806ab4abfb1838a1b5eec279f5e93ce5c703685d --- taskcluster/docker/image-builder/build-image.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/taskcluster/docker/image-builder/build-image.sh b/taskcluster/docker/image-builder/build-image.sh index d99a414ba..7da8da97c 100755 --- a/taskcluster/docker/image-builder/build-image.sh +++ b/taskcluster/docker/image-builder/build-image.sh @@ -23,6 +23,8 @@ test -n "$IMAGE_NAME" || raise_error "IMAGE_NAME must be provided." # proxy socket that the worker user can use. export DOCKER_SOCKET=/var/run/docker.proxy socat UNIX-LISTEN:$DOCKER_SOCKET,fork,group=worker,mode=0775 UNIX-CLIENT:/var/run/docker.sock Date: Fri, 4 May 2018 17:23:31 -0700 Subject: [PATCH 43/74] Bug 1459737 - Move run-task into taskcluster/scripts; r=dustin In preparation for making it usable on Windows, after which point having it in a directory with "docker" in it doesn't make much sense. MozReview-Commit-ID: Hgu0buFyJwF --HG-- rename : taskcluster/docker/recipes/run-task => taskcluster/scripts/run-task extra : rebase_source : 3c0b502d28b5aad54bd04069efbfda88e25bbb20 --- taskcluster/docker/image-builder/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/taskcluster/docker/image-builder/Dockerfile b/taskcluster/docker/image-builder/Dockerfile index be081d4d4..36549f9ed 100644 --- a/taskcluster/docker/image-builder/Dockerfile +++ b/taskcluster/docker/image-builder/Dockerfile @@ -15,8 +15,8 @@ ADD topsrcdir/testing/mozharness/external_tools/robustcheckout.py /usr/local/mer # %include taskcluster/docker/recipes/hgrc COPY topsrcdir/taskcluster/docker/recipes/hgrc /etc/mercurial/hgrc.d/mozilla.rc -# %include taskcluster/docker/recipes/run-task -ADD topsrcdir/taskcluster/docker/recipes/run-task /usr/local/bin/run-task +# %include taskcluster/scripts/run-task +ADD topsrcdir/taskcluster/scripts/run-task /usr/local/bin/run-task # Add and run setup script ADD build-image.sh /usr/local/bin/build-image.sh From 6ce0e805acec3c63db53e6ef92cd65bbe8b7239b Mon Sep 17 00:00:00 2001 From: Gregory Szorc Date: Wed, 9 May 2018 17:45:39 -0700 Subject: [PATCH 44/74] Bug 1460475 - Install Python 3 on image_builder; r=dustin We want Python 3 available everywhere because it is 2018. MozReview-Commit-ID: L3wufNXKdnp --HG-- extra : rebase_source : c260923e3c13f8b28e30eaaf6e1bd38f79500052 --- taskcluster/docker/image-builder/setup.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/taskcluster/docker/image-builder/setup.sh b/taskcluster/docker/image-builder/setup.sh index bbf05149e..8038c1c9f 100644 --- a/taskcluster/docker/image-builder/setup.sh +++ b/taskcluster/docker/image-builder/setup.sh @@ -9,6 +9,8 @@ apt-get update -y # Install dependencies apt-get install -y --no-install-recommends \ socat \ + python3.5 \ + python3-minimal \ python \ python-requests \ python-requests-unixsocket From 2071da466ed31b9ce21746e307cc848ec870406e Mon Sep 17 00:00:00 2001 From: Gregory Szorc Date: Wed, 9 May 2018 17:54:38 -0700 Subject: [PATCH 45/74] Bug 1460475 - Upgrade python-zstandard in image_builder; r=dustin Version 0.9.0 bundles a newer version of the zstandard library, which is a little faster and has a few minor bug fixes (none that we were likely hitting, however). MozReview-Commit-ID: 9YgSZ0G41eg --HG-- extra : rebase_source : 8f5a68323b1e1fe7e9f1dd1a92e132434972d21d --- taskcluster/docker/image-builder/download-and-compress | 2 +- taskcluster/docker/image-builder/setup.sh | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/taskcluster/docker/image-builder/download-and-compress b/taskcluster/docker/image-builder/download-and-compress index 2dd737855..a9da58c54 100755 --- a/taskcluster/docker/image-builder/download-and-compress +++ b/taskcluster/docker/image-builder/download-and-compress @@ -9,7 +9,7 @@ import time import requests import requests_unixsocket -import zstd +import zstandard as zstd # Allow requests to fetch from UNIX domain sockets. requests_unixsocket.monkeypatch() diff --git a/taskcluster/docker/image-builder/setup.sh b/taskcluster/docker/image-builder/setup.sh index 8038c1c9f..22bb10117 100644 --- a/taskcluster/docker/image-builder/setup.sh +++ b/taskcluster/docker/image-builder/setup.sh @@ -39,17 +39,17 @@ cd /setup tooltool_fetch < Date: Wed, 9 May 2018 19:41:07 -0700 Subject: [PATCH 46/74] Bug 1460475 - Port download-and-compress to Python 3; r=dustin download-and-compress isn't very complicated and should work on Python 3 with minimal effort. So let's switch it to use Python 3. MozReview-Commit-ID: 9G1WfcbbKEY --HG-- extra : rebase_source : 3a6bab06c8500a90413e8b7642a7bf7bdff04a46 --- .../docker/image-builder/download-and-compress | 2 +- taskcluster/docker/image-builder/setup.sh | 14 +++++++++----- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/taskcluster/docker/image-builder/download-and-compress b/taskcluster/docker/image-builder/download-and-compress index a9da58c54..4cf7bedb9 100755 --- a/taskcluster/docker/image-builder/download-and-compress +++ b/taskcluster/docker/image-builder/download-and-compress @@ -1,4 +1,4 @@ -#!/usr/bin/python2.7 +#!/usr/bin/python3 -u # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. diff --git a/taskcluster/docker/image-builder/setup.sh b/taskcluster/docker/image-builder/setup.sh index 22bb10117..041f1b2fd 100644 --- a/taskcluster/docker/image-builder/setup.sh +++ b/taskcluster/docker/image-builder/setup.sh @@ -9,15 +9,16 @@ apt-get update -y # Install dependencies apt-get install -y --no-install-recommends \ socat \ + python-requests \ + python-requests-unixsocket \ python3.5 \ python3-minimal \ - python \ - python-requests \ - python-requests-unixsocket + python3-requests \ + python3-requests-unixsocket # Extra dependencies only needed for image building. Will be removed at # end of script. -apt-get install -y python-pip +apt-get install -y python-pip python3-pip # Install mercurial # shellcheck disable=SC1091 @@ -49,11 +50,14 @@ tooltool_fetch < Date: Wed, 31 Oct 2018 23:02:42 +0000 Subject: [PATCH 47/74] Bug 1498640 - deploy latest image_builder image r=glandium This uses the latest image_builder image (on docker hub) to build even the image_builder image. The change to `docker.py` handles a new API response (`aux`) from the Docker daemon. It's unclear what this key means, but displaying it is simple. Differential Revision: https://phabricator.services.mozilla.com/D8441 --HG-- extra : moz-landing-system : lando --- taskcluster/docker/image-builder/VERSION | 1 + 1 file changed, 1 insertion(+) create mode 100644 taskcluster/docker/image-builder/VERSION diff --git a/taskcluster/docker/image-builder/VERSION b/taskcluster/docker/image-builder/VERSION new file mode 100644 index 000000000..4a36342fc --- /dev/null +++ b/taskcluster/docker/image-builder/VERSION @@ -0,0 +1 @@ +3.0.0 From d1f5774f8e75b4ac62a6dd93ad24e98bbf8c2187 Mon Sep 17 00:00:00 2001 From: shindli Date: Thu, 1 Nov 2018 06:20:05 +0200 Subject: [PATCH 48/74] Backed out changeset f33b9e6aca06 (bug 1498640) for mda permafailures with "Error in test execution: NotFoundError: The object can not be found here." message --- taskcluster/docker/image-builder/VERSION | 1 - 1 file changed, 1 deletion(-) delete mode 100644 taskcluster/docker/image-builder/VERSION diff --git a/taskcluster/docker/image-builder/VERSION b/taskcluster/docker/image-builder/VERSION deleted file mode 100644 index 4a36342fc..000000000 --- a/taskcluster/docker/image-builder/VERSION +++ /dev/null @@ -1 +0,0 @@ -3.0.0 From 02889a8b917ee67f3485d17c7bf596a3b816f042 Mon Sep 17 00:00:00 2001 From: "Dustin J. Mitchell" Date: Wed, 31 Oct 2018 23:02:42 +0000 Subject: [PATCH 49/74] Bug 1498640 - deploy latest image_builder image r=glandium This uses the latest image_builder image (on docker hub) to build even the image_builder image. The change to `docker.py` handles a new API response (`aux`) from the Docker daemon. It's unclear what this key means, but displaying it is simple. Differential Revision: https://phabricator.services.mozilla.com/D8441 --HG-- extra : rebase_source : 2c069da57e416d5e1821e55653d37b23d633ae78 extra : source : f33b9e6aca06b7883f048c6c068704680cbfbbb2 --- taskcluster/docker/image-builder/VERSION | 1 + 1 file changed, 1 insertion(+) create mode 100644 taskcluster/docker/image-builder/VERSION diff --git a/taskcluster/docker/image-builder/VERSION b/taskcluster/docker/image-builder/VERSION new file mode 100644 index 000000000..4a36342fc --- /dev/null +++ b/taskcluster/docker/image-builder/VERSION @@ -0,0 +1 @@ +3.0.0 From 90ec358b6dcfddc49cc9ec58fe7d4a9f83082064 Mon Sep 17 00:00:00 2001 From: Andreea Pavel Date: Thu, 15 Nov 2018 07:53:41 +0200 Subject: [PATCH 50/74] Backed out 5 changesets (bug 1498640, bug 1436037, bug 1497660, bug 1505579) for mass failures on a CLOSED TREE Backed out changeset 914a7a899dd1 (bug 1436037) Backed out changeset e072757bf691 (bug 1436037) Backed out changeset b0805a8cf6eb (bug 1505579) Backed out changeset aa32ba9e6df3 (bug 1498640) Backed out changeset ce441b8a784c (bug 1497660) --- taskcluster/docker/image-builder/VERSION | 1 - 1 file changed, 1 deletion(-) delete mode 100644 taskcluster/docker/image-builder/VERSION diff --git a/taskcluster/docker/image-builder/VERSION b/taskcluster/docker/image-builder/VERSION deleted file mode 100644 index 4a36342fc..000000000 --- a/taskcluster/docker/image-builder/VERSION +++ /dev/null @@ -1 +0,0 @@ -3.0.0 From 3e3adf169e57d7508e500d46fac14a7c3e6b8129 Mon Sep 17 00:00:00 2001 From: "Dustin J. Mitchell" Date: Wed, 31 Oct 2018 23:02:42 +0000 Subject: [PATCH 51/74] Bug 1498640 - deploy latest image_builder image r=glandium This uses the latest image_builder image (on docker hub) to build even the image_builder image. The change to `docker.py` handles a new API response (`aux`) from the Docker daemon. It's unclear what this key means, but displaying it is simple. Differential Revision: https://phabricator.services.mozilla.com/D8441 --HG-- extra : rebase_source : b6a2c2de231bd623521a0a7a0dc595fed059b758 extra : intermediate-source : aa32ba9e6df3e20aab896a63538c8e8a432b4988 extra : source : f33b9e6aca06b7883f048c6c068704680cbfbbb2 --- taskcluster/docker/image-builder/VERSION | 1 + 1 file changed, 1 insertion(+) create mode 100644 taskcluster/docker/image-builder/VERSION diff --git a/taskcluster/docker/image-builder/VERSION b/taskcluster/docker/image-builder/VERSION new file mode 100644 index 000000000..4a36342fc --- /dev/null +++ b/taskcluster/docker/image-builder/VERSION @@ -0,0 +1 @@ +3.0.0 From 264f49931d11e796ef964d5f3ab15ceb7dda0cd3 Mon Sep 17 00:00:00 2001 From: Gregory Szorc Date: Tue, 11 Dec 2018 19:50:43 +0000 Subject: [PATCH 52/74] Bug 1512188 - Rename --vcs-checkout to --firefox-checkout; r=tomprince We now have multiple things we may check out. "vcs" meaning "firefox" is not obvious. Let's change the terminology to be more specific. Differential Revision: https://phabricator.services.mozilla.com/D13813 --HG-- extra : moz-landing-system : lando --- taskcluster/docker/image-builder/build-image.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/taskcluster/docker/image-builder/build-image.sh b/taskcluster/docker/image-builder/build-image.sh index 7da8da97c..049005589 100755 --- a/taskcluster/docker/image-builder/build-image.sh +++ b/taskcluster/docker/image-builder/build-image.sh @@ -38,7 +38,7 @@ fi # Build image run-task \ - --vcs-checkout "/builds/worker/checkouts/gecko" \ + --firefox-checkout "/builds/worker/checkouts/gecko" \ --sparse-profile build/sparse-profiles/docker-image \ -- \ sh -x -c "$LOAD_COMMAND \ From 3fb2433d173c2924ba071b3e1621a2a7b32a1af3 Mon Sep 17 00:00:00 2001 From: Gregory Szorc Date: Tue, 11 Dec 2018 19:50:45 +0000 Subject: [PATCH 53/74] Bug 1512188 - Rename --sparse-profile to --firefox-sparse-profile; r=tomprince We have multiple source checkouts. --sparse-profile is ambiguous as to which one it could refer to. Let's rename the argument so it is prefixed with the repo/project we are checking out. Differential Revision: https://phabricator.services.mozilla.com/D13814 --HG-- extra : moz-landing-system : lando --- taskcluster/docker/image-builder/build-image.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/taskcluster/docker/image-builder/build-image.sh b/taskcluster/docker/image-builder/build-image.sh index 049005589..2d9912e38 100755 --- a/taskcluster/docker/image-builder/build-image.sh +++ b/taskcluster/docker/image-builder/build-image.sh @@ -39,7 +39,7 @@ fi # Build image run-task \ --firefox-checkout "/builds/worker/checkouts/gecko" \ - --sparse-profile build/sparse-profiles/docker-image \ + --firefox-sparse-profile build/sparse-profiles/docker-image \ -- \ sh -x -c "$LOAD_COMMAND \ /builds/worker/checkouts/gecko/mach taskcluster-build-image \ From 999227f734faf75ed977bc87fee19751d0cff596 Mon Sep 17 00:00:00 2001 From: Gregory Szorc Date: Tue, 11 Dec 2018 19:50:56 +0000 Subject: [PATCH 54/74] Bug 1512188 - Revert to "gecko" for vcs naming; r=tomprince For historical consistency and consistency with index paths. "env_prefix" is no longer used after this change, so it has been removed. Differential Revision: https://phabricator.services.mozilla.com/D13876 --HG-- extra : moz-landing-system : lando --- taskcluster/docker/image-builder/build-image.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/taskcluster/docker/image-builder/build-image.sh b/taskcluster/docker/image-builder/build-image.sh index 2d9912e38..cd00b8bb3 100755 --- a/taskcluster/docker/image-builder/build-image.sh +++ b/taskcluster/docker/image-builder/build-image.sh @@ -38,8 +38,8 @@ fi # Build image run-task \ - --firefox-checkout "/builds/worker/checkouts/gecko" \ - --firefox-sparse-profile build/sparse-profiles/docker-image \ + --gecko-checkout "/builds/worker/checkouts/gecko" \ + --gecko-sparse-profile build/sparse-profiles/docker-image \ -- \ sh -x -c "$LOAD_COMMAND \ /builds/worker/checkouts/gecko/mach taskcluster-build-image \ From da9bae328a318dac77634c26675d670463061373 Mon Sep 17 00:00:00 2001 From: Dorel Luca Date: Tue, 11 Dec 2018 23:08:27 +0200 Subject: [PATCH 55/74] Backed out 12 changesets (bug 1512285, bug 1512188) for toolchain failure. CLOSED TREE Backed out changeset 59813ae1b6ea (bug 1512188) Backed out changeset 1f56f1e581e2 (bug 1512188) Backed out changeset 0b3259dc10bd (bug 1512188) Backed out changeset d0eb311b3c8f (bug 1512188) Backed out changeset d9585e5d7a3a (bug 1512188) Backed out changeset b4f3dc9b0956 (bug 1512188) Backed out changeset 7d88308ca069 (bug 1512285) Backed out changeset 838f49d718a7 (bug 1512188) Backed out changeset 6bd036a5166d (bug 1512188) Backed out changeset 3bcedff402fa (bug 1512188) Backed out changeset 4f546b3e0b18 (bug 1512188) Backed out changeset bc4232cfcfb8 (bug 1512188) --- taskcluster/docker/image-builder/build-image.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/taskcluster/docker/image-builder/build-image.sh b/taskcluster/docker/image-builder/build-image.sh index cd00b8bb3..7da8da97c 100755 --- a/taskcluster/docker/image-builder/build-image.sh +++ b/taskcluster/docker/image-builder/build-image.sh @@ -38,8 +38,8 @@ fi # Build image run-task \ - --gecko-checkout "/builds/worker/checkouts/gecko" \ - --gecko-sparse-profile build/sparse-profiles/docker-image \ + --vcs-checkout "/builds/worker/checkouts/gecko" \ + --sparse-profile build/sparse-profiles/docker-image \ -- \ sh -x -c "$LOAD_COMMAND \ /builds/worker/checkouts/gecko/mach taskcluster-build-image \ From 62b3f4b69728a8c56975f156e20aa00a9225442c Mon Sep 17 00:00:00 2001 From: Gregory Szorc Date: Wed, 19 Dec 2018 03:12:19 +0000 Subject: [PATCH 56/74] Bug 1513429 - Upgrade image_builder to Ubuntu 18.04; r=dustin This appears to "just work." While I would like to convert this image to Debian and make it deterministic, that is more effect than I'm willing to invest at the moment. The impetus for this change is unblocking partial clones. Mercurial's SQLite storage backend apparently hits a SQLite bug in version 3.11 of SQLite (what Ubuntu 16.04 runs) where SQLite complains about database corruption when there are readers from multiple processes. Ubuntu 18.04 is running SQLite 3.22 and doesn't exhibit the buggy behavior. Differential Revision: https://phabricator.services.mozilla.com/D14228 --HG-- extra : moz-landing-system : lando --- taskcluster/docker/image-builder/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/taskcluster/docker/image-builder/Dockerfile b/taskcluster/docker/image-builder/Dockerfile index 36549f9ed..93470373e 100644 --- a/taskcluster/docker/image-builder/Dockerfile +++ b/taskcluster/docker/image-builder/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:16.04 +FROM ubuntu:18.04 # %include python/mozbuild/mozbuild/action/tooltool.py ADD topsrcdir/python/mozbuild/mozbuild/action/tooltool.py /setup/tooltool.py From 5e2d74bde0e9dedfeed63980656a5aa31c07b682 Mon Sep 17 00:00:00 2001 From: Bogdan Tara Date: Wed, 19 Dec 2018 06:06:30 +0200 Subject: [PATCH 57/74] Backed out changeset e52ecfe8a7b1 (bug 1513429) for toolchain failure CLOSED TREE --- taskcluster/docker/image-builder/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/taskcluster/docker/image-builder/Dockerfile b/taskcluster/docker/image-builder/Dockerfile index 93470373e..36549f9ed 100644 --- a/taskcluster/docker/image-builder/Dockerfile +++ b/taskcluster/docker/image-builder/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:18.04 +FROM ubuntu:16.04 # %include python/mozbuild/mozbuild/action/tooltool.py ADD topsrcdir/python/mozbuild/mozbuild/action/tooltool.py /setup/tooltool.py From a6e56ee89adcec973bbbba7030b48e4102ba88f0 Mon Sep 17 00:00:00 2001 From: Gregory Szorc Date: Wed, 19 Dec 2018 17:18:45 +0000 Subject: [PATCH 58/74] Bug 1513429 - Upgrade image_builder to Ubuntu 18.04; r=dustin This appears to "just work." While I would like to convert this image to Debian and make it deterministic, that is more effect than I'm willing to invest at the moment. The impetus for this change is unblocking partial clones. Mercurial's SQLite storage backend apparently hits a SQLite bug in version 3.11 of SQLite (what Ubuntu 16.04 runs) where SQLite complains about database corruption when there are readers from multiple processes. Ubuntu 18.04 is running SQLite 3.22 and doesn't exhibit the buggy behavior. Differential Revision: https://phabricator.services.mozilla.com/D14228 --HG-- extra : moz-landing-system : lando --- taskcluster/docker/image-builder/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/taskcluster/docker/image-builder/Dockerfile b/taskcluster/docker/image-builder/Dockerfile index 36549f9ed..93470373e 100644 --- a/taskcluster/docker/image-builder/Dockerfile +++ b/taskcluster/docker/image-builder/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:16.04 +FROM ubuntu:18.04 # %include python/mozbuild/mozbuild/action/tooltool.py ADD topsrcdir/python/mozbuild/mozbuild/action/tooltool.py /setup/tooltool.py From 7b00d6c27872e7475d5af753bd99839c86fa3152 Mon Sep 17 00:00:00 2001 From: Gregory Szorc Date: Sat, 29 Dec 2018 05:13:09 +0000 Subject: [PATCH 59/74] Bug 1512188 - Rename --vcs-checkout to --gecko-checkout; r=tomprince,dustin We now have multiple things we may check out. "vcs" meaning "gecko" is not obvious. Let's change the terminology to be more specific. Differential Revision: https://phabricator.services.mozilla.com/D13813 --HG-- extra : moz-landing-system : lando --- taskcluster/docker/image-builder/build-image.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/taskcluster/docker/image-builder/build-image.sh b/taskcluster/docker/image-builder/build-image.sh index 7da8da97c..654ff6ecc 100755 --- a/taskcluster/docker/image-builder/build-image.sh +++ b/taskcluster/docker/image-builder/build-image.sh @@ -38,7 +38,7 @@ fi # Build image run-task \ - --vcs-checkout "/builds/worker/checkouts/gecko" \ + --gecko-checkout "/builds/worker/checkouts/gecko" \ --sparse-profile build/sparse-profiles/docker-image \ -- \ sh -x -c "$LOAD_COMMAND \ From 74cfb506a917129045a8bd176e094ff7ea8d1102 Mon Sep 17 00:00:00 2001 From: Gregory Szorc Date: Sat, 29 Dec 2018 05:13:17 +0000 Subject: [PATCH 60/74] Bug 1512188 - Rename --sparse-profile to --gecko-sparse-profile; r=tomprince,dustin We have multiple source checkouts. --sparse-profile is ambiguous as to which one it could refer to. Let's rename the argument so it is prefixed with the repo/project we are checking out. Differential Revision: https://phabricator.services.mozilla.com/D13814 --HG-- extra : moz-landing-system : lando --- taskcluster/docker/image-builder/build-image.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/taskcluster/docker/image-builder/build-image.sh b/taskcluster/docker/image-builder/build-image.sh index 654ff6ecc..cd00b8bb3 100755 --- a/taskcluster/docker/image-builder/build-image.sh +++ b/taskcluster/docker/image-builder/build-image.sh @@ -39,7 +39,7 @@ fi # Build image run-task \ --gecko-checkout "/builds/worker/checkouts/gecko" \ - --sparse-profile build/sparse-profiles/docker-image \ + --gecko-sparse-profile build/sparse-profiles/docker-image \ -- \ sh -x -c "$LOAD_COMMAND \ /builds/worker/checkouts/gecko/mach taskcluster-build-image \ From 6c0d397d940506dd88394f8bfc45cbe8c33c9983 Mon Sep 17 00:00:00 2001 From: Bogdan Tara Date: Sat, 29 Dec 2018 07:56:56 +0200 Subject: [PATCH 61/74] Backed out 8 changesets (bug 1512285, bug 1512188) for fetch bustages CLOSED TREE Backed out changeset 5e7aa7d98012 (bug 1512188) Backed out changeset 55874a956ae1 (bug 1512188) Backed out changeset ac6e2cd6eb32 (bug 1512188) Backed out changeset f68df074eac1 (bug 1512188) Backed out changeset f01d2662582f (bug 1512188) Backed out changeset 35d74dd16706 (bug 1512285) Backed out changeset 9e95b9bf1201 (bug 1512188) Backed out changeset ac4a49eb95f5 (bug 1512188) --- taskcluster/docker/image-builder/build-image.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/taskcluster/docker/image-builder/build-image.sh b/taskcluster/docker/image-builder/build-image.sh index cd00b8bb3..7da8da97c 100755 --- a/taskcluster/docker/image-builder/build-image.sh +++ b/taskcluster/docker/image-builder/build-image.sh @@ -38,8 +38,8 @@ fi # Build image run-task \ - --gecko-checkout "/builds/worker/checkouts/gecko" \ - --gecko-sparse-profile build/sparse-profiles/docker-image \ + --vcs-checkout "/builds/worker/checkouts/gecko" \ + --sparse-profile build/sparse-profiles/docker-image \ -- \ sh -x -c "$LOAD_COMMAND \ /builds/worker/checkouts/gecko/mach taskcluster-build-image \ From 767a0892397e64421a566d2a0be384282794294c Mon Sep 17 00:00:00 2001 From: Gregory Szorc Date: Mon, 31 Dec 2018 18:51:27 +0000 Subject: [PATCH 62/74] Bug 1512188 - Rename --vcs-checkout to --gecko-checkout; r=tomprince,dustin We now have multiple things we may check out. "vcs" meaning "gecko" is not obvious. Let's change the terminology to be more specific. Differential Revision: https://phabricator.services.mozilla.com/D13813 --HG-- extra : moz-landing-system : lando --- taskcluster/docker/image-builder/build-image.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/taskcluster/docker/image-builder/build-image.sh b/taskcluster/docker/image-builder/build-image.sh index 7da8da97c..654ff6ecc 100755 --- a/taskcluster/docker/image-builder/build-image.sh +++ b/taskcluster/docker/image-builder/build-image.sh @@ -38,7 +38,7 @@ fi # Build image run-task \ - --vcs-checkout "/builds/worker/checkouts/gecko" \ + --gecko-checkout "/builds/worker/checkouts/gecko" \ --sparse-profile build/sparse-profiles/docker-image \ -- \ sh -x -c "$LOAD_COMMAND \ From ef0519cf1ec735660a34dc7e954b9ffe8a81af2f Mon Sep 17 00:00:00 2001 From: Gregory Szorc Date: Mon, 31 Dec 2018 18:52:26 +0000 Subject: [PATCH 63/74] Bug 1512188 - Rename --sparse-profile to --gecko-sparse-profile; r=tomprince,dustin We have multiple source checkouts. --sparse-profile is ambiguous as to which one it could refer to. Let's rename the argument so it is prefixed with the repo/project we are checking out. Differential Revision: https://phabricator.services.mozilla.com/D13814 --HG-- extra : moz-landing-system : lando --- taskcluster/docker/image-builder/build-image.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/taskcluster/docker/image-builder/build-image.sh b/taskcluster/docker/image-builder/build-image.sh index 654ff6ecc..cd00b8bb3 100755 --- a/taskcluster/docker/image-builder/build-image.sh +++ b/taskcluster/docker/image-builder/build-image.sh @@ -39,7 +39,7 @@ fi # Build image run-task \ --gecko-checkout "/builds/worker/checkouts/gecko" \ - --sparse-profile build/sparse-profiles/docker-image \ + --gecko-sparse-profile build/sparse-profiles/docker-image \ -- \ sh -x -c "$LOAD_COMMAND \ /builds/worker/checkouts/gecko/mach taskcluster-build-image \ From 79c7dbc80a84dd29284db7e00f9b445640499366 Mon Sep 17 00:00:00 2001 From: Chris AtLee Date: Sat, 23 Feb 2019 17:16:43 +0000 Subject: [PATCH 64/74] Bug 1527394: Squash docker images before exporting/compressing them r=tomprince Differential Revision: https://phabricator.services.mozilla.com/D19541 --HG-- extra : moz-landing-system : lando --- taskcluster/docker/image-builder/Dockerfile | 2 + .../docker/image-builder/build-image.sh | 6 +- .../docker/image-builder/requirements/py2.in | 2 + .../docker/image-builder/requirements/py2.txt | 67 +++++++++++ .../docker/image-builder/requirements/py3.in | 8 ++ .../docker/image-builder/requirements/py3.txt | 104 ++++++++++++++++++ taskcluster/docker/image-builder/setup.sh | 22 +--- 7 files changed, 192 insertions(+), 19 deletions(-) create mode 100644 taskcluster/docker/image-builder/requirements/py2.in create mode 100644 taskcluster/docker/image-builder/requirements/py2.txt create mode 100644 taskcluster/docker/image-builder/requirements/py3.in create mode 100644 taskcluster/docker/image-builder/requirements/py3.txt diff --git a/taskcluster/docker/image-builder/Dockerfile b/taskcluster/docker/image-builder/Dockerfile index 93470373e..a74264739 100644 --- a/taskcluster/docker/image-builder/Dockerfile +++ b/taskcluster/docker/image-builder/Dockerfile @@ -22,6 +22,8 @@ ADD topsrcdir/taskcluster/scripts/run-task /usr/local/bin/run-task ADD build-image.sh /usr/local/bin/build-image.sh ADD download-and-compress /usr/local/bin/download-and-compress ADD setup.sh /setup/setup.sh +ADD requirements/py2.txt /setup/requirements-py2.txt +ADD requirements/py3.txt /setup/requirements-py3.txt RUN bash /setup/setup.sh # Setup a workspace that won't use AUFS. diff --git a/taskcluster/docker/image-builder/build-image.sh b/taskcluster/docker/image-builder/build-image.sh index cd00b8bb3..aaf3e3ae6 100755 --- a/taskcluster/docker/image-builder/build-image.sh +++ b/taskcluster/docker/image-builder/build-image.sh @@ -43,9 +43,13 @@ run-task \ -- \ sh -x -c "$LOAD_COMMAND \ /builds/worker/checkouts/gecko/mach taskcluster-build-image \ - -t \"$IMAGE_NAME:$HASH\" \ + -t \"${IMAGE_NAME}:${HASH}-pre\" \ \"$IMAGE_NAME\"" +# Squash the image +export DOCKER_HOST=unix:/$DOCKER_SOCKET +/usr/local/bin/docker-squash -v -t "${IMAGE_NAME}:${HASH}" "${IMAGE_NAME}:${HASH}-pre" + # Create artifact folder (note that this must occur after run-task) mkdir -p /builds/worker/workspace/artifacts diff --git a/taskcluster/docker/image-builder/requirements/py2.in b/taskcluster/docker/image-builder/requirements/py2.in new file mode 100644 index 000000000..8f4aae712 --- /dev/null +++ b/taskcluster/docker/image-builder/requirements/py2.in @@ -0,0 +1,2 @@ +# For compressing docker images +zstandard diff --git a/taskcluster/docker/image-builder/requirements/py2.txt b/taskcluster/docker/image-builder/requirements/py2.txt new file mode 100644 index 000000000..cbd986a3a --- /dev/null +++ b/taskcluster/docker/image-builder/requirements/py2.txt @@ -0,0 +1,67 @@ +# +# This file is autogenerated by pip-compile +# To update, run: +# +# pip-compile --generate-hashes --output-file requirements/py2.txt requirements/py2.in +# +cffi==1.12.1 \ + --hash=sha256:0b5f895714a7a9905148fc51978c62e8a6cbcace30904d39dcd0d9e2265bb2f6 \ + --hash=sha256:27cdc7ba35ee6aa443271d11583b50815c4bb52be89a909d0028e86c21961709 \ + --hash=sha256:2d4a38049ea93d5ce3c7659210393524c1efc3efafa151bd85d196fa98fce50a \ + --hash=sha256:3262573d0d60fc6b9d0e0e6e666db0e5045cbe8a531779aa0deb3b425ec5a282 \ + --hash=sha256:358e96cfffc185ab8f6e7e425c7bb028931ed08d65402fbcf3f4e1bff6e66556 \ + --hash=sha256:37c7db824b5687fbd7ea5519acfd054c905951acc53503547c86be3db0580134 \ + --hash=sha256:39b9554dfe60f878e0c6ff8a460708db6e1b1c9cc6da2c74df2955adf83e355d \ + --hash=sha256:42b96a77acf8b2d06821600fa87c208046decc13bd22a4a0e65c5c973443e0da \ + --hash=sha256:5b37dde5035d3c219324cac0e69d96495970977f310b306fa2df5910e1f329a1 \ + --hash=sha256:5d35819f5566d0dd254f273d60cf4a2dcdd3ae3003dfd412d40b3fe8ffd87509 \ + --hash=sha256:5df73aa465e53549bd03c819c1bc69fb85529a5e1a693b7b6cb64408dd3970d1 \ + --hash=sha256:7075b361f7a4d0d4165439992d0b8a3cdfad1f302bf246ed9308a2e33b046bd3 \ + --hash=sha256:7678b5a667b0381c173abe530d7bdb0e6e3b98e062490618f04b80ca62686d96 \ + --hash=sha256:7dfd996192ff8a535458c17f22ff5eb78b83504c34d10eefac0c77b1322609e2 \ + --hash=sha256:8a3be5d31d02c60f84c4fd4c98c5e3a97b49f32e16861367f67c49425f955b28 \ + --hash=sha256:9812e53369c469506b123aee9dcb56d50c82fad60c5df87feb5ff59af5b5f55c \ + --hash=sha256:9b6f7ba4e78c52c1a291d0c0c0bd745d19adde1a9e1c03cb899f0c6efd6f8033 \ + --hash=sha256:a85bc1d7c3bba89b3d8c892bc0458de504f8b3bcca18892e6ed15b5f7a52ad9d \ + --hash=sha256:aa6b9c843ad645ebb12616de848cc4e25a40f633ccc293c3c9fe34107c02c2ea \ + --hash=sha256:bae1aa56ee00746798beafe486daa7cfb586cd395c6ce822ba3068e48d761bc0 \ + --hash=sha256:bae96e26510e4825d5910a196bf6b5a11a18b87d9278db6d08413be8ea799469 \ + --hash=sha256:bd78df3b594013b227bf31d0301566dc50ba6f40df38a70ded731d5a8f2cb071 \ + --hash=sha256:c2711197154f46d06f73542c539a0ff5411f1951fab391e0a4ac8359badef719 \ + --hash=sha256:d998c20e3deed234fca993fd6c8314cb7cbfda05fd170f1bd75bb5d7421c3c5a \ + --hash=sha256:df4f840d77d9e37136f8e6b432fecc9d6b8730f18f896e90628712c793466ce6 \ + --hash=sha256:f5653c2581acb038319e6705d4e3593677676df14b112f13e0b5b44b6a18df1a \ + --hash=sha256:f7c7aa485a2e2250d455148470ffd0195eecc3d845122635202d7467d6f7b4cf \ + --hash=sha256:f9e2c66a6493147de835f207f198540a56b26745ce4f272fbc7c2f2cfebeb729 \ + # via zstandard +pycparser==2.19 \ + --hash=sha256:a988718abfad80b6b157acce7bf130a30876d27603738ac39f140993246b25b3 \ + # via cffi +zstandard==0.10.2 \ + --hash=sha256:08114ac056944e7f70c0faf99d0afbce08b078eacf8ee6698985654c7e725234 \ + --hash=sha256:087276799ddf3200b4724e3d6f57b11ba975d9243b4af9e95721397d795a2497 \ + --hash=sha256:0c21feac9f7c850a457b1c707c3cc4f3b8f475a3c9120f8cec82ebc3b215b80a \ + --hash=sha256:0fe6403a01e996a7247239691101148dc4071ccf7fe12b680d7b6c91a04aefbb \ + --hash=sha256:1383412acd5356ff543c434723f2e7794c77e1ed4efc1062464cc2112c09af50 \ + --hash=sha256:2acd18eeac4fcecef8c1b95d4ffaa606222aa1ba0d4372e829dc516b0504e6ef \ + --hash=sha256:302bd7b3bc7281015cd6f975207755c534551d0a32c79147518f2de0459dbef4 \ + --hash=sha256:390acfced0106fb12247e12c2aa399836e6686f5ba9daec332957ff830f215cd \ + --hash=sha256:43ec51075547d498ec6e7952e459c3817e610d6e4ca68f4fa43a16ccea01d496 \ + --hash=sha256:53f89a65d52d6fb56b2c5dd0445f30ca25852f344ba20de325ce6767dd842fca \ + --hash=sha256:5f4f650b83b8085862de9e555d87f6053ca577b4070f4c6610a870116c4dd1f4 \ + --hash=sha256:72ef2361d90a717457376351acb5b1b0c189a09dbd95adcb51907a96b79a6add \ + --hash=sha256:7ef5c7ede8e8cda2a37c0ecab456f4cfae2c42049f51b24edb5303dbfe318ea6 \ + --hash=sha256:86c9dee0fe6d4ea5bf394767929fdf5f924d161d9a6d23adcd58a690c5e160b0 \ + --hash=sha256:8b587c9a17f4b050274d9b7f9284d5fae0a8d6a8021f88f779345593326bc33d \ + --hash=sha256:91025801859a60b7761dea6a8b645f25be6d3639ef828423f094d90b3f60850e \ + --hash=sha256:9d2940e2801cc768d2cb71e71dca3b025ca3737e9d1d0fad0c95b2e7db0c947a \ + --hash=sha256:aa520b90eede823632013a319e91652d8226a6309a104cffdc7e00d5a2b5e66b \ + --hash=sha256:b10fba39049595827f228e77e7b5070cb39c46466bf8fef51da73220a20cc717 \ + --hash=sha256:c794b5c21485fb3232f5693995ba1a497267b1aecb70b218107cf131f8dc1d3d \ + --hash=sha256:d05516bc197c5b7b2aa2f834ea7c5ee9fd9aa3034f4193cc05d899b18251aa9c \ + --hash=sha256:d085c2c676f03357e5d6b11dbbf4e8c1b0d20b1066ac87e6cccc45d4b6c19675 \ + --hash=sha256:dd40e26aaee67b9078618b0fce3d5f209e328852f2c72c6772cf6352f57d2ed1 \ + --hash=sha256:e7b84c10ed30c1c997d81ef271945372fba9e18ac58d77a17d43fd9c42392ed4 \ + --hash=sha256:e982d8af9618d45b25456f1f80e6d628295772d74d755f9a46b90711b7a56067 \ + --hash=sha256:ef24c8ec97f93b2bdf1080553cdf38ea9ab195846b679cdcfe683c945ed2f1ee \ + --hash=sha256:f46c5021c3663f82c2ff994295a8574638d56a831ca2a26d736d47fbcf4f9187 diff --git a/taskcluster/docker/image-builder/requirements/py3.in b/taskcluster/docker/image-builder/requirements/py3.in new file mode 100644 index 000000000..75aeac2c7 --- /dev/null +++ b/taskcluster/docker/image-builder/requirements/py3.in @@ -0,0 +1,8 @@ +# The docker module removed support for docker engines < 1.21 starting with +# docker 3.0. Once we upgrade the docker running on our workers, we can remove +# this restriction here. +docker<3.0 +docker-squash + +# For compressing docker images +zstandard diff --git a/taskcluster/docker/image-builder/requirements/py3.txt b/taskcluster/docker/image-builder/requirements/py3.txt new file mode 100644 index 000000000..1da0def30 --- /dev/null +++ b/taskcluster/docker/image-builder/requirements/py3.txt @@ -0,0 +1,104 @@ +# +# This file is autogenerated by pip-compile +# To update, run: +# +# pip-compile --generate-hashes --output-file requirements/py3.txt requirements/py3.in +# +certifi==2018.11.29 \ + --hash=sha256:47f9c83ef4c0c621eaef743f133f09fa8a74a9b75f037e8624f83bd1b6626cb7 \ + --hash=sha256:993f830721089fef441cdfeb4b2c8c9df86f0c63239f06bd025a76a7daddb033 \ + # via requests +cffi==1.12.1 \ + --hash=sha256:0b5f895714a7a9905148fc51978c62e8a6cbcace30904d39dcd0d9e2265bb2f6 \ + --hash=sha256:27cdc7ba35ee6aa443271d11583b50815c4bb52be89a909d0028e86c21961709 \ + --hash=sha256:2d4a38049ea93d5ce3c7659210393524c1efc3efafa151bd85d196fa98fce50a \ + --hash=sha256:3262573d0d60fc6b9d0e0e6e666db0e5045cbe8a531779aa0deb3b425ec5a282 \ + --hash=sha256:358e96cfffc185ab8f6e7e425c7bb028931ed08d65402fbcf3f4e1bff6e66556 \ + --hash=sha256:37c7db824b5687fbd7ea5519acfd054c905951acc53503547c86be3db0580134 \ + --hash=sha256:39b9554dfe60f878e0c6ff8a460708db6e1b1c9cc6da2c74df2955adf83e355d \ + --hash=sha256:42b96a77acf8b2d06821600fa87c208046decc13bd22a4a0e65c5c973443e0da \ + --hash=sha256:5b37dde5035d3c219324cac0e69d96495970977f310b306fa2df5910e1f329a1 \ + --hash=sha256:5d35819f5566d0dd254f273d60cf4a2dcdd3ae3003dfd412d40b3fe8ffd87509 \ + --hash=sha256:5df73aa465e53549bd03c819c1bc69fb85529a5e1a693b7b6cb64408dd3970d1 \ + --hash=sha256:7075b361f7a4d0d4165439992d0b8a3cdfad1f302bf246ed9308a2e33b046bd3 \ + --hash=sha256:7678b5a667b0381c173abe530d7bdb0e6e3b98e062490618f04b80ca62686d96 \ + --hash=sha256:7dfd996192ff8a535458c17f22ff5eb78b83504c34d10eefac0c77b1322609e2 \ + --hash=sha256:8a3be5d31d02c60f84c4fd4c98c5e3a97b49f32e16861367f67c49425f955b28 \ + --hash=sha256:9812e53369c469506b123aee9dcb56d50c82fad60c5df87feb5ff59af5b5f55c \ + --hash=sha256:9b6f7ba4e78c52c1a291d0c0c0bd745d19adde1a9e1c03cb899f0c6efd6f8033 \ + --hash=sha256:a85bc1d7c3bba89b3d8c892bc0458de504f8b3bcca18892e6ed15b5f7a52ad9d \ + --hash=sha256:aa6b9c843ad645ebb12616de848cc4e25a40f633ccc293c3c9fe34107c02c2ea \ + --hash=sha256:bae1aa56ee00746798beafe486daa7cfb586cd395c6ce822ba3068e48d761bc0 \ + --hash=sha256:bae96e26510e4825d5910a196bf6b5a11a18b87d9278db6d08413be8ea799469 \ + --hash=sha256:bd78df3b594013b227bf31d0301566dc50ba6f40df38a70ded731d5a8f2cb071 \ + --hash=sha256:c2711197154f46d06f73542c539a0ff5411f1951fab391e0a4ac8359badef719 \ + --hash=sha256:d998c20e3deed234fca993fd6c8314cb7cbfda05fd170f1bd75bb5d7421c3c5a \ + --hash=sha256:df4f840d77d9e37136f8e6b432fecc9d6b8730f18f896e90628712c793466ce6 \ + --hash=sha256:f5653c2581acb038319e6705d4e3593677676df14b112f13e0b5b44b6a18df1a \ + --hash=sha256:f7c7aa485a2e2250d455148470ffd0195eecc3d845122635202d7467d6f7b4cf \ + --hash=sha256:f9e2c66a6493147de835f207f198540a56b26745ce4f272fbc7c2f2cfebeb729 \ + # via zstandard +chardet==3.0.4 \ + --hash=sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae \ + --hash=sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691 \ + # via requests +docker-pycreds==0.4.0 \ + --hash=sha256:6ce3270bcaf404cc4c3e27e4b6c70d3521deae82fb508767870fdbf772d584d4 \ + --hash=sha256:7266112468627868005106ec19cd0d722702d2b7d5912a28e19b826c3d37af49 \ + # via docker +docker-squash==1.0.7 \ + --hash=sha256:95ca24fbeffa915c3d467b7ad538c6437a02f68aaa4e31e16c451c47a30a2169 +docker==2.7.0 \ + --hash=sha256:144248308e8ea31c4863c6d74e1b55daf97cc190b61d0fe7b7313ab920d6a76c \ + --hash=sha256:c1d4e37b1ea03b2b6efdd0379640f6ea372fefe56efa65d4d17c34c6b9d54558 +idna==2.8 \ + --hash=sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407 \ + --hash=sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c \ + # via requests +pycparser==2.19 \ + --hash=sha256:a988718abfad80b6b157acce7bf130a30876d27603738ac39f140993246b25b3 \ + # via cffi +requests==2.21.0 \ + --hash=sha256:502a824f31acdacb3a35b6690b5fbf0bc41d63a24a45c4004352b0242707598e \ + --hash=sha256:7bf2a778576d825600030a110f3c0e3e8edc51dfaafe1c146e39a2027784957b \ + # via docker +six==1.12.0 \ + --hash=sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c \ + --hash=sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73 \ + # via docker, docker-pycreds, docker-squash, websocket-client +urllib3==1.24.1 \ + --hash=sha256:61bf29cada3fc2fbefad4fdf059ea4bd1b4a86d2b6d15e1c7c0b582b9752fe39 \ + --hash=sha256:de9529817c93f27c8ccbfead6985011db27bd0ddfcdb2d86f3f663385c6a9c22 \ + # via requests +websocket-client==0.54.0 \ + --hash=sha256:8c8bf2d4f800c3ed952df206b18c28f7070d9e3dcbd6ca6291127574f57ee786 \ + --hash=sha256:e51562c91ddb8148e791f0155fdb01325d99bb52c4cdbb291aee7a3563fd0849 \ + # via docker +zstandard==0.10.2 \ + --hash=sha256:08114ac056944e7f70c0faf99d0afbce08b078eacf8ee6698985654c7e725234 \ + --hash=sha256:087276799ddf3200b4724e3d6f57b11ba975d9243b4af9e95721397d795a2497 \ + --hash=sha256:0c21feac9f7c850a457b1c707c3cc4f3b8f475a3c9120f8cec82ebc3b215b80a \ + --hash=sha256:0fe6403a01e996a7247239691101148dc4071ccf7fe12b680d7b6c91a04aefbb \ + --hash=sha256:1383412acd5356ff543c434723f2e7794c77e1ed4efc1062464cc2112c09af50 \ + --hash=sha256:2acd18eeac4fcecef8c1b95d4ffaa606222aa1ba0d4372e829dc516b0504e6ef \ + --hash=sha256:302bd7b3bc7281015cd6f975207755c534551d0a32c79147518f2de0459dbef4 \ + --hash=sha256:390acfced0106fb12247e12c2aa399836e6686f5ba9daec332957ff830f215cd \ + --hash=sha256:43ec51075547d498ec6e7952e459c3817e610d6e4ca68f4fa43a16ccea01d496 \ + --hash=sha256:53f89a65d52d6fb56b2c5dd0445f30ca25852f344ba20de325ce6767dd842fca \ + --hash=sha256:5f4f650b83b8085862de9e555d87f6053ca577b4070f4c6610a870116c4dd1f4 \ + --hash=sha256:72ef2361d90a717457376351acb5b1b0c189a09dbd95adcb51907a96b79a6add \ + --hash=sha256:7ef5c7ede8e8cda2a37c0ecab456f4cfae2c42049f51b24edb5303dbfe318ea6 \ + --hash=sha256:86c9dee0fe6d4ea5bf394767929fdf5f924d161d9a6d23adcd58a690c5e160b0 \ + --hash=sha256:8b587c9a17f4b050274d9b7f9284d5fae0a8d6a8021f88f779345593326bc33d \ + --hash=sha256:91025801859a60b7761dea6a8b645f25be6d3639ef828423f094d90b3f60850e \ + --hash=sha256:9d2940e2801cc768d2cb71e71dca3b025ca3737e9d1d0fad0c95b2e7db0c947a \ + --hash=sha256:aa520b90eede823632013a319e91652d8226a6309a104cffdc7e00d5a2b5e66b \ + --hash=sha256:b10fba39049595827f228e77e7b5070cb39c46466bf8fef51da73220a20cc717 \ + --hash=sha256:c794b5c21485fb3232f5693995ba1a497267b1aecb70b218107cf131f8dc1d3d \ + --hash=sha256:d05516bc197c5b7b2aa2f834ea7c5ee9fd9aa3034f4193cc05d899b18251aa9c \ + --hash=sha256:d085c2c676f03357e5d6b11dbbf4e8c1b0d20b1066ac87e6cccc45d4b6c19675 \ + --hash=sha256:dd40e26aaee67b9078618b0fce3d5f209e328852f2c72c6772cf6352f57d2ed1 \ + --hash=sha256:e7b84c10ed30c1c997d81ef271945372fba9e18ac58d77a17d43fd9c42392ed4 \ + --hash=sha256:e982d8af9618d45b25456f1f80e6d628295772d74d755f9a46b90711b7a56067 \ + --hash=sha256:ef24c8ec97f93b2bdf1080553cdf38ea9ab195846b679cdcfe683c945ed2f1ee \ + --hash=sha256:f46c5021c3663f82c2ff994295a8574638d56a831ca2a26d736d47fbcf4f9187 diff --git a/taskcluster/docker/image-builder/setup.sh b/taskcluster/docker/image-builder/setup.sh index 041f1b2fd..2aa0c70b2 100644 --- a/taskcluster/docker/image-builder/setup.sh +++ b/taskcluster/docker/image-builder/setup.sh @@ -34,26 +34,12 @@ chmod +x /usr/local/bin/download-and-compress # Create workspace mkdir -p /builds/worker/workspace -# Install python-zstandard. -( -cd /setup -tooltool_fetch < Date: Thu, 25 Jul 2019 14:53:58 +0900 Subject: [PATCH 65/74] Bug 1569355 - Upgrade python-zstandard to 0.11.1. r=tomprince Differential Revision: https://phabricator.services.mozilla.com/D39583 MANUAL PUSH: avoid closing autoland while all docker images and toolchains are rebuilt due to both changes. --- .../docker/image-builder/requirements/py2.txt | 56 +++++++++---------- .../docker/image-builder/requirements/py3.txt | 56 +++++++++---------- 2 files changed, 56 insertions(+), 56 deletions(-) diff --git a/taskcluster/docker/image-builder/requirements/py2.txt b/taskcluster/docker/image-builder/requirements/py2.txt index cbd986a3a..9b290c99c 100644 --- a/taskcluster/docker/image-builder/requirements/py2.txt +++ b/taskcluster/docker/image-builder/requirements/py2.txt @@ -37,31 +37,31 @@ cffi==1.12.1 \ pycparser==2.19 \ --hash=sha256:a988718abfad80b6b157acce7bf130a30876d27603738ac39f140993246b25b3 \ # via cffi -zstandard==0.10.2 \ - --hash=sha256:08114ac056944e7f70c0faf99d0afbce08b078eacf8ee6698985654c7e725234 \ - --hash=sha256:087276799ddf3200b4724e3d6f57b11ba975d9243b4af9e95721397d795a2497 \ - --hash=sha256:0c21feac9f7c850a457b1c707c3cc4f3b8f475a3c9120f8cec82ebc3b215b80a \ - --hash=sha256:0fe6403a01e996a7247239691101148dc4071ccf7fe12b680d7b6c91a04aefbb \ - --hash=sha256:1383412acd5356ff543c434723f2e7794c77e1ed4efc1062464cc2112c09af50 \ - --hash=sha256:2acd18eeac4fcecef8c1b95d4ffaa606222aa1ba0d4372e829dc516b0504e6ef \ - --hash=sha256:302bd7b3bc7281015cd6f975207755c534551d0a32c79147518f2de0459dbef4 \ - --hash=sha256:390acfced0106fb12247e12c2aa399836e6686f5ba9daec332957ff830f215cd \ - --hash=sha256:43ec51075547d498ec6e7952e459c3817e610d6e4ca68f4fa43a16ccea01d496 \ - --hash=sha256:53f89a65d52d6fb56b2c5dd0445f30ca25852f344ba20de325ce6767dd842fca \ - --hash=sha256:5f4f650b83b8085862de9e555d87f6053ca577b4070f4c6610a870116c4dd1f4 \ - --hash=sha256:72ef2361d90a717457376351acb5b1b0c189a09dbd95adcb51907a96b79a6add \ - --hash=sha256:7ef5c7ede8e8cda2a37c0ecab456f4cfae2c42049f51b24edb5303dbfe318ea6 \ - --hash=sha256:86c9dee0fe6d4ea5bf394767929fdf5f924d161d9a6d23adcd58a690c5e160b0 \ - --hash=sha256:8b587c9a17f4b050274d9b7f9284d5fae0a8d6a8021f88f779345593326bc33d \ - --hash=sha256:91025801859a60b7761dea6a8b645f25be6d3639ef828423f094d90b3f60850e \ - --hash=sha256:9d2940e2801cc768d2cb71e71dca3b025ca3737e9d1d0fad0c95b2e7db0c947a \ - --hash=sha256:aa520b90eede823632013a319e91652d8226a6309a104cffdc7e00d5a2b5e66b \ - --hash=sha256:b10fba39049595827f228e77e7b5070cb39c46466bf8fef51da73220a20cc717 \ - --hash=sha256:c794b5c21485fb3232f5693995ba1a497267b1aecb70b218107cf131f8dc1d3d \ - --hash=sha256:d05516bc197c5b7b2aa2f834ea7c5ee9fd9aa3034f4193cc05d899b18251aa9c \ - --hash=sha256:d085c2c676f03357e5d6b11dbbf4e8c1b0d20b1066ac87e6cccc45d4b6c19675 \ - --hash=sha256:dd40e26aaee67b9078618b0fce3d5f209e328852f2c72c6772cf6352f57d2ed1 \ - --hash=sha256:e7b84c10ed30c1c997d81ef271945372fba9e18ac58d77a17d43fd9c42392ed4 \ - --hash=sha256:e982d8af9618d45b25456f1f80e6d628295772d74d755f9a46b90711b7a56067 \ - --hash=sha256:ef24c8ec97f93b2bdf1080553cdf38ea9ab195846b679cdcfe683c945ed2f1ee \ - --hash=sha256:f46c5021c3663f82c2ff994295a8574638d56a831ca2a26d736d47fbcf4f9187 +zstandard==0.11.1 \ + --hash=sha256:19f5ad81590acd20dbdfb930b87a035189778662fdc67ab8cbcc106269ed1be8 \ + --hash=sha256:1a1db0c9774181e806a418c32d511aa085c7e2c28c257a58f6c107f5decb3109 \ + --hash=sha256:22d7aa898f36f78108cc1ef0c8da8225f0add518441d815ad4fdd1d577378209 \ + --hash=sha256:357873afdd7cd0e653d169c36ce837ce2b3e5926dd4a5c0f0476c813f6765373 \ + --hash=sha256:3c31da5d78a7b07e722e8a3e0b1295bc9b316b7e90a1666659c451a42750ffe4 \ + --hash=sha256:3f76562ec63fabc6f4b5be0cd986f911c97105c35c31b4d655b90c4d2fe07f40 \ + --hash=sha256:42fa4462e0563fe17e73dfeb95eef9b00429b86282f8f6ca0e2765b1855a8324 \ + --hash=sha256:51aad01a5709ca6f45768c69ffd4c887528e5ad9e09302426b735560752c4e82 \ + --hash=sha256:6cd81819a02e57e38e27c53c5c0a7015e059b0e148a18bf27b46b4f808840879 \ + --hash=sha256:717fd2494f222164396e03d08ef57174d2a889920b81ca49f276caf9381e6405 \ + --hash=sha256:71c8711458212c973a9b719275db8111f22803e0caf675affde50703b96e9be1 \ + --hash=sha256:76a331b5a6258fce3906551557db9be83bdd89a62f66f509a55a4a307239c782 \ + --hash=sha256:7c92dfcdf7e0c540f9718b40b4c54516a968ef6b81567b75df81866a1af2189d \ + --hash=sha256:7f3db21223a8bb4ffcf6c36b9c20d38278967723b47fce249dcb6ec6d4082b83 \ + --hash=sha256:7fa9deba4c904e76870e08324adff94ec3a4bc56a50bbe1a9f859a4aed11c0d2 \ + --hash=sha256:88912cbcf68cc40037c113460a166ebfbbb24864ceebb89ad221ea346f22e995 \ + --hash=sha256:94aa5bb817f1c747b21214f6ef83a022bcb63bf81e4dae2954768165c13a510b \ + --hash=sha256:951e382a2ea47179ecb3e314e8c70f2e5189e3652ccbbcb71c6443dd71bc20fc \ + --hash=sha256:978a500ae1184f602dc902977ec208c7cf02c10caae9c159b10976a7cb29f879 \ + --hash=sha256:991c4a40171d87854b219cdf2ba56c1c34b3b3a8ebe5d1ab63bd357ff71271b2 \ + --hash=sha256:9ca84187182743d2e6bbf9d3f79d3834db205cddc98add27ad20f2189d080a60 \ + --hash=sha256:ae50bc839cf1ff549f55a3e55922563f246fb692f77497175a8d8d4cddc294da \ + --hash=sha256:b7abae5b17e82d5f78aaa641077b4619c6ad204e30c6f3445d422acff5f35d3e \ + --hash=sha256:b8fce0c961654f77c81a6ae1f2cd40633b41ef16a12ae02f0382ed6692f9bb90 \ + --hash=sha256:d8f047d3647a5cd1b77b4580f35208c938da00c101a092571c85bcefaa2d725d \ + --hash=sha256:f1785b31bf428e964a9670dd4f721023f2741ef7fd67c663bf01e3d4d3f9ec2a \ + --hash=sha256:fcf70e1e9d38035a15482e954ba064f3b701cf84cfe571576d15af93ac2a2fb1 diff --git a/taskcluster/docker/image-builder/requirements/py3.txt b/taskcluster/docker/image-builder/requirements/py3.txt index 1da0def30..8389ac886 100644 --- a/taskcluster/docker/image-builder/requirements/py3.txt +++ b/taskcluster/docker/image-builder/requirements/py3.txt @@ -74,31 +74,31 @@ websocket-client==0.54.0 \ --hash=sha256:8c8bf2d4f800c3ed952df206b18c28f7070d9e3dcbd6ca6291127574f57ee786 \ --hash=sha256:e51562c91ddb8148e791f0155fdb01325d99bb52c4cdbb291aee7a3563fd0849 \ # via docker -zstandard==0.10.2 \ - --hash=sha256:08114ac056944e7f70c0faf99d0afbce08b078eacf8ee6698985654c7e725234 \ - --hash=sha256:087276799ddf3200b4724e3d6f57b11ba975d9243b4af9e95721397d795a2497 \ - --hash=sha256:0c21feac9f7c850a457b1c707c3cc4f3b8f475a3c9120f8cec82ebc3b215b80a \ - --hash=sha256:0fe6403a01e996a7247239691101148dc4071ccf7fe12b680d7b6c91a04aefbb \ - --hash=sha256:1383412acd5356ff543c434723f2e7794c77e1ed4efc1062464cc2112c09af50 \ - --hash=sha256:2acd18eeac4fcecef8c1b95d4ffaa606222aa1ba0d4372e829dc516b0504e6ef \ - --hash=sha256:302bd7b3bc7281015cd6f975207755c534551d0a32c79147518f2de0459dbef4 \ - --hash=sha256:390acfced0106fb12247e12c2aa399836e6686f5ba9daec332957ff830f215cd \ - --hash=sha256:43ec51075547d498ec6e7952e459c3817e610d6e4ca68f4fa43a16ccea01d496 \ - --hash=sha256:53f89a65d52d6fb56b2c5dd0445f30ca25852f344ba20de325ce6767dd842fca \ - --hash=sha256:5f4f650b83b8085862de9e555d87f6053ca577b4070f4c6610a870116c4dd1f4 \ - --hash=sha256:72ef2361d90a717457376351acb5b1b0c189a09dbd95adcb51907a96b79a6add \ - --hash=sha256:7ef5c7ede8e8cda2a37c0ecab456f4cfae2c42049f51b24edb5303dbfe318ea6 \ - --hash=sha256:86c9dee0fe6d4ea5bf394767929fdf5f924d161d9a6d23adcd58a690c5e160b0 \ - --hash=sha256:8b587c9a17f4b050274d9b7f9284d5fae0a8d6a8021f88f779345593326bc33d \ - --hash=sha256:91025801859a60b7761dea6a8b645f25be6d3639ef828423f094d90b3f60850e \ - --hash=sha256:9d2940e2801cc768d2cb71e71dca3b025ca3737e9d1d0fad0c95b2e7db0c947a \ - --hash=sha256:aa520b90eede823632013a319e91652d8226a6309a104cffdc7e00d5a2b5e66b \ - --hash=sha256:b10fba39049595827f228e77e7b5070cb39c46466bf8fef51da73220a20cc717 \ - --hash=sha256:c794b5c21485fb3232f5693995ba1a497267b1aecb70b218107cf131f8dc1d3d \ - --hash=sha256:d05516bc197c5b7b2aa2f834ea7c5ee9fd9aa3034f4193cc05d899b18251aa9c \ - --hash=sha256:d085c2c676f03357e5d6b11dbbf4e8c1b0d20b1066ac87e6cccc45d4b6c19675 \ - --hash=sha256:dd40e26aaee67b9078618b0fce3d5f209e328852f2c72c6772cf6352f57d2ed1 \ - --hash=sha256:e7b84c10ed30c1c997d81ef271945372fba9e18ac58d77a17d43fd9c42392ed4 \ - --hash=sha256:e982d8af9618d45b25456f1f80e6d628295772d74d755f9a46b90711b7a56067 \ - --hash=sha256:ef24c8ec97f93b2bdf1080553cdf38ea9ab195846b679cdcfe683c945ed2f1ee \ - --hash=sha256:f46c5021c3663f82c2ff994295a8574638d56a831ca2a26d736d47fbcf4f9187 +zstandard==0.11.1 \ + --hash=sha256:19f5ad81590acd20dbdfb930b87a035189778662fdc67ab8cbcc106269ed1be8 \ + --hash=sha256:1a1db0c9774181e806a418c32d511aa085c7e2c28c257a58f6c107f5decb3109 \ + --hash=sha256:22d7aa898f36f78108cc1ef0c8da8225f0add518441d815ad4fdd1d577378209 \ + --hash=sha256:357873afdd7cd0e653d169c36ce837ce2b3e5926dd4a5c0f0476c813f6765373 \ + --hash=sha256:3c31da5d78a7b07e722e8a3e0b1295bc9b316b7e90a1666659c451a42750ffe4 \ + --hash=sha256:3f76562ec63fabc6f4b5be0cd986f911c97105c35c31b4d655b90c4d2fe07f40 \ + --hash=sha256:42fa4462e0563fe17e73dfeb95eef9b00429b86282f8f6ca0e2765b1855a8324 \ + --hash=sha256:51aad01a5709ca6f45768c69ffd4c887528e5ad9e09302426b735560752c4e82 \ + --hash=sha256:6cd81819a02e57e38e27c53c5c0a7015e059b0e148a18bf27b46b4f808840879 \ + --hash=sha256:717fd2494f222164396e03d08ef57174d2a889920b81ca49f276caf9381e6405 \ + --hash=sha256:71c8711458212c973a9b719275db8111f22803e0caf675affde50703b96e9be1 \ + --hash=sha256:76a331b5a6258fce3906551557db9be83bdd89a62f66f509a55a4a307239c782 \ + --hash=sha256:7c92dfcdf7e0c540f9718b40b4c54516a968ef6b81567b75df81866a1af2189d \ + --hash=sha256:7f3db21223a8bb4ffcf6c36b9c20d38278967723b47fce249dcb6ec6d4082b83 \ + --hash=sha256:7fa9deba4c904e76870e08324adff94ec3a4bc56a50bbe1a9f859a4aed11c0d2 \ + --hash=sha256:88912cbcf68cc40037c113460a166ebfbbb24864ceebb89ad221ea346f22e995 \ + --hash=sha256:94aa5bb817f1c747b21214f6ef83a022bcb63bf81e4dae2954768165c13a510b \ + --hash=sha256:951e382a2ea47179ecb3e314e8c70f2e5189e3652ccbbcb71c6443dd71bc20fc \ + --hash=sha256:978a500ae1184f602dc902977ec208c7cf02c10caae9c159b10976a7cb29f879 \ + --hash=sha256:991c4a40171d87854b219cdf2ba56c1c34b3b3a8ebe5d1ab63bd357ff71271b2 \ + --hash=sha256:9ca84187182743d2e6bbf9d3f79d3834db205cddc98add27ad20f2189d080a60 \ + --hash=sha256:ae50bc839cf1ff549f55a3e55922563f246fb692f77497175a8d8d4cddc294da \ + --hash=sha256:b7abae5b17e82d5f78aaa641077b4619c6ad204e30c6f3445d422acff5f35d3e \ + --hash=sha256:b8fce0c961654f77c81a6ae1f2cd40633b41ef16a12ae02f0382ed6692f9bb90 \ + --hash=sha256:d8f047d3647a5cd1b77b4580f35208c938da00c101a092571c85bcefaa2d725d \ + --hash=sha256:f1785b31bf428e964a9670dd4f721023f2741ef7fd67c663bf01e3d4d3f9ec2a \ + --hash=sha256:fcf70e1e9d38035a15482e954ba064f3b701cf84cfe571576d15af93ac2a2fb1 From 16d778b9f81f229900384fc355bd65094b3b641e Mon Sep 17 00:00:00 2001 From: Tom Prince Date: Tue, 7 Jul 2020 19:53:32 +0000 Subject: [PATCH 66/74] Bug 1626058: Build docker images with kaniko, instead of dind; r=glandium,hwine Differential Revision: https://phabricator.services.mozilla.com/D77864 --- taskcluster/docker/image-builder/Dockerfile | 127 +- taskcluster/docker/image-builder/VERSION | 2 +- taskcluster/docker/image-builder/apt.conf | 5 + .../docker/image-builder/build-image.sh | 66 - .../image-builder/build-image/Cargo.lock | 1078 +++++++++++++++++ .../image-builder/build-image/Cargo.toml | 22 + .../image-builder/build-image/src/config.rs | 112 ++ .../image-builder/build-image/src/main.rs | 169 +++ .../build-image/src/taskcluster.rs | 55 + .../image-builder/download-and-compress | 85 -- taskcluster/docker/image-builder/policy.json | 11 + .../docker/image-builder/requirements/py2.in | 2 - .../docker/image-builder/requirements/py2.txt | 67 - .../docker/image-builder/requirements/py3.in | 8 - .../docker/image-builder/requirements/py3.txt | 104 -- taskcluster/docker/image-builder/setup.sh | 54 - 16 files changed, 1544 insertions(+), 423 deletions(-) create mode 100644 taskcluster/docker/image-builder/apt.conf delete mode 100755 taskcluster/docker/image-builder/build-image.sh create mode 100644 taskcluster/docker/image-builder/build-image/Cargo.lock create mode 100644 taskcluster/docker/image-builder/build-image/Cargo.toml create mode 100644 taskcluster/docker/image-builder/build-image/src/config.rs create mode 100644 taskcluster/docker/image-builder/build-image/src/main.rs create mode 100644 taskcluster/docker/image-builder/build-image/src/taskcluster.rs delete mode 100755 taskcluster/docker/image-builder/download-and-compress create mode 100644 taskcluster/docker/image-builder/policy.json delete mode 100644 taskcluster/docker/image-builder/requirements/py2.in delete mode 100644 taskcluster/docker/image-builder/requirements/py2.txt delete mode 100644 taskcluster/docker/image-builder/requirements/py3.in delete mode 100644 taskcluster/docker/image-builder/requirements/py3.txt delete mode 100644 taskcluster/docker/image-builder/setup.sh diff --git a/taskcluster/docker/image-builder/Dockerfile b/taskcluster/docker/image-builder/Dockerfile index a74264739..b5804ce95 100644 --- a/taskcluster/docker/image-builder/Dockerfile +++ b/taskcluster/docker/image-builder/Dockerfile @@ -1,47 +1,102 @@ -FROM ubuntu:18.04 +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. -# %include python/mozbuild/mozbuild/action/tooltool.py -ADD topsrcdir/python/mozbuild/mozbuild/action/tooltool.py /setup/tooltool.py +FROM golang:1.14 as skopeo -# %include taskcluster/docker/recipes/common.sh -ADD topsrcdir/taskcluster/docker/recipes/common.sh /setup/common.sh +WORKDIR /go/src/ +RUN ["git", "clone", "--no-checkout", "--depth=1", "--branch=v1.1.0", "https://github.com/containers/skopeo", "."] +RUN ["git", "checkout", "63085f5bef1131aa9ec0907a5c8d66b67de7c4b2"] +ENV GO111MODULE=on CGO_ENABLED=0 +RUN ["go", "build", \ + "-mod=vendor", "-o", "out/skopeo", \ + "-tags", "exclude_graphdriver_devicemapper exclude_graphdriver_btrfs containers_image_openpgp", \ + # Set unixTempDirForBigFiles so skopeo will extract in a directory hidden by kaniko + # We create the directory below. + "-ldflags", " -X github.com/containers/image/v5/internal/tmpdir.unixTempDirForBigFiles=/workspace/tmp -X github.com/containers/image/v5/signature.systemDefaultPolicyPath=/kaniko/containers/policy.json -extldflags \"-static\" -w -s", \ + "./cmd/skopeo"] -# %include taskcluster/docker/recipes/install-mercurial.sh -ADD topsrcdir/taskcluster/docker/recipes/install-mercurial.sh /setup/install-mercurial.sh +FROM golang:1.14 as kaniko +WORKDIR /go/src/ +RUN ["git", "clone", "--no-checkout", "--depth=1", "--branch=v0.24.0", "https://github.com/GoogleContainerTools/kaniko", "."] +RUN ["git", "checkout", "cdbd8af0578c56e2801b57461e9f417f9479d303"] +RUN ["make"] -# %include testing/mozharness/external_tools/robustcheckout.py -ADD topsrcdir/testing/mozharness/external_tools/robustcheckout.py /usr/local/mercurial/robustcheckout.py +# Build the `build-image` command as a static binary using musl +# The setup is loosely based on a stripped down version of +# https://github.com/emk/rust-musl-builder/blob/master/Dockerfile +FROM debian:buster as build-image -# %include taskcluster/docker/recipes/hgrc -COPY topsrcdir/taskcluster/docker/recipes/hgrc /etc/mercurial/hgrc.d/mozilla.rc +COPY apt.conf /etc/apt/apt.conf.d/99taskcluster -# %include taskcluster/scripts/run-task -ADD topsrcdir/taskcluster/scripts/run-task /usr/local/bin/run-task +RUN apt-get update && \ + apt-get install \ + build-essential \ + ca-certificates \ + curl \ + musl-dev \ + musl-tools \ + && \ + useradd rust --user-group --create-home --shell /bin/bash -# Add and run setup script -ADD build-image.sh /usr/local/bin/build-image.sh -ADD download-and-compress /usr/local/bin/download-and-compress -ADD setup.sh /setup/setup.sh -ADD requirements/py2.txt /setup/requirements-py2.txt -ADD requirements/py3.txt /setup/requirements-py3.txt -RUN bash /setup/setup.sh +# Run all further code as user `rust`, and create our working directories +# as the appropriate user. +USER rust -# Setup a workspace that won't use AUFS. -VOLUME /builds/worker/checkouts -VOLUME /builds/worker/workspace +# Set up our path with all our binary directories, including those for the +# musl-gcc toolchain and for our Rust toolchain. +ENV PATH=/home/rust/.cargo/bin:$PATH -# Set variable normally configured at login, by the shells parent process, these -# are taken from GNU su manual -ENV HOME /builds/worker -ENV SHELL /bin/bash -ENV USER worker -ENV LOGNAME worker -ENV HOSTNAME taskcluster-worker -ENV LC_ALL C +# The Rust toolchain to use when building our image. Set by `hooks/build`. +ENV TOOLCHAIN=1.42.0 \ + TARGET=x86_64-unknown-linux-musl -# Create worker user -RUN useradd -d /builds/worker -s /bin/bash -m worker +# Install our Rust toolchain and the `musl` target. We patch the +# command-line we pass to the installer so that it won't attempt to +# interact with the user or fool around with TTYs. We also set the default +# `--target` to musl so that our users don't need to keep overriding it +# manually. +RUN curl https://sh.rustup.rs -sSf | \ + sh -s -- -y \ + --profile minimal \ + --default-toolchain $TOOLCHAIN \ + --target $TARGET -# Set some sane defaults -WORKDIR /builds/worker/ -CMD build-image.sh +# Expect our source code to live in /home/rust/src. We'll run the build as +# user `rust`, which will be uid 1000, gid 1000 outside the container. +RUN mkdir -p /home/rust/src +WORKDIR /home/rust/src +# Add our source code. +ADD --chown=rust:rust build-image/ ./ + +# --out-dir is not yet stable +ENV RUSTC_BOOTSTRAP=1 +# Build our application. +RUN ["cargo", "build", "--target", "x86_64-unknown-linux-musl", "--out-dir=bin", "--release", "-Zunstable-options"] + +FROM scratch as empty + +FROM scratch + +COPY --from=skopeo /go/src/out/skopeo /kaniko/skopeo +COPY --from=kaniko /go/src/out/executor /kaniko/executor +COPY --from=build-image \ + /home/rust/src/bin/build-image \ + /kaniko/build-image + +ADD https://mkcert.org/generate/ /kaniko/ssl/certs/ca-certificats.crt +ENV SSL_CERT_DIR=/kaniko/ssl/certs + +ADD policy.json /kaniko/containers/policy.json + +ENV HOME /root +ENV USER /root +WORKDIR /workspace + +ENV PATH /usr/local/bin:/kaniko + +VOLUME /workspace +# Create an empty temporary directory for skopeo +COPY --from=empty / /workspace/tmp +COPY --from=empty / /workspace/cache +ENTRYPOINT ["/kaniko/build-image"] diff --git a/taskcluster/docker/image-builder/VERSION b/taskcluster/docker/image-builder/VERSION index 4a36342fc..fcdb2e109 100644 --- a/taskcluster/docker/image-builder/VERSION +++ b/taskcluster/docker/image-builder/VERSION @@ -1 +1 @@ -3.0.0 +4.0.0 diff --git a/taskcluster/docker/image-builder/apt.conf b/taskcluster/docker/image-builder/apt.conf new file mode 100644 index 000000000..84c0cf10e --- /dev/null +++ b/taskcluster/docker/image-builder/apt.conf @@ -0,0 +1,5 @@ +quiet "true"; +APT::Get::Assume-Yes "true"; +APT::Install-Recommends "false"; +Acquire::Check-Valid-Until "false"; +Acquire::Retries "5"; diff --git a/taskcluster/docker/image-builder/build-image.sh b/taskcluster/docker/image-builder/build-image.sh deleted file mode 100755 index aaf3e3ae6..000000000 --- a/taskcluster/docker/image-builder/build-image.sh +++ /dev/null @@ -1,66 +0,0 @@ -#!/bin/bash -vex - -# Set bash options to exit immediately if a pipeline exists non-zero, expand -# print a trace of commands, and make output verbose (print shell input as it's -# read) -# See https://www.gnu.org/software/bash/manual/html_node/The-Set-Builtin.html -set -x -e -v -o pipefail - -# Prefix errors with taskcluster error prefix so that they are parsed by Treeherder -raise_error() { - echo - echo "[taskcluster-image-build:error] $1" - exit 1 -} - -# Ensure that the PROJECT is specified so the image can be indexed -test -n "$PROJECT" || raise_error "PROJECT must be provided." -test -n "$HASH" || raise_error "Context HASH must be provided." -test -n "$IMAGE_NAME" || raise_error "IMAGE_NAME must be provided." - -# The docker socket is mounted by the taskcluster worker in a way that prevents -# us changing its permissions to allow the worker user to access it. Create a -# proxy socket that the worker user can use. -export DOCKER_SOCKET=/var/run/docker.proxy -socat UNIX-LISTEN:$DOCKER_SOCKET,fork,group=worker,mode=0775 UNIX-CLIENT:/var/run/docker.sock "] +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +reqwest = { version= "0.10.4", features = ["rustls-tls", "blocking"], default-features = false} +zstd = "0.5.1" +url = "2.1.1" +anyhow = "1.0.27" +serde = { version = "1.0.105", features = ["derive"]} +serde_json = "1.0.50" +envy = "0.4.1" + +[workspace] diff --git a/taskcluster/docker/image-builder/build-image/src/config.rs b/taskcluster/docker/image-builder/build-image/src/config.rs new file mode 100644 index 000000000..94c1d55a1 --- /dev/null +++ b/taskcluster/docker/image-builder/build-image/src/config.rs @@ -0,0 +1,112 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +use anyhow::Result; +use serde::de::Error; +use serde::Deserialize; +use std::collections::HashMap; + +fn default_image_name() -> String { + "mozilla.org/taskgraph/default-image:latest".into() +} +fn default_zstd_level() -> i32 { + 3 +} + +fn from_json<'de, D, T>(deserializer: D) -> Result +where + D: serde::de::Deserializer<'de>, + T: serde::de::DeserializeOwned, +{ + let value: String = serde::Deserialize::deserialize(deserializer)?; + serde_json::from_str(&value).map_err(|err| { + D::Error::invalid_value(serde::de::Unexpected::Str(&value), &&*err.to_string()) + }) +} + +#[derive(Deserialize, Debug, PartialEq, Eq)] +pub struct Config { + pub context_task_id: String, + pub context_path: String, + pub parent_task_id: Option, + #[serde(default = "default_image_name")] + pub image_name: String, + #[serde(default = "default_zstd_level")] + pub docker_image_zstd_level: i32, + #[serde(default)] + pub debug: bool, + #[serde(default, deserialize_with = "from_json")] + pub docker_build_args: HashMap, +} + +impl Config { + pub fn from_env() -> Result { + Ok(envy::from_env()?) + } +} + +#[cfg(test)] +mod test { + use anyhow::Result; + + #[test] + fn test() -> Result<()> { + let env: Vec<(String, String)> = vec![ + ("CONTEXT_TASK_ID".into(), "xGRRgzG6QlCCwsFsyuqm0Q".into()), + ( + "CONTEXT_PATH".into(), + "public/docker-contexts/image.tar.gz".into(), + ), + ]; + let config: super::Config = envy::from_iter(env.into_iter())?; + assert_eq!( + config, + super::Config { + context_task_id: "xGRRgzG6QlCCwsFsyuqm0Q".into(), + context_path: "public/docker-contexts/image.tar.gz".into(), + parent_task_id: None, + image_name: "mozilla.org/taskgraph/default-image:latest".into(), + docker_image_zstd_level: 3, + debug: false, + docker_build_args: Default::default() + } + ); + Ok(()) + } + + #[test] + fn test_docker_build_args() -> Result<()> { + let env: Vec<(String, String)> = vec![ + ("CONTEXT_TASK_ID".into(), "xGRRgzG6QlCCwsFsyuqm0Q".into()), + ( + "CONTEXT_PATH".into(), + "public/docker-contexts/image.tar.gz".into(), + ), + ( + "DOCKER_BUILD_ARGS".into(), + serde_json::json! ({ + "test": "Value", + }) + .to_string(), + ), + ]; + let config: super::Config = envy::from_iter(env.into_iter())?; + assert_eq!( + config, + super::Config { + context_task_id: "xGRRgzG6QlCCwsFsyuqm0Q".into(), + context_path: "public/docker-contexts/image.tar.gz".into(), + parent_task_id: None, + image_name: "mozilla.org/taskgraph/default-image:latest".into(), + docker_image_zstd_level: 3, + debug: false, + docker_build_args: [("test".to_string(), "Value".to_string())] + .iter() + .cloned() + .collect(), + } + ); + Ok(()) + } +} diff --git a/taskcluster/docker/image-builder/build-image/src/main.rs b/taskcluster/docker/image-builder/build-image/src/main.rs new file mode 100644 index 000000000..85a0f9562 --- /dev/null +++ b/taskcluster/docker/image-builder/build-image/src/main.rs @@ -0,0 +1,169 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#![forbid(unsafe_code)] + +use std::collections::HashMap; +use std::process::Command; + +use anyhow::{ensure, Context, Result}; +use serde::Deserialize; + +mod config; +mod taskcluster; + +use config::Config; + +fn log_step(msg: &str) { + println!("[build-image] {}", msg); +} + +fn read_image_digest(path: &str) -> Result { + let output = Command::new("/kaniko/skopeo") + .arg("inspect") + .arg(format!("docker-archive:{}", path)) + .stdout(std::process::Stdio::piped()) + .spawn()? + .wait_with_output()?; + ensure!(output.status.success(), "Could not inspect parent image."); + + #[derive(Deserialize, Debug)] + #[serde(rename_all = "PascalCase")] + struct ImageInfo { + #[serde(skip_serializing_if = "Option::is_none")] + name: Option, + #[serde(skip_serializing_if = "Option::is_none")] + tag: Option, + digest: String, + // ... + } + + let image_info: ImageInfo = serde_json::from_slice(&output.stdout) + .with_context(|| format!("Could parse image info from {:?}", path))?; + Ok(image_info.digest) +} + +fn download_parent_image( + cluster: &taskcluster::TaskCluster, + task_id: &str, + dest: &str, +) -> Result { + zstd::stream::copy_decode( + cluster.stream_artifact(&task_id, "public/image.tar.zst")?, + std::fs::File::create(dest)?, + ) + .context("Could not download parent image.")?; + + read_image_digest(dest) +} + +fn build_image( + context_path: &str, + dest: &str, + debug: bool, + build_args: HashMap, +) -> Result<()> { + let mut command = Command::new("/kaniko/executor"); + command + .stderr(std::process::Stdio::inherit()) + .args(&["--context", &format!("tar://{}", context_path)]) + .args(&["--destination", "image"]) + .args(&["--dockerfile", "Dockerfile"]) + .arg("--no-push") + .args(&["--cache-dir", "/workspace/cache"]) + .arg("--single-snapshot") + // FIXME: Generating reproducible layers currently causes OOM. + // .arg("--reproducible") + .arg("--whitelist-var-run=false") + .args(&["--tarPath", dest]); + if debug { + command.args(&["-v", "debug"]); + } + for (key, value) in build_args { + command.args(&["--build-arg", &format!("{}={}", key, value)]); + } + let status = command.status()?; + ensure!(status.success(), "Could not build image."); + Ok(()) +} + +fn repack_image(source: &str, dest: &str, image_name: &str) -> Result<()> { + let status = Command::new("/kaniko/skopeo") + .arg("copy") + .arg(format!("docker-archive:{}", source)) + .arg(format!("docker-archive:{}:{}", dest, image_name)) + .stderr(std::process::Stdio::inherit()) + .status()?; + ensure!(status.success(), "Could repack image."); + Ok(()) +} + +fn main() -> Result<()> { + let config = Config::from_env().context("Could not parse environment variables.")?; + + let cluster = taskcluster::TaskCluster::from_env()?; + + let mut build_args = config.docker_build_args; + + build_args.insert("TASKCLUSTER_ROOT_URL".into(), cluster.root_url()); + + log_step("Downloading context."); + + std::io::copy( + &mut cluster.stream_artifact(&config.context_task_id, &config.context_path)?, + &mut std::fs::File::create("/workspace/context.tar.gz")?, + ) + .context("Could not download image context.")?; + + if let Some(parent_task_id) = config.parent_task_id { + log_step("Downloading image."); + let digest = download_parent_image(&cluster, &parent_task_id, "/workspace/parent.tar")?; + + log_step(&format!("Parent image digest {}", &digest)); + std::fs::rename( + "/workspace/parent.tar", + format!("/workspace/cache/{}", digest), + )?; + + build_args.insert( + "DOCKER_IMAGE_PARENT".into(), + format!("parent:latest@{}", digest), + ); + } + + log_step("Building image."); + build_image( + "/workspace/context.tar.gz", + "/workspace/image-pre.tar", + config.debug, + build_args, + )?; + log_step("Repacking image."); + repack_image( + "/workspace/image-pre.tar", + "/workspace/image.tar", + &config.image_name, + )?; + + log_step("Compressing image."); + compress_file( + "/workspace/image.tar", + "/workspace/image.tar.zst", + config.docker_image_zstd_level, + )?; + + Ok(()) +} + +fn compress_file( + source: impl AsRef, + dest: impl AsRef, + zstd_level: i32, +) -> Result<()> { + Ok(zstd::stream::copy_encode( + std::fs::File::open(source)?, + std::fs::File::create(dest)?, + zstd_level, + )?) +} diff --git a/taskcluster/docker/image-builder/build-image/src/taskcluster.rs b/taskcluster/docker/image-builder/build-image/src/taskcluster.rs new file mode 100644 index 000000000..3b39d669f --- /dev/null +++ b/taskcluster/docker/image-builder/build-image/src/taskcluster.rs @@ -0,0 +1,55 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +use anyhow::{Context, Result}; + +pub struct TaskCluster { + root_url: url::Url, + client: reqwest::blocking::Client, +} + +impl TaskCluster { + pub fn from_env() -> Result { + std::env::var("TASKCLUSTER_ROOT_URL") + .context("TASKCLUSTER_ROOT_URL not set.") + .and_then(|var| var.parse().context("Couldn't parse TASKCLUSTER_ROOT_URL.")) + .map(|root_url| TaskCluster { + root_url, + client: reqwest::blocking::Client::new(), + }) + } + + /// Return the root URL as suitable for passing to other processes. + /// + /// In particular, any trailing slashes are removed. + pub fn root_url(&self) -> String { + self.root_url.as_str().trim_end_matches("/").to_string() + } + + pub fn task_artifact_url(&self, task_id: &str, path: &str) -> url::Url { + let mut url = self.root_url.clone(); + url.set_path(&format!("api/queue/v1/task/{}/artifacts/{}", task_id, path)); + url + } + + pub fn stream_artifact(&self, task_id: &str, path: &str) -> Result { + let url = self.task_artifact_url(task_id, path); + Ok(self.client.get(url).send()?.error_for_status()?) + } +} + +#[cfg(test)] +mod test { + #[test] + fn test_url() { + let cluster = super::TaskCluster { + root_url: url::Url::parse("http://taskcluster.example").unwrap(), + client: reqwest::blocking::Client::new(), + }; + assert_eq!( + cluster.task_artifact_url("QzDLgP4YRwanIvgPt6ClfA","public/docker-contexts/decision.tar.gz"), + url::Url::parse("http://taskcluster.example/api/queue/v1/task/QzDLgP4YRwanIvgPt6ClfA/artifacts/public/docker-contexts/decision.tar.gz").unwrap(), + ); + } +} diff --git a/taskcluster/docker/image-builder/download-and-compress b/taskcluster/docker/image-builder/download-and-compress deleted file mode 100755 index 4cf7bedb9..000000000 --- a/taskcluster/docker/image-builder/download-and-compress +++ /dev/null @@ -1,85 +0,0 @@ -#!/usr/bin/python3 -u -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -import os -import sys -import time - -import requests -import requests_unixsocket -import zstandard as zstd - -# Allow requests to fetch from UNIX domain sockets. -requests_unixsocket.monkeypatch() - - -def download_and_compress(url, path, level): - r = requests.get(url, stream=True) - - if r.status_code != 200: - raise Exception('non-200 response: %d' % r.status_code) - - in_size = 0 - out_size = 0 - last_progress = time.time() - - # Use all available CPU cores for multi-threaded compression. - cctx = zstd.ZstdCompressor(threads=-1, level=level, write_checksum=True) - cobj = cctx.compressobj() - with open(path, 'wb') as fh: - for raw in r.iter_content(zstd.COMPRESSION_RECOMMENDED_INPUT_SIZE): - # Print output periodically, for humans. - now = time.time() - if now - last_progress > 5.0: - print('%d -> %d' % (in_size, out_size)) - last_progress = now - - in_size += len(raw) - chunk = cobj.compress(raw) - if not chunk: - continue - - out_size += len(chunk) - fh.write(chunk) - - chunk = cobj.flush() - out_size += len(chunk) - fh.write(chunk) - - return in_size, out_size - - -if __name__ == '__main__': - url, temp_path, final_path = sys.argv[1:] - - # Default zstd level is 3. We default to 10 because multi-threaded - # compression allows us to burn lots of CPU for significant image - # size reductions without a major wall time penalty. - level = int(os.environ.get('DOCKER_IMAGE_ZSTD_LEVEL', '10')) - print('using zstandard compression level %d' % level) - - count = 0 - while count < 10: - count += 1 - - try: - t_start = time.time() - raw_size, compress_size = download_and_compress(url, temp_path, - level) - elapsed = time.time() - t_start - # Move to final path at end so partial image isn't uploaded as - # an artifact. - os.rename(temp_path, final_path) - speed = int(raw_size / elapsed) / 1000000 - print('compression ratio: %.2f (%d -> %d) @ %d MB/s' % ( - float(compress_size) / float(raw_size), - raw_size, compress_size, speed)) - sys.exit(0) - except Exception as e: - print('exception: %s' % e) - time.sleep(5) - - print('reached maximum retry attempts; giving up') - sys.exit(1) diff --git a/taskcluster/docker/image-builder/policy.json b/taskcluster/docker/image-builder/policy.json new file mode 100644 index 000000000..c709def1e --- /dev/null +++ b/taskcluster/docker/image-builder/policy.json @@ -0,0 +1,11 @@ +{ + "default": [{"type": "reject"}], + "transports": { + "docker-archive": { + "": [{"type": "insecureAcceptAnything"}] + }, + "dir": { + "": [{"type": "insecureAcceptAnything"}] + } + } +} diff --git a/taskcluster/docker/image-builder/requirements/py2.in b/taskcluster/docker/image-builder/requirements/py2.in deleted file mode 100644 index 8f4aae712..000000000 --- a/taskcluster/docker/image-builder/requirements/py2.in +++ /dev/null @@ -1,2 +0,0 @@ -# For compressing docker images -zstandard diff --git a/taskcluster/docker/image-builder/requirements/py2.txt b/taskcluster/docker/image-builder/requirements/py2.txt deleted file mode 100644 index 9b290c99c..000000000 --- a/taskcluster/docker/image-builder/requirements/py2.txt +++ /dev/null @@ -1,67 +0,0 @@ -# -# This file is autogenerated by pip-compile -# To update, run: -# -# pip-compile --generate-hashes --output-file requirements/py2.txt requirements/py2.in -# -cffi==1.12.1 \ - --hash=sha256:0b5f895714a7a9905148fc51978c62e8a6cbcace30904d39dcd0d9e2265bb2f6 \ - --hash=sha256:27cdc7ba35ee6aa443271d11583b50815c4bb52be89a909d0028e86c21961709 \ - --hash=sha256:2d4a38049ea93d5ce3c7659210393524c1efc3efafa151bd85d196fa98fce50a \ - --hash=sha256:3262573d0d60fc6b9d0e0e6e666db0e5045cbe8a531779aa0deb3b425ec5a282 \ - --hash=sha256:358e96cfffc185ab8f6e7e425c7bb028931ed08d65402fbcf3f4e1bff6e66556 \ - --hash=sha256:37c7db824b5687fbd7ea5519acfd054c905951acc53503547c86be3db0580134 \ - --hash=sha256:39b9554dfe60f878e0c6ff8a460708db6e1b1c9cc6da2c74df2955adf83e355d \ - --hash=sha256:42b96a77acf8b2d06821600fa87c208046decc13bd22a4a0e65c5c973443e0da \ - --hash=sha256:5b37dde5035d3c219324cac0e69d96495970977f310b306fa2df5910e1f329a1 \ - --hash=sha256:5d35819f5566d0dd254f273d60cf4a2dcdd3ae3003dfd412d40b3fe8ffd87509 \ - --hash=sha256:5df73aa465e53549bd03c819c1bc69fb85529a5e1a693b7b6cb64408dd3970d1 \ - --hash=sha256:7075b361f7a4d0d4165439992d0b8a3cdfad1f302bf246ed9308a2e33b046bd3 \ - --hash=sha256:7678b5a667b0381c173abe530d7bdb0e6e3b98e062490618f04b80ca62686d96 \ - --hash=sha256:7dfd996192ff8a535458c17f22ff5eb78b83504c34d10eefac0c77b1322609e2 \ - --hash=sha256:8a3be5d31d02c60f84c4fd4c98c5e3a97b49f32e16861367f67c49425f955b28 \ - --hash=sha256:9812e53369c469506b123aee9dcb56d50c82fad60c5df87feb5ff59af5b5f55c \ - --hash=sha256:9b6f7ba4e78c52c1a291d0c0c0bd745d19adde1a9e1c03cb899f0c6efd6f8033 \ - --hash=sha256:a85bc1d7c3bba89b3d8c892bc0458de504f8b3bcca18892e6ed15b5f7a52ad9d \ - --hash=sha256:aa6b9c843ad645ebb12616de848cc4e25a40f633ccc293c3c9fe34107c02c2ea \ - --hash=sha256:bae1aa56ee00746798beafe486daa7cfb586cd395c6ce822ba3068e48d761bc0 \ - --hash=sha256:bae96e26510e4825d5910a196bf6b5a11a18b87d9278db6d08413be8ea799469 \ - --hash=sha256:bd78df3b594013b227bf31d0301566dc50ba6f40df38a70ded731d5a8f2cb071 \ - --hash=sha256:c2711197154f46d06f73542c539a0ff5411f1951fab391e0a4ac8359badef719 \ - --hash=sha256:d998c20e3deed234fca993fd6c8314cb7cbfda05fd170f1bd75bb5d7421c3c5a \ - --hash=sha256:df4f840d77d9e37136f8e6b432fecc9d6b8730f18f896e90628712c793466ce6 \ - --hash=sha256:f5653c2581acb038319e6705d4e3593677676df14b112f13e0b5b44b6a18df1a \ - --hash=sha256:f7c7aa485a2e2250d455148470ffd0195eecc3d845122635202d7467d6f7b4cf \ - --hash=sha256:f9e2c66a6493147de835f207f198540a56b26745ce4f272fbc7c2f2cfebeb729 \ - # via zstandard -pycparser==2.19 \ - --hash=sha256:a988718abfad80b6b157acce7bf130a30876d27603738ac39f140993246b25b3 \ - # via cffi -zstandard==0.11.1 \ - --hash=sha256:19f5ad81590acd20dbdfb930b87a035189778662fdc67ab8cbcc106269ed1be8 \ - --hash=sha256:1a1db0c9774181e806a418c32d511aa085c7e2c28c257a58f6c107f5decb3109 \ - --hash=sha256:22d7aa898f36f78108cc1ef0c8da8225f0add518441d815ad4fdd1d577378209 \ - --hash=sha256:357873afdd7cd0e653d169c36ce837ce2b3e5926dd4a5c0f0476c813f6765373 \ - --hash=sha256:3c31da5d78a7b07e722e8a3e0b1295bc9b316b7e90a1666659c451a42750ffe4 \ - --hash=sha256:3f76562ec63fabc6f4b5be0cd986f911c97105c35c31b4d655b90c4d2fe07f40 \ - --hash=sha256:42fa4462e0563fe17e73dfeb95eef9b00429b86282f8f6ca0e2765b1855a8324 \ - --hash=sha256:51aad01a5709ca6f45768c69ffd4c887528e5ad9e09302426b735560752c4e82 \ - --hash=sha256:6cd81819a02e57e38e27c53c5c0a7015e059b0e148a18bf27b46b4f808840879 \ - --hash=sha256:717fd2494f222164396e03d08ef57174d2a889920b81ca49f276caf9381e6405 \ - --hash=sha256:71c8711458212c973a9b719275db8111f22803e0caf675affde50703b96e9be1 \ - --hash=sha256:76a331b5a6258fce3906551557db9be83bdd89a62f66f509a55a4a307239c782 \ - --hash=sha256:7c92dfcdf7e0c540f9718b40b4c54516a968ef6b81567b75df81866a1af2189d \ - --hash=sha256:7f3db21223a8bb4ffcf6c36b9c20d38278967723b47fce249dcb6ec6d4082b83 \ - --hash=sha256:7fa9deba4c904e76870e08324adff94ec3a4bc56a50bbe1a9f859a4aed11c0d2 \ - --hash=sha256:88912cbcf68cc40037c113460a166ebfbbb24864ceebb89ad221ea346f22e995 \ - --hash=sha256:94aa5bb817f1c747b21214f6ef83a022bcb63bf81e4dae2954768165c13a510b \ - --hash=sha256:951e382a2ea47179ecb3e314e8c70f2e5189e3652ccbbcb71c6443dd71bc20fc \ - --hash=sha256:978a500ae1184f602dc902977ec208c7cf02c10caae9c159b10976a7cb29f879 \ - --hash=sha256:991c4a40171d87854b219cdf2ba56c1c34b3b3a8ebe5d1ab63bd357ff71271b2 \ - --hash=sha256:9ca84187182743d2e6bbf9d3f79d3834db205cddc98add27ad20f2189d080a60 \ - --hash=sha256:ae50bc839cf1ff549f55a3e55922563f246fb692f77497175a8d8d4cddc294da \ - --hash=sha256:b7abae5b17e82d5f78aaa641077b4619c6ad204e30c6f3445d422acff5f35d3e \ - --hash=sha256:b8fce0c961654f77c81a6ae1f2cd40633b41ef16a12ae02f0382ed6692f9bb90 \ - --hash=sha256:d8f047d3647a5cd1b77b4580f35208c938da00c101a092571c85bcefaa2d725d \ - --hash=sha256:f1785b31bf428e964a9670dd4f721023f2741ef7fd67c663bf01e3d4d3f9ec2a \ - --hash=sha256:fcf70e1e9d38035a15482e954ba064f3b701cf84cfe571576d15af93ac2a2fb1 diff --git a/taskcluster/docker/image-builder/requirements/py3.in b/taskcluster/docker/image-builder/requirements/py3.in deleted file mode 100644 index 75aeac2c7..000000000 --- a/taskcluster/docker/image-builder/requirements/py3.in +++ /dev/null @@ -1,8 +0,0 @@ -# The docker module removed support for docker engines < 1.21 starting with -# docker 3.0. Once we upgrade the docker running on our workers, we can remove -# this restriction here. -docker<3.0 -docker-squash - -# For compressing docker images -zstandard diff --git a/taskcluster/docker/image-builder/requirements/py3.txt b/taskcluster/docker/image-builder/requirements/py3.txt deleted file mode 100644 index 8389ac886..000000000 --- a/taskcluster/docker/image-builder/requirements/py3.txt +++ /dev/null @@ -1,104 +0,0 @@ -# -# This file is autogenerated by pip-compile -# To update, run: -# -# pip-compile --generate-hashes --output-file requirements/py3.txt requirements/py3.in -# -certifi==2018.11.29 \ - --hash=sha256:47f9c83ef4c0c621eaef743f133f09fa8a74a9b75f037e8624f83bd1b6626cb7 \ - --hash=sha256:993f830721089fef441cdfeb4b2c8c9df86f0c63239f06bd025a76a7daddb033 \ - # via requests -cffi==1.12.1 \ - --hash=sha256:0b5f895714a7a9905148fc51978c62e8a6cbcace30904d39dcd0d9e2265bb2f6 \ - --hash=sha256:27cdc7ba35ee6aa443271d11583b50815c4bb52be89a909d0028e86c21961709 \ - --hash=sha256:2d4a38049ea93d5ce3c7659210393524c1efc3efafa151bd85d196fa98fce50a \ - --hash=sha256:3262573d0d60fc6b9d0e0e6e666db0e5045cbe8a531779aa0deb3b425ec5a282 \ - --hash=sha256:358e96cfffc185ab8f6e7e425c7bb028931ed08d65402fbcf3f4e1bff6e66556 \ - --hash=sha256:37c7db824b5687fbd7ea5519acfd054c905951acc53503547c86be3db0580134 \ - --hash=sha256:39b9554dfe60f878e0c6ff8a460708db6e1b1c9cc6da2c74df2955adf83e355d \ - --hash=sha256:42b96a77acf8b2d06821600fa87c208046decc13bd22a4a0e65c5c973443e0da \ - --hash=sha256:5b37dde5035d3c219324cac0e69d96495970977f310b306fa2df5910e1f329a1 \ - --hash=sha256:5d35819f5566d0dd254f273d60cf4a2dcdd3ae3003dfd412d40b3fe8ffd87509 \ - --hash=sha256:5df73aa465e53549bd03c819c1bc69fb85529a5e1a693b7b6cb64408dd3970d1 \ - --hash=sha256:7075b361f7a4d0d4165439992d0b8a3cdfad1f302bf246ed9308a2e33b046bd3 \ - --hash=sha256:7678b5a667b0381c173abe530d7bdb0e6e3b98e062490618f04b80ca62686d96 \ - --hash=sha256:7dfd996192ff8a535458c17f22ff5eb78b83504c34d10eefac0c77b1322609e2 \ - --hash=sha256:8a3be5d31d02c60f84c4fd4c98c5e3a97b49f32e16861367f67c49425f955b28 \ - --hash=sha256:9812e53369c469506b123aee9dcb56d50c82fad60c5df87feb5ff59af5b5f55c \ - --hash=sha256:9b6f7ba4e78c52c1a291d0c0c0bd745d19adde1a9e1c03cb899f0c6efd6f8033 \ - --hash=sha256:a85bc1d7c3bba89b3d8c892bc0458de504f8b3bcca18892e6ed15b5f7a52ad9d \ - --hash=sha256:aa6b9c843ad645ebb12616de848cc4e25a40f633ccc293c3c9fe34107c02c2ea \ - --hash=sha256:bae1aa56ee00746798beafe486daa7cfb586cd395c6ce822ba3068e48d761bc0 \ - --hash=sha256:bae96e26510e4825d5910a196bf6b5a11a18b87d9278db6d08413be8ea799469 \ - --hash=sha256:bd78df3b594013b227bf31d0301566dc50ba6f40df38a70ded731d5a8f2cb071 \ - --hash=sha256:c2711197154f46d06f73542c539a0ff5411f1951fab391e0a4ac8359badef719 \ - --hash=sha256:d998c20e3deed234fca993fd6c8314cb7cbfda05fd170f1bd75bb5d7421c3c5a \ - --hash=sha256:df4f840d77d9e37136f8e6b432fecc9d6b8730f18f896e90628712c793466ce6 \ - --hash=sha256:f5653c2581acb038319e6705d4e3593677676df14b112f13e0b5b44b6a18df1a \ - --hash=sha256:f7c7aa485a2e2250d455148470ffd0195eecc3d845122635202d7467d6f7b4cf \ - --hash=sha256:f9e2c66a6493147de835f207f198540a56b26745ce4f272fbc7c2f2cfebeb729 \ - # via zstandard -chardet==3.0.4 \ - --hash=sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae \ - --hash=sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691 \ - # via requests -docker-pycreds==0.4.0 \ - --hash=sha256:6ce3270bcaf404cc4c3e27e4b6c70d3521deae82fb508767870fdbf772d584d4 \ - --hash=sha256:7266112468627868005106ec19cd0d722702d2b7d5912a28e19b826c3d37af49 \ - # via docker -docker-squash==1.0.7 \ - --hash=sha256:95ca24fbeffa915c3d467b7ad538c6437a02f68aaa4e31e16c451c47a30a2169 -docker==2.7.0 \ - --hash=sha256:144248308e8ea31c4863c6d74e1b55daf97cc190b61d0fe7b7313ab920d6a76c \ - --hash=sha256:c1d4e37b1ea03b2b6efdd0379640f6ea372fefe56efa65d4d17c34c6b9d54558 -idna==2.8 \ - --hash=sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407 \ - --hash=sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c \ - # via requests -pycparser==2.19 \ - --hash=sha256:a988718abfad80b6b157acce7bf130a30876d27603738ac39f140993246b25b3 \ - # via cffi -requests==2.21.0 \ - --hash=sha256:502a824f31acdacb3a35b6690b5fbf0bc41d63a24a45c4004352b0242707598e \ - --hash=sha256:7bf2a778576d825600030a110f3c0e3e8edc51dfaafe1c146e39a2027784957b \ - # via docker -six==1.12.0 \ - --hash=sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c \ - --hash=sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73 \ - # via docker, docker-pycreds, docker-squash, websocket-client -urllib3==1.24.1 \ - --hash=sha256:61bf29cada3fc2fbefad4fdf059ea4bd1b4a86d2b6d15e1c7c0b582b9752fe39 \ - --hash=sha256:de9529817c93f27c8ccbfead6985011db27bd0ddfcdb2d86f3f663385c6a9c22 \ - # via requests -websocket-client==0.54.0 \ - --hash=sha256:8c8bf2d4f800c3ed952df206b18c28f7070d9e3dcbd6ca6291127574f57ee786 \ - --hash=sha256:e51562c91ddb8148e791f0155fdb01325d99bb52c4cdbb291aee7a3563fd0849 \ - # via docker -zstandard==0.11.1 \ - --hash=sha256:19f5ad81590acd20dbdfb930b87a035189778662fdc67ab8cbcc106269ed1be8 \ - --hash=sha256:1a1db0c9774181e806a418c32d511aa085c7e2c28c257a58f6c107f5decb3109 \ - --hash=sha256:22d7aa898f36f78108cc1ef0c8da8225f0add518441d815ad4fdd1d577378209 \ - --hash=sha256:357873afdd7cd0e653d169c36ce837ce2b3e5926dd4a5c0f0476c813f6765373 \ - --hash=sha256:3c31da5d78a7b07e722e8a3e0b1295bc9b316b7e90a1666659c451a42750ffe4 \ - --hash=sha256:3f76562ec63fabc6f4b5be0cd986f911c97105c35c31b4d655b90c4d2fe07f40 \ - --hash=sha256:42fa4462e0563fe17e73dfeb95eef9b00429b86282f8f6ca0e2765b1855a8324 \ - --hash=sha256:51aad01a5709ca6f45768c69ffd4c887528e5ad9e09302426b735560752c4e82 \ - --hash=sha256:6cd81819a02e57e38e27c53c5c0a7015e059b0e148a18bf27b46b4f808840879 \ - --hash=sha256:717fd2494f222164396e03d08ef57174d2a889920b81ca49f276caf9381e6405 \ - --hash=sha256:71c8711458212c973a9b719275db8111f22803e0caf675affde50703b96e9be1 \ - --hash=sha256:76a331b5a6258fce3906551557db9be83bdd89a62f66f509a55a4a307239c782 \ - --hash=sha256:7c92dfcdf7e0c540f9718b40b4c54516a968ef6b81567b75df81866a1af2189d \ - --hash=sha256:7f3db21223a8bb4ffcf6c36b9c20d38278967723b47fce249dcb6ec6d4082b83 \ - --hash=sha256:7fa9deba4c904e76870e08324adff94ec3a4bc56a50bbe1a9f859a4aed11c0d2 \ - --hash=sha256:88912cbcf68cc40037c113460a166ebfbbb24864ceebb89ad221ea346f22e995 \ - --hash=sha256:94aa5bb817f1c747b21214f6ef83a022bcb63bf81e4dae2954768165c13a510b \ - --hash=sha256:951e382a2ea47179ecb3e314e8c70f2e5189e3652ccbbcb71c6443dd71bc20fc \ - --hash=sha256:978a500ae1184f602dc902977ec208c7cf02c10caae9c159b10976a7cb29f879 \ - --hash=sha256:991c4a40171d87854b219cdf2ba56c1c34b3b3a8ebe5d1ab63bd357ff71271b2 \ - --hash=sha256:9ca84187182743d2e6bbf9d3f79d3834db205cddc98add27ad20f2189d080a60 \ - --hash=sha256:ae50bc839cf1ff549f55a3e55922563f246fb692f77497175a8d8d4cddc294da \ - --hash=sha256:b7abae5b17e82d5f78aaa641077b4619c6ad204e30c6f3445d422acff5f35d3e \ - --hash=sha256:b8fce0c961654f77c81a6ae1f2cd40633b41ef16a12ae02f0382ed6692f9bb90 \ - --hash=sha256:d8f047d3647a5cd1b77b4580f35208c938da00c101a092571c85bcefaa2d725d \ - --hash=sha256:f1785b31bf428e964a9670dd4f721023f2741ef7fd67c663bf01e3d4d3f9ec2a \ - --hash=sha256:fcf70e1e9d38035a15482e954ba064f3b701cf84cfe571576d15af93ac2a2fb1 diff --git a/taskcluster/docker/image-builder/setup.sh b/taskcluster/docker/image-builder/setup.sh deleted file mode 100644 index 2aa0c70b2..000000000 --- a/taskcluster/docker/image-builder/setup.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/bin/bash -vex -set -v -e -x - -export DEBIAN_FRONTEND=noninteractive - -# Update apt-get lists -apt-get update -y - -# Install dependencies -apt-get install -y --no-install-recommends \ - socat \ - python-requests \ - python-requests-unixsocket \ - python3.5 \ - python3-minimal \ - python3-requests \ - python3-requests-unixsocket - -# Extra dependencies only needed for image building. Will be removed at -# end of script. -apt-get install -y python-pip python3-pip - -# Install mercurial -# shellcheck disable=SC1091 -. /setup/common.sh -# shellcheck disable=SC1091 -. /setup/install-mercurial.sh - -# Install build-image.sh script -chmod +x /usr/local/bin/build-image.sh -chmod +x /usr/local/bin/run-task -chmod +x /usr/local/bin/download-and-compress - -# Create workspace -mkdir -p /builds/worker/workspace - -# We need to install for both Python 2 and 3 because `mach taskcluster-load-image` -# uses Python 2 and `download-and-compress` uses Python 3. -# We also need to make sure to explicitly install python3-distutils so that it doesn't get purged later -apt-get install -y python3-distutils -/usr/bin/pip -v install -r /setup/requirements-py2.txt -/usr/bin/pip3 -v install -r /setup/requirements-py3.txt - -# python-pip only needed to install python-zstandard. Removing it removes -# several hundred MB of dependencies from the image. -apt-get purge -y python-pip python3-pip - -# Purge apt-get caches to minimize image size -apt-get auto-remove -y -apt-get clean -y -rm -rf /var/lib/apt/lists/ - -# Remove this script -rm -rf /setup/ From ef30b99f503a46a871d3ac6a6e320f4e7ae14313 Mon Sep 17 00:00:00 2001 From: Mike Hommey Date: Thu, 9 Jul 2020 03:54:46 +0000 Subject: [PATCH 67/74] Bug 1651227 - Use the in-tree kaniko image builder image to build docker images. r=tomprince Differential Revision: https://phabricator.services.mozilla.com/D82622 --- taskcluster/docker/image-builder/Dockerfile | 29 +++++++++---------- .../image-builder/build-image/Cargo.lock | 7 +++++ .../image-builder/build-image/Cargo.toml | 1 + .../image-builder/build-image/src/main.rs | 13 +++++++++ 4 files changed, 34 insertions(+), 16 deletions(-) diff --git a/taskcluster/docker/image-builder/Dockerfile b/taskcluster/docker/image-builder/Dockerfile index b5804ce95..3058619a2 100644 --- a/taskcluster/docker/image-builder/Dockerfile +++ b/taskcluster/docker/image-builder/Dockerfile @@ -5,10 +5,10 @@ FROM golang:1.14 as skopeo WORKDIR /go/src/ -RUN ["git", "clone", "--no-checkout", "--depth=1", "--branch=v1.1.0", "https://github.com/containers/skopeo", "."] -RUN ["git", "checkout", "63085f5bef1131aa9ec0907a5c8d66b67de7c4b2"] +RUN ["/usr/bin/git", "clone", "--no-checkout", "--depth=1", "--branch=v1.1.0", "https://github.com/containers/skopeo", "."] +RUN ["/usr/bin/git", "checkout", "63085f5bef1131aa9ec0907a5c8d66b67de7c4b2"] ENV GO111MODULE=on CGO_ENABLED=0 -RUN ["go", "build", \ +RUN ["/usr/local/go/bin/go", "build", \ "-mod=vendor", "-o", "out/skopeo", \ "-tags", "exclude_graphdriver_devicemapper exclude_graphdriver_btrfs containers_image_openpgp", \ # Set unixTempDirForBigFiles so skopeo will extract in a directory hidden by kaniko @@ -18,9 +18,9 @@ RUN ["go", "build", \ FROM golang:1.14 as kaniko WORKDIR /go/src/ -RUN ["git", "clone", "--no-checkout", "--depth=1", "--branch=v0.24.0", "https://github.com/GoogleContainerTools/kaniko", "."] -RUN ["git", "checkout", "cdbd8af0578c56e2801b57461e9f417f9479d303"] -RUN ["make"] +RUN ["/usr/bin/git", "clone", "--no-checkout", "--depth=1", "--branch=v0.24.0", "https://github.com/GoogleContainerTools/kaniko", "."] +RUN ["/usr/bin/git", "checkout", "cdbd8af0578c56e2801b57461e9f417f9479d303"] +RUN ["/usr/bin/make"] # Build the `build-image` command as a static binary using musl # The setup is loosely based on a stripped down version of @@ -72,22 +72,22 @@ ADD --chown=rust:rust build-image/ ./ # --out-dir is not yet stable ENV RUSTC_BOOTSTRAP=1 # Build our application. -RUN ["cargo", "build", "--target", "x86_64-unknown-linux-musl", "--out-dir=bin", "--release", "-Zunstable-options"] +RUN ["/home/rust/.cargo/bin/cargo", "build", "--target", "x86_64-unknown-linux-musl", "--out-dir=bin", "--release", "-Zunstable-options"] FROM scratch as empty FROM scratch -COPY --from=skopeo /go/src/out/skopeo /kaniko/skopeo -COPY --from=kaniko /go/src/out/executor /kaniko/executor +COPY --from=skopeo /go/src/out/skopeo /kaniko-bootstrap/skopeo +COPY --from=kaniko /go/src/out/executor /kaniko-bootstrap/executor COPY --from=build-image \ /home/rust/src/bin/build-image \ - /kaniko/build-image + /kaniko-bootstrap/build-image -ADD https://mkcert.org/generate/ /kaniko/ssl/certs/ca-certificats.crt +ADD https://mkcert.org/generate/ /kaniko-bootstrap/ssl/certs/ca-certificats.crt ENV SSL_CERT_DIR=/kaniko/ssl/certs -ADD policy.json /kaniko/containers/policy.json +ADD policy.json /kaniko-bootstrap/containers/policy.json ENV HOME /root ENV USER /root @@ -96,7 +96,4 @@ WORKDIR /workspace ENV PATH /usr/local/bin:/kaniko VOLUME /workspace -# Create an empty temporary directory for skopeo -COPY --from=empty / /workspace/tmp -COPY --from=empty / /workspace/cache -ENTRYPOINT ["/kaniko/build-image"] +ENTRYPOINT ["/kaniko-bootstrap/build-image"] diff --git a/taskcluster/docker/image-builder/build-image/Cargo.lock b/taskcluster/docker/image-builder/build-image/Cargo.lock index f010a003c..7e0792059 100644 --- a/taskcluster/docker/image-builder/build-image/Cargo.lock +++ b/taskcluster/docker/image-builder/build-image/Cargo.lock @@ -31,6 +31,7 @@ version = "0.1.0" dependencies = [ "anyhow 1.0.31 (registry+https://github.com/rust-lang/crates.io-index)", "envy 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", + "fs_extra 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "reqwest 0.10.6 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.114 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 1.0.56 (registry+https://github.com/rust-lang/crates.io-index)", @@ -95,6 +96,11 @@ name = "fnv" version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "fs_extra" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "fuchsia-zircon" version = "0.3.3" @@ -976,6 +982,7 @@ dependencies = [ "checksum encoding_rs 0.8.23 (registry+https://github.com/rust-lang/crates.io-index)" = "e8ac63f94732332f44fe654443c46f6375d1939684c17b0afb6cb56b0456e171" "checksum envy 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f938a4abd5b75fe3737902dbc2e79ca142cc1526827a9e40b829a086758531a9" "checksum fnv 1.0.7 (registry+https://github.com/rust-lang/crates.io-index)" = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +"checksum fs_extra 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5f2a4a2034423744d2cc7ca2068453168dcdb82c438419e639a26bd87839c674" "checksum fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82" "checksum fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" "checksum futures-channel 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "f366ad74c28cca6ba456d95e6422883cfb4b252a83bed929c83abfdbbf2967d5" diff --git a/taskcluster/docker/image-builder/build-image/Cargo.toml b/taskcluster/docker/image-builder/build-image/Cargo.toml index 5998412fc..be42fe1e3 100644 --- a/taskcluster/docker/image-builder/build-image/Cargo.toml +++ b/taskcluster/docker/image-builder/build-image/Cargo.toml @@ -18,5 +18,6 @@ anyhow = "1.0.27" serde = { version = "1.0.105", features = ["derive"]} serde_json = "1.0.50" envy = "0.4.1" +fs_extra = "1.1" [workspace] diff --git a/taskcluster/docker/image-builder/build-image/src/main.rs b/taskcluster/docker/image-builder/build-image/src/main.rs index 85a0f9562..559b828b0 100644 --- a/taskcluster/docker/image-builder/build-image/src/main.rs +++ b/taskcluster/docker/image-builder/build-image/src/main.rs @@ -5,9 +5,11 @@ #![forbid(unsafe_code)] use std::collections::HashMap; +use std::path::Path; use std::process::Command; use anyhow::{ensure, Context, Result}; +use fs_extra::dir::{CopyOptions, move_dir}; use serde::Deserialize; mod config; @@ -100,6 +102,16 @@ fn repack_image(source: &str, dest: &str, image_name: &str) -> Result<()> { } fn main() -> Result<()> { + // Kaniko expects everything to be in /kaniko, so if not running from there, move + // everything there. + if let Some(path) = std::env::current_exe()?.parent() { + if path != Path::new("/kaniko") { + let mut options = CopyOptions::new(); + options.copy_inside = true; + move_dir(path, "/kaniko", &options)?; + } + } + let config = Config::from_env().context("Could not parse environment variables.")?; let cluster = taskcluster::TaskCluster::from_env()?; @@ -121,6 +133,7 @@ fn main() -> Result<()> { let digest = download_parent_image(&cluster, &parent_task_id, "/workspace/parent.tar")?; log_step(&format!("Parent image digest {}", &digest)); + std::fs::create_dir_all("/workspace/cache")?; std::fs::rename( "/workspace/parent.tar", format!("/workspace/cache/{}", digest), From d7b0077da9a28b9ffe09cf9c7615b439421a1cb8 Mon Sep 17 00:00:00 2001 From: Sylvestre Ledru Date: Sun, 12 Jul 2020 17:01:13 +0000 Subject: [PATCH 68/74] Bug 1617369 - Reformat recent rust changes with rustfmt r=emilio # ignore-this-changeset Differential Revision: https://phabricator.services.mozilla.com/D83259 --- taskcluster/docker/image-builder/build-image/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/taskcluster/docker/image-builder/build-image/src/main.rs b/taskcluster/docker/image-builder/build-image/src/main.rs index 559b828b0..997617c84 100644 --- a/taskcluster/docker/image-builder/build-image/src/main.rs +++ b/taskcluster/docker/image-builder/build-image/src/main.rs @@ -9,7 +9,7 @@ use std::path::Path; use std::process::Command; use anyhow::{ensure, Context, Result}; -use fs_extra::dir::{CopyOptions, move_dir}; +use fs_extra::dir::{move_dir, CopyOptions}; use serde::Deserialize; mod config; From 04026e16ce394b9fd13ecdab4c05bee4f5e97fb0 Mon Sep 17 00:00:00 2001 From: Mike Hommey Date: Tue, 25 Aug 2020 20:25:36 +0000 Subject: [PATCH 69/74] Bug 1660964 - Upgrade skopeo to 1.1.1 and kaniko to 1.0.0. r=aki Differential Revision: https://phabricator.services.mozilla.com/D88116 --- taskcluster/docker/image-builder/Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/taskcluster/docker/image-builder/Dockerfile b/taskcluster/docker/image-builder/Dockerfile index 3058619a2..6ac2f81e1 100644 --- a/taskcluster/docker/image-builder/Dockerfile +++ b/taskcluster/docker/image-builder/Dockerfile @@ -5,8 +5,8 @@ FROM golang:1.14 as skopeo WORKDIR /go/src/ -RUN ["/usr/bin/git", "clone", "--no-checkout", "--depth=1", "--branch=v1.1.0", "https://github.com/containers/skopeo", "."] -RUN ["/usr/bin/git", "checkout", "63085f5bef1131aa9ec0907a5c8d66b67de7c4b2"] +RUN ["/usr/bin/git", "clone", "--no-checkout", "--depth=1", "--branch=v1.1.1", "https://github.com/containers/skopeo", "."] +RUN ["/usr/bin/git", "checkout", "67abbb3cefbdc876447583d5ea45e76bf441eba7"] ENV GO111MODULE=on CGO_ENABLED=0 RUN ["/usr/local/go/bin/go", "build", \ "-mod=vendor", "-o", "out/skopeo", \ @@ -18,8 +18,8 @@ RUN ["/usr/local/go/bin/go", "build", \ FROM golang:1.14 as kaniko WORKDIR /go/src/ -RUN ["/usr/bin/git", "clone", "--no-checkout", "--depth=1", "--branch=v0.24.0", "https://github.com/GoogleContainerTools/kaniko", "."] -RUN ["/usr/bin/git", "checkout", "cdbd8af0578c56e2801b57461e9f417f9479d303"] +RUN ["/usr/bin/git", "clone", "--no-checkout", "--depth=1", "--branch=v1.0.0", "https://github.com/GoogleContainerTools/kaniko", "."] +RUN ["/usr/bin/git", "checkout", "146ec6a9cd6f87b4a12e8119ded575d5edca35ac"] RUN ["/usr/bin/make"] # Build the `build-image` command as a static binary using musl From 772b5cdc759edd8ea8e39f02071c5274ced88c15 Mon Sep 17 00:00:00 2001 From: Razvan Maries Date: Wed, 26 Aug 2020 07:47:14 +0300 Subject: [PATCH 70/74] Backed out changeset a716e6202487 (bug 1660964) for Gecko Decision Task bustage. CLOSED TREE --- taskcluster/docker/image-builder/Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/taskcluster/docker/image-builder/Dockerfile b/taskcluster/docker/image-builder/Dockerfile index 6ac2f81e1..3058619a2 100644 --- a/taskcluster/docker/image-builder/Dockerfile +++ b/taskcluster/docker/image-builder/Dockerfile @@ -5,8 +5,8 @@ FROM golang:1.14 as skopeo WORKDIR /go/src/ -RUN ["/usr/bin/git", "clone", "--no-checkout", "--depth=1", "--branch=v1.1.1", "https://github.com/containers/skopeo", "."] -RUN ["/usr/bin/git", "checkout", "67abbb3cefbdc876447583d5ea45e76bf441eba7"] +RUN ["/usr/bin/git", "clone", "--no-checkout", "--depth=1", "--branch=v1.1.0", "https://github.com/containers/skopeo", "."] +RUN ["/usr/bin/git", "checkout", "63085f5bef1131aa9ec0907a5c8d66b67de7c4b2"] ENV GO111MODULE=on CGO_ENABLED=0 RUN ["/usr/local/go/bin/go", "build", \ "-mod=vendor", "-o", "out/skopeo", \ @@ -18,8 +18,8 @@ RUN ["/usr/local/go/bin/go", "build", \ FROM golang:1.14 as kaniko WORKDIR /go/src/ -RUN ["/usr/bin/git", "clone", "--no-checkout", "--depth=1", "--branch=v1.0.0", "https://github.com/GoogleContainerTools/kaniko", "."] -RUN ["/usr/bin/git", "checkout", "146ec6a9cd6f87b4a12e8119ded575d5edca35ac"] +RUN ["/usr/bin/git", "clone", "--no-checkout", "--depth=1", "--branch=v0.24.0", "https://github.com/GoogleContainerTools/kaniko", "."] +RUN ["/usr/bin/git", "checkout", "cdbd8af0578c56e2801b57461e9f417f9479d303"] RUN ["/usr/bin/make"] # Build the `build-image` command as a static binary using musl From 7eac121c1405c31cfb882d8dce8bd2ddd0bcf8f6 Mon Sep 17 00:00:00 2001 From: Mike Hommey Date: Wed, 26 Aug 2020 04:53:42 +0000 Subject: [PATCH 71/74] Bug 1660964 - Upgrade skopeo to 1.1.1 and kaniko to 1.0.0. r=aki Differential Revision: https://phabricator.services.mozilla.com/D88116 --- taskcluster/docker/image-builder/Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/taskcluster/docker/image-builder/Dockerfile b/taskcluster/docker/image-builder/Dockerfile index 3058619a2..6ac2f81e1 100644 --- a/taskcluster/docker/image-builder/Dockerfile +++ b/taskcluster/docker/image-builder/Dockerfile @@ -5,8 +5,8 @@ FROM golang:1.14 as skopeo WORKDIR /go/src/ -RUN ["/usr/bin/git", "clone", "--no-checkout", "--depth=1", "--branch=v1.1.0", "https://github.com/containers/skopeo", "."] -RUN ["/usr/bin/git", "checkout", "63085f5bef1131aa9ec0907a5c8d66b67de7c4b2"] +RUN ["/usr/bin/git", "clone", "--no-checkout", "--depth=1", "--branch=v1.1.1", "https://github.com/containers/skopeo", "."] +RUN ["/usr/bin/git", "checkout", "67abbb3cefbdc876447583d5ea45e76bf441eba7"] ENV GO111MODULE=on CGO_ENABLED=0 RUN ["/usr/local/go/bin/go", "build", \ "-mod=vendor", "-o", "out/skopeo", \ @@ -18,8 +18,8 @@ RUN ["/usr/local/go/bin/go", "build", \ FROM golang:1.14 as kaniko WORKDIR /go/src/ -RUN ["/usr/bin/git", "clone", "--no-checkout", "--depth=1", "--branch=v0.24.0", "https://github.com/GoogleContainerTools/kaniko", "."] -RUN ["/usr/bin/git", "checkout", "cdbd8af0578c56e2801b57461e9f417f9479d303"] +RUN ["/usr/bin/git", "clone", "--no-checkout", "--depth=1", "--branch=v1.0.0", "https://github.com/GoogleContainerTools/kaniko", "."] +RUN ["/usr/bin/git", "checkout", "146ec6a9cd6f87b4a12e8119ded575d5edca35ac"] RUN ["/usr/bin/make"] # Build the `build-image` command as a static binary using musl From aee764c02d64e8a3b3a167cafb51150d28c8bb88 Mon Sep 17 00:00:00 2001 From: Mike Hommey Date: Tue, 1 Sep 2020 00:35:22 +0000 Subject: [PATCH 72/74] Bug 1662305 - Remove workaround for shortcomings from kaniko < 1. r=taskgraph-reviewers,aki Older versions of kaniko didn't handle $PATH correctly in `RUN` commands, and we worked around this by using full paths for the executables. Now that the base image builder is upgraded to kaniko 1, we can remove those workarounds. Differential Revision: https://phabricator.services.mozilla.com/D88932 --- taskcluster/docker/image-builder/Dockerfile | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/taskcluster/docker/image-builder/Dockerfile b/taskcluster/docker/image-builder/Dockerfile index 6ac2f81e1..ad38a48c4 100644 --- a/taskcluster/docker/image-builder/Dockerfile +++ b/taskcluster/docker/image-builder/Dockerfile @@ -5,10 +5,10 @@ FROM golang:1.14 as skopeo WORKDIR /go/src/ -RUN ["/usr/bin/git", "clone", "--no-checkout", "--depth=1", "--branch=v1.1.1", "https://github.com/containers/skopeo", "."] -RUN ["/usr/bin/git", "checkout", "67abbb3cefbdc876447583d5ea45e76bf441eba7"] +RUN ["git", "clone", "--no-checkout", "--depth=1", "--branch=v1.1.1", "https://github.com/containers/skopeo", "."] +RUN ["git", "checkout", "67abbb3cefbdc876447583d5ea45e76bf441eba7"] ENV GO111MODULE=on CGO_ENABLED=0 -RUN ["/usr/local/go/bin/go", "build", \ +RUN ["go", "build", \ "-mod=vendor", "-o", "out/skopeo", \ "-tags", "exclude_graphdriver_devicemapper exclude_graphdriver_btrfs containers_image_openpgp", \ # Set unixTempDirForBigFiles so skopeo will extract in a directory hidden by kaniko @@ -18,9 +18,9 @@ RUN ["/usr/local/go/bin/go", "build", \ FROM golang:1.14 as kaniko WORKDIR /go/src/ -RUN ["/usr/bin/git", "clone", "--no-checkout", "--depth=1", "--branch=v1.0.0", "https://github.com/GoogleContainerTools/kaniko", "."] -RUN ["/usr/bin/git", "checkout", "146ec6a9cd6f87b4a12e8119ded575d5edca35ac"] -RUN ["/usr/bin/make"] +RUN ["git", "clone", "--no-checkout", "--depth=1", "--branch=v1.0.0", "https://github.com/GoogleContainerTools/kaniko", "."] +RUN ["git", "checkout", "146ec6a9cd6f87b4a12e8119ded575d5edca35ac"] +RUN ["make"] # Build the `build-image` command as a static binary using musl # The setup is loosely based on a stripped down version of @@ -72,7 +72,7 @@ ADD --chown=rust:rust build-image/ ./ # --out-dir is not yet stable ENV RUSTC_BOOTSTRAP=1 # Build our application. -RUN ["/home/rust/.cargo/bin/cargo", "build", "--target", "x86_64-unknown-linux-musl", "--out-dir=bin", "--release", "-Zunstable-options"] +RUN ["cargo", "build", "--target", "x86_64-unknown-linux-musl", "--out-dir=bin", "--release", "-Zunstable-options"] FROM scratch as empty From 91ccdcef085f13010bc4144be4fb3a886c83036f Mon Sep 17 00:00:00 2001 From: Mark Banner Date: Tue, 9 May 2023 16:14:51 +0000 Subject: [PATCH 73/74] Bug 1829618 - Enable Prettier on json files (automatic fixes). r=perftest-reviewers,webcompat-reviewers,extension-reviewers,andi,desktop-theme-reviewers,pip-reviewers,devtools-reviewers,sync-reviewers,mossop,denschub,dao,sparky,robwu,geckoview-reviewers,owlish Differential Revision: https://phabricator.services.mozilla.com/D176336 --- taskcluster/docker/image-builder/policy.json | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/taskcluster/docker/image-builder/policy.json b/taskcluster/docker/image-builder/policy.json index c709def1e..c9a9b225c 100644 --- a/taskcluster/docker/image-builder/policy.json +++ b/taskcluster/docker/image-builder/policy.json @@ -1,11 +1,11 @@ { - "default": [{"type": "reject"}], - "transports": { - "docker-archive": { - "": [{"type": "insecureAcceptAnything"}] - }, - "dir": { - "": [{"type": "insecureAcceptAnything"}] - } + "default": [{ "type": "reject" }], + "transports": { + "docker-archive": { + "": [{ "type": "insecureAcceptAnything" }] + }, + "dir": { + "": [{ "type": "insecureAcceptAnything" }] } + } } From 4227f27b4d69aa9dabc9ed72502a37f306302674 Mon Sep 17 00:00:00 2001 From: Andrew Halberstadt Date: Wed, 15 May 2024 12:14:25 -0400 Subject: [PATCH 74/74] ci: add 'docker-image' task for 'image-builder' It might be a little odd to use the `image_builder` image to build the `image_builder` image. But doing this should be fine and is analogous to compilers using earlier versions of themselves to compile themselves. Besides, if we ever break the image we'd just be back to where we are today, needing to build it manually. Note for now I'm simply focusing on adding builds and not worrying about any release mechanisms. A later PR will add a `push-image` task and somehow figure out how to automatically bootstrap to this. --- taskcluster/kinds/docker-image/kind.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/taskcluster/kinds/docker-image/kind.yml b/taskcluster/kinds/docker-image/kind.yml index 4043efa4a..b71479b4d 100644 --- a/taskcluster/kinds/docker-image/kind.yml +++ b/taskcluster/kinds/docker-image/kind.yml @@ -21,6 +21,8 @@ tasks: parent: run-task fetch: symbol: I(fetch) + image-builder: + symbol: I(ib) index-task: symbol: I(idx) python: