From c0d1314adaab7353a0fab8c69785bbe25c81f73c Mon Sep 17 00:00:00 2001 From: Kylian Serrania Date: Sun, 7 Jun 2020 00:52:03 +0200 Subject: [PATCH 1/8] Reduce number of layers --- windows/Dockerfile | 101 +++++++++++++++++++++++---------------------- 1 file changed, 51 insertions(+), 50 deletions(-) diff --git a/windows/Dockerfile b/windows/Dockerfile index 316cbebd0..d41ab0510 100644 --- a/windows/Dockerfile +++ b/windows/Dockerfile @@ -24,44 +24,42 @@ ARG TARGET_ARCH ENV TARGET_ARCH=${TARGET_ARCH:-x64} # Chocolatey package versions -ENV GIT_VERSION "2.26.2" -ENV SEVENZIP_VERSION "19.0" -ENV VS2017BUILDTOOLS_VERSION "15.9.23.0" -ENV VCPYTHON27_VERSION "9.0.0.30729" -ENV GO_VERSION "1.13.8" -ENV RUBY_VERSION "2.4.3.1" -ENV PYTHON_VERSION "2.7.17" -ENV WIX_VERSION "3.11.2" -ENV CMAKE_VERSION "3.17.2" -ENV MSYS_VERSION "20190524.0.0.20191030" - -ENV EMBEDDED_PYTHON_2_VERSION "2.7.17" -ENV EMBEDDED_PYTHON_3_VERSION "3.8.1" - -ENV CACERTS_HASH "ADF770DFD574A0D6026BFAA270CB6879B063957177A991D453FF1D302C02081F" - -LABEL target_agent="Agent 6/7" -LABEL target_arch=${TARGET_ARCH} -LABEL windows_version=${WINDOWS_VERSION} -LABEL git_version=${GIT_VERSION} -LABEL sevenzip_version=${SEVENZIP_VERSION} -LABEL vs2017buildtools_version=${VS2017BUILDTOOLS_VERSION} -LABEL vcpython27_version=${VCPYTHON27_VERSION} -LABEL go_version=${GO_VERSION} -LABEL ruby_version=${RUBY_VERSION} -LABEL wix_version=${WIX_VERSION} -LABEL cmake_version=${CMAKE_VERSION} -LABEL msys_version=${MSYS_VERSION} -LABEL system_python_version=${PYTHON_VERSION} -LABEL embedded_py2_version=${EMBEDDED_PYTHON_2_VERSION} -LABEL embedded_py3_version=${EMBEDDED_PYTHON_3_VERSION} +ENV GIT_VERSION="2.26.2" \ + SEVENZIP_VERSION="19.0" \ + VS2017BUILDTOOLS_VERSION="15.9.23.0" \ + VCPYTHON27_VERSION="9.0.0.30729" \ + GO_VERSION="1.13.8" \ + RUBY_VERSION="2.4.3.1" \ + PYTHON_VERSION="2.7.17" \ + WIX_VERSION="3.11.2" \ + CMAKE_VERSION="3.17.2" \ + MSYS_VERSION="20190524.0.0.20191030" \ + EMBEDDED_PYTHON_2_VERSION="2.7.17" \ + EMBEDDED_PYTHON_3_VERSION="3.8.1" \ + CACERTS_HASH="ADF770DFD574A0D6026BFAA270CB6879B063957177A991D453FF1D302C02081F" + +LABEL target_agent="Agent 6/7" \ + target_arch=${TARGET_ARCH} \ + windows_version=${WINDOWS_VERSION} \ + git_version=${GIT_VERSION} \ + sevenzip_version=${SEVENZIP_VERSION} \ + vs2017buildtools_version=${VS2017BUILDTOOLS_VERSION} \ + vcpython27_version=${VCPYTHON27_VERSION} \ + go_version=${GO_VERSION} \ + ruby_version=${RUBY_VERSION} \ + wix_version=${WIX_VERSION} \ + cmake_version=${CMAKE_VERSION} \ + msys_version=${MSYS_VERSION} \ + system_python_version=${PYTHON_VERSION} \ + embedded_py2_version=${EMBEDDED_PYTHON_2_VERSION} \ + embedded_py3_version=${EMBEDDED_PYTHON_3_VERSION} # Add certificates needed for build & check certificates file hash # We need to trust the DigiCert High Assurance EV Root CA certificate, which signs python.org, # to be able to download some Python components during the Agent build. -RUN (New-Object System.Net.WebClient).DownloadFile(\"https://curl.haxx.se/ca/cacert.pem\", \"cacert.pem\") -RUN if ((Get-FileHash .\cacert.pem).Hash -ne \"$ENV:CACERTS_HASH\") { Write-Host \"Wrong hashsum for cacert.pem: got '$((Get-FileHash .\cacert.pem).Hash)', expected '$ENV:CACERTS_HASH'.\"; exit 1 } -RUN setx SSL_CERT_FILE \"C:\cacert.pem\" +RUN (New-Object System.Net.WebClient).DownloadFile(\"https://curl.haxx.se/ca/cacert.pem\", \"cacert.pem\") ; \ + if ((Get-FileHash .\cacert.pem).Hash -ne \"$ENV:CACERTS_HASH\") { Write-Host \"Wrong hashsum for cacert.pem: got '$((Get-FileHash .\cacert.pem).Hash)', expected '$ENV:CACERTS_HASH'.\"; exit 1 } ; \ + setx SSL_CERT_FILE \"C:\cacert.pem\" ### Preliminary step: we need both the .NET 3.5 runtime and ### the .NET 4.8 runtime. To do this, we get 4.8 from a base image and we @@ -88,50 +86,53 @@ RUN if ($Env:TARGET_ARCH -eq 'x86') { setx CHOCO_ARCH_FLAG '-x86' } RUN $env:chocolateyUseWindowsCompression = 'true'; Invoke-Expression ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1')) # Install git -RUN $env:chocolateyUseWindowsCompression = 'true'; cinst -y --no-progress git $ENV:CHOCO_ARCH_FLAG --version $ENV:GIT_VERSION ### HACK: we disable symbolic links when cloning repositories ### to work around a symlink-related failure in the agent-binaries omnibus project ### when copying the datadog-agent project twice. + +RUN $env:chocolateyUseWindowsCompression = 'true'; cinst -y --no-progress git $ENV:CHOCO_ARCH_FLAG --version $ENV:GIT_VERSION RUN git config --system core.symlinks false # Install 7zip RUN $env:chocolateyUseWindowsCompression = 'true'; cinst -y --no-progress 7zip $ENV:CHOCO_ARCH_FLAG --version $ENV:SEVENZIP_VERSION # Install VS2017 -RUN cinst -y --no-progress visualstudio2017buildtools $ENV:CHOCO_ARCH_FLAG --version $ENV:VS2017BUILDTOOLS_VERSION --params \"--add Microsoft.VisualStudio.ComponentGroup.NativeDesktop.Win81 --add Microsoft.VisualStudio.Workload.VCTools\" -RUN setx VSTUDIO_ROOT \"${env:ProgramFiles(x86)}\Microsoft Visual Studio\2017\BuildTools\" +RUN cinst -y --no-progress visualstudio2017buildtools $ENV:CHOCO_ARCH_FLAG --version $ENV:VS2017BUILDTOOLS_VERSION --params \"--add Microsoft.VisualStudio.ComponentGroup.NativeDesktop.Win81 --add Microsoft.VisualStudio.Workload.VCTools\" ; \ + setx VSTUDIO_ROOT \"${env:ProgramFiles(x86)}\Microsoft Visual Studio\2017\BuildTools\" # Install VC compiler for Python 2.7 RUN cinst -y --no-progress vcpython27 $ENV:CHOCO_ARCH_FLAG --version $ENV:VCPYTHON27_VERSION # Install Wix and update PATH to include it -RUN cinst -y --no-progress wixtoolset $ENV:CHOCO_ARCH_FLAG --version $ENV:WIX_VERSION -RUN [Environment]::SetEnvironmentVariable(\"Path\", [Environment]::GetEnvironmentVariable(\"Path\", [EnvironmentVariableTarget]::Machine) + \";${env:ProgramFiles(x86)}\WiX Toolset v3.11\bin\", [System.EnvironmentVariableTarget]::Machine) +RUN cinst -y --no-progress wixtoolset $ENV:CHOCO_ARCH_FLAG --version $ENV:WIX_VERSION ; \ + [Environment]::SetEnvironmentVariable(\"Path\", [Environment]::GetEnvironmentVariable(\"Path\", [EnvironmentVariableTarget]::Machine) + \";${env:ProgramFiles(x86)}\WiX Toolset v3.11\bin\", [System.EnvironmentVariableTarget]::Machine) # Install Cmake and update PATH to include it -RUN cinst -y --no-progress cmake $ENV:CHOCO_ARCH_FLAG --version $ENV:CMAKE_VERSION -RUN if ($Env:TARGET_ARCH -eq 'x86') { [Environment]::SetEnvironmentVariable(\"Path\", [Environment]::GetEnvironmentVariable(\"Path\", [EnvironmentVariableTarget]::Machine) + \";${Env:ProgramFiles(x86)}\CMake\bin\", [System.EnvironmentVariableTarget]::Machine) } -RUN if ($Env:TARGET_ARCH -eq 'x64') { [Environment]::SetEnvironmentVariable(\"Path\", [Environment]::GetEnvironmentVariable(\"Path\", [EnvironmentVariableTarget]::Machine) + \";${env:ProgramFiles}\CMake\bin\", [System.EnvironmentVariableTarget]::Machine) } +RUN cinst -y --no-progress cmake $ENV:CHOCO_ARCH_FLAG --version $ENV:CMAKE_VERSION ; \ + if ($Env:TARGET_ARCH -eq 'x86') { [Environment]::SetEnvironmentVariable(\"Path\", [Environment]::GetEnvironmentVariable(\"Path\", [EnvironmentVariableTarget]::Machine) + \";${Env:ProgramFiles(x86)}\CMake\bin\", [System.EnvironmentVariableTarget]::Machine) } ; \ + if ($Env:TARGET_ARCH -eq 'x64') { [Environment]::SetEnvironmentVariable(\"Path\", [Environment]::GetEnvironmentVariable(\"Path\", [EnvironmentVariableTarget]::Machine) + \";${env:ProgramFiles}\CMake\bin\", [System.EnvironmentVariableTarget]::Machine) } # Install golang and set GOPATH to the dev path used in builds & tests -RUN cinst -y --no-progress golang $ENV:CHOCO_ARCH_FLAG --version $ENV:GO_VERSION -RUN setx GOPATH C:\dev\go +RUN cinst -y --no-progress golang $ENV:CHOCO_ARCH_FLAG --version $ENV:GO_VERSION ; \ + setx GOPATH C:\dev\go # Install system Python 2 (to use invoke) RUN cinst -y --no-progress python2 $ENV:CHOCO_ARCH_FLAG --version $ENV:PYTHON_VERSION # Install 64-bit ruby and bundler (for omnibus builds) RUN cinst -y --no-progress ruby --version $ENV:RUBY_VERSION -RUN setx RIDK ((Get-Command ridk).Path) -RUN gem install bundler +RUN setx RIDK ((Get-Command ridk).Path) ; \ + gem install bundler # Install msys2 system & install 64-bit C/C++ compilation toolchain -RUN cinst -y --no-progress msys2 --params \"/NoUpdate\" --version $ENV:MSYS_VERSION -RUN ridk install 3 +RUN cinst -y --no-progress msys2 --params \"/NoUpdate\" --version $ENV:MSYS_VERSION ; \ + ridk install 3 # (32-bit only) Install 32-bit C/C++ compilation toolchain -RUN if ($Env:TARGET_ARCH -eq 'x86') { ridk enable; bash -c \"pacman -S --needed --noconfirm mingw-w64-i686-binutils mingw-w64-i686-crt-git mingw-w64-i686-gcc mingw-w64-i686-gcc-libs mingw-w64-i686-headers-git mingw-w64-i686-libmangle-git mingw-w64-i686-libwinpthread-git mingw-w64-i686-make mingw-w64-i686-pkg-config mingw-w64-i686-tools-git mingw-w64-i686-winpthreads-git\" } -RUN if ($Env:TARGET_ARCH -eq 'x86') { [Environment]::SetEnvironmentVariable(\"Path\", [Environment]::GetEnvironmentVariable(\"Path\", [EnvironmentVariableTarget]::Machine) + \";C:\tools\msys64\mingw32\bin;C:\tools\msys64\usr\bin\", [System.EnvironmentVariableTarget]::Machine) } +RUN if ($Env:TARGET_ARCH -eq 'x86') { \ + ridk enable; bash -c \"pacman -S --needed --noconfirm mingw-w64-i686-binutils mingw-w64-i686-crt-git mingw-w64-i686-gcc mingw-w64-i686-gcc-libs mingw-w64-i686-headers-git mingw-w64-i686-libmangle-git mingw-w64-i686-libwinpthread-git mingw-w64-i686-make mingw-w64-i686-pkg-config mingw-w64-i686-tools-git mingw-w64-i686-winpthreads-git\" ; \ + [Environment]::SetEnvironmentVariable(\"Path\", [Environment]::GetEnvironmentVariable(\"Path\", [EnvironmentVariableTarget]::Machine) + \";C:\tools\msys64\mingw32\bin;C:\tools\msys64\usr\bin\", [System.EnvironmentVariableTarget]::Machine) \ + } # Install aws cli COPY ./windows/install_awscli.ps1 install_awscli.ps1 From 53f4d5ba26ca6c911388a11923e98e897225f3e8 Mon Sep 17 00:00:00 2001 From: Kylian Serrania Date: Sun, 7 Jun 2020 10:29:53 +0200 Subject: [PATCH 2/8] Reorganize Dockerfiles --- .gitlab-ci.yml | 4 +- windows/Dockerfile | 98 +++++------- windows/Dockerfile-x86 | 141 ++++++++++++++++++ windows/{ => scripts}/aws_networking.ps1 | 0 windows/{ => scripts}/entrypoint.bat | 0 windows/{ => scripts}/install_awscli.ps1 | 0 windows/{ => scripts}/install_docker.ps1 | 0 .../install_embedded_pythons.ps1 | 0 windows/{ => scripts}/install_net35_1809.bat | 0 windows/{ => scripts}/install_net35_1909.bat | 0 10 files changed, 179 insertions(+), 64 deletions(-) create mode 100644 windows/Dockerfile-x86 rename windows/{ => scripts}/aws_networking.ps1 (100%) rename windows/{ => scripts}/entrypoint.bat (100%) rename windows/{ => scripts}/install_awscli.ps1 (100%) rename windows/{ => scripts}/install_docker.ps1 (100%) rename windows/{ => scripts}/install_embedded_pythons.ps1 (100%) rename windows/{ => scripts}/install_net35_1809.bat (100%) rename windows/{ => scripts}/install_net35_1909.bat (100%) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c699e5683..48ed69721 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -112,7 +112,7 @@ build_windows_1809_x86: extends: .winbuild tags: [ "runner:windows-docker", "windowsversion:1809" ] variables: - DOCKERFILE: windows/Dockerfile + DOCKERFILE: windows/Dockerfile-x86 IMAGE: windows_1809_x86 BASE_IMAGE: mcr.microsoft.com/dotnet/framework/runtime:4.8-20200114-windowsservercore-ltsc2019 TARGET_ARCH: x86 @@ -134,7 +134,7 @@ build_windows_1909_x86: extends: .winbuild tags: [ "runner:windows-docker", "windowsversion:1909" ] variables: - DOCKERFILE: windows/Dockerfile + DOCKERFILE: windows/Dockerfile-x86 IMAGE: windows_1909_x86 BASE_IMAGE: mcr.microsoft.com/dotnet/framework/runtime:4.8-20200310-windowsservercore-1909 TARGET_ARCH: x86 diff --git a/windows/Dockerfile b/windows/Dockerfile index d41ab0510..3696855be 100644 --- a/windows/Dockerfile +++ b/windows/Dockerfile @@ -20,9 +20,6 @@ SHELL ["powershell", "-Command"] ARG WINDOWS_VERSION ENV WINDOWS_VERSION=${WINDOWS_VERSION:-1809} -ARG TARGET_ARCH -ENV TARGET_ARCH=${TARGET_ARCH:-x64} - # Chocolatey package versions ENV GIT_VERSION="2.26.2" \ SEVENZIP_VERSION="19.0" \ @@ -54,6 +51,8 @@ LABEL target_agent="Agent 6/7" \ embedded_py2_version=${EMBEDDED_PYTHON_2_VERSION} \ embedded_py3_version=${EMBEDDED_PYTHON_3_VERSION} +COPY ./windows/scripts scripts + # Add certificates needed for build & check certificates file hash # We need to trust the DigiCert High Assurance EV Root CA certificate, which signs python.org, # to be able to download some Python components during the Agent build. @@ -61,27 +60,6 @@ RUN (New-Object System.Net.WebClient).DownloadFile(\"https://curl.haxx.se/ca/cac if ((Get-FileHash .\cacert.pem).Hash -ne \"$ENV:CACERTS_HASH\") { Write-Host \"Wrong hashsum for cacert.pem: got '$((Get-FileHash .\cacert.pem).Hash)', expected '$ENV:CACERTS_HASH'.\"; exit 1 } ; \ setx SSL_CERT_FILE \"C:\cacert.pem\" -### Preliminary step: we need both the .NET 3.5 runtime and -### the .NET 4.8 runtime. To do this, we get 4.8 from a base image and we -### manually the install .NET Framework 3.5 runtime using the instructions in -### the mcr.microsoft.com/dotnet/framework/runtime:3.5 Dockerfile: -### https://github.com/microsoft/dotnet-framework-docker/blob/26597e42d157cc1e09d1e0dc8f23c32e6c3d1467/3.5/runtime/windowsservercore-ltsc2019/Dockerfile - -### The .NET Fx 3.5 is needed for the Visual C++ Compiler for Python 2.7 -### (https://www.microsoft.com/en-us/download/details.aspx?id=44266) -### and to work around a bug in the WiX 3.11 installer -### (https://github.com/wixtoolset/issues/issues/5661). - -# Install .NET Fx 3.5 -COPY ./windows/install_net35_1809.bat install_net35_1809.bat -COPY ./windows/install_net35_1909.bat install_net35_1909.bat -RUN if ($Env:WINDOWS_VERSION -eq '1809') { .\install_net35_1809.bat } -RUN if ($Env:WINDOWS_VERSION -eq '1909') { .\install_net35_1909.bat } - -### End of preliminary step - -RUN if ($Env:TARGET_ARCH -eq 'x86') { setx CHOCO_ARCH_FLAG '-x86' } - # Install Chocolatey RUN $env:chocolateyUseWindowsCompression = 'true'; Invoke-Expression ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1')) @@ -89,35 +67,22 @@ RUN $env:chocolateyUseWindowsCompression = 'true'; Invoke-Expression ((New-Objec ### HACK: we disable symbolic links when cloning repositories ### to work around a symlink-related failure in the agent-binaries omnibus project ### when copying the datadog-agent project twice. - -RUN $env:chocolateyUseWindowsCompression = 'true'; cinst -y --no-progress git $ENV:CHOCO_ARCH_FLAG --version $ENV:GIT_VERSION +RUN $env:chocolateyUseWindowsCompression = 'true'; cinst -y --no-progress git --version $ENV:GIT_VERSION RUN git config --system core.symlinks false # Install 7zip -RUN $env:chocolateyUseWindowsCompression = 'true'; cinst -y --no-progress 7zip $ENV:CHOCO_ARCH_FLAG --version $ENV:SEVENZIP_VERSION - -# Install VS2017 -RUN cinst -y --no-progress visualstudio2017buildtools $ENV:CHOCO_ARCH_FLAG --version $ENV:VS2017BUILDTOOLS_VERSION --params \"--add Microsoft.VisualStudio.ComponentGroup.NativeDesktop.Win81 --add Microsoft.VisualStudio.Workload.VCTools\" ; \ - setx VSTUDIO_ROOT \"${env:ProgramFiles(x86)}\Microsoft Visual Studio\2017\BuildTools\" - -# Install VC compiler for Python 2.7 -RUN cinst -y --no-progress vcpython27 $ENV:CHOCO_ARCH_FLAG --version $ENV:VCPYTHON27_VERSION - -# Install Wix and update PATH to include it -RUN cinst -y --no-progress wixtoolset $ENV:CHOCO_ARCH_FLAG --version $ENV:WIX_VERSION ; \ - [Environment]::SetEnvironmentVariable(\"Path\", [Environment]::GetEnvironmentVariable(\"Path\", [EnvironmentVariableTarget]::Machine) + \";${env:ProgramFiles(x86)}\WiX Toolset v3.11\bin\", [System.EnvironmentVariableTarget]::Machine) +RUN $env:chocolateyUseWindowsCompression = 'true'; cinst -y --no-progress 7zip --version $ENV:SEVENZIP_VERSION # Install Cmake and update PATH to include it -RUN cinst -y --no-progress cmake $ENV:CHOCO_ARCH_FLAG --version $ENV:CMAKE_VERSION ; \ - if ($Env:TARGET_ARCH -eq 'x86') { [Environment]::SetEnvironmentVariable(\"Path\", [Environment]::GetEnvironmentVariable(\"Path\", [EnvironmentVariableTarget]::Machine) + \";${Env:ProgramFiles(x86)}\CMake\bin\", [System.EnvironmentVariableTarget]::Machine) } ; \ - if ($Env:TARGET_ARCH -eq 'x64') { [Environment]::SetEnvironmentVariable(\"Path\", [Environment]::GetEnvironmentVariable(\"Path\", [EnvironmentVariableTarget]::Machine) + \";${env:ProgramFiles}\CMake\bin\", [System.EnvironmentVariableTarget]::Machine) } +RUN cinst -y --no-progress cmake --version $ENV:CMAKE_VERSION ; \ + [Environment]::SetEnvironmentVariable(\"Path\", [Environment]::GetEnvironmentVariable(\"Path\", [EnvironmentVariableTarget]::Machine) + \";${env:ProgramFiles}\CMake\bin\", [System.EnvironmentVariableTarget]::Machine) # Install golang and set GOPATH to the dev path used in builds & tests -RUN cinst -y --no-progress golang $ENV:CHOCO_ARCH_FLAG --version $ENV:GO_VERSION ; \ +RUN cinst -y --no-progress golang --version $ENV:GO_VERSION ; \ setx GOPATH C:\dev\go # Install system Python 2 (to use invoke) -RUN cinst -y --no-progress python2 $ENV:CHOCO_ARCH_FLAG --version $ENV:PYTHON_VERSION +RUN cinst -y --no-progress python2 --version $ENV:PYTHON_VERSION # Install 64-bit ruby and bundler (for omnibus builds) RUN cinst -y --no-progress ruby --version $ENV:RUBY_VERSION @@ -128,31 +93,40 @@ RUN setx RIDK ((Get-Command ridk).Path) ; \ RUN cinst -y --no-progress msys2 --params \"/NoUpdate\" --version $ENV:MSYS_VERSION ; \ ridk install 3 -# (32-bit only) Install 32-bit C/C++ compilation toolchain -RUN if ($Env:TARGET_ARCH -eq 'x86') { \ - ridk enable; bash -c \"pacman -S --needed --noconfirm mingw-w64-i686-binutils mingw-w64-i686-crt-git mingw-w64-i686-gcc mingw-w64-i686-gcc-libs mingw-w64-i686-headers-git mingw-w64-i686-libmangle-git mingw-w64-i686-libwinpthread-git mingw-w64-i686-make mingw-w64-i686-pkg-config mingw-w64-i686-tools-git mingw-w64-i686-winpthreads-git\" ; \ - [Environment]::SetEnvironmentVariable(\"Path\", [Environment]::GetEnvironmentVariable(\"Path\", [EnvironmentVariableTarget]::Machine) + \";C:\tools\msys64\mingw32\bin;C:\tools\msys64\usr\bin\", [System.EnvironmentVariableTarget]::Machine) \ - } - # Install aws cli -COPY ./windows/install_awscli.ps1 install_awscli.ps1 -RUN powershell -C .\install_awscli.ps1 +RUN powershell -C .\scripts\install_awscli.ps1 # Install docker, manifest-tool and notary -COPY ./windows/install_docker.ps1 install_docker.ps1 -RUN powershell -Command .\install_docker.ps1 +RUN powershell -Command .\scripts\install_docker.ps1 # Install embedded pythons (for unit testing) -COPY ./windows/install_embedded_pythons.ps1 install_embedded_pythons.ps1 -RUN powershell -C .\install_embedded_pythons.ps1 +RUN powershell -C .\scripts\install_embedded_pythons.ps1 + +### We need both the .NET 3.5 runtime and +### the .NET 4.8 runtime. To do this, we get 4.8 from a base image and we +### manually the install .NET Framework 3.5 runtime using the instructions in +### the mcr.microsoft.com/dotnet/framework/runtime:3.5 Dockerfile: +### https://github.com/microsoft/dotnet-framework-docker/blob/26597e42d157cc1e09d1e0dc8f23c32e6c3d1467/3.5/runtime/windowsservercore-ltsc2019/Dockerfile + +### The .NET Fx 3.5 is needed for the Visual C++ Compiler for Python 2.7 +### (https://www.microsoft.com/en-us/download/details.aspx?id=44266) +### and to work around a bug in the WiX 3.11 installer +### (https://github.com/wixtoolset/issues/issues/5661). + +# Install .NET Fx 3.5 +RUN if ($Env:WINDOWS_VERSION -eq '1809') { .\scripts\install_net35_1809.bat } +RUN if ($Env:WINDOWS_VERSION -eq '1909') { .\scripts\install_net35_1909.bat } -# Add signtool to path -RUN [Environment]::SetEnvironmentVariable(\"Path\", [Environment]::GetEnvironmentVariable(\"Path\", [EnvironmentVariableTarget]::Machine) + \";${env:ProgramFiles(x86)}\Windows Kits\8.1\bin\x64\", [System.EnvironmentVariableTarget]::Machine) +# Install VS2017 +RUN cinst -y --no-progress visualstudio2017buildtools --version $ENV:VS2017BUILDTOOLS_VERSION --params \"--add Microsoft.VisualStudio.ComponentGroup.NativeDesktop.Win81 --add Microsoft.VisualStudio.Workload.VCTools\" ; \ + setx VSTUDIO_ROOT \"${env:ProgramFiles(x86)}\Microsoft Visual Studio\2017\BuildTools\" ; \ + [Environment]::SetEnvironmentVariable(\"Path\", [Environment]::GetEnvironmentVariable(\"Path\", [EnvironmentVariableTarget]::Machine) + \";${env:ProgramFiles(x86)}\Windows Kits\8.1\bin\x64\", [System.EnvironmentVariableTarget]::Machine) -# Set 32-bit flag env var -RUN if ($Env:TARGET_ARCH -eq 'x86') { setx WINDOWS_BUILD_32_BIT 1 } +# Install VC compiler for Python 2.7 +RUN cinst -y --no-progress vcpython27 --version $ENV:VCPYTHON27_VERSION -COPY ./windows/entrypoint.bat /entrypoint.bat -COPY ./windows/aws_networking.ps1 /aws_networking.ps1 +# Install Wix and update PATH to include it +RUN cinst -y --no-progress wixtoolset --version $ENV:WIX_VERSION ; \ + [Environment]::SetEnvironmentVariable(\"Path\", [Environment]::GetEnvironmentVariable(\"Path\", [EnvironmentVariableTarget]::Machine) + \";${env:ProgramFiles(x86)}\WiX Toolset v3.11\bin\", [System.EnvironmentVariableTarget]::Machine) -ENTRYPOINT ["/entrypoint.bat"] +ENTRYPOINT ["scripts/entrypoint.bat"] diff --git a/windows/Dockerfile-x86 b/windows/Dockerfile-x86 new file mode 100644 index 000000000..c7387edfd --- /dev/null +++ b/windows/Dockerfile-x86 @@ -0,0 +1,141 @@ +# Use the Microsoft-provided .NET Runtime 4.8 image as the base image +# because installing it in the image with Chocolatey requires a reboot. + +# There is a bug in the February 11, 2020 update that makes commands fail in +# docker containers (see: https://support.microsoft.com/en-us/help/4542617/you-might-encounter-issues-when-using-windows-server-containers-with-t) +# To avoid that, there are two solutions: +# - both the host and the container must have the February 11, 2020 update, or +# - neither the host and the container must have the February 11, 2020 update. +# Since our 1809 windows-docker host image does not have this update, we use a base +# container image that does not have this update either (thus the 20200114 tag). +# On the contrary, since our 1909 windows-docker host image does have this update, +# we use a base container image that does have this update. +# TODO: Once the 1809 windows-docker host image is updated, update the base container image. +ARG BASE_IMAGE=mcr.microsoft.com/dotnet/framework/runtime:4.8-20200114-windowsservercore-ltsc2019 + +FROM ${BASE_IMAGE} + +SHELL ["powershell", "-Command"] + +ARG WINDOWS_VERSION +ENV WINDOWS_VERSION=${WINDOWS_VERSION:-1809} + +# Chocolatey package versions +ENV GIT_VERSION="2.26.2" \ + SEVENZIP_VERSION="19.0" \ + VS2017BUILDTOOLS_VERSION="15.9.23.0" \ + VCPYTHON27_VERSION="9.0.0.30729" \ + GO_VERSION="1.13.8" \ + RUBY_VERSION="2.4.3.1" \ + PYTHON_VERSION="2.7.17" \ + WIX_VERSION="3.11.2" \ + CMAKE_VERSION="3.17.2" \ + MSYS_VERSION="20190524.0.0.20191030" \ + EMBEDDED_PYTHON_2_VERSION="2.7.17" \ + EMBEDDED_PYTHON_3_VERSION="3.8.1" \ + CACERTS_HASH="ADF770DFD574A0D6026BFAA270CB6879B063957177A991D453FF1D302C02081F" + +LABEL target_agent="Agent 6/7" \ + target_arch="x86" \ + windows_version=${WINDOWS_VERSION} \ + git_version=${GIT_VERSION} \ + sevenzip_version=${SEVENZIP_VERSION} \ + vs2017buildtools_version=${VS2017BUILDTOOLS_VERSION} \ + vcpython27_version=${VCPYTHON27_VERSION} \ + go_version=${GO_VERSION} \ + ruby_version=${RUBY_VERSION} \ + wix_version=${WIX_VERSION} \ + cmake_version=${CMAKE_VERSION} \ + msys_version=${MSYS_VERSION} \ + system_python_version=${PYTHON_VERSION} \ + embedded_py2_version=${EMBEDDED_PYTHON_2_VERSION} \ + embedded_py3_version=${EMBEDDED_PYTHON_3_VERSION} + +COPY ./windows/scripts scripts + +# Add certificates needed for build & check certificates file hash +# We need to trust the DigiCert High Assurance EV Root CA certificate, which signs python.org, +# to be able to download some Python components during the Agent build. +RUN (New-Object System.Net.WebClient).DownloadFile(\"https://curl.haxx.se/ca/cacert.pem\", \"cacert.pem\") ; \ + if ((Get-FileHash .\cacert.pem).Hash -ne \"$ENV:CACERTS_HASH\") { Write-Host \"Wrong hashsum for cacert.pem: got '$((Get-FileHash .\cacert.pem).Hash)', expected '$ENV:CACERTS_HASH'.\"; exit 1 } ; \ + setx SSL_CERT_FILE \"C:\cacert.pem\" + +# Set 32-bit flag env var +RUN setx WINDOWS_BUILD_32_BIT 1 + +# Install Chocolatey +RUN $env:chocolateyUseWindowsCompression = 'true'; Invoke-Expression ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1')) + +# Install git +### HACK: we disable symbolic links when cloning repositories +### to work around a symlink-related failure in the agent-binaries omnibus project +### when copying the datadog-agent project twice. + +RUN $env:chocolateyUseWindowsCompression = 'true'; cinst -y --no-progress git -x86 --version $ENV:GIT_VERSION +RUN git config --system core.symlinks false + +# Install 7zip +RUN $env:chocolateyUseWindowsCompression = 'true'; cinst -y --no-progress 7zip -x86 --version $ENV:SEVENZIP_VERSION + +# Install Cmake and update PATH to include it +RUN cinst -y --no-progress cmake -x86 --version $ENV:CMAKE_VERSION ; \ + [Environment]::SetEnvironmentVariable(\"Path\", [Environment]::GetEnvironmentVariable(\"Path\", [EnvironmentVariableTarget]::Machine) + \";${Env:ProgramFiles(x86)}\CMake\bin\", [System.EnvironmentVariableTarget]::Machine) ; \ + [Environment]::SetEnvironmentVariable(\"Path\", [Environment]::GetEnvironmentVariable(\"Path\", [EnvironmentVariableTarget]::Machine) + \";${env:ProgramFiles}\CMake\bin\", [System.EnvironmentVariableTarget]::Machine) + +# Install golang and set GOPATH to the dev path used in builds & tests +RUN cinst -y --no-progress golang -x86 --version $ENV:GO_VERSION ; \ + setx GOPATH C:\dev\go + +# Install system Python 2 (to use invoke) +RUN cinst -y --no-progress python2 -x86 --version $ENV:PYTHON_VERSION + +# Install 64-bit ruby and bundler (for omnibus builds) +RUN cinst -y --no-progress ruby --version $ENV:RUBY_VERSION +RUN setx RIDK ((Get-Command ridk).Path) ; \ + gem install bundler + +# Install msys2 system & install 64-bit C/C++ compilation toolchain +RUN cinst -y --no-progress msys2 --params \"/NoUpdate\" --version $ENV:MSYS_VERSION ; \ + ridk install 3 + +# (32-bit only) Install 32-bit C/C++ compilation toolchain +RUN ridk enable; bash -c \"pacman -S --needed --noconfirm mingw-w64-i686-binutils mingw-w64-i686-crt-git mingw-w64-i686-gcc mingw-w64-i686-gcc-libs mingw-w64-i686-headers-git mingw-w64-i686-libmangle-git mingw-w64-i686-libwinpthread-git mingw-w64-i686-make mingw-w64-i686-pkg-config mingw-w64-i686-tools-git mingw-w64-i686-winpthreads-git\" ; \ + [Environment]::SetEnvironmentVariable(\"Path\", [Environment]::GetEnvironmentVariable(\"Path\", [EnvironmentVariableTarget]::Machine) + \";C:\tools\msys64\mingw32\bin;C:\tools\msys64\usr\bin\", [System.EnvironmentVariableTarget]::Machine) + +# Install aws cli +RUN powershell -C .\scripts\install_awscli.ps1 + +# Install docker, manifest-tool and notary +RUN powershell -Command .\scripts\install_docker.ps1 + +# Install embedded pythons (for unit testing) +RUN powershell -C .\scripts\install_embedded_pythons.ps1 + +### We need both the .NET 3.5 runtime and +### the .NET 4.8 runtime. To do this, we get 4.8 from a base image and we +### manually the install .NET Framework 3.5 runtime using the instructions in +### the mcr.microsoft.com/dotnet/framework/runtime:3.5 Dockerfile: +### https://github.com/microsoft/dotnet-framework-docker/blob/26597e42d157cc1e09d1e0dc8f23c32e6c3d1467/3.5/runtime/windowsservercore-ltsc2019/Dockerfile + +### The .NET Fx 3.5 is needed for the Visual C++ Compiler for Python 2.7 +### (https://www.microsoft.com/en-us/download/details.aspx?id=44266) +### and to work around a bug in the WiX 3.11 installer +### (https://github.com/wixtoolset/issues/issues/5661). + +# Install .NET Fx 3.5 +RUN if ($Env:WINDOWS_VERSION -eq '1809') { .\scripts\install_net35_1809.bat } +RUN if ($Env:WINDOWS_VERSION -eq '1909') { .\scripts\install_net35_1909.bat } + +# Install VS2017 +RUN cinst -y --no-progress visualstudio2017buildtools -x86 --version $ENV:VS2017BUILDTOOLS_VERSION --params \"--add Microsoft.VisualStudio.ComponentGroup.NativeDesktop.Win81 --add Microsoft.VisualStudio.Workload.VCTools\" ; \ + setx VSTUDIO_ROOT \"${env:ProgramFiles(x86)}\Microsoft Visual Studio\2017\BuildTools\" ; \ + [Environment]::SetEnvironmentVariable(\"Path\", [Environment]::GetEnvironmentVariable(\"Path\", [EnvironmentVariableTarget]::Machine) + \";${env:ProgramFiles(x86)}\Windows Kits\8.1\bin\x64\", [System.EnvironmentVariableTarget]::Machine) + +# Install VC compiler for Python 2.7 +RUN cinst -y --no-progress vcpython27 -x86 --version $ENV:VCPYTHON27_VERSION + +# Install Wix and update PATH to include it +RUN cinst -y --no-progress wixtoolset -x86 --version $ENV:WIX_VERSION ; \ + [Environment]::SetEnvironmentVariable(\"Path\", [Environment]::GetEnvironmentVariable(\"Path\", [EnvironmentVariableTarget]::Machine) + \";${env:ProgramFiles(x86)}\WiX Toolset v3.11\bin\", [System.EnvironmentVariableTarget]::Machine) + +ENTRYPOINT ["scripts/entrypoint.bat"] diff --git a/windows/aws_networking.ps1 b/windows/scripts/aws_networking.ps1 similarity index 100% rename from windows/aws_networking.ps1 rename to windows/scripts/aws_networking.ps1 diff --git a/windows/entrypoint.bat b/windows/scripts/entrypoint.bat similarity index 100% rename from windows/entrypoint.bat rename to windows/scripts/entrypoint.bat diff --git a/windows/install_awscli.ps1 b/windows/scripts/install_awscli.ps1 similarity index 100% rename from windows/install_awscli.ps1 rename to windows/scripts/install_awscli.ps1 diff --git a/windows/install_docker.ps1 b/windows/scripts/install_docker.ps1 similarity index 100% rename from windows/install_docker.ps1 rename to windows/scripts/install_docker.ps1 diff --git a/windows/install_embedded_pythons.ps1 b/windows/scripts/install_embedded_pythons.ps1 similarity index 100% rename from windows/install_embedded_pythons.ps1 rename to windows/scripts/install_embedded_pythons.ps1 diff --git a/windows/install_net35_1809.bat b/windows/scripts/install_net35_1809.bat similarity index 100% rename from windows/install_net35_1809.bat rename to windows/scripts/install_net35_1809.bat diff --git a/windows/install_net35_1909.bat b/windows/scripts/install_net35_1909.bat similarity index 100% rename from windows/install_net35_1909.bat rename to windows/scripts/install_net35_1909.bat From 87cc565f3d5d594ae82c125f6f4ea088fffd4c82 Mon Sep 17 00:00:00 2001 From: Kylian Serrania Date: Sun, 7 Jun 2020 11:26:52 +0200 Subject: [PATCH 3/8] Use bundled 3.5 and 4.8 on 1909 --- .gitlab-ci.yml | 41 +++++++++++++++++++++-------------------- windows/Dockerfile | 6 ++++-- windows/Dockerfile-x86 | 6 ++++-- 3 files changed, 29 insertions(+), 24 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 48ed69721..c68ae6675 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -92,53 +92,54 @@ build_system-probe_arm64: - $SRC_IMAGE = "486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/${IMAGE}:${SRC_TAG}" - echo "build --build-arg BASE_IMAGE=$BASE_IMAGE --build-arg TARGET_ARCH=$TARGET_ARCH --build-arg WINDOWS_VERSION=$WINDOWS_VERSION --tag $SRC_IMAGE --file $DOCKERFILE ." - powershell -Command "docker build --no-cache --build-arg BASE_IMAGE=$BASE_IMAGE --build-arg TARGET_ARCH=$TARGET_ARCH --build-arg WINDOWS_VERSION=$WINDOWS_VERSION --tag $SRC_IMAGE --file $DOCKERFILE ." + - docker images - docker push $SRC_IMAGE after_script: - docker image prune -f - docker rmi 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/${IMAGE}:v${CI_PIPELINE_ID}-$($CI_COMMIT_SHA.SubString(0,7)) -build_windows_1809_x64: +build_windows_1909_x64: extends: .winbuild - tags: [ "runner:windows-docker", "windowsversion:1809" ] + tags: [ "runner:windows-docker", "windowsversion:1909" ] variables: DOCKERFILE: windows/Dockerfile - IMAGE: windows_1809_x64 - BASE_IMAGE: mcr.microsoft.com/dotnet/framework/runtime:4.8-20200114-windowsservercore-ltsc2019 + IMAGE: windows_1909_x64 + BASE_IMAGE: mcr.microsoft.com/dotnet/framework/runtime:3.5-20200512-windowsservercore-1909 TARGET_ARCH: x64 - WINDOWS_VERSION: 1809 + WINDOWS_VERSION: 1909 resource_group: windows_x64 -build_windows_1809_x86: +build_windows_1909_x86: extends: .winbuild - tags: [ "runner:windows-docker", "windowsversion:1809" ] + tags: [ "runner:windows-docker", "windowsversion:1909" ] variables: DOCKERFILE: windows/Dockerfile-x86 - IMAGE: windows_1809_x86 - BASE_IMAGE: mcr.microsoft.com/dotnet/framework/runtime:4.8-20200114-windowsservercore-ltsc2019 + IMAGE: windows_1909_x86 + BASE_IMAGE: mcr.microsoft.com/dotnet/framework/runtime:3.5-20200512-windowsservercore-1909 TARGET_ARCH: x86 - WINDOWS_VERSION: 1809 + WINDOWS_VERSION: 1909 resource_group: windows_x86 -build_windows_1909_x64: +build_windows_1809_x64: extends: .winbuild - tags: [ "runner:windows-docker", "windowsversion:1909" ] + tags: [ "runner:windows-docker", "windowsversion:1809" ] variables: DOCKERFILE: windows/Dockerfile - IMAGE: windows_1909_x64 - BASE_IMAGE: mcr.microsoft.com/dotnet/framework/runtime:4.8-20200310-windowsservercore-1909 + IMAGE: windows_1809_x64 + BASE_IMAGE: mcr.microsoft.com/dotnet/framework/runtime:4.8-20200114-windowsservercore-ltsc2019 TARGET_ARCH: x64 - WINDOWS_VERSION: 1909 + WINDOWS_VERSION: 1809 resource_group: windows_x64 -build_windows_1909_x86: +build_windows_1809_x86: extends: .winbuild - tags: [ "runner:windows-docker", "windowsversion:1909" ] + tags: [ "runner:windows-docker", "windowsversion:1809" ] variables: DOCKERFILE: windows/Dockerfile-x86 - IMAGE: windows_1909_x86 - BASE_IMAGE: mcr.microsoft.com/dotnet/framework/runtime:4.8-20200310-windowsservercore-1909 + IMAGE: windows_1809_x86 + BASE_IMAGE: mcr.microsoft.com/dotnet/framework/runtime:4.8-20200114-windowsservercore-ltsc2019 TARGET_ARCH: x86 - WINDOWS_VERSION: 1909 + WINDOWS_VERSION: 1809 resource_group: windows_x86 .release: diff --git a/windows/Dockerfile b/windows/Dockerfile index 3696855be..6b1dfa4ce 100644 --- a/windows/Dockerfile +++ b/windows/Dockerfile @@ -103,11 +103,14 @@ RUN powershell -Command .\scripts\install_docker.ps1 RUN powershell -C .\scripts\install_embedded_pythons.ps1 ### We need both the .NET 3.5 runtime and -### the .NET 4.8 runtime. To do this, we get 4.8 from a base image and we +### the .NET 4.8 runtime. To do this, on 1809 we get 4.8 from a base image and we ### manually the install .NET Framework 3.5 runtime using the instructions in ### the mcr.microsoft.com/dotnet/framework/runtime:3.5 Dockerfile: ### https://github.com/microsoft/dotnet-framework-docker/blob/26597e42d157cc1e09d1e0dc8f23c32e6c3d1467/3.5/runtime/windowsservercore-ltsc2019/Dockerfile +### 1909 includes .NET Framework 4.8 by default, so we use the image that includes +### .NET Framework 3.5 as well, and thus we don't need to install it manually here. + ### The .NET Fx 3.5 is needed for the Visual C++ Compiler for Python 2.7 ### (https://www.microsoft.com/en-us/download/details.aspx?id=44266) ### and to work around a bug in the WiX 3.11 installer @@ -115,7 +118,6 @@ RUN powershell -C .\scripts\install_embedded_pythons.ps1 # Install .NET Fx 3.5 RUN if ($Env:WINDOWS_VERSION -eq '1809') { .\scripts\install_net35_1809.bat } -RUN if ($Env:WINDOWS_VERSION -eq '1909') { .\scripts\install_net35_1909.bat } # Install VS2017 RUN cinst -y --no-progress visualstudio2017buildtools --version $ENV:VS2017BUILDTOOLS_VERSION --params \"--add Microsoft.VisualStudio.ComponentGroup.NativeDesktop.Win81 --add Microsoft.VisualStudio.Workload.VCTools\" ; \ diff --git a/windows/Dockerfile-x86 b/windows/Dockerfile-x86 index c7387edfd..07926b131 100644 --- a/windows/Dockerfile-x86 +++ b/windows/Dockerfile-x86 @@ -112,11 +112,14 @@ RUN powershell -Command .\scripts\install_docker.ps1 RUN powershell -C .\scripts\install_embedded_pythons.ps1 ### We need both the .NET 3.5 runtime and -### the .NET 4.8 runtime. To do this, we get 4.8 from a base image and we +### the .NET 4.8 runtime. To do this, on 1809 we get 4.8 from a base image and we ### manually the install .NET Framework 3.5 runtime using the instructions in ### the mcr.microsoft.com/dotnet/framework/runtime:3.5 Dockerfile: ### https://github.com/microsoft/dotnet-framework-docker/blob/26597e42d157cc1e09d1e0dc8f23c32e6c3d1467/3.5/runtime/windowsservercore-ltsc2019/Dockerfile +### 1909 includes .NET Framework 4.8 by default, so we use the image that includes +### .NET Framework 3.5 as well, and thus we don't need to install it manually here. + ### The .NET Fx 3.5 is needed for the Visual C++ Compiler for Python 2.7 ### (https://www.microsoft.com/en-us/download/details.aspx?id=44266) ### and to work around a bug in the WiX 3.11 installer @@ -124,7 +127,6 @@ RUN powershell -C .\scripts\install_embedded_pythons.ps1 # Install .NET Fx 3.5 RUN if ($Env:WINDOWS_VERSION -eq '1809') { .\scripts\install_net35_1809.bat } -RUN if ($Env:WINDOWS_VERSION -eq '1909') { .\scripts\install_net35_1909.bat } # Install VS2017 RUN cinst -y --no-progress visualstudio2017buildtools -x86 --version $ENV:VS2017BUILDTOOLS_VERSION --params \"--add Microsoft.VisualStudio.ComponentGroup.NativeDesktop.Win81 --add Microsoft.VisualStudio.Workload.VCTools\" ; \ From 81c4916a37a351d72b7b2911bca18cdb2bd7769a Mon Sep 17 00:00:00 2001 From: Kylian Serrania Date: Mon, 8 Jun 2020 17:46:01 +0200 Subject: [PATCH 4/8] Try even less layers --- .gitlab-ci.yml | 44 ++++---- windows/Dockerfile | 78 ++------------ windows/Dockerfile-x86 | 143 ------------------------- windows/scripts/install_net35_1909.bat | 21 ---- windows/scripts/postsetup.ps1 | 18 ++++ windows/scripts/setup.ps1 | 69 ++++++++++++ 6 files changed, 117 insertions(+), 256 deletions(-) delete mode 100644 windows/Dockerfile-x86 delete mode 100644 windows/scripts/install_net35_1909.bat create mode 100644 windows/scripts/postsetup.ps1 create mode 100644 windows/scripts/setup.ps1 diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c68ae6675..f83574f04 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -98,48 +98,48 @@ build_system-probe_arm64: - docker image prune -f - docker rmi 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/${IMAGE}:v${CI_PIPELINE_ID}-$($CI_COMMIT_SHA.SubString(0,7)) -build_windows_1909_x64: +build_windows_1809_x64: extends: .winbuild - tags: [ "runner:windows-docker", "windowsversion:1909" ] + tags: [ "runner:windows-docker", "windowsversion:1809" ] variables: DOCKERFILE: windows/Dockerfile - IMAGE: windows_1909_x64 - BASE_IMAGE: mcr.microsoft.com/dotnet/framework/runtime:3.5-20200512-windowsservercore-1909 + IMAGE: windows_1809_x64 + BASE_IMAGE: mcr.microsoft.com/dotnet/framework/runtime:4.8-20200114-windowsservercore-ltsc2019 TARGET_ARCH: x64 - WINDOWS_VERSION: 1909 + WINDOWS_VERSION: 1809 resource_group: windows_x64 -build_windows_1909_x86: +build_windows_1809_x86: extends: .winbuild - tags: [ "runner:windows-docker", "windowsversion:1909" ] + tags: [ "runner:windows-docker", "windowsversion:1809" ] variables: - DOCKERFILE: windows/Dockerfile-x86 - IMAGE: windows_1909_x86 - BASE_IMAGE: mcr.microsoft.com/dotnet/framework/runtime:3.5-20200512-windowsservercore-1909 + DOCKERFILE: windows/Dockerfile + IMAGE: windows_1809_x86 + BASE_IMAGE: mcr.microsoft.com/dotnet/framework/runtime:4.8-20200114-windowsservercore-ltsc2019 TARGET_ARCH: x86 - WINDOWS_VERSION: 1909 + WINDOWS_VERSION: 1809 resource_group: windows_x86 -build_windows_1809_x64: +build_windows_1909_x64: extends: .winbuild - tags: [ "runner:windows-docker", "windowsversion:1809" ] + tags: [ "runner:windows-docker", "windowsversion:1909" ] variables: DOCKERFILE: windows/Dockerfile - IMAGE: windows_1809_x64 - BASE_IMAGE: mcr.microsoft.com/dotnet/framework/runtime:4.8-20200114-windowsservercore-ltsc2019 + IMAGE: windows_1909_x64 + BASE_IMAGE: mcr.microsoft.com/dotnet/framework/runtime:3.5-20200512-windowsservercore-1909 TARGET_ARCH: x64 - WINDOWS_VERSION: 1809 + WINDOWS_VERSION: 1909 resource_group: windows_x64 -build_windows_1809_x86: +build_windows_1909_x86: extends: .winbuild - tags: [ "runner:windows-docker", "windowsversion:1809" ] + tags: [ "runner:windows-docker", "windowsversion:1909" ] variables: - DOCKERFILE: windows/Dockerfile-x86 - IMAGE: windows_1809_x86 - BASE_IMAGE: mcr.microsoft.com/dotnet/framework/runtime:4.8-20200114-windowsservercore-ltsc2019 + DOCKERFILE: windows/Dockerfile + IMAGE: windows_1909_x86 + BASE_IMAGE: mcr.microsoft.com/dotnet/framework/runtime:3.5-20200512-windowsservercore-1909 TARGET_ARCH: x86 - WINDOWS_VERSION: 1809 + WINDOWS_VERSION: 1909 resource_group: windows_x86 .release: diff --git a/windows/Dockerfile b/windows/Dockerfile index 6b1dfa4ce..de3b807dd 100644 --- a/windows/Dockerfile +++ b/windows/Dockerfile @@ -20,6 +20,9 @@ SHELL ["powershell", "-Command"] ARG WINDOWS_VERSION ENV WINDOWS_VERSION=${WINDOWS_VERSION:-1809} +ARG TARGET_ARCH +ENV TARGET_ARCH=${TARGET_ARCH:-x64} + # Chocolatey package versions ENV GIT_VERSION="2.26.2" \ SEVENZIP_VERSION="19.0" \ @@ -53,82 +56,17 @@ LABEL target_agent="Agent 6/7" \ COPY ./windows/scripts scripts -# Add certificates needed for build & check certificates file hash -# We need to trust the DigiCert High Assurance EV Root CA certificate, which signs python.org, -# to be able to download some Python components during the Agent build. -RUN (New-Object System.Net.WebClient).DownloadFile(\"https://curl.haxx.se/ca/cacert.pem\", \"cacert.pem\") ; \ - if ((Get-FileHash .\cacert.pem).Hash -ne \"$ENV:CACERTS_HASH\") { Write-Host \"Wrong hashsum for cacert.pem: got '$((Get-FileHash .\cacert.pem).Hash)', expected '$ENV:CACERTS_HASH'.\"; exit 1 } ; \ - setx SSL_CERT_FILE \"C:\cacert.pem\" - -# Install Chocolatey -RUN $env:chocolateyUseWindowsCompression = 'true'; Invoke-Expression ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1')) - -# Install git -### HACK: we disable symbolic links when cloning repositories -### to work around a symlink-related failure in the agent-binaries omnibus project -### when copying the datadog-agent project twice. -RUN $env:chocolateyUseWindowsCompression = 'true'; cinst -y --no-progress git --version $ENV:GIT_VERSION -RUN git config --system core.symlinks false - -# Install 7zip -RUN $env:chocolateyUseWindowsCompression = 'true'; cinst -y --no-progress 7zip --version $ENV:SEVENZIP_VERSION - -# Install Cmake and update PATH to include it -RUN cinst -y --no-progress cmake --version $ENV:CMAKE_VERSION ; \ - [Environment]::SetEnvironmentVariable(\"Path\", [Environment]::GetEnvironmentVariable(\"Path\", [EnvironmentVariableTarget]::Machine) + \";${env:ProgramFiles}\CMake\bin\", [System.EnvironmentVariableTarget]::Machine) - -# Install golang and set GOPATH to the dev path used in builds & tests -RUN cinst -y --no-progress golang --version $ENV:GO_VERSION ; \ - setx GOPATH C:\dev\go - -# Install system Python 2 (to use invoke) -RUN cinst -y --no-progress python2 --version $ENV:PYTHON_VERSION - -# Install 64-bit ruby and bundler (for omnibus builds) -RUN cinst -y --no-progress ruby --version $ENV:RUBY_VERSION -RUN setx RIDK ((Get-Command ridk).Path) ; \ - gem install bundler +RUN scripts/setup.ps1 -# Install msys2 system & install 64-bit C/C++ compilation toolchain -RUN cinst -y --no-progress msys2 --params \"/NoUpdate\" --version $ENV:MSYS_VERSION ; \ - ridk install 3 +RUN scripts/postsetup.ps1 # Install aws cli -RUN powershell -C .\scripts\install_awscli.ps1 +RUN scripts/install_awscli.ps1 # Install docker, manifest-tool and notary -RUN powershell -Command .\scripts\install_docker.ps1 +RUN scripts/install_docker.ps1 # Install embedded pythons (for unit testing) -RUN powershell -C .\scripts\install_embedded_pythons.ps1 - -### We need both the .NET 3.5 runtime and -### the .NET 4.8 runtime. To do this, on 1809 we get 4.8 from a base image and we -### manually the install .NET Framework 3.5 runtime using the instructions in -### the mcr.microsoft.com/dotnet/framework/runtime:3.5 Dockerfile: -### https://github.com/microsoft/dotnet-framework-docker/blob/26597e42d157cc1e09d1e0dc8f23c32e6c3d1467/3.5/runtime/windowsservercore-ltsc2019/Dockerfile - -### 1909 includes .NET Framework 4.8 by default, so we use the image that includes -### .NET Framework 3.5 as well, and thus we don't need to install it manually here. - -### The .NET Fx 3.5 is needed for the Visual C++ Compiler for Python 2.7 -### (https://www.microsoft.com/en-us/download/details.aspx?id=44266) -### and to work around a bug in the WiX 3.11 installer -### (https://github.com/wixtoolset/issues/issues/5661). - -# Install .NET Fx 3.5 -RUN if ($Env:WINDOWS_VERSION -eq '1809') { .\scripts\install_net35_1809.bat } - -# Install VS2017 -RUN cinst -y --no-progress visualstudio2017buildtools --version $ENV:VS2017BUILDTOOLS_VERSION --params \"--add Microsoft.VisualStudio.ComponentGroup.NativeDesktop.Win81 --add Microsoft.VisualStudio.Workload.VCTools\" ; \ - setx VSTUDIO_ROOT \"${env:ProgramFiles(x86)}\Microsoft Visual Studio\2017\BuildTools\" ; \ - [Environment]::SetEnvironmentVariable(\"Path\", [Environment]::GetEnvironmentVariable(\"Path\", [EnvironmentVariableTarget]::Machine) + \";${env:ProgramFiles(x86)}\Windows Kits\8.1\bin\x64\", [System.EnvironmentVariableTarget]::Machine) - -# Install VC compiler for Python 2.7 -RUN cinst -y --no-progress vcpython27 --version $ENV:VCPYTHON27_VERSION - -# Install Wix and update PATH to include it -RUN cinst -y --no-progress wixtoolset --version $ENV:WIX_VERSION ; \ - [Environment]::SetEnvironmentVariable(\"Path\", [Environment]::GetEnvironmentVariable(\"Path\", [EnvironmentVariableTarget]::Machine) + \";${env:ProgramFiles(x86)}\WiX Toolset v3.11\bin\", [System.EnvironmentVariableTarget]::Machine) +RUN scripts/install_embedded_pythons.ps1 ENTRYPOINT ["scripts/entrypoint.bat"] diff --git a/windows/Dockerfile-x86 b/windows/Dockerfile-x86 deleted file mode 100644 index 07926b131..000000000 --- a/windows/Dockerfile-x86 +++ /dev/null @@ -1,143 +0,0 @@ -# Use the Microsoft-provided .NET Runtime 4.8 image as the base image -# because installing it in the image with Chocolatey requires a reboot. - -# There is a bug in the February 11, 2020 update that makes commands fail in -# docker containers (see: https://support.microsoft.com/en-us/help/4542617/you-might-encounter-issues-when-using-windows-server-containers-with-t) -# To avoid that, there are two solutions: -# - both the host and the container must have the February 11, 2020 update, or -# - neither the host and the container must have the February 11, 2020 update. -# Since our 1809 windows-docker host image does not have this update, we use a base -# container image that does not have this update either (thus the 20200114 tag). -# On the contrary, since our 1909 windows-docker host image does have this update, -# we use a base container image that does have this update. -# TODO: Once the 1809 windows-docker host image is updated, update the base container image. -ARG BASE_IMAGE=mcr.microsoft.com/dotnet/framework/runtime:4.8-20200114-windowsservercore-ltsc2019 - -FROM ${BASE_IMAGE} - -SHELL ["powershell", "-Command"] - -ARG WINDOWS_VERSION -ENV WINDOWS_VERSION=${WINDOWS_VERSION:-1809} - -# Chocolatey package versions -ENV GIT_VERSION="2.26.2" \ - SEVENZIP_VERSION="19.0" \ - VS2017BUILDTOOLS_VERSION="15.9.23.0" \ - VCPYTHON27_VERSION="9.0.0.30729" \ - GO_VERSION="1.13.8" \ - RUBY_VERSION="2.4.3.1" \ - PYTHON_VERSION="2.7.17" \ - WIX_VERSION="3.11.2" \ - CMAKE_VERSION="3.17.2" \ - MSYS_VERSION="20190524.0.0.20191030" \ - EMBEDDED_PYTHON_2_VERSION="2.7.17" \ - EMBEDDED_PYTHON_3_VERSION="3.8.1" \ - CACERTS_HASH="ADF770DFD574A0D6026BFAA270CB6879B063957177A991D453FF1D302C02081F" - -LABEL target_agent="Agent 6/7" \ - target_arch="x86" \ - windows_version=${WINDOWS_VERSION} \ - git_version=${GIT_VERSION} \ - sevenzip_version=${SEVENZIP_VERSION} \ - vs2017buildtools_version=${VS2017BUILDTOOLS_VERSION} \ - vcpython27_version=${VCPYTHON27_VERSION} \ - go_version=${GO_VERSION} \ - ruby_version=${RUBY_VERSION} \ - wix_version=${WIX_VERSION} \ - cmake_version=${CMAKE_VERSION} \ - msys_version=${MSYS_VERSION} \ - system_python_version=${PYTHON_VERSION} \ - embedded_py2_version=${EMBEDDED_PYTHON_2_VERSION} \ - embedded_py3_version=${EMBEDDED_PYTHON_3_VERSION} - -COPY ./windows/scripts scripts - -# Add certificates needed for build & check certificates file hash -# We need to trust the DigiCert High Assurance EV Root CA certificate, which signs python.org, -# to be able to download some Python components during the Agent build. -RUN (New-Object System.Net.WebClient).DownloadFile(\"https://curl.haxx.se/ca/cacert.pem\", \"cacert.pem\") ; \ - if ((Get-FileHash .\cacert.pem).Hash -ne \"$ENV:CACERTS_HASH\") { Write-Host \"Wrong hashsum for cacert.pem: got '$((Get-FileHash .\cacert.pem).Hash)', expected '$ENV:CACERTS_HASH'.\"; exit 1 } ; \ - setx SSL_CERT_FILE \"C:\cacert.pem\" - -# Set 32-bit flag env var -RUN setx WINDOWS_BUILD_32_BIT 1 - -# Install Chocolatey -RUN $env:chocolateyUseWindowsCompression = 'true'; Invoke-Expression ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1')) - -# Install git -### HACK: we disable symbolic links when cloning repositories -### to work around a symlink-related failure in the agent-binaries omnibus project -### when copying the datadog-agent project twice. - -RUN $env:chocolateyUseWindowsCompression = 'true'; cinst -y --no-progress git -x86 --version $ENV:GIT_VERSION -RUN git config --system core.symlinks false - -# Install 7zip -RUN $env:chocolateyUseWindowsCompression = 'true'; cinst -y --no-progress 7zip -x86 --version $ENV:SEVENZIP_VERSION - -# Install Cmake and update PATH to include it -RUN cinst -y --no-progress cmake -x86 --version $ENV:CMAKE_VERSION ; \ - [Environment]::SetEnvironmentVariable(\"Path\", [Environment]::GetEnvironmentVariable(\"Path\", [EnvironmentVariableTarget]::Machine) + \";${Env:ProgramFiles(x86)}\CMake\bin\", [System.EnvironmentVariableTarget]::Machine) ; \ - [Environment]::SetEnvironmentVariable(\"Path\", [Environment]::GetEnvironmentVariable(\"Path\", [EnvironmentVariableTarget]::Machine) + \";${env:ProgramFiles}\CMake\bin\", [System.EnvironmentVariableTarget]::Machine) - -# Install golang and set GOPATH to the dev path used in builds & tests -RUN cinst -y --no-progress golang -x86 --version $ENV:GO_VERSION ; \ - setx GOPATH C:\dev\go - -# Install system Python 2 (to use invoke) -RUN cinst -y --no-progress python2 -x86 --version $ENV:PYTHON_VERSION - -# Install 64-bit ruby and bundler (for omnibus builds) -RUN cinst -y --no-progress ruby --version $ENV:RUBY_VERSION -RUN setx RIDK ((Get-Command ridk).Path) ; \ - gem install bundler - -# Install msys2 system & install 64-bit C/C++ compilation toolchain -RUN cinst -y --no-progress msys2 --params \"/NoUpdate\" --version $ENV:MSYS_VERSION ; \ - ridk install 3 - -# (32-bit only) Install 32-bit C/C++ compilation toolchain -RUN ridk enable; bash -c \"pacman -S --needed --noconfirm mingw-w64-i686-binutils mingw-w64-i686-crt-git mingw-w64-i686-gcc mingw-w64-i686-gcc-libs mingw-w64-i686-headers-git mingw-w64-i686-libmangle-git mingw-w64-i686-libwinpthread-git mingw-w64-i686-make mingw-w64-i686-pkg-config mingw-w64-i686-tools-git mingw-w64-i686-winpthreads-git\" ; \ - [Environment]::SetEnvironmentVariable(\"Path\", [Environment]::GetEnvironmentVariable(\"Path\", [EnvironmentVariableTarget]::Machine) + \";C:\tools\msys64\mingw32\bin;C:\tools\msys64\usr\bin\", [System.EnvironmentVariableTarget]::Machine) - -# Install aws cli -RUN powershell -C .\scripts\install_awscli.ps1 - -# Install docker, manifest-tool and notary -RUN powershell -Command .\scripts\install_docker.ps1 - -# Install embedded pythons (for unit testing) -RUN powershell -C .\scripts\install_embedded_pythons.ps1 - -### We need both the .NET 3.5 runtime and -### the .NET 4.8 runtime. To do this, on 1809 we get 4.8 from a base image and we -### manually the install .NET Framework 3.5 runtime using the instructions in -### the mcr.microsoft.com/dotnet/framework/runtime:3.5 Dockerfile: -### https://github.com/microsoft/dotnet-framework-docker/blob/26597e42d157cc1e09d1e0dc8f23c32e6c3d1467/3.5/runtime/windowsservercore-ltsc2019/Dockerfile - -### 1909 includes .NET Framework 4.8 by default, so we use the image that includes -### .NET Framework 3.5 as well, and thus we don't need to install it manually here. - -### The .NET Fx 3.5 is needed for the Visual C++ Compiler for Python 2.7 -### (https://www.microsoft.com/en-us/download/details.aspx?id=44266) -### and to work around a bug in the WiX 3.11 installer -### (https://github.com/wixtoolset/issues/issues/5661). - -# Install .NET Fx 3.5 -RUN if ($Env:WINDOWS_VERSION -eq '1809') { .\scripts\install_net35_1809.bat } - -# Install VS2017 -RUN cinst -y --no-progress visualstudio2017buildtools -x86 --version $ENV:VS2017BUILDTOOLS_VERSION --params \"--add Microsoft.VisualStudio.ComponentGroup.NativeDesktop.Win81 --add Microsoft.VisualStudio.Workload.VCTools\" ; \ - setx VSTUDIO_ROOT \"${env:ProgramFiles(x86)}\Microsoft Visual Studio\2017\BuildTools\" ; \ - [Environment]::SetEnvironmentVariable(\"Path\", [Environment]::GetEnvironmentVariable(\"Path\", [EnvironmentVariableTarget]::Machine) + \";${env:ProgramFiles(x86)}\Windows Kits\8.1\bin\x64\", [System.EnvironmentVariableTarget]::Machine) - -# Install VC compiler for Python 2.7 -RUN cinst -y --no-progress vcpython27 -x86 --version $ENV:VCPYTHON27_VERSION - -# Install Wix and update PATH to include it -RUN cinst -y --no-progress wixtoolset -x86 --version $ENV:WIX_VERSION ; \ - [Environment]::SetEnvironmentVariable(\"Path\", [Environment]::GetEnvironmentVariable(\"Path\", [EnvironmentVariableTarget]::Machine) + \";${env:ProgramFiles(x86)}\WiX Toolset v3.11\bin\", [System.EnvironmentVariableTarget]::Machine) - -ENTRYPOINT ["scripts/entrypoint.bat"] diff --git a/windows/scripts/install_net35_1909.bat b/windows/scripts/install_net35_1909.bat deleted file mode 100644 index 2bf39aded..000000000 --- a/windows/scripts/install_net35_1909.bat +++ /dev/null @@ -1,21 +0,0 @@ -:: -:: Installs .NET Runtime 3.5, used by Wix 3.11 and the Visual C++ Compiler for Python 2.7 -:: Taken from the mcr.microsoft.com/dotnet/framework/runtime:3.5 Dockerfile: -:: https://github.com/microsoft/dotnet-framework-docker/blob/abc2ca65b28058f7c71ec8cd8763a8fbf2a9c03f/3.5/runtime/windowsservercore-1909/Dockerfile -:: - -set DOTNET_RUNNING_IN_CONTAINER true - -curl -fSLo microsoft-windows-netfx3.zip https://dotnetbinaries.blob.core.windows.net/dockerassets/microsoft-windows-netfx3-1909.zip -tar -zxf microsoft-windows-netfx3.zip -del /F /Q microsoft-windows-netfx3.zip -DISM /Online /Quiet /Add-Package /PackagePath:.\microsoft-windows-netfx3-ondemand-package~31bf3856ad364e35~amd64~~.cab -del microsoft-windows-netfx3-ondemand-package~31bf3856ad364e35~amd64~~.cab -powershell Remove-Item -Force -Recurse ${Env:TEMP}\* - -curl -fSLo patch.msu http://download.windowsupdate.com/d/msdownload/update/software/updt/2020/01/windows10.0-kb4534132-x64-ndp48_21067bd5f9c305ee6a6cee79db6ca38587cb6ad8.msu -mkdir patch -expand patch.msu patch -F:* -del /F /Q patch.msu -DISM /Online /Quiet /Add-Package /PackagePath:C:\patch\windows10.0-kb4534132-x64-ndp48.cab -rmdir /S /Q patch \ No newline at end of file diff --git a/windows/scripts/postsetup.ps1 b/windows/scripts/postsetup.ps1 new file mode 100644 index 000000000..3961d5045 --- /dev/null +++ b/windows/scripts/postsetup.ps1 @@ -0,0 +1,18 @@ +$ErrorActionPreference = 'Stop' + +# git, ridk and bundler aren't available yet right after they're installed, you need to restart a shell + +### HACK: we disable symbolic links when cloning repositories +### to work around a symlink-related failure in the agent-binaries omnibus project +### when copying the datadog-agent project twice. +git config --system core.symlinks false + +# Install 64-bit C/C++ compilation toolchain +ridk install 3 +setx RIDK ((Get-Command ridk).Path) + +# (32-bit only) Install 32-bit C/C++ compilation toolchain +if ($Env:TARGET_ARCH -eq 'x86') { ridk enable; bash -c "pacman -S --needed --noconfirm mingw-w64-i686-binutils mingw-w64-i686-crt-git mingw-w64-i686-gcc mingw-w64-i686-gcc-libs mingw-w64-i686-headers-git mingw-w64-i686-libmangle-git mingw-w64-i686-libwinpthread-git mingw-w64-i686-make mingw-w64-i686-pkg-config mingw-w64-i686-tools-git mingw-w64-i686-winpthreads-git" } +if ($Env:TARGET_ARCH -eq 'x86') { [Environment]::SetEnvironmentVariable("Path", [Environment]::GetEnvironmentVariable("Path", [EnvironmentVariableTarget]::Machine) + ";C:\tools\msys64\mingw32\bin;C:\tools\msys64\usr\bin", [System.EnvironmentVariableTarget]::Machine) } + +gem install bundler diff --git a/windows/scripts/setup.ps1 b/windows/scripts/setup.ps1 new file mode 100644 index 000000000..9bcb69d48 --- /dev/null +++ b/windows/scripts/setup.ps1 @@ -0,0 +1,69 @@ +$ErrorActionPreference = 'Stop' + +# Add certificates needed for build & check certificates file hash +# We need to trust the DigiCert High Assurance EV Root CA certificate, which signs python.org, +# to be able to download some Python components during the Agent build. +(New-Object System.Net.WebClient).DownloadFile("https://curl.haxx.se/ca/cacert.pem", "cacert.pem") +if ((Get-FileHash .\cacert.pem).Hash -ne "$ENV:CACERTS_HASH") { Write-Host "Wrong hashsum for cacert.pem: got '$((Get-FileHash .\cacert.pem).Hash)', expected '$ENV:CACERTS_HASH'."; exit 1 } +setx SSL_CERT_FILE "C:\cacert.pem" + +# Set 32-bit flag env var +if ($Env:TARGET_ARCH -eq 'x86') { setx CHOCO_ARCH_FLAG '-x86' ; setx WINDOWS_BUILD_32_BIT 1 } + +# Install Chocolatey +$env:chocolateyUseWindowsCompression = 'true'; Invoke-Expression ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1')) + +# Install git +$env:chocolateyUseWindowsCompression = 'true'; cinst -y --no-progress git $ENV:CHOCO_ARCH_FLAG --version $ENV:GIT_VERSION + +# Install 7zip +$env:chocolateyUseWindowsCompression = 'true'; cinst -y --no-progress 7zip $ENV:CHOCO_ARCH_FLAG --version $ENV:SEVENZIP_VERSION + +# Install Cmake and update PATH to include it +cinst -y --no-progress cmake $ENV:CHOCO_ARCH_FLAG --version $ENV:CMAKE_VERSION +if ($Env:TARGET_ARCH -eq 'x86') { [Environment]::SetEnvironmentVariable("Path", [Environment]::GetEnvironmentVariable("Path", [EnvironmentVariableTarget]::Machine) + ";${Env:ProgramFiles(x86)}\CMake\bin", [System.EnvironmentVariableTarget]::Machine) } +if ($Env:TARGET_ARCH -eq 'x64') { [Environment]::SetEnvironmentVariable("Path", [Environment]::GetEnvironmentVariable("Path", [EnvironmentVariableTarget]::Machine) + ";${env:ProgramFiles}\CMake\bin", [System.EnvironmentVariableTarget]::Machine) } + +# Install golang and set GOPATH to the dev path used in builds & tests +cinst -y --no-progress golang $ENV:CHOCO_ARCH_FLAG --version $ENV:GO_VERSION +setx GOPATH C:\dev\go + +# Install system Python 2 (to use invoke) +cinst -y --no-progress python2 $ENV:CHOCO_ARCH_FLAG --version $ENV:PYTHON_VERSION + +# Install 64-bit ruby and bundler (for omnibus builds) +cinst -y --no-progress ruby $ENV:CHOCO_ARCH_FLAG --version $ENV:RUBY_VERSION + +### We need both the .NET 3.5 runtime and +### the .NET 4.8 runtime. To do this, on 1809 we get 4.8 from a base image and we +### manually the install .NET Framework 3.5 runtime using the instructions in +### the mcr.microsoft.com/dotnet/framework/runtime:3.5 Dockerfile: +### https://github.com/microsoft/dotnet-framework-docker/blob/26597e42d157cc1e09d1e0dc8f23c32e6c3d1467/3.5/runtime/windowsservercore-ltsc2019/Dockerfile + +### 1909 includes .NET Framework 4.8 by default, so we use the image that includes +### .NET Framework 3.5 as well, and thus we don't need to install it manually here. + +### The .NET Fx 3.5 is needed for the Visual C++ Compiler for Python 2.7 +### (https://www.microsoft.com/en-us/download/details.aspx?id=44266) +### and to work around a bug in the WiX 3.11 installer +### (https://github.com/wixtoolset/issues/issues/5661). + +# Install .NET Fx 3.5 +if ($Env:WINDOWS_VERSION -eq '1809') { .\scripts\install_net35_1809.bat } + +# Install VS2017 +cinst -y --no-progress visualstudio2017buildtools $ENV:CHOCO_ARCH_FLAG --version $ENV:VS2017BUILDTOOLS_VERSION --params "--add Microsoft.VisualStudio.ComponentGroup.NativeDesktop.Win81 --add Microsoft.VisualStudio.Workload.VCTools" +setx VSTUDIO_ROOT "${env:ProgramFiles(x86)}\Microsoft Visual Studio\2017\BuildTools" + +# Add signtool to path +[Environment]::SetEnvironmentVariable("Path", [Environment]::GetEnvironmentVariable("Path", [EnvironmentVariableTarget]::Machine) + ";${env:ProgramFiles(x86)}\Windows Kits\8.1\bin\x64", [System.EnvironmentVariableTarget]::Machine) + +# Install VC compiler for Python 2.7 +cinst -y --no-progress vcpython27 $ENV:CHOCO_ARCH_FLAG --version $ENV:VCPYTHON27_VERSION + +# Install Wix and update PATH to include it +cinst -y --no-progress wixtoolset $ENV:CHOCO_ARCH_FLAG --version $ENV:WIX_VERSION +[Environment]::SetEnvironmentVariable("Path", [Environment]::GetEnvironmentVariable("Path", [EnvironmentVariableTarget]::Machine) + ";${env:ProgramFiles(x86)}\WiX Toolset v3.11\bin", [System.EnvironmentVariableTarget]::Machine) + +# Install msys2 system +cinst -y --no-progress msys2 --params "/NoUpdate" --version $ENV:MSYS_VERSION From 9fa3bc4bbfebe87d5f81c57e32aa520a56e81465 Mon Sep 17 00:00:00 2001 From: Kylian Serrania Date: Mon, 8 Jun 2020 21:46:35 +0200 Subject: [PATCH 5/8] Correct entrypoint --- windows/Dockerfile | 2 +- windows/scripts/entrypoint.bat | 2 +- windows/scripts/postsetup.ps1 | 1 + windows/scripts/setup.ps1 | 4 ++-- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/windows/Dockerfile b/windows/Dockerfile index de3b807dd..bb8183a10 100644 --- a/windows/Dockerfile +++ b/windows/Dockerfile @@ -69,4 +69,4 @@ RUN scripts/install_docker.ps1 # Install embedded pythons (for unit testing) RUN scripts/install_embedded_pythons.ps1 -ENTRYPOINT ["scripts/entrypoint.bat"] +ENTRYPOINT ["/scripts/entrypoint.bat"] diff --git a/windows/scripts/entrypoint.bat b/windows/scripts/entrypoint.bat index 045b54602..6e778d88d 100644 --- a/windows/scripts/entrypoint.bat +++ b/windows/scripts/entrypoint.bat @@ -14,7 +14,7 @@ @echo AWS_NETWORKING is %AWS_NETWORKING% if defined AWS_NETWORKING ( @echo Detected AWS container, setting up networking - powershell -C "c:\aws_networking.ps1" + powershell -C "c:\scripts\aws_networking.ps1" ) %* exit /b %ERRORLEVEL% diff --git a/windows/scripts/postsetup.ps1 b/windows/scripts/postsetup.ps1 index 3961d5045..7ad922c80 100644 --- a/windows/scripts/postsetup.ps1 +++ b/windows/scripts/postsetup.ps1 @@ -15,4 +15,5 @@ setx RIDK ((Get-Command ridk).Path) if ($Env:TARGET_ARCH -eq 'x86') { ridk enable; bash -c "pacman -S --needed --noconfirm mingw-w64-i686-binutils mingw-w64-i686-crt-git mingw-w64-i686-gcc mingw-w64-i686-gcc-libs mingw-w64-i686-headers-git mingw-w64-i686-libmangle-git mingw-w64-i686-libwinpthread-git mingw-w64-i686-make mingw-w64-i686-pkg-config mingw-w64-i686-tools-git mingw-w64-i686-winpthreads-git" } if ($Env:TARGET_ARCH -eq 'x86') { [Environment]::SetEnvironmentVariable("Path", [Environment]::GetEnvironmentVariable("Path", [EnvironmentVariableTarget]::Machine) + ";C:\tools\msys64\mingw32\bin;C:\tools\msys64\usr\bin", [System.EnvironmentVariableTarget]::Machine) } +# Install bundler for omnibus builds gem install bundler diff --git a/windows/scripts/setup.ps1 b/windows/scripts/setup.ps1 index 9bcb69d48..f75ca4534 100644 --- a/windows/scripts/setup.ps1 +++ b/windows/scripts/setup.ps1 @@ -31,8 +31,8 @@ setx GOPATH C:\dev\go # Install system Python 2 (to use invoke) cinst -y --no-progress python2 $ENV:CHOCO_ARCH_FLAG --version $ENV:PYTHON_VERSION -# Install 64-bit ruby and bundler (for omnibus builds) -cinst -y --no-progress ruby $ENV:CHOCO_ARCH_FLAG --version $ENV:RUBY_VERSION +# Install 64-bit ruby (for omnibus builds) +cinst -y --no-progress ruby --version $ENV:RUBY_VERSION ### We need both the .NET 3.5 runtime and ### the .NET 4.8 runtime. To do this, on 1809 we get 4.8 from a base image and we From b98cddb0391a752fe37340aceee3b77ad73e1712 Mon Sep 17 00:00:00 2001 From: Kylian Serrania Date: Tue, 9 Jun 2020 11:40:31 +0200 Subject: [PATCH 6/8] Update comments --- windows/Dockerfile | 2 ++ windows/scripts/setup.ps1 | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/windows/Dockerfile b/windows/Dockerfile index bb8183a10..193fdca9e 100644 --- a/windows/Dockerfile +++ b/windows/Dockerfile @@ -56,8 +56,10 @@ LABEL target_agent="Agent 6/7" \ COPY ./windows/scripts scripts +# Install builder tools RUN scripts/setup.ps1 +# Run setup commands that need builder tools RUN scripts/postsetup.ps1 # Install aws cli diff --git a/windows/scripts/setup.ps1 b/windows/scripts/setup.ps1 index f75ca4534..f57052bdd 100644 --- a/windows/scripts/setup.ps1 +++ b/windows/scripts/setup.ps1 @@ -34,8 +34,8 @@ cinst -y --no-progress python2 $ENV:CHOCO_ARCH_FLAG --version $ENV:PYTHON_VERSIO # Install 64-bit ruby (for omnibus builds) cinst -y --no-progress ruby --version $ENV:RUBY_VERSION -### We need both the .NET 3.5 runtime and -### the .NET 4.8 runtime. To do this, on 1809 we get 4.8 from a base image and we +### We need both the .NET 3.5 runtime and the .NET 4.8 runtime. +### To do this, on 1809 we get 4.8 from a base image and we ### manually the install .NET Framework 3.5 runtime using the instructions in ### the mcr.microsoft.com/dotnet/framework/runtime:3.5 Dockerfile: ### https://github.com/microsoft/dotnet-framework-docker/blob/26597e42d157cc1e09d1e0dc8f23c32e6c3d1467/3.5/runtime/windowsservercore-ltsc2019/Dockerfile From 320d4b9ef40a4fd804229b51b4132655b4dd1e6e Mon Sep 17 00:00:00 2001 From: Kylian Serrania Date: Tue, 9 Jun 2020 16:49:35 +0200 Subject: [PATCH 7/8] Add checks --- windows/scripts/postsetup.ps1 | 15 +++++++++++++++ windows/scripts/setup.ps1 | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+) diff --git a/windows/scripts/postsetup.ps1 b/windows/scripts/postsetup.ps1 index 7ad922c80..652e788f4 100644 --- a/windows/scripts/postsetup.ps1 +++ b/windows/scripts/postsetup.ps1 @@ -1,5 +1,13 @@ $ErrorActionPreference = 'Stop' +function Code-Check { + param ($Code, $Element) + if ($Code -ne 0) { + Write-Host -ForegroundColor Red "$Element install failed" + [Environment]::Exit(1) + } +} + # git, ridk and bundler aren't available yet right after they're installed, you need to restart a shell ### HACK: we disable symbolic links when cloning repositories @@ -9,11 +17,18 @@ git config --system core.symlinks false # Install 64-bit C/C++ compilation toolchain ridk install 3 +Code-Check -Code $LASTEXITCODE -Element "ridk" + + setx RIDK ((Get-Command ridk).Path) # (32-bit only) Install 32-bit C/C++ compilation toolchain if ($Env:TARGET_ARCH -eq 'x86') { ridk enable; bash -c "pacman -S --needed --noconfirm mingw-w64-i686-binutils mingw-w64-i686-crt-git mingw-w64-i686-gcc mingw-w64-i686-gcc-libs mingw-w64-i686-headers-git mingw-w64-i686-libmangle-git mingw-w64-i686-libwinpthread-git mingw-w64-i686-make mingw-w64-i686-pkg-config mingw-w64-i686-tools-git mingw-w64-i686-winpthreads-git" } +Code-Check -Code $LASTEXITCODE -Element "32-bit ridk" + + if ($Env:TARGET_ARCH -eq 'x86') { [Environment]::SetEnvironmentVariable("Path", [Environment]::GetEnvironmentVariable("Path", [EnvironmentVariableTarget]::Machine) + ";C:\tools\msys64\mingw32\bin;C:\tools\msys64\usr\bin", [System.EnvironmentVariableTarget]::Machine) } # Install bundler for omnibus builds gem install bundler +Code-Check -Code $LASTEXITCODE -Element "bundler" diff --git a/windows/scripts/setup.ps1 b/windows/scripts/setup.ps1 index f57052bdd..58d0d9a14 100644 --- a/windows/scripts/setup.ps1 +++ b/windows/scripts/setup.ps1 @@ -1,5 +1,13 @@ $ErrorActionPreference = 'Stop' +function Code-Check { + param ($Code, $Element) + if ($Code -ne 0) { + Write-Host -ForegroundColor Red "$Element install failed" + [Environment]::Exit(1) + } +} + # Add certificates needed for build & check certificates file hash # We need to trust the DigiCert High Assurance EV Root CA certificate, which signs python.org, # to be able to download some Python components during the Agent build. @@ -12,27 +20,43 @@ if ($Env:TARGET_ARCH -eq 'x86') { setx CHOCO_ARCH_FLAG '-x86' ; setx WINDOWS_BUI # Install Chocolatey $env:chocolateyUseWindowsCompression = 'true'; Invoke-Expression ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1')) +Code-Check -Code $LASTEXITCODE -Element "chocolatey" + # Install git $env:chocolateyUseWindowsCompression = 'true'; cinst -y --no-progress git $ENV:CHOCO_ARCH_FLAG --version $ENV:GIT_VERSION +Code-Check -Code $LASTEXITCODE -Element "git" + # Install 7zip $env:chocolateyUseWindowsCompression = 'true'; cinst -y --no-progress 7zip $ENV:CHOCO_ARCH_FLAG --version $ENV:SEVENZIP_VERSION +Code-Check -Code $LASTEXITCODE -Element "7zip" + # Install Cmake and update PATH to include it cinst -y --no-progress cmake $ENV:CHOCO_ARCH_FLAG --version $ENV:CMAKE_VERSION +Code-Check -Code $LASTEXITCODE -Element "cmake" + + if ($Env:TARGET_ARCH -eq 'x86') { [Environment]::SetEnvironmentVariable("Path", [Environment]::GetEnvironmentVariable("Path", [EnvironmentVariableTarget]::Machine) + ";${Env:ProgramFiles(x86)}\CMake\bin", [System.EnvironmentVariableTarget]::Machine) } if ($Env:TARGET_ARCH -eq 'x64') { [Environment]::SetEnvironmentVariable("Path", [Environment]::GetEnvironmentVariable("Path", [EnvironmentVariableTarget]::Machine) + ";${env:ProgramFiles}\CMake\bin", [System.EnvironmentVariableTarget]::Machine) } # Install golang and set GOPATH to the dev path used in builds & tests cinst -y --no-progress golang $ENV:CHOCO_ARCH_FLAG --version $ENV:GO_VERSION +Code-Check -Code $LASTEXITCODE -Element "golang" + + setx GOPATH C:\dev\go # Install system Python 2 (to use invoke) cinst -y --no-progress python2 $ENV:CHOCO_ARCH_FLAG --version $ENV:PYTHON_VERSION +Code-Check -Code $LASTEXITCODE -Element "python2" + # Install 64-bit ruby (for omnibus builds) cinst -y --no-progress ruby --version $ENV:RUBY_VERSION +Code-Check -Code $LASTEXITCODE -Element "ruby" + ### We need both the .NET 3.5 runtime and the .NET 4.8 runtime. ### To do this, on 1809 we get 4.8 from a base image and we @@ -50,9 +74,14 @@ cinst -y --no-progress ruby --version $ENV:RUBY_VERSION # Install .NET Fx 3.5 if ($Env:WINDOWS_VERSION -eq '1809') { .\scripts\install_net35_1809.bat } +Code-Check -Code $LASTEXITCODE -Element ".NET 3.5" + # Install VS2017 cinst -y --no-progress visualstudio2017buildtools $ENV:CHOCO_ARCH_FLAG --version $ENV:VS2017BUILDTOOLS_VERSION --params "--add Microsoft.VisualStudio.ComponentGroup.NativeDesktop.Win81 --add Microsoft.VisualStudio.Workload.VCTools" +Code-Check -Code $LASTEXITCODE -Element "visualstudio2017buildtools" + + setx VSTUDIO_ROOT "${env:ProgramFiles(x86)}\Microsoft Visual Studio\2017\BuildTools" # Add signtool to path @@ -60,10 +89,16 @@ setx VSTUDIO_ROOT "${env:ProgramFiles(x86)}\Microsoft Visual Studio\2017\BuildTo # Install VC compiler for Python 2.7 cinst -y --no-progress vcpython27 $ENV:CHOCO_ARCH_FLAG --version $ENV:VCPYTHON27_VERSION +Code-Check -Code $LASTEXITCODE -Element "vcpython27" + # Install Wix and update PATH to include it cinst -y --no-progress wixtoolset $ENV:CHOCO_ARCH_FLAG --version $ENV:WIX_VERSION +Code-Check -Code $LASTEXITCODE -Element "wixtoolset" + + [Environment]::SetEnvironmentVariable("Path", [Environment]::GetEnvironmentVariable("Path", [EnvironmentVariableTarget]::Machine) + ";${env:ProgramFiles(x86)}\WiX Toolset v3.11\bin", [System.EnvironmentVariableTarget]::Machine) # Install msys2 system cinst -y --no-progress msys2 --params "/NoUpdate" --version $ENV:MSYS_VERSION +Code-Check -Code $LASTEXITCODE -Element "msys2" From 8ec90fe7d858d61b507e927c3ba14e57cf537de0 Mon Sep 17 00:00:00 2001 From: Kylian Serrania Date: Thu, 11 Jun 2020 13:54:10 +0200 Subject: [PATCH 8/8] Split long lines --- windows/scripts/postsetup.ps1 | 15 +++++++++++++-- windows/scripts/setup.ps1 | 30 ++++++++++++++++++++++-------- 2 files changed, 35 insertions(+), 10 deletions(-) diff --git a/windows/scripts/postsetup.ps1 b/windows/scripts/postsetup.ps1 index 652e788f4..fe4c356bd 100644 --- a/windows/scripts/postsetup.ps1 +++ b/windows/scripts/postsetup.ps1 @@ -23,11 +23,22 @@ Code-Check -Code $LASTEXITCODE -Element "ridk" setx RIDK ((Get-Command ridk).Path) # (32-bit only) Install 32-bit C/C++ compilation toolchain -if ($Env:TARGET_ARCH -eq 'x86') { ridk enable; bash -c "pacman -S --needed --noconfirm mingw-w64-i686-binutils mingw-w64-i686-crt-git mingw-w64-i686-gcc mingw-w64-i686-gcc-libs mingw-w64-i686-headers-git mingw-w64-i686-libmangle-git mingw-w64-i686-libwinpthread-git mingw-w64-i686-make mingw-w64-i686-pkg-config mingw-w64-i686-tools-git mingw-w64-i686-winpthreads-git" } +if ($Env:TARGET_ARCH -eq 'x86') { + ridk enable + bash -c "pacman -S --needed --noconfirm mingw-w64-i686-binutils mingw-w64-i686-crt-git \ + mingw-w64-i686-gcc mingw-w64-i686-gcc-libs mingw-w64-i686-headers-git mingw-w64-i686-libmangle-git \ + mingw-w64-i686-libwinpthread-git mingw-w64-i686-make mingw-w64-i686-pkg-config mingw-w64-i686-tools-git \ + mingw-w64-i686-winpthreads-git" +} + Code-Check -Code $LASTEXITCODE -Element "32-bit ridk" -if ($Env:TARGET_ARCH -eq 'x86') { [Environment]::SetEnvironmentVariable("Path", [Environment]::GetEnvironmentVariable("Path", [EnvironmentVariableTarget]::Machine) + ";C:\tools\msys64\mingw32\bin;C:\tools\msys64\usr\bin", [System.EnvironmentVariableTarget]::Machine) } +if ($Env:TARGET_ARCH -eq 'x86') { + [Environment]::SetEnvironmentVariable("Path", + [Environment]::GetEnvironmentVariable("Path", [EnvironmentVariableTarget]::Machine) + + ";C:\tools\msys64\mingw32\bin;C:\tools\msys64\usr\bin", [System.EnvironmentVariableTarget]::Machine) +} # Install bundler for omnibus builds gem install bundler diff --git a/windows/scripts/setup.ps1 b/windows/scripts/setup.ps1 index 58d0d9a14..5feeb2141 100644 --- a/windows/scripts/setup.ps1 +++ b/windows/scripts/setup.ps1 @@ -19,27 +19,38 @@ setx SSL_CERT_FILE "C:\cacert.pem" if ($Env:TARGET_ARCH -eq 'x86') { setx CHOCO_ARCH_FLAG '-x86' ; setx WINDOWS_BUILD_32_BIT 1 } # Install Chocolatey -$env:chocolateyUseWindowsCompression = 'true'; Invoke-Expression ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1')) +$env:chocolateyUseWindowsCompression = 'true' +Invoke-Expression ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1')) Code-Check -Code $LASTEXITCODE -Element "chocolatey" # Install git -$env:chocolateyUseWindowsCompression = 'true'; cinst -y --no-progress git $ENV:CHOCO_ARCH_FLAG --version $ENV:GIT_VERSION +cinst -y --no-progress git $ENV:CHOCO_ARCH_FLAG --version $ENV:GIT_VERSION Code-Check -Code $LASTEXITCODE -Element "git" # Install 7zip -$env:chocolateyUseWindowsCompression = 'true'; cinst -y --no-progress 7zip $ENV:CHOCO_ARCH_FLAG --version $ENV:SEVENZIP_VERSION +cinst -y --no-progress 7zip $ENV:CHOCO_ARCH_FLAG --version $ENV:SEVENZIP_VERSION Code-Check -Code $LASTEXITCODE -Element "7zip" +# We have 7zip now, no need for Windows compression +$env:chocolateyUseWindowsCompression = 'false' # Install Cmake and update PATH to include it cinst -y --no-progress cmake $ENV:CHOCO_ARCH_FLAG --version $ENV:CMAKE_VERSION Code-Check -Code $LASTEXITCODE -Element "cmake" -if ($Env:TARGET_ARCH -eq 'x86') { [Environment]::SetEnvironmentVariable("Path", [Environment]::GetEnvironmentVariable("Path", [EnvironmentVariableTarget]::Machine) + ";${Env:ProgramFiles(x86)}\CMake\bin", [System.EnvironmentVariableTarget]::Machine) } -if ($Env:TARGET_ARCH -eq 'x64') { [Environment]::SetEnvironmentVariable("Path", [Environment]::GetEnvironmentVariable("Path", [EnvironmentVariableTarget]::Machine) + ";${env:ProgramFiles}\CMake\bin", [System.EnvironmentVariableTarget]::Machine) } +if ($Env:TARGET_ARCH -eq 'x86') { + [Environment]::SetEnvironmentVariable("Path", + [Environment]::GetEnvironmentVariable("Path", [EnvironmentVariableTarget]::Machine) + + ";${Env:ProgramFiles(x86)}\CMake\bin", [System.EnvironmentVariableTarget]::Machine) +} +if ($Env:TARGET_ARCH -eq 'x64') { + [Environment]::SetEnvironmentVariable("Path", + [Environment]::GetEnvironmentVariable("Path", [EnvironmentVariableTarget]::Machine) + + ";${env:ProgramFiles}\CMake\bin", [System.EnvironmentVariableTarget]::Machine) +} # Install golang and set GOPATH to the dev path used in builds & tests cinst -y --no-progress golang $ENV:CHOCO_ARCH_FLAG --version $ENV:GO_VERSION @@ -85,7 +96,9 @@ Code-Check -Code $LASTEXITCODE -Element "visualstudio2017buildtools" setx VSTUDIO_ROOT "${env:ProgramFiles(x86)}\Microsoft Visual Studio\2017\BuildTools" # Add signtool to path -[Environment]::SetEnvironmentVariable("Path", [Environment]::GetEnvironmentVariable("Path", [EnvironmentVariableTarget]::Machine) + ";${env:ProgramFiles(x86)}\Windows Kits\8.1\bin\x64", [System.EnvironmentVariableTarget]::Machine) +[Environment]::SetEnvironmentVariable("Path", + [Environment]::GetEnvironmentVariable("Path", [EnvironmentVariableTarget]::Machine) + + ";${env:ProgramFiles(x86)}\Windows Kits\8.1\bin\x64", [System.EnvironmentVariableTarget]::Machine) # Install VC compiler for Python 2.7 cinst -y --no-progress vcpython27 $ENV:CHOCO_ARCH_FLAG --version $ENV:VCPYTHON27_VERSION @@ -96,8 +109,9 @@ Code-Check -Code $LASTEXITCODE -Element "vcpython27" cinst -y --no-progress wixtoolset $ENV:CHOCO_ARCH_FLAG --version $ENV:WIX_VERSION Code-Check -Code $LASTEXITCODE -Element "wixtoolset" - -[Environment]::SetEnvironmentVariable("Path", [Environment]::GetEnvironmentVariable("Path", [EnvironmentVariableTarget]::Machine) + ";${env:ProgramFiles(x86)}\WiX Toolset v3.11\bin", [System.EnvironmentVariableTarget]::Machine) +[Environment]::SetEnvironmentVariable("Path", + [Environment]::GetEnvironmentVariable("Path", [EnvironmentVariableTarget]::Machine) + + ";${env:ProgramFiles(x86)}\WiX Toolset v3.11\bin", [System.EnvironmentVariableTarget]::Machine) # Install msys2 system cinst -y --no-progress msys2 --params "/NoUpdate" --version $ENV:MSYS_VERSION