DataDog · KSerrania · Jun 6, 2020 · Jun 7, 2020 · Jun 7, 2020 · Jun 8, 2020
@@ -170,6 +170,7 @@ test_system-probe_arm64:
     - $SRC_IMAGE = "486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/${IMAGE}:${SRC_TAG}"
     - echo "build --build-arg BASE_IMAGE=$BASE_IMAGE --build-arg DD_TARGET_ARCH=$DD_TARGET_ARCH --build-arg WINDOWS_VERSION=$WINDOWS_VERSION --tag $SRC_IMAGE --file $DOCKERFILE ."
     - powershell -Command "docker build --no-cache --build-arg BASE_IMAGE=$BASE_IMAGE --build-arg DD_TARGET_ARCH=$DD_TARGET_ARCH --build-arg WINDOWS_VERSION=$WINDOWS_VERSION --tag $SRC_IMAGE --file $DOCKERFILE ."
+    - docker images
     - docker push $SRC_IMAGE
   after_script:
     - docker image prune -f
@@ -203,7 +204,7 @@ build_windows_1909_x64:
   variables:
     DOCKERFILE: windows/Dockerfile
     IMAGE: windows_1909_x64
-    BASE_IMAGE: mcr.microsoft.com/dotnet/framework/runtime:4.8-20200310-windowsservercore-1909
+    BASE_IMAGE: mcr.microsoft.com/dotnet/framework/runtime:3.5-20200512-windowsservercore-1909
     DD_TARGET_ARCH: x64
     WINDOWS_VERSION: 1909
   resource_group: windows_x64
@@ -214,7 +215,7 @@ build_windows_1909_x86:
   variables:
     DOCKERFILE: windows/Dockerfile
     IMAGE: windows_1909_x86
-    BASE_IMAGE: mcr.microsoft.com/dotnet/framework/runtime:4.8-20200310-windowsservercore-1909
+    BASE_IMAGE: mcr.microsoft.com/dotnet/framework/runtime:3.5-20200512-windowsservercore-1909
     DD_TARGET_ARCH: x86
     WINDOWS_VERSION: 1909
   resource_group: windows_x86

@@ -24,134 +24,51 @@ ARG TARGET_ARCH
 ENV TARGET_ARCH=${TARGET_ARCH:-x64}
 
 # Chocolatey package versions
-ENV GIT_VERSION "2.26.2"
-ENV SEVENZIP_VERSION "19.0"
-ENV VS2017BUILDTOOLS_VERSION "15.9.23.0"
-ENV VCPYTHON27_VERSION "9.0.0.30729"
-ENV GO_VERSION "1.13.8"
-ENV RUBY_VERSION "2.4.3.1"
-ENV PYTHON_VERSION "2.7.17"
-ENV WIX_VERSION "3.11.2"
-ENV CMAKE_VERSION "3.17.2"
-ENV MSYS_VERSION "20190524.0.0.20191030"
-
-ENV EMBEDDED_PYTHON_2_VERSION "2.7.17"
-ENV EMBEDDED_PYTHON_3_VERSION "3.8.1"
-
-ENV CACERTS_HASH "ADF770DFD574A0D6026BFAA270CB6879B063957177A991D453FF1D302C02081F"
-
-LABEL target_agent="Agent 6/7"
-LABEL target_arch=${TARGET_ARCH}
-LABEL windows_version=${WINDOWS_VERSION}
-LABEL git_version=${GIT_VERSION}
-LABEL sevenzip_version=${SEVENZIP_VERSION}
-LABEL vs2017buildtools_version=${VS2017BUILDTOOLS_VERSION}
-LABEL vcpython27_version=${VCPYTHON27_VERSION}
-LABEL go_version=${GO_VERSION}
-LABEL ruby_version=${RUBY_VERSION}
-LABEL wix_version=${WIX_VERSION}
-LABEL cmake_version=${CMAKE_VERSION}
-LABEL msys_version=${MSYS_VERSION}
-LABEL system_python_version=${PYTHON_VERSION}
-LABEL embedded_py2_version=${EMBEDDED_PYTHON_2_VERSION}
-LABEL embedded_py3_version=${EMBEDDED_PYTHON_3_VERSION}
-
-# Add certificates needed for build & check certificates file hash
-# We need to trust the DigiCert High Assurance EV Root CA certificate, which signs python.org,
-# to be able to download some Python components during the Agent build.
-RUN (New-Object System.Net.WebClient).DownloadFile(\"https://curl.haxx.se/ca/cacert.pem\", \"cacert.pem\")
-RUN if ((Get-FileHash .\cacert.pem).Hash -ne \"$ENV:CACERTS_HASH\") { Write-Host \"Wrong hashsum for cacert.pem: got '$((Get-FileHash .\cacert.pem).Hash)', expected '$ENV:CACERTS_HASH'.\"; exit 1 }
-RUN setx SSL_CERT_FILE \"C:\cacert.pem\"
-
-### Preliminary step: we need both the .NET 3.5 runtime and
-### the .NET 4.8 runtime. To do this, we get 4.8 from a base image and we
-### manually the install .NET Framework 3.5 runtime using the instructions in
-### the mcr.microsoft.com/dotnet/framework/runtime:3.5 Dockerfile:
-### https://github.com/microsoft/dotnet-framework-docker/blob/26597e42d157cc1e09d1e0dc8f23c32e6c3d1467/3.5/runtime/windowsservercore-ltsc2019/Dockerfile
-
-### The .NET Fx 3.5 is needed for the Visual C++ Compiler for Python 2.7
-### (https://www.microsoft.com/en-us/download/details.aspx?id=44266)
-### and to work around a bug in the WiX 3.11 installer
-### (https://github.com/wixtoolset/issues/issues/5661).
-
-# Install .NET Fx 3.5
-COPY ./windows/install_net35_1809.bat install_net35_1809.bat
-COPY ./windows/install_net35_1909.bat install_net35_1909.bat
-RUN if ($Env:WINDOWS_VERSION -eq '1809') { .\install_net35_1809.bat }
-RUN if ($Env:WINDOWS_VERSION -eq '1909') { .\install_net35_1909.bat }
-
-### End of preliminary step
-
-RUN if ($Env:TARGET_ARCH -eq 'x86') { setx CHOCO_ARCH_FLAG '-x86' }
-
-# Install Chocolatey
-RUN $env:chocolateyUseWindowsCompression = 'true'; Invoke-Expression ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1'))
-
-# Install git
-RUN $env:chocolateyUseWindowsCompression = 'true'; cinst -y --no-progress git $ENV:CHOCO_ARCH_FLAG --version $ENV:GIT_VERSION
-### HACK: we disable symbolic links when cloning repositories
-### to work around a symlink-related failure in the agent-binaries omnibus project
-### when copying the datadog-agent project twice.
-RUN git config --system core.symlinks false
-
-# Install 7zip
-RUN $env:chocolateyUseWindowsCompression = 'true'; cinst -y --no-progress 7zip $ENV:CHOCO_ARCH_FLAG --version $ENV:SEVENZIP_VERSION
-
-# Install VS2017
-RUN cinst -y --no-progress visualstudio2017buildtools $ENV:CHOCO_ARCH_FLAG --version $ENV:VS2017BUILDTOOLS_VERSION --params \"--add Microsoft.VisualStudio.ComponentGroup.NativeDesktop.Win81 --add Microsoft.VisualStudio.Workload.VCTools\"
-RUN setx VSTUDIO_ROOT \"${env:ProgramFiles(x86)}\Microsoft Visual Studio\2017\BuildTools\"
-
-# Install VC compiler for Python 2.7
-RUN cinst -y --no-progress vcpython27 $ENV:CHOCO_ARCH_FLAG --version $ENV:VCPYTHON27_VERSION
-
-# Install Wix and update PATH to include it
-RUN cinst -y --no-progress wixtoolset $ENV:CHOCO_ARCH_FLAG --version $ENV:WIX_VERSION
-RUN [Environment]::SetEnvironmentVariable(\"Path\", [Environment]::GetEnvironmentVariable(\"Path\", [EnvironmentVariableTarget]::Machine) + \";${env:ProgramFiles(x86)}\WiX Toolset v3.11\bin\", [System.EnvironmentVariableTarget]::Machine)
-
-# Install Cmake and update PATH to include it
-RUN cinst -y --no-progress cmake $ENV:CHOCO_ARCH_FLAG --version $ENV:CMAKE_VERSION
-RUN if ($Env:TARGET_ARCH -eq 'x86') { [Environment]::SetEnvironmentVariable(\"Path\", [Environment]::GetEnvironmentVariable(\"Path\", [EnvironmentVariableTarget]::Machine) + \";${Env:ProgramFiles(x86)}\CMake\bin\", [System.EnvironmentVariableTarget]::Machine) }
-RUN if ($Env:TARGET_ARCH -eq 'x64') { [Environment]::SetEnvironmentVariable(\"Path\", [Environment]::GetEnvironmentVariable(\"Path\", [EnvironmentVariableTarget]::Machine) + \";${env:ProgramFiles}\CMake\bin\", [System.EnvironmentVariableTarget]::Machine) }
-
-# Install golang and set GOPATH to the dev path used in builds & tests
-RUN cinst -y --no-progress golang $ENV:CHOCO_ARCH_FLAG --version $ENV:GO_VERSION
-RUN setx GOPATH C:\dev\go
-
-# Install system Python 2 (to use invoke)
-RUN cinst -y --no-progress python2 $ENV:CHOCO_ARCH_FLAG --version $ENV:PYTHON_VERSION
-
-# Install 64-bit ruby and bundler (for omnibus builds)
-RUN cinst -y --no-progress ruby --version $ENV:RUBY_VERSION
-RUN setx RIDK ((Get-Command ridk).Path)
-RUN gem install bundler
-
-# Install msys2 system & install 64-bit C/C++ compilation toolchain
-RUN cinst -y --no-progress msys2 --params \"/NoUpdate\" --version $ENV:MSYS_VERSION
-RUN ridk install 3
-
-# (32-bit only) Install 32-bit C/C++ compilation toolchain
-RUN if ($Env:TARGET_ARCH -eq 'x86') { ridk enable; bash -c \"pacman -S --needed --noconfirm mingw-w64-i686-binutils mingw-w64-i686-crt-git mingw-w64-i686-gcc mingw-w64-i686-gcc-libs mingw-w64-i686-headers-git mingw-w64-i686-libmangle-git mingw-w64-i686-libwinpthread-git mingw-w64-i686-make mingw-w64-i686-pkg-config mingw-w64-i686-tools-git mingw-w64-i686-winpthreads-git\" }
-RUN if ($Env:TARGET_ARCH -eq 'x86') { [Environment]::SetEnvironmentVariable(\"Path\", [Environment]::GetEnvironmentVariable(\"Path\", [EnvironmentVariableTarget]::Machine) + \";C:\tools\msys64\mingw32\bin;C:\tools\msys64\usr\bin\", [System.EnvironmentVariableTarget]::Machine) }
+ENV GIT_VERSION="2.26.2" \
+    SEVENZIP_VERSION="19.0" \
+    VS2017BUILDTOOLS_VERSION="15.9.23.0" \
+    VCPYTHON27_VERSION="9.0.0.30729" \
+    GO_VERSION="1.13.8" \
+    RUBY_VERSION="2.4.3.1" \
+    PYTHON_VERSION="2.7.17" \
+    WIX_VERSION="3.11.2" \
+    CMAKE_VERSION="3.17.2" \
+    MSYS_VERSION="20190524.0.0.20191030" \
+    EMBEDDED_PYTHON_2_VERSION="2.7.17" \
+    EMBEDDED_PYTHON_3_VERSION="3.8.1" \
+    CACERTS_HASH="ADF770DFD574A0D6026BFAA270CB6879B063957177A991D453FF1D302C02081F"
+
+LABEL target_agent="Agent 6/7" \
+      target_arch=${TARGET_ARCH} \
+      windows_version=${WINDOWS_VERSION} \
+      git_version=${GIT_VERSION} \
+      sevenzip_version=${SEVENZIP_VERSION} \
+      vs2017buildtools_version=${VS2017BUILDTOOLS_VERSION} \
+      vcpython27_version=${VCPYTHON27_VERSION} \
+      go_version=${GO_VERSION} \
+      ruby_version=${RUBY_VERSION} \
+      wix_version=${WIX_VERSION} \
+      cmake_version=${CMAKE_VERSION} \
+      msys_version=${MSYS_VERSION} \
+      system_python_version=${PYTHON_VERSION} \
+      embedded_py2_version=${EMBEDDED_PYTHON_2_VERSION} \
+      embedded_py3_version=${EMBEDDED_PYTHON_3_VERSION}
+
+COPY ./windows/scripts scripts
+
+# Install builder tools
+RUN scripts/setup.ps1
+
+# Run setup commands that need builder tools
+RUN scripts/postsetup.ps1
 
 # Install aws cli
-COPY ./windows/install_awscli.ps1 install_awscli.ps1
-RUN powershell -C .\install_awscli.ps1
+RUN scripts/install_awscli.ps1
 
 # Install docker, manifest-tool and notary
-COPY ./windows/install_docker.ps1 install_docker.ps1
-RUN powershell -Command .\install_docker.ps1
+RUN scripts/install_docker.ps1
 
 # Install embedded pythons (for unit testing)
-COPY ./windows/install_embedded_pythons.ps1 install_embedded_pythons.ps1
-RUN powershell -C .\install_embedded_pythons.ps1
-
-# Add signtool to path
-RUN [Environment]::SetEnvironmentVariable(\"Path\", [Environment]::GetEnvironmentVariable(\"Path\", [EnvironmentVariableTarget]::Machine) + \";${env:ProgramFiles(x86)}\Windows Kits\8.1\bin\x64\", [System.EnvironmentVariableTarget]::Machine)
-
-# Set 32-bit flag env var
-RUN if ($Env:TARGET_ARCH -eq 'x86') { setx WINDOWS_BUILD_32_BIT 1 }
-
-COPY ./windows/entrypoint.bat /entrypoint.bat
-COPY ./windows/aws_networking.ps1 /aws_networking.ps1
+RUN scripts/install_embedded_pythons.ps1
 
-ENTRYPOINT ["/entrypoint.bat"]
+ENTRYPOINT ["/scripts/entrypoint.bat"]
@@ -14,7 +14,7 @@
 @echo AWS_NETWORKING is %AWS_NETWORKING%
 if defined AWS_NETWORKING (
     @echo Detected AWS container, setting up networking
-    powershell -C "c:\aws_networking.ps1"
+    powershell -C "c:\scripts\aws_networking.ps1"
 )
 %*
 exit /b %ERRORLEVEL%
@@ -0,0 +1,45 @@
+$ErrorActionPreference = 'Stop'
+
+function Code-Check {
+    param ($Code, $Element)
+    if ($Code -ne 0) {
+        Write-Host -ForegroundColor Red "$Element install failed"
+        [Environment]::Exit(1)
+    }
+}
+
+# git, ridk and bundler aren't available yet right after they're installed, you need to restart a shell
+
+### HACK: we disable symbolic links when cloning repositories
+### to work around a symlink-related failure in the agent-binaries omnibus project
+### when copying the datadog-agent project twice.
+git config --system core.symlinks false
+
+# Install 64-bit C/C++ compilation toolchain
+ridk install 3
+Code-Check -Code $LASTEXITCODE -Element "ridk"
+
+
+setx RIDK ((Get-Command ridk).Path)
+
+# (32-bit only) Install 32-bit C/C++ compilation toolchain
+if ($Env:TARGET_ARCH -eq 'x86') {
+    ridk enable
+    bash -c "pacman -S --needed --noconfirm mingw-w64-i686-binutils mingw-w64-i686-crt-git \
+        mingw-w64-i686-gcc mingw-w64-i686-gcc-libs mingw-w64-i686-headers-git mingw-w64-i686-libmangle-git \
+        mingw-w64-i686-libwinpthread-git mingw-w64-i686-make mingw-w64-i686-pkg-config mingw-w64-i686-tools-git \
+        mingw-w64-i686-winpthreads-git"
+}
+
+Code-Check -Code $LASTEXITCODE -Element "32-bit ridk"
+
+
+if ($Env:TARGET_ARCH -eq 'x86') {
+    [Environment]::SetEnvironmentVariable("Path",
+        [Environment]::GetEnvironmentVariable("Path", [EnvironmentVariableTarget]::Machine) +
+        ";C:\tools\msys64\mingw32\bin;C:\tools\msys64\usr\bin", [System.EnvironmentVariableTarget]::Machine)
+}
+
+# Install bundler for omnibus builds
+gem install bundler
+Code-Check -Code $LASTEXITCODE -Element "bundler"
@@ -0,0 +1,118 @@
+$ErrorActionPreference = 'Stop'
+
+function Code-Check {
+    param ($Code, $Element)
+    if ($Code -ne 0) {
+        Write-Host -ForegroundColor Red "$Element install failed"
+        [Environment]::Exit(1)
+    }
+}
+
+# Add certificates needed for build & check certificates file hash
+# We need to trust the DigiCert High Assurance EV Root CA certificate, which signs python.org,
+# to be able to download some Python components during the Agent build.
+(New-Object System.Net.WebClient).DownloadFile("https://curl.haxx.se/ca/cacert.pem", "cacert.pem")
+if ((Get-FileHash .\cacert.pem).Hash -ne "$ENV:CACERTS_HASH") { Write-Host "Wrong hashsum for cacert.pem: got '$((Get-FileHash .\cacert.pem).Hash)', expected '$ENV:CACERTS_HASH'."; exit 1 }
+setx SSL_CERT_FILE "C:\cacert.pem"
+
+# Set 32-bit flag env var
+if ($Env:TARGET_ARCH -eq 'x86') { setx CHOCO_ARCH_FLAG '-x86' ; setx WINDOWS_BUILD_32_BIT 1 }
+
+# Install Chocolatey
+$env:chocolateyUseWindowsCompression = 'true'
+Invoke-Expression ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1'))
+Code-Check -Code $LASTEXITCODE -Element "chocolatey"
+
+
+# Install git
+cinst -y --no-progress git $ENV:CHOCO_ARCH_FLAG --version $ENV:GIT_VERSION
+Code-Check -Code $LASTEXITCODE -Element "git"
+
+
+# Install 7zip
+cinst -y --no-progress 7zip $ENV:CHOCO_ARCH_FLAG --version $ENV:SEVENZIP_VERSION
+Code-Check -Code $LASTEXITCODE -Element "7zip"
+
+# We have 7zip now, no need for Windows compression
+$env:chocolateyUseWindowsCompression = 'false'
+
+# Install Cmake and update PATH to include it
+cinst -y --no-progress cmake $ENV:CHOCO_ARCH_FLAG --version $ENV:CMAKE_VERSION
+Code-Check -Code $LASTEXITCODE -Element "cmake"
+
+
+if ($Env:TARGET_ARCH -eq 'x86') {
+    [Environment]::SetEnvironmentVariable("Path",
+        [Environment]::GetEnvironmentVariable("Path", [EnvironmentVariableTarget]::Machine) +
+        ";${Env:ProgramFiles(x86)}\CMake\bin", [System.EnvironmentVariableTarget]::Machine)
+}
+if ($Env:TARGET_ARCH -eq 'x64') {
+    [Environment]::SetEnvironmentVariable("Path",
+        [Environment]::GetEnvironmentVariable("Path", [EnvironmentVariableTarget]::Machine) +
+        ";${env:ProgramFiles}\CMake\bin", [System.EnvironmentVariableTarget]::Machine)
+}
+
+# Install golang and set GOPATH to the dev path used in builds & tests
+cinst -y --no-progress golang $ENV:CHOCO_ARCH_FLAG --version $ENV:GO_VERSION
+Code-Check -Code $LASTEXITCODE -Element "golang"
+
+
+setx GOPATH C:\dev\go
+
+# Install system Python 2 (to use invoke)
+cinst -y --no-progress python2 $ENV:CHOCO_ARCH_FLAG --version $ENV:PYTHON_VERSION
+Code-Check -Code $LASTEXITCODE -Element "python2"
+
+
+# Install 64-bit ruby (for omnibus builds)
+cinst -y --no-progress ruby --version $ENV:RUBY_VERSION
+Code-Check -Code $LASTEXITCODE -Element "ruby"
+
+
+### We need both the .NET 3.5 runtime and the .NET 4.8 runtime.
+### To do this, on 1809 we get 4.8 from a base image and we
+### manually the install .NET Framework 3.5 runtime using the instructions in
+### the mcr.microsoft.com/dotnet/framework/runtime:3.5 Dockerfile:
+### https://github.com/microsoft/dotnet-framework-docker/blob/26597e42d157cc1e09d1e0dc8f23c32e6c3d1467/3.5/runtime/windowsservercore-ltsc2019/Dockerfile
+
+### 1909 includes .NET Framework 4.8 by default, so we use the image that includes
+### .NET Framework 3.5 as well, and thus we don't need to install it manually here.
+
+### The .NET Fx 3.5 is needed for the Visual C++ Compiler for Python 2.7
+### (https://www.microsoft.com/en-us/download/details.aspx?id=44266)
+### and to work around a bug in the WiX 3.11 installer
+### (https://github.com/wixtoolset/issues/issues/5661).
+
+# Install .NET Fx 3.5
+if ($Env:WINDOWS_VERSION -eq '1809') { .\scripts\install_net35_1809.bat }
+Code-Check -Code $LASTEXITCODE -Element ".NET 3.5"
+
+
+# Install VS2017
+cinst -y --no-progress visualstudio2017buildtools $ENV:CHOCO_ARCH_FLAG --version $ENV:VS2017BUILDTOOLS_VERSION --params "--add Microsoft.VisualStudio.ComponentGroup.NativeDesktop.Win81 --add Microsoft.VisualStudio.Workload.VCTools"
+Code-Check -Code $LASTEXITCODE -Element "visualstudio2017buildtools"
+
+
+setx VSTUDIO_ROOT "${env:ProgramFiles(x86)}\Microsoft Visual Studio\2017\BuildTools"
+
+# Add signtool to path
+[Environment]::SetEnvironmentVariable("Path",
+    [Environment]::GetEnvironmentVariable("Path", [EnvironmentVariableTarget]::Machine) +
+    ";${env:ProgramFiles(x86)}\Windows Kits\8.1\bin\x64", [System.EnvironmentVariableTarget]::Machine)
+
+# Install VC compiler for Python 2.7
+cinst -y --no-progress vcpython27 $ENV:CHOCO_ARCH_FLAG --version $ENV:VCPYTHON27_VERSION
+Code-Check -Code $LASTEXITCODE -Element "vcpython27"
+
+
+# Install Wix and update PATH to include it
+cinst -y --no-progress wixtoolset $ENV:CHOCO_ARCH_FLAG --version $ENV:WIX_VERSION
+Code-Check -Code $LASTEXITCODE -Element "wixtoolset"
+
+[Environment]::SetEnvironmentVariable("Path",
+    [Environment]::GetEnvironmentVariable("Path", [EnvironmentVariableTarget]::Machine) +
+    ";${env:ProgramFiles(x86)}\WiX Toolset v3.11\bin", [System.EnvironmentVariableTarget]::Machine)
+
+# Install msys2 system
+cinst -y --no-progress msys2 --params "/NoUpdate" --version $ENV:MSYS_VERSION
+Code-Check -Code $LASTEXITCODE -Element "msys2"