From 04fbb2bac5a1461d7dde573729aed479257b93ca Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Sat, 2 Dec 2023 13:12:18 +0000 Subject: [PATCH 1/2] Update dependencies from https://github.com/dotnet/arcade build 20231201.1 Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.Build.Tasks.Installers , Microsoft.DotNet.CMake.Sdk , Microsoft.DotNet.XliffTasks From Version 9.0.0-beta.23580.7 -> To Version 9.0.0-beta.23601.1 --- eng/Version.Details.xml | 16 ++-- eng/Versions.props | 2 +- eng/common/post-build/redact-logs.ps1 | 81 +++++++++++++++++++ .../templates/job/publish-build-assets.yml | 4 +- .../templates/post-build/common-variables.yml | 2 + .../templates/post-build/post-build.yml | 1 + eng/common/templates/steps/publish-logs.yml | 28 ++++++- global.json | 4 +- 8 files changed, 124 insertions(+), 14 deletions(-) create mode 100644 eng/common/post-build/redact-logs.ps1 diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index ed9f51434b62..a805df0d336a 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -206,18 +206,18 @@ - + https://github.com/dotnet/arcade - 9cb7d80460fcd74ce5b31633d2deb32a93fd30de + 38aa367e014493c6f6ebfe94d9029bea01d072c4 - + https://github.com/dotnet/arcade - 9cb7d80460fcd74ce5b31633d2deb32a93fd30de + 38aa367e014493c6f6ebfe94d9029bea01d072c4 - + https://github.com/dotnet/arcade - 9cb7d80460fcd74ce5b31633d2deb32a93fd30de + 38aa367e014493c6f6ebfe94d9029bea01d072c4 https://github.com/dotnet/arcade-services @@ -227,9 +227,9 @@ https://github.com/dotnet/arcade-services 5263b603d90991a0c200aca8b8892c3d7cfe4751 - + https://github.com/dotnet/arcade - 9cb7d80460fcd74ce5b31633d2deb32a93fd30de + 38aa367e014493c6f6ebfe94d9029bea01d072c4 https://github.com/dotnet/runtime diff --git a/eng/Versions.props b/eng/Versions.props index 4c9357ee6e20..d17537ffa133 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -40,7 +40,7 @@ - 9.0.0-beta.23580.7 + 9.0.0-beta.23601.1 diff --git a/eng/common/post-build/redact-logs.ps1 b/eng/common/post-build/redact-logs.ps1 new file mode 100644 index 000000000000..82d91f6fd022 --- /dev/null +++ b/eng/common/post-build/redact-logs.ps1 @@ -0,0 +1,81 @@ +[CmdletBinding(PositionalBinding=$False)] +param( + [Parameter(Mandatory=$true, Position=0)][string] $InputPath, + [Parameter(Mandatory=$true)][string] $BinlogToolVersion, + [Parameter(Mandatory=$false)][string] $DotnetPath, + [Parameter(Mandatory=$false)][string] $PackageFeed = 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public/nuget/v3/index.json', + # File with strings to redact - separated by newlines. + # For comments start the line with '# ' - such lines are ignored + [Parameter(Mandatory=$false)][string] $TokensFilePath, + [Parameter(ValueFromRemainingArguments=$true)][String[]]$TokensToRedact +) + +try { + . $PSScriptRoot\post-build-utils.ps1 + + $packageName = 'binlogtool' + + $dotnet = $DotnetPath + + if (!$dotnet) { + $dotnetRoot = InitializeDotNetCli -install:$true + $dotnet = "$dotnetRoot\dotnet.exe" + } + + $toolList = & "$dotnet" tool list -g + + if ($toolList -like "*$packageName*") { + & "$dotnet" tool uninstall $packageName -g + } + + $toolPath = "$PSScriptRoot\..\..\..\.tools" + $verbosity = 'minimal' + + New-Item -ItemType Directory -Force -Path $toolPath + + Push-Location -Path $toolPath + + try { + Write-Host "Installing Binlog redactor CLI..." + Write-Host "'$dotnet' new tool-manifest" + & "$dotnet" new tool-manifest + Write-Host "'$dotnet' tool install $packageName --local --add-source '$PackageFeed' -v $verbosity --version $BinlogToolVersion" + & "$dotnet" tool install $packageName --local --add-source "$PackageFeed" -v $verbosity --version $BinlogToolVersion + + if (Test-Path $TokensFilePath) { + Write-Host "Adding additional sensitive data for redaction from file: " $TokensFilePath + $TokensToRedact += Get-Content -Path $TokensFilePath | Foreach {$_.Trim()} | Where { $_ -notmatch "^# " } + } + + $optionalParams = [System.Collections.ArrayList]::new() + + Foreach ($p in $TokensToRedact) + { + if($p -match '^\$\(.*\)$') + { + Write-Host ("Ignoring token {0} as it is probably unexpanded AzDO variable" -f $p) + } + elseif($p) + { + $optionalParams.Add("-p:" + $p) | Out-Null + } + } + + & $dotnet binlogtool redact --input:$InputPath --recurse --in-place ` + @optionalParams + + if ($LastExitCode -ne 0) { + Write-PipelineTelemetryError -Category 'Redactor' -Type 'warning' -Message "Problems using Redactor tool (exit code: $LastExitCode). But ignoring them now." + } + } + finally { + Pop-Location + } + + Write-Host 'done.' +} +catch { + Write-Host $_ + Write-PipelineTelemetryError -Category 'Redactor' -Message "There was an error while trying to redact logs. Error: $_" + ExitWithExitCode 1 +} diff --git a/eng/common/templates/job/publish-build-assets.yml b/eng/common/templates/job/publish-build-assets.yml index bb0d9f8b0f12..3b25fd97923c 100644 --- a/eng/common/templates/job/publish-build-assets.yml +++ b/eng/common/templates/job/publish-build-assets.yml @@ -48,8 +48,8 @@ jobs: - group: AzureDevOps-Artifact-Feeds-Pats - name: runCodesignValidationInjection value: false - - ${{ if eq(parameters.publishAssetsImmediately, 'true') }}: - - template: /eng/common/templates/post-build/common-variables.yml + # unconditional - needed for logs publishing (redactor tool version) + - template: /eng/common/templates/post-build/common-variables.yml pool: # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) diff --git a/eng/common/templates/post-build/common-variables.yml b/eng/common/templates/post-build/common-variables.yml index 173914f2364a..4ef7bd271fbd 100644 --- a/eng/common/templates/post-build/common-variables.yml +++ b/eng/common/templates/post-build/common-variables.yml @@ -17,6 +17,8 @@ variables: value: 3.0.0 - name: SymbolToolVersion value: 1.0.1 + - name: BinlogToolVersion + value: 1.0.8 - name: runCodesignValidationInjection value: false diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml index ef720f9d7819..d64236b28cfb 100644 --- a/eng/common/templates/post-build/post-build.yml +++ b/eng/common/templates/post-build/post-build.yml @@ -187,6 +187,7 @@ stages: parameters: StageLabel: 'Validation' JobLabel: 'Signing' + BinlogToolVersion: $(BinlogToolVersion) - job: displayName: SourceLink Validation diff --git a/eng/common/templates/steps/publish-logs.yml b/eng/common/templates/steps/publish-logs.yml index 88f238f36bfd..dadf1c464cd1 100644 --- a/eng/common/templates/steps/publish-logs.yml +++ b/eng/common/templates/steps/publish-logs.yml @@ -1,6 +1,9 @@ parameters: StageLabel: '' JobLabel: '' + CustomSensitiveDataList: '' + # A default - in case value from eng/common/templates/post-build/common-variables.yml is not passed + BinlogToolVersion: '1.0.8' steps: - task: Powershell@2 @@ -12,7 +15,30 @@ steps: Move-Item -Path $(Build.SourcesDirectory)/artifacts/log/Debug/* $(Build.SourcesDirectory)/PostBuildLogs/${{parameters.StageLabel}}/${{parameters.JobLabel}}/ continueOnError: true condition: always() - + +- task: PowerShell@2 + displayName: Redact Logs + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/redact-logs.ps1 + # For now this needs to have explicit list of all sensitive data. Taken from eng/publishing/v3/publish.yml + # Sensitive data can as well be added to $(Build.SourcesDirectory)/eng/BinlogSecretsRedactionFile.txt' + # If the file exists - sensitive data for redaction will be sourced from it + # (single entry per line, lines starting with '# ' are considered comments and skipped) + arguments: -InputPath '$(Build.SourcesDirectory)/PostBuildLogs' + -BinlogToolVersion ${{parameters.BinlogToolVersion}} + -TokensFilePath '$(Build.SourcesDirectory)/eng/BinlogSecretsRedactionFile.txt' + '$(publishing-dnceng-devdiv-code-r-build-re)' + '$(MaestroAccessToken)' + '$(dn-bot-all-orgs-artifact-feeds-rw)' + '$(akams-client-id)' + '$(akams-client-secret)' + '$(microsoft-symbol-server-pat)' + '$(symweb-symbol-server-pat)' + '$(dn-bot-all-orgs-build-rw-code-rw)' + ${{parameters.CustomSensitiveDataList}} + continueOnError: true + condition: always() + - task: PublishBuildArtifacts@1 displayName: Publish Logs inputs: diff --git a/global.json b/global.json index a99b52842fd3..b74be4a43b22 100644 --- a/global.json +++ b/global.json @@ -11,7 +11,7 @@ "cmake": "3.21.0" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.23580.7", - "Microsoft.DotNet.CMake.Sdk": "9.0.0-beta.23580.7" + "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.23601.1", + "Microsoft.DotNet.CMake.Sdk": "9.0.0-beta.23601.1" } } From aac66ea891578d75675536b969c171a57f4bb5cf Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Sun, 3 Dec 2023 13:10:19 +0000 Subject: [PATCH 2/2] Update dependencies from https://github.com/dotnet/arcade build 20231201.1 Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.Build.Tasks.Installers , Microsoft.DotNet.CMake.Sdk , Microsoft.DotNet.XliffTasks From Version 9.0.0-beta.23580.7 -> To Version 9.0.0-beta.23601.1