Sync eng/common directory with azure-sdk-tools repository (#4787)

eng/common/TestResources/New-TestResources.ps1

@@ -244,10 +244,16 @@ try {
                 $context = Get-AzContext
             }
         } else {
-            if ($currentSubcriptionId -ne 'faa080af-c1d8-40ad-9cce-e1a450ca5b57') {
+            if ($context.Tenant.Name -like '*TME*') {
+                if ($currentSubscriptionId -ne '4d042dc6-fe17-4698-a23f-ec6a8d1e98f4') {
+                    Log "Attempting to select subscription 'Azure SDK Test Resources - TME (4d042dc6-fe17-4698-a23f-ec6a8d1e98f4)'"
+                    $null = Select-AzSubscription -Subscription '4d042dc6-fe17-4698-a23f-ec6a8d1e98f4' -ErrorAction Ignore
+                    # Update the context.
+                    $context = Get-AzContext
+                }
+            } elseif ($currentSubcriptionId -ne 'faa080af-c1d8-40ad-9cce-e1a450ca5b57') {
                 Log "Attempting to select subscription 'Azure SDK Developer Playground (faa080af-c1d8-40ad-9cce-e1a450ca5b57)'"
                 $null = Select-AzSubscription -Subscription 'faa080af-c1d8-40ad-9cce-e1a450ca5b57' -ErrorAction Ignore
-
                 # Update the context.
                 $context = Get-AzContext
             }
@@ -261,6 +267,7 @@ try {
             'faa080af-c1d8-40ad-9cce-e1a450ca5b57' = 'Azure SDK Developer Playground'
             'a18897a6-7e44-457d-9260-f2854c0aca42' = 'Azure SDK Engineering System'
             '2cd617ea-1866-46b1-90e3-fffb087ebf9b' = 'Azure SDK Test Resources'
+            '4d042dc6-fe17-4698-a23f-ec6a8d1e98f4' = 'Azure SDK Test Resources - TME '
         }
 
         # Print which subscription is currently selected.
@@ -313,7 +320,14 @@ try {
     # Make sure the provisioner OID is set so we can pass it through to the deployment.
     if (!$ProvisionerApplicationId -and !$ProvisionerApplicationOid) {
         if ($context.Account.Type -eq 'User') {
-            $user = Get-AzADUser -UserPrincipalName $context.Account.Id
+            # Support corp tenant and TME tenant user id lookups
+            $user = Get-AzADUser -Mail $context.Account.Id
+            if ($user -eq $null -or !$user.Id) {
+                $user = Get-AzADUser -UserPrincipalName $context.Account.Id
+            }
+            if ($user -eq $null -or !$user.Id) {
+                throw "Failed to find entra object ID for the current user"
+            }
             $ProvisionerApplicationOid = $user.Id
         } elseif ($context.Account.Type -eq 'ServicePrincipal') {
             $sp = Get-AzADServicePrincipal -ApplicationId $context.Account.Id
@@ -383,7 +397,14 @@ try {
             Write-Warning "The specified TestApplicationId '$TestApplicationId' will be ignored when -ServicePrincipalAutth is not set."
         }
 
-        $userAccount = (Get-AzADUser -UserPrincipalName (Get-AzContext).Account)
+        # Support corp tenant and TME tenant user id lookups
+        $userAccount = (Get-AzADUser -Mail (Get-AzContext).Account.Id)
+        if ($userAccount -eq $null -or !$userAccount.Id) {
+            $userAccount = (Get-AzADUser -UserPrincipalName (Get-AzContext).Account)
+        }
+        if ($userAccount -eq $null -or !$userAccount.Id) {
+            throw "Failed to find entra object ID for the current user"
+        }
         $TestApplicationOid = $userAccount.Id
         $TestApplicationId = $testApplicationOid
         $userAccountName = $userAccount.UserPrincipalName
@@ -506,6 +527,8 @@ try {
     if ($CI -and $Environment -eq 'AzureCloud' -and $env:PoolSubnet) {
         $templateParameters.Add('azsdkPipelineSubnetList', @($env:PoolSubnet))
     }
+    # Some arm/bicep templates may want to change deployment settings (e.g. local auth) in sandboxed TME tenants
+    $templateParameters.Add('supportsSafeSecretStandard', ($context.Tenant.Name -notlike '*TME*'))
 
     $defaultCloudParameters = LoadCloudConfig $Environment
     MergeHashes $defaultCloudParameters $(Get-Variable templateParameters)

eng/common/TestResources/TestResources-Helpers.ps1

@@ -131,7 +131,7 @@ function MergeHashes([hashtable] $source, [psvariable] $dest) {
 
 function BuildBicepFile([System.IO.FileSystemInfo] $file) {
     if (!(Get-Command bicep -ErrorAction Ignore)) {
-        Write-Error "A bicep file was found at '$($file.FullName)' but the Azure Bicep CLI is not installed. See aka.ms/bicep-install"
+        Write-Error "A bicep file was found at '$($file.FullName)' but the Azure Bicep CLI is not installed. See https://aka.ms/bicep-install"
         throw
     }
 

eng/common/docgeneration/Generate-DocIndex.ps1

@@ -25,13 +25,14 @@ function Get-BlobStorage-Artifacts(
     # "--query '[].name'" returns a list of only blob names
     # "--num-results *" handles pagination so the caller does not have to
     $artifacts = az storage blob list `
+        --auth-mode login `
         --account-name $storageAccountName `
         --container-name $storageContainerName `
         --prefix $storagePrefix `
         --delimiter / `
         --only-show-errors `
         --query '[].name' `
-        --num-results * | ConvertFrom-Json
+        --num-results * | ConvertFrom-Json 
     LogDebug "Number of artifacts found: $($artifacts.Length)"
 
     # example: "python/azure-storage-blob" -> "azure-storage-blob"

eng/common/docgeneration/templates/matthews/styles/main.js

@@ -108,7 +108,7 @@ function httpGetAsyncFallbackOnFail(targetUrl, successCallback, failureCallback)
 }
 
 function populateOptions(selector, packageName) {
-    var versionRequestUrl = "https://azuresdkdocs.blob.core.windows.net/$web/" + SELECTED_LANGUAGE + "/" + packageName + "/versioning/versions"
+    var versionRequestUrl = "https://azuresdkdocs.z19.web.core.windows.net/" + SELECTED_LANGUAGE + "/" + packageName + "/versioning/versions"
 
     httpGetAsync(versionRequestUrl, function (responseText) {
         var versionselector = document.createElement("select")
@@ -181,9 +181,9 @@ function loadedOtherVersions(url, latestVersions, publishedVersions, selector, c
 }
 
 function populateIndexList(selector, packageName) {
-    var url = "https://azuresdkdocs.blob.core.windows.net/$web/" + SELECTED_LANGUAGE + "/" + packageName + "/versioning/versions"
-    var latestGAUrl = "https://azuresdkdocs.blob.core.windows.net/$web/" + SELECTED_LANGUAGE + "/" + packageName + "/versioning/latest-ga"
-    var latestPreviewUrl = "https://azuresdkdocs.blob.core.windows.net/$web/" + SELECTED_LANGUAGE + "/" + packageName + "/versioning/latest-preview"
+    var url = "https://azuresdkdocs.z19.web.core.windows.net/" + SELECTED_LANGUAGE + "/" + packageName + "/versioning/versions"
+    var latestGAUrl = "https://azuresdkdocs.z19.web.core.windows.net/" + SELECTED_LANGUAGE + "/" + packageName + "/versioning/latest-ga"
+    var latestPreviewUrl = "https://azuresdkdocs.z19.web.core.windows.net/" + SELECTED_LANGUAGE + "/" + packageName + "/versioning/latest-preview"
     var latestVersions = document.createElement("ul")
     httpGetLatestAsync(latestGAUrl, latestVersions, packageName)
     httpGetLatestAsync(latestPreviewUrl, latestVersions, packageName)
@@ -217,7 +217,7 @@ function populateIndexList(selector, packageName) {
 }
 
 function getPackageUrl(language, package, version) {
-    return "https://azuresdkdocs.blob.core.windows.net/$web/" + language + "/" + package + "/" + version + "/index.html"
+    return "https://azuresdkdocs.z19.web.core.windows.net/" + language + "/" + package + "/" + version + "/index.html"
 }
 
 // Populate Versions

eng/common/pipelines/templates/archetype-typespec-emitter.yml

@@ -62,6 +62,11 @@ parameters:
   type: boolean
   default: false
 
+# Paths to sparse checkout
+- name: SparseCheckoutPaths
+  type: object
+  default: []
+
 extends:
   template: /eng/pipelines/templates/stages/1es-redirect.yml
   parameters:
@@ -79,6 +84,8 @@ extends:
       - job: Build
         steps:
         - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml
+          parameters:
+            Paths: ${{ parameters.SparseCheckoutPaths }}
 
         - ${{ parameters.InitializationSteps }}
 
@@ -411,7 +418,9 @@ extends:
             buildArtifactsPath: $(Pipeline.Workspace)/build_artifacts
           steps:
           - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml
-
+            parameters:
+              Paths: ${{ parameters.SparseCheckoutPaths }}
+
           - download: current
             artifact: build_artifacts
             displayName: Download build artifacts
@@ -438,3 +447,14 @@ extends:
             parameters:
               artifactName: test_artifacts_$(System.JobName)
               artifactPath: $(Build.ArtifactStagingDirectory)
+
+          - ${{ if and(eq(variables['System.TeamProject'], 'internal'), ne(variables['Build.Reason'], 'PullRequest')) }}:
+              - task: AzureCLI@2
+                displayName: "Upload Spector Standard Coverage Report"
+                condition: and(ne(variables['SpectorName'], ''), succeeded())
+                inputs:
+                  azureSubscription: "TypeSpec Storage"
+                  scriptType: "bash"
+                  scriptLocation: "inlineScript"
+                  inlineScript: npx tsp-spector upload-coverage --coverageFile $(Build.ArtifactStagingDirectory)/tsp-spector-coverage-azure.json --generatorName @azure-typespec/$(SpectorName) --storageAccountName typespec --containerName coverages --generatorVersion $(node -p -e "require('./package.json').version") --generatorMode azure
+                  workingDirectory: $(Build.SourcesDirectory)/eng/packages/$(SpectorName)

eng/common/pipelines/templates/jobs/docindex.yml

@@ -4,9 +4,9 @@ jobs:
       name: azsdk-pool-mms-win-2022-general
     steps:
       - task: UsePythonVersion@0
-        displayName: 'Use Python 3.9'
+        displayName: 'Use Python 3.11'
         inputs:
-          versionSpec: '3.9'
+          versionSpec: '3.11'
 
       - pwsh: |
           Invoke-WebRequest -Uri "https://github.com/dotnet/docfx/releases/download/v2.43.2/docfx.zip" `
@@ -15,12 +15,14 @@ jobs:
         workingDirectory: $(Build.BinariesDirectory)
         displayName: Download and Extract DocFX
 
-      - task: PowerShell@2
+      - task: AzureCLI@2
         displayName: 'Generate Doc Index'
         inputs:
-          pwsh: true
-          filePath: $(Build.SourcesDirectory)/eng/common/docgeneration/Generate-DocIndex.ps1 
-          arguments: >
+          azureSubscription: 'Azure SDK Artifacts'
+          scriptType: pscore
+          scriptLocation: inlineScript
+          inlineScript: >
+            $(Build.SourcesDirectory)/eng/common/docgeneration/Generate-DocIndex.ps1
             -Docfx $(docfxPath)
             -RepoRoot $(Build.SourcesDirectory)
             -DocGenDir "$(Build.SourcesDirectory)/eng/common/docgeneration"

eng/common/pipelines/templates/jobs/generate-job-matrix.yml

@@ -42,6 +42,21 @@ parameters:
 - name: PreGenerationSteps
   type: stepList
   default: []
+- name: EnablePRGeneration
+  type: boolean
+  default: false
+- name: PRMatrixSetting
+  type: string
+  default: 'ArtifactPackageNames'
+- name: PRJobBatchSize
+  type: number
+  default: 10
+- name: PRMatrixIndirectFilters
+  type: object
+  default: []
+- name: PRMatrixSparseIndirect
+  type: boolean
+  default: true
 # Mappings to OS name required at template compile time by 1es pipeline templates
 - name: Pools
   type: object
@@ -84,57 +99,89 @@ jobs:
 
     - ${{ parameters.PreGenerationSteps }}
 
-    - ${{ each config in parameters.MatrixConfigs }}:
+    - ${{ if eq(parameters.EnablePRGeneration, false) }}:
+      - ${{ each config in parameters.MatrixConfigs }}:
+        - ${{ each pool in parameters.Pools }}:
+          - ${{ if eq(config.GenerateVMJobs, 'true') }}:
+            - task: Powershell@2
+              inputs:
+                pwsh: true
+                filePath: eng/common/scripts/job-matrix/Create-JobMatrix.ps1
+                arguments: >
+                  -ConfigPath ${{ config.Path }}
+                  -Selection ${{ config.Selection }}
+                  -DisplayNameFilter '$(displayNameFilter)'
+                  -Filters '${{ join(''',''', parameters.MatrixFilters) }}', 'container=^$', 'SupportedClouds=^$|${{ parameters.CloudConfig.Cloud }}', 'Pool=${{ pool.filter }}'
+                  -Replace '${{ join(''',''', parameters.MatrixReplace) }}'
+                  -NonSparseParameters '${{ join(''',''', config.NonSparseParameters) }}'
+              displayName: Create ${{ pool.name }} Matrix ${{ config.Name }}
+              name: vm_job_matrix_${{ config.Name }}_${{ pool.name }}
+          - ${{ if eq(config.GenerateContainerJobs, 'true') }}:
+            - task: Powershell@2
+              inputs:
+                pwsh: true
+                filePath: eng/common/scripts/job-matrix/Create-JobMatrix.ps1
+                arguments: >
+                  -ConfigPath ${{ config.Path }}
+                  -Selection ${{ config.Selection }}
+                  -DisplayNameFilter '$(displayNameFilter)'
+                  -Filters '${{ join(''',''', parameters.MatrixFilters) }}', 'container=^$', 'SupportedClouds=^$|${{ parameters.CloudConfig.Cloud }}', 'Pool=${{ pool.filter }}'
+                  -NonSparseParameters '${{ join(''',''', config.NonSparseParameters) }}'
+              displayName: Create ${{ pool.name }} Container Matrix ${{ config.Name }}
+              name: container_job_matrix_${{ config.Name }}_${{ pool.name }}
+
+      # This else being set also currently assumes that the $(Build.ArtifactStagingDirectory)/PackageInfo folder is populated by PreGenerationSteps.
+      # Not currently not hardcoded, so not doing the needful and populating this folder before we hit this step will result in generation errors.
+    - ${{ else }}:
       - ${{ each pool in parameters.Pools }}:
-        - ${{ if eq(config.GenerateVMJobs, 'true') }}:
-          - task: Powershell@2
-            inputs:
-              pwsh: true
-              filePath: eng/common/scripts/job-matrix/Create-JobMatrix.ps1
-              arguments: >
-                -ConfigPath ${{ config.Path }}
-                -Selection ${{ config.Selection }}
-                -DisplayNameFilter '$(displayNameFilter)'
-                -Filters '${{ join(''',''', parameters.MatrixFilters) }}', 'container=^$', 'SupportedClouds=^$|${{ parameters.CloudConfig.Cloud }}', 'Pool=${{ pool.filter }}'
-                -Replace '${{ join(''',''', parameters.MatrixReplace) }}'
-                -NonSparseParameters '${{ join(''',''', config.NonSparseParameters) }}'
-            displayName: Create ${{ pool.name }} Matrix ${{ config.Name }}
-            name: vm_job_matrix_${{ config.Name }}_${{ pool.name }}
+        - pwsh: |
+            '${{ convertToJson(parameters.MatrixConfigs) }}' | Set-Content matrix.json
 
-        - ${{ if eq(config.GenerateContainerJobs, 'true') }}:
-          - task: Powershell@2
-            inputs:
-              pwsh: true
-              filePath: eng/common/scripts/job-matrix/Create-JobMatrix.ps1
-              arguments: >
-                -ConfigPath ${{ config.Path }}
-                -Selection ${{ config.Selection }}
-                -DisplayNameFilter '$(displayNameFilter)'
-                -Filters '${{ join(''',''', parameters.MatrixFilters) }}', 'container=^$', 'SupportedClouds=^$|${{ parameters.CloudConfig.Cloud }}', 'Pool=${{ pool.filter }}'
-                -NonSparseParameters '${{ join(''',''', config.NonSparseParameters) }}'
-            displayName: Create ${{ pool.name }} Container Matrix ${{ config.Name }}
-            name: container_job_matrix_${{ config.Name }}_${{ pool.name }}
+            ./eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 `
+              -PackagePropertiesFolder $(Build.ArtifactStagingDirectory)/PackageInfo `
+              -PRMatrixFile matrix.json `
+              -PRMatrixSetting ${{ parameters.PRMatrixSetting }} `
+              -DisplayNameFilter '$(displayNameFilter)' `
+              -Filters '${{ join(''',''', parameters.MatrixFilters) }}', 'container=^$', 'SupportedClouds=^$|${{ parameters.CloudConfig.Cloud }}', 'Pool=${{ pool.filter }}' `
+              -IndirectFilters '${{ join(''',''', parameters.PRMatrixIndirectFilters) }}' `
+              -Replace '${{ join(''',''', parameters.MatrixReplace) }}' `
+              -PackagesPerPRJob ${{ parameters.PRJobBatchSize }} `
+              -SparseIndirect $${{ parameters.PRMatrixSparseIndirect }}
+          displayName: Create ${{ pool.name }} PR Matrix
+          name: vm_job_matrix_pr_${{ pool.name }}
 
-- ${{ each config in parameters.MatrixConfigs }}:
-  - ${{ each pool in parameters.Pools }}:
-    - ${{ if eq(config.GenerateVMJobs, 'true') }}:
-      - template: ${{ parameters.JobTemplatePath }}
-        parameters:
-          UsePlatformContainer: false
-          OSName: ${{ pool.os }}
-          Matrix: dependencies.${{ parameters.GenerateJobName }}.outputs['vm_job_matrix_${{ config.Name }}_${{ pool.name }}.matrix']
-          DependsOn: ${{ parameters.GenerateJobName }}
-          CloudConfig: ${{ parameters.CloudConfig }}
-          ${{ each param in parameters.AdditionalParameters }}:
-            ${{ param.key }}: ${{ param.value }}
+- ${{ if eq(parameters.EnablePRGeneration, false) }}:
+  - ${{ each config in parameters.MatrixConfigs }}:
+    - ${{ each pool in parameters.Pools }}:
+      - ${{ if eq(config.GenerateVMJobs, 'true') }}:
+        - template: ${{ parameters.JobTemplatePath }}
+          parameters:
+            UsePlatformContainer: false
+            OSName: ${{ pool.os }}
+            Matrix: dependencies.${{ parameters.GenerateJobName }}.outputs['vm_job_matrix_${{ config.Name }}_${{ pool.name }}.matrix']
+            DependsOn: ${{ parameters.GenerateJobName }}
+            CloudConfig: ${{ parameters.CloudConfig }}
+            ${{ each param in parameters.AdditionalParameters }}:
+              ${{ param.key }}: ${{ param.value }}
 
-    - ${{ if eq(config.GenerateContainerJobs, 'true') }}:
-      - template: ${{ parameters.JobTemplatePath }}
-        parameters:
-          UsePlatformContainer: true
-          OSName: ${{ pool.os }}
-          Matrix: dependencies.${{ parameters.GenerateJobName }}.outputs['vm_job_matrix_${{ config.Name }}_${{ pool.name }}.matrix']
-          DependsOn: ${{ parameters.GenerateJobName }}
-          CloudConfig: ${{ parameters.CloudConfig }}
-          ${{ each param in parameters.AdditionalParameters }}:
-            ${{ param.key }}: ${{ param.value }}
+      - ${{ if eq(config.GenerateContainerJobs, 'true') }}:
+        - template: ${{ parameters.JobTemplatePath }}
+          parameters:
+            UsePlatformContainer: true
+            OSName: ${{ pool.os }}
+            Matrix: dependencies.${{ parameters.GenerateJobName }}.outputs['vm_job_matrix_${{ config.Name }}_${{ pool.name }}.matrix']
+            DependsOn: ${{ parameters.GenerateJobName }}
+            CloudConfig: ${{ parameters.CloudConfig }}
+            ${{ each param in parameters.AdditionalParameters }}:
+              ${{ param.key }}: ${{ param.value }}
+- ${{ else }}:
+  - ${{ each pool in parameters.Pools }}:
+    - template: ${{ parameters.JobTemplatePath }}
+      parameters:
+        UsePlatformContainer: false
+        OSName: ${{ pool.os }}
+        Matrix: dependencies.${{ parameters.GenerateJobName }}.outputs['vm_job_matrix_pr_${{ pool.name }}.matrix']
+        DependsOn: ${{ parameters.GenerateJobName }}
+        CloudConfig: ${{ parameters.CloudConfig }}
+        ${{ each param in parameters.AdditionalParameters }}:
+          ${{ param.key }}: ${{ param.value }}