From 4510d1570f592363af9cd1fb6fdfe5e1f1466a58 Mon Sep 17 00:00:00 2001 From: Chunan Ye Date: Wed, 8 Jul 2026 10:31:58 -0700 Subject: [PATCH 1/2] feature: retire azure-rbac skill --- .github/CODEOWNERS | 1 - .../references/blob-structure.md | 1 - .github/skills/file-test-bug/SKILL.md | 2 +- .github/workflows/test-all-integration.yml | 2 +- evals/azure-rbac/eval.yaml | 172 ------------------ plugin/skills/azure-rbac/SKILL.md | 17 -- plugin/skills/azure-rbac/version.json | 6 - plugin/skills/entra-agent-id/SKILL.md | 2 +- plugin/skills/entra-app-registration/SKILL.md | 2 +- tests/skills.json | 3 +- 10 files changed, 5 insertions(+), 203 deletions(-) delete mode 100644 evals/azure-rbac/eval.yaml delete mode 100644 plugin/skills/azure-rbac/SKILL.md delete mode 100644 plugin/skills/azure-rbac/version.json diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index a6e0407cb..a9dd86e24 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -25,7 +25,6 @@ /plugin/skills/azure-messaging/ @kashifkhan @RickWinter /plugin/skills/azure-prepare/ @tmeschter @kvenkatrajan @RickWinter /plugin/skills/azure-quotas/ @rakal-dyh @RickWinter -/plugin/skills/azure-rbac/ @JasonYeMSFT @msalaman @RickWinter /plugin/skills/azure-reliability/ @MadhuraBharadwaj-MSFT @saikoumudi @RickWinter /plugin/skills/azure-resource-lookup/ @JasonYeMSFT @RickWinter /plugin/skills/azure-resource-visualizer/ @tmeschter @RickWinter diff --git a/.github/skills/analyze-skill-issues/references/blob-structure.md b/.github/skills/analyze-skill-issues/references/blob-structure.md index c581b0130..0b3c4168b 100644 --- a/.github/skills/analyze-skill-issues/references/blob-structure.md +++ b/.github/skills/analyze-skill-issues/references/blob-structure.md @@ -15,7 +15,6 @@ | azure-kubernetes, kubernetes, aks | `azure-kubernetes` | | azure-kusto, kusto | `azure-kusto` | | azure-storage, storage | `azure-storage` | -| azure-rbac, rbac | `azure-rbac` | | (any other name) | use as-is | ## Blob Path Layout diff --git a/.github/skills/file-test-bug/SKILL.md b/.github/skills/file-test-bug/SKILL.md index a626829b3..8aa6cf992 100644 --- a/.github/skills/file-test-bug/SKILL.md +++ b/.github/skills/file-test-bug/SKILL.md @@ -13,7 +13,7 @@ Creates a GitHub issue in `microsoft/github-copilot-for-azure` for integration t ## Input -- **Skill name** (required): e.g., `azure-rbac`, `appinsights-instrumentation` +- **Skill name** (required): e.g., `azure-ai`, `appinsights-instrumentation` - **Test run** (optional): Timestamp of test run. Defaults to most recent with logs for the skill. ## Steps diff --git a/.github/workflows/test-all-integration.yml b/.github/workflows/test-all-integration.yml index 078965a8c..67f29cf2f 100644 --- a/.github/workflows/test-all-integration.yml +++ b/.github/workflows/test-all-integration.yml @@ -22,7 +22,7 @@ on: workflow_dispatch: inputs: skills: - description: 'Comma separated list of skills to test (e.g. "appinsights-instrumentation,azure-rbac")' + description: 'Comma separated list of skills to test (e.g. "appinsights-instrumentation,azure-ai")' required: true type: string model-override: diff --git a/evals/azure-rbac/eval.yaml b/evals/azure-rbac/eval.yaml deleted file mode 100644 index 992b76698..000000000 --- a/evals/azure-rbac/eval.yaml +++ /dev/null @@ -1,172 +0,0 @@ -# Vally eval config — migrated from Jest integration tests -# Source: tests/azure-rbac/integration.test.ts -# Migration: Jest → Vally -# -# Global graders (evaluate#125 workaround): no_runtime_failure -# is duplicated into every stimulus block below. - -name: azure-rbac-integration-eval -description: | - Integration evaluation for azure-rbac skill — migrated from Jest integration tests. - Covers routing/skill invocation and output quality for least-privilege role guidance, - including role recommendation, Azure CLI role assignment commands, and Bicep snippets. - -tags: - type: integration - skill: azure-rbac - -defaults: - runs: 5 - timeout: "10m" - executor: integration-test-agent-runner - model: claude-sonnet-4.6 - -scoring: - threshold: 0.8 - -stimuli: - # ── routing-acr-role-prompt ── - # Jest: "invokes azure-rbac skill for role recommendation prompt" - # Assertions: softCheckSkill + isSkillInvoked (invocation rate ≥ 80%) - - name: "Routing: managed identity pull from ACR" - prompt: "What role should I assign to my managed identity to read images in an Azure Container Registry?" - tags: - type: integration - tier: smoke - cost: llm - area: routing - earlyTerminate: '[{"type":"skill-call","skill":"azure-rbac"},{"type":"tool-call-count","count":3}]' - graders: - - type: skill-invocation - config: - required: - - azure-rbac - - type: output-not-matches - config: - pattern: "(?i)fatal error|unhandled exception|stack trace" - - # ── routing-blob-reader-prompt ── - # Jest: "invokes azure-rbac skill for least privilege role prompt" - # Assertions: softCheckSkill + isSkillInvoked (invocation rate ≥ 80%) - - name: "Routing: least privilege for blob read" - prompt: "What is the least privilege role for reading blob storage?" - tags: - type: integration - tier: smoke - cost: llm - area: routing - earlyTerminate: '[{"type":"skill-call","skill":"azure-rbac"},{"type":"tool-call-count","count":3}]' - graders: - - type: skill-invocation - config: - required: - - azure-rbac - - type: output-not-matches - config: - pattern: "(?i)fatal error|unhandled exception|stack trace" - - # ── acr-pull-output-prompt ── - # Jest: "invokes azure-rbac skill for AcrPull prompt" - # Expects skill invocation + AcrPull + az role assignment + Bicep roleAssignments resource - - name: "Output: recommend AcrPull with CLI and Bicep" - prompt: "What role should I assign to my managed identity to read images in an Azure Container Registry?" - tags: - type: integration - tier: full - cost: llm - area: output - graders: - - type: skill-invocation - config: - required: - - azure-rbac - - type: output-contains - config: - substring: "AcrPull" - - type: output-contains - config: - substring: "az role assignment" - - type: output-contains - config: - substring: "Microsoft.Authorization/roleAssignments" - - type: output-not-matches - config: - pattern: "(?i)fatal error|unhandled exception|stack trace" - - # ── blob-reader-output-prompt ── - # Jest: "recommends Storage Blob Data Reader for blob read access" - - name: "Output: Storage Blob Data Reader with CLI and Bicep" - prompt: "What Azure role should I use to give my app read-only access to blob storage?" - tags: - type: integration - tier: full - cost: llm - area: output - graders: - - type: skill-invocation - config: - required: - - azure-rbac - - type: output-contains - config: - substring: "Storage Blob Data Reader" - - type: output-contains - config: - substring: "az role assignment" - - type: output-contains - config: - substring: "Microsoft.Authorization/roleAssignments" - - type: output-not-matches - config: - pattern: "(?i)fatal error|unhandled exception|stack trace" - - # ── keyvault-secrets-user-output-prompt ── - # Jest: "recommends Key Vault Secrets User for secret access" - - name: "Output: Key Vault Secrets User with CLI and Bicep" - prompt: "What role do I need to read secrets from Azure Key Vault?" - tags: - type: integration - tier: full - cost: llm - area: output - graders: - - type: skill-invocation - config: - required: - - azure-rbac - - type: output-contains - config: - substring: "Key Vault Secrets User" - - type: output-contains - config: - substring: "az role assignment" - - type: output-contains - config: - substring: "Microsoft.Authorization/roleAssignments" - - type: output-not-matches - config: - pattern: "(?i)fatal error|unhandled exception|stack trace" - - # ── bicep-contributor-output-prompt ── - # Jest: "provides Bicep code for role assignment" - - name: "Output: Contributor assignment Bicep snippet" - prompt: "Show me Bicep code to assign Contributor role to a managed identity on a storage account" - tags: - type: integration - tier: full - cost: llm - area: output - graders: - - type: skill-invocation - config: - required: - - azure-rbac - - type: output-contains - config: - substring: "Contributor" - - type: output-contains - config: - substring: "Microsoft.Authorization/roleAssignments" - - type: output-not-matches - config: - pattern: "(?i)fatal error|unhandled exception|stack trace" diff --git a/plugin/skills/azure-rbac/SKILL.md b/plugin/skills/azure-rbac/SKILL.md deleted file mode 100644 index a056de52c..000000000 --- a/plugin/skills/azure-rbac/SKILL.md +++ /dev/null @@ -1,17 +0,0 @@ ---- -name: azure-rbac -description: "Helps users find the right Azure RBAC role for an identity with least privilege access, then generate CLI commands and Bicep code to assign it. Also provides guidance on permissions required to grant roles. WHEN: bicep for role assignment, what role should I assign, least privilege role, RBAC role for, role to read blobs, role for managed identity, custom role definition, assign role to identity, what role do I need to grant access, permissions to assign roles." -license: MIT -metadata: - author: Microsoft - version: "0.0.0-placeholder" ---- -Use the 'azure__documentation' tool to find the minimal role definition that matches the desired permissions the user wants to assign to an identity. If no built-in role matches the desired permissions, use the 'azure__extension_cli_generate' tool to create a custom role definition with the desired permissions. Then use the 'azure__extension_cli_generate' tool to generate the CLI commands needed to assign that role to the identity. Finally, use the 'azure__bicepschema' and 'azure__get_azure_bestpractices' tools to provide a Bicep code snippet for adding the role assignment. If user is asking about role necessary to set access, refer to Prerequisites for Granting Roles down below: - -## Prerequisites for Granting Roles - -To assign RBAC roles to identities, you need a role that includes the `Microsoft.Authorization/roleAssignments/write` permission. The most common roles with this permission are: - -- **User Access Administrator** (least privilege - recommended for role assignment only) -- **Owner** (full access including role assignment) -- **Custom Role** with `Microsoft.Authorization/roleAssignments/write` diff --git a/plugin/skills/azure-rbac/version.json b/plugin/skills/azure-rbac/version.json deleted file mode 100644 index af73f6419..000000000 --- a/plugin/skills/azure-rbac/version.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "version": "1.1", - "pathFilters": [ - "." - ] -} diff --git a/plugin/skills/entra-agent-id/SKILL.md b/plugin/skills/entra-agent-id/SKILL.md index 1c3ed5093..f4bacd532 100644 --- a/plugin/skills/entra-agent-id/SKILL.md +++ b/plugin/skills/entra-agent-id/SKILL.md @@ -1,6 +1,6 @@ --- name: entra-agent-id -description: "Provision Microsoft Entra Agent Identity Blueprints, BlueprintPrincipals, and per-instance Agent Identities via Microsoft Graph, and configure OAuth 2.0 token exchange (fmi_path, OBO, cross-tenant) including the Microsoft Entra SDK for AgentID sidecar. USE FOR: Agent Identity Blueprint, BlueprintPrincipal, agent OAuth, fmi_path token exchange, agent OBO, Workload Identity Federation for agents, polyglot agent auth, Microsoft.Identity.Web.AgentIdentities. DO NOT USE FOR: standard Entra app registration (use entra-app-registration), Azure RBAC (use azure-rbac), Microsoft Foundry agent authoring (use microsoft-foundry)." +description: "Provision Microsoft Entra Agent Identity Blueprints, BlueprintPrincipals, and per-instance Agent Identities via Microsoft Graph, and configure OAuth 2.0 token exchange (fmi_path, OBO, cross-tenant) including the Microsoft Entra SDK for AgentID sidecar. USE FOR: Agent Identity Blueprint, BlueprintPrincipal, agent OAuth, fmi_path token exchange, agent OBO, Workload Identity Federation for agents, polyglot agent auth, Microsoft.Identity.Web.AgentIdentities. DO NOT USE FOR: standard Entra app registration (use entra-app-registration), Microsoft Foundry agent authoring (use microsoft-foundry)." license: MIT metadata: author: Microsoft diff --git a/plugin/skills/entra-app-registration/SKILL.md b/plugin/skills/entra-app-registration/SKILL.md index 3b0e43e4b..60986a6aa 100644 --- a/plugin/skills/entra-app-registration/SKILL.md +++ b/plugin/skills/entra-app-registration/SKILL.md @@ -1,6 +1,6 @@ --- name: entra-app-registration -description: "Guides Microsoft Entra ID app registration, OAuth 2.0 authentication, and MSAL integration. USE FOR: create app registration, register Azure AD app, configure OAuth, set up authentication, add API permissions, generate service principal, MSAL example, console app auth, Entra ID setup, Azure AD authentication. DO NOT USE FOR: Azure RBAC or role assignments (use azure-rbac), Key Vault secrets (use azure-keyvault-expiration-audit), general Azure resource security guidance." +description: "Guides Microsoft Entra ID app registration, OAuth 2.0 authentication, and MSAL integration. USE FOR: create app registration, register Azure AD app, configure OAuth, set up authentication, add API permissions, generate service principal, MSAL example, console app auth, Entra ID setup, Azure AD authentication. DO NOT USE FOR: Key Vault secrets (use azure-keyvault-expiration-audit), general Azure resource security guidance." license: MIT metadata: author: Microsoft diff --git a/tests/skills.json b/tests/skills.json index 67eb0a0d2..4f4beac69 100644 --- a/tests/skills.json +++ b/tests/skills.json @@ -16,7 +16,6 @@ "azure-messaging", "azure-prepare", "azure-quotas", - "azure-rbac", "azure-reliability", "azure-resource-lookup", "azure-resource-visualizer", @@ -31,6 +30,6 @@ "integrationTestSchedule": { "0 5 * * 2-6": "microsoft-foundry", "0 8 * * 2-6": "azure-deploy", - "0 12 * * 2-6": "airunway-aks-setup,appinsights-instrumentation,azure-ai,azure-aigateway,azure-cloud-migrate,azure-compliance,azure-compute,azure-cost,azure-diagnostics,azure-enterprise-infra-planner,azure-kubernetes,azure-kusto,azure-messaging,azure-prepare,azure-quotas,azure-rbac,azure-resource-lookup,azure-resource-visualizer,azure-storage,azure-upgrade,azure-validate,entra-agent-id,entra-app-registration,azure-reliability,python-appservice-deploy" + "0 12 * * 2-6": "airunway-aks-setup,appinsights-instrumentation,azure-ai,azure-aigateway,azure-cloud-migrate,azure-compliance,azure-compute,azure-cost,azure-diagnostics,azure-enterprise-infra-planner,azure-kubernetes,azure-kusto,azure-messaging,azure-prepare,azure-quotas,azure-resource-lookup,azure-resource-visualizer,azure-storage,azure-upgrade,azure-validate,entra-agent-id,entra-app-registration,azure-reliability,python-appservice-deploy" } } \ No newline at end of file From a4a46269093058a96651b9285c1b738fd88d57af Mon Sep 17 00:00:00 2001 From: Chunan Ye Date: Wed, 15 Jul 2026 10:09:03 -0700 Subject: [PATCH 2/2] remove azure-rbac from ci test --- .github/workflows/test-all-integration.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-all-integration.yml b/.github/workflows/test-all-integration.yml index 67f29cf2f..20903ae78 100644 --- a/.github/workflows/test-all-integration.yml +++ b/.github/workflows/test-all-integration.yml @@ -177,7 +177,7 @@ jobs: runs-on: ubuntu-latest env: # Skills that should be tested using vally in a JSON array - VALLY_SKILLS: '["azure-ai","azure-aigateway","azure-compliance","azure-diagnostics","azure-kusto","azure-storage","entra-app-registration","azure-messaging","airunway-aks-setup","azure-rbac","appinsights-instrumentation","azure-cost","azure-quotas","azure-kubernetes","azure-enterprise-infra-planner","azure-resource-lookup","azure-resource-visualizer","azure-reliability","entra-agent-id","python-appservice-deploy","azure-upgrade","azure-prepare","azure-compute","azure-validate"]' + VALLY_SKILLS: '["azure-ai","azure-aigateway","azure-compliance","azure-diagnostics","azure-kusto","azure-storage","entra-app-registration","azure-messaging","airunway-aks-setup","appinsights-instrumentation","azure-cost","azure-quotas","azure-kubernetes","azure-enterprise-infra-planner","azure-resource-lookup","azure-resource-visualizer","azure-reliability","entra-agent-id","python-appservice-deploy","azure-upgrade","azure-prepare","azure-compute","azure-validate"]' AZURE_CLIENT_ID: ${{ vars.AZURE_CLIENT_ID }} AZURE_TENANT_ID: ${{ vars.AZURE_TENANT_ID }} AZURE_SUBSCRIPTION_ID: ${{ vars.AZURE_SUBSCRIPTION_ID }}