From 42dadfb5550373c864a0404afde95b15beebd4fa Mon Sep 17 00:00:00 2001 From: Shane Ochotny Date: Fri, 3 Dec 2021 07:56:36 -0500 Subject: [PATCH] Updates to Bicep private networking --- Bicep/main.bicep | 10 ++ Bicep/main.parameters.json | 11 +- Bicep/modules/storageAccount.bicep | 123 ++++++++++++++++- Bicep/modules/synapseAnalytics.bicep | 198 +++++++++++++++++++++++---- README.md | 2 +- deploySynapse.sh | 63 ++++----- 6 files changed, 348 insertions(+), 59 deletions(-) diff --git a/Bicep/main.bicep b/Bicep/main.bicep index cbebdac..4459645 100644 --- a/Bicep/main.bicep +++ b/Bicep/main.bicep @@ -54,6 +54,12 @@ param private_endpoint_virtual_network string @description('Name of the Subnet within the Virtual Network where you want to create the Private Endpoints. (i.e. private-endpoint-subnet)') param private_endpoint_virtual_network_subnet string +@description('Name of the Resource Group that contains the Virtual Network where you want to create the Private Endpoints. (i.e. prod-network)') +param private_endpoint_virtual_network_resource_group string + +@description('Name of the Resource Group that contains the Private DNS Zones for Storage and Synapse if Private Endpoints are enabled. (i.e. prod-network)') +param private_endpoint_private_dns_zone_resource_group string + // Add a random suffix to ensure global uniqueness among the resources created // Bicep: https://docs.microsoft.com/en-us/azure/azure-resource-manager/bicep/bicep-functions-string#uniquestring var suffix = '${substring(uniqueString(subscription().subscriptionId, deployment().name), 0, 3)}' @@ -91,6 +97,8 @@ module synapseStorageAccount 'modules/storageAccount.bicep' = { enable_private_endpoints: enable_private_endpoints private_endpoint_virtual_network: private_endpoint_virtual_network private_endpoint_virtual_network_subnet: private_endpoint_virtual_network_subnet + private_endpoint_virtual_network_resource_group: private_endpoint_virtual_network_resource_group + private_endpoint_private_dns_zone_resource_group: private_endpoint_private_dns_zone_resource_group } dependsOn: [ @@ -113,6 +121,8 @@ module synapseAnalytics 'modules/synapseAnalytics.bicep' = { enable_private_endpoints: enable_private_endpoints private_endpoint_virtual_network: private_endpoint_virtual_network private_endpoint_virtual_network_subnet: private_endpoint_virtual_network_subnet + private_endpoint_virtual_network_resource_group: private_endpoint_virtual_network_resource_group + private_endpoint_private_dns_zone_resource_group: private_endpoint_private_dns_zone_resource_group } dependsOn: [ diff --git a/Bicep/main.parameters.json b/Bicep/main.parameters.json index 4db46e7..3c697a3 100644 --- a/Bicep/main.parameters.json +++ b/Bicep/main.parameters.json @@ -18,7 +18,7 @@ "value": "Pass@word123" }, "synapse_azure_ad_admin_object_id": { - "value": "REPLACE_SYNAPSE_AZURE_AD_ADMIN_OBJECT_ID" + "value": "REPLACE_SYNAPSE_AZURE_AD_ADMIN_OBJECT_ID" }, "enable_private_endpoints": { "value": false @@ -27,8 +27,13 @@ "value": "" }, "private_endpoint_virtual_network_subnet": { - "value": "" + "value": "" + }, + "private_endpoint_virtual_network_resource_group": { + "value": "" + }, + "private_endpoint_private_dns_zone_resource_group": { + "value": "" } - } } diff --git a/Bicep/modules/storageAccount.bicep b/Bicep/modules/storageAccount.bicep index 41dd90b..898dc03 100644 --- a/Bicep/modules/storageAccount.bicep +++ b/Bicep/modules/storageAccount.bicep @@ -15,11 +15,13 @@ param logAnalyticsId string param enable_private_endpoints bool param private_endpoint_virtual_network string param private_endpoint_virtual_network_subnet string +param private_endpoint_virtual_network_resource_group string +param private_endpoint_private_dns_zone_resource_group string // Azure Data Lake Storage Gen2: Storage for the Synapse Workspace configuration data and test data // Azure: https://docs.microsoft.com/en-us/azure/storage/blobs/data-lake-storage-introduction // Bicep: https://docs.microsoft.com/en-us/azure/templates/microsoft.storage/storageaccounts -resource synapseStorageAccount 'Microsoft.Storage/storageAccounts@2019-06-01' = { +resource synapseStorageAccount 'Microsoft.Storage/storageAccounts@2021-06-01' = { name: 'pocsynapseadls${suffix}' location: azure_region kind: 'StorageV2' @@ -28,6 +30,17 @@ resource synapseStorageAccount 'Microsoft.Storage/storageAccounts@2019-06-01' = } properties: { isHnsEnabled: true + networkAcls: { + bypass: (enable_private_endpoints) ? 'None' : 'AzureServices' + defaultAction: (enable_private_endpoints) ? 'Deny' : 'Allow' + resourceAccessRules: [ + { + resourceId: '/subscriptions/${subscription().subscriptionId}/resourcegroups/${resourceGroup().name}/providers/Microsoft.Synapse/workspaces/*' + tenantId: subscription().tenantId + } + ] + } + publicNetworkAccess: (enable_private_endpoints) ? 'Disabled' : 'Enabled' } } @@ -110,6 +123,114 @@ resource synapseStorageDiagnostics 'Microsoft.Insights/diagnosticSettings@2021-0 } } +// Reference to the existing Virtual Network if Private Endpoints we're enabling Private Endpoints +resource existingVirtualNetwork 'Microsoft.Network/virtualNetworks@2020-08-01' existing = if (enable_private_endpoints) { + name: private_endpoint_virtual_network + scope: resourceGroup(private_endpoint_virtual_network_resource_group) +} + +// Reference to the existing Virtual Network Subnet to create the Private Endpoints if we're enabling them +resource existingVirtualNetworkSubnet 'Microsoft.Network/virtualNetworks/subnets@2020-04-01' existing = if (enable_private_endpoints) { + parent: existingVirtualNetwork + name: private_endpoint_virtual_network_subnet +} + +// Azure Data Lake Storage Gen2 Blob Private Endpoint +// Azure: https://docs.microsoft.com/en-us/azure/storage/common/storage-private-endpoints +// Bicep: https://docs.microsoft.com/en-us/azure/templates/microsoft.network/privateendpoints +resource synapseStorageAccountBlobPrivateEndpoint 'Microsoft.Network/privateEndpoints@2021-03-01' = if (enable_private_endpoints) { + name: 'pocsynapsestorage-blob-endpoint' + location: azure_region + properties: { + subnet: { + id: existingVirtualNetworkSubnet.id + } + privateLinkServiceConnections: [ + { + name: 'pocsynapsestorage-blob-privateserviceconnection' + properties: { + privateLinkServiceId: synapseStorageAccount.id + groupIds: [ + 'blob' + ] + } + } + ] + } +} + +// Azure Data Lake Storage Gen2 DFS Private Endpoint +// Azure: https://docs.microsoft.com/en-us/azure/storage/common/storage-private-endpoints +// Bicep: https://docs.microsoft.com/en-us/azure/templates/microsoft.network/privateendpoints +resource synapseStorageAccountDfsPrivateEndpoint 'Microsoft.Network/privateEndpoints@2021-03-01' = if (enable_private_endpoints) { + name: 'pocsynapsestorage-dfs-endpoint' + location: azure_region + properties: { + subnet: { + id: existingVirtualNetworkSubnet.id + } + privateLinkServiceConnections: [ + { + name: 'pocsynapsestorage-dfs-privateserviceconnection' + properties: { + privateLinkServiceId: synapseStorageAccount.id + groupIds: [ + 'dfs' + ] + } + } + ] + } +} + +// Reference to the existing Storage Blob Private DNS Zone if Private Endpoints are enabled so we can auto-register +resource privateDnsZoneBlob 'Microsoft.Network/privateDnsZones@2020-01-01' existing = if (enable_private_endpoints) { + name: 'privatelink.blob.${environment().suffixes.storage}' + scope: resourceGroup(private_endpoint_private_dns_zone_resource_group) +} + +// Reference to the existing Storage DFS Private DNS Zone if Private Endpoints are enabled so we can auto-register +resource privateDnsZoneDfs 'Microsoft.Network/privateDnsZones@2020-01-01' existing = if (enable_private_endpoints) { + name: 'privatelink.dfs.${environment().suffixes.storage}' + scope: resourceGroup(private_endpoint_private_dns_zone_resource_group) +} + +// Azure Data Lake Storage Gen2 Blob Private Endpoint DNS Registration +// Azure: +// Bicep: +resource blobPrivateDnsZoneGroup 'Microsoft.Network/privateEndpoints/privateDnsZoneGroups@2021-03-01' = if (enable_private_endpoints) { + parent: synapseStorageAccountBlobPrivateEndpoint + name: 'default' + properties: { + privateDnsZoneConfigs: [ + { + name: 'default' + properties: { + privateDnsZoneId: privateDnsZoneBlob.id + } + } + ] + } +} + +// Azure Data Lake Storage Gen2 DFS Private Endpoint DNS Registration +// Azure: +// Bicep: +resource dfsPrivateDnsZoneGroup 'Microsoft.Network/privateEndpoints/privateDnsZoneGroups@2021-03-01' = if (enable_private_endpoints) { + parent: synapseStorageAccountDfsPrivateEndpoint + name: 'default' + properties: { + privateDnsZoneConfigs: [ + { + name: 'default' + properties: { + privateDnsZoneId: privateDnsZoneDfs.id + } + } + ] + } +} + // Outputs for reference in the Post-Deployment Configuration output synapseStorageAccountDFS string = synapseStorageAccount.properties.primaryEndpoints.dfs output datalake_name string = synapseStorageAccount.name diff --git a/Bicep/modules/synapseAnalytics.bicep b/Bicep/modules/synapseAnalytics.bicep index 388d57b..ce3b3c2 100644 --- a/Bicep/modules/synapseAnalytics.bicep +++ b/Bicep/modules/synapseAnalytics.bicep @@ -18,6 +18,8 @@ param logAnalyticsId string param enable_private_endpoints bool param private_endpoint_virtual_network string param private_endpoint_virtual_network_subnet string +param private_endpoint_virtual_network_resource_group string +param private_endpoint_private_dns_zone_resource_group string // Synapse Workspace // Azure: https://docs.microsoft.com/en-us/azure/synapse-analytics/overview-what-is @@ -30,7 +32,7 @@ resource synapseWorkspace 'Microsoft.Synapse/workspaces@2021-06-01' = { } properties: { - publicNetworkAccess: 'Enabled' + //publicNetworkAccess: (enable_private_endpoints) ? 'Disabled' : 'Enabled' managedVirtualNetwork: 'default' defaultDataLakeStorage: { accountUrl: synapseStorageAccountDFS @@ -62,28 +64,6 @@ resource synapseStorageWorkspacePermissions 'Microsoft.Authorization/roleAssignm ] } -// Synapse Workspace Firewall: Allow authenticated access from anywhere if Private Endpoints are disabled -// Azure: https://docs.microsoft.com/en-us/azure/synapse-analytics/security/synapse-workspace-ip-firewall -// Bicep: https://docs.microsoft.com/en-us/azure/templates/microsoft.synapse/workspaces/firewallrules -resource synapseFirewallAllowAzureServices 'Microsoft.Synapse/workspaces/firewallRules@2021-06-01' = { - name: '${synapseWorkspace.name}/AllowAllWindowsAzureIps' - properties: { - endIpAddress: '0.0.0.0' - startIpAddress: '0.0.0.0' - } -} - -// Synapse Workspace Firewall: Allow authenticated access from anywhere if Private Endpoints are disabled -// Azure: https://docs.microsoft.com/en-us/azure/synapse-analytics/security/synapse-workspace-ip-firewall -// Bicep: https://docs.microsoft.com/en-us/azure/templates/microsoft.synapse/workspaces/firewallrules -resource synapseFirewallAllowAll 'Microsoft.Synapse/workspaces/firewallRules@2021-06-01' = { - name: '${synapseWorkspace.name}/AllowAll' - properties: { - endIpAddress: '255.255.255.255' - startIpAddress: '0.0.0.0' - } -} - // Diagnostic Logs for Synapse // Bicep: https://docs.microsoft.com/en-us/azure/templates/microsoft.insights/diagnosticsettings resource synapseDiagnostics 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' = { @@ -193,5 +173,177 @@ resource synapseSQLPoolDiagnostics 'Microsoft.Insights/diagnosticSettings@2021-0 ] } +// Synapse Workspace Firewall: Allow authenticated access from anywhere if Private Endpoints are disabled +// Azure: https://docs.microsoft.com/en-us/azure/synapse-analytics/security/synapse-workspace-ip-firewall +// Bicep: https://docs.microsoft.com/en-us/azure/templates/microsoft.synapse/workspaces/firewallrules +resource synapseFirewallAllowAzureServices 'Microsoft.Synapse/workspaces/firewallRules@2021-06-01' = if (!enable_private_endpoints) { + name: '${synapseWorkspace.name}/AllowAllWindowsAzureIps' + properties: { + endIpAddress: '0.0.0.0' + startIpAddress: '0.0.0.0' + } +} + +// Synapse Workspace Firewall: Allow authenticated access from anywhere if Private Endpoints are disabled +// Azure: https://docs.microsoft.com/en-us/azure/synapse-analytics/security/synapse-workspace-ip-firewall +// Bicep: https://docs.microsoft.com/en-us/azure/templates/microsoft.synapse/workspaces/firewallrules +resource synapseFirewallAllowAll 'Microsoft.Synapse/workspaces/firewallRules@2021-06-01' = if (!enable_private_endpoints) { + name: '${synapseWorkspace.name}/AllowAll' + properties: { + endIpAddress: '255.255.255.255' + startIpAddress: '0.0.0.0' + } +} + +// Reference to the existing Virtual Network if Private Endpoints we're enabling Private Endpoints +resource existingVirtualNetwork 'Microsoft.Network/virtualNetworks@2020-08-01' existing = if (enable_private_endpoints) { + name: private_endpoint_virtual_network + scope: resourceGroup(private_endpoint_virtual_network_resource_group) +} + +// Reference to the existing Virtual Network Subnet to create the Private Endpoints if we're enabling them +resource existingVirtualNetworkSubnet 'Microsoft.Network/virtualNetworks/subnets@2020-04-01' existing = if (enable_private_endpoints) { + parent: existingVirtualNetwork + name: private_endpoint_virtual_network_subnet +} + +// Create a Private Endpoint for Synapse Dedicated SQL Pools +// Azure: https://docs.microsoft.com/en-us/azure/synapse-analytics/security/how-to-connect-to-workspace-with-private-links +// Bicep: https://docs.microsoft.com/en-us/azure/templates/microsoft.network/privateendpoints +resource synapseWorkspaceSqlPrivateEndpoint 'Microsoft.Network/privateEndpoints@2021-03-01' = if (enable_private_endpoints) { + name: 'pocsynapseanalytics-sql-endpoint' + location: azure_region + properties: { + subnet: { + id: existingVirtualNetworkSubnet.id + } + privateLinkServiceConnections: [ + { + name: 'pocsynapseanalytics-sql-privateserviceconnection' + properties: { + privateLinkServiceId: synapseWorkspace.id + groupIds: [ + 'sql' + ] + } + } + ] + } +} + +// Create a Private Endpoint for Synapse Serverless SQL +// Azure: https://docs.microsoft.com/en-us/azure/synapse-analytics/security/how-to-connect-to-workspace-with-private-links +// Bicep: https://docs.microsoft.com/en-us/azure/templates/microsoft.network/privateendpoints +resource synapseWorkspaceServerlessSqlPrivateEndpoint 'Microsoft.Network/privateEndpoints@2021-03-01' = if (enable_private_endpoints) { + name: 'pocsynapseanalytics-sqlondemand-endpoint' + location: azure_region + properties: { + subnet: { + id: existingVirtualNetworkSubnet.id + } + privateLinkServiceConnections: [ + { + name: 'pocsynapseanalytics-sqlondemand-privateserviceconnection' + properties: { + privateLinkServiceId: synapseWorkspace.id + groupIds: [ + 'sqlondemand' + ] + } + } + ] + } +} + +// Create a Private Endpoint for Synapse Workspace +// Azure: https://docs.microsoft.com/en-us/azure/synapse-analytics/security/how-to-connect-to-workspace-with-private-links +// Bicep: https://docs.microsoft.com/en-us/azure/templates/microsoft.network/privateendpoints +resource synapseWorkspaceDevPrivateEndpoint 'Microsoft.Network/privateEndpoints@2021-03-01' = if (enable_private_endpoints) { + name: 'pocsynapseanalytics-dev-endpoint' + location: azure_region + properties: { + subnet: { + id: existingVirtualNetworkSubnet.id + } + privateLinkServiceConnections: [ + { + name: 'pocsynapseanalytics-dev-privateserviceconnection' + properties: { + privateLinkServiceId: synapseWorkspace.id + groupIds: [ + 'dev' + ] + } + } + ] + } +} + +// Reference to the existing Synapse Dedicated SQL Private DNS Zone if Private Endpoints are enabled so we can auto-register +resource privateDnsZoneSynapseSql 'Microsoft.Network/privateDnsZones@2020-01-01' existing = if (enable_private_endpoints) { + name: 'privatelink.sql.azuresynapse.net' + scope: resourceGroup(private_endpoint_private_dns_zone_resource_group) +} + +// Synapse Dedicated SQL Private Endpoint DNS Registration +// Azure: https://docs.microsoft.com/en-us/azure/private-link/private-endpoint-dns +// Bicep: https://docs.microsoft.com/en-us/azure/templates/microsoft.network/privateendpoints/privatednszonegroups +resource synapseSqlPrivateDnsZoneGroup 'Microsoft.Network/privateEndpoints/privateDnsZoneGroups@2021-03-01' = if (enable_private_endpoints) { + parent: synapseWorkspaceSqlPrivateEndpoint + name: 'synapseSql' + properties: { + privateDnsZoneConfigs: [ + { + name: 'default' + properties: { + privateDnsZoneId: privateDnsZoneSynapseSql.id + } + } + ] + } +} + +// Synapse Serverless SQL Private Endpoint DNS Registration +// Azure: https://docs.microsoft.com/en-us/azure/private-link/private-endpoint-dns +// Bicep: https://docs.microsoft.com/en-us/azure/templates/microsoft.network/privateendpoints/privatednszonegroups +resource synapseServerlessSqlPrivateDnsZoneGroup 'Microsoft.Network/privateEndpoints/privateDnsZoneGroups@2021-03-01' = if (enable_private_endpoints) { + parent: synapseWorkspaceServerlessSqlPrivateEndpoint + name: 'synapseSqlServerless' + properties: { + privateDnsZoneConfigs: [ + { + name: 'default' + properties: { + privateDnsZoneId: privateDnsZoneSynapseSql.id + } + } + ] + } +} + +// Reference to the existing Synapse Workspace Dev Private DNS Zone if Private Endpoints are enabled so we can auto-register +resource privateDnsZoneSynapseDev 'Microsoft.Network/privateDnsZones@2020-01-01' existing = if (enable_private_endpoints) { + name: 'privatelink.dev.azuresynapse.net' + scope: resourceGroup(private_endpoint_private_dns_zone_resource_group) +} + +// Synapse Workspace Private Endpoint DNS Registration +// Azure: https://docs.microsoft.com/en-us/azure/private-link/private-endpoint-dns +// Bicep: https://docs.microsoft.com/en-us/azure/templates/microsoft.network/privateendpoints/privatednszonegroups +resource synapseDevPrivateDnsZoneGroup 'Microsoft.Network/privateEndpoints/privateDnsZoneGroups@2021-03-01' = if (enable_private_endpoints) { + parent: synapseWorkspaceDevPrivateEndpoint + name: 'synapseDev' + properties: { + privateDnsZoneConfigs: [ + { + name: 'default' + properties: { + privateDnsZoneId: privateDnsZoneSynapseDev.id + } + } + ] + } +} + // Outputs for reference in the Post-Deployment Configuration output synapse_analytics_workspace_name string = synapseWorkspace.name diff --git a/README.md b/README.md index 98836ba..21a5563 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Azure-Synapse-Analytics-PoC +# Azure Synapse Analytics PoC Accelerator ![alt tag](https://raw.githubusercontent.com/shaneochotny/Azure-Synapse-Analytics-PoC\/main/Images/Synapse-Analytics-PoC-Architecture.gif) diff --git a/deploySynapse.sh b/deploySynapse.sh index 6269805..1f040a2 100644 --- a/deploySynapse.sh +++ b/deploySynapse.sh @@ -27,20 +27,20 @@ # Make sure this configuration script hasn't been executed already if [ -f "deploySynapse.complete" ]; then - echo "ERROR: It appears this configuration has already been completed."; + echo "ERROR: It appears this configuration has already been completed." | tee -a deploySynapse.log exit 1; fi # Try and determine if we're executing from within the Azure Cloud Shell if [ ! "${AZUREPS_HOST_ENVIRONMENT}" = "cloud-shell/1.0" ]; then - echo "ERROR: It doesn't appear like your executing this from the Azure Cloud Shell. Please use the Azure Cloud Shell at https://shell.azure.com"; + echo "ERROR: It doesn't appear like your executing this from the Azure Cloud Shell. Please use the Azure Cloud Shell at https://shell.azure.com" | tee -a deploySynapse.log exit 1; fi # Try and get a token to validate that we're logged into Azure CLI aadToken=$(az account get-access-token --resource=https://dev.azuresynapse.net --query accessToken --output tsv 2>&1) if echo "$aadToken" | grep -q "ERROR"; then - echo "ERROR: You don't appear to be logged in to Azure CLI. Please login to the Azure CLI using 'az login'"; + echo "ERROR: You don't appear to be logged in to Azure CLI. Please login to the Azure CLI using 'az login'" | tee -a deploySynapse.log exit 1; fi @@ -59,7 +59,7 @@ bicepDeploymentCheck=$(az deployment sub show --name Azure-Synapse-Analytics-PoC if [ "$bicepDeploymentCheck" == "Succeeded" ]; then deploymentType="bicep" elif [ "$bicepDeploymentCheck" == "Failed" ] || [ "$bicepDeploymentCheck" == "Canceled" ]; then - echo "ERROR: It looks like a Bicep deployment was attempted, but failed." + echo "ERROR: It looks like a Bicep deployment was attempted, but failed." | tee -a deploySynapse.log exit 1; fi @@ -72,13 +72,13 @@ if echo "$bicepDeploymentCheck" | grep -q "DeploymentNotFound"; then # There was no Bicep or Terraform deployment so we're taking the easy button approach and deploying the Synapse # environment on behalf of the user via Terraform. - echo "Deploying Synapse Analytics environment. This will take several minutes..." + echo "Deploying Synapse Analytics environment. This will take several minutes..." | tee -a deploySynapse.log # Terraform init and validation echo "Executing 'terraform -chdir=Terraform init'" terraformInit=$(terraform -chdir=Terraform init 2>&1) if ! echo "$terraformInit" | grep -q "Terraform has been successfully initialized!"; then - echo "ERROR: Failed to perform 'terraform -chdir=Terraform init'" + echo "ERROR: Failed to perform 'terraform -chdir=Terraform init'" | tee -a deploySynapse.log exit 1; fi @@ -86,7 +86,7 @@ if echo "$bicepDeploymentCheck" | grep -q "DeploymentNotFound"; then echo "Executing 'terraform -chdir=Terraform plan'" terraformPlan=$(terraform -chdir=Terraform plan) if echo "$terraformPlan" | grep -q "Error:"; then - echo "ERROR: Failed to perform 'terraform -chdir=Terraform plan'" + echo "ERROR: Failed to perform 'terraform -chdir=Terraform plan'" | tee -a deploySynapse.log exit 1; fi @@ -96,7 +96,7 @@ if echo "$bicepDeploymentCheck" | grep -q "DeploymentNotFound"; then if echo "$terraformApply" | grep -q "Apply complete!"; then deploymentType="terraform" else - echo "ERROR: Failed to perform 'terraform -chdir=Terraform apply'" + echo "ERROR: Failed to perform 'terraform -chdir=Terraform apply'" | tee -a deploySynapse.log exit 1; fi fi @@ -120,7 +120,7 @@ fi # Get the output variables from the Bicep deployment if [ "$deploymentType" == "bicep" ]; then - resourceGroup=$(jq -r .parameters.resource_group_name.value Bicep/main.parameters.json 2>&1) + resourceGroup=$(az deployment sub show --name Azure-Synapse-Analytics-PoC --query properties.parameters.resource_group_name.value --output tsv 2>&1) synapseAnalyticsWorkspaceName=$(az deployment sub show --name Azure-Synapse-Analytics-PoC --query properties.outputs.synapse_analytics_workspace_name.value --output tsv 2>&1) synapseAnalyticsSQLPoolName=$(az deployment sub show --name Azure-Synapse-Analytics-PoC --query properties.outputs.synapse_sql_pool_name.value --output tsv 2>&1) synapseAnalyticsSQLAdmin=$(az deployment sub show --name Azure-Synapse-Analytics-PoC --query properties.outputs.synapse_sql_administrator_login.value --output tsv 2>&1) @@ -130,26 +130,27 @@ if [ "$deploymentType" == "bicep" ]; then privateEndpointsEnabled=$(az deployment sub show --name Azure-Synapse-Analytics-PoC --query properties.outputs.private_endpoints_enabled.value --output tsv 2>&1) fi -echo "Deployment Type: ${deploymentType}" -echo "Azure Subscription: ${azureSubscriptionName}" -echo "Azure Subscription ID: ${azureSubscriptionID}" -echo "Azure AD Username: ${azureUsername}" -echo "Synapse Analytics Workspace Resource Group: ${resourceGroup}" -echo "Synapse Analytics Workspace: ${synapseAnalyticsWorkspaceName}" -echo "Synapse Analytics SQL Admin: ${synapseAnalyticsSQLAdmin}" -echo "Data Lake Name: ${datalakeName}" +echo "Deployment Type: ${deploymentType}" | tee -a deploySynapse.log +echo "Azure Subscription: ${azureSubscriptionName}" | tee -a deploySynapse.log +echo "Azure Subscription ID: ${azureSubscriptionID}" | tee -a deploySynapse.log +echo "Azure AD Username: ${azureUsername}" | tee -a deploySynapse.log +echo "Synapse Analytics Workspace Resource Group: ${resourceGroup}" | tee -a deploySynapse.log +echo "Synapse Analytics Workspace: ${synapseAnalyticsWorkspaceName}" | tee -a deploySynapse.log +echo "Synapse Analytics SQL Admin: ${synapseAnalyticsSQLAdmin}" | tee -a deploySynapse.log +echo "Data Lake Name: ${datalakeName}" | tee -a deploySynapse.log # If Private Endpoints are enabled, temporarily disable the firewalls so we can copy files and perform additional configuration if [ "$privateEndpointsEnabled" == "true" ]; then - az storage account update --name ${datalakeName} --resource-group ${resourceGroup} --default-action Allow --only-show-errors -o none - az synapse workspace firewall-rule create --name AllowAllWindowsAzureIps --resource-group ${resourceGroup} --workspace-name ${synapseAnalyticsWorkspaceName} --start-ip-address 0.0.0.0 --end-ip-address 0.0.0.0 --only-show-errors -o none + az storage account update --name ${datalakeName} --resource-group ${resourceGroup} --default-action Allow >> deploySynapse.log 2>&1 + az synapse workspace firewall-rule create --name AllowAll --resource-group ${resourceGroup} --workspace-name ${synapseAnalyticsWorkspaceName} --start-ip-address 0.0.0.0 --end-ip-address 255.255.255.255 >> deploySynapse.log 2>&1 + az synapse workspace firewall-rule create --name AllowAllWindowsAzureIps --resource-group ${resourceGroup} --workspace-name ${synapseAnalyticsWorkspaceName} --start-ip-address 0.0.0.0 --end-ip-address 0.0.0.0 >> deploySynapse.log 2>&1 fi # Enable Result Set Cache -echo "Enabling Result Set Caching..." -sqlcmd -U ${synapseAnalyticsSQLAdmin} -P ${synapseAnalyticsSQLAdminPassword} -S tcp:${synapseAnalyticsWorkspaceName}.sql.azuresynapse.net -d master -I -Q "ALTER DATABASE ${synapseAnalyticsSQLPoolName} SET RESULT_SET_CACHING ON;" +echo "Enabling Result Set Caching..." | tee -a deploySynapse.log +sqlcmd -U ${synapseAnalyticsSQLAdmin} -P ${synapseAnalyticsSQLAdminPassword} -S tcp:${synapseAnalyticsWorkspaceName}.sql.azuresynapse.net -d master -I -Q "ALTER DATABASE ${synapseAnalyticsSQLPoolName} SET RESULT_SET_CACHING ON;" >> deploySynapse.log 2>&1 -echo "Creating the Auto Pause and Resume pipeline..." +echo "Creating the Auto Pause and Resume pipeline..." | tee -a deploySynapse.log # Copy the Auto_Pause_and_Resume Pipeline template and update the variables cp artifacts/Auto_Pause_and_Resume.json.tmpl artifacts/Auto_Pause_and_Resume.json 2>&1 @@ -165,7 +166,7 @@ az synapse pipeline create --only-show-errors -o none --workspace-name ${synapse az synapse trigger create --only-show-errors -o none --workspace-name ${synapseAnalyticsWorkspaceName} --name Pause --file @artifacts/triggerPause.json az synapse trigger create --only-show-errors -o none --workspace-name ${synapseAnalyticsWorkspaceName} --name Resume --file @artifacts/triggerResume.json -echo "Creating the Parquet Auto Ingestion pipeline..." +echo "Creating the Parquet Auto Ingestion pipeline..." | tee -a deploySynapse.log # Create the Resource Class Logins cp artifacts/Create_Resource_Class_Logins.sql.tmpl artifacts/Create_Resource_Class_Logins.sql 2>&1 @@ -192,12 +193,12 @@ az storage copy --only-show-errors -o none --destination https://${datalakeName} # Copy sample data for the Parquet Auto Ingestion pipeline sampleDataStorageSAS="?sv=2020-10-02&st=2021-11-23T05%3A00%3A00Z&se=2022-11-24T05%3A00%3A00Z&sr=c&sp=rl&sig=PMi22pEYzw1dHNrOI9gqrwcbi3AJLq%2BxWoSX9SOTLuw%3D" -az storage copy --only-show-errors -o none --destination "https://${datalakeName}.blob.core.windows.net/data/Sample/" --source "https://synapseanalyticspocdata.blob.core.windows.net/sample/AdventureWorks/${sampleDataStorageSAS}" --recursive > /dev/null 2>&1 +az storage copy --only-show-errors -o none --destination "https://${datalakeName}.blob.core.windows.net/data/Sample/" --source "https://synapseanalyticspocdata.blob.core.windows.net/sample/AdventureWorks/${sampleDataStorageSAS}" --recursive >> deploySynapse.log 2>&1 # Create the Auto_Pause_and_Resume Pipeline in the Synapse Analytics Workspace -az synapse pipeline create --only-show-errors -o none --workspace-name ${synapseAnalyticsWorkspaceName} --name "Parquet Auto Ingestion" --file @artifacts/Parquet_Auto_Ingestion.json +az synapse pipeline create --only-show-errors -o none --workspace-name ${synapseAnalyticsWorkspaceName} --name "Parquet Auto Ingestion" --file @artifacts/Parquet_Auto_Ingestion.json >> deploySynapse.log 2>&1 -echo "Creating the Demo Data database using Synapse Serverless SQL..." +echo "Creating the Demo Data database using Synapse Serverless SQL..." | tee -a deploySynapse.log # Create a Demo Data database using Synapse Serverless SQL sqlcmd -U ${synapseAnalyticsSQLAdmin} -P ${synapseAnalyticsSQLAdminPassword} -S tcp:${synapseAnalyticsWorkspaceName}-ondemand.sql.azuresynapse.net -d master -I -Q "CREATE DATABASE [Demo Data (Serverless)];" @@ -207,11 +208,11 @@ sqlcmd -U ${synapseAnalyticsSQLAdmin} -P ${synapseAnalyticsSQLAdminPassword} -S # Restore the firewall rules on ADLS an Azure Synapse Analytics. That was needed temporarily to apply these settings. if [ "$privateEndpointsEnabled" == "true" ]; then - echo "Restoring firewall rules..." - az storage account update --name ${datalakeName} --resource-group ${resourceGroup} --default-action Deny --only-show-errors -o none - az synapse workspace firewall-rule delete --name AllowAllWindowsAzureIps --resource-group ${resourceGroup} --workspace-name ${synapseAnalyticsWorkspaceName} --only-show-errors -o none --yes - az resource update --name ${synapseAnalyticsWorkspaceName} --resource-group ${resourceGroup} --resource-type Microsoft.Synapse/workspaces --set Properties.publicNetworkAccess="Disabled" --only-show-errors -o none + echo "Restoring firewall rules..." | tee -a deploySynapse.log + az storage account update --name ${datalakeName} --resource-group ${resourceGroup} --default-action Deny >> deploySynapse.log 2>&1 + az synapse workspace firewall-rule delete --name AllowAll --resource-group ${resourceGroup} --workspace-name ${synapseAnalyticsWorkspaceName} --yes >> deploySynapse.log 2>&1 + az synapse workspace firewall-rule delete --name AllowAllWindowsAzureIps --resource-group ${resourceGroup} --workspace-name ${synapseAnalyticsWorkspaceName} --yes >> deploySynapse.log 2>&1 fi -echo "Deployment complete!" +echo "Deployment complete!" | tee -a deploySynapse.log touch deploySynapse.complete \ No newline at end of file