added the first draft of the az-svc-data-integration-mlw

ran the terraform recursive and fixed it

changed the cares name into sample

Add adf and fix AF to use id

Author: Mahsa Hanifi

infra/modules/providers/azure/data-factory/README.md

@@ -17,15 +17,13 @@ An instance of the `data-factory` module deploys the _**Data Factory**_ in order
 - Ability to provision a single Data Factory instance
 - Ability to provision a configurable Pipeline
 - Ability to configure Trigger
-- Ability to configure SQL server Dataset
-- Ability to configure SQL server Linked Service
+
 
 ## Out Of Scope
 
 The following are not support in the time being
 
 - Creating Multiple pipelines
-- Only SQL server Dataset/Linked Service are implemented.
 
 ## Definition
 
@@ -35,8 +33,6 @@ Terraform resources used to define the `data-factory` module include the followi
 - [azurerm_data_factory_integration_runtime_managed](https://www.terraform.io/docs/providers/azurerm/r/data_factory_integration_runtime_managed.html)
 - [azurerm_data_factory_pipeline](https://www.terraform.io/docs/providers/azurerm/r/data_factory_pipeline.html)
 - [azurerm_data_factory_trigger_schedule](https://www.terraform.io/docs/providers/azurerm/r/data_factory_trigger_schedule.html)
-- [azurerm_data_factory_dataset_sql_server](https://www.terraform.io/docs/providers/azurerm/r/data_factory_dataset_sql_server_table.html)
-- [azurerm_data_factory_linked_service_sql_server](https://www.terraform.io/docs/providers/azurerm/r/data_factory_linked_service_sql_server.html)
 
 ## Usage
 
@@ -60,11 +56,6 @@ module "data_factory" {
   data_factory_trigger_name                 = "adftrigger"
   data_factory_trigger_interval             = 1
   data_factory_trigger_frequency            = "Minute"
-  data_factory_dataset_sql_name             = "adfsqldataset"
-  data_factory_dataset_sql_table_name       = "adfsqldatasettable"
-  data_factory_dataset_sql_folder           = ""
-  data_factory_linked_sql_name              = "adfsqllinked"
-  data_factory_linked_sql_connection_string = "Server=tcp:adfsql..."
 }
 ```
 

infra/modules/providers/azure/data-factory/datasets.tf

@@ -28,16 +28,6 @@ output "trigger_interval" {
   value       = azurerm_data_factory_trigger_schedule.main.interval
 }
 
-output "sql_dataset_id" {
-  description = "The ID of the SQL server dataset created"
-  value       = azurerm_data_factory_dataset_sql_server_table.main.id
-}
-
-output "sql_linked_service_id" {
-  description = "The ID of the SQL server Linked service created"
-  value       = azurerm_data_factory_linked_service_sql_server.main.id
-}
-
 output "adf_identity_principal_id" {
   description = "The ID of the principal(client) in Azure active directory"
   value       = azurerm_data_factory.main.identity[0].principal_id

infra/modules/providers/azure/data-factory/linkedservices.tf

@@ -2,10 +2,7 @@ resource_group_name                       = ""
 data_factory_name                         = ""
 data_factory_runtime_name                 = ""
 data_factory_pipeline_name                = ""
-data_factory_dataset_sql_name             = ""
-data_factory_dataset_sql_table_name       = ""
-data_factory_linked_sql_name              = ""
-data_factory_linked_sql_connection_string = ""
+data_factory_trigger_name = ""
 vnet_integration = {
   vnet_id     = ""
   subnet_name = ""

infra/modules/providers/azure/data-factory/output.tf

@@ -25,16 +25,6 @@ func TestDataFactory(t *testing.T) {
 				"data_factory_name",
 				"pipeline_name",
 			),
-			VerifyCreatedDataset(subscription,
-				"resource_group_name",
-				"data_factory_name",
-				"sql_dataset_id",
-			),
-			VerifyCreatedLinkedService(subscription,
-				"resource_group_name",
-				"data_factory_name",
-				"sql_linked_service_id",
-			),
 		},
 	}
 	integration.RunIntegrationTests(&testFixture)

infra/modules/providers/azure/data-factory/terraform.tfvars.template

@@ -1,13 +1,9 @@
-resource_group_name                       = "adftest"
-data_factory_name                         = "adftest"
-data_factory_runtime_name                 = "adfrttest"
-data_factory_pipeline_name                = "testpipeline"
-data_factory_trigger_name                 = "testtrigger"
-data_factory_dataset_sql_name             = "testsql"
-data_factory_dataset_sql_table_name       = "adfsqltableheba"
-data_factory_linked_sql_name              = "testlinkedsql"
-data_factory_linked_sql_connection_string = "connectionstring"
+resource_group_name                       = ""
+data_factory_name                         = ""
+data_factory_runtime_name                 = ""
+data_factory_pipeline_name                = ""
+data_factory_trigger_name = ""
 vnet_integration = {
-  vnet_id     = "/subscriptions/resourceGroups/providers/Microsoft.Network/virtualNetworks/testvnet"
-  subnet_name = "default"
+  vnet_id     = ""
+  subnet_name = ""
 }

infra/modules/providers/azure/data-factory/tests/integration/data_factory_integration_test.go

@@ -1,24 +1,11 @@
 package unit
 
 import (
-	"encoding/json"
-	"strings"
 	"testing"
-
-	"github.com/gruntwork-io/terratest/modules/random"
 	tests "github.com/microsoft/cobalt/infra/modules/providers/azure/data-factory/tests"
 	"github.com/microsoft/terratest-abstraction/unit"
 )
 
-// helper function to parse blocks of JSON into a generic Go map
-func asMap(t *testing.T, jsonString string) map[string]interface{} {
-	var theMap map[string]interface{}
-	if err := json.Unmarshal([]byte(jsonString), &theMap); err != nil {
-		t.Fatal(err)
-	}
-	return theMap
-}
-
 func TestTemplate(t *testing.T) {
 
 	expectedDataFactory := map[string]interface{}{
@@ -53,27 +40,16 @@ func TestTemplate(t *testing.T) {
 		"frequency": "Minute",
 	}
 
-	expectedDatasetSQL := map[string]interface{}{
-		"name": "testsql",
-	}
-
-	expectedLinkedSQL := map[string]interface{}{
-		"name":              "testlinkedsql",
-		"connection_string": "connectionstring",
-	}
-
 	testFixture := unit.UnitTestFixture{
 		GoTest:                t,
 		TfOptions:             tests.DataFactoryTFOptions,
 		PlanAssertions:        nil,
-		ExpectedResourceCount: 6,
+		ExpectedResourceCount: 4,
 		ExpectedResourceAttributeValues: unit.ResourceDescription{
 			"azurerm_data_factory.main":                             expectedDataFactory,
 			"azurerm_data_factory_integration_runtime_managed.main": expectedDFIntRunTime,
 			"azurerm_data_factory_pipeline.main":                    expectedPipeline,
 			"azurerm_data_factory_trigger_schedule.main":            expectedTrigger,
-			"azurerm_data_factory_dataset_sql_server_table.main":    expectedDatasetSQL,
-			"azurerm_data_factory_linked_service_sql_server.main":   expectedLinkedSQL,
 		},
 	}
 

infra/modules/providers/azure/data-factory/tests/test.tfvars

@@ -68,34 +68,4 @@ variable "data_factory_trigger_frequency" {
   description = "The trigger freqency. Valid values include Minute, Hour, Day, Week, Month. Defaults to Minute."
   type        = string
   default     = "Minute"
-}
-
-variable "data_factory_dataset_sql_name" {
-  description = "Specifies the name of the Data Factory Dataset SQL Server Table. Only letters, numbers and '_' are allowed."
-  type        = string
-  default     = ""
-}
-
-variable "data_factory_dataset_sql_table_name" {
-  description = "The table name of the Data Factory Dataset SQL Server Table."
-  type        = string
-  default     = ""
-}
-
-variable "data_factory_dataset_sql_folder" {
-  description = "The folder that this Dataset is in. If not specified, the Dataset will appear at the root level."
-  type        = string
-  default     = ""
-}
-
-variable "data_factory_linked_sql_name" {
-  description = "Specifies the name of the Data Factory Linked Service SQL Server. Changing this forces a new resource to be created."
-  type        = string
-  default     = ""
-}
-
-variable "data_factory_linked_sql_connection_string" {
-  description = "The connection string in which to authenticate with the SQL Server."
-  type        = string
-  default     = ""
 }

infra/modules/providers/azure/data-factory/tests/unit/data_factory_unit_test.go

@@ -0,0 +1,12 @@
+export ARM_ACCESS_KEY=
+export ARM_CLIENT_ID=
+export ARM_CLIENT_SECRET=
+export ARM_SUBSCRIPTION_ID=
+export ARM_TENANT_ID=
+export BUILD_BUILDID=1
+export GO_VERSION=1.12.5
+export TF_VAR_remote_state_account=
+export TF_VAR_remote_state_container=
+export TF_VERSION=0.12.4
+export TF_WARN_OUTPUT_ERRORS=1
+export TF_VAR_resource_group_location=eastus