diff --git a/azure/terraform/main.tf b/azure/terraform/main.tf index 5c02c1d..7d932e6 100644 --- a/azure/terraform/main.tf +++ b/azure/terraform/main.tf @@ -9,7 +9,7 @@ terraform { version = "3.14.0" } helm = { - source = "hashicorp/helm" + source = "hashicorp/helm" version = "2.6.0" } } @@ -17,27 +17,27 @@ terraform { data "azurerm_kubernetes_cluster" "default" { - depends_on = [module.infra] # refresh cluster state before reading + depends_on = [module.infra] # refresh cluster state before reading resource_group_name = local.metaflow_resource_group_name name = local.kubernetes_cluster_name } data "azurerm_postgresql_flexible_server" "default" { - depends_on = [module.infra] # refresh cluster state before reading + depends_on = [module.infra] # refresh cluster state before reading resource_group_name = local.metaflow_resource_group_name name = local.database_server_name } data "azurerm_storage_account" "default" { - depends_on = [module.infra] # refresh cluster state before reading + depends_on = [module.infra] # refresh cluster state before reading resource_group_name = local.metaflow_resource_group_name name = local.storage_account_name - + } data "azurerm_storage_container" "default" { - depends_on = [module.infra] # refresh cluster state before reading - name = local.storage_container_name + depends_on = [module.infra] # refresh cluster state before reading + name = local.storage_container_name storage_account_name = local.storage_account_name } @@ -95,15 +95,16 @@ module "services" { metaflow_db_user = local.metaflow_database_server_admin_login metaflow_db_password = local.metaflow_db_password metaflow_kubernetes_secret_name = local.metaflow_kubernetes_secret_name - azure_storage_credentials = { + azure_storage_credentials = { AZURE_CLIENT_ID = module.infra.service_principal_client_id AZURE_TENANT_ID = module.infra.service_principal_tenant_id AZURE_CLIENT_SECRET = module.infra.service_principal_client_secret } - - deploy_airflow = var.deploy_airflow - deploy_argo = var.deploy_argo - - airflow_version = local.airflow_version - airflow_frenet_secret = local.airflow_frenet_secret + + deploy_airflow = var.deploy_airflow + deploy_argo = var.deploy_argo + deploy_argo_events = var.deploy_argo_events + + airflow_version = local.airflow_version + airflow_frenet_secret = local.airflow_frenet_secret } diff --git a/azure/terraform/services/argo.tf b/azure/terraform/services/argo.tf index 9235bc8..e194be6 100644 --- a/azure/terraform/services/argo.tf +++ b/azure/terraform/services/argo.tf @@ -5,9 +5,20 @@ resource "kubernetes_namespace" "argo" { } } +resource "kubernetes_namespace" "argo-events" { + count = var.deploy_argo_events ? 1 : 0 + metadata { + name = "argo-events" + } +} + locals { - is_windows = substr(pathexpand("~"), 0, 1) == "/" ? false : true - _kubectl_cmd = "kubectl apply -n argo -f https://raw.githubusercontent.com/argoproj/argo-workflows/master/manifests/quick-start-postgres.yaml" + is_windows = substr(pathexpand("~"), 0, 1) == "/" ? false : true + _argo_cmd = "kubectl apply -n argo -f https://raw.githubusercontent.com/argoproj/argo-workflows/master/manifests/quick-start-postgres.yaml" + _argo_events_cmd = "kubectl apply -f https://raw.githubusercontent.com/argoproj/argo-events/v1.7.3/manifests/install.yaml" + _service_accts_cmd = "kubectl apply -n argo -f ${path.module}/argo_events/service_accounts.yaml" + _event_bus_cmd = "kubectl apply -n argo -f https://raw.githubusercontent.com/argoproj/argo-events/v1.7.3/examples/eventbus/native.yaml" + _webhook_source_cmd = "kubectl apply -n argo -f ${path.module}/argo_events/webhook_source.yaml" } # Yes local-exec is unfortunate. @@ -17,10 +28,58 @@ locals { resource "null_resource" "argo-quick-start-installation" { count = var.deploy_argo ? 1 : 0 triggers = { - cmd = local._kubectl_cmd + cmd = local._argo_cmd + } + provisioner "local-exec" { + interpreter = local.is_windows ? ["PowerShell"] : null + command = local.is_windows ? "$env:KUBECONFIG='${local_file.kubeconfig.filename}'; ${local._argo_cmd}" : "KUBECONFIG=${local_file.kubeconfig.filename} ${local._argo_cmd}" + } +} + +resource "null_resource" "argo-events-quick-start" { + count = var.deploy_argo_events ? 1 : 0 + depends_on = [null_resource.argo-quick-start-installation] + triggers = { + cmd = local._argo_events_cmd + } + provisioner "local-exec" { + interpreter = local.is_windows ? ["PowerShell"] : null + command = local.is_windows ? "$env:KUBECONFIG='${local_file.kubeconfig.filename}'; ${local._argo_events_cmd}" : "KUBECONFIG=${local_file.kubeconfig.filename} ${local._argo_events_cmd}" + } +} + +resource "null_resource" "argo-events-service-accounts" { + count = var.deploy_argo_events ? 1 : 0 + depends_on = [null_resource.argo-events-quick-start] + triggers = { + cmd = local._service_accts_cmd + } + provisioner "local-exec" { + interpreter = local.is_windows ? ["PowerShell"] : null + command = local.is_windows ? "$env:KUBECONFIG='${local_file.kubeconfig.filename}'; ${local._service_accts_cmd}" : "KUBECONFIG=${local_file.kubeconfig.filename} ${local._service_accts_cmd}" + } +} + +resource "null_resource" "argo-events-event-bus" { + count = var.deploy_argo_events ? 1 : 0 + depends_on = [null_resource.argo-events-quick-start] + triggers = { + cmd = local._event_bus_cmd + } + provisioner "local-exec" { + interpreter = local.is_windows ? ["PowerShell"] : null + command = local.is_windows ? "$env:KUBECONFIG='${local_file.kubeconfig.filename}'; ${local._event_bus_cmd}" : "KUBECONFIG=${local_file.kubeconfig.filename} ${local._event_bus_cmd}" + } +} + +resource "null_resource" "argo-events-webhook-source" { + count = var.deploy_argo_events ? 1 : 0 + depends_on = [null_resource.argo-events-event-bus] + triggers = { + cmd = local._webhook_source_cmd } provisioner "local-exec" { interpreter = local.is_windows ? ["PowerShell"] : null - command = local.is_windows ? "$env:KUBECONFIG='${local_file.kubeconfig.filename}'; ${local._kubectl_cmd}" : "KUBECONFIG=${local_file.kubeconfig.filename} ${local._kubectl_cmd}" + command = local.is_windows ? "$env:KUBECONFIG='${local_file.kubeconfig.filename}'; ${local._webhook_source_cmd}" : "KUBECONFIG=${local_file.kubeconfig.filename} ${local._webhook_source_cmd}" } } diff --git a/azure/terraform/services/argo_events/service_accounts.yaml b/azure/terraform/services/argo_events/service_accounts.yaml new file mode 100644 index 0000000..7cee29e --- /dev/null +++ b/azure/terraform/services/argo_events/service_accounts.yaml @@ -0,0 +1,31 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: operate-workflow-sa +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: operate-workflow-role +rules: + - apiGroups: + - argoproj.io + verbs: + - "*" + resources: + - workflows + - workflowtemplates + - cronworkflows + - clusterworkflowtemplates +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: operate-workflow-role-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: operate-workflow-role +subjects: + - kind: ServiceAccount + name: operate-workflow-sa \ No newline at end of file diff --git a/azure/terraform/services/argo_events/webhook_source.yaml b/azure/terraform/services/argo_events/webhook_source.yaml new file mode 100644 index 0000000..3ba72c5 --- /dev/null +++ b/azure/terraform/services/argo_events/webhook_source.yaml @@ -0,0 +1,15 @@ +apiVersion: argoproj.io/v1alpha1 +kind: EventSource +metadata: + name: metaflow-webhook + namespace: argo +spec: + service: + ports: + - port: 12000 + targetPort: 12000 + webhook: + event: + port: "12000" + endpoint: /event + method: POST \ No newline at end of file diff --git a/azure/terraform/services/variables.tf b/azure/terraform/services/variables.tf index 8d38f34..9086b09 100644 --- a/azure/terraform/services/variables.tf +++ b/azure/terraform/services/variables.tf @@ -42,10 +42,10 @@ variable "metaflow_azure_storage_blob_service_endpoint" { } variable "azure_storage_credentials" { - type = map + type = map(any) } -variable "metaflow_kubernetes_secret_name"{ +variable "metaflow_kubernetes_secret_name" { type = string } @@ -61,6 +61,10 @@ variable "deploy_argo" { type = bool } +variable "deploy_argo_events" { + type = bool +} + variable "deploy_airflow" { type = bool } diff --git a/azure/terraform/variables.tf b/azure/terraform/variables.tf index f377a8b..75bb0e6 100644 --- a/azure/terraform/variables.tf +++ b/azure/terraform/variables.tf @@ -3,35 +3,35 @@ locals { - storage_container_name = "metaflow-storage-container" + storage_container_name = "metaflow-storage-container" metaflow_datastore_sysroot_azure = "${local.storage_container_name}/tf-full-stack-sysroot" - location = "westus" - metaflow_resource_group_name = "rg-metaflow-${terraform.workspace}-${local.location}" + location = "westus" + metaflow_resource_group_name = "rg-metaflow-${terraform.workspace}-${local.location}" # MUST be globally unique (entire Azure). Would recommend user to add a meaningful prefix. kubernetes_cluster_name = "aks-${var.org_prefix}-metaflow-${terraform.workspace}" # This MUST be globally unique (entire Azure). Pick a meaningful and unique value for org_prefix database_server_name = "psql-${var.org_prefix}-metaflow-${terraform.workspace}" # This MUST be globally unique (entire Azure). Pick a meaningful and unique value for org_prefix - storage_account_name = "st${var.org_prefix}metaflow${terraform.workspace}" + storage_account_name = "st${var.org_prefix}metaflow${terraform.workspace}" storage_service_principal_name = "Metaflow storage service principal (${terraform.workspace})" - virtual_network_name = "vnet-${var.org_prefix}-metaflow-${local.location}-${terraform.workspace}" - db_subnet_name = "snet-${var.org_prefix}-metaflow-db-${local.location}-${terraform.workspace}" - k8s_subnet_name = "snet-${var.org_prefix}-metaflow-k8s-${local.location}-${terraform.workspace}" + virtual_network_name = "vnet-${var.org_prefix}-metaflow-${local.location}-${terraform.workspace}" + db_subnet_name = "snet-${var.org_prefix}-metaflow-db-${local.location}-${terraform.workspace}" + k8s_subnet_name = "snet-${var.org_prefix}-metaflow-k8s-${local.location}-${terraform.workspace}" # Changeable after initial "terraform apply" (e.g. image upgrades, secret rotation) - metadata_service_image = "public.ecr.aws/outerbounds/metaflow_metadata_service:2.3.3" - metaflow_ui_static_service_image = "public.ecr.aws/outerbounds/metaflow_ui:v1.1.4" + metadata_service_image = "public.ecr.aws/outerbounds/metaflow_metadata_service:2.3.3" + metaflow_ui_static_service_image = "public.ecr.aws/outerbounds/metaflow_ui:v1.1.4" metaflow_ui_backend_service_image = "public.ecr.aws/outerbounds/metaflow_metadata_service:2.3.3" - metaflow_kubernetes_secret_name = "metaflow-azure-storage-credentials" + metaflow_kubernetes_secret_name = "metaflow-azure-storage-credentials" # Forever constants metaflow_database_server_admin_login = "metaflow" - metaflow_db_name = "metaflow" - metaflow_db_password = "metaflow" # DB is private, accessible only within vnet. - metaflow_db_port = 5432 + metaflow_db_name = "metaflow" + metaflow_db_password = "metaflow" # DB is private, accessible only within vnet. + metaflow_db_port = 5432 # Airflow Related Options - airflow_version = "2.3.3" + airflow_version = "2.3.3" airflow_frenet_secret = "myverysecretvalue" } @@ -45,11 +45,16 @@ variable "org_prefix" { } variable "deploy_argo" { - type = bool + type = bool + default = true +} + +variable "deploy_argo_events" { + type = bool default = true } variable "deploy_airflow" { - type = bool + type = bool default = false -} \ No newline at end of file +}